From cb338991325e452b67dd2bd45d9fce00d21e159a Mon Sep 17 00:00:00 2001 From: Samuel Paccoud Date: Mon, 2 Sep 2024 20:15:41 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(backend)=20first=20working=20prototyp?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is built from our boilerplate and adds a first working prototype of indexing (single document or bulk) and searching (with basic filtering and sorting. No authentication yet). Run `make demo` to generate an index for development. --- .dockerignore | 36 ++ .github/ISSUE_TEMPLATE.md | 6 + .github/ISSUE_TEMPLATE/Bug_report.md | 28 + .github/ISSUE_TEMPLATE/Feature_request.md | 23 + .github/ISSUE_TEMPLATE/Support_question.md | 22 + .github/PULL_REQUEST_TEMPLATE.md | 11 + .github/workflows/deploy.yml | 52 ++ .github/workflows/docker-hub.yml | 140 +++++ .github/workflows/drive.yml | 203 +++++++ .gitignore | 81 +++ .gitlint | 78 +++ .gitmodules | 3 + CHANGELOG.md | 7 + Dockerfile | 138 +++++ LICENSE | 21 + Makefile | 265 +++++++++ UPGRADE.md | 17 + bin/Tiltfile | 57 ++ bin/_config.sh | 93 +++ bin/compose | 6 + bin/manage | 6 + bin/pylint | 38 ++ bin/pytest | 8 + bin/start-kind.sh | 103 ++++ bin/state | 25 + bin/update_openapi_schema | 12 + bin/updatekeys.sh | 3 + crowdin/config.yml | 29 + docker-compose.yml | 96 +++ docker/files/usr/local/bin/entrypoint | 35 ++ docker/files/usr/local/etc/gunicorn/drive.py | 16 + env.d/development/common.dist | 31 + env.d/development/crowdin.dist | 3 + env.d/development/postgresql.dist | 11 + gitlint/gitlint_emoji.py | 37 ++ renovate.json | 13 + scripts/install-hooks.sh | 30 + scripts/update-git-submodule.sh | 4 + scripts/updatekeys.sh | 3 + src/backend/.pylintrc | 472 +++++++++++++++ src/backend/MANIFEST.in | 3 + src/backend/__init__.py | 0 src/backend/core/__init__.py | 0 src/backend/core/admin.py | 15 + src/backend/core/api.py | 37 ++ src/backend/core/authentication.py | 34 ++ src/backend/core/enums.py | 12 + src/backend/core/factories.py | 53 ++ src/backend/core/migrations/0001_initial.py | 64 ++ src/backend/core/migrations/__init__.py | 0 src/backend/core/models.py | 58 ++ src/backend/core/opensearch.py | 45 ++ src/backend/core/permissions.py | 12 + src/backend/core/schemas.py | 94 +++ src/backend/core/tests/__init__.py | 0 .../tests/test_api_documents_index_bulk.py | 290 ++++++++++ .../tests/test_api_documents_index_single.py | 254 ++++++++ .../core/tests/test_api_documents_search.py | 448 ++++++++++++++ .../core/tests/test_models_services.py | 47 ++ src/backend/core/urls.py | 8 + src/backend/core/views.py | 239 ++++++++ src/backend/demo/__init__.py | 0 src/backend/demo/defaults.py | 6 + src/backend/demo/management/__init__.py | 0 .../demo/management/commands/__init__.py | 0 .../demo/management/commands/create_demo.py | 198 +++++++ src/backend/demo/tests/__init__.py | 0 .../demo/tests/test_commands_create_demo.py | 29 + src/backend/drive/__init__.py | 0 src/backend/drive/celery_app.py | 22 + src/backend/drive/settings.py | 547 ++++++++++++++++++ src/backend/drive/urls.py | 38 ++ src/backend/drive/wsgi.py | 17 + src/backend/manage.py | 14 + src/backend/pyproject.toml | 132 +++++ src/backend/setup.py | 7 + src/helm/drive/Chart.yaml | 4 + src/helm/drive/README.md | 128 ++++ src/helm/drive/generate-readme.sh | 10 + src/helm/drive/templates/_helpers.tpl | 184 ++++++ .../drive/templates/backend_deployment.yaml | 136 +++++ src/helm/drive/templates/backend_job.yaml | 121 ++++ .../backend_job_createsuperuser.yaml | 121 ++++ src/helm/drive/templates/backend_svc.yaml | 21 + .../drive/templates/frontend_deployment.yaml | 136 +++++ src/helm/drive/templates/frontend_svc.yaml | 21 + src/helm/drive/templates/ingress.yaml | 118 ++++ src/helm/drive/templates/ingress_admin.yaml | 98 ++++ src/helm/drive/templates/ingress_media.yaml | 84 +++ src/helm/drive/templates/secret.yaml | 9 + src/helm/drive/values.yaml | 265 +++++++++ src/helm/env.d/dev/secrets.enc.yaml | 62 ++ src/helm/env.d/dev/values.drive.yaml.gotmpl | 89 +++ src/helm/env.d/preprod/secrets.enc.yaml | 1 + .../env.d/preprod/values.drive.yaml.gotmpl | 137 +++++ src/helm/env.d/production/secrets.enc.yaml | 1 + .../env.d/production/values.drive.yaml.gotmpl | 137 +++++ src/helm/env.d/staging/secrets.enc.yaml | 1 + .../env.d/staging/values.drive.yaml.gotmpl | 134 +++++ src/helm/extra/Chart.yaml | 5 + src/helm/extra/templates/keydb.yaml | 7 + src/helm/extra/templates/postgresql.yaml | 7 + src/helm/extra/templates/s3.yaml | 8 + src/helm/extra/templates/secrets.yaml | 10 + src/helm/helmfile.yaml | 67 +++ 105 files changed, 7107 insertions(+) create mode 100644 .dockerignore create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/ISSUE_TEMPLATE/Bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/Feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/Support_question.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/workflows/deploy.yml create mode 100644 .github/workflows/docker-hub.yml create mode 100644 .github/workflows/drive.yml create mode 100644 .gitignore create mode 100644 .gitlint create mode 100644 .gitmodules create mode 100644 CHANGELOG.md create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 UPGRADE.md create mode 100644 bin/Tiltfile create mode 100644 bin/_config.sh create mode 100755 bin/compose create mode 100755 bin/manage create mode 100755 bin/pylint create mode 100755 bin/pytest create mode 100755 bin/start-kind.sh create mode 100755 bin/state create mode 100755 bin/update_openapi_schema create mode 100644 bin/updatekeys.sh create mode 100644 crowdin/config.yml create mode 100644 docker-compose.yml create mode 100755 docker/files/usr/local/bin/entrypoint create mode 100644 docker/files/usr/local/etc/gunicorn/drive.py create mode 100644 env.d/development/common.dist create mode 100644 env.d/development/crowdin.dist create mode 100644 env.d/development/postgresql.dist create mode 100644 gitlint/gitlint_emoji.py create mode 100644 renovate.json create mode 100755 scripts/install-hooks.sh create mode 100755 scripts/update-git-submodule.sh create mode 100755 scripts/updatekeys.sh create mode 100644 src/backend/.pylintrc create mode 100644 src/backend/MANIFEST.in create mode 100644 src/backend/__init__.py create mode 100644 src/backend/core/__init__.py create mode 100644 src/backend/core/admin.py create mode 100644 src/backend/core/api.py create mode 100644 src/backend/core/authentication.py create mode 100644 src/backend/core/enums.py create mode 100644 src/backend/core/factories.py create mode 100644 src/backend/core/migrations/0001_initial.py create mode 100644 src/backend/core/migrations/__init__.py create mode 100644 src/backend/core/models.py create mode 100644 src/backend/core/opensearch.py create mode 100644 src/backend/core/permissions.py create mode 100644 src/backend/core/schemas.py create mode 100644 src/backend/core/tests/__init__.py create mode 100644 src/backend/core/tests/test_api_documents_index_bulk.py create mode 100644 src/backend/core/tests/test_api_documents_index_single.py create mode 100644 src/backend/core/tests/test_api_documents_search.py create mode 100644 src/backend/core/tests/test_models_services.py create mode 100644 src/backend/core/urls.py create mode 100644 src/backend/core/views.py create mode 100644 src/backend/demo/__init__.py create mode 100644 src/backend/demo/defaults.py create mode 100644 src/backend/demo/management/__init__.py create mode 100644 src/backend/demo/management/commands/__init__.py create mode 100644 src/backend/demo/management/commands/create_demo.py create mode 100644 src/backend/demo/tests/__init__.py create mode 100644 src/backend/demo/tests/test_commands_create_demo.py create mode 100644 src/backend/drive/__init__.py create mode 100644 src/backend/drive/celery_app.py create mode 100755 src/backend/drive/settings.py create mode 100644 src/backend/drive/urls.py create mode 100644 src/backend/drive/wsgi.py create mode 100644 src/backend/manage.py create mode 100644 src/backend/pyproject.toml create mode 100644 src/backend/setup.py create mode 100644 src/helm/drive/Chart.yaml create mode 100644 src/helm/drive/README.md create mode 100644 src/helm/drive/generate-readme.sh create mode 100644 src/helm/drive/templates/_helpers.tpl create mode 100644 src/helm/drive/templates/backend_deployment.yaml create mode 100644 src/helm/drive/templates/backend_job.yaml create mode 100644 src/helm/drive/templates/backend_job_createsuperuser.yaml create mode 100644 src/helm/drive/templates/backend_svc.yaml create mode 100644 src/helm/drive/templates/frontend_deployment.yaml create mode 100644 src/helm/drive/templates/frontend_svc.yaml create mode 100644 src/helm/drive/templates/ingress.yaml create mode 100644 src/helm/drive/templates/ingress_admin.yaml create mode 100644 src/helm/drive/templates/ingress_media.yaml create mode 100644 src/helm/drive/templates/secret.yaml create mode 100644 src/helm/drive/values.yaml create mode 100644 src/helm/env.d/dev/secrets.enc.yaml create mode 100644 src/helm/env.d/dev/values.drive.yaml.gotmpl create mode 120000 src/helm/env.d/preprod/secrets.enc.yaml create mode 100644 src/helm/env.d/preprod/values.drive.yaml.gotmpl create mode 120000 src/helm/env.d/production/secrets.enc.yaml create mode 100644 src/helm/env.d/production/values.drive.yaml.gotmpl create mode 120000 src/helm/env.d/staging/secrets.enc.yaml create mode 100644 src/helm/env.d/staging/values.drive.yaml.gotmpl create mode 100644 src/helm/extra/Chart.yaml create mode 100644 src/helm/extra/templates/keydb.yaml create mode 100644 src/helm/extra/templates/postgresql.yaml create mode 100644 src/helm/extra/templates/s3.yaml create mode 100644 src/helm/extra/templates/secrets.yaml create mode 100644 src/helm/helmfile.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..fe9c333 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,36 @@ +# Python +__pycache__ +*.pyc +**/__pycache__ +**/*.pyc +venv +.venv + +# System-specific files +.DS_Store +**/.DS_Store + +# Docker +docker compose.* +env.d + +# Docs +docs +*.md +*.log + +# Development/test cache & configurations +data +.cache +.circleci +.git +.vscode +.iml +.idea +db.sqlite3 +.mypy_cache +.pylint.d +.pytest_cache + +# Frontend +node_modules diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..24e79d0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,6 @@ + diff --git a/.github/ISSUE_TEMPLATE/Bug_report.md b/.github/ISSUE_TEMPLATE/Bug_report.md new file mode 100644 index 0000000..e9244eb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/Bug_report.md @@ -0,0 +1,28 @@ +--- +name: 🐛 Bug Report +about: If something is not working as expected 🤔. + +--- + +## Bug Report + +**Problematic behavior** +A clear and concise description of the behavior. + +**Expected behavior/code** +A clear and concise description of what you expected to happen (or code). + +**Steps to Reproduce** +1. Do this... +2. Then this... +3. And then the bug happens! + +**Environment** +- Drive version: +- Platform: + +**Possible Solution** + + +**Additional context/Screenshots** +Add any other context about the problem here. If applicable, add screenshots to help explain. diff --git a/.github/ISSUE_TEMPLATE/Feature_request.md b/.github/ISSUE_TEMPLATE/Feature_request.md new file mode 100644 index 0000000..51d3170 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/Feature_request.md @@ -0,0 +1,23 @@ +--- +name: ✨ Feature Request +about: I have a suggestion (and may want to build it 💪)! + +--- + +## Feature Request + +**Is your feature request related to a problem or unsupported use case? Please describe.** +A clear and concise description of what the problem is. For example: I need to do some task and I have an issue... + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. Add any considered drawbacks. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Discovery, Documentation, Adoption, Migration Strategy** +If you can, explain how users will be able to use this and possibly write out a version the docs (if applicable). +Maybe a screenshot or design? + +**Do you want to work on it through a Pull Request?** + diff --git a/.github/ISSUE_TEMPLATE/Support_question.md b/.github/ISSUE_TEMPLATE/Support_question.md new file mode 100644 index 0000000..3f32d2a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/Support_question.md @@ -0,0 +1,22 @@ +--- +name: 🤗 Support Question +about: If you have a question 💬, or something was not clear from the docs! + +--- + + + +--- + +Please make sure you have read our [main Readme](https://github.com/numerique-gouv/drive). + +Also make sure it was not already answered in [an open or close issue](https://github.com/numerique-gouv/drive/issues). + +If your question was not covered, and you feel like it should be, fire away! We'd love to improve our docs! 👌 + +**Topic** +What's the general area of your question: for example, docker setup, database schema, search functionality,... + +**Question** +Try to be as specific as possible so we can help you as best we can. Please be patient 🙏 diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..85cfbe6 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,11 @@ +## Purpose + +Description... + + +## Proposal + +Description... + +- [] item 1... +- [] item 2... diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..0373f8a --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,52 @@ +name: Deploy + +on: + push: + tags: + - 'preprod' + - 'production' + + +jobs: + notify-argocd: + runs-on: ubuntu-latest + steps: + - + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "drive,secrets" + - + name: Checkout repository + uses: actions/checkout@v2 + with: + submodules: recursive + token: ${{ steps.app-token.outputs.token }} + - + name: Load sops secrets + uses: rouja/actions-sops@main + with: + secret-file: secrets/numerique-gouv/drive/secrets.enc.env + age-key: ${{ secrets.SOPS_PRIVATE }} + - + name: Call argocd github webhook + run: | + data='{"ref": "'$GITHUB_REF'","repository": {"html_url":"'$GITHUB_SERVER_URL'/'$GITHUB_REPOSITORY'"}}' + sig=$(echo -n ${data} | openssl dgst -sha1 -hmac ''${ARGOCD_WEBHOOK_SECRET}'' | awk '{print "X-Hub-Signature: sha1="$2}') + curl -X POST -H 'X-GitHub-Event:push' -H "Content-Type: application/json" -H "${sig}" --data "${data}" $ARGOCD_WEBHOOK_URL + sig=$(echo -n ${data} | openssl dgst -sha1 -hmac ''${ARGOCD_PRODUCTION_WEBHOOK_SECRET}'' | awk '{print "X-Hub-Signature: sha1="$2}') + curl -X POST -H 'X-GitHub-Event:push' -H "Content-Type: application/json" -H "${sig}" --data "${data}" $ARGOCD_PRODUCTION_WEBHOOK_URL + + start-test-on-preprod: + needs: + - notify-argocd + runs-on: ubuntu-latest + if: startsWith(github.event.ref, 'refs/tags/preprod') + steps: + - + name: Debug + run: | + echo "Start test when preprod is ready" diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml new file mode 100644 index 0000000..28570b6 --- /dev/null +++ b/.github/workflows/docker-hub.yml @@ -0,0 +1,140 @@ +name: Docker Hub Workflow + +on: + workflow_dispatch: + push: + branches: + - 'main' + tags: + - 'v*' + pull_request: + branches: + - 'main' + +env: + DOCKER_USER: 1001:127 + +jobs: + build-and-push-backend: + runs-on: ubuntu-latest + steps: + - + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "drive,secrets" + - + name: Checkout repository + uses: actions/checkout@v2 + with: + submodules: recursive + token: ${{ steps.app-token.outputs.token }} + - + name: Load sops secrets + uses: rouja/actions-sops@main + with: + secret-file: secrets/numerique-gouv/drive/secrets.enc.env + age-key: ${{ secrets.SOPS_PRIVATE }} + - + name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: lasuite/drive-backend + - + name: Login to DockerHub + if: github.event_name != 'pull_request' + run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin + - + name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + target: backend-production + build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + build-and-push-frontend: + runs-on: ubuntu-latest + steps: + - + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "drive,secrets" + - + name: Checkout repository + uses: actions/checkout@v2 + with: + submodules: recursive + token: ${{ steps.app-token.outputs.token }} + - + name: Load sops secrets + uses: rouja/actions-sops@main + with: + secret-file: secrets/numerique-gouv/drive/secrets.enc.env + age-key: ${{ secrets.SOPS_PRIVATE }} + - + name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: lasuite/drive-frontend + - + name: Login to DockerHub + if: github.event_name != 'pull_request' + run: echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USER" --password-stdin + - + name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: ./src/frontend/Dockerfile + target: frontend-production + build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + notify-argocd: + needs: + - build-and-push-frontend + - build-and-push-backend + runs-on: ubuntu-latest + if: | + github.event_name != 'pull_request' + steps: + - + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "drive,secrets" + - + name: Checkout repository + uses: actions/checkout@v2 + with: + submodules: recursive + token: ${{ steps.app-token.outputs.token }} + - + name: Load sops secrets + uses: rouja/actions-sops@main + with: + secret-file: secrets/numerique-gouv/drive/secrets.enc.env + age-key: ${{ secrets.SOPS_PRIVATE }} + - + name: Call argocd github webhook + run: | + data='{"ref": "'$GITHUB_REF'","repository": {"html_url":"'$GITHUB_SERVER_URL'/'$GITHUB_REPOSITORY'"}}' + sig=$(echo -n ${data} | openssl dgst -sha1 -hmac ''${ARGOCD_WEBHOOK_SECRET}'' | awk '{print "X-Hub-Signature: sha1="$2}') + curl -X POST -H 'X-GitHub-Event:push' -H "Content-Type: application/json" -H "${sig}" --data "${data}" $ARGOCD_WEBHOOK_URL diff --git a/.github/workflows/drive.yml b/.github/workflows/drive.yml new file mode 100644 index 0000000..1b06bc1 --- /dev/null +++ b/.github/workflows/drive.yml @@ -0,0 +1,203 @@ +name: Main Workflow + +on: + push: + branches: + - main + pull_request: + branches: + - "*" + +jobs: + lint-git: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' # Makes sense only for pull requests + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: show + run: git log + - name: Enforce absence of print statements in code + run: | + ! git diff origin/${{ github.event.pull_request.base.ref }}..HEAD -- . ':(exclude)**/drive.yml' | grep "print(" + - name: Check absence of fixup commits + run: | + ! git log | grep 'fixup!' + - name: Install gitlint + run: pip install --user requests gitlint + - name: Lint commit messages added to main + run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD + + check-changelog: + runs-on: ubuntu-latest + if: | + contains(github.event.pull_request.labels.*.name, 'noChangeLog') == false && + github.event_name == 'pull_request' + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 50 + - name: Check that the CHANGELOG has been modified in the current branch + run: git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.after }} | grep 'CHANGELOG.md' + + lint-changelog: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Check CHANGELOG max line length + run: | + max_line_length=$(cat CHANGELOG.md | grep -Ev "^\[.*\]: https://github.com" | wc -L) + if [ $max_line_length -ge 80 ]; then + echo "ERROR: CHANGELOG has lines longer than 80 characters." + exit 1 + fi + + lint-back: + runs-on: ubuntu-latest + defaults: + run: + working-directory: src/backend + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Python + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install development dependencies + run: pip install --user .[dev] + - name: Check code formatting with ruff + run: ~/.local/bin/ruff format . --diff + - name: Lint code with ruff + run: ~/.local/bin/ruff check . + - name: Lint code with pylint + run: ~/.local/bin/pylint . + + test-back: + runs-on: ubuntu-latest + needs: build-mails + + defaults: + run: + working-directory: src/backend + + services: + postgres: + image: postgres:16 + env: + POSTGRES_DB: drive + POSTGRES_USER: dinum + POSTGRES_PASSWORD: pass + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + env: + DJANGO_CONFIGURATION: Test + DJANGO_SETTINGS_MODULE: drive.settings + DJANGO_SECRET_KEY: ThisIsAnExampleKeyForTestPurposeOnly + OIDC_OP_JWKS_ENDPOINT: /endpoint-for-test-purpose-only + DB_HOST: localhost + DB_NAME: drive + DB_USER: dinum + DB_PASSWORD: pass + DB_PORT: 5432 + STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Create writable /data + run: | + sudo mkdir -p /data/media && \ + sudo mkdir -p /data/static + + - name: Install Python + uses: actions/setup-python@v3 + with: + python-version: "3.10" + + - name: Install development dependencies + run: pip install --user .[dev] + + - name: Install gettext (required to compile messages) + run: | + sudo apt-get update + sudo apt-get install -y gettext + + - name: Generate a MO file from strings extracted from the project + run: python manage.py compilemessages + + - name: Run tests + run: ~/.local/bin/pytest -n 2 + + i18n-crowdin: + runs-on: ubuntu-latest + steps: + - + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "infrastructure,secrets" + - + name: Checkout repository + uses: actions/checkout@v2 + with: + submodules: recursive + token: ${{ steps.app-token.outputs.token }} + - + name: Load sops secrets + uses: rouja/actions-sops@main + with: + secret-file: secrets/numerique-gouv/drive/secrets.enc.env + age-key: ${{ secrets.SOPS_PRIVATE }} + + - name: Install gettext (required to make messages) + run: | + sudo apt-get update + sudo apt-get install -y gettext + + - name: Install Python + uses: actions/setup-python@v3 + with: + python-version: "3.10" + + - name: Install development dependencies + working-directory: src/backend + run: pip install --user .[dev] + + - name: Generate the translation base file + run: ~/.local/bin/django-admin makemessages --keep-pot --all + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "18.x" + cache: "yarn" + cache-dependency-path: src/frontend/yarn.lock + + - name: Install dependencies + run: cd src/frontend/ && yarn install --frozen-lockfile + + - name: Extract the frontend translation + run: make frontend-i18n-extract + + - name: Upload files to Crowdin + run: | + docker run \ + --rm \ + -e CROWDIN_API_TOKEN=$CROWDIN_API_TOKEN \ + -e CROWDIN_PROJECT_ID=$CROWDIN_PROJECT_ID \ + -e CROWDIN_BASE_PATH=$CROWDIN_BASE_PATH \ + -v "${{ github.workspace }}:/app" \ + crowdin/cli:3.16.0 \ + crowdin upload sources -c /app/crowdin/config.yml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2464394 --- /dev/null +++ b/.gitignore @@ -0,0 +1,81 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +.DS_Store +.next/ + +# Translations # Translations +*.pot + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +env.d/development/* +!env.d/development/*.dist +env.d/terraform + +# npm +node_modules + +# Mails +src/backend/core/templates/mail/ + +# Typescript client +src/frontend/tsclient + +# Swagger +**/swagger.json + +# Logs +*.log + +# Terraform +.terraform +*.tfstate +*.tfstate.backup + +# Test & lint +.coverage +.pylint.d +.pytest_cache +db.sqlite3 +.mypy_cache + +# Site media +/data/ + +# IDEs +.idea/ +.vscode/ +*.iml +.devcontainer diff --git a/.gitlint b/.gitlint new file mode 100644 index 0000000..f7373b6 --- /dev/null +++ b/.gitlint @@ -0,0 +1,78 @@ +# All these sections are optional, edit this file as you like. +[general] +# Ignore certain rules, you can reference them by their id or by their full name +# ignore=title-trailing-punctuation, T3 + +# verbosity should be a value between 1 and 3, the commandline -v flags take precedence over this +# verbosity = 2 + +# By default gitlint will ignore merge commits. Set to 'false' to disable. +# ignore-merge-commits=true + +# By default gitlint will ignore fixup commits. Set to 'false' to disable. +# ignore-fixup-commits=true + +# By default gitlint will ignore squash commits. Set to 'false' to disable. +# ignore-squash-commits=true + +# Enable debug mode (prints more output). Disabled by default. +# debug=true + +# Set the extra-path where gitlint will search for user defined rules +# See http://jorisroovers.github.io/gitlint/user_defined_rules for details +extra-path=gitlint/ + +# [title-max-length] +# line-length=80 + +[title-must-not-contain-word] +# Comma-separated list of words that should not occur in the title. Matching is case +# insensitive. It's fine if the keyword occurs as part of a larger word (so "WIPING" +# will not cause a violation, but "WIP: my title" will. +words=wip + +#[title-match-regex] +# python like regex (https://docs.python.org/2/library/re.html) that the +# commit-msg title must be matched to. +# Note that the regex can contradict with other rules if not used correctly +# (e.g. title-must-not-contain-word). +#regex= + +# [B1] +# B1 = body-max-line-length +# line-length=120 +# [body-min-length] +# min-length=5 + +# [body-is-missing] +# Whether to ignore this rule on merge commits (which typically only have a title) +# default = True +# ignore-merge-commits=false + +# [body-changed-file-mention] +# List of files that need to be explicitly mentioned in the body when they are changed +# This is useful for when developers often erroneously edit certain files or git submodules. +# By specifying this rule, developers can only change the file when they explicitly reference +# it in the commit message. +# files=gitlint/rules.py,README.md + +# [author-valid-email] +# python like regex (https://docs.python.org/2/library/re.html) that the +# commit author email address should be matched to +# For example, use the following regex if you only want to allow email addresses from foo.com +# regex=[^@]+@foo.com + +[ignore-by-title] +# Allow empty body & wrong title pattern only when bots (pyup/greenkeeper) +# upgrade dependencies +regex=^(⬆️.*|Update (.*) from (.*) to (.*)|(chore|fix)\(package\): update .*)$ +ignore=B6,UC1 + +# [ignore-by-body] +# Ignore certain rules for commits of which the body has a line that matches a regex +# E.g. Match bodies that have a line that that contain "release" +# regex=(.*)release(.*) +# +# Ignore certain rules, you can reference them by their id or by their full name +# Use 'all' to ignore all rules +# ignore=T1,body-min-length diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..0846cc2 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "secrets"] + path = secrets + url = ../secrets diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..28a9b52 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0), +and this project adheres to +[Semantic Versioning](https://semver.org/spec/v2.0.0.html). diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..45bd7fc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,138 @@ +# Django drive + +# ---- base image to inherit from ---- +FROM python:3.10-slim-bullseye AS base + +# Upgrade pip to its latest release to speed up dependencies installation +RUN python -m pip install --upgrade pip + +# Upgrade system packages to install security updates +RUN apt-get update && \ + apt-get -y upgrade && \ + rm -rf /var/lib/apt/lists/* + +# ---- Back-end builder image ---- +FROM base AS back-builder + +WORKDIR /builder + +# Copy required python dependencies +COPY ./src/backend /builder + +RUN mkdir /install && \ + pip install --prefix=/install . + +# ---- static link collector ---- +FROM base AS link-collector +ARG DRIVE_STATIC_ROOT=/data/static + +# Install libpangocairo & rdfind +RUN apt-get update && \ + apt-get install -y \ + libpangocairo-1.0-0 \ + rdfind && \ + rm -rf /var/lib/apt/lists/* + +# Copy installed python dependencies +COPY --from=back-builder /install /usr/local + +# Copy drive application (see .dockerignore) +COPY ./src/backend /app/ + +WORKDIR /app + +# collectstatic +RUN DJANGO_CONFIGURATION=Build DJANGO_JWT_PRIVATE_SIGNING_KEY=Dummy \ + python manage.py collectstatic --noinput + +# Replace duplicated file by a symlink to decrease the overall size of the +# final image +RUN rdfind -makesymlinks true -followsymlinks true -makeresultsfile false ${DRIVE_STATIC_ROOT} + +# ---- Core application image ---- +FROM base AS core + +ENV PYTHONUNBUFFERED=1 + +# Install required system libs +RUN apt-get update && \ + apt-get install -y \ + gettext \ + libcairo2 \ + libffi-dev \ + libgdk-pixbuf2.0-0 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + shared-mime-info && \ + rm -rf /var/lib/apt/lists/* + +# Copy entrypoint +COPY ./docker/files/usr/local/bin/entrypoint /usr/local/bin/entrypoint + +# Give the "root" group the same permissions as the "root" user on /etc/passwd +# to allow a user belonging to the root group to add new users; typically the +# docker user (see entrypoint). +RUN chmod g=u /etc/passwd + +# Copy installed python dependencies +COPY --from=back-builder /install /usr/local + +# Copy drive application (see .dockerignore) +COPY ./src/backend /app/ + +WORKDIR /app + +# We wrap commands run in this container by the following entrypoint that +# creates a user on-the-fly with the container user ID (see USER) and root group +# ID. +ENTRYPOINT [ "/usr/local/bin/entrypoint" ] + +# ---- Development image ---- +FROM core AS backend-development + +# Switch back to the root user to install development dependencies +USER root:root + +# Install psql +RUN apt-get update && \ + apt-get install -y postgresql-client && \ + rm -rf /var/lib/apt/lists/* + +# Uninstall drive and re-install it in editable mode along with development +# dependencies +RUN pip uninstall -y drive +RUN pip install -e .[dev] + +# Restore the un-privileged user running the application +ARG DOCKER_USER +USER ${DOCKER_USER} + +# Target database host (e.g. database engine following docker compose services +# name) & port +ENV DB_HOST=postgresql \ + DB_PORT=5432 + +# Run django development server +CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"] + +# ---- Production image ---- +FROM core AS backend-production + +ARG DRIVE_STATIC_ROOT=/data/static + +# Gunicorn +RUN mkdir -p /usr/local/etc/gunicorn +COPY docker/files/usr/local/etc/gunicorn/drive.py /usr/local/etc/gunicorn/drive.py + +# Un-privileged user running the application +ARG DOCKER_USER +USER ${DOCKER_USER} + +# Copy statics +COPY --from=link-collector ${DRIVE_STATIC_ROOT} ${DRIVE_STATIC_ROOT} + +# Copy drive mails +COPY --from=mail-builder /mail/backend/core/templates/mail /app/core/templates/mail + +# The default command runs gunicorn WSGI server in drive's main module +CMD ["gunicorn", "-c", "/usr/local/etc/gunicorn/drive.py", "drive.wsgi:application"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8830452 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Direction Interministérielle du Numérique - Gouvernement Français + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2462b7c --- /dev/null +++ b/Makefile @@ -0,0 +1,265 @@ +# /!\ /!\ /!\ /!\ /!\ /!\ /!\ DISCLAIMER /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ +# +# This Makefile is only meant to be used for DEVELOPMENT purpose as we are +# changing the user id that will run in the container. +# +# PLEASE DO NOT USE IT FOR YOUR CI/PRODUCTION/WHATEVER... +# +# /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ +# +# Note to developers: +# +# While editing this file, please respect the following statements: +# +# 1. Every variable should be defined in the ad hoc VARIABLES section with a +# relevant subsection +# 2. Every new rule should be defined in the ad hoc RULES section with a +# relevant subsection depending on the targeted service +# 3. Rules should be sorted alphabetically within their section +# 4. When a rule has multiple dependencies, you should: +# - duplicate the rule name to add the help string (if required) +# - write one dependency per line to increase readability and diffs +# 5. .PHONY rule statement should be written after the corresponding rule +# ============================================================================== +# VARIABLES + +BOLD := \033[1m +RESET := \033[0m +GREEN := \033[1;32m + + +# -- Database + +DB_HOST = postgresql +DB_PORT = 5432 + +# -- Docker +# Get the current user ID to use for docker run and docker exec commands +DOCKER_UID = $(shell id -u) +DOCKER_GID = $(shell id -g) +DOCKER_USER = $(DOCKER_UID):$(DOCKER_GID) +COMPOSE = DOCKER_USER=$(DOCKER_USER) docker compose +COMPOSE_EXEC = $(COMPOSE) exec +COMPOSE_EXEC_APP = $(COMPOSE_EXEC) app +COMPOSE_RUN = $(COMPOSE) run --rm +COMPOSE_RUN_APP = $(COMPOSE_RUN) app +COMPOSE_RUN_CROWDIN = $(COMPOSE_RUN) crowdin crowdin +WAIT_DB = @$(COMPOSE_RUN) dockerize -wait tcp://$(DB_HOST):$(DB_PORT) -timeout 60s + +# -- Backend +MANAGE = $(COMPOSE_RUN_APP) python manage.py + +# ============================================================================== +# RULES + +default: help + +data/opensearch: + @mkdir -p data/opensearch + +data/static: + @mkdir -p data/static + +# -- Project + +create-env-files: ## Copy the dist env files to env files +create-env-files: \ + env.d/development/common \ + env.d/development/crowdin \ + env.d/development/postgresql +.PHONY: create-env-files + +bootstrap: ## Prepare Docker images for the project +bootstrap: \ + data/opensearch \ + data/static \ + create-env-files \ + build \ + migrate \ + demo \ + back-i18n-compile +.PHONY: bootstrap + +# -- Docker/compose +build: ## build the app container + @$(COMPOSE) build app --no-cache +.PHONY: build + +down: ## stop and remove containers, networks, images, and volumes + @$(COMPOSE) down +.PHONY: down + +logs: ## display app logs (follow mode) + @$(COMPOSE) logs -f app +.PHONY: logs + +run: ## start the wsgi (production) and development server + @$(COMPOSE) up --force-recreate -d celery + @echo "Wait for postgresql to be up..." + @$(WAIT_DB) +.PHONY: run + +status: ## an alias for "docker compose ps" + @$(COMPOSE) ps +.PHONY: status + +stop: ## stop the development server using Docker + @$(COMPOSE) stop +.PHONY: stop + +# -- Backend + +demo: ## flush db then create a demo for load testing purpose + @$(MAKE) resetdb + @$(MANAGE) create_demo +.PHONY: demo + +# Nota bene: Black should come after isort just in case they don't agree... +lint: ## lint back-end python sources +lint: \ + lint-ruff-format \ + lint-ruff-check \ + lint-pylint +.PHONY: lint + +lint-ruff-format: ## format back-end python sources with ruff + @echo 'lint:ruff-format started…' + @$(COMPOSE_RUN_APP) ruff format . +.PHONY: lint-ruff-format + +lint-ruff-check: ## lint back-end python sources with ruff + @echo 'lint:ruff-check started…' + @$(COMPOSE_RUN_APP) ruff check . --fix +.PHONY: lint-ruff-check + +lint-pylint: ## lint back-end python sources with pylint only on changed files from main + @echo 'lint:pylint started…' + bin/pylint . +.PHONY: lint-pylint + +test: ## run project tests + @$(MAKE) test-back-parallel +.PHONY: test + +test-back: ## run back-end tests + @args="$(filter-out $@,$(MAKECMDGOALS))" && \ + bin/pytest $${args:-${1}} +.PHONY: test-back + +test-back-parallel: ## run all back-end tests in parallel + @args="$(filter-out $@,$(MAKECMDGOALS))" && \ + bin/pytest -n auto $${args:-${1}} +.PHONY: test-back-parallel + +makemigrations: ## run django makemigrations for the drive project. + @echo "$(BOLD)Running makemigrations$(RESET)" + @$(COMPOSE) up -d postgresql + @$(WAIT_DB) + @$(MANAGE) makemigrations +.PHONY: makemigrations + +migrate: ## run django migrations for the drive project. + @echo "$(BOLD)Running migrations$(RESET)" + @$(COMPOSE) up -d postgresql + @$(WAIT_DB) + @$(MANAGE) migrate +.PHONY: migrate + +superuser: ## Create an admin superuser with password "admin" + @echo "$(BOLD)Creating a Django superuser$(RESET)" + @$(WAIT_DB) + @$(MANAGE) shell -c "from core.models import User; not User.objects.filter(username='admin').exists() and User.objects.create_superuser('admin', 'admin@example.com', 'admin')" +.PHONY: superuser + +back-i18n-compile: ## compile the gettext files + @$(MANAGE) compilemessages --ignore="venv/**/*" +.PHONY: back-i18n-compile + +back-i18n-generate: ## create the .pot files used for i18n + @$(MANAGE) makemessages -a --keep-pot +.PHONY: back-i18n-generate + +shell: ## connect to database shell + @$(MANAGE) shell #_plus +.PHONY: dbshell + +# -- Database + +dbshell: ## connect to database shell + docker compose exec app python manage.py dbshell +.PHONY: dbshell + +resetdb: FLUSH_ARGS ?= +resetdb: ## flush database and create a superuser "admin" + @echo "$(BOLD)Flush database$(RESET)" + @$(MANAGE) flush $(FLUSH_ARGS) + @${MAKE} superuser +.PHONY: resetdb + +env.d/development/common: + cp -n env.d/development/common.dist env.d/development/common + +env.d/development/postgresql: + cp -n env.d/development/postgresql.dist env.d/development/postgresql + +# -- Internationalization + +env.d/development/crowdin: + cp -n env.d/development/crowdin.dist env.d/development/crowdin + +crowdin-download: ## Download translated message from crowdin + @$(COMPOSE_RUN_CROWDIN) download -c crowdin/config.yml +.PHONY: crowdin-download + +crowdin-download-sources: ## Download sources from Crowdin + @$(COMPOSE_RUN_CROWDIN) download sources -c crowdin/config.yml +.PHONY: crowdin-download-sources + +crowdin-upload: ## Upload source translations to crowdin + @$(COMPOSE_RUN_CROWDIN) upload sources -c crowdin/config.yml +.PHONY: crowdin-upload + +i18n-compile: ## compile all translations +i18n-compile: \ + back-i18n-compile \ + frontend-i18n-compile +.PHONY: i18n-compile + +i18n-generate: ## create the .pot files and extract frontend messages +i18n-generate: \ + back-i18n-generate \ + frontend-i18n-generate +.PHONY: i18n-generate + +i18n-download-and-compile: ## download all translated messages and compile them to be used by all applications +i18n-download-and-compile: \ + crowdin-download \ + i18n-compile +.PHONY: i18n-download-and-compile + +i18n-generate-and-upload: ## generate source translations for all applications and upload them to Crowdin +i18n-generate-and-upload: \ + i18n-generate \ + crowdin-upload +.PHONY: i18n-generate-and-upload + +# -- Misc +clean: ## restore repository state as it was freshly cloned + git clean -idx +.PHONY: clean + +help: + @echo "$(BOLD)drive Makefile" + @echo "Please use 'make $(BOLD)target$(RESET)' where $(BOLD)target$(RESET) is one of:" + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(firstword $(MAKEFILE_LIST)) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(GREEN)%-30s$(RESET) %s\n", $$1, $$2}' +.PHONY: help + +# -- K8S +build-k8s-cluster: ## build the kubernetes cluster using kind + ./bin/start-kind.sh +.PHONY: build-k8s-cluster + +start-tilt: ## start the kubernetes cluster using kind + tilt up -f ./bin/Tiltfile +.PHONY: build-k8s-cluster + diff --git a/UPGRADE.md b/UPGRADE.md new file mode 100644 index 0000000..a905f77 --- /dev/null +++ b/UPGRADE.md @@ -0,0 +1,17 @@ +# Upgrade + +All instructions to upgrade this project from one release to the next will be +documented in this file. Upgrades must be run sequentially, meaning you should +not skip minor/major releases while upgrading (fix releases can be skipped). + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +For most upgrades, you just need to run the django migrations with +the following command inside your docker container: + +`python manage.py migrate` + +(Note : in your development environment, you can `make migrate`.) + +## [Unreleased] diff --git a/bin/Tiltfile b/bin/Tiltfile new file mode 100644 index 0000000..909cb51 --- /dev/null +++ b/bin/Tiltfile @@ -0,0 +1,57 @@ +load('ext://uibutton', 'cmd_button', 'bool_input', 'location') +load('ext://namespace', 'namespace_create', 'namespace_inject') +namespace_create('drive') + +docker_build( + 'localhost:5001/drive-backend:latest', + context='..', + dockerfile='../Dockerfile', + only=['./src/backend', './docker'], + target = 'backend-production', + live_update=[ + sync('../src/backend', '/app'), + run( + 'pip install -r /app/requirements.txt', + trigger=['./api/requirements.txt'] + ) + ] +) + +docker_build( + 'localhost:5001/drive-frontend:latest', + context='..', + dockerfile='../src/frontend/Dockerfile', + only=['./src/frontend', './docker', './.dockerignore'], + target = 'drive', + live_update=[ + sync('../src/frontend', '/home/frontend'), + ] +) + +k8s_yaml(local('cd ../src/helm && helmfile -n drive -e dev template .')) + +migration = ''' +set -eu +# get k8s pod name from tilt resource name +POD_NAME="$(tilt get kubernetesdiscovery drive-backend -ojsonpath='{.status.pods[0].name}')" +kubectl -n drive exec "$POD_NAME" -- python manage.py makemigrations +''' +cmd_button('Make migration', + argv=['sh', '-c', migration], + resource='drive-backend', + icon_name='developer_board', + text='Run makemigration', +) + +pod_migrate = ''' +set -eu +# get k8s pod name from tilt resource name +POD_NAME="$(tilt get kubernetesdiscovery drive-backend -ojsonpath='{.status.pods[0].name}')" +kubectl -n drive exec "$POD_NAME" -- python manage.py migrate --no-input +''' +cmd_button('Migrate db', + argv=['sh', '-c', pod_migrate], + resource='drive-backend', + icon_name='developer_board', + text='Run database migration', +) diff --git a/bin/_config.sh b/bin/_config.sh new file mode 100644 index 0000000..e8e6c80 --- /dev/null +++ b/bin/_config.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +set -eo pipefail + +REPO_DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd)" +UNSET_USER=0 + +COMPOSE_FILE="${REPO_DIR}/docker-compose.yml" +COMPOSE_PROJECT="drive" + + +# _set_user: set (or unset) default user id used to run docker commands +# +# usage: _set_user +# +# You can override default user ID (the current host user ID), by defining the +# USER_ID environment variable. +# +# To avoid running docker commands with a custom user, please set the +# $UNSET_USER environment variable to 1. +function _set_user() { + + if [ $UNSET_USER -eq 1 ]; then + USER_ID="" + return + fi + + # USER_ID = USER_ID or `id -u` if USER_ID is not set + USER_ID=${USER_ID:-$(id -u)} + + echo "🙋(user) ID: ${USER_ID}" +} + +# docker_compose: wrap docker compose command +# +# usage: docker_compose [options] [ARGS...] +# +# options: docker compose command options +# ARGS : docker compose command arguments +function _docker_compose() { + + echo "🐳(compose) project: '${COMPOSE_PROJECT}' file: '${COMPOSE_FILE}'" + docker compose \ + -p "${COMPOSE_PROJECT}" \ + -f "${COMPOSE_FILE}" \ + --project-directory "${REPO_DIR}" \ + "$@" +} + +# _dc_run: wrap docker compose run command +# +# usage: _dc_run [options] [ARGS...] +# +# options: docker compose run command options +# ARGS : docker compose run command arguments +function _dc_run() { + _set_user + + user_args="--user=$USER_ID" + if [ -z $USER_ID ]; then + user_args="" + fi + + _docker_compose run --rm $user_args "$@" +} + +# _dc_exec: wrap docker compose exec command +# +# usage: _dc_exec [options] [ARGS...] +# +# options: docker compose exec command options +# ARGS : docker compose exec command arguments +function _dc_exec() { + _set_user + + echo "🐳(compose) exec command: '\$@'" + + user_args="--user=$USER_ID" + if [ -z $USER_ID ]; then + user_args="" + fi + + _docker_compose exec $user_args "$@" +} + +# _django_manage: wrap django's manage.py command with docker compose +# +# usage : _django_manage [ARGS...] +# +# ARGS : django's manage.py command arguments +function _django_manage() { + _dc_run app python manage.py "$@" +} diff --git a/bin/compose b/bin/compose new file mode 100755 index 0000000..1adb3d8 --- /dev/null +++ b/bin/compose @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# shellcheck source=bin/_config.sh +source "$(dirname "${BASH_SOURCE[0]}")/_config.sh" + +_docker_compose "$@" diff --git a/bin/manage b/bin/manage new file mode 100755 index 0000000..b6c82d9 --- /dev/null +++ b/bin/manage @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# shellcheck source=bin/_config.sh +source "$(dirname "${BASH_SOURCE[0]}")/_config.sh" + +_django_manage "$@" diff --git a/bin/pylint b/bin/pylint new file mode 100755 index 0000000..7cacb5c --- /dev/null +++ b/bin/pylint @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# shellcheck source=bin/_config.sh +source "$(dirname "${BASH_SOURCE[0]}")/_config.sh" + +declare diff_from +declare -a paths +declare -a args + +# Parse options +for arg in "$@" +do + case $arg in + --diff-only=*) + diff_from="${arg#*=}" + shift + ;; + -*) + args+=("$arg") + shift + ;; + *) + paths+=("$arg") + shift + ;; + esac +done + +if [[ -n "${diff_from}" ]]; then + # Run pylint only on modified files located in src/backend + # (excluding deleted files and migration files) + # shellcheck disable=SC2207 + paths=($(git diff "${diff_from}" --name-only --diff-filter=d -- src/backend ':!**/migrations/*.py' | grep -E '^src/backend/.*\.py$')) +fi + +# Fix docker vs local path when project sources are mounted as a volume +read -ra paths <<< "$(echo "${paths[@]}" | sed "s|src/backend/||g")" +_dc_run app pylint "${paths[@]}" "${args[@]}" diff --git a/bin/pytest b/bin/pytest new file mode 100755 index 0000000..abca9a6 --- /dev/null +++ b/bin/pytest @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +source "$(dirname "${BASH_SOURCE[0]}")/_config.sh" + +_dc_run \ + -e DJANGO_CONFIGURATION=Test \ + app \ + pytest "$@" diff --git a/bin/start-kind.sh b/bin/start-kind.sh new file mode 100755 index 0000000..a77f12f --- /dev/null +++ b/bin/start-kind.sh @@ -0,0 +1,103 @@ +#!/bin/sh +set -o errexit + +CURRENT_DIR=$(pwd) + +echo "0. Create ca" +# 0. Create ca +mkcert -install +cd /tmp +mkcert "127.0.0.1.nip.io" "*.127.0.0.1.nip.io" +cd $CURRENT_DIR + +echo "1. Create registry container unless it already exists" +# 1. Create registry container unless it already exists +reg_name='kind-registry' +reg_port='5001' +if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then + docker run \ + -d --restart=always -p "127.0.0.1:${reg_port}:5000" --network bridge --name "${reg_name}" \ + registry:2 +fi + +echo "2. Create kind cluster with containerd registry config dir enabled" +# 2. Create kind cluster with containerd registry config dir enabled +# TODO: kind will eventually enable this by default and this patch will +# be unnecessary. +# +# See: +# https://github.com/kubernetes-sigs/kind/issues/2875 +# https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration +# See: https://github.com/containerd/containerd/blob/main/docs/hosts.md +cat < /dev/null 2>&1; then + if [ -w /etc/passwd ]; then + echo "${USER_NAME:-default}:x:$(id -u):$(id -g):${USER_NAME:-default} user:${HOME}:/sbin/nologin" >> /etc/passwd + fi +fi + +echo "🐳(entrypoint) running your command: ${*}" +exec "$@" diff --git a/docker/files/usr/local/etc/gunicorn/drive.py b/docker/files/usr/local/etc/gunicorn/drive.py new file mode 100644 index 0000000..8b870d5 --- /dev/null +++ b/docker/files/usr/local/etc/gunicorn/drive.py @@ -0,0 +1,16 @@ +# Gunicorn-django settings +bind = ["0.0.0.0:8000"] +name = "drive" +python_path = "/app" + +# Run +graceful_timeout = 90 +timeout = 90 +workers = 3 + +# Logging +# Using '-' for the access log file makes gunicorn log accesses to stdout +accesslog = "-" +# Using '-' for the error log file makes gunicorn log errors to stderr +errorlog = "-" +loglevel = "info" diff --git a/env.d/development/common.dist b/env.d/development/common.dist new file mode 100644 index 0000000..b118e83 --- /dev/null +++ b/env.d/development/common.dist @@ -0,0 +1,31 @@ +# Django +DJANGO_ALLOWED_HOSTS=* +DJANGO_SECRET_KEY=ThisIsAnExampleKeyForDevPurposeOnly +DJANGO_SETTINGS_MODULE=drive.settings +DJANGO_SUPERUSER_PASSWORD=admin + +# Python +PYTHONPATH=/app + +# drive settings + +# Backend url +DRIVE_BASE_URL="http://localhost:8072" + +# Opensearch +OPENSEARCH_PASSWORD=drive.PASS123 +OPENSEARCH_USE_SSL=false + +# OIDC +OIDC_OP_JWKS_ENDPOINT=http://nginx:8083/realms/drive/protocol/openid-connect/certs +OIDC_OP_AUTHORIZATION_ENDPOINT=http://localhost:8083/realms/drive/protocol/openid-connect/auth +OIDC_OP_TOKEN_ENDPOINT=http://nginx:8083/realms/drive/protocol/openid-connect/token +OIDC_OP_USER_ENDPOINT=http://nginx:8083/realms/drive/protocol/openid-connect/userinfo + +OIDC_RP_CLIENT_ID=drive +OIDC_RP_CLIENT_SECRET=ThisIsAnExampleKeyForDevPurposeOnly +OIDC_RP_SIGN_ALGO=RS256 +OIDC_RP_SCOPES="openid email" + +OIDC_REDIRECT_ALLOWED_HOSTS=["http://localhost:8083", "http://localhost:3000"] +OIDC_AUTH_REQUEST_EXTRA_PARAMS={"acr_values": "eidas1"} diff --git a/env.d/development/crowdin.dist b/env.d/development/crowdin.dist new file mode 100644 index 0000000..1218e3b --- /dev/null +++ b/env.d/development/crowdin.dist @@ -0,0 +1,3 @@ +CROWDIN_API_TOKEN=Your-Api-Token +CROWDIN_PROJECT_ID=Your-Project-Id +CROWDIN_BASE_PATH=/app/src diff --git a/env.d/development/postgresql.dist b/env.d/development/postgresql.dist new file mode 100644 index 0000000..21f03ea --- /dev/null +++ b/env.d/development/postgresql.dist @@ -0,0 +1,11 @@ +# Postgresql db container configuration +POSTGRES_DB=drive +POSTGRES_USER=dinum +POSTGRES_PASSWORD=pass + +# App database configuration +DB_HOST=postgresql +DB_NAME=drive +DB_USER=dinum +DB_PASSWORD=pass +DB_PORT=5432 \ No newline at end of file diff --git a/gitlint/gitlint_emoji.py b/gitlint/gitlint_emoji.py new file mode 100644 index 0000000..59c86ea --- /dev/null +++ b/gitlint/gitlint_emoji.py @@ -0,0 +1,37 @@ +""" +Gitlint extra rule to validate that the message title is of the form +"() " +""" +from __future__ import unicode_literals + +import re + +import requests + +from gitlint.rules import CommitMessageTitle, LineRule, RuleViolation + + +class GitmojiTitle(LineRule): + """ + This rule will enforce that each commit title is of the form "() " + where gitmoji is an emoji from the list defined in https://gitmoji.carloscuesta.me and + subject should be all lowercase + """ + + id = "UC1" + name = "title-should-have-gitmoji-and-scope" + target = CommitMessageTitle + + def validate(self, title, _commit): + """ + Download the list possible gitmojis from the project's github repository and check that + title contains one of them. + """ + gitmojis = requests.get( + "https://raw.githubusercontent.com/carloscuesta/gitmoji/master/packages/gitmojis/src/gitmojis.json" + ).json()["gitmojis"] + emojis = [item["emoji"] for item in gitmojis] + pattern = r"^({:s})\(.*\)\s[a-z].*$".format("|".join(emojis)) + if not re.search(pattern, title): + violation_msg = 'Title does not match regex "() "' + return [RuleViolation(self.id, violation_msg, title)] diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..ce81df3 --- /dev/null +++ b/renovate.json @@ -0,0 +1,13 @@ +{ + "extends": ["github>numerique-gouv/renovate-configuration"], + "dependencyDashboard": true, + "labels": ["dependencies", "noChangeLog"], + "packageRules": [ + { + "enabled": false, + "groupName": "ignored python dependencies", + "matchManagers": ["pep621"], + "matchPackageNames": [] + } + ] +} diff --git a/scripts/install-hooks.sh b/scripts/install-hooks.sh new file mode 100755 index 0000000..7d1c790 --- /dev/null +++ b/scripts/install-hooks.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +mkdir -p "$(dirname -- "${BASH_SOURCE[0]}")/../.git/hooks/" +PRE_COMMIT_FILE="$(dirname -- "${BASH_SOURCE[0]}")/../.git/hooks/pre-commit" + +cat <<'EOF' >$PRE_COMMIT_FILE +#!/bin/bash + +# directories containing potential secrets +DIRS="." + +bold=$(tput bold) +normal=$(tput sgr0) + +# allow to read user input, assigns stdin to keyboard +exec ?$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=yes + +# Minimum lines number of a similarity. +# First implementations of CMS wizards have common fields we do not want to factorize for now +min-similarity-lines=35 + + +[BASIC] + +# Naming style matching correct argument names +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style +#argument-rgx= + +# Naming style matching correct attribute names +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Naming style matching correct class attribute names +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style +#class-attribute-rgx= + +# Naming style matching correct class names +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming-style +#class-rgx= + +# Naming style matching correct constant names +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|urlpatterns|logger)$ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma +good-names=i, + j, + k, + cm, + ex, + Run, + _ + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Naming style matching correct inline iteration names +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style +#inlinevar-rgx= + +# Naming style matching correct method names +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style +method-rgx=([a-z_][a-z0-9_]{2,50}|setUp|set[Uu]pClass|tearDown|tear[Dd]ownClass|assert[A-Z]\w*|maxDiff|test_[a-z0-9_]+)$ + +# Naming style matching correct module names +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style +#variable-rgx= + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of statements in function / method body +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=0 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=builtins.Exception diff --git a/src/backend/MANIFEST.in b/src/backend/MANIFEST.in new file mode 100644 index 0000000..d55d1c4 --- /dev/null +++ b/src/backend/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE +include README.md +recursive-include src/backend/drive *.html *.png *.gif *.css *.ico *.jpg *.jpeg *.po *.mo *.eot *.svg *.ttf *.woff *.woff2 diff --git a/src/backend/__init__.py b/src/backend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/core/__init__.py b/src/backend/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/core/admin.py b/src/backend/core/admin.py new file mode 100644 index 0000000..b637411 --- /dev/null +++ b/src/backend/core/admin.py @@ -0,0 +1,15 @@ +"""Admin config for drive's core app""" +from django.contrib import admin + +from .models import Service + + +@admin.register(Service) +class ServiceAdmin(admin.ModelAdmin): + """Register the serivce model for the admin site""" + + list_display = ("name", "created_at", "is_active") + search_fields = ("name",) + list_filter = ("is_active", "created_at") + ordering = ("-created_at",) + readonly_fields = ("created_at", "token") diff --git a/src/backend/core/api.py b/src/backend/core/api.py new file mode 100644 index 0000000..46454a3 --- /dev/null +++ b/src/backend/core/api.py @@ -0,0 +1,37 @@ +"""Impress core API endpoints""" + +from django.core.exceptions import ValidationError + +from pydantic import ValidationError as PydanticValidationError +from rest_framework import exceptions as drf_exceptions +from rest_framework import status +from rest_framework import views as drf_views +from rest_framework.response import Response + + +def exception_handler(exc, context): + """Handle Django ValidationError as an accepted exception. + + For the parameters, see ``exception_handler`` + This code comes from twidi's gist: + https://gist.github.com/twidi/9d55486c36b6a51bdcb05ce3a763e79f + """ + if isinstance(exc, ValidationError): + detail = exc.message_dict + + if hasattr(exc, "message"): + detail = exc.message + elif hasattr(exc, "messages"): + detail = exc.messages + exc = drf_exceptions.ValidationError(detail=detail) + + elif isinstance(exc, PydanticValidationError): + return Response( + [ + {key: error[key] for key in ("msg", "type", "loc")} + for error in exc.errors() + ], + status=status.HTTP_400_BAD_REQUEST, + ) + + return drf_views.exception_handler(exc, context) diff --git a/src/backend/core/authentication.py b/src/backend/core/authentication.py new file mode 100644 index 0000000..a72aa44 --- /dev/null +++ b/src/backend/core/authentication.py @@ -0,0 +1,34 @@ +"""Token authentication.""" +from django.contrib.auth.models import AnonymousUser + +from rest_framework import authentication, exceptions + +from .models import Service + + +class ServiceTokenAuthentication(authentication.BaseAuthentication): + """A custom authentication looking for valid tokens among registered services""" + + model = Service + + def authenticate(self, request): + """Authenticate token from the "Authorization" header.""" + token = request.headers.get("Authorization") + if not token: + raise exceptions.NotAuthenticated() + + token = token.split(" ")[-1] # Extract token if prefixed with "Token" + + return self.authenticate_credentials(token) + + def authenticate_credentials(self, token): + """Check that the token is registered and valid.""" + try: + service_name = ( + self.model.objects.only("name").get(token=token, is_active=True).name + ) + except self.model.DoesNotExist as excpt: + raise exceptions.AuthenticationFailed("Invalid token.") from excpt + + # We don't associate tokens with a user + return AnonymousUser(), service_name diff --git a/src/backend/core/enums.py b/src/backend/core/enums.py new file mode 100644 index 0000000..5618c66 --- /dev/null +++ b/src/backend/core/enums.py @@ -0,0 +1,12 @@ +"""Enums for drive's core app.""" + +RELEVANCE = "relevance" + +CREATED_AT = "created_at" +IS_PUBLIC = "is_public" +SIZE = "size" +TITLE = "title" +UPDATED_AT = "updated_at" + +ORDER_BY_OPTIONS = (RELEVANCE, TITLE, CREATED_AT, UPDATED_AT, SIZE, IS_PUBLIC) +SOURCE_FIELDS = (TITLE, SIZE, CREATED_AT, UPDATED_AT, IS_PUBLIC) diff --git a/src/backend/core/factories.py b/src/backend/core/factories.py new file mode 100644 index 0000000..f112f2b --- /dev/null +++ b/src/backend/core/factories.py @@ -0,0 +1,53 @@ +"""Factories for the drive's core app""" + +from uuid import uuid4 + +from django.utils import timezone +from django.utils.text import slugify + +import factory +from faker import Faker + +from .models import Service + +fake = Faker() + + +class DocumentSchemaFactory(factory.DictFactory): + """ + A factory for generating dictionaries that represent a document for + indexation for testing and development purposes. + """ + + id = factory.LazyFunction(uuid4) + title = factory.Sequence(lambda n: f"Test title {n!s}") + content = factory.Sequence(lambda n: f"Test content {n!s}") + created_at = factory.LazyFunction( + lambda: fake.date_time_this_decade(tzinfo=timezone.get_current_timezone()) + ) + size = factory.LazyFunction(lambda: fake.random_int(min=0, max=1024**2)) + users = factory.LazyFunction(lambda: [uuid4() for _ in range(3)]) + groups = factory.LazyFunction(lambda: [slugify(fake.word()) for _ in range(3)]) + is_public = factory.Faker("boolean") + + @factory.lazy_attribute + def updated_at(self): + """Ensure updated_at is after created_at and before now""" + return fake.date_time_between( + start_date=self.created_at, + end_date=timezone.now(), + tzinfo=timezone.get_current_timezone(), + ) + + +class ServiceFactory(factory.django.DjangoModelFactory): + """ + A factory for generating service instances for testing and development purposes. + """ + + name = factory.Sequence(lambda n: f"test-index-{n!s}") + created_at = factory.Faker("date_time_this_year", tzinfo=None) + is_active = True + + class Meta: + model = Service diff --git a/src/backend/core/migrations/0001_initial.py b/src/backend/core/migrations/0001_initial.py new file mode 100644 index 0000000..7c72745 --- /dev/null +++ b/src/backend/core/migrations/0001_initial.py @@ -0,0 +1,64 @@ +# Generated by Django 5.0.7 on 2024-09-01 05:31 + +import django.contrib.auth.models +import django.contrib.auth.validators +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('auth', '0012_alter_user_first_name_max_length'), + ] + + operations = [ + migrations.CreateModel( + name='Service', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.SlugField(max_length=20, unique=True)), + ('token', models.CharField(max_length=50)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('is_active', models.BooleanField(default=True)), + ], + options={ + 'verbose_name': 'service', + 'verbose_name_plural': 'services', + 'db_table': 'drive_service', + 'ordering': ['-is_active', '-created_at'], + }, + ), + migrations.CreateModel( + name='User', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('password', models.CharField(max_length=128, verbose_name='password')), + ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), + ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), + ('username', models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username')), + ('first_name', models.CharField(blank=True, max_length=150, verbose_name='first name')), + ('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')), + ('email', models.EmailField(blank=True, max_length=254, verbose_name='email address')), + ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), + ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), + ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.group', verbose_name='groups')), + ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.permission', verbose_name='user permissions')), + ], + options={ + 'verbose_name': 'user', + 'verbose_name_plural': 'users', + 'abstract': False, + }, + managers=[ + ('objects', django.contrib.auth.models.UserManager()), + ], + ), + migrations.AddConstraint( + model_name='service', + constraint=models.CheckConstraint(check=models.Q(('token__length', 50)), name='token_length_exact_50'), + ), + ] diff --git a/src/backend/core/migrations/__init__.py b/src/backend/core/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/core/models.py b/src/backend/core/models.py new file mode 100644 index 0000000..9a0dc46 --- /dev/null +++ b/src/backend/core/models.py @@ -0,0 +1,58 @@ +"""Models for drive's core app""" +import secrets +import string + +from django.contrib.auth.models import AbstractUser +from django.db import models +from django.db.models.functions import Length +from django.utils.text import slugify +from django.utils.translation import gettext_lazy as _ + +models.CharField.register_lookup(Length) +TOKEN_LENGTH = 50 + + +class User(AbstractUser): + """User for the drive application""" + + +class Service(models.Model): + """Service registered to index its documents to our drive""" + + name = models.SlugField(max_length=20, unique=True) + token = models.CharField(max_length=TOKEN_LENGTH) + created_at = models.DateTimeField(auto_now_add=True) + is_active = models.BooleanField(default=True) + + class Meta: + db_table = "drive_service" + verbose_name = _("service") + verbose_name_plural = _("services") + ordering = ["-is_active", "-created_at"] + constraints = [ + models.CheckConstraint( + check=models.Q(token__length=TOKEN_LENGTH), name="token_length_exact_50" + ), + ] + + def __str__(self): + return self.name + + def save(self, *args, **kwargs): + """Automatically slugify the service name and generate a token on creation""" + self.name = slugify(self.name) + if not self.token: + self.token = self.generate_secure_token() + super().save(*args, **kwargs) + + @staticmethod + def generate_secure_token(): + """Generate a secure token with with Python secret module""" + characters = string.ascii_letters + string.digits + string.punctuation + token = "".join(secrets.choice(characters) for _ in range(TOKEN_LENGTH)) + return token + + @property + def index_name(self): + """Compute index name from service name""" + return f"drive-{self.name:s}" diff --git a/src/backend/core/opensearch.py b/src/backend/core/opensearch.py new file mode 100644 index 0000000..802fcf5 --- /dev/null +++ b/src/backend/core/opensearch.py @@ -0,0 +1,45 @@ +"""Opensearch related utils.""" +from django.conf import settings + +from opensearchpy import OpenSearch +from opensearchpy.exceptions import NotFoundError + +client = OpenSearch( + hosts=[{"host": settings.OPENSEARCH_HOST, "port": settings.OPENSEARCH_PORT}], + http_auth=(settings.OPENSEARCH_USER, settings.OPENSEARCH_PASSWORD), + timeout=50, + use_ssl=settings.OPENSEARCH_USE_SSL, + verify_certs=False, +) + + +def ensure_index_exists(index_name): + """Create index if it does not exist""" + try: + client.indices.get(index=index_name) + except NotFoundError: + client.indices.create( + index=index_name, + body={ + "mappings": { + "dynamic": "strict", + "properties": { + "title": { + "type": "keyword", # Primary field for exact matches and sorting + "fields": { + "text": { + "type": "text" # Sub-field for full-text search + } + }, + }, + "content": {"type": "text"}, + "created_at": {"type": "date"}, + "updated_at": {"type": "date"}, + "size": {"type": "long"}, + "users": {"type": "keyword"}, + "groups": {"type": "keyword"}, + "is_public": {"type": "boolean"}, + }, + } + }, + ) diff --git a/src/backend/core/permissions.py b/src/backend/core/permissions.py new file mode 100644 index 0000000..cee7f92 --- /dev/null +++ b/src/backend/core/permissions.py @@ -0,0 +1,12 @@ +"""Permission classes for drive's core app""" + +from rest_framework import permissions + + +class IsAuthAuthenticated(permissions.BasePermission): + """ + Allows access only to auth authenticated users. + """ + + def has_permission(self, request, view): + return bool(request.auth) diff --git a/src/backend/core/schemas.py b/src/backend/core/schemas.py new file mode 100644 index 0000000..3e658ad --- /dev/null +++ b/src/backend/core/schemas.py @@ -0,0 +1,94 @@ +"""Pydantic model to validate documents before indexation.""" +from typing import Annotated, List, Literal, Optional + +from django.utils import timezone +from django.utils.text import slugify + +from pydantic import ( + UUID4, + AwareDatetime, + BaseModel, + ConfigDict, + Field, + conint, + field_validator, + model_validator, +) + +from . import enums + + +class DocumentSchema(BaseModel): + """Schema for validating the documents submitted to our API for indexing""" + + id: UUID4 + title: Annotated[str, Field(max_length=300)] + content: str + created_at: AwareDatetime + updated_at: AwareDatetime + size: Annotated[int, Field(ge=0, le=100 * 1024**3)] # File size limited to 100GB + users: List[UUID4] = Field(default_factory=list) + groups: List[Annotated[str, Field(pattern=r"^[a-z0-9]+(?:-[a-z0-9]+)*$")]] = Field( + default_factory=list + ) + is_public: bool = Field(default=False) + + model_config = ConfigDict( + str_min_length=1, str_strip_whitespace=True, use_enum_values=True + ) + + @field_validator("title") + @staticmethod + def normalize_title(value): + """Normalize the title field by stripping whitespace and converting to lowercase""" + return value.strip().lower() + + @field_validator("created_at", "updated_at") + @staticmethod + def must_be_past(value, info): + """Validate that `created_at` and `updated_at` fields are in the past""" + if value >= timezone.now(): + raise ValueError(f"{info.field_name} must be earlier than now") + return value + + @model_validator(mode="after") + def check_update_at_after_created_at(self): + """Date and time of last modification should be later than date and time of creation""" + if self.created_at > self.updated_at: + raise ValueError("updated_at must be later than created_at") + return self + + @field_validator("groups") + @staticmethod + def validate_groups(groups): + """Validate that group slugs are properly formatted as lowercase and hyphen-separated""" + validated_groups = [] + for value in groups: + slug = slugify(value) + if value != slug: + raise ValueError( + f"Groups must be slugs (lowercase, hyphen-separated): {slug:s}" + ) + validated_groups.append(value) + return validated_groups + + +class SearchQueryParametersSchema(BaseModel): + """Schema for validating the querystring on the search API endpoint""" + + q: str + is_public: Optional[bool] = None + order_by: Optional[Literal[enums.ORDER_BY_OPTIONS]] = Field(default="relevance") + order_direction: Optional[Literal["asc", "desc"]] = Field(default="desc") + page_number: Optional[conint(ge=1)] = Field(default=1) + page_size: Optional[conint(ge=1, le=100)] = Field(default=50) + + @model_validator(mode="before") + @staticmethod + def handle_lists(values): + """Make sure we get strings and ignore multiple values.""" + for key, value in values.items(): + if isinstance(value, list): + # Take the first item if it's a list + values[key] = value[0] if value else None + return values diff --git a/src/backend/core/tests/__init__.py b/src/backend/core/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/core/tests/test_api_documents_index_bulk.py b/src/backend/core/tests/test_api_documents_index_bulk.py new file mode 100644 index 0000000..63f1ce1 --- /dev/null +++ b/src/backend/core/tests/test_api_documents_index_bulk.py @@ -0,0 +1,290 @@ +"""Tests indexing documents in OpenSearch over the API""" +import datetime + +from django.utils import timezone + +import pytest +from rest_framework.test import APIClient + +from core import factories, opensearch + +pytestmark = pytest.mark.django_db + + +def test_api_documents_index_bulk_anonymous(): + """Anonymous requests should not be allowed to index documents in bulk.""" + documents = factories.DocumentSchemaFactory.build_batch(3) + + response = APIClient().post("/api/v1.0/documents/", documents, format="json") + + assert response.status_code == 403 + assert response.json() == { + "detail": "Authentication credentials were not provided." + } + + +def test_api_documents_index_bulk_invalid_token(): + """Requests with invalid tokens should not be allowed to index documents in bulk.""" + documents = factories.DocumentSchemaFactory.build_batch(3) + + response = APIClient().post( + "/api/v1.0/documents/", + documents, + HTTP_AUTHORIZATION="Bearer invalid", + format="json", + ) + + assert response.status_code == 403 + assert response.json() == {"detail": "Invalid token."} + + +def test_api_documents_index_bulk_success(): + """A registered service should be able to index documents in bulk with a valid token.""" + service = factories.ServiceFactory(name="test-service") + documents = factories.DocumentSchemaFactory.build_batch(3) + + response = APIClient().post( + "/api/v1.0/documents/", + documents, + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + format="json", + ) + + assert response.status_code == 207 + responses = response.json() + assert len(responses) == 3 + for result in response.json(): + assert result["status"] == "success" + + +@pytest.mark.parametrize( + "field, invalid_value, error_type, error_message", + [ + ( + "id", + "0f9b1c9d-030f-427a-8a0e-6b7c202c5daz", # invalid UUID b/c contains a z + "uuid_parsing", + ( + "Input should be a valid UUID, invalid character: expected an optional " + "prefix of `urn:uuid:` followed by [0-9a-fA-F-], found `z` at 36" + ), + ), + ("title", 1, "string_type", "Input should be a valid string"), + ( + "title", + "a" * 301, + "string_too_long", + "String should have at most 300 characters", + ), + ("content", 1, "string_type", "Input should be a valid string"), + ( + "created_at", + "invalid_date", + "datetime_from_date_parsing", + "Input should be a valid datetime or date, invalid character in year", + ), + ( + "updated_at", + "invalid_date", + "datetime_from_date_parsing", + "Input should be a valid datetime or date, invalid character in year", + ), + ( + "size", + 64448894017.3, + "int_from_float", + "Input should be a valid integer, got a number with a fractional part", + ), + ( + "size", + "not an integer", + "int_parsing", + "Input should be a valid integer, unable to parse string as an integer", + ), + ( + "users", + "33052c8b-3181-4420-aede-f8396fc0f9a1", + "list_type", + "Input should be a valid list", + ), + ( + "users", + ["33052c8b-3181-4420-aede-f8396fc0f9az"], # invalid UUID b/c contains a z + "uuid_parsing", + ( + "Input should be a valid UUID, invalid character: expected an optional " + "prefix of `urn:uuid:` followed by [0-9a-fA-F-], found `z` at 36" + ), + ), + ( + "groups", + ["not_a_slug"], + "string_pattern_mismatch", + "String should match pattern '^[a-z0-9]+(?:-[a-z0-9]+)*$'", + ), + ( + "groups", + "not-a-list", + "list_type", + "Input should be a valid list", + ), + ( + "is_public", + "invalid_boolean", + "bool_parsing", + "Input should be a valid boolean, unable to interpret input", + ), + ], +) +def test_api_documents_index_bulk_invalid_document( + field, invalid_value, error_type, error_message +): + """Test bulk document indexing with various invalid fields.""" + service = factories.ServiceFactory(name="test-service") + documents = factories.DocumentSchemaFactory.build_batch(3) + + # Modify the first document with the invalid value for the specified field + documents[0][field] = invalid_value + + response = APIClient().post( + "/api/v1.0/documents/", + documents, + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + format="json", + ) + + assert response.status_code == 400 + responses = response.json() + + assert responses[0]["status"] == "error" + assert len(responses[0]["errors"]) == 1 + assert responses[0]["errors"][0]["msg"] == error_message + assert responses[0]["errors"][0]["type"] == error_type + + for i in [1, 2]: + assert responses[i]["status"] == "valid" + + +@pytest.mark.parametrize( + "field", ["id", "title", "content", "size", "created_at", "updated_at"] +) +def test_api_documents_index_bulk_required(field): + """Test bulk document indexing with a required field missing.""" + service = factories.ServiceFactory(name="test-service") + documents = factories.DocumentSchemaFactory.build_batch(3) + + del documents[0][field] + + response = APIClient().post( + "/api/v1.0/documents/", + documents, + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + format="json", + ) + + assert response.status_code == 400 + responses = response.json() + assert responses[0]["status"] == "error" + assert len(responses[0]["errors"]) == 1 + assert responses[0]["errors"][0]["msg"] == "Field required" + assert responses[0]["errors"][0]["type"] == "missing" + + for i in [1, 2]: + assert responses[i]["status"] == "valid" + + +@pytest.mark.parametrize( + "field,default_value", + [ + ("users", []), + ("groups", []), + ("is_public", False), + ], +) +def test_api_documents_index_bulk_default(field, default_value): + """Test bulk document indexing while removing optional fields that have default values.""" + service = factories.ServiceFactory(name="test-service") + documents = factories.DocumentSchemaFactory.build_batch(3) + + del documents[0][field] + + response = APIClient().post( + "/api/v1.0/documents/", + documents, + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + format="json", + ) + + assert response.status_code == 207 + responses = response.json() + assert len(responses) == 3 + for result in response.json(): + assert result["status"] == "success" + + indexed_document = opensearch.client.get( + index=service.index_name, id=responses[0]["_id"] + )["_source"] + assert indexed_document[field] == default_value + + +def test_api_documents_index_bulk_updated_at_before_created_at(): + """Test bulk document indexing with updated_at before created_at.""" + service = factories.ServiceFactory(name="test-service") + documents = factories.DocumentSchemaFactory.build_batch(3) + + documents[0]["updated_at"] = documents[0]["created_at"] - datetime.timedelta( + seconds=1 + ) + + response = APIClient().post( + "/api/v1.0/documents/", + documents, + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + format="json", + ) + + assert response.status_code == 400 + responses = response.json() + assert responses[0]["status"] == "error" + assert len(responses[0]["errors"]) == 1 + assert ( + responses[0]["errors"][0]["msg"] + == "Value error, updated_at must be later than created_at" + ) + assert responses[0]["errors"][0]["type"] == "value_error" + + for i in [1, 2]: + assert responses[i]["status"] == "valid" + + +@pytest.mark.parametrize( + "field", + ["created_at", "updated_at"], +) +def test_api_documents_index_bulk_datetime_future(field): + """Test bulk document indexing with datetimes in the future.""" + service = factories.ServiceFactory(name="test-service") + documents = factories.DocumentSchemaFactory.build_batch(3) + + now = timezone.now() + documents[0][field] = now + datetime.timedelta(seconds=3) + + response = APIClient().post( + "/api/v1.0/documents/", + documents, + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + format="json", + ) + + assert response.status_code == 400 + responses = response.json() + assert responses[0]["status"] == "error" + assert len(responses[0]["errors"]) == 1 + assert ( + responses[0]["errors"][0]["msg"] + == f"Value error, {field:s} must be earlier than now" + ) + assert responses[0]["errors"][0]["type"] == "value_error" + + for i in [1, 2]: + assert responses[i]["status"] == "valid" diff --git a/src/backend/core/tests/test_api_documents_index_single.py b/src/backend/core/tests/test_api_documents_index_single.py new file mode 100644 index 0000000..31bb728 --- /dev/null +++ b/src/backend/core/tests/test_api_documents_index_single.py @@ -0,0 +1,254 @@ +"""Tests indexing documents in OpenSearch over the API""" +import datetime + +from django.utils import timezone + +import pytest +from rest_framework.test import APIClient + +from core import factories, opensearch + +pytestmark = pytest.mark.django_db + + +def test_api_documents_index_single_anonymous(): + """Anonymous requests should not be allowed to index documents.""" + document = factories.DocumentSchemaFactory.build() + + response = APIClient().post("/api/v1.0/documents/", document, format="json") + + assert response.status_code == 403 + assert response.json() == { + "detail": "Authentication credentials were not provided." + } + + +def test_api_documents_index_single_invalid_token(): + """Requests with invalid tokens should not be allowed to index documents.""" + document = factories.DocumentSchemaFactory.build() + + response = APIClient().post( + "/api/v1.0/documents/", + document, + HTTP_AUTHORIZATION="Bearer invalid", + format="json", + ) + + assert response.status_code == 403 + assert response.json() == {"detail": "Invalid token."} + + +def test_api_documents_index_single_success(): + """A registered service should be able to index document with a valid token.""" + service = factories.ServiceFactory(name="test-service") + document = factories.DocumentSchemaFactory.build() + + response = APIClient().post( + "/api/v1.0/documents/", + document, + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + format="json", + ) + + assert response.status_code == 201 + assert response.json()["_id"] == str(document["id"]) + + +@pytest.mark.parametrize( + "field, invalid_value, error_type, error_message", + [ + ( + "id", + "0f9b1c9d-030f-427a-8a0e-6b7c202c5daz", # invalid UUID b/c contains a z* + "uuid_parsing", + ( + "Input should be a valid UUID, invalid character: expected an optional " + "prefix of `urn:uuid:` followed by [0-9a-fA-F-], found `z` at 36" + ), + ), + ("title", 1, "string_type", "Input should be a valid string"), + ( + "title", + "a" * 301, + "string_too_long", + "String should have at most 300 characters", + ), + ("content", 1, "string_type", "Input should be a valid string"), + ( + "created_at", + "invalid_date", + "datetime_from_date_parsing", + "Input should be a valid datetime or date, invalid character in year", + ), + ( + "updated_at", + "invalid_date", + "datetime_from_date_parsing", + "Input should be a valid datetime or date, invalid character in year", + ), + ( + "size", + 64448894017.3, + "int_from_float", + "Input should be a valid integer, got a number with a fractional part", + ), + ( + "size", + "not an integer", + "int_parsing", + "Input should be a valid integer, unable to parse string as an integer", + ), + ( + "users", + "33052c8b-3181-4420-aede-f8396fc0f9a1", + "list_type", + ("Input should be a valid list"), + ), + ( + "users", + ["33052c8b-3181-4420-aede-f8396fc0f9az"], # invalid UUID b/c contains a z* + "uuid_parsing", + ( + "Input should be a valid UUID, invalid character: expected an optional " + "prefix of `urn:uuid:` followed by [0-9a-fA-F-], found `z` at 36" + ), + ), + ( + "groups", + ["not_a_slug"], + "string_pattern_mismatch", + "String should match pattern '^[a-z0-9]+(?:-[a-z0-9]+)*$'", + ), + ( + "groups", + "not-a-list", + "list_type", + ("Input should be a valid list"), + ), + ( + "is_public", + "invalid_boolean", + "bool_parsing", + "Input should be a valid boolean, unable to interpret input", + ), + ], +) +def test_api_documents_index_single_invalid_document( + field, invalid_value, error_type, error_message +): + """Test document indexing with various invalid fields.""" + service = factories.ServiceFactory(name="test-service") + document = factories.DocumentSchemaFactory.build() + + # Modify the document with the invalid value for the specified field + document[field] = invalid_value + + response = APIClient().post( + "/api/v1.0/documents/", + document, + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + format="json", + ) + + assert response.status_code == 400 + assert response.data[0]["msg"] == error_message + assert response.data[0]["type"] == error_type + + +@pytest.mark.parametrize( + "field", ["id", "title", "content", "size", "created_at", "updated_at"] +) +def test_api_documents_index_single_required(field): + """Test document indexing with a required field missing.""" + service = factories.ServiceFactory(name="test-service") + document = factories.DocumentSchemaFactory.build() + + del document[field] + + response = APIClient().post( + "/api/v1.0/documents/", + document, + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + format="json", + ) + + assert response.status_code == 400 + assert response.data[0]["msg"] == "Field required" + assert response.data[0]["type"] == "missing" + + +@pytest.mark.parametrize( + "field,default_value", + [ + ("users", []), + ("groups", []), + ("is_public", False), + ], +) +def test_api_documents_index_single_default(field, default_value): + """Test document indexing while removing optional fields that have default values.""" + service = factories.ServiceFactory(name="test-service") + document = factories.DocumentSchemaFactory.build() + + del document[field] + + response = APIClient().post( + "/api/v1.0/documents/", + document, + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + format="json", + ) + + assert response.status_code == 201 + assert response.json()["_id"] == str(document["id"]) + + indexed_document = opensearch.client.get( + index=service.index_name, id=str(document["id"]) + )["_source"] + assert indexed_document[field] == default_value + + +def test_api_documents_index_single_udpated_at_before_created(): + """Test document indexing with updated_at before created_at.""" + service = factories.ServiceFactory(name="test-service") + document = factories.DocumentSchemaFactory.build() + + document["updated_at"] = document["created_at"] - datetime.timedelta(seconds=1) + + response = APIClient().post( + "/api/v1.0/documents/", + document, + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + format="json", + ) + + assert response.status_code == 400 + assert ( + response.data[0]["msg"] + == "Value error, updated_at must be later than created_at" + ) + assert response.data[0]["type"] == "value_error" + + +@pytest.mark.parametrize( + "field", + ["created_at", "updated_at"], +) +def test_api_documents_index_single_datetime_future(field): + """Test document indexing with datetimes in the future.""" + service = factories.ServiceFactory(name="test-service") + document = factories.DocumentSchemaFactory.build() + + now = timezone.now() + document[field] = now + datetime.timedelta(seconds=3) + + response = APIClient().post( + "/api/v1.0/documents/", + document, + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + format="json", + ) + + assert response.status_code == 400 + assert response.data[0]["msg"] == f"Value error, {field:s} must be earlier than now" + assert response.data[0]["type"] == "value_error" diff --git a/src/backend/core/tests/test_api_documents_search.py b/src/backend/core/tests/test_api_documents_search.py new file mode 100644 index 0000000..e08dfea --- /dev/null +++ b/src/backend/core/tests/test_api_documents_search.py @@ -0,0 +1,448 @@ +""" +Test suite for searching documents in OpenSearch over the API. + +Don't use pytest parametrized tests because batch generation and indexing +of documents is slow and better done only once. +""" +import operator + +import pytest +from opensearchpy.helpers import bulk +from rest_framework.test import APIClient + +from core import enums, factories, opensearch + +pytestmark = pytest.mark.django_db + + +def prepare_index(index_name, documents): + """Prepare the search index before testing a query on it.""" + + documents = documents if isinstance(documents, list) else [documents] + + # Ensure existence of an empty index + opensearch.client.indices.delete(index=index_name) + opensearch.ensure_index_exists(index_name) + + # Bulk index documents + actions = [] + for document in documents: + _source = document.copy() + _id = _source.pop("id") + actions.append( + { + "_op_type": "index", + "_index": index_name, + "_id": _id, + "_source": _source, + } + ) + bulk(opensearch.client, actions) + + opensearch.client.indices.refresh(index=index_name) + assert opensearch.client.count(index=index_name)["count"] == len(documents) + + +def test_api_documents_search_query_title(): + """Searching a document by its title should work as expected""" + service = factories.ServiceFactory(name="test-service") + document = factories.DocumentSchemaFactory.build( + title="The quick brown fox", content="the wolf" + ) + + # Add other documents + other_fox_document = factories.DocumentSchemaFactory.build( + title="The blue fox", content="the wolf" + ) + no_fox_document = factories.DocumentSchemaFactory.build( + title="The brown goat", content="the wolf" + ) + documents = [document, other_fox_document, no_fox_document] + prepare_index(service.index_name, documents) + + response = APIClient().get( + "/api/v1.0/documents/", + {"q": "a quick fox"}, + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + ) + + assert response.status_code == 200 + assert len(response.json()) == 2 + + fox_data = response.json()[0] + assert list(fox_data.keys()) == ["_index", "_id", "_score", "_source", "fields"] + assert fox_data["_id"] == str(document["id"]) + assert fox_data["_source"] == { + "size": document["size"], + "created_at": document["created_at"].isoformat(), + "updated_at": document["updated_at"].isoformat(), + "is_public": document["is_public"], + "title": "The quick brown fox", + } + assert fox_data["fields"] == {"number_of_users": [3], "number_of_groups": [3]} + + other_fox_data = response.json()[1] + assert list(other_fox_data.keys()) == [ + "_index", + "_id", + "_score", + "_source", + "fields", + ] + assert other_fox_data["_id"] == str(other_fox_document["id"]) + assert other_fox_data["_source"] == { + "size": other_fox_document["size"], + "created_at": other_fox_document["created_at"].isoformat(), + "updated_at": other_fox_document["updated_at"].isoformat(), + "is_public": other_fox_document["is_public"], + "title": "The blue fox", + } + assert other_fox_data["fields"] == {"number_of_users": [3], "number_of_groups": [3]} + + +def test_api_documents_search_query_content(): + """Searching a document by its content should work as expected""" + service = factories.ServiceFactory(name="test-service") + document = factories.DocumentSchemaFactory.build( + title="the wolf", content="The quick brown fox" + ) + + # Add other documents + other_fox_document = factories.DocumentSchemaFactory.build( + title="the wolf", content="The blue fox" + ) + no_fox_document = factories.DocumentSchemaFactory.build( + title="the wolf", content="The brown goat" + ) + prepare_index(service.index_name, [document, other_fox_document, no_fox_document]) + + response = APIClient().get( + "/api/v1.0/documents/", + {"q": "a quick fox"}, + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + ) + + assert response.status_code == 200 + assert len(response.json()) == 2 + + fox_data = response.json()[0] + assert list(fox_data.keys()) == ["_index", "_id", "_score", "_source", "fields"] + assert fox_data["_id"] == str(document["id"]) + assert fox_data["_source"] == { + "size": document["size"], + "created_at": document["created_at"].isoformat(), + "updated_at": document["updated_at"].isoformat(), + "is_public": document["is_public"], + "title": document["title"], + } + assert fox_data["fields"] == {"number_of_users": [3], "number_of_groups": [3]} + + other_fox_data = response.json()[1] + assert list(other_fox_data.keys()) == [ + "_index", + "_id", + "_score", + "_source", + "fields", + ] + assert other_fox_data["_id"] == str(other_fox_document["id"]) + assert other_fox_data["_source"] == { + "size": other_fox_document["size"], + "created_at": other_fox_document["created_at"].isoformat(), + "updated_at": other_fox_document["updated_at"].isoformat(), + "is_public": other_fox_document["is_public"], + "title": other_fox_document["title"], + } + assert other_fox_data["fields"] == {"number_of_users": [3], "number_of_groups": [3]} + + +def test_api_documents_search_ordering_by_fields(): + """It should be possible to order by several fields""" + service = factories.ServiceFactory(name="test-service") + documents = factories.DocumentSchemaFactory.build_batch(4) + prepare_index(service.index_name, documents) + + parameters = [ + (enums.TITLE, "asc"), + (enums.TITLE, "desc"), + (enums.CREATED_AT, "asc"), + (enums.CREATED_AT, "desc"), + (enums.UPDATED_AT, "asc"), + (enums.UPDATED_AT, "desc"), + (enums.SIZE, "asc"), + (enums.SIZE, "desc"), + (enums.IS_PUBLIC, "asc"), + (enums.IS_PUBLIC, "desc"), + ] + + for field, direction in parameters: + response = APIClient().get( + f"/api/v1.0/documents/?q=*&order_by={field}&order_direction={direction}", + HTTP_AUTHORIZATION=f"Bearer {service.token}", + ) + + assert response.status_code == 200 + responses = response.json() + assert len(responses) == 4 + + # Check that results are sorted by the field as expected + compare = operator.le if direction == "asc" else operator.ge + for i in range(len(responses) - 1): + assert compare( + responses[i]["_source"][field], responses[i + 1]["_source"][field] + ) + + +def test_api_documents_search_ordering_by_relevance(): + """It should be possible to order by relevance (score)""" + service = factories.ServiceFactory(name="test-service") + documents = factories.DocumentSchemaFactory.build_batch(4) + prepare_index(service.index_name, documents) + + for direction in ["asc", "desc"]: + response = APIClient().get( + f"/api/v1.0/documents/?q=*&order_by=relevance&order_direction={direction}", + HTTP_AUTHORIZATION=f"Bearer {service.token}", + ) + + assert response.status_code == 200 + responses = response.json() + assert len(responses) == 4 + + # Check that results are sorted by score as expected + compare = operator.le if direction == "asc" else operator.ge + for i in range(len(responses) - 1): + assert compare(responses[i]["_score"], responses[i + 1]["_score"]) + + +def test_api_documents_search_ordering_by_unknown_field(): + """Trying to sort by an unknown field should return a 400 error""" + + # Setup: Initialize the service and documents only once + service = factories.ServiceFactory(name="test-service") + documents = factories.DocumentSchemaFactory.build_batch(2) + prepare_index(service.index_name, documents) + + # Define the parameters manually + directions = ["asc", "desc"] + + # Perform the parameterized tests + for direction in directions: + response = APIClient().get( + f"/api/v1.0/documents/?q=*&order_by=unknown&order_direction={direction}", + HTTP_AUTHORIZATION=f"Bearer {service.token}", + ) + + assert response.status_code == 400 + assert response.json() == [ + { + "loc": ["order_by"], + "msg": ( + "Input should be 'relevance', 'title', 'created_at', " + "'updated_at', 'size' or 'is_public'" + ), + "type": "literal_error", + } + ] + + +def test_api_documents_search_ordering_by_unknown_direction(): + """Trying to sort with an unknown direction should return a 400 error""" + service = factories.ServiceFactory(name="test-service") + documents = factories.DocumentSchemaFactory.build_batch(2) + prepare_index(service.index_name, documents) + + for field in enums.ORDER_BY_OPTIONS: + response = APIClient().get( + f"/api/v1.0/documents/?q=*&order_by={field}&order_direction=unknown", + HTTP_AUTHORIZATION=f"Bearer {service.token}", + ) + + assert response.status_code == 400 + assert response.json() == [ + { + "loc": ["order_direction"], + "msg": "Input should be 'asc' or 'desc'", + "type": "literal_error", + } + ] + + +def test_api_documents_search_filtering_by_is_public(): + """It should be possible to filter results by their publication status""" + service = factories.ServiceFactory(name="test-service") + documents = factories.DocumentSchemaFactory.build_batch(4) + prepare_index(service.index_name, documents) + + parameters = [ + [True, "True"], + [False, "False"], + [True, "true"], + [False, "false"], + [True, "1"], + [False, "0"], + ] + + for public_boolean, public_string in parameters: + response = APIClient().get( + f"/api/v1.0/documents/?q=*&is_public={public_string}", + HTTP_AUTHORIZATION=f"Bearer {service.token}", + ) + + assert response.status_code == 200 + responses = response.json() + + for result in responses: + assert public_boolean == result["_source"]["is_public"] + + +# Pagination + + +def test_api_documents_search_pagination_basic(): + """Pagination should correctly return documents for the specified page and page size""" + service = factories.ServiceFactory(name="test-service") + documents = factories.DocumentSchemaFactory.build_batch(9) + ids = [str(doc["id"]) for doc in documents] + prepare_index(service.index_name, documents) + + # Request the first page with a page size of 3 + response = APIClient().get( + "/api/v1.0/documents/?q=*&page_number=1&page_size=3", + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + ) + assert response.status_code == 200 + responses = response.json() + assert len(responses) == 3 # Page size is 3 + assert [r["_id"] for r in responses] == ids[0:3] + + # Request the second page with a page size of 3 + response = APIClient().get( + "/api/v1.0/documents/?q=*&page_number=2&page_size=3", + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + ) + assert response.status_code == 200 + responses = response.json() + assert len(responses) == 3 + assert [r["_id"] for r in responses] == ids[3:6] + + # Request the third page with a page size of 5 (should contain the remaining 3 documents) + response = APIClient().get( + "/api/v1.0/documents/?q=*&page_number=3&page_size=3", + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + ) + + assert response.status_code == 200 + responses = response.json() + assert len(responses) == 3 + assert [r["_id"] for r in responses] == ids[6:9] + + +def test_api_documents_search_pagination_last_page_edge_case(): + """Requesting the last page should return the correct number of remaining documents""" + service = factories.ServiceFactory(name="test-service") + documents = factories.DocumentSchemaFactory.build_batch(8) + ids = [str(doc["id"]) for doc in documents] + prepare_index(service.index_name, documents) + + # Request the first page with a page size of 3 + response = APIClient().get( + "/api/v1.0/documents/?q=*&page_number=1&page_size=3", + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + ) + + assert response.status_code == 200 + assert len(response.json()) == 3 + assert [r["_id"] for r in response.json()] == ids[0:3] + + # Request the third page with a page size of 3 (should contain the last 1 document) + response = APIClient().get( + "/api/v1.0/documents/?q=*&page_number=3&page_size=3", + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + ) + + assert response.status_code == 200 + assert len(response.json()) == 2 # Only 2 documents should be on the last page + assert [r["_id"] for r in response.json()] == ids[6:] + + +def test_api_documents_search_pagination_out_of_bounds(): + """ + Requesting a page number that exceeds the total number of pages should return an empty list + """ + service = factories.ServiceFactory(name="test-service") + documents = factories.DocumentSchemaFactory.build_batch(4) + prepare_index(service.index_name, documents) + + # Request the fourth page with a page size of 2 (there are only 2 pages) + response = APIClient().get( + "/api/v1.0/documents/?q=*&page_number=4&page_size=2", + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + ) + + assert response.status_code == 200 + assert len(response.json()) == 0 # No documents should be returned + + +def test_api_documents_search_pagination_invalid_parameters(): + """Invalid pagination parameters should result in a 400 error""" + service = factories.ServiceFactory(name="test-service") + documents = factories.DocumentSchemaFactory.build_batch(4) + prepare_index(service.index_name, documents) + + parameters = [ + ( + "invalid", + 10, + "int_parsing", + "Input should be a valid integer, unable to parse string as an integer", + ), + ( + 1, + "invalid", + "int_parsing", + "Input should be a valid integer, unable to parse string as an integer", + ), + (-1, 10, "greater_than_equal", "Input should be greater than or equal to 1"), + (1, -10, "greater_than_equal", "Input should be greater than or equal to 1"), + (0, 10, "greater_than_equal", "Input should be greater than or equal to 1"), + (1, 0, "greater_than_equal", "Input should be greater than or equal to 1"), + ] + + for page_number, page_size, error_type, error_message in parameters: + response = APIClient().get( + f"/api/v1.0/documents/?q=*&page_number={page_number}&page_size={page_size}", + HTTP_AUTHORIZATION=f"Bearer {service.token}", + ) + + assert response.status_code == 400 + assert response.data[0]["msg"] == error_message + assert response.data[0]["type"] == error_type + + +def test_api_documents_search_pagination_with_filtering(): + """Pagination should work correctly when combined with filtering by is_public""" + service = factories.ServiceFactory(name="test-service") + public_documents = factories.DocumentSchemaFactory.build_batch(3, is_public=True) + public_ids = [str(doc["id"]) for doc in public_documents] + private_documents = factories.DocumentSchemaFactory.build_batch(2, is_public=False) + prepare_index(service.index_name, public_documents + private_documents) + + # Filter by public documents, request first page + response = APIClient().get( + "/api/v1.0/documents/?q=*&is_public=true&page_number=1&page_size=2", + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + ) + assert response.status_code == 200 + assert len(response.json()) == 2 + assert [r["_id"] for r in response.json()] == public_ids[0:2] + + # Request second page for public documents (remaining 1 document) + response = APIClient().get( + "/api/v1.0/documents/?q=*&is_public=true&page_number=2&page_size=2", + HTTP_AUTHORIZATION=f"Bearer {service.token:s}", + ) + + assert response.status_code == 200 + assert len(response.json()) == 1 + assert [r["_id"] for r in response.json()] == public_ids[2:] diff --git a/src/backend/core/tests/test_models_services.py b/src/backend/core/tests/test_models_services.py new file mode 100644 index 0000000..908c71c --- /dev/null +++ b/src/backend/core/tests/test_models_services.py @@ -0,0 +1,47 @@ +"""Tests Service model for drive's core app.""" + +from django.db import DataError, IntegrityError + +import pytest + +from core import factories + +pytestmark = pytest.mark.django_db + + +def test_models_services_name_unique(): + """The name field should be unique across services.""" + service = factories.ServiceFactory() + + with pytest.raises(IntegrityError): + factories.ServiceFactory(name=service.name) + + +def test_models_services_name_slugified(): + """The name field should be slugified.""" + service = factories.ServiceFactory(name="My service name") + assert service.name == "my-service-name" + + +def test_models_services_index_name(): + """The index name should be computed as a property from the service name.""" + service = factories.ServiceFactory(name="My service name") + assert service.index_name == "drive-my-service-name" + + +def test_models_services_token_50_characters_exact(): + """The token field should be 50 characters long.""" + service = factories.ServiceFactory() + assert len(service.token) == 50 + + +def test_models_services_token_50_characters_less(): + """The token field should not be less than 50 characters long.""" + with pytest.raises(IntegrityError): + factories.ServiceFactory(token="a" * 49) + + +def test_models_services_token_50_characters_more(): + """The token field should be 50 characters long.""" + with pytest.raises(DataError): + factories.ServiceFactory(token="a" * 51) diff --git a/src/backend/core/urls.py b/src/backend/core/urls.py new file mode 100644 index 0000000..d24a21e --- /dev/null +++ b/src/backend/core/urls.py @@ -0,0 +1,8 @@ +"""URL configuration for drive's core app.""" +from django.urls import path + +from .views import DocumentView + +urlpatterns = [ + path("documents/", DocumentView.as_view(), name="document"), +] diff --git a/src/backend/core/views.py b/src/backend/core/views.py new file mode 100644 index 0000000..051e702 --- /dev/null +++ b/src/backend/core/views.py @@ -0,0 +1,239 @@ +"""Views for drive's core app.""" +from pydantic import ValidationError as PydanticValidationError +from rest_framework import status, views +from rest_framework.response import Response +from urllib3.exceptions import ReadTimeoutError + +from . import enums, schemas +from .authentication import ServiceTokenAuthentication +from .opensearch import client, ensure_index_exists +from .permissions import IsAuthAuthenticated + + +class DocumentView(views.APIView): + """ + API view for managing documents in an OpenSearch index. + + This view provides functionality for both indexing and searching documents + within an OpenSearch index dedicated to the authenticated service. The class + supports the following operations: + + 1. **Document Indexing (POST)**: + - Handles both single document and bulk document indexing. + - The index is dynamically determined based on the service authentication token, + ensuring that each service has its own isolated index. + + 2. **Document Search (GET)**: + - Enables searching through indexed documents with support for various filters + and sorting options. + - The search results can be sorted or filtered via querystring parameters. + """ + + authentication_classes = [ServiceTokenAuthentication] + permission_classes = [IsAuthAuthenticated] + + @property + def index_name(self): + """Compute index name from the service name extracted during authentication""" + return f"drive-{self.request.auth}" + + # pylint: disable=too-many-locals + def post(self, request, *args, **kwargs): + """ + API view for indexing documents into OpenSearch index of the authenticated service. + + This view supports both single document indexing and bulk indexing. It handles + the following scenarios based on the type of request data: + + 1. **Single Document Indexing**: If the request contains a single document (as a + dictionary), it will be indexed into an OpenSearch index named `drive-{auth_token}`. + On success, the indexed document is returned with a `201 Created` status. If an + error occurs, a `400 Bad Request` response with an error message is returned. + + 2. **Bulk Indexing**: If the request contains a list of documents, each document is + validated and indexed in bulk. The response includes a detailed status of each + document, indicating whether it was successfully indexed or if an error occurred. + The HTTP status code for the bulk indexing operation is `207 Multi-Status`, and the + response body contains information about the success or failure of each individual + document. + + Methods: + ------- + post(request, *args, **kwargs): + Handles POST requests to index either a single document or a list of documents. + - **Single Document**: Expects a dictionary representing a document. + - **Bulk Indexing**: Expects a list of dictionaries, each representing a document. + + Request Data: + ------------- + For single document indexing: + - Content-Type: application/json + - Body: JSON object representing a document. + + For bulk indexing: + - Content-Type: application/json + - Body: JSON array of JSON objects, each representing a document. + + Responses: + ----------- + - **Single Document Indexing**: + - 201 Created: Returns the indexed document. + - 400 Bad Request: Returns an error message if the document is invalid or if + indexing fails. + + - **Bulk Indexing**: + - 207 Multi-Status if all documents formatting is correct, 400 Bad Request otherwise: + - Returns a list of results for all documents, with details of success and indexing + errors. + """ + + if isinstance(request.data, list): + # Bulk indexing several documents + results = [] + actions = [] + has_errors = False + + for i, document_data in enumerate(request.data): + try: + document = schemas.DocumentSchema(**document_data) + except PydanticValidationError as excpt: + errors = [ + {key: error[key] for key in ("msg", "type", "loc")} + for error in excpt.errors() + ] + results.append({"index": i, "status": "error", "errors": errors}) + has_errors = True + else: + document_dict = document.model_dump() + _id = document_dict.pop("id") + actions.append({"index": {"_id": _id}}) + actions.append(document_dict) + results.append({"index": i, "_id": _id, "status": "valid"}) + if has_errors: + return Response(results, status=status.HTTP_400_BAD_REQUEST) + + ensure_index_exists(self.index_name) + response = client.bulk(index=self.index_name, body=actions) + for i, item in enumerate(response["items"]): + if item["index"]["status"] != 201: + results[i]["status"] = "error" + results[i]["message"] = ( + item["index"].get("error", {}).get("reason", "Unknown error") + ) + else: + results[i]["status"] = "success" + + return Response(results, status=status.HTTP_207_MULTI_STATUS) + + # Indexing a single document + document = schemas.DocumentSchema(**request.data) + document_dict = document.model_dump() + _id = document_dict.pop("id") + try: + client.index(index=self.index_name, body=document_dict, id=_id) + except ReadTimeoutError: + ensure_index_exists(self.index_name) + client.index(index=self.index_name, body=document_dict, id=_id) + + return Response( + {"status": "created", "_id": _id}, status=status.HTTP_201_CREATED + ) + + def get(self, request, *args, **kwargs): + """ + Handle GET requests to perform a search on indexed documents with optional filtering + and ordering. + + The search query should be provided as a query parameter 'q'. The method constructs a + search request to OpenSearch using the specified query, with the option to filter by + 'is_public' and order by 'relevance', 'created_at', 'updated_at', or 'size'. + The results are further filtered by 'users' and 'groups' based on the authentication + header. + + Query Parameters: + --------------- + q : str + The search query string. This is a required parameter. + is_public : bool, optional + Filter results based on the 'is_public' field. + order_by : str, optional + Order results by 'relevance', 'created_at', 'updated_at', or 'size'. + Defaults to 'relevance' if not specified. + order_direction : str, optional + Order direction, 'asc' for ascending or 'desc' for descending. + Defaults to 'desc'. + page_number : int, optional + The page number to retrieve. + Defaults to 1 if not specified. + page_size : int, optional + The number of results to return per page. + Defaults to 50 if not specified. + + Returns: + -------- + Response : rest_framework.response.Response + - 200 OK: Returns a list of search results matching the query. + - 400 Bad Request: If the query parameter 'q' is not provided or invalid. + """ + # Extract and validate query parameters using Pydantic schema + params = schemas.SearchQueryParametersSchema(**request.GET) + + # Compute pagination parameters + from_value = (params.page_number - 1) * params.page_size + size_value = params.page_size + + # Prepare the search query + search_body = { + "_source": enums.SOURCE_FIELDS, # limit the fields to return + "script_fields": { + "number_of_users": {"script": {"source": "doc['users'].size()"}}, + "number_of_groups": {"script": {"source": "doc['groups'].size()"}}, + }, + "query": {"bool": {"must": [], "filter": []}}, + "sort": [], + "from": from_value, + "size": size_value, + } + + # Adding the text query + if params.q == "*": + search_body["query"]["bool"]["must"].append({"match_all": {}}) + else: + search_body["query"]["bool"]["must"].append( + { + "multi_match": { + "query": params.q, + # Give title more importance over content by a power of 3 + "fields": ["title.text^3", "content"], + } + } + ) + + # Add sorting logic based on relevance or specified field + if params.order_by == enums.RELEVANCE: + search_body["sort"].append({"_score": {"order": params.order_direction}}) + else: + search_body["sort"].append( + {params.order_by: {"order": params.order_direction}} + ) + + # Filter by is_public if provided + if params.is_public is not None: + search_body["query"]["bool"]["filter"].append( + {"term": {enums.IS_PUBLIC: params.is_public}} + ) + + # Filter by users and groups based on authentication$ + # user = request.user + # groups = user.get_teams() + + # search_body['query']['bool']['filter'].append({ + # "terms": {"users": [user.sub]} + # }) + + # search_body['query']['bool']['filter'].append({ + # "terms": {"groups": list(groups)} + # }) + + response = client.search(index=self.index_name, body=search_body) + return Response(response["hits"]["hits"], status=status.HTTP_200_OK) diff --git a/src/backend/demo/__init__.py b/src/backend/demo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/demo/defaults.py b/src/backend/demo/defaults.py new file mode 100644 index 0000000..4fa4f51 --- /dev/null +++ b/src/backend/demo/defaults.py @@ -0,0 +1,6 @@ +"""Parameters that define how the demo site will be built.""" + +NB_OBJECTS = { + "documents": 1000, + "services": 5, +} diff --git a/src/backend/demo/management/__init__.py b/src/backend/demo/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/demo/management/commands/__init__.py b/src/backend/demo/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/demo/management/commands/create_demo.py b/src/backend/demo/management/commands/create_demo.py new file mode 100644 index 0000000..5ba9c44 --- /dev/null +++ b/src/backend/demo/management/commands/create_demo.py @@ -0,0 +1,198 @@ +# ruff: noqa: S311, S106 +"""create_demo management command""" + +import logging +import random +import time +from uuid import uuid4 + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.utils import timezone +from django.utils.text import slugify + +from faker import Faker +from opensearchpy.helpers import bulk + +from core import factories, opensearch + +from demo import defaults + +fake = Faker() + +logger = logging.getLogger("drive.commands.demo.create_demo") + + +class BulkIndexing: + """A utility class to index to OpenSearch in bulk by just pushing to a queue.""" + + BATCH_SIZE = 20000 + + def __init__(self, stdout, *args, **kwargs): + """Define actions as a list.""" + self.actions = [] + self.stdout = stdout + self.logger = logging.getLogger(__name__) + + def bulk_index(self): + """Actually index documents in bulk to OpenSearch.""" + _success, failed = bulk(opensearch.client, self.actions, stats_only=False) + + if failed: + self.handle_failures(failed) + + # Clear the actions list after bulk indexing and display progress + self.actions.clear() + self.stdout.write(".", ending="") + + def handle_failures(self, failed): + """Handle the failed bulk operations.""" + for failure in failed: + self.logger.error("Failed to index document: %s", failure) + + def push(self, index_name, _id, document): + """Add a document to queue so that it gets created in bulk.""" + self.actions.append( + { + "_op_type": "index", + "_index": index_name, + "_id": _id, + "_source": document, + } + ) + + if len(self.actions) >= self.BATCH_SIZE: + self.bulk_index() + + def flush(self): + """Flush any remaining documents in the queue.""" + if self.actions: + self.bulk_index() + + def __del__(self): + """Ensure that any remaining documents are indexed when the instance is deleted.""" + self.flush() + + +class Timeit: + """A utility context manager/method decorator to time execution.""" + + total_time = 0 + + def __init__(self, stdout, sentence=None): + """Set the sentence to be displayed for timing information.""" + self.sentence = sentence + self.start = None + self.stdout = stdout + + def __call__(self, func): + """Behavior on call for use as a method decorator.""" + + def timeit_wrapper(*args, **kwargs): + """wrapper to trigger/stop the timer before/after function call.""" + self.__enter__() + result = func(*args, **kwargs) + self.__exit__(None, None, None) + return result + + return timeit_wrapper + + def __enter__(self): + """Start timer upon entering context manager.""" + self.start = time.perf_counter() + if self.sentence: + self.stdout.write(self.sentence, ending=".") + + def __exit__(self, exc_type, exc_value, exc_tb): + """Stop timer and display result upon leaving context manager.""" + if exc_type is not None: + raise exc_type(exc_value) + end = time.perf_counter() + elapsed_time = end - self.start + if self.sentence: + self.stdout.write(f" Took {elapsed_time:g} seconds") + + self.__class__.total_time += elapsed_time + return elapsed_time + + +def generate_document(): + """Generate a realistic document dictionary faster than factory_boy does it.""" + created_at = fake.past_datetime(tzinfo=timezone.get_current_timezone()) + updated_at = fake.date_time_between( + start_date=created_at, + end_date=timezone.now(), + tzinfo=timezone.get_current_timezone(), + ) + + return { + "title": fake.sentence(nb_words=10, variable_nb_words=True), + "content": "\n".join(fake.paragraphs(nb=5)), + "created_at": created_at, + "updated_at": updated_at, + "size": random.randint(0, 100 * 1024**2), + "users": [str(uuid4()) for _ in range(3)], + "groups": [slugify(fake.word()) for _ in range(3)], + "is_public": fake.boolean(), + } + + +def create_demo(stdout): + """ + Create a database with demo data for developers to work in a realistic environment. + """ + opensearch.client.indices.delete("*") + + with Timeit(stdout, "Creating services"): + services = factories.ServiceFactory.create_batch( + defaults.NB_OBJECTS["services"] + ) + for service in services: + opensearch.ensure_index_exists(service.index_name) + opensearch.client.indices.refresh(index=service.index_name) + + with Timeit(stdout, "Creating documents"): + actions = BulkIndexing(stdout) + for _ in range(defaults.NB_OBJECTS["documents"]): + service = random.choice(services) + document = generate_document() + actions.push(service.index_name, uuid4(), document) + actions.flush() + + # Check and report on indexed documents + total_indexed = 0 + for service in services: + opensearch.client.indices.refresh(index=service.index_name) + indexed = opensearch.client.count(index=service.index_name)["count"] + stdout.write(f" - {service.index_name:s}: {indexed:d} documents") + total_indexed += indexed + + stdout.write(f" TOTAL: {total_indexed:d} documents") + + +class Command(BaseCommand): + """A management command to create a demo database.""" + + help = __doc__ + + def add_arguments(self, parser): + """Add argument to require forcing execution when not in debug mode.""" + parser.add_argument( + "-f", + "--force", + action="store_true", + default=False, + help="Force command execution despite DEBUG is set to False", + ) + + def handle(self, *args, **options): + """Handling of the management command.""" + if not settings.DEBUG and not options["force"]: + raise CommandError( + ( + "This command is not meant to be used in production environment " + "except you know what you are doing, if so use --force parameter" + ) + ) + + create_demo(self.stdout) diff --git a/src/backend/demo/tests/__init__.py b/src/backend/demo/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/demo/tests/test_commands_create_demo.py b/src/backend/demo/tests/test_commands_create_demo.py new file mode 100644 index 0000000..d848efe --- /dev/null +++ b/src/backend/demo/tests/test_commands_create_demo.py @@ -0,0 +1,29 @@ +"""Test the `create_demo` management command""" + +from unittest import mock + +from django.core.management import call_command +from django.test import override_settings + +import pytest + +from core import models, opensearch + +from demo import defaults + +pytestmark = pytest.mark.django_db + +TEST_NB_OBJECTS = { + "documents": 4, + "services": 2, +} + + +@override_settings(DEBUG=True) +@mock.patch.dict(defaults.NB_OBJECTS, TEST_NB_OBJECTS) +def test_commands_create_demo(): + """The create_demo management command should create objects as expected.""" + call_command("create_demo") + + assert models.Service.objects.count() == 2 + assert opensearch.client.count()["count"] == 4 diff --git a/src/backend/drive/__init__.py b/src/backend/drive/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/drive/celery_app.py b/src/backend/drive/celery_app.py new file mode 100644 index 0000000..ac81604 --- /dev/null +++ b/src/backend/drive/celery_app.py @@ -0,0 +1,22 @@ +"""Drive celery configuration file.""" +import os + +from celery import Celery +from configurations.importer import install + +# Set the default Django settings module for the 'celery' program. +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "drive.settings") +os.environ.setdefault("DJANGO_CONFIGURATION", "Development") + +install(check_options=True) + +app = Celery("drive") + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object("django.conf:settings", namespace="CELERY") + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() diff --git a/src/backend/drive/settings.py b/src/backend/drive/settings.py new file mode 100755 index 0000000..79a859a --- /dev/null +++ b/src/backend/drive/settings.py @@ -0,0 +1,547 @@ +""" +Django settings for drive project. + +Generated by 'django-admin startproject' using Django 3.1.5. + +For more information on this file, see +https://docs.djangoproject.com/en/3.1/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.1/ref/settings/ +""" +import json +import os + +from django.utils.translation import gettext_lazy as _ + +import sentry_sdk +from configurations import Configuration, values +from sentry_sdk.integrations.django import DjangoIntegration + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +DATA_DIR = os.path.join("/", "data") + + +def get_release(): + """ + Get the current release of the application + + By release, we mean the release from the version.json file à la Mozilla [1] + (if any). If this file has not been found, it defaults to "NA". + + [1] + https://github.com/mozilla-services/Dockerflow/blob/master/docs/version_object.md + """ + # Try to get the current release from the version.json file generated by the + # CI during the Docker image build + try: + with open(os.path.join(BASE_DIR, "version.json"), encoding="utf8") as version: + return json.load(version)["version"] + except FileNotFoundError: + return "NA" # Default: not available + + +class Base(Configuration): + """ + This is the base configuration every configuration (aka environment) should inherit from. It + is recommended to configure third-party applications by creating a configuration mixins in + ./configurations and compose the Base configuration with those mixins. + + It depends on an environment variable that SHOULD be defined: + + * DJANGO_SECRET_KEY + + You may also want to override default configuration by setting the following environment + variables: + + * DJANGO_SENTRY_DSN + * DB_NAME + * DB_HOST + * DB_PASSWORD + * DB_USER + """ + + DEBUG = False + USE_SWAGGER = False + + API_VERSION = "v1.0" + + # Security + ALLOWED_HOSTS = values.ListValue([]) + SECRET_KEY = values.Value(None) + + # Application definition + ROOT_URLCONF = "drive.urls" + WSGI_APPLICATION = "drive.wsgi.application" + + # Database + DATABASES = { + "default": { + "ENGINE": values.Value( + "django.db.backends.postgresql_psycopg2", + environ_name="DB_ENGINE", + environ_prefix=None, + ), + "NAME": values.Value("drive", environ_name="DB_NAME", environ_prefix=None), + "USER": values.Value("dinum", environ_name="DB_USER", environ_prefix=None), + "PASSWORD": values.Value( + "pass", environ_name="DB_PASSWORD", environ_prefix=None + ), + "HOST": values.Value( + "localhost", environ_name="DB_HOST", environ_prefix=None + ), + "PORT": values.Value(5432, environ_name="DB_PORT", environ_prefix=None), + } + } + DEFAULT_AUTO_FIELD = "django.db.models.AutoField" + + # Static files (CSS, JavaScript, Images) + STATIC_URL = "/static/" + STATIC_ROOT = os.path.join(DATA_DIR, "static") + MEDIA_URL = "/media/" + MEDIA_ROOT = os.path.join(DATA_DIR, "media") + + SITE_ID = 1 + + STORAGES = { + "default": { + "BACKEND": "storages.backends.s3.S3Storage", + }, + "staticfiles": { + "BACKEND": values.Value( + "whitenoise.storage.CompressedManifestStaticFilesStorage", + environ_name="STORAGES_STATICFILES_BACKEND", + ), + }, + } + + # Internationalization + # https://docs.djangoproject.com/en/3.1/topics/i18n/ + + # Languages + LANGUAGE_CODE = values.Value("en-us") + + # Careful! Languages should be ordered by priority, as this tuple is used to get + # fallback/default languages throughout the app. + LANGUAGES = values.SingleNestedTupleValue( + ( + ("en-us", _("English")), + ("fr-fr", _("French")), + ) + ) + + LOCALE_PATHS = (os.path.join(BASE_DIR, "locale"),) + + TIME_ZONE = "UTC" + USE_I18N = True + USE_TZ = True + + # Templates + TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [os.path.join(BASE_DIR, "templates")], + "OPTIONS": { + "context_processors": [ + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + "django.template.context_processors.csrf", + "django.template.context_processors.debug", + "django.template.context_processors.i18n", + "django.template.context_processors.media", + "django.template.context_processors.request", + "django.template.context_processors.tz", + ], + "loaders": [ + "django.template.loaders.filesystem.Loader", + "django.template.loaders.app_directories.Loader", + ], + }, + }, + ] + + MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "whitenoise.middleware.WhiteNoiseMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.locale.LocaleMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "corsheaders.middleware.CorsMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "dockerflow.django.middleware.DockerflowMiddleware", + ] + + AUTHENTICATION_BACKENDS = [ + "django.contrib.auth.backends.ModelBackend", + ] + + # Django applications from the highest priority to the lowest + INSTALLED_APPS = [ + # drive + "core", + "demo", + # Third party apps + "corsheaders", + "dockerflow.django", + "drf_spectacular", + "rest_framework", + # Django + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.postgres", + "django.contrib.sessions", + "django.contrib.sites", + "django.contrib.messages", + "django.contrib.staticfiles", + # OIDC third party + "mozilla_django_oidc", + ] + + # Cache + CACHES = { + "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}, + } + + REST_FRAMEWORK = { + "DEFAULT_AUTHENTICATION_CLASSES": ( + "rest_framework.authentication.TokenAuthentication", + ), + "DEFAULT_PARSER_CLASSES": [ + "rest_framework.parsers.JSONParser", + ], + "DEFAULT_PERMISSION_CLASSES": [ + "rest_framework.permissions.IsAuthenticated", + ], + "EXCEPTION_HANDLER": "core.api.exception_handler", + "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination", + "PAGE_SIZE": 20, + "DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.URLPathVersioning", + "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", + } + + # OpenSearch settings + OPENSEARCH_HOST = values.Value( + default="opensearch", environ_name="OPENSEARCH_HOST", environ_prefix=None + ) + OPENSEARCH_PORT = values.Value( + default=9200, environ_name="OPENSEARCH_PORT", environ_prefix=None + ) + OPENSEARCH_USER = values.Value( + default="admin", environ_name="OPENSEARCH_USER", environ_prefix=None + ) + OPENSEARCH_PASSWORD = values.SecretValue( + environ_name="OPENSEARCH_PASSWORD", environ_prefix=None + ) + OPENSEARCH_USE_SSL = values.BooleanValue( + default=True, environ_name="OPENSEARCH_USE_SSL", environ_prefix=None + ) + + SPECTACULAR_SETTINGS = { + "TITLE": "Drive API", + "DESCRIPTION": "This is the drive API schema.", + "VERSION": "0.1.0", + "SERVE_INCLUDE_SCHEMA": False, + "ENABLE_DJANGO_DEPLOY_CHECK": values.BooleanValue( + default=False, + environ_name="SPECTACULAR_SETTINGS_ENABLE_DJANGO_DEPLOY_CHECK", + ), + "COMPONENT_SPLIT_REQUEST": True, + # OTHER SETTINGS + "SWAGGER_UI_DIST": "SIDECAR", # shorthand to use the sidecar instead + "SWAGGER_UI_FAVICON_HREF": "SIDECAR", + "REDOC_DIST": "SIDECAR", + } + + AUTH_USER_MODEL = "core.User" + + # CORS + CORS_ALLOW_CREDENTIALS = True + CORS_ALLOW_ALL_ORIGINS = values.BooleanValue(True) + CORS_ALLOWED_ORIGINS = values.ListValue([]) + CORS_ALLOWED_ORIGIN_REGEXES = values.ListValue([]) + + # Sentry + SENTRY_DSN = values.Value(None, environ_name="SENTRY_DSN") + + # Celery + CELERY_BROKER_URL = values.Value("redis://redis:6379/0") + CELERY_BROKER_TRANSPORT_OPTIONS = values.DictValue({}) + + # Session + SESSION_ENGINE = "django.contrib.sessions.backends.cache" + SESSION_CACHE_ALIAS = "default" + SESSION_COOKIE_AGE = 60 * 60 * 12 + + # OIDC - Authorization Code Flow + OIDC_CREATE_USER = values.BooleanValue( + default=True, + environ_name="OIDC_CREATE_USER", + ) + OIDC_RP_SIGN_ALGO = values.Value( + "RS256", environ_name="OIDC_RP_SIGN_ALGO", environ_prefix=None + ) + OIDC_RP_CLIENT_ID = values.Value( + "drive", environ_name="OIDC_RP_CLIENT_ID", environ_prefix=None + ) + OIDC_RP_CLIENT_SECRET = values.Value( + None, + environ_name="OIDC_RP_CLIENT_SECRET", + environ_prefix=None, + ) + OIDC_OP_JWKS_ENDPOINT = values.Value( + environ_name="OIDC_OP_JWKS_ENDPOINT", environ_prefix=None + ) + OIDC_OP_AUTHORIZATION_ENDPOINT = values.Value( + environ_name="OIDC_OP_AUTHORIZATION_ENDPOINT", environ_prefix=None + ) + OIDC_OP_TOKEN_ENDPOINT = values.Value( + None, environ_name="OIDC_OP_TOKEN_ENDPOINT", environ_prefix=None + ) + OIDC_OP_USER_ENDPOINT = values.Value( + None, environ_name="OIDC_OP_USER_ENDPOINT", environ_prefix=None + ) + OIDC_OP_LOGOUT_ENDPOINT = values.Value( + None, environ_name="OIDC_OP_LOGOUT_ENDPOINT", environ_prefix=None + ) + OIDC_AUTH_REQUEST_EXTRA_PARAMS = values.DictValue( + {}, environ_name="OIDC_AUTH_REQUEST_EXTRA_PARAMS", environ_prefix=None + ) + OIDC_RP_SCOPES = values.Value( + "openid email", environ_name="OIDC_RP_SCOPES", environ_prefix=None + ) + LOGIN_REDIRECT_URL = values.Value( + None, environ_name="LOGIN_REDIRECT_URL", environ_prefix=None + ) + LOGIN_REDIRECT_URL_FAILURE = values.Value( + None, environ_name="LOGIN_REDIRECT_URL_FAILURE", environ_prefix=None + ) + LOGOUT_REDIRECT_URL = values.Value( + None, environ_name="LOGOUT_REDIRECT_URL", environ_prefix=None + ) + OIDC_USE_NONCE = values.BooleanValue( + default=True, environ_name="OIDC_USE_NONCE", environ_prefix=None + ) + OIDC_REDIRECT_REQUIRE_HTTPS = values.BooleanValue( + default=False, environ_name="OIDC_REDIRECT_REQUIRE_HTTPS", environ_prefix=None + ) + OIDC_REDIRECT_ALLOWED_HOSTS = values.ListValue( + default=[], environ_name="OIDC_REDIRECT_ALLOWED_HOSTS", environ_prefix=None + ) + OIDC_STORE_ID_TOKEN = values.BooleanValue( + default=True, environ_name="OIDC_STORE_ID_TOKEN", environ_prefix=None + ) + ALLOW_LOGOUT_GET_METHOD = values.BooleanValue( + default=True, environ_name="ALLOW_LOGOUT_GET_METHOD", environ_prefix=None + ) + + # pylint: disable=invalid-name + @property + def ENVIRONMENT(self): + """Environment in which the application is launched.""" + return self.__class__.__name__.lower() + + # pylint: disable=invalid-name + @property + def RELEASE(self): + """ + Return the release information. + + Delegate to the module function to enable easier testing. + """ + return get_release() + + @classmethod + def post_setup(cls): + """Post setup configuration. + This is the place where you can configure settings that require other + settings to be loaded. + """ + super().post_setup() + + # The SENTRY_DSN setting should be available to activate sentry for an environment + if cls.SENTRY_DSN is not None: + sentry_sdk.init( + dsn=cls.SENTRY_DSN, + environment=cls.__name__.lower(), + release=get_release(), + integrations=[DjangoIntegration()], + ) + with sentry_sdk.configure_scope() as scope: + scope.set_extra("application", "backend") + + +class Build(Base): + """Settings used when the application is built. + + This environment should not be used to run the application. Just to build it with non-blocking + settings. + """ + + SECRET_KEY = values.Value("DummyKey") + STORAGES = { + "default": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + }, + "staticfiles": { + "BACKEND": values.Value( + "whitenoise.storage.CompressedManifestStaticFilesStorage", + environ_name="STORAGES_STATICFILES_BACKEND", + ), + }, + } + + +class Development(Base): + """ + Development environment settings + + We set DEBUG to True and configure the server to respond from all hosts. + """ + + ALLOWED_HOSTS = ["*"] + CORS_ALLOW_ALL_ORIGINS = True + CSRF_TRUSTED_ORIGINS = ["http://localhost:8072", "http://localhost:3000"] + DEBUG = True + + SESSION_COOKIE_NAME = "drive_sessionid" + + USE_SWAGGER = True + + def __init__(self): + # pylint: disable=invalid-name + self.INSTALLED_APPS += ["django_extensions", "drf_spectacular_sidecar"] + + +class Test(Base): + """Test environment settings""" + + LOGGING = values.DictValue( + { + "version": 1, + "disable_existing_loggers": False, + "handlers": { + "console": { + "class": "logging.StreamHandler", + }, + }, + "loggers": { + "drive": { + "handlers": ["console"], + "level": "DEBUG", + }, + }, + } + ) + PASSWORD_HASHERS = [ + "django.contrib.auth.hashers.MD5PasswordHasher", + ] + USE_SWAGGER = True + + CELERY_TASK_ALWAYS_EAGER = values.BooleanValue(True) + + def __init__(self): + # pylint: disable=invalid-name + self.INSTALLED_APPS += ["drf_spectacular_sidecar"] + + +class ContinuousIntegration(Test): + """ + Continuous Integration environment settings + + nota bene: it should inherit from the Test environment. + """ + + +class Production(Base): + """ + Production environment settings + + You must define the ALLOWED_HOSTS environment variable in Production + configuration (and derived configurations): + ALLOWED_HOSTS=["foo.com", "foo.fr"] + """ + + # Security + ALLOWED_HOSTS = values.ListValue(None) + CSRF_TRUSTED_ORIGINS = values.ListValue([]) + SECURE_BROWSER_XSS_FILTER = True + SECURE_CONTENT_TYPE_NOSNIFF = True + + # SECURE_PROXY_SSL_HEADER allows to fix the scheme in Django's HttpRequest + # object when your application is behind a reverse proxy. + # + # Keep this SECURE_PROXY_SSL_HEADER configuration only if : + # - your Django app is behind a proxy. + # - your proxy strips the X-Forwarded-Proto header from all incoming requests + # - Your proxy sets the X-Forwarded-Proto header and sends it to Django + # + # In other cases, you should comment the following line to avoid security issues. + # SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") + SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") + + # Modern browsers require to have the `secure` attribute on cookies with `Samesite=none` + CSRF_COOKIE_SECURE = True + SESSION_COOKIE_SECURE = True + + # Privacy + SECURE_REFERRER_POLICY = "same-origin" + + CACHES = { + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": values.Value( + "redis://redis:6379/1", + environ_name="REDIS_URL", + environ_prefix=None, + ), + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + }, + }, + } + + +class Feature(Production): + """ + Feature environment settings + + nota bene: it should inherit from the Production environment. + """ + + +class Staging(Production): + """ + Staging environment settings + + nota bene: it should inherit from the Production environment. + """ + + +class PreProduction(Production): + """ + Pre-production environment settings + + nota bene: it should inherit from the Production environment. + """ + + +class Demo(Production): + """ + Demonstration environment settings + + nota bene: it should inherit from the Production environment. + """ + + STORAGES = { + "default": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, + } diff --git a/src/backend/drive/urls.py b/src/backend/drive/urls.py new file mode 100644 index 0000000..1040740 --- /dev/null +++ b/src/backend/drive/urls.py @@ -0,0 +1,38 @@ +"""URL configuration for the drive project""" + +from django.conf import settings +from django.contrib import admin +from django.urls import include, path, re_path + +from drf_spectacular.views import ( + SpectacularJSONAPIView, + SpectacularRedocView, + SpectacularSwaggerView, +) + +urlpatterns = [ + path("admin/", admin.site.urls), + path(f"api/{settings.API_VERSION}/", include("core.urls")), +] + +if settings.USE_SWAGGER or settings.DEBUG: + urlpatterns += [ + path( + f"{settings.API_VERSION}/swagger.json", + SpectacularJSONAPIView.as_view( + api_version=settings.API_VERSION, + urlconf="core.urls", + ), + name="client-api-schema", + ), + path( + f"{settings.API_VERSION}//swagger/", + SpectacularSwaggerView.as_view(url_name="client-api-schema"), + name="swagger-ui-schema", + ), + re_path( + f"{settings.API_VERSION}//redoc/", + SpectacularRedocView.as_view(url_name="client-api-schema"), + name="redoc-schema", + ), + ] diff --git a/src/backend/drive/wsgi.py b/src/backend/drive/wsgi.py new file mode 100644 index 0000000..a021ef2 --- /dev/null +++ b/src/backend/drive/wsgi.py @@ -0,0 +1,17 @@ +""" +WSGI config for the drive project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.1/howto/deployment/wsgi/ +""" + +import os + +from configurations.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "drive.settings") +os.environ.setdefault("DJANGO_CONFIGURATION", "Development") + +application = get_wsgi_application() diff --git a/src/backend/manage.py b/src/backend/manage.py new file mode 100644 index 0000000..34c7eb4 --- /dev/null +++ b/src/backend/manage.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +""" +drive's sandbox management script. +""" +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "drive.settings") + os.environ.setdefault("DJANGO_CONFIGURATION", "Development") + + from configurations.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml new file mode 100644 index 0000000..1a847d2 --- /dev/null +++ b/src/backend/pyproject.toml @@ -0,0 +1,132 @@ +# +# drive package +# +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "drive" +version = "1.1.0" +authors = [{ "name" = "DINUM", "email" = "dev@mail.numerique.gouv.fr" }] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Framework :: Django", + "Framework :: Django :: 5", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", +] +description = "An application to print markdown to pdf from a set of managed templates." +keywords = ["Django", "Contacts", "Templates", "RBAC"] +license = { file = "LICENSE" } +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "celery[redis]==5.3.6", + "django-configurations==2.5", + "django-cors-headers==4.3.1", + "redis==5.0.3", + "django-redis==5.4.0", + "django==5.0.7", + "djangorestframework==3.14.0", + "drf_spectacular==0.26.5", + "dockerflow==2022.8.0", + "factory_boy==3.3.0", + "gunicorn==22.0.0", + "psycopg[binary]==3.1.14", + "requests==2.32.2", + "sentry-sdk==2.8.0", + "url-normalize==1.4.3", + "whitenoise==6.6.0", + "mozilla-django-oidc==4.0.0", + "opensearch-py==2.7.1", + "pydantic==2.8.2", + "pyjwt==2.9.0", +] + +[project.urls] +"Bug Tracker" = "https://github.com/numerique-gouv/drive/issues/new" +"Changelog" = "https://github.com/numerique-gouv/drive/blob/main/CHANGELOG.md" +"Homepage" = "https://github.com/numerique-gouv/drive" +"Repository" = "https://github.com/numerique-gouv/drive" + +[project.optional-dependencies] +dev = [ + "django-extensions==3.2.3", + "drf-spectacular-sidecar==2023.12.1", + "faker==28.0.0", + "ipdb==0.13.13", + "ipython==8.18.1", + "pyfakefs==5.3.2", + "pylint-django==2.5.5", + "pylint==3.0.3", + "pytest-cov==4.1.0", + "pytest-django==4.7.0", + "pytest==7.4.3", + "pytest-icdiff==0.8", + "pytest-xdist==3.5.0", + "responses==0.24.1", + "ruff==0.1.6", + "types-requests==2.31.0.10", +] + +[tool.setuptools] +packages = { find = { where = ["."], exclude = ["tests"] } } +zip-safe = true + +[tool.distutils.bdist_wheel] +universal = true + +[tool.ruff] +exclude = [ + ".git", + ".venv", + "build", + "venv", + "__pycache__", + "*/migrations/*", +] +ignore= ["DJ001", "PLR2004"] +line-length = 88 + + +[tool.ruff.lint] +select = [ + "B", # flake8-bugbear + "BLE", # flake8-blind-except + "C4", # flake8-comprehensions + "DJ", # flake8-django + "I", # isort + "PLC", # pylint-convention + "PLE", # pylint-error + "PLR", # pylint-refactoring + "PLW", # pylint-warning + "RUF100", # Ruff unused-noqa + "RUF200", # Ruff check pyproject.toml + "S", # flake8-bandit + "SLF", # flake8-self + "T20", # flake8-print +] + +[tool.ruff.lint.isort] +section-order = ["future","standard-library","django","third-party","drive","first-party","local-folder"] +sections = { drive=["core"], django=["django"] } + +[tool.ruff.per-file-ignores] +"**/tests/*" = ["S", "SLF"] + +[tool.pytest.ini_options] +addopts = [ + "-v", + "--cov-report", + "term-missing", + # Allow test files to have the same name in different directories. + "--import-mode=importlib", +] +python_files = [ + "test_*.py", + "tests.py", +] diff --git a/src/backend/setup.py b/src/backend/setup.py new file mode 100644 index 0000000..5573f43 --- /dev/null +++ b/src/backend/setup.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +"""Setup file for the drive module. All configuration stands in the setup.cfg file.""" +# coding: utf-8 + +from setuptools import setup + +setup() diff --git a/src/helm/drive/Chart.yaml b/src/helm/drive/Chart.yaml new file mode 100644 index 0000000..d9d9f09 --- /dev/null +++ b/src/helm/drive/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +type: application +name: drive +version: 0.0.1 diff --git a/src/helm/drive/README.md b/src/helm/drive/README.md new file mode 100644 index 0000000..959e9cd --- /dev/null +++ b/src/helm/drive/README.md @@ -0,0 +1,128 @@ +# Drive helm chart + +## Parameters + +### General configuration + +| Name | Description | Value | +| ------------------------------------------ | ---------------------------------------------------- | ------------------------ | +| `image.repository` | Repository to use to pull drive's container image | `lasuite/drive-backend` | +| `image.tag` | drive's container tag | `latest` | +| `image.pullPolicy` | Container image pull policy | `IfNotPresent` | +| `image.credentials.username` | Username for container registry authentication | | +| `image.credentials.password` | Password for container registry authentication | | +| `image.credentials.registry` | Registry url for which the credentials are specified | | +| `image.credentials.name` | Name of the generated secret for imagePullSecrets | | +| `nameOverride` | Override the chart name | `""` | +| `fullnameOverride` | Override the full application name | `""` | +| `ingress.enabled` | whether to enable the Ingress or not | `false` | +| `ingress.className` | IngressClass to use for the Ingress | `nil` | +| `ingress.host` | Host for the Ingress | `drive.example.com` | +| `ingress.path` | Path to use for the Ingress | `/` | +| `ingress.hosts` | Additional host to configure for the Ingress | `[]` | +| `ingress.tls.enabled` | Weather to enable TLS for the Ingress | `true` | +| `ingress.tls.additional[].secretName` | Secret name for additional TLS config | | +| `ingress.tls.additional[].hosts[]` | Hosts for additional TLS config | | +| `ingress.customBackends` | Add custom backends to ingress | `[]` | +| `ingressAdmin.enabled` | whether to enable the Ingress or not | `false` | +| `ingressAdmin.className` | IngressClass to use for the Ingress | `nil` | +| `ingressAdmin.host` | Host for the Ingress | `drive.example.com` | +| `ingressAdmin.path` | Path to use for the Ingress | `/admin` | +| `ingressAdmin.hosts` | Additional host to configure for the Ingress | `[]` | +| `ingressAdmin.tls.enabled` | Weather to enable TLS for the Ingress | `true` | +| `ingressAdmin.tls.additional[].secretName` | Secret name for additional TLS config | | +| `ingressAdmin.tls.additional[].hosts[]` | Hosts for additional TLS config | | + +### backend + +| Name | Description | Value | +| ----------------------------------------------------- | ---------------------------------------------------------------------------------- | ----------------------------------------------- | +| `backend.command` | Override the backend container command | `[]` | +| `backend.args` | Override the backend container args | `[]` | +| `backend.replicas` | Amount of backend replicas | `3` | +| `backend.shareProcessNamespace` | Enable share process namespace between containers | `false` | +| `backend.sidecars` | Add sidecars containers to backend deployment | `[]` | +| `backend.securityContext` | Configure backend Pod security context | `nil` | +| `backend.envVars` | Configure backend container environment variables | `undefined` | +| `backend.envVars.BY_VALUE` | Example environment variable by setting value directly | | +| `backend.envVars.FROM_CONFIGMAP.configMapKeyRef.name` | Name of a ConfigMap when configuring env vars from a ConfigMap | | +| `backend.envVars.FROM_CONFIGMAP.configMapKeyRef.key` | Key within a ConfigMap when configuring env vars from a ConfigMap | | +| `backend.envVars.FROM_SECRET.secretKeyRef.name` | Name of a Secret when configuring env vars from a Secret | | +| `backend.envVars.FROM_SECRET.secretKeyRef.key` | Key within a Secret when configuring env vars from a Secret | | +| `backend.podAnnotations` | Annotations to add to the backend Pod | `{}` | +| `backend.service.type` | backend Service type | `ClusterIP` | +| `backend.service.port` | backend Service listening port | `80` | +| `backend.service.targetPort` | backend container listening port | `8000` | +| `backend.service.annotations` | Annotations to add to the backend Service | `{}` | +| `backend.migrate.command` | backend migrate command | `["python","manage.py","migrate","--no-input"]` | +| `backend.migrate.restartPolicy` | backend migrate job restart policy | `Never` | +| `backend.probes.liveness.path` | Configure path for backend HTTP liveness probe | `/__heartbeat__` | +| `backend.probes.liveness.targetPort` | Configure port for backend HTTP liveness probe | `undefined` | +| `backend.probes.liveness.initialDelaySeconds` | Configure initial delay for backend liveness probe | `10` | +| `backend.probes.liveness.initialDelaySeconds` | Configure timeout for backend liveness probe | `10` | +| `backend.probes.startup.path` | Configure path for backend HTTP startup probe | `undefined` | +| `backend.probes.startup.targetPort` | Configure port for backend HTTP startup probe | `undefined` | +| `backend.probes.startup.initialDelaySeconds` | Configure initial delay for backend startup probe | `undefined` | +| `backend.probes.startup.initialDelaySeconds` | Configure timeout for backend startup probe | `undefined` | +| `backend.probes.readiness.path` | Configure path for backend HTTP readiness probe | `/__lbheartbeat__` | +| `backend.probes.readiness.targetPort` | Configure port for backend HTTP readiness probe | `undefined` | +| `backend.probes.readiness.initialDelaySeconds` | Configure initial delay for backend readiness probe | `10` | +| `backend.probes.readiness.initialDelaySeconds` | Configure timeout for backend readiness probe | `10` | +| `backend.resources` | Resource requirements for the backend container | `{}` | +| `backend.nodeSelector` | Node selector for the backend Pod | `{}` | +| `backend.tolerations` | Tolerations for the backend Pod | `[]` | +| `backend.affinity` | Affinity for the backend Pod | `{}` | +| `backend.persistence` | Additional volumes to create and mount on the backend. Used for debugging purposes | `{}` | +| `backend.persistence.volume-name.size` | Size of the additional volume | | +| `backend.persistence.volume-name.type` | Type of the additional volume, persistentVolumeClaim or emptyDir | | +| `backend.persistence.volume-name.mountPath` | Path where the volume should be mounted to | | +| `backend.extraVolumeMounts` | Additional volumes to mount on the backend. | `[]` | +| `backend.extraVolumes` | Additional volumes to mount on the backend. | `[]` | + +### frontend + +| Name | Description | Value | +| ------------------------------------------------------ | ----------------------------------------------------------------------------------- | ------------------------- | +| `frontend.image.repository` | Repository to use to pull drive's frontend container image | `lasuite/drive-frontend` | +| `frontend.image.tag` | drive's frontend container tag | `latest` | +| `frontend.image.pullPolicy` | frontend container image pull policy | `IfNotPresent` | +| `frontend.command` | Override the frontend container command | `[]` | +| `frontend.args` | Override the frontend container args | `[]` | +| `frontend.replicas` | Amount of frontend replicas | `3` | +| `frontend.shareProcessNamespace` | Enable share process namefrontend between containers | `false` | +| `frontend.sidecars` | Add sidecars containers to frontend deployment | `[]` | +| `frontend.securityContext` | Configure frontend Pod security context | `nil` | +| `frontend.envVars` | Configure frontend container environment variables | `undefined` | +| `frontend.envVars.BY_VALUE` | Example environment variable by setting value directly | | +| `frontend.envVars.FROM_CONFIGMAP.configMapKeyRef.name` | Name of a ConfigMap when configuring env vars from a ConfigMap | | +| `frontend.envVars.FROM_CONFIGMAP.configMapKeyRef.key` | Key within a ConfigMap when configuring env vars from a ConfigMap | | +| `frontend.envVars.FROM_SECRET.secretKeyRef.name` | Name of a Secret when configuring env vars from a Secret | | +| `frontend.envVars.FROM_SECRET.secretKeyRef.key` | Key within a Secret when configuring env vars from a Secret | | +| `frontend.podAnnotations` | Annotations to add to the frontend Pod | `{}` | +| `frontend.service.type` | frontend Service type | `ClusterIP` | +| `frontend.service.port` | frontend Service listening port | `80` | +| `frontend.service.targetPort` | frontend container listening port | `8080` | +| `frontend.service.annotations` | Annotations to add to the frontend Service | `{}` | +| `frontend.probes` | Configure probe for frontend | `{}` | +| `frontend.probes.liveness.path` | Configure path for frontend HTTP liveness probe | | +| `frontend.probes.liveness.targetPort` | Configure port for frontend HTTP liveness probe | | +| `frontend.probes.liveness.initialDelaySeconds` | Configure initial delay for frontend liveness probe | | +| `frontend.probes.liveness.initialDelaySeconds` | Configure timeout for frontend liveness probe | | +| `frontend.probes.startup.path` | Configure path for frontend HTTP startup probe | | +| `frontend.probes.startup.targetPort` | Configure port for frontend HTTP startup probe | | +| `frontend.probes.startup.initialDelaySeconds` | Configure initial delay for frontend startup probe | | +| `frontend.probes.startup.initialDelaySeconds` | Configure timeout for frontend startup probe | | +| `frontend.probes.readiness.path` | Configure path for frontend HTTP readiness probe | | +| `frontend.probes.readiness.targetPort` | Configure port for frontend HTTP readiness probe | | +| `frontend.probes.readiness.initialDelaySeconds` | Configure initial delay for frontend readiness probe | | +| `frontend.probes.readiness.initialDelaySeconds` | Configure timeout for frontend readiness probe | | +| `frontend.resources` | Resource requirements for the frontend container | `{}` | +| `frontend.nodeSelector` | Node selector for the frontend Pod | `{}` | +| `frontend.tolerations` | Tolerations for the frontend Pod | `[]` | +| `frontend.affinity` | Affinity for the frontend Pod | `{}` | +| `frontend.persistence` | Additional volumes to create and mount on the frontend. Used for debugging purposes | `{}` | +| `frontend.persistence.volume-name.size` | Size of the additional volume | | +| `frontend.persistence.volume-name.type` | Type of the additional volume, persistentVolumeClaim or emptyDir | | +| `frontend.persistence.volume-name.mountPath` | Path where the volume should be mounted to | | +| `frontend.extraVolumeMounts` | Additional volumes to mount on the frontend. | `[]` | +| `frontend.extraVolumes` | Additional volumes to mount on the frontend. | `[]` | diff --git a/src/helm/drive/generate-readme.sh b/src/helm/drive/generate-readme.sh new file mode 100644 index 0000000..edbd280 --- /dev/null +++ b/src/helm/drive/generate-readme.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +docker image ls | grep readme-generator-for-helm +if [ "$?" -ne "0" ]; then + git clone https://github.com/bitnami/readme-generator-for-helm.git /tmp/readme-generator-for-helm + cd /tmp/readme-generator-for-helm + docker build -t readme-generator-for-helm:latest . + cd $(dirname -- "${BASH_SOURCE[0]}") +fi +docker run --rm -it -v ./values.yaml:/app/values.yaml -v ./README.md:/app/README.md readme-generator-for-helm:latest readme-generator -v values.yaml -r README.md diff --git a/src/helm/drive/templates/_helpers.tpl b/src/helm/drive/templates/_helpers.tpl new file mode 100644 index 0000000..18518d7 --- /dev/null +++ b/src/helm/drive/templates/_helpers.tpl @@ -0,0 +1,184 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "drive.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "drive.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "drive.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +drive.labels +*/}} +{{- define "drive.labels" -}} +helm.sh/chart: {{ include "drive.chart" . }} +{{ include "drive.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "drive.selectorLabels" -}} +app.kubernetes.io/name: {{ include "drive.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +transform dictionnary of environment variables +Usage : {{ include "drive.env.transformDict" .Values.envVars }} + +Example: +envVars: + # Using simple strings as env vars + ENV_VAR_NAME: "envVar value" + # Using a value from a configMap + ENV_VAR_FROM_CM: + configMapKeyRef: + name: cm-name + key: "key_in_cm" + # Using a value from a secret + ENV_VAR_FROM_SECRET: + secretKeyRef: + name: secret-name + key: "key_in_secret" +*/}} +{{- define "drive.env.transformDict" -}} +{{- range $key, $value := . }} +- name: {{ $key | quote }} +{{- if $value | kindIs "map" }} + valueFrom: {{ $value | toYaml | nindent 4 }} +{{- else }} + value: {{ $value | quote }} +{{- end }} +{{- end }} +{{- end }} + + +{{/* +drive env vars +*/}} +{{- define "drive.common.env" -}} +{{- $topLevelScope := index . 0 -}} +{{- $workerScope := index . 1 -}} +{{- include "drive.env.transformDict" $workerScope.envVars -}} +{{- end }} + +{{/* +Common labels + +Requires array with top level scope and component name +*/}} +{{- define "drive.common.labels" -}} +{{- $topLevelScope := index . 0 -}} +{{- $component := index . 1 -}} +{{- include "drive.labels" $topLevelScope }} +app.kubernetes.io/component: {{ $component }} +{{- end }} + +{{/* +Common selector labels + +Requires array with top level scope and component name +*/}} +{{- define "drive.common.selectorLabels" -}} +{{- $topLevelScope := index . 0 -}} +{{- $component := index . 1 -}} +{{- include "drive.selectorLabels" $topLevelScope }} +app.kubernetes.io/component: {{ $component }} +{{- end }} + +{{- define "drive.probes.abstract" -}} +{{- if .exec -}} +exec: +{{- toYaml .exec | nindent 2 }} +{{- else if .tcpSocket -}} +tcpSocket: +{{- toYaml .tcpSocket | nindent 2 }} +{{- else -}} +httpGet: + path: {{ .path }} + port: {{ .targetPort }} +{{- end }} +initialDelaySeconds: {{ .initialDelaySeconds | eq nil | ternary 0 .initialDelaySeconds }} +timeoutSeconds: {{ .timeoutSeconds | eq nil | ternary 1 .timeoutSeconds }} +{{- end }} + +{{/* +Full name for the backend + +Requires top level scope +*/}} +{{- define "drive.backend.fullname" -}} +{{ include "drive.fullname" . }}-backend +{{- end }} + +{{/* +Full name for the frontend + +Requires top level scope +*/}} +{{- define "drive.frontend.fullname" -}} +{{ include "drive.fullname" . }}-frontend +{{- end }} + +{{/* +Full name for the webrtc + +Requires top level scope +*/}} +{{- define "drive.webrtc.fullname" -}} +{{ include "drive.fullname" . }}-webrtc +{{- end }} + +{{/* +Usage : {{ include "drive.secret.dockerconfigjson.name" (dict "fullname" (include "drive.fullname" .) "imageCredentials" .Values.path.to.the.image1) }} +*/}} +{{- define "drive.secret.dockerconfigjson.name" }} +{{- if (default (dict) .imageCredentials).name }}{{ .imageCredentials.name }}{{ else }}{{ .fullname | trunc 63 | trimSuffix "-" }}-dockerconfig{{ end -}} +{{- end }} + +{{/* +Usage : {{ include "drive.secret.dockerconfigjson" (dict "fullname" (include "drive.fullname" .) "imageCredentials" .Values.path.to.the.image1) }} +*/}} +{{- define "drive.secret.dockerconfigjson" }} +{{- if .imageCredentials -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "drive.secret.dockerconfigjson.name" (dict "fullname" .fullname "imageCredentials" .imageCredentials) }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": before-hook-creation +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: {{ template "drive.secret.dockerconfigjson.data" .imageCredentials }} +{{- end -}} +{{- end }} diff --git a/src/helm/drive/templates/backend_deployment.yaml b/src/helm/drive/templates/backend_deployment.yaml new file mode 100644 index 0000000..0aa7b1b --- /dev/null +++ b/src/helm/drive/templates/backend_deployment.yaml @@ -0,0 +1,136 @@ +{{- $envVars := include "drive.common.env" (list . .Values.backend) -}} +{{- $fullName := include "drive.backend.fullname" . -}} +{{- $component := "backend" -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "drive.common.labels" (list . $component) | nindent 4 }} +spec: + replicas: {{ .Values.backend.replicas }} + selector: + matchLabels: + {{- include "drive.common.selectorLabels" (list . $component) | nindent 6 }} + template: + metadata: + annotations: + {{- with .Values.backend.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "drive.common.selectorLabels" (list . $component) | nindent 8 }} + spec: + {{- if $.Values.image.credentials }} + imagePullSecrets: + - name: {{ include "drive.secret.dockerconfigjson.name" (dict "fullname" (include "drive.fullname" .) "imageCredentials" $.Values.image.credentials) }} + {{- end}} + shareProcessNamespace: {{ .Values.backend.shareProcessNamespace }} + containers: + {{- with .Values.backend.sidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} + - name: {{ .Chart.Name }} + image: "{{ (.Values.backend.image | default dict).repository | default .Values.image.repository }}:{{ (.Values.backend.image | default dict).tag | default .Values.image.tag }}" + imagePullPolicy: {{ (.Values.backend.image | default dict).pullPolicy | default .Values.image.pullPolicy }} + {{- with .Values.backend.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.backend.args }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- if $envVars}} + {{- $envVars | indent 12 }} + {{- end }} + {{- with .Values.backend.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.backend.service.targetPort }} + protocol: TCP + {{- if .Values.backend.probes.liveness }} + livenessProbe: + {{- include "drive.probes.abstract" (merge .Values.backend.probes.liveness (dict "targetPort" .Values.backend.service.targetPort )) | nindent 12 }} + {{- end }} + {{- if .Values.backend.probes.readiness }} + readinessProbe: + {{- include "drive.probes.abstract" (merge .Values.backend.probes.readiness (dict "targetPort" .Values.backend.service.targetPort )) | nindent 12 }} + {{- end }} + {{- if .Values.backend.probes.startup }} + startupProbe: + {{- include "drive.probes.abstract" (merge .Values.backend.probes.startup (dict "targetPort" .Values.backend.service.targetPort )) | nindent 12 }} + {{- end }} + {{- with .Values.backend.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + mountPath: {{ $value.path }} + subPath: content + {{- end }} + {{- range $name, $volume := .Values.backend.persistence }} + - name: "{{ $name }}" + mountPath: "{{ $volume.mountPath }}" + {{- end }} + {{- range .Values.backend.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- with .Values.backend.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.backend.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.backend.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + configMap: + name: "{{ include "drive.fullname" $ }}-files-{{ $index }}" + {{- end }} + {{- range $name, $volume := .Values.backend.persistence }} + - name: "{{ $name }}" + {{- if eq $volume.type "emptyDir" }} + emptyDir: {} + {{- else }} + persistentVolumeClaim: + claimName: "{{ $fullName }}-{{ $name }}" + {{- end }} + {{- end }} + {{- range .Values.backend.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 12 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 12 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 12 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 12 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} diff --git a/src/helm/drive/templates/backend_job.yaml b/src/helm/drive/templates/backend_job.yaml new file mode 100644 index 0000000..e8152b7 --- /dev/null +++ b/src/helm/drive/templates/backend_job.yaml @@ -0,0 +1,121 @@ +{{- $envVars := include "drive.common.env" (list . .Values.backend) -}} +{{- $fullName := include "drive.backend.fullname" . -}} +{{- $component := "backend" -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $fullName }}-migrate + namespace: {{ .Release.Namespace | quote }} + {{- with .Values.backend.migrateJobAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "drive.common.labels" (list . $component) | nindent 4 }} +spec: + template: + metadata: + annotations: + {{- with .Values.backend.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "drive.common.selectorLabels" (list . $component) | nindent 8 }} + spec: + {{- if $.Values.image.credentials }} + imagePullSecrets: + - name: {{ include "drive.secret.dockerconfigjson.name" (dict "fullname" (include "drive.fullname" .) "imageCredentials" $.Values.image.credentials) }} + {{- end}} + shareProcessNamespace: {{ .Values.backend.shareProcessNamespace }} + containers: + {{- with .Values.backend.sidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} + - name: {{ .Chart.Name }} + image: "{{ (.Values.backend.image | default dict).repository | default .Values.image.repository }}:{{ (.Values.backend.image | default dict).tag | default .Values.image.tag }}" + imagePullPolicy: {{ (.Values.backend.image | default dict).pullPolicy | default .Values.image.pullPolicy }} + {{- with .Values.backend.migrate.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.backend.args }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- if $envVars}} + {{- $envVars | indent 12 }} + {{- end }} + {{- with .Values.backend.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.backend.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + mountPath: {{ $value.path }} + subPath: content + {{- end }} + {{- range $name, $volume := .Values.backend.persistence }} + - name: "{{ $name }}" + mountPath: "{{ $volume.mountPath }}" + {{- end }} + {{- range .Values.backend.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- with .Values.backend.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.backend.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.backend.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + restartPolicy: {{ .Values.backend.migrate.restartPolicy }} + volumes: + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + configMap: + name: "{{ include "drive.fullname" $ }}-files-{{ $index }}" + {{- end }} + {{- range $name, $volume := .Values.backend.persistence }} + - name: "{{ $name }}" + {{- if eq $volume.type "emptyDir" }} + emptyDir: {} + {{- else }} + persistentVolumeClaim: + claimName: "{{ $fullName }}-{{ $name }}" + {{- end }} + {{- end }} + {{- range .Values.backend.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 12 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 12 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 12 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 12 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} diff --git a/src/helm/drive/templates/backend_job_createsuperuser.yaml b/src/helm/drive/templates/backend_job_createsuperuser.yaml new file mode 100644 index 0000000..842565d --- /dev/null +++ b/src/helm/drive/templates/backend_job_createsuperuser.yaml @@ -0,0 +1,121 @@ +{{- $envVars := include "drive.common.env" (list . .Values.backend) -}} +{{- $fullName := include "drive.backend.fullname" . -}} +{{- $component := "backend" -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $fullName }}-createsuperuser + namespace: {{ .Release.Namespace | quote }} + {{- with .Values.backend.migrateJobAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "drive.common.labels" (list . $component) | nindent 4 }} +spec: + template: + metadata: + annotations: + {{- with .Values.backend.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "drive.common.selectorLabels" (list . $component) | nindent 8 }} + spec: + {{- if $.Values.image.credentials }} + imagePullSecrets: + - name: {{ include "drive.secret.dockerconfigjson.name" (dict "fullname" (include "drive.fullname" .) "imageCredentials" $.Values.image.credentials) }} + {{- end}} + shareProcessNamespace: {{ .Values.backend.shareProcessNamespace }} + containers: + {{- with .Values.backend.sidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} + - name: {{ .Chart.Name }} + image: "{{ (.Values.backend.image | default dict).repository | default .Values.image.repository }}:{{ (.Values.backend.image | default dict).tag | default .Values.image.tag }}" + imagePullPolicy: {{ (.Values.backend.image | default dict).pullPolicy | default .Values.image.pullPolicy }} + {{- with .Values.backend.createsuperuser.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.backend.args }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- if $envVars}} + {{- $envVars | indent 12 }} + {{- end }} + {{- with .Values.backend.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.backend.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + mountPath: {{ $value.path }} + subPath: content + {{- end }} + {{- range $name, $volume := .Values.backend.persistence }} + - name: "{{ $name }}" + mountPath: "{{ $volume.mountPath }}" + {{- end }} + {{- range .Values.backend.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- with .Values.backend.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.backend.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.backend.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + restartPolicy: {{ .Values.backend.createsuperuser.restartPolicy }} + volumes: + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + configMap: + name: "{{ include "drive.fullname" $ }}-files-{{ $index }}" + {{- end }} + {{- range $name, $volume := .Values.backend.persistence }} + - name: "{{ $name }}" + {{- if eq $volume.type "emptyDir" }} + emptyDir: {} + {{- else }} + persistentVolumeClaim: + claimName: "{{ $fullName }}-{{ $name }}" + {{- end }} + {{- end }} + {{- range .Values.backend.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 12 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 12 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 12 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 12 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} diff --git a/src/helm/drive/templates/backend_svc.yaml b/src/helm/drive/templates/backend_svc.yaml new file mode 100644 index 0000000..bcb0bec --- /dev/null +++ b/src/helm/drive/templates/backend_svc.yaml @@ -0,0 +1,21 @@ +{{- $envVars := include "drive.common.env" (list . .Values.backend) -}} +{{- $fullName := include "drive.backend.fullname" . -}} +{{- $component := "backend" -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "drive.common.labels" (list . $component) | nindent 4 }} + annotations: + {{- toYaml $.Values.backend.service.annotations | nindent 4 }} +spec: + type: {{ .Values.backend.service.type }} + ports: + - port: {{ .Values.backend.service.port }} + targetPort: {{ .Values.backend.service.targetPort }} + protocol: TCP + name: http + selector: + {{- include "drive.common.selectorLabels" (list . $component) | nindent 4 }} diff --git a/src/helm/drive/templates/frontend_deployment.yaml b/src/helm/drive/templates/frontend_deployment.yaml new file mode 100644 index 0000000..a2f93b2 --- /dev/null +++ b/src/helm/drive/templates/frontend_deployment.yaml @@ -0,0 +1,136 @@ +{{- $envVars := include "drive.common.env" (list . .Values.frontend) -}} +{{- $fullName := include "drive.frontend.fullname" . -}} +{{- $component := "frontend" -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "drive.common.labels" (list . $component) | nindent 4 }} +spec: + replicas: {{ .Values.frontend.replicas }} + selector: + matchLabels: + {{- include "drive.common.selectorLabels" (list . $component) | nindent 6 }} + template: + metadata: + annotations: + {{- with .Values.frontend.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "drive.common.selectorLabels" (list . $component) | nindent 8 }} + spec: + {{- if $.Values.image.credentials }} + imagePullSecrets: + - name: {{ include "drive.secret.dockerconfigjson.name" (dict "fullname" (include "drive.fullname" .) "imageCredentials" $.Values.image.credentials) }} + {{- end}} + shareProcessNamespace: {{ .Values.frontend.shareProcessNamespace }} + containers: + {{- with .Values.frontend.sidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} + - name: {{ .Chart.Name }} + image: "{{ (.Values.frontend.image | default dict).repository | default .Values.image.repository }}:{{ (.Values.frontend.image | default dict).tag | default .Values.image.tag }}" + imagePullPolicy: {{ (.Values.frontend.image | default dict).pullPolicy | default .Values.image.pullPolicy }} + {{- with .Values.frontend.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.frontend.args }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- if $envVars}} + {{- $envVars | indent 12 }} + {{- end }} + {{- with .Values.frontend.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.frontend.service.targetPort }} + protocol: TCP + {{- if .Values.frontend.probes.liveness }} + livenessProbe: + {{- include "drive.probes.abstract" (merge .Values.frontend.probes.liveness (dict "targetPort" .Values.frontend.service.targetPort )) | nindent 12 }} + {{- end }} + {{- if .Values.frontend.probes.readiness }} + readinessProbe: + {{- include "drive.probes.abstract" (merge .Values.frontend.probes.readiness (dict "targetPort" .Values.frontend.service.targetPort )) | nindent 12 }} + {{- end }} + {{- if .Values.frontend.probes.startup }} + startupProbe: + {{- include "drive.probes.abstract" (merge .Values.frontend.probes.startup (dict "targetPort" .Values.frontend.service.targetPort )) | nindent 12 }} + {{- end }} + {{- with .Values.frontend.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + mountPath: {{ $value.path }} + subPath: content + {{- end }} + {{- range $name, $volume := .Values.frontend.persistence }} + - name: "{{ $name }}" + mountPath: "{{ $volume.mountPath }}" + {{- end }} + {{- range .Values.frontend.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- with .Values.frontend.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.frontend.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.frontend.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + configMap: + name: "{{ include "drive.fullname" $ }}-files-{{ $index }}" + {{- end }} + {{- range $name, $volume := .Values.frontend.persistence }} + - name: "{{ $name }}" + {{- if eq $volume.type "emptyDir" }} + emptyDir: {} + {{- else }} + persistentVolumeClaim: + claimName: "{{ $fullName }}-{{ $name }}" + {{- end }} + {{- end }} + {{- range .Values.frontend.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 12 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 12 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 12 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 12 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} diff --git a/src/helm/drive/templates/frontend_svc.yaml b/src/helm/drive/templates/frontend_svc.yaml new file mode 100644 index 0000000..5c2f96a --- /dev/null +++ b/src/helm/drive/templates/frontend_svc.yaml @@ -0,0 +1,21 @@ +{{- $envVars := include "drive.common.env" (list . .Values.frontend) -}} +{{- $fullName := include "drive.frontend.fullname" . -}} +{{- $component := "frontend" -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "drive.common.labels" (list . $component) | nindent 4 }} + annotations: + {{- toYaml $.Values.frontend.service.annotations | nindent 4 }} +spec: + type: {{ .Values.frontend.service.type }} + ports: + - port: {{ .Values.frontend.service.port }} + targetPort: {{ .Values.frontend.service.targetPort }} + protocol: TCP + name: http + selector: + {{- include "drive.common.selectorLabels" (list . $component) | nindent 4 }} diff --git a/src/helm/drive/templates/ingress.yaml b/src/helm/drive/templates/ingress.yaml new file mode 100644 index 0000000..bd628ec --- /dev/null +++ b/src/helm/drive/templates/ingress.yaml @@ -0,0 +1,118 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "drive.fullname" . -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "drive.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls.enabled }} + tls: + {{- if .Values.ingress.host }} + - secretName: {{ $fullName }}-tls + hosts: + - {{ .Values.ingress.host | quote }} + {{- end }} + {{- range .Values.ingress.tls.additional }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.ingress.host }} + - host: {{ .Values.ingress.host | quote }} + http: + paths: + - path: {{ .Values.ingress.path | quote }} + {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: Prefix + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "drive.frontend.fullname" . }} + port: + number: {{ .Values.frontend.service.port }} + {{- else }} + serviceName: {{ include "drive.frontend.fullname" . }} + servicePort: {{ .Values.frontend.service.port }} + {{- end }} + - path: /api + {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: Prefix + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "drive.backend.fullname" . }} + port: + number: {{ .Values.backend.service.port }} + {{- else }} + serviceName: {{ include "drive.backend.fullname" . }} + servicePort: {{ .Values.backend.service.port }} + {{- end }} + {{- with .Values.ingress.customBackends }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- range .Values.ingress.hosts }} + - host: {{ . | quote }} + http: + paths: + - path: {{ $.Values.ingress.path | quote }} + {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: Prefix + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "drive.frontend.fullname" $ }} + port: + number: {{ $.Values.frontend.service.port }} + {{- else }} + serviceName: {{ include "drive.frontend.fullname" $ }} + servicePort: {{ $.Values.frontend.service.port }} + {{- end }} + - path: /api + {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: Prefix + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "drive.backend.fullname" $ }} + port: + number: {{ $.Values.backend.service.port }} + {{- else }} + serviceName: {{ include "drive.backend.fullname" $ }} + servicePort: {{ $.Values.backend.service.port }} + {{- end }} + {{- with $.Values.ingress.customBackends }} + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} +{{- end }} + diff --git a/src/helm/drive/templates/ingress_admin.yaml b/src/helm/drive/templates/ingress_admin.yaml new file mode 100644 index 0000000..855a6e4 --- /dev/null +++ b/src/helm/drive/templates/ingress_admin.yaml @@ -0,0 +1,98 @@ +{{- if .Values.ingressAdmin.enabled -}} +{{- $fullName := include "drive.fullname" . -}} +{{- if and .Values.ingressAdmin.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingressAdmin.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingressAdmin.annotations "kubernetes.io/ingress.class" .Values.ingressAdmin.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }}-admin + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "drive.labels" . | nindent 4 }} + {{- with .Values.ingressAdmin.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingressAdmin.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingressAdmin.className }} + {{- end }} + {{- if .Values.ingressAdmin.tls.enabled }} + tls: + {{- if .Values.ingressAdmin.host }} + - secretName: {{ $fullName }}-tls + hosts: + - {{ .Values.ingressAdmin.host | quote }} + {{- end }} + {{- range .Values.ingressAdmin.tls.additional }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.ingressAdmin.host }} + - host: {{ .Values.ingressAdmin.host | quote }} + http: + paths: + - path: {{ .Values.ingressAdmin.path | quote }} + {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: Prefix + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "drive.backend.fullname" . }} + port: + number: {{ .Values.backend.service.port }} + {{- else }} + serviceName: {{ include "drive.backend.fullname" . }} + servicePort: {{ .Values.backend.service.port }} + {{- end }} + - path: /static + {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: Prefix + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "drive.backend.fullname" . }} + port: + number: {{ .Values.backend.service.port }} + {{- else }} + serviceName: {{ include "drive.backend.fullname" . }} + servicePort: {{ .Values.backend.service.port }} + {{- end }} + {{- end }} + {{- range .Values.ingressAdmin.hosts }} + - host: {{ . | quote }} + http: + paths: + - path: {{ $.Values.ingressAdmin.path | quote }} + {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: Prefix + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "drive.backend.fullname" $ }} + port: + number: {{ $.Values.backend.service.port }} + {{- else }} + serviceName: {{ include "drive.backend.fullname" $ }} + servicePort: {{ $.Values.backend.service.port }} + {{- end }} + {{- end }} +{{- end }} + diff --git a/src/helm/drive/templates/ingress_media.yaml b/src/helm/drive/templates/ingress_media.yaml new file mode 100644 index 0000000..8aa40e9 --- /dev/null +++ b/src/helm/drive/templates/ingress_media.yaml @@ -0,0 +1,84 @@ +{{- if .Values.ingressMedia.enabled -}} +{{- $fullName := include "drive.fullname" . -}} +{{- if and .Values.ingressMedia.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingressMedia.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingressMedia.annotations "kubernetes.io/ingress.class" .Values.ingressMedia.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }}-media + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "drive.labels" . | nindent 4 }} + {{- with .Values.ingressMedia.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingressMedia.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingressMedia.className }} + {{- end }} + {{- if .Values.ingressMedia.tls.enabled }} + tls: + {{- if .Values.ingressMedia.host }} + - secretName: {{ $fullName }}-tls + hosts: + - {{ .Values.ingressMedia.host | quote }} + {{- end }} + {{- range .Values.ingressMedia.tls.additional }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.ingressMedia.host }} + - host: {{ .Values.ingressMedia.host | quote }} + http: + paths: + - path: {{ .Values.ingressMedia.path | quote }} + {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: Prefix + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "drive.backend.fullname" . }} + port: + number: {{ .Values.backend.service.port }} + {{- else }} + serviceName: {{ include "drive.backend.fullname" . }} + servicePort: {{ .Values.backend.service.port }} + {{- end }} + {{- end }} + {{- range .Values.ingressMedia.hosts }} + - host: {{ . | quote }} + http: + paths: + - path: {{ $.Values.ingressMedia.path | quote }} + {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: Prefix + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "drive.backend.fullname" $ }} + port: + number: {{ $.Values.backend.service.port }} + {{- else }} + serviceName: {{ include "drive.backend.fullname" $ }} + servicePort: {{ $.Values.backend.service.port }} + {{- end }} + {{- end }} +{{- end }} + diff --git a/src/helm/drive/templates/secret.yaml b/src/helm/drive/templates/secret.yaml new file mode 100644 index 0000000..1a06a2d --- /dev/null +++ b/src/helm/drive/templates/secret.yaml @@ -0,0 +1,9 @@ +{{ if .Values.htaccess }} +apiVersion: v1 +kind: Secret +metadata: + name: htaccess + namespace: {{ .Release.Namespace | quote }} +stringData: + auth: {{ .Values.htaccess }} +{{ end }} diff --git a/src/helm/drive/values.yaml b/src/helm/drive/values.yaml new file mode 100644 index 0000000..32ceae6 --- /dev/null +++ b/src/helm/drive/values.yaml @@ -0,0 +1,265 @@ +# Default values for drive. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +## @section General configuration + +## @param image.repository Repository to use to pull drive's container image +## @param image.tag drive's container tag +## @param image.pullPolicy Container image pull policy +## @extra image.credentials.username Username for container registry authentication +## @extra image.credentials.password Password for container registry authentication +## @extra image.credentials.registry Registry url for which the credentials are specified +## @extra image.credentials.name Name of the generated secret for imagePullSecrets +image: + repository: lasuite/drive-backend + pullPolicy: IfNotPresent + tag: "latest" + +## @param nameOverride Override the chart name +## @param fullnameOverride Override the full application name +nameOverride: "" +fullnameOverride: "" + +## @skip commonEnvVars +commonEnvVars: &commonEnvVars + <<: [] + +## @param ingress.enabled whether to enable the Ingress or not +## @param ingress.className IngressClass to use for the Ingress +## @param ingress.host Host for the Ingress +## @param ingress.path Path to use for the Ingress +ingress: + enabled: false + className: null + host: drive.example.com + path: / + ## @param ingress.hosts Additional host to configure for the Ingress + hosts: [] + # - chart-example.local + ## @param ingress.tls.enabled Weather to enable TLS for the Ingress + ## @skip ingress.tls.additional + ## @extra ingress.tls.additional[].secretName Secret name for additional TLS config + ## @extra ingress.tls.additional[].hosts[] Hosts for additional TLS config + tls: + enabled: true + additional: [] + + ## @param ingress.customBackends Add custom backends to ingress + customBackends: [] + + +## @param ingressAdmin.enabled whether to enable the Ingress or not +## @param ingressAdmin.className IngressClass to use for the Ingress +## @param ingressAdmin.host Host for the Ingress +## @param ingressAdmin.path Path to use for the Ingress +ingressAdmin: + enabled: false + className: null + host: drive.example.com + path: /admin + ## @param ingressAdmin.hosts Additional host to configure for the Ingress + hosts: [ ] + # - chart-example.local + ## @param ingressAdmin.tls.enabled Weather to enable TLS for the Ingress + ## @skip ingressAdmin.tls.additional + ## @extra ingressAdmin.tls.additional[].secretName Secret name for additional TLS config + ## @extra ingressAdmin.tls.additional[].hosts[] Hosts for additional TLS config + tls: + enabled: true + additional: [] + + +## @section backend + +backend: + + ## @param backend.command Override the backend container command + command: [] + + ## @param backend.args Override the backend container args + args: [] + + ## @param backend.replicas Amount of backend replicas + replicas: 3 + + ## @param backend.shareProcessNamespace Enable share process namespace between containers + shareProcessNamespace: false + + ## @param backend.sidecars Add sidecars containers to backend deployment + sidecars: [] + + ## @param backend.migrateJobAnnotations Annotations for the migrate job + migrateJobAnnotations: {} + + ## @param backend.securityContext Configure backend Pod security context + securityContext: null + + ## @param backend.envVars Configure backend container environment variables + ## @extra backend.envVars.BY_VALUE Example environment variable by setting value directly + ## @extra backend.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap + ## @extra backend.envVars.FROM_CONFIGMAP.configMapKeyRef.key Key within a ConfigMap when configuring env vars from a ConfigMap + ## @extra backend.envVars.FROM_SECRET.secretKeyRef.name Name of a Secret when configuring env vars from a Secret + ## @extra backend.envVars.FROM_SECRET.secretKeyRef.key Key within a Secret when configuring env vars from a Secret + ## @skip backend.envVars + envVars: + <<: *commonEnvVars + + ## @param backend.podAnnotations Annotations to add to the backend Pod + podAnnotations: {} + + ## @param backend.service.type backend Service type + ## @param backend.service.port backend Service listening port + ## @param backend.service.targetPort backend container listening port + ## @param backend.service.annotations Annotations to add to the backend Service + service: + type: ClusterIP + port: 80 + targetPort: 8000 + annotations: {} + + ## @param backend.migrate.command backend migrate command + ## @param backend.migrate.restartPolicy backend migrate job restart policy + migrate: + command: + - "python" + - "manage.py" + - "migrate" + - "--no-input" + restartPolicy: Never + + ## @param backend.probes.liveness.path [nullable] Configure path for backend HTTP liveness probe + ## @param backend.probes.liveness.targetPort [nullable] Configure port for backend HTTP liveness probe + ## @param backend.probes.liveness.initialDelaySeconds [nullable] Configure initial delay for backend liveness probe + ## @param backend.probes.liveness.initialDelaySeconds [nullable] Configure timeout for backend liveness probe + ## @param backend.probes.startup.path [nullable] Configure path for backend HTTP startup probe + ## @param backend.probes.startup.targetPort [nullable] Configure port for backend HTTP startup probe + ## @param backend.probes.startup.initialDelaySeconds [nullable] Configure initial delay for backend startup probe + ## @param backend.probes.startup.initialDelaySeconds [nullable] Configure timeout for backend startup probe + ## @param backend.probes.readiness.path [nullable] Configure path for backend HTTP readiness probe + ## @param backend.probes.readiness.targetPort [nullable] Configure port for backend HTTP readiness probe + ## @param backend.probes.readiness.initialDelaySeconds [nullable] Configure initial delay for backend readiness probe + ## @param backend.probes.readiness.initialDelaySeconds [nullable] Configure timeout for backend readiness probe + probes: + liveness: + path: /__heartbeat__ + initialDelaySeconds: 10 + readiness: + path: /__lbheartbeat__ + initialDelaySeconds: 10 + + ## @param backend.resources Resource requirements for the backend container + resources: {} + + ## @param backend.nodeSelector Node selector for the backend Pod + nodeSelector: {} + + ## @param backend.tolerations Tolerations for the backend Pod + tolerations: [] + + ## @param backend.affinity Affinity for the backend Pod + affinity: {} + + ## @param backend.persistence Additional volumes to create and mount on the backend. Used for debugging purposes + ## @extra backend.persistence.volume-name.size Size of the additional volume + ## @extra backend.persistence.volume-name.type Type of the additional volume, persistentVolumeClaim or emptyDir + ## @extra backend.persistence.volume-name.mountPath Path where the volume should be mounted to + persistence: {} + + ## @param backend.extraVolumeMounts Additional volumes to mount on the backend. + extraVolumeMounts: [] + + ## @param backend.extraVolumes Additional volumes to mount on the backend. + extraVolumes: [] + + +## @section frontend + +frontend: + ## @param frontend.image.repository Repository to use to pull drive's frontend container image + ## @param frontend.image.tag drive's frontend container tag + ## @param frontend.image.pullPolicy frontend container image pull policy + image: + repository: lasuite/drive-frontend + pullPolicy: IfNotPresent + tag: "latest" + + ## @param frontend.command Override the frontend container command + command: [] + + ## @param frontend.args Override the frontend container args + args: [] + + ## @param frontend.replicas Amount of frontend replicas + replicas: 3 + + ## @param frontend.shareProcessNamespace Enable share process namefrontend between containers + shareProcessNamespace: false + + ## @param frontend.sidecars Add sidecars containers to frontend deployment + sidecars: [] + + ## @param frontend.securityContext Configure frontend Pod security context + securityContext: null + + ## @param frontend.envVars Configure frontend container environment variables + ## @extra frontend.envVars.BY_VALUE Example environment variable by setting value directly + ## @extra frontend.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap + ## @extra frontend.envVars.FROM_CONFIGMAP.configMapKeyRef.key Key within a ConfigMap when configuring env vars from a ConfigMap + ## @extra frontend.envVars.FROM_SECRET.secretKeyRef.name Name of a Secret when configuring env vars from a Secret + ## @extra frontend.envVars.FROM_SECRET.secretKeyRef.key Key within a Secret when configuring env vars from a Secret + ## @skip frontend.envVars + envVars: + <<: *commonEnvVars + + ## @param frontend.podAnnotations Annotations to add to the frontend Pod + podAnnotations: {} + + ## @param frontend.service.type frontend Service type + ## @param frontend.service.port frontend Service listening port + ## @param frontend.service.targetPort frontend container listening port + ## @param frontend.service.annotations Annotations to add to the frontend Service + service: + type: ClusterIP + port: 80 + targetPort: 8080 + annotations: {} + + ## @param frontend.probes Configure probe for frontend + ## @extra frontend.probes.liveness.path Configure path for frontend HTTP liveness probe + ## @extra frontend.probes.liveness.targetPort Configure port for frontend HTTP liveness probe + ## @extra frontend.probes.liveness.initialDelaySeconds Configure initial delay for frontend liveness probe + ## @extra frontend.probes.liveness.initialDelaySeconds Configure timeout for frontend liveness probe + ## @extra frontend.probes.startup.path Configure path for frontend HTTP startup probe + ## @extra frontend.probes.startup.targetPort Configure port for frontend HTTP startup probe + ## @extra frontend.probes.startup.initialDelaySeconds Configure initial delay for frontend startup probe + ## @extra frontend.probes.startup.initialDelaySeconds Configure timeout for frontend startup probe + ## @extra frontend.probes.readiness.path Configure path for frontend HTTP readiness probe + ## @extra frontend.probes.readiness.targetPort Configure port for frontend HTTP readiness probe + ## @extra frontend.probes.readiness.initialDelaySeconds Configure initial delay for frontend readiness probe + ## @extra frontend.probes.readiness.initialDelaySeconds Configure timeout for frontend readiness probe + probes: {} + + ## @param frontend.resources Resource requirements for the frontend container + resources: {} + + ## @param frontend.nodeSelector Node selector for the frontend Pod + nodeSelector: {} + + ## @param frontend.tolerations Tolerations for the frontend Pod + tolerations: [] + + ## @param frontend.affinity Affinity for the frontend Pod + affinity: {} + + ## @param frontend.persistence Additional volumes to create and mount on the frontend. Used for debugging purposes + ## @extra frontend.persistence.volume-name.size Size of the additional volume + ## @extra frontend.persistence.volume-name.type Type of the additional volume, persistentVolumeClaim or emptyDir + ## @extra frontend.persistence.volume-name.mountPath Path where the volume should be mounted to + persistence: {} + + ## @param frontend.extraVolumeMounts Additional volumes to mount on the frontend. + extraVolumeMounts: [] + + ## @param frontend.extraVolumes Additional volumes to mount on the frontend. + extraVolumes: [] diff --git a/src/helm/env.d/dev/secrets.enc.yaml b/src/helm/env.d/dev/secrets.enc.yaml new file mode 100644 index 0000000..83ad7de --- /dev/null +++ b/src/helm/env.d/dev/secrets.enc.yaml @@ -0,0 +1,62 @@ +djangoSuperUserEmail: ENC[AES256_GCM,data:7b1xfYmr1g0RlBmsHBRA39ZPV/6+1DrtHQ==,iv:/GW7oLxPTZYmRWVPvyAQMoZl1owHM4Fo0XAOtyEh2rA=,tag:DaqoW+dglyAOXMm5+mrDfA==,type:str] +djangoSuperUserPass: ENC[AES256_GCM,data:RQgX,iv:q3CdfmwGfHSTjLXTimDk/1MyoFLviRuwmZa2E7GUzhY=,tag:HCtdtqgSxdJIHFhI8xpegQ==,type:str] +djangoSecretKey: ENC[AES256_GCM,data:mtJCf6mKfj/fJkg4wmfIvvU1vkUEF77BI8TUFikp/M3nPveDXhKmy3Cw3cXFpOYiFZ0=,iv:qwPRKsPS1Jhylj5asbmknXm1xOX3nfp9iccuorUrcj0=,tag:ENVfAt4i3PttoqD8+Kc4wQ==,type:str] +oidc: + clientId: ENC[AES256_GCM,data:wndPCbysbWDybdHglcG+wkMWk1rrD40hKqFxct9T3TLEGOk/,iv:RH1OdBX1GYIT90sSq0AGz49fFi6dL0m49Pegs6Ko9tQ=,tag:/tKytQwoZkBX1Tf96gAjIA==,type:str] + clientSecret: ENC[AES256_GCM,data:MUJ0wsg+LC2QZ1jZ0Twd3FS3dQevmJq9/97qVI3ARHuJIVlQz0Qah4vE7/iR+sn7ME2o1s1AzV4c1Yx/F3nHBg==,iv:LvinICSzF/8EvrHZD4Jp6lt7g3yxSOEgVHPrc3SShjo=,tag:yvkyyBXmhEkmGL7jZevUCA==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age15fyxdwmg5mvldtqqus87xspuws2u0cpvwheehrtvkexj4tnsqqysw6re2x + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMMjFCeWhkUmRWTnlIM1JM + dVFock1DWmtXQnpQZWZMWW1YdndhSS93MlVFCmxKVDUwOUt0NjJIZiswSm5aRi9U + VEllelBZVmFKdVFzcVJPUm50VHo5RTgKLS0tIDlkU3htTEdSREFOSUxlTGVtUm1n + RzJZbzhFcDNZKzdxMWFHTWx6Uy9GVFkKTw8LbhzAACp0NUHDfNcXpZyr2pJyNxxw + C7j/UB0cAejlSJHaUUiZ6TEcslXRpqnNagwUw4z/uzo7m4temay22A== + -----END AGE ENCRYPTED FILE----- + - recipient: age16hnlml8yv4ynwy0seer57g8qww075crd0g7nsundz3pj4wk7m3vqftszg7 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQQjBNMnVlNURQVWdjSyty + RGozcmN5eTUwRHJIWnhhc1E3U1NXQ3AwTWxBCnFjbmJNZnFiRVJ6VHhmQmt1Vk5n + OTVXWVh3RzhoMWNrbUl6OHphTjFLQVUKLS0tIGJjUlNhK0dHQ2R3SCtrbTRnaFJT + Q1pyRXhSVm8xQWk2NG1MK0srVU1pL2sKkoxGCM00UM2leTNCn5H8499uwJw1NIXs + PoRNgplehrHFptrAwGEpSYMXbxu88N7EWa/rtOp+sHWK5zpxscMkjA== + -----END AGE ENCRYPTED FILE----- + - recipient: age1plkp8td6zzfcavjusmsfrlk54t9vn8jjxm8zaz7cmnr7kzl2nfnsd54hwg + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzYnpkYnJnYnJjVFRHRzRa + N09JOXVnQkVrcVcwdk9kR1k1azNib2lkMVZFCmhvOHlpVnJ0RlRpYWZ1TkVoaklV + NmNzY3BEeWN1MUtKWmZFT2RaMUxBRW8KLS0tIG92ZmhsZ29LSkRSREhiaG9kWXhH + akREb0ttYVpNWTJHb1pjaWRFbWpxUjgKgZp3cN2rZw4ktbpb5cUnDEtsT/KWszGi + pmpJHgsMADigyUc+Pjw+1pwpn0FtXVEXGedbf8bBuJavvbS2PuJBsg== + -----END AGE ENCRYPTED FILE----- + - recipient: age12g6f5fse25tgrwweleh4jls3qs52hey2edh759smulwmk5lnzadslu2cp3 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxaHZJeStiVnBzTGNTNzdo + UDFVTU51ZWp0WWorUnBlSzVBSU9IU2JnbUNNCkpMZGdNV3FUYkZOcWNLK0JWci81 + WGNwYi9Jb0QrV0lkUzNJWTcrUjIzUmMKLS0tIHlTKzNsVzNsSGFuYjJ0RFp0Y1Nr + a1VOcDBPTTYvNjkxN092N1UrYk1CM2cKNifC3ZLOrFTFKA9iKg8nPpZb+3DxnTwq + grsrxQa40b/Vv/aPoiPBMeSENDcH48X/EhMFNKX7dvl+7HEaY+QPlA== + -----END AGE ENCRYPTED FILE----- + - recipient: age1hnhuzj96ktkhpyygvmz0x9h8mfvssz7ss6emmukags644mdhf4msajk93r + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoZ2ZlcllJeGlKUDNxUk1w + ekZ3TSttaXREV1FBRWwzNW54cjlYbHpLdWpRCnhSL2hEVVBEWEJKQWF0YTk1YzhJ + RTBGN25sT0hBM3V4QndiTVkveDBwQ2cKLS0tIEdoZGRLRXdCME1wcUJHQXhtSHBQ + UVEyNUVIanF6Z3ZSUjU1aTk0NFRBR0EKGuH5vzOV9lP/qRew0maECapKtLILaf/4 + XoSgPnjh8pIbJG7i9VKnFORlzkNJ6OPhZlX3ax15hd1qQv0PSCMBDA== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-07-02T10:03:17Z" + mac: ENC[AES256_GCM,data:qx236E1cFtBmbYyUf6B95/Fwu2hoi9ZAhUcYiY/tsG9h1+kwXntfkvbH3ekyI7A5ZrpJXMeQZ7gLc+ohci4m5Ju+/G39MjMt+ww0Y6gBMqe59YlHfeFD2mYsnn9j1pqtbrIJ6+8fLDmhaXtGtXP3qRmFTc9LwL6Rm+5gn8cjcnA=,iv:TC7zBnQ0hRz0JSytrYVnyJiI1eMWRTBqctLajZYUhvU=,tag:wCBeo2xD5UpdRqGjkZxbXA==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.8.1 diff --git a/src/helm/env.d/dev/values.drive.yaml.gotmpl b/src/helm/env.d/dev/values.drive.yaml.gotmpl new file mode 100644 index 0000000..74b6901 --- /dev/null +++ b/src/helm/env.d/dev/values.drive.yaml.gotmpl @@ -0,0 +1,89 @@ +image: + repository: localhost:5001/drive-backend + pullPolicy: Always + tag: "latest" + +backend: + replicas: 1 + envVars: + DJANGO_CSRF_TRUSTED_ORIGINS: https://drive.127.0.0.1.nip.io,http://drive.127.0.0.1.nip.io + DJANGO_CONFIGURATION: Production + DJANGO_ALLOWED_HOSTS: "*" + DJANGO_SECRET_KEY: {{ .Values.djangoSecretKey }} + DJANGO_SETTINGS_MODULE: drive.settings + DJANGO_SUPERUSER_PASSWORD: admin + DJANGO_EMAIL_HOST: "mailcatcher" + DJANGO_EMAIL_PORT: 1025 + DJANGO_EMAIL_USE_SSL: False + OIDC_OP_JWKS_ENDPOINT: https://fca.integ01.dev-agentconnect.fr/api/v2/jwks + OIDC_OP_AUTHORIZATION_ENDPOINT: https://fca.integ01.dev-agentconnect.fr/api/v2/authorize + OIDC_OP_TOKEN_ENDPOINT: https://fca.integ01.dev-agentconnect.fr/api/v2/token + OIDC_OP_USER_ENDPOINT: https://fca.integ01.dev-agentconnect.fr/api/v2/userinfo + OIDC_OP_LOGOUT_ENDPOINT: https://fca.integ01.dev-agentconnect.fr/api/v2/session/end + OIDC_RP_CLIENT_ID: {{ .Values.oidc.clientId }} + OIDC_RP_CLIENT_SECRET: {{ .Values.oidc.clientSecret }} + OIDC_RP_SIGN_ALGO: RS256 + OIDC_RP_SCOPES: "openid email" + OIDC_REDIRECT_ALLOWED_HOSTS: https://drive.127.0.0.1.nip.io + OIDC_AUTH_REQUEST_EXTRA_PARAMS: "{'acr_values': 'eidas1'}" + LOGIN_REDIRECT_URL: https://drive.127.0.0.1.nip.io + LOGIN_REDIRECT_URL_FAILURE: https://drive.127.0.0.1.nip.io + LOGOUT_REDIRECT_URL: https://drive.127.0.0.1.nip.io + DB_HOST: postgres-postgresql + DB_NAME: drive + DB_USER: dinum + DB_PASSWORD: pass + DB_PORT: 5432 + POSTGRES_DB: drive + POSTGRES_USER: dinum + POSTGRES_PASSWORD: pass + REDIS_URL: redis://default:pass@redis-master:6379/1 + STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage + + migrate: + command: + - "/bin/sh" + - "-c" + - | + python manage.py migrate --no-input && + python manage.py create_demo --force + restartPolicy: Never + + command: + - "gunicorn" + - "-c" + - "/usr/local/etc/gunicorn/drive.py" + - "drive.wsgi:application" + - "--reload" + + createsuperuser: + command: + - "/bin/sh" + - "-c" + - | + shell -c "from core.models import User; not User.objects.filter(username='admin').exists() and User.objects.create_superuser('admin', 'admin@example.com', 'admin')" + restartPolicy: Never + +frontend: + envVars: + PORT: 8080 + NEXT_PUBLIC_API_ORIGIN: https://drive.127.0.0.1.nip.io + NEXT_PUBLIC_SIGNALING_URL: wss://drive.127.0.0.1.nip.io/ws + + replicas: 1 + command: + - yarn + - dev + + image: + repository: localhost:5001/drive-frontend + pullPolicy: Always + tag: "latest" + +ingress: + enabled: true + host: drive.127.0.0.1.nip.io + +ingressAdmin: + enabled: true + host: drive.127.0.0.1.nip.io diff --git a/src/helm/env.d/preprod/secrets.enc.yaml b/src/helm/env.d/preprod/secrets.enc.yaml new file mode 120000 index 0000000..25e608e --- /dev/null +++ b/src/helm/env.d/preprod/secrets.enc.yaml @@ -0,0 +1 @@ +../../../../secrets/numerique-gouv/impress/env/preprod/secrets.enc.yaml \ No newline at end of file diff --git a/src/helm/env.d/preprod/values.drive.yaml.gotmpl b/src/helm/env.d/preprod/values.drive.yaml.gotmpl new file mode 100644 index 0000000..f469a46 --- /dev/null +++ b/src/helm/env.d/preprod/values.drive.yaml.gotmpl @@ -0,0 +1,137 @@ +image: + repository: lasuite/drive-backend + pullPolicy: Always + tag: "v1.1.0" + +backend: + migrateJobAnnotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/hook-delete-policy: HookSucceeded + envVars: + DJANGO_CSRF_TRUSTED_ORIGINS: http://drive-preprod.beta.numerique.gouv.fr,https://drive-preprod.beta.numerique.gouv.fr + DJANGO_CONFIGURATION: Production + DJANGO_ALLOWED_HOSTS: "*" + DJANGO_SUPERUSER_EMAIL: + secretKeyRef: + name: backend + key: DJANGO_SUPERUSER_EMAIL + DJANGO_SECRET_KEY: + secretKeyRef: + name: backend + key: DJANGO_SECRET_KEY + DJANGO_SETTINGS_MODULE: drive.settings + DJANGO_SUPERUSER_PASSWORD: + secretKeyRef: + name: backend + key: DJANGO_SUPERUSER_PASSWORD + DJANGO_EMAIL_HOST: "snap-mail.numerique.gouv.fr" + DJANGO_EMAIL_PORT: 465 + DJANGO_EMAIL_USE_SSL: True + DJANGO_SILENCED_SYSTEM_CHECKS: security.W008,security.W004 + OIDC_OP_JWKS_ENDPOINT: https://fca.integ01.dev-agentconnect.fr/api/v2/jwks + OIDC_OP_AUTHORIZATION_ENDPOINT: https://fca.integ01.dev-agentconnect.fr/api/v2/authorize + OIDC_OP_TOKEN_ENDPOINT: https://fca.integ01.dev-agentconnect.fr/api/v2/token + OIDC_OP_USER_ENDPOINT: https://fca.integ01.dev-agentconnect.fr/api/v2/userinfo + OIDC_OP_LOGOUT_ENDPOINT: https://fca.integ01.dev-agentconnect.fr/api/v2/session/end + OIDC_RP_CLIENT_ID: + secretKeyRef: + name: backend + key: OIDC_RP_CLIENT_ID + OIDC_RP_CLIENT_SECRET: + secretKeyRef: + name: backend + key: OIDC_RP_CLIENT_SECRET + OIDC_RP_SIGN_ALGO: RS256 + OIDC_RP_SCOPES: "openid email" + OIDC_REDIRECT_ALLOWED_HOSTS: https://drive-preprod.beta.numerique.gouv.fr + OIDC_AUTH_REQUEST_EXTRA_PARAMS: "{'acr_values': 'eidas1'}" + LOGIN_REDIRECT_URL: https://drive-preprod.beta.numerique.gouv.fr + LOGIN_REDIRECT_URL_FAILURE: https://drive-preprod.beta.numerique.gouv.fr + LOGOUT_REDIRECT_URL: https://drive-preprod.beta.numerique.gouv.fr + DB_HOST: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: host + DB_NAME: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: database + DB_USER: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: username + DB_PASSWORD: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: password + DB_PORT: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: port + POSTGRES_USER: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: username + POSTGRES_DB: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: database + POSTGRES_PASSWORD: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: password + REDIS_URL: + secretKeyRef: + name: redis.redis.libre.sh + key: url + AWS_S3_ENDPOINT_URL: + secretKeyRef: + name: drive-media-storage.bucket.libre.sh + key: url + AWS_S3_ACCESS_KEY_ID: + secretKeyRef: + name: drive-media-storage.bucket.libre.sh + key: accessKey + AWS_S3_SECRET_ACCESS_KEY: + secretKeyRef: + name: drive-media-storage.bucket.libre.sh + key: secretKey + AWS_STORAGE_BUCKET_NAME: + secretKeyRef: + name: drive-media-storage.bucket.libre.sh + key: bucket + AWS_S3_REGION_NAME: local + STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage + + createsuperuser: + command: + - "/bin/sh" + - "-c" + - | + python manage.py createsuperuser --email $DJANGO_SUPERUSER_EMAIL --password $DJANGO_SUPERUSER_PASSWORD + restartPolicy: Never + +frontend: + image: + repository: lasuite/drive-frontend + pullPolicy: Always + tag: "v1.1.0" + +ingress: + enabled: true + host: drive-preprod.beta.numerique.gouv.fr + className: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/auth-type: basic + nginx.ingress.kubernetes.io/auth-secret: htaccess + nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required' + +ingressAdmin: + enabled: true + host: drive-preprod.beta.numerique.gouv.fr + className: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/auth-signin: https://oauth2-proxy-preprod.beta.numerique.gouv.fr/oauth2/start + nginx.ingress.kubernetes.io/auth-url: https://oauth2-proxy-preprod.beta.numerique.gouv.fr/oauth2/auth diff --git a/src/helm/env.d/production/secrets.enc.yaml b/src/helm/env.d/production/secrets.enc.yaml new file mode 120000 index 0000000..1efef50 --- /dev/null +++ b/src/helm/env.d/production/secrets.enc.yaml @@ -0,0 +1 @@ +../../../../secrets/numerique-gouv/impress/env/production/secrets.enc.yaml \ No newline at end of file diff --git a/src/helm/env.d/production/values.drive.yaml.gotmpl b/src/helm/env.d/production/values.drive.yaml.gotmpl new file mode 100644 index 0000000..781a899 --- /dev/null +++ b/src/helm/env.d/production/values.drive.yaml.gotmpl @@ -0,0 +1,137 @@ +image: + repository: lasuite/drive-backend + pullPolicy: Always + tag: "v1.1.0" + +backend: + migrateJobAnnotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: HookSucceeded + envVars: + DJANGO_CSRF_TRUSTED_ORIGINS: https://docs.numerique.gouv.fr + DJANGO_CONFIGURATION: Production + DJANGO_ALLOWED_HOSTS: "*" + DJANGO_SECRET_KEY: + secretKeyRef: + name: backend + key: DJANGO_SECRET_KEY + DJANGO_SETTINGS_MODULE: drive.settings + DJANGO_SUPERUSER_EMAIL: + secretKeyRef: + name: backend + key: DJANGO_SUPERUSER_EMAIL + DJANGO_SUPERUSER_PASSWORD: + secretKeyRef: + name: backend + key: DJANGO_SUPERUSER_PASSWORD + DJANGO_EMAIL_HOST: "snap-mail.numerique.gouv.fr" + DJANGO_EMAIL_PORT: 465 + DJANGO_EMAIL_USE_SSL: True + DJANGO_SILENCED_SYSTEM_CHECKS: security.W008,security.W004 + OIDC_OP_JWKS_ENDPOINT: https://auth.agentconnect.gouv.fr/api/v2/jwks + OIDC_OP_AUTHORIZATION_ENDPOINT: https://auth.agentconnect.gouv.fr/api/v2/authorize + OIDC_OP_TOKEN_ENDPOINT: https://auth.agentconnect.gouv.fr/api/v2/token + OIDC_OP_USER_ENDPOINT: https://auth.agentconnect.gouv.fr/api/v2/userinfo + OIDC_OP_LOGOUT_ENDPOINT: https://auth.agentconnect.gouv.fr/api/v2/session/end + OIDC_RP_CLIENT_ID: + secretKeyRef: + name: backend + key: OIDC_RP_CLIENT_ID + OIDC_RP_CLIENT_SECRET: + secretKeyRef: + name: backend + key: OIDC_RP_CLIENT_SECRET + OIDC_RP_SIGN_ALGO: RS256 + OIDC_RP_SCOPES: "openid email" + OIDC_REDIRECT_ALLOWED_HOSTS: https://docs.numerique.gouv.fr + OIDC_AUTH_REQUEST_EXTRA_PARAMS: "{'acr_values': 'eidas1'}" + LOGIN_REDIRECT_URL: https://docs.numerique.gouv.fr + LOGIN_REDIRECT_URL_FAILURE: https://docs.numerique.gouv.fr + LOGOUT_REDIRECT_URL: https://docs.numerique.gouv.fr + DB_HOST: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: host + DB_NAME: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: database + DB_USER: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: username + DB_PASSWORD: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: password + DB_PORT: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: port + POSTGRES_USER: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: username + POSTGRES_DB: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: database + POSTGRES_PASSWORD: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: password + REDIS_URL: + secretKeyRef: + name: redis.redis.libre.sh + key: url + AWS_S3_ENDPOINT_URL: + secretKeyRef: + name: drive-media-storage.bucket.libre.sh + key: url + AWS_S3_ACCESS_KEY_ID: + secretKeyRef: + name: drive-media-storage.bucket.libre.sh + key: accessKey + AWS_S3_SECRET_ACCESS_KEY: + secretKeyRef: + name: drive-media-storage.bucket.libre.sh + key: secretKey + AWS_STORAGE_BUCKET_NAME: + secretKeyRef: + name: drive-media-storage.bucket.libre.sh + key: bucket + AWS_S3_REGION_NAME: local + STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage + + createsuperuser: + command: + - "/bin/sh" + - "-c" + - | + python manage.py createsuperuser --email $DJANGO_SUPERUSER_EMAIL --password $DJANGO_SUPERUSER_PASSWORD + restartPolicy: Never + +frontend: + image: + repository: lasuite/drive-frontend + pullPolicy: Always + tag: "v1.1.0" + +ingress: + enabled: true + host: docs.numerique.gouv.fr + className: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt + nginx.ingress.kubernetes.io/auth-type: basic + nginx.ingress.kubernetes.io/auth-secret: htaccess + nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required' + +ingressAdmin: + enabled: true + host: docs.numerique.gouv.fr + className: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt + nginx.ingress.kubernetes.io/auth-signin: https://oauth2-proxy.beta.numerique.gouv.fr/oauth2/start + nginx.ingress.kubernetes.io/auth-url: https://oauth2-proxy.beta.numerique.gouv.fr/oauth2/auth diff --git a/src/helm/env.d/staging/secrets.enc.yaml b/src/helm/env.d/staging/secrets.enc.yaml new file mode 120000 index 0000000..ca63795 --- /dev/null +++ b/src/helm/env.d/staging/secrets.enc.yaml @@ -0,0 +1 @@ +../../../../secrets/numerique-gouv/impress/env/staging/secrets.enc.yaml \ No newline at end of file diff --git a/src/helm/env.d/staging/values.drive.yaml.gotmpl b/src/helm/env.d/staging/values.drive.yaml.gotmpl new file mode 100644 index 0000000..4af5632 --- /dev/null +++ b/src/helm/env.d/staging/values.drive.yaml.gotmpl @@ -0,0 +1,134 @@ +image: + repository: lasuite/drive-backend + pullPolicy: Always + tag: "main" + +backend: + migrateJobAnnotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/hook-delete-policy: HookSucceeded + envVars: + DJANGO_CSRF_TRUSTED_ORIGINS: http://drive-staging.beta.numerique.gouv.fr,https://drive-staging.beta.numerique.gouv.fr + DJANGO_CONFIGURATION: Production + DJANGO_ALLOWED_HOSTS: "*" + DJANGO_SECRET_KEY: + secretKeyRef: + name: backend + key: DJANGO_SECRET_KEY + DJANGO_SETTINGS_MODULE: drive.settings + DJANGO_SUPERUSER_EMAIL: + secretKeyRef: + name: backend + key: DJANGO_SUPERUSER_EMAIL + DJANGO_SUPERUSER_PASSWORD: + secretKeyRef: + name: backend + key: DJANGO_SUPERUSER_PASSWORD + DJANGO_EMAIL_HOST: "snap-mail.numerique.gouv.fr" + DJANGO_EMAIL_PORT: 465 + DJANGO_EMAIL_USE_SSL: True + DJANGO_SILENCED_SYSTEM_CHECKS: security.W008,security.W004 + OIDC_OP_JWKS_ENDPOINT: https://fca.integ01.dev-agentconnect.fr/api/v2/jwks + OIDC_OP_AUTHORIZATION_ENDPOINT: https://fca.integ01.dev-agentconnect.fr/api/v2/authorize + OIDC_OP_TOKEN_ENDPOINT: https://fca.integ01.dev-agentconnect.fr/api/v2/token + OIDC_OP_USER_ENDPOINT: https://fca.integ01.dev-agentconnect.fr/api/v2/userinfo + OIDC_OP_LOGOUT_ENDPOINT: https://fca.integ01.dev-agentconnect.fr/api/v2/session/end + OIDC_RP_CLIENT_ID: + secretKeyRef: + name: backend + key: OIDC_RP_CLIENT_ID + OIDC_RP_CLIENT_SECRET: + secretKeyRef: + name: backend + key: OIDC_RP_CLIENT_SECRET + OIDC_RP_SIGN_ALGO: RS256 + OIDC_RP_SCOPES: "openid email" + OIDC_REDIRECT_ALLOWED_HOSTS: https://drive-staging.beta.numerique.gouv.fr + OIDC_AUTH_REQUEST_EXTRA_PARAMS: "{'acr_values': 'eidas1'}" + LOGIN_REDIRECT_URL: https://drive-staging.beta.numerique.gouv.fr + LOGIN_REDIRECT_URL_FAILURE: https://drive-staging.beta.numerique.gouv.fr + LOGOUT_REDIRECT_URL: https://drive-staging.beta.numerique.gouv.fr + DB_HOST: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: host + DB_NAME: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: database + DB_USER: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: username + DB_PASSWORD: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: password + DB_PORT: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: port + POSTGRES_USER: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: username + POSTGRES_DB: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: database + POSTGRES_PASSWORD: + secretKeyRef: + name: postgresql.postgres.libre.sh + key: password + REDIS_URL: + secretKeyRef: + name: redis.redis.libre.sh + key: url + AWS_S3_ENDPOINT_URL: + secretKeyRef: + name: drive-media-storage.bucket.libre.sh + key: url + AWS_S3_ACCESS_KEY_ID: + secretKeyRef: + name: drive-media-storage.bucket.libre.sh + key: accessKey + AWS_S3_SECRET_ACCESS_KEY: + secretKeyRef: + name: drive-media-storage.bucket.libre.sh + key: secretKey + AWS_STORAGE_BUCKET_NAME: + secretKeyRef: + name: drive-media-storage.bucket.libre.sh + key: bucket + AWS_S3_REGION_NAME: local + STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage + + createsuperuser: + command: + - "/bin/sh" + - "-c" + - | + python manage.py createsuperuser --email $DJANGO_SUPERUSER_EMAIL --password $DJANGO_SUPERUSER_PASSWORD + restartPolicy: Never + +frontend: + image: + repository: lasuite/drive-frontend + pullPolicy: Always + tag: "main" + +ingress: + enabled: true + host: drive-staging.beta.numerique.gouv.fr + className: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + +ingressAdmin: + enabled: true + host: drive-staging.beta.numerique.gouv.fr + className: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/auth-signin: https://oauth2-proxy-preprod.beta.numerique.gouv.fr/oauth2/start + nginx.ingress.kubernetes.io/auth-url: https://oauth2-proxy-preprod.beta.numerique.gouv.fr/oauth2/auth diff --git a/src/helm/extra/Chart.yaml b/src/helm/extra/Chart.yaml new file mode 100644 index 0000000..4dea1bb --- /dev/null +++ b/src/helm/extra/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: extra +description: A Helm chart to add some manifests to drive +type: application +version: 0.1.0 diff --git a/src/helm/extra/templates/keydb.yaml b/src/helm/extra/templates/keydb.yaml new file mode 100644 index 0000000..00898ae --- /dev/null +++ b/src/helm/extra/templates/keydb.yaml @@ -0,0 +1,7 @@ +apiVersion: core.libre.sh/v1alpha1 +kind: Redis +metadata: + name: redis + namespace: {{ .Release.Namespace | quote }} +spec: + disableAuth: false diff --git a/src/helm/extra/templates/postgresql.yaml b/src/helm/extra/templates/postgresql.yaml new file mode 100644 index 0000000..4812f49 --- /dev/null +++ b/src/helm/extra/templates/postgresql.yaml @@ -0,0 +1,7 @@ +apiVersion: core.libre.sh/v1alpha1 +kind: Postgres +metadata: + name: postgresql + namespace: {{ .Release.Namespace | quote }} +spec: + database: drive diff --git a/src/helm/extra/templates/s3.yaml b/src/helm/extra/templates/s3.yaml new file mode 100644 index 0000000..10b108c --- /dev/null +++ b/src/helm/extra/templates/s3.yaml @@ -0,0 +1,8 @@ +apiVersion: core.libre.sh/v1alpha1 +kind: Bucket +metadata: + name: drive-media-storage + namespace: {{ .Release.Namespace | quote }} +spec: + provider: data + versioned: true diff --git a/src/helm/extra/templates/secrets.yaml b/src/helm/extra/templates/secrets.yaml new file mode 100644 index 0000000..e0e7849 --- /dev/null +++ b/src/helm/extra/templates/secrets.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: backend +stringData: + DJANGO_SUPERUSER_EMAIL: {{ .Values.djangoSuperUserEmail }} + DJANGO_SUPERUSER_PASSWORD: {{ .Values.djangoSuperUserPass }} + DJANGO_SECRET_KEY: {{ .Values.djangoSecretKey }} + OIDC_RP_CLIENT_ID: {{ .Values.oidc.clientId }} + OIDC_RP_CLIENT_SECRET: {{ .Values.oidc.clientSecret }} diff --git a/src/helm/helmfile.yaml b/src/helm/helmfile.yaml new file mode 100644 index 0000000..178ab23 --- /dev/null +++ b/src/helm/helmfile.yaml @@ -0,0 +1,67 @@ +repositories: +- name: bitnami + url: registry-1.docker.io/bitnamicharts + oci: true + +releases: + - name: postgres + installed: {{ eq .Environment.Name "dev" | toYaml }} + namespace: {{ .Namespace }} + chart: bitnami/postgresql + version: 13.1.5 + values: + - auth: + username: dinum + password: pass + database: drive + - tls: + enabled: true + autoGenerated: true + + - name: redis + installed: {{ eq .Environment.Name "dev" | toYaml }} + namespace: {{ .Namespace }} + chart: bitnami/redis + version: 18.19.2 + values: + - auth: + password: pass + architecture: standalone + + - name: extra + installed: {{ ne .Environment.Name "dev" | toYaml }} + namespace: {{ .Namespace }} + chart: ./extra + secrets: + - env.d/{{ .Environment.Name }}/secrets.enc.yaml + + - name: drive + version: {{ .Values.version }} + namespace: {{ .Namespace }} + chart: ./drive + values: + - env.d/{{ .Environment.Name }}/values.drive.yaml.gotmpl + secrets: + - env.d/{{ .Environment.Name }}/secrets.enc.yaml + +environments: + dev: + values: + - version: 0.0.1 + secrets: + - env.d/{{ .Environment.Name }}/secrets.enc.yaml + staging: + values: + - version: 0.0.1 + secrets: + - env.d/{{ .Environment.Name }}/secrets.enc.yaml + preprod: + values: + - version: 0.0.1 + secrets: + - env.d/{{ .Environment.Name }}/secrets.enc.yaml + production: + values: + - version: 0.0.1 + secrets: + - env.d/{{ .Environment.Name }}/secrets.enc.yaml