diff --git a/.env b/.env index 3394e74..6c2e673 100644 --- a/.env +++ b/.env @@ -9,3 +9,6 @@ DB_PASSWORD=pass DB_PORT=15432 STORAGES_STATICFILES_BACKEND=django.contrib.staticfiles.storage.StaticFilesStorage +AWS_S3_ENDPOINT_URL=http://localhost:9000 +AWS_S3_ACCESS_KEY_ID=conversations +AWS_S3_SECRET_ACCESS_KEY=password diff --git a/.github/workflows/conversations.yml b/.github/workflows/conversations.yml index 631ee98..bdd3cc9 100644 --- a/.github/workflows/conversations.yml +++ b/.github/workflows/conversations.yml @@ -131,6 +131,9 @@ jobs: DB_PASSWORD: pass DB_PORT: 5432 STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage + AWS_S3_ENDPOINT_URL: http://localhost:9000 + AWS_S3_ACCESS_KEY_ID: conversations + AWS_S3_SECRET_ACCESS_KEY: password steps: - name: Checkout repository @@ -149,11 +152,34 @@ jobs: key: mail-templates-${{ hashFiles('src/mail/mjml') }} fail-on-cache-miss: true + - name: Start MinIO + run: | + docker pull minio/minio + docker run -d --name minio \ + -p 9000:9000 \ + -e "MINIO_ACCESS_KEY=conversations" \ + -e "MINIO_SECRET_KEY=password" \ + -v /data/media:/data \ + minio/minio server --console-address :9001 /data + # Tool to wait for a service to be ready - name: Install Dockerize run: | curl -sSL https://github.com/jwilder/dockerize/releases/download/v0.8.0/dockerize-linux-amd64-v0.8.0.tar.gz | sudo tar -C /usr/local/bin -xzv + - name: Wait for MinIO to be ready + run: | + dockerize -wait tcp://localhost:9000 -timeout 10s + + - name: Configure MinIO + run: | + MINIO=$(docker ps | grep minio/minio | sed -E 's/.*\s+([a-zA-Z0-9_-]+)$/\1/') + docker exec ${MINIO} sh -c \ + "mc alias set conversations http://localhost:9000 conversations password && \ + mc alias ls && \ + mc mb conversations/conversations-media-storage && \ + mc version enable conversations/conversations-media-storage" + - name: Install Python uses: actions/setup-python@v5 with: diff --git a/compose.yml b/compose.yml index 60c6d6e..c72dca0 100644 --- a/compose.yml +++ b/compose.yml @@ -21,6 +21,38 @@ services: ports: - "1081:1080" + minio: + user: ${DOCKER_USER:-1000} + image: minio/minio + environment: + - MINIO_ROOT_USER=conversations + - MINIO_ROOT_PASSWORD=password + ports: + - '9000:9000' + - '9001:9001' + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 1s + timeout: 20s + retries: 300 + entrypoint: "" + command: minio server --console-address :9001 /data + volumes: + - ./data/media:/data + + createbuckets: + image: minio/mc + depends_on: + minio: + condition: service_healthy + restart: true + entrypoint: > + sh -c " + /usr/bin/mc alias set conversations http://minio:9000 conversations password && \ + /usr/bin/mc mb conversations/conversations-media-storage && \ + /usr/bin/mc version enable conversations/conversations-media-storage && \ + exit 0;" + app-dev: build: context: . @@ -50,6 +82,8 @@ services: condition: service_started redis: condition: service_started + createbuckets: + condition: service_started nginx: image: nginx:1.25 diff --git a/docker/files/etc/nginx/conf.d/default.conf b/docker/files/etc/nginx/conf.d/default.conf index 875fcaf..0b508e6 100644 --- a/docker/files/etc/nginx/conf.d/default.conf +++ b/docker/files/etc/nginx/conf.d/default.conf @@ -4,6 +4,39 @@ server { server_name localhost; charset utf-8; + # Proxy auth for media + location /media/ { + # Auth request configuration + auth_request /media-auth; + auth_request_set $authHeader $upstream_http_authorization; + auth_request_set $authDate $upstream_http_x_amz_date; + auth_request_set $authContentSha256 $upstream_http_x_amz_content_sha256; + + # Pass specific headers from the auth response + proxy_set_header Authorization $authHeader; + proxy_set_header X-Amz-Date $authDate; + proxy_set_header X-Amz-Content-SHA256 $authContentSha256; + + # Get resource from Minio + proxy_pass http://minio:9000/conversations-media-storage/; + proxy_set_header Host minio:9000; + + add_header Content-Security-Policy "default-src 'none'" always; + } + + location /media-auth { + proxy_pass http://app-dev:8000/api/v1.0/chats/media-auth/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Original-URL $request_uri; + + # Prevent the body from being passed + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-Method $request_method; + } + location / { proxy_pass http://keycloak:8080; proxy_set_header Host $host; diff --git a/docs/architecture.md b/docs/architecture.md index 48cc8bb..1170715 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,4 +10,5 @@ flowchart TD Back -- REST API --> Yserver Back --> DB("Database (PostgreSQL)") Back <--> Celery --> DB + Back ----> S3("Minio (S3)") ``` diff --git a/docs/env.md b/docs/env.md index a5cf881..8f04917 100644 --- a/docs/env.md +++ b/docs/env.md @@ -19,6 +19,11 @@ These are the environment variables you can set for the `conversations-backend` | DB_PORT | port of the database | 5432 | | MEDIA_BASE_URL | | | | STORAGES_STATICFILES_BACKEND | | whitenoise.storage.CompressedManifestStaticFilesStorage | +| AWS_S3_ENDPOINT_URL | S3 endpoint | | +| AWS_S3_ACCESS_KEY_ID | access id for s3 endpoint | | +| AWS_S3_SECRET_ACCESS_KEY | access key for s3 endpoint | | +| AWS_S3_REGION_NAME | region name for s3 endpoint | | +| AWS_STORAGE_BUCKET_NAME | bucket name for s3 endpoint | conversations-media-storage | | DOCUMENT_IMAGE_MAX_SIZE | maximum size of document in bytes | 10485760 | | LANGUAGE_CODE | default language | en-us | | API_USERS_LIST_THROTTLE_RATE_SUSTAINED | throttle rate for api | 180/hour | diff --git a/docs/examples/conversations.values.yaml b/docs/examples/conversations.values.yaml index 9fe564d..c9b0fd1 100644 --- a/docs/examples/conversations.values.yaml +++ b/docs/examples/conversations.values.yaml @@ -48,6 +48,10 @@ backend: POSTGRES_USER: dinum POSTGRES_PASSWORD: pass REDIS_URL: redis://default:pass@redis-master:6379/1 + AWS_S3_ENDPOINT_URL: http://minio.conversations.svc.cluster.local:9000 + AWS_S3_ACCESS_KEY_ID: root + AWS_S3_SECRET_ACCESS_KEY: password + AWS_STORAGE_BUCKET_NAME: conversations-media-storage STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage migrate: @@ -113,3 +117,18 @@ ingress: ingressAdmin: enabled: true host: conversations.127.0.0.1.nip.io + +ingressMedia: + enabled: true + host: conversations.127.0.0.1.nip.io + + annotations: + nginx.ingress.kubernetes.io/auth-url: https://conversations.127.0.0.1.nip.io/api/v1.0/chats/media-auth/ + nginx.ingress.kubernetes.io/auth-response-headers: "Authorization, X-Amz-Date, X-Amz-Content-SHA256" + nginx.ingress.kubernetes.io/upstream-vhost: minio.conversations.svc.cluster.local:9000 + nginx.ingress.kubernetes.io/rewrite-target: /conversations-media-storage/$1 + +serviceMedia: + host: minio.conversations.svc.cluster.local + port: 9000 + diff --git a/docs/installation.md b/docs/installation.md index 5b0d961..679df16 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -8,6 +8,7 @@ This document is a step-by-step guide that describes how to install Conversation - an OIDC provider (if you don't have one, we provide an example) - a PostgreSQL server (if you don't have one, we provide an example) - a Memcached server (if you don't have one, we provide an example) +- a S3 bucket (if you don't have one, we provide an example) ### Test cluster @@ -172,6 +173,22 @@ POSTGRES_USER: dinum POSTGRES_PASSWORD: pass ``` +### Find s3 bucket connection values + +Conversations uses an s3 bucket to store documents, so if you have a provider obtain the necessary information to use it. If you don't, you can install a local minio testing environment as follow: + +``` +$ helm install minio oci://registry-1.docker.io/bitnamicharts/minio -f examples/minio.values.yaml +$ kubectl get po +NAME READY STATUS RESTARTS AGE +keycloak-0 1/1 Running 0 38m +keycloak-postgresql-0 1/1 Running 0 38m +minio-84f5c66895-bbhsk 1/1 Running 0 42s +minio-provisioning-2b5sq 0/1 Completed 0 42s +postgresql-0 1/1 Running 0 24m +redis-master-0 1/1 Running 0 10m +``` + ## Deployment Now you are ready to deploy Conversations without AI. AI requires more dependencies (OpenAI API). To deploy Conversations you need to provide all previous information to the helm chart. @@ -188,6 +205,8 @@ conversations-conversations-backend-migrate-c949s 0/1 Completed conversations-conversations-frontend-6749f644f7-p5s42 1/1 Running 0 79s keycloak-0 1/1 Running 0 48m keycloak-postgresql-0 1/1 Running 0 48m +minio-84f5c66895-bbhsk 1/1 Running 0 10m +minio-provisioning-2b5sq 0/1 Completed 0 10m postgresql-0 1/1 Running 0 34m redis-master-0 1/1 Running 0 20m ``` @@ -201,6 +220,7 @@ $ kubectl get ingress NAME CLASS HOSTS ADDRESS PORTS AGE conversations-conversations conversations.127.0.0.1.nip.io localhost 80, 443 114s conversations-conversations-admin conversations.127.0.0.1.nip.io localhost 80, 443 114s +conversations-conversations-media conversations.127.0.0.1.nip.io localhost 80, 443 114s conversations-conversations-ws conversations.127.0.0.1.nip.io localhost 80, 443 114s keycloak keycloak.127.0.0.1.nip.io localhost 80 49m ``` diff --git a/docs/system-requirements.md b/docs/system-requirements.md index 2daee4e..2c0c7f2 100644 --- a/docs/system-requirements.md +++ b/docs/system-requirements.md @@ -19,6 +19,7 @@ Memory is the first bottleneck; CPU matters only when Celery or the Next.js buil | PostgreSQL | **1 – 2 GB** | `shared_buffers` starting point ≈ 25% RAM ([postgresql.org][1]) | | Keycloak | **≈ 1.3 GB** | 70% of limit for heap + ~300 MB non-heap ([keycloak.org][2]) | | Redis | **≤ 256 MB** | Empty instance ≈ 3 MB; budget 256 MB to allow small datasets ([stackoverflow.com][3]) | +| MinIO | **2 GB (dev) / 32 GB (prod)** | Pre-allocates 1–2 GiB; docs recommend 32 GB per host for ≤ 100 Ti storage ([min.io][4]) | | Django API (+ Celery) | **0.8 – 1.5 GB** | Empirical in-house metrics | | Next.js frontend | **0.5 – 1 GB** | Dev build chain | | Nginx | **< 100 MB** | Static reverse-proxy footprint | @@ -26,6 +27,7 @@ Memory is the first bottleneck; CPU matters only when Celery or the Next.js buil [1]: https://www.postgresql.org/docs/9.1/runtime-config-resource.html "PostgreSQL: Documentation: 9.1: Resource Consumption" [2]: https://www.keycloak.org/high-availability/concepts-memory-and-cpu-sizing "Concepts for sizing CPU and memory resources - Keycloak" [3]: https://stackoverflow.com/questions/45233052/memory-footprint-for-redis-empty-instance "Memory footprint for Redis empty instance - Stack Overflow" +[4]: https://min.io/docs/minio/kubernetes/upstream/operations/checklists/hardware.html "Hardware Checklist — MinIO Object Storage for Kubernetes" > **Rule of thumb:** add 2 GB for OS/overhead, then sum only the rows you actually run. @@ -38,6 +40,7 @@ Production deployments differ significantly from development environments. The t | PostgreSQL | **2 – 8 GB** | Higher `shared_buffers` and connection pooling for concurrent users | | OIDC Provider (optional) | **Variable** | Any OIDC-compatible provider (Keycloak, Auth0, Azure AD, etc.) - external or self-hosted | | Redis | **256 MB – 2 GB** | Session storage and caching; scales with active user sessions | +| Object Storage (optional) | **External or self-hosted** | Can use AWS S3, Azure Blob, Google Cloud Storage, or self-hosted MinIO | | Django API (+ Celery) | **1 – 3 GB** | Production workloads with background tasks and higher concurrency | | Static Files (Nginx) | **< 200 MB** | Serves Next.js build output and static assets; no development overhead | | Nginx (Load Balancer) | **< 200 MB** | Reverse proxy, SSL termination, static file serving | @@ -46,6 +49,7 @@ Production deployments differ significantly from development environments. The t - **Frontend**: Uses pre-built Next.js static assets served by Nginx (no Node.js runtime needed) - **Authentication**: Any OIDC-compatible provider can be used instead of self-hosted Keycloak +- **Object Storage**: External services (S3, Azure Blob) or self-hosted solutions (MinIO) are both viable - **Database**: Consider PostgreSQL clustering or managed database services for high availability - **Scaling**: Horizontal scaling is recommended for Django API service @@ -83,6 +87,7 @@ Production deployments differ significantly from development environments. The t | 8071 | Django | | 8080 | Keycloak | | 8083 | Nginx proxy | +| 9000/9001 | MinIO | | 15432 | PostgreSQL (main) | | 5433 | PostgreSQL (Keycloak) | | 1081 | Maildev | @@ -96,3 +101,5 @@ Production deployments differ significantly from development environments. The t **CPU** – budget one vCPU per busy container until Celery or Next.js builds saturate. **Disk** – SSD; add 10 GB extra for the Docker layer cache. + +**MinIO** – for demos, mount a local folder instead of running MinIO to save 2 GB+ of RAM. diff --git a/docs/troubleshoot.md b/docs/troubleshoot.md index dfc9484..0416306 100644 --- a/docs/troubleshoot.md +++ b/docs/troubleshoot.md @@ -83,6 +83,55 @@ If you already have CRLF line endings in your local repository, the **best appro git commit -m "✏️(project) Fix line endings to LF" ``` +## Minio Permission Issues on Windows + +### Problem Description + +On Windows, you may encounter permission-related errors when running Minio in development mode with Docker Compose. This typically happens because: + +- **Windows file permissions** don't map well to Unix-style user IDs used in Docker containers +- **Docker Desktop** may have issues with user mapping when using the `DOCKER_USER` environment variable +- **Minio container** fails to start or access volumes due to permission conflicts + +### Common Symptoms + +- Minio container fails to start with permission denied errors +- Error messages related to file system permissions in Minio logs +- Unable to create or access buckets in the development environment +- Docker Compose showing Minio service as unhealthy or exited + +### Solution for Windows Users + +If you encounter Minio permission issues on Windows, you can temporarily disable user mapping for the Minio service: + +1. **Open the `compose.yml` file** + +2. **Comment out the user directive** in the `minio` service section: + ```yaml + minio: + # user: ${DOCKER_USER:-1000} # Comment this line on Windows if permission issues occur + image: minio/minio + environment: + - MINIO_ROOT_USER=conversations + - MINIO_ROOT_PASSWORD=password + # ... rest of the configuration + ``` + +3. **Restart the services**: + ```bash + make run + ``` + +### Why This Works + +- Commenting out the `user` directive allows the Minio container to run with its default user +- This bypasses Windows-specific permission mapping issues +- The container will have the necessary permissions to access and manage the mounted volumes + +### Note + +This is a **development-only workaround**. In production environments, proper user mapping and security considerations should be maintained according to your deployment requirements. + ## Frontend File Watching Issues on Windows ### Problem Description @@ -140,4 +189,4 @@ Add the `WATCHPACK_POLLING=true` environment variable to the frontend-developmen ### Note -This setting is primarily needed for Windows users. Linux and macOS users typically don't need this setting as file watching works correctly by default on those platforms. +This setting is primarily needed for Windows users. Linux and macOS users typically don't need this setting as file watching works correctly by default on those platforms. \ No newline at end of file diff --git a/env.d/development/common.dist b/env.d/development/common.dist index cc6a320..dee107a 100644 --- a/env.d/development/common.dist +++ b/env.d/development/common.dist @@ -26,6 +26,9 @@ CONVERSATIONS_BASE_URL="http://localhost:8072" # Media STORAGES_STATICFILES_BACKEND=django.contrib.staticfiles.storage.StaticFilesStorage +AWS_S3_ENDPOINT_URL=http://minio:9000 +AWS_S3_ACCESS_KEY_ID=conversations +AWS_S3_SECRET_ACCESS_KEY=password MEDIA_BASE_URL=http://localhost:8083 # OIDC diff --git a/src/backend/conversations/settings.py b/src/backend/conversations/settings.py index 2e676cf..3cc4e26 100755 --- a/src/backend/conversations/settings.py +++ b/src/backend/conversations/settings.py @@ -109,7 +109,7 @@ class Base(BraveSettings, Configuration): STORAGES = { "default": { - "BACKEND": "django.core.files.storage.FileSystemStorage", + "BACKEND": "storages.backends.s3.S3Storage", }, "staticfiles": { "BACKEND": values.Value( @@ -119,6 +119,19 @@ class Base(BraveSettings, Configuration): }, } + # Media + AWS_S3_ENDPOINT_URL = values.Value(environ_name="AWS_S3_ENDPOINT_URL", environ_prefix=None) + AWS_S3_ACCESS_KEY_ID = values.Value(environ_name="AWS_S3_ACCESS_KEY_ID", environ_prefix=None) + AWS_S3_SECRET_ACCESS_KEY = values.Value( + environ_name="AWS_S3_SECRET_ACCESS_KEY", environ_prefix=None + ) + AWS_S3_REGION_NAME = values.Value(environ_name="AWS_S3_REGION_NAME", environ_prefix=None) + AWS_STORAGE_BUCKET_NAME = values.Value( + "conversations-media-storage", + environ_name="AWS_STORAGE_BUCKET_NAME", + environ_prefix=None, + ) + # Internationalization # https://docs.djangoproject.com/en/3.1/topics/i18n/ @@ -880,6 +893,17 @@ class Build(Base): """ SECRET_KEY = values.Value("DummyKey") + STORAGES = { + "default": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + }, + "staticfiles": { + "BACKEND": values.Value( + "whitenoise.storage.CompressedManifestStaticFilesStorage", + environ_name="STORAGES_STATICFILES_BACKEND", + ), + }, + } class Development(Base): diff --git a/src/helm/conversations/README.md b/src/helm/conversations/README.md index e508e52..7383186 100644 --- a/src/helm/conversations/README.md +++ b/src/helm/conversations/README.md @@ -4,36 +4,53 @@ ### General configuration -| Name | Description | Value | -| ------------------------------------------ | --------------------------------------------------------- | ------------------------------- | -| `image.repository` | Repository to use to pull conversations's container image | `lasuite/conversations-backend` | -| `image.tag` | conversations's container tag | `latest` | -| `image.pullPolicy` | Container image pull policy | `IfNotPresent` | -| `image.credentials.username` | Username for container registry authentication | | -| `image.credentials.password` | Password for container registry authentication | | -| `image.credentials.registry` | Registry url for which the credentials are specified | | -| `image.credentials.name` | Name of the generated secret for imagePullSecrets | | -| `nameOverride` | Override the chart name | `""` | -| `fullnameOverride` | Override the full application name | `""` | -| `ingress.enabled` | whether to enable the Ingress or not | `false` | -| `ingress.className` | IngressClass to use for the Ingress | `nil` | -| `ingress.host` | Host for the Ingress | `conversations.example.com` | -| `ingress.path` | Path to use for the Ingress | `/` | -| `ingress.hosts` | Additional host to configure for the Ingress | `[]` | -| `ingress.tls.enabled` | Weather to enable TLS for the Ingress | `true` | -| `ingress.tls.secretName` | Secret name for TLS config | `nil` | -| `ingress.tls.additional[].secretName` | Secret name for additional TLS config | | -| `ingress.tls.additional[].hosts[]` | Hosts for additional TLS config | | -| `ingress.customBackends` | Add custom backends to ingress | `[]` | -| `ingressAdmin.enabled` | whether to enable the Ingress or not | `false` | -| `ingressAdmin.className` | IngressClass to use for the Ingress | `nil` | -| `ingressAdmin.host` | Host for the Ingress | `conversations.example.com` | -| `ingressAdmin.path` | Path to use for the Ingress | `/admin` | -| `ingressAdmin.hosts` | Additional host to configure for the Ingress | `[]` | -| `ingressAdmin.tls.enabled` | Weather to enable TLS for the Ingress | `true` | -| `ingressAdmin.tls.secretName` | Secret name for TLS config | `nil` | -| `ingressAdmin.tls.additional[].secretName` | Secret name for additional TLS config | | -| `ingressAdmin.tls.additional[].hosts[]` | Hosts for additional TLS config | | +| Name | Description | Value | +|------------------------------------------------------------------------------|-----------------------------------------------------------|------------------------------------------------------------------| +| `image.repository` | Repository to use to pull conversations's container image | `lasuite/conversations-backend` | +| `image.tag` | conversations's container tag | `latest` | +| `image.pullPolicy` | Container image pull policy | `IfNotPresent` | +| `image.credentials.username` | Username for container registry authentication | | +| `image.credentials.password` | Password for container registry authentication | | +| `image.credentials.registry` | Registry url for which the credentials are specified | | +| `image.credentials.name` | Name of the generated secret for imagePullSecrets | | +| `nameOverride` | Override the chart name | `""` | +| `fullnameOverride` | Override the full application name | `""` | +| `ingress.enabled` | whether to enable the Ingress or not | `false` | +| `ingress.className` | IngressClass to use for the Ingress | `nil` | +| `ingress.host` | Host for the Ingress | `conversations.example.com` | +| `ingress.path` | Path to use for the Ingress | `/` | +| `ingress.hosts` | Additional host to configure for the Ingress | `[]` | +| `ingress.tls.enabled` | Weather to enable TLS for the Ingress | `true` | +| `ingress.tls.secretName` | Secret name for TLS config | `nil` | +| `ingress.tls.additional[].secretName` | Secret name for additional TLS config | | +| `ingress.tls.additional[].hosts[]` | Hosts for additional TLS config | | +| `ingress.customBackends` | Add custom backends to ingress | `[]` | +| `ingressAdmin.enabled` | whether to enable the Ingress or not | `false` | +| `ingressAdmin.className` | IngressClass to use for the Ingress | `nil` | +| `ingressAdmin.host` | Host for the Ingress | `conversations.example.com` | +| `ingressAdmin.path` | Path to use for the Ingress | `/admin` | +| `ingressAdmin.hosts` | Additional host to configure for the Ingress | `[]` | +| `ingressAdmin.tls.enabled` | Weather to enable TLS for the Ingress | `true` | +| `ingressAdmin.tls.secretName` | Secret name for TLS config | `nil` | +| `ingressAdmin.tls.additional[].secretName` | Secret name for additional TLS config | | +| `ingressAdmin.tls.additional[].hosts[]` | Hosts for additional TLS config | | +| `ingressMedia.enabled` | whether to enable the Ingress or not | `false` | +| `ingressMedia.className` | IngressClass to use for the Ingress | `nil` | +| `ingressMedia.host` | Host for the Ingress | `conversations.example.com` | +| `ingressMedia.path` | Path to use for the Ingress | `/media/(.*)` | +| `ingressMedia.hosts` | Additional host to configure for the Ingress | `[]` | +| `ingressMedia.tls.enabled` | Weather to enable TLS for the Ingress | `true` | +| `ingressMedia.tls.secretName` | Secret name for TLS config | `nil` | +| `ingressMedia.tls.additional[].secretName` | Secret name for additional TLS config | | +| `ingressMedia.tls.additional[].hosts[]` | Hosts for additional TLS config | | +| `ingressMedia.annotations.nginx.ingress.kubernetes.io/auth-url` | | `https://conversations.example.com/api/v1.0/chats/media-auth/` | +| `ingressMedia.annotations.nginx.ingress.kubernetes.io/auth-response-headers` | | `Authorization, X-Amz-Date, X-Amz-Content-SHA256` | +| `ingressMedia.annotations.nginx.ingress.kubernetes.io/upstream-vhost` | | `minio.conversations.svc.cluster.local:9000` | +| `ingressMedia.annotations.nginx.ingress.kubernetes.io/configuration-snippet` | | `add_header Content-Security-Policy "default-src 'none'" always; | +| ` | | | +| `serviceMedia.host` | | `minio.conversations.svc.cluster.local` | +| `serviceMedia.port` | | `9000` | +| `serviceMedia.annotations` | | `{}` | ### backend @@ -102,7 +119,7 @@ ### frontend | Name | Description | Value | -| ------------------------------------------------------ | ----------------------------------------------------------------------------------- | -------------------------------- | +|--------------------------------------------------------|-------------------------------------------------------------------------------------|----------------------------------| | `frontend.image.repository` | Repository to use to pull conversations's frontend container image | `lasuite/conversations-frontend` | | `frontend.image.tag` | conversations's frontend container tag | `latest` | | `frontend.image.pullPolicy` | frontend container image pull policy | `IfNotPresent` | @@ -152,7 +169,7 @@ ### posthog | Name | Description | Value | -| -------------------------------------- | ----------------------------------------------------------- | --------------------------- | +|----------------------------------------|-------------------------------------------------------------|-----------------------------| | `posthog.ingress.enabled` | Enable or disable the ingress resource creation | `false` | | `posthog.ingress.className` | Kubernetes ingress class name to use (e.g., nginx, traefik) | `nil` | | `posthog.ingress.host` | Primary hostname for the ingress resource | `conversations.example.com` | diff --git a/src/helm/conversations/templates/ingress_media.yaml b/src/helm/conversations/templates/ingress_media.yaml new file mode 100644 index 0000000..938dd1a --- /dev/null +++ b/src/helm/conversations/templates/ingress_media.yaml @@ -0,0 +1,83 @@ +{{- if .Values.ingressMedia.enabled -}} +{{- $fullName := include "conversations.fullname" . -}} +{{- if and .Values.ingressMedia.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingressMedia.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingressMedia.annotations "kubernetes.io/ingress.class" .Values.ingressMedia.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }}-media + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "conversations.labels" . | nindent 4 }} + {{- with .Values.ingressMedia.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingressMedia.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingressMedia.className }} + {{- end }} + {{- if .Values.ingressMedia.tls.enabled }} + tls: + {{- if .Values.ingressMedia.host }} + - secretName: {{ .Values.ingressMedia.tls.secretName | default (printf "%s-tls" $fullName) | quote }} + hosts: + - {{ .Values.ingressMedia.host | quote }} + {{- end }} + {{- range .Values.ingressMedia.tls.additional }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- if .Values.ingressMedia.host }} + - host: {{ .Values.ingressMedia.host | quote }} + http: + paths: + - path: {{ .Values.ingressMedia.path | quote }} + {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: ImplementationSpecific + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }}-media + port: + number: {{ .Values.serviceMedia.port }} + {{- else }} + serviceName: {{ $fullName }}-media + servicePort: {{ .Values.serviceMedia.port }} + {{- end }} + {{- end }} + {{- range .Values.ingressMedia.hosts }} + - host: {{ . | quote }} + http: + paths: + - path: {{ $.Values.ingressMedia.path | quote }} + {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: ImplementationSpecific + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }}-media + port: + number: {{ .Values.serviceMedia.port }} + {{- else }} + serviceName: {{ $fullName }}-media + servicePort: {{ .Values.serviceMedia.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/src/helm/conversations/templates/media_svc.yaml b/src/helm/conversations/templates/media_svc.yaml new file mode 100644 index 0000000..66f3058 --- /dev/null +++ b/src/helm/conversations/templates/media_svc.yaml @@ -0,0 +1,14 @@ +{{- $fullName := include "conversations.fullname" . -}} +{{- $component := "media" -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ $fullName }}-media + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "conversations.common.labels" (list . $component) | nindent 4 }} + annotations: + {{- toYaml $.Values.serviceMedia.annotations | nindent 4 }} +spec: + type: ExternalName + externalName: {{ $.Values.serviceMedia.host }} diff --git a/src/helm/conversations/values.yaml b/src/helm/conversations/values.yaml index 3b183ac..884c53c 100644 --- a/src/helm/conversations/values.yaml +++ b/src/helm/conversations/values.yaml @@ -72,6 +72,47 @@ ingressAdmin: secretName: null additional: [] +## @param ingressMedia.enabled whether to enable the Ingress or not +## @param ingressMedia.className IngressClass to use for the Ingress +## @param ingressMedia.host Host for the Ingress +## @param ingressMedia.path Path to use for the Ingress +ingressMedia: + enabled: false + className: null + host: conversations.example.com + path: /media/(.*) + ## @param ingressMedia.hosts Additional host to configure for the Ingress + hosts: [ ] + # - chart-example.local + ## @param ingressMedia.tls.enabled Weather to enable TLS for the Ingress + ## @param ingressMedia.tls.secretName Secret name for TLS config + ## @skip ingressMedia.tls.additional + ## @extra ingressMedia.tls.additional[].secretName Secret name for additional TLS config + ## @extra ingressMedia.tls.additional[].hosts[] Hosts for additional TLS config + tls: + enabled: true + secretName: null + additional: [] + + ## @param ingressMedia.annotations.nginx.ingress.kubernetes.io/auth-url + ## @param ingressMedia.annotations.nginx.ingress.kubernetes.io/auth-response-headers + ## @param ingressMedia.annotations.nginx.ingress.kubernetes.io/upstream-vhost + ## @param ingressMedia.annotations.nginx.ingress.kubernetes.io/configuration-snippet + annotations: + nginx.ingress.kubernetes.io/auth-url: https://conversations.example.com/api/v1.0/chats/media-auth/ + nginx.ingress.kubernetes.io/auth-response-headers: "Authorization, X-Amz-Date, X-Amz-Content-SHA256" + nginx.ingress.kubernetes.io/upstream-vhost: minio.conversations.svc.cluster.local:9000 + nginx.ingress.kubernetes.io/configuration-snippet: | + add_header Content-Security-Policy "default-src 'none'" always; + +## @param serviceMedia.host +## @param serviceMedia.port +## @param serviceMedia.annotations +serviceMedia: + host: minio.conversations.svc.cluster.local + port: 9000 + annotations: {} + ## @section backend diff --git a/src/helm/env.d/dev-staging/values.conversations.yaml.gotmpl b/src/helm/env.d/dev-staging/values.conversations.yaml.gotmpl index c288862..b62f7fe 100644 --- a/src/helm/env.d/dev-staging/values.conversations.yaml.gotmpl +++ b/src/helm/env.d/dev-staging/values.conversations.yaml.gotmpl @@ -56,6 +56,10 @@ backend: POSTGRES_USER: dinum POSTGRES_PASSWORD: pass REDIS_URL: redis://default:pass@redis-master:6379/2 + AWS_S3_ENDPOINT_URL: http://minio.conversations.svc.cluster.local:9000 + AWS_S3_ACCESS_KEY_ID: root + AWS_S3_SECRET_ACCESS_KEY: password + AWS_STORAGE_BUCKET_NAME: conversations-media-storage STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage OIDC_STORE_ACCESS_TOKEN: True OIDC_STORE_REFRESH_TOKEN: True @@ -128,3 +132,17 @@ posthog: ingressAssets: enabled: false + +ingressMedia: + enabled: true + host: conversations.127.0.0.1.nip.io + + annotations: + nginx.ingress.kubernetes.io/auth-url: https://conversations.127.0.0.1.nip.io/api/v1.0/chats/media-auth/ + nginx.ingress.kubernetes.io/auth-response-headers: "Authorization, X-Amz-Date, X-Amz-Content-SHA256" + nginx.ingress.kubernetes.io/upstream-vhost: minio.conversations.svc.cluster.local:9000 + nginx.ingress.kubernetes.io/rewrite-target: /conversations-media-storage/$1 + +serviceMedia: + host: minio.conversations.svc.cluster.local + port: 9000 diff --git a/src/helm/env.d/dev/values.conversations.yaml.gotmpl b/src/helm/env.d/dev/values.conversations.yaml.gotmpl index b43507f..b6360fb 100644 --- a/src/helm/env.d/dev/values.conversations.yaml.gotmpl +++ b/src/helm/env.d/dev/values.conversations.yaml.gotmpl @@ -55,6 +55,10 @@ backend: POSTGRES_PASSWORD: pass REDIS_URL: redis://default:pass@redis-master:6379/1 DJANGO_CELERY_BROKER_URL: redis://default:pass@redis-master:6379/1 + AWS_S3_ENDPOINT_URL: http://minio.conversations.svc.cluster.local:9000 + AWS_S3_ACCESS_KEY_ID: root + AWS_S3_SECRET_ACCESS_KEY: password + AWS_STORAGE_BUCKET_NAME: conversations-media-storage STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage CACHES_KEY_PREFIX: "{{ now | unixEpoch }}" migrate: @@ -148,3 +152,17 @@ posthog: ingressAssets: enabled: false + +ingressMedia: + enabled: true + host: conversations.127.0.0.1.nip.io + + annotations: + nginx.ingress.kubernetes.io/auth-url: https://conversations.127.0.0.1.nip.io/api/v1.0/chats/media-auth/ + nginx.ingress.kubernetes.io/auth-response-headers: "Authorization, X-Amz-Date, X-Amz-Content-SHA256" + nginx.ingress.kubernetes.io/upstream-vhost: minio.conversations.svc.cluster.local:9000 + nginx.ingress.kubernetes.io/rewrite-target: /conversations-media-storage/$1 + +serviceMedia: + host: minio.conversations.svc.cluster.local + port: 9000 diff --git a/src/helm/helmfile.yaml.gotmpl b/src/helm/helmfile.yaml.gotmpl index 321254f..2aa933d 100644 --- a/src/helm/helmfile.yaml.gotmpl +++ b/src/helm/helmfile.yaml.gotmpl @@ -65,6 +65,21 @@ releases: enabled: true autoGenerated: true + - name: minio + installed: {{ eq .Environment.Name "dev" | toYaml }} + namespace: {{ .Namespace }} + chart: bitnami/minio + version: 12.10.10 + values: + - auth: + rootUser: root + rootPassword: password + - provisioning: + enabled: true + buckets: + - name: conversations-media-storage + versioning: true + - name: redis installed: {{ eq .Environment.Name "dev" | toYaml }} namespace: {{ .Namespace }} @@ -80,4 +95,4 @@ releases: namespace: {{ .Namespace }} chart: ./conversations values: - - env.d/{{ .Environment.Name }}/values.conversations.yaml.gotmpl + - env.d/{{ .Environment.Name }}/values.conversations.yaml.gotmpl \ No newline at end of file