electron-rare/conversations
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

⚰️(storage) remove S3 media storage

Browse Source 
Unused for now and not clear if we will need it later.
This commit is contained in:
Quentin BEY
2025-08-26 15:27:00 +02:00
parent 86bcebadf8
commit 697b040a40
17 changed files with 2 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
-3
View File
@@ -9,6 +9,3 @@ DB_PASSWORD=pass
DB_PORT=15432
STORAGES_STATICFILES_BACKEND=django.contrib.staticfiles.storage.StaticFilesStorage
AWS_S3_ENDPOINT_URL=http://localhost:9000
AWS_S3_ACCESS_KEY_ID=conversations
AWS_S3_SECRET_ACCESS_KEY=password
.github/workflows/conversations.yml
-3
View File
@@ -126,9 +126,6 @@ jobs:
DB_PASSWORD: pass
DB_PORT: 5432
STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
AWS_S3_ENDPOINT_URL: http://localhost:9000
AWS_S3_ACCESS_KEY_ID: conversations
AWS_S3_SECRET_ACCESS_KEY: password
steps:
- name: Checkout repository
docker/files/etc/nginx/conf.d/default.conf
-33
View File
@@ -4,39 +4,6 @@ server {
server_name localhost;
charset utf-8;
# Proxy auth for media
location /media/ {
# Auth request configuration
auth_request /media-auth;
auth_request_set $authHeader $upstream_http_authorization;
auth_request_set $authDate $upstream_http_x_amz_date;
auth_request_set $authContentSha256 $upstream_http_x_amz_content_sha256;
# Pass specific headers from the auth response
proxy_set_header Authorization $authHeader;
proxy_set_header X-Amz-Date $authDate;
proxy_set_header X-Amz-Content-SHA256 $authContentSha256;
# Get resource from Minio
proxy_pass http://minio:9000/conversations-media-storage/;
proxy_set_header Host minio:9000;
add_header Content-Security-Policy "default-src 'none'" always;
}
location /media-auth {
proxy_pass http://app-dev:8000/api/v1.0/documents/media-auth/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Original-URL $request_uri;
# Prevent the body from being passed
proxy_pass_request_body off;
proxy_set_header Content-Length "";
proxy_set_header X-Original-Method $request_method;
}
location / {
proxy_pass http://keycloak:8080;
proxy_set_header Host $host;
docs/architecture.md
-1
View File
@@ -10,5 +10,4 @@ flowchart TD
Back -- REST API --> Yserver
Back --> DB("Database (PostgreSQL)")
Back <--> Celery --> DB
Back ----> S3("Minio (S3)")
```
docs/env.md
-5
View File
@@ -19,11 +19,6 @@ These are the environment variables you can set for the `conversations-backend`
| DB_PORT | port of the database | 5432 |
| MEDIA_BASE_URL | | |
| STORAGES_STATICFILES_BACKEND | | whitenoise.storage.CompressedManifestStaticFilesStorage |
| AWS_S3_ENDPOINT_URL | S3 endpoint | |
| AWS_S3_ACCESS_KEY_ID | access id for s3 endpoint | |
| AWS_S3_SECRET_ACCESS_KEY | access key for s3 endpoint | |
| AWS_S3_REGION_NAME | region name for s3 endpoint | |
| AWS_STORAGE_BUCKET_NAME | bucket name for s3 endpoint | conversations-media-storage |
| DOCUMENT_IMAGE_MAX_SIZE | maximum size of document in bytes | 10485760 |
| LANGUAGE_CODE | default language | en-us |
| API_USERS_LIST_THROTTLE_RATE_SUSTAINED | throttle rate for api | 180/hour |
docs/examples/conversations.values.yaml
-19
View File
@@ -48,10 +48,6 @@ backend:
POSTGRES_USER: dinum
POSTGRES_PASSWORD: pass
REDIS_URL: redis://default:pass@redis-master:6379/1
AWS_S3_ENDPOINT_URL: http://minio.conversations.svc.cluster.local:9000
AWS_S3_ACCESS_KEY_ID: root
AWS_S3_SECRET_ACCESS_KEY: password
AWS_STORAGE_BUCKET_NAME: conversations-media-storage
STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
migrate:
@@ -117,18 +113,3 @@ ingress:
ingressAdmin:
enabled: true
host: conversations.127.0.0.1.nip.io
ingressMedia:
enabled: true
host: conversations.127.0.0.1.nip.io
annotations:
nginx.ingress.kubernetes.io/auth-url: https://conversations.127.0.0.1.nip.io/api/v1.0/documents/media-auth/
nginx.ingress.kubernetes.io/auth-response-headers: "Authorization, X-Amz-Date, X-Amz-Content-SHA256"
nginx.ingress.kubernetes.io/upstream-vhost: minio.conversations.svc.cluster.local:9000
nginx.ingress.kubernetes.io/rewrite-target: /conversations-media-storage/$1
serviceMedia:
host: minio.conversations.svc.cluster.local
port: 9000
docs/installation.md
-20
View File
@@ -8,7 +8,6 @@ This document is a step-by-step guide that describes how to install Conversation
- an OIDC provider (if you don't have one, we provide an example)
- a PostgreSQL server (if you don't have one, we provide an example)
- a Memcached server (if you don't have one, we provide an example)
- a S3 bucket (if you don't have one, we provide an example)
### Test cluster
@@ -173,22 +172,6 @@ POSTGRES_USER: dinum
POSTGRES_PASSWORD: pass
```
### Find s3 bucket connection values
Conversations uses an s3 bucket to store documents, so if you have a provider obtain the necessary information to use it. If you don't, you can install a local minio testing environment as follow:
```
$ helm install minio oci://registry-1.docker.io/bitnamicharts/minio -f examples/minio.values.yaml
$ kubectl get po
NAME READY STATUS RESTARTS AGE
keycloak-0 1/1 Running 0 38m
keycloak-postgresql-0 1/1 Running 0 38m
minio-84f5c66895-bbhsk 1/1 Running 0 42s
minio-provisioning-2b5sq 0/1 Completed 0 42s
postgresql-0 1/1 Running 0 24m
redis-master-0 1/1 Running 0 10m
```
## Deployment
Now you are ready to deploy Conversations without AI. AI requires more dependencies (OpenAI API). To deploy Conversations you need to provide all previous information to the helm chart.
@@ -205,8 +188,6 @@ conversations-conversations-backend-migrate-c949s 0/1 Completed
conversations-conversations-frontend-6749f644f7-p5s42 1/1 Running 0 79s
keycloak-0 1/1 Running 0 48m
keycloak-postgresql-0 1/1 Running 0 48m
minio-84f5c66895-bbhsk 1/1 Running 0 10m
minio-provisioning-2b5sq 0/1 Completed 0 10m
postgresql-0 1/1 Running 0 34m
redis-master-0 1/1 Running 0 20m
```
@@ -220,7 +201,6 @@ $ kubectl get ingress
NAME CLASS HOSTS ADDRESS PORTS AGE
conversations-conversations <none> conversations.127.0.0.1.nip.io localhost 80, 443 114s
conversations-conversations-admin <none> conversations.127.0.0.1.nip.io localhost 80, 443 114s
conversations-conversations-media <none> conversations.127.0.0.1.nip.io localhost 80, 443 114s
conversations-conversations-ws <none> conversations.127.0.0.1.nip.io localhost 80, 443 114s
keycloak <none> keycloak.127.0.0.1.nip.io localhost 80 49m
```
docs/system-requirements.md
-2
View File
@@ -41,7 +41,6 @@ Production deployments differ significantly from development environments. The t
| PostgreSQL | **2 – 8 GB** | Higher `shared_buffers` and connection pooling for concurrent users |
| OIDC Provider (optional) | **Variable** | Any OIDC-compatible provider (Keycloak, Auth0, Azure AD, etc.) - external or self-hosted |
| Redis | **256 MB – 2 GB** | Session storage and caching; scales with active user sessions |
| Object Storage (optional) | **External or self-hosted** | Can use AWS S3, Azure Blob, Google Cloud Storage, or self-hosted MinIO |
| Django API (+ Celery) | **1 – 3 GB** | Production workloads with background tasks and higher concurrency |
| Static Files (Nginx) | **< 200 MB** | Serves Next.js build output and static assets; no development overhead |
| Nginx (Load Balancer) | **< 200 MB** | Reverse proxy, SSL termination, static file serving |
@@ -50,7 +49,6 @@ Production deployments differ significantly from development environments. The t
- **Frontend**: Uses pre-built Next.js static assets served by Nginx (no Node.js runtime needed)
- **Authentication**: Any OIDC-compatible provider can be used instead of self-hosted Keycloak
- **Object Storage**: External services (S3, Azure Blob) or self-hosted solutions (MinIO) are both viable
- **Database**: Consider PostgreSQL clustering or managed database services for high availability
- **Scaling**: Horizontal scaling is recommended for Django API service
env.d/development/common.dist
-3
View File
@@ -26,9 +26,6 @@ CONVERSATIONS_BASE_URL="http://localhost:8072"
# Media
STORAGES_STATICFILES_BACKEND=django.contrib.staticfiles.storage.StaticFilesStorage
AWS_S3_ENDPOINT_URL=http://minio:9000
AWS_S3_ACCESS_KEY_ID=conversations
AWS_S3_SECRET_ACCESS_KEY=password
MEDIA_BASE_URL=http://localhost:8083
# OIDC
src/backend/conversations/settings.py
+1 -25
View File
@@ -98,7 +98,7 @@ class Base(Configuration):
STORAGES = {
"default": {
"BACKEND": "storages.backends.s3.S3Storage",
"BACKEND": "django.core.files.storage.FileSystemStorage",
},
"staticfiles": {
"BACKEND": values.Value(
@@ -108,19 +108,6 @@ class Base(Configuration):
},
}
# Media
AWS_S3_ENDPOINT_URL = values.Value(environ_name="AWS_S3_ENDPOINT_URL", environ_prefix=None)
AWS_S3_ACCESS_KEY_ID = values.Value(environ_name="AWS_S3_ACCESS_KEY_ID", environ_prefix=None)
AWS_S3_SECRET_ACCESS_KEY = values.Value(
environ_name="AWS_S3_SECRET_ACCESS_KEY", environ_prefix=None
)
AWS_S3_REGION_NAME = values.Value(environ_name="AWS_S3_REGION_NAME", environ_prefix=None)
AWS_STORAGE_BUCKET_NAME = values.Value(
"conversations-media-storage",
environ_name="AWS_STORAGE_BUCKET_NAME",
environ_prefix=None,
)
# Internationalization
# https://docs.djangoproject.com/en/3.1/topics/i18n/
@@ -778,17 +765,6 @@ class Build(Base):
"""
SECRET_KEY = values.Value("DummyKey")
STORAGES = {
"default": {
"BACKEND": "django.core.files.storage.FileSystemStorage",
},
"staticfiles": {
"BACKEND": values.Value(
"whitenoise.storage.CompressedManifestStaticFilesStorage",
environ_name="STORAGES_STATICFILES_BACKEND",
),
},
}
class Development(Base):
src/helm/conversations/README.md
-18
View File
@@ -34,24 +34,6 @@
| `ingressAdmin.tls.secretName` | Secret name for TLS config | `nil` |
| `ingressAdmin.tls.additional[].secretName` | Secret name for additional TLS config | |
| `ingressAdmin.tls.additional[].hosts[]` | Hosts for additional TLS config | |
| `ingressMedia.enabled` | whether to enable the Ingress or not | `false` |
| `ingressMedia.className` | IngressClass to use for the Ingress | `nil` |
| `ingressMedia.host` | Host for the Ingress | `conversations.example.com` |
| `ingressMedia.path` | Path to use for the Ingress | `/media/(.*)` |
| `ingressMedia.hosts` | Additional host to configure for the Ingress | `[]` |
| `ingressMedia.tls.enabled` | Weather to enable TLS for the Ingress | `true` |
| `ingressMedia.tls.secretName` | Secret name for TLS config | `nil` |
| `ingressMedia.tls.additional[].secretName` | Secret name for additional TLS config | |
| `ingressMedia.tls.additional[].hosts[]` | Hosts for additional TLS config | |
| `ingressMedia.annotations.nginx.ingress.kubernetes.io/auth-url` | | `https://conversations.example.com/api/v1.0/documents/media-auth/` |
| `ingressMedia.annotations.nginx.ingress.kubernetes.io/auth-response-headers` | | `Authorization, X-Amz-Date, X-Amz-Content-SHA256` |
| `ingressMedia.annotations.nginx.ingress.kubernetes.io/upstream-vhost` | | `minio.conversations.svc.cluster.local:9000` |
| `ingressMedia.annotations.nginx.ingress.kubernetes.io/configuration-snippet` | | `add_header Content-Security-Policy "default-src 'none'" always; |
| ` | | |
| `serviceMedia.host` | | `minio.conversations.svc.cluster.local` |
| `serviceMedia.port` | | `9000` |
| `serviceMedia.annotations` | | `{}` |
### backend
| Name | Description | Value |
src/helm/conversations/templates/ingress_media.yaml
-83
View File
@@ -1,83 +0,0 @@
{{- if .Values.ingressMedia.enabled -}}
{{- $fullName := include "conversations.fullname" . -}}
{{- if and .Values.ingressMedia.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
{{- if not (hasKey .Values.ingressMedia.annotations "kubernetes.io/ingress.class") }}
{{- $_ := set .Values.ingressMedia.annotations "kubernetes.io/ingress.class" .Values.ingressMedia.className}}
{{- end }}
{{- end }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ $fullName }}-media
namespace: {{ .Release.Namespace | quote }}
labels:
{{- include "conversations.labels" . | nindent 4 }}
{{- with .Values.ingressMedia.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingressMedia.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
ingressClassName: {{ .Values.ingressMedia.className }}
{{- end }}
{{- if .Values.ingressMedia.tls.enabled }}
tls:
{{- if .Values.ingressMedia.host }}
- secretName: {{ .Values.ingressMedia.tls.secretName | default (printf "%s-tls" $fullName) | quote }}
hosts:
- {{ .Values.ingressMedia.host | quote }}
{{- end }}
{{- range .Values.ingressMedia.tls.additional }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- if .Values.ingressMedia.host }}
- host: {{ .Values.ingressMedia.host | quote }}
http:
paths:
- path: {{ .Values.ingressMedia.path | quote }}
{{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }}
pathType: ImplementationSpecific
{{- end }}
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $fullName }}-media
port:
number: {{ .Values.serviceMedia.port }}
{{- else }}
serviceName: {{ $fullName }}-media
servicePort: {{ .Values.serviceMedia.port }}
{{- end }}
{{- end }}
{{- range .Values.ingressMedia.hosts }}
- host: {{ . | quote }}
http:
paths:
- path: {{ $.Values.ingressMedia.path | quote }}
{{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }}
pathType: ImplementationSpecific
{{- end }}
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $fullName }}-media
port:
number: {{ .Values.serviceMedia.port }}
{{- else }}
serviceName: {{ $fullName }}-media
servicePort: {{ .Values.serviceMedia.port }}
{{- end }}
{{- end }}
{{- end }}
src/helm/conversations/templates/media_svc.yaml
-14
View File
@@ -1,14 +0,0 @@
{{- $fullName := include "conversations.fullname" . -}}
{{- $component := "media" -}}
apiVersion: v1
kind: Service
metadata:
name: {{ $fullName }}-media
namespace: {{ .Release.Namespace | quote }}
labels:
{{- include "conversations.common.labels" (list . $component) | nindent 4 }}
annotations:
{{- toYaml $.Values.serviceMedia.annotations | nindent 4 }}
spec:
type: ExternalName
externalName: {{ $.Values.serviceMedia.host }}
src/helm/conversations/values.yaml
-41
View File
@@ -72,47 +72,6 @@ ingressAdmin:
secretName: null
additional: []
## @param ingressMedia.enabled whether to enable the Ingress or not
## @param ingressMedia.className IngressClass to use for the Ingress
## @param ingressMedia.host Host for the Ingress
## @param ingressMedia.path Path to use for the Ingress
ingressMedia:
enabled: false
className: null
host: conversations.example.com
path: /media/(.*)
## @param ingressMedia.hosts Additional host to configure for the Ingress
hosts: [ ]
# - chart-example.local
## @param ingressMedia.tls.enabled Weather to enable TLS for the Ingress
## @param ingressMedia.tls.secretName Secret name for TLS config
## @skip ingressMedia.tls.additional
## @extra ingressMedia.tls.additional[].secretName Secret name for additional TLS config
## @extra ingressMedia.tls.additional[].hosts[] Hosts for additional TLS config
tls:
enabled: true
secretName: null
additional: []
## @param ingressMedia.annotations.nginx.ingress.kubernetes.io/auth-url
## @param ingressMedia.annotations.nginx.ingress.kubernetes.io/auth-response-headers
## @param ingressMedia.annotations.nginx.ingress.kubernetes.io/upstream-vhost
## @param ingressMedia.annotations.nginx.ingress.kubernetes.io/configuration-snippet
annotations:
nginx.ingress.kubernetes.io/auth-url: https://conversations.example.com/api/v1.0/documents/media-auth/
nginx.ingress.kubernetes.io/auth-response-headers: "Authorization, X-Amz-Date, X-Amz-Content-SHA256"
nginx.ingress.kubernetes.io/upstream-vhost: minio.conversations.svc.cluster.local:9000
nginx.ingress.kubernetes.io/configuration-snippet: |
add_header Content-Security-Policy "default-src 'none'" always;
## @param serviceMedia.host
## @param serviceMedia.port
## @param serviceMedia.annotations
serviceMedia:
host: minio.conversations.svc.cluster.local
port: 9000
annotations: {}
## @section backend
src/helm/env.d/dev-staging/values.conversations.yaml.gotmpl
-18
View File
@@ -56,10 +56,6 @@ backend:
POSTGRES_USER: dinum
POSTGRES_PASSWORD: pass
REDIS_URL: redis://default:pass@redis-master:6379/2
AWS_S3_ENDPOINT_URL: http://minio.conversations.svc.cluster.local:9000
AWS_S3_ACCESS_KEY_ID: root
AWS_S3_SECRET_ACCESS_KEY: password
AWS_STORAGE_BUCKET_NAME: conversations-media-storage
STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
OIDC_STORE_ACCESS_TOKEN: True
OIDC_STORE_REFRESH_TOKEN: True
@@ -132,17 +128,3 @@ posthog:
ingressAssets:
enabled: false
ingressMedia:
enabled: true
host: conversations.127.0.0.1.nip.io
annotations:
nginx.ingress.kubernetes.io/auth-url: https://conversations.127.0.0.1.nip.io/api/v1.0/documents/media-auth/
nginx.ingress.kubernetes.io/auth-response-headers: "Authorization, X-Amz-Date, X-Amz-Content-SHA256"
nginx.ingress.kubernetes.io/upstream-vhost: minio.conversations.svc.cluster.local:9000
nginx.ingress.kubernetes.io/rewrite-target: /conversations-media-storage/$1
serviceMedia:
host: minio.conversations.svc.cluster.local
port: 9000
src/helm/env.d/dev/values.conversations.yaml.gotmpl
-18
View File
@@ -55,10 +55,6 @@ backend:
POSTGRES_PASSWORD: pass
REDIS_URL: redis://default:pass@redis-master:6379/1
DJANGO_CELERY_BROKER_URL: redis://default:pass@redis-master:6379/1
AWS_S3_ENDPOINT_URL: http://minio.conversations.svc.cluster.local:9000
AWS_S3_ACCESS_KEY_ID: root
AWS_S3_SECRET_ACCESS_KEY: password
AWS_STORAGE_BUCKET_NAME: conversations-media-storage
STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
CACHES_KEY_PREFIX: "{{ now | unixEpoch }}"
migrate:
@@ -148,17 +144,3 @@ posthog:
ingressAssets:
enabled: false
ingressMedia:
enabled: true
host: conversations.127.0.0.1.nip.io
annotations:
nginx.ingress.kubernetes.io/auth-url: https://conversations.127.0.0.1.nip.io/api/v1.0/documents/media-auth/
nginx.ingress.kubernetes.io/auth-response-headers: "Authorization, X-Amz-Date, X-Amz-Content-SHA256"
nginx.ingress.kubernetes.io/upstream-vhost: minio.conversations.svc.cluster.local:9000
nginx.ingress.kubernetes.io/rewrite-target: /conversations-media-storage/$1
serviceMedia:
host: minio.conversations.svc.cluster.local
port: 9000
src/helm/helmfile.yaml.gotmpl
+1 -16
View File
@@ -65,21 +65,6 @@ releases:
enabled: true
autoGenerated: true
- name: minio
installed: {{ eq .Environment.Name "dev" | toYaml }}
namespace: {{ .Namespace }}
chart: bitnami/minio
version: 12.10.10
values:
- auth:
rootUser: root
rootPassword: password
- provisioning:
enabled: true
buckets:
- name: conversations-media-storage
versioning: true
- name: redis
installed: {{ eq .Environment.Name "dev" | toYaml }}
namespace: {{ .Namespace }}
@@ -95,4 +80,4 @@ releases:
namespace: {{ .Namespace }}
chart: ./conversations
values:
- env.d/{{ .Environment.Name }}/values.conversations.yaml.gotmpl
- env.d/{{ .Environment.Name }}/values.conversations.yaml.gotmpl