diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..c485594 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,12 @@ +## Architecture + +### Global system architecture + +```mermaid +flowchart TD + Docs -- REST API --> Back("Backend (Django)") + Back --> DB("Database (PostgreSQL)") + Back -- REST API --> Opensearch + Back <--> Celery --> DB + User -- HTTP --> Dashboard --> Opensearch +``` diff --git a/docs/env.md b/docs/env.md new file mode 100644 index 0000000..b85a473 --- /dev/null +++ b/docs/env.md @@ -0,0 +1,105 @@ +# Find variables + +Here we describe all environment variables that can be set for the find application. + +## find-backend container + +These are the environment variables you can set for the `find-backend` container. + +| Option | Description | default | +|-------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------| +| API_USERS_LIST_LIMIT | Limit on API users | 5 | +| API_USERS_LIST_THROTTLE_RATE_BURST | Throttle rate for api on burst | 30/minute | +| API_USERS_LIST_THROTTLE_RATE_SUSTAINED | Throttle rate for api | 180/hour | +| CACHES_DEFAULT_TIMEOUT | Cache default timeout | 30 | +| CACHES_KEY_PREFIX | The prefix used to every cache keys. | docs | +| DB_ENGINE | Engine to use for database connections | django.db.backends.postgresql_psycopg2 | +| DB_HOST | Host of the database | localhost | +| DB_NAME | Name of the database | impress | +| DB_PASSWORD | Password to authenticate with | pass | +| DB_PORT | Port of the database | 5432 | +| DB_USER | User to authenticate with | dinum | +| DJANGO_ALLOWED_HOSTS | Allowed hosts | [] | +| DJANGO_CELERY_BROKER_TRANSPORT_OPTIONS | Celery broker transport options | {} | +| DJANGO_CELERY_BROKER_URL | Celery broker url | redis://redis:6379/0 | +| DJANGO_CORS_ALLOW_ALL_ORIGINS | Allow all CORS origins | false | +| DJANGO_CORS_ALLOWED_ORIGIN_REGEXES | List of origins allowed for CORS using regulair expressions | [] | +| DJANGO_CORS_ALLOWED_ORIGINS | List of origins allowed for CORS | [] | +| DJANGO_CSRF_TRUSTED_ORIGINS | CSRF trusted origins | [] | +| DJANGO_EMAIL_BACKEND | Email backend library | django.core.mail.backends.smtp.EmailBackend | +| DJANGO_EMAIL_BRAND_NAME | Brand name for email | | +| DJANGO_EMAIL_FROM | Email address used as sender | from@example.com | +| DJANGO_EMAIL_HOST | Hostname of email | | +| DJANGO_EMAIL_HOST_PASSWORD | Password to authenticate with on the email host | | +| DJANGO_EMAIL_HOST_USER | User to authenticate with on the email host | | +| DJANGO_EMAIL_LOGO_IMG | Logo for the email | | +| DJANGO_EMAIL_PORT | Port used to connect to email host | | +| DJANGO_EMAIL_USE_SSL | Use ssl for email host connection | false | +| DJANGO_EMAIL_USE_TLS | Use tls for email host connection | false | +| DJANGO_SECRET_KEY | Secret key | | +| DJANGO_SERVER_TO_SERVER_API_TOKENS | | [] | +| DOCUMENT_IMAGE_MAX_SIZE | Maximum size of document in bytes | 10485760 | +| FRONTEND_CSS_URL | To add a external css file to the app | | +| FRONTEND_HOMEPAGE_FEATURE_ENABLED | Frontend feature flag to display the homepage | false | +| FRONTEND_THEME | Frontend theme to use | | +| LANGUAGE_CODE | Default language | en-us | +| LOGGING_LEVEL_LOGGERS_APP | Application logging level. options are "DEBUG", "INFO", "WARN", "ERROR", "CRITICAL" | INFO | +| LOGGING_LEVEL_LOGGERS_ROOT | Default logging level. options are "DEBUG", "INFO", "WARN", "ERROR", "CRITICAL" | INFO | +| LOGIN_REDIRECT_URL | Login redirect url | | +| LOGIN_REDIRECT_URL_FAILURE | Login redirect url on failure | | +| LOGOUT_REDIRECT_URL | Logout redirect url | | +| MALWARE_DETECTION_BACKEND | The malware detection backend use from the django-lasuite package | lasuite.malware_detection.backends.dummy.DummyBackend | +| MALWARE_DETECTION_PARAMETERS | A dict containing all the parameters to initiate the malware detection backend | {"callback_path": "core.malware_detection.malware_detection_callback",} | +| MEDIA_BASE_URL | | | +| NO_WEBSOCKET_CACHE_TIMEOUT | Cache used to store current editor session key when only users without websocket are editing a document | 120 | +| OIDC_ALLOW_DUPLICATE_EMAILS | Allow duplicate emails | false | +| OIDC_AUTH_REQUEST_EXTRA_PARAMS | OIDC extra auth parameters | {} | +| OIDC_CREATE_USER | Create used on OIDC | false | +| OIDC_DRF_AUTH_BACKEND | DRF Authentication backend class for OIDC | lasuite.oidc_login.backends.OIDCAuthenticationBackend | +| OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION | Fallback to email for identification | true | +| OIDC_OP_AUTHORIZATION_ENDPOINT | Authorization endpoint for OIDC | | +| OIDC_OP_INTROSPECTION_ENDPOINT | Introspection endpoint for OIDC | | +| OIDC_OP_JWKS_ENDPOINT | JWKS endpoint for OIDC | | +| OIDC_OP_LOGOUT_ENDPOINT | Logout endpoint for OIDC | | +| OIDC_OP_TOKEN_ENDPOINT | Token endpoint for OIDC | | +| OIDC_OP_USER_ENDPOINT | User endpoint for OIDC | | +| OIDC_REDIRECT_ALLOWED_HOSTS | Allowed hosts for OIDC redirect url | [] | +| OIDC_REDIRECT_REQUIRE_HTTPS | Require https for OIDC redirect url | false | +| OIDC_RP_CLIENT_ID | Client id used for OIDC | impress | +| OIDC_RP_CLIENT_SECRET | Client secret used for OIDC | | +| OIDC_RP_SCOPES | Scopes requested for OIDC | openid email | +| OIDC_RP_SIGN_ALGO | verification algorithm used OIDC tokens | RS256 | +| OIDC_RS_BACKEND_CLASS | Resource server backend class for OIDC | core.authentication.FinderResourceServerBackend | +| OIDC_RS_AUDIENCE_CLAIM | The claim used to identify the audience | client_id | +| OIDC_RS_CLIENT_ID | Client id used for OIDC resource server | | +| OIDC_RS_CLIENT_SECRET | Client secret used for OIDC resource server | | +| OIDC_RS_SIGNING_ALGO | Signing algorithm | ES256 | +| OIDC_RS_SCOPES | Required scopes for authentication | ["openid"] | +| OIDC_RS_PRIVATE_KEY_STR | Private key for encryption/decryption | | +| OIDC_RS_ENCRYPTION_KEY_TYPE | Encryption key type (RSA, EC, etc.) | RSA | +| OIDC_RS_ENCRYPTION_ALGO | Encryption algorithm | RSA-OAEP | +| OIDC_RS_ENCRYPTION_ENCODING | Encryption encoding algorithm | A256GCM | +| OIDC_STORE_ID_TOKEN | Store OIDC token | true | +| OIDC_USE_NONCE | Use nonce for OIDC | true | +| OIDC_USERINFO_FULLNAME_FIELDS | OIDC token claims to create full name | ["first_name", "last_name"] | +| OIDC_USERINFO_SHORTNAME_FIELD | OIDC token claims to create shortname | first_name | +| OIDC_VERIFY_SSL | Enable SSL validation for OIDC | true | +| OIDC_TIMEOUT | Timeout delay for OID requests | | +| OIDC_PROXY | Defines a proxy for all requests to the OpenID Connect provider | | +| OPENSEARCH_HOST | Opensearch database url | default | +| OPENSEARCH_PORT | Port of Opensearch database | 9200 | +| OPENSEARCH_USER | Opensearch database user | admin | +| OPENSEARCH_PASSWORD | Opensearch database user password | | +| OPENSEARCH_USE_SSL | Enable SSL connection for Opensearch database | true | +| POSTHOG_KEY | Posthog key for analytics | | +| REDIS_URL | Cache url | redis://redis:6379/1 | +| SENTRY_DSN | Sentry host | | +| SESSION_COOKIE_AGE | duration of the cookie session | 60*60*12 | +| SPECTACULAR_SETTINGS_ENABLE_DJANGO_DEPLOY_CHECK | | false | +| STORAGES_STATICFILES_BACKEND | | whitenoise.storage.CompressedManifestStaticFilesStorage | +| THEME_CUSTOMIZATION_CACHE_TIMEOUT | Cache duration for the customization settings | 86400 | +| THEME_CUSTOMIZATION_FILE_PATH | Full path to the file customizing the theme. An example is provided in src/backend/impress/configuration/theme/default.json | BASE_DIR/impress/configuration/theme/default.json | +| TRASHBIN_CUTOFF_DAYS | Trashbin cutoff | 30 | +| USER_OIDC_ESSENTIAL_CLAIMS | Essential claims in OIDC token | [] | +| Y_PROVIDER_API_BASE_URL | Y Provider url | | +| Y_PROVIDER_API_KEY | Y provider API key | | diff --git a/docs/release.md b/docs/release.md new file mode 100644 index 0000000..5beb3fb --- /dev/null +++ b/docs/release.md @@ -0,0 +1,58 @@ +# Releasing a new version + +Whenever we are cooking a new release (e.g. `0.0.1`) we should follow a standard procedure described below: + +1. Create a new branch named: `release/0.0.1`. + +2. Bump the release number for backend project and Helm files: + + - for backend, update the version number by hand in `pyproject.toml`, + - for Helm, update Docker image tag in files located at `src/helm/env.d` for both `preprod` and `production` environments: + + ```yaml + image: + repository: lasuite/find + pullPolicy: Always + tag: "v0.0.1" # Replace with your new version number, without forgetting the "v" prefix + ... + + The new images don't exist _yet_: they will be created automatically later in the process. + +3. Update the project's `Changelog` following the [keepachangelog](https://keepachangelog.com/en/0.3.0/) recommendations + +4. Commit your changes with the following format: the ๐Ÿ”– release emoji, the type of release (patch/minor/patch) and the release version: + + ```text + ๐Ÿ”–(minor) bump release to 0.0.1 + ``` + +5. Open a pull request, wait for an approval from your peers and merge it. +6. Checkout and pull changes from the `main` branch to ensure you have the latest updates. +7. Tag and push your commit: + + ```bash + git tag v0.0.1 && git push origin tag v0.0.1 + ``` + + Doing this triggers the CI and tells it to build the new Docker image versions that you targeted earlier in the Helm files. + +8. Ensure the new [backend](https://hub.docker.com/r/lasuite/find/tags) image tags are on Docker Hub. +9. The release is now done! + +# Deploying + +> [!TIP] +> The `staging` platform is deployed automatically with every update of the `main` branch. + +Making a new release doesn't publish it automatically in production. + +Deployment is done by ArgoCD. ArgoCD checks for the `production` tag and automatically deploys the production platform with the targeted commit. + +To publish, we mark the commit we want with the `production` tag. ArgoCD is then notified that the tag has changed. It then deploys the Docker image tags specified in the Helm files of the targeted commit. + +To publish the release you just made: + +```bash +git tag --force production v0.0.1 +git push --force origin production +``` diff --git a/docs/setup-for-docs.md b/docs/setup-for-docs.md new file mode 100644 index 0000000..353a99c --- /dev/null +++ b/docs/setup-for-docs.md @@ -0,0 +1,40 @@ +# Setup the Find search for Impress + +This configuration will enable the fulltext search feature for Docs : +- Each save on **core.Document** or **core.DocumentAccess** will trigger the indexer +- The `api/v1.0/documents/search/` will work as a proxy with the Find API for fulltext search. + +## Create an index service for Docs + +Configure a **Service** for Docs application with these settings + +- **Name**: `docs`
_request.auth.name of the Docs application._ +- **Client id**: `impress`
_Name of the token audience or client_id of the Docs application._ + +See [how-to-use-indexer.md](how-to-use-indexer.md) for details. + +## Configure settings of Docs + +Add those Django settings the Docs application to enable the feature. + +```shell +SEARCH_INDEXER_CLASS="core.services.search_indexers.FindDocumentIndexer" +SEARCH_INDEXER_COUNTDOWN=10 # Debounce delay in seconds for the indexer calls. + +# Indexation endpoint. +SEARCH_INDEXER_SECRET=my-token-from-the-find-impress-service +# The token from service "docs" of Find application. +SEARCH_INDEXER_URL="http://app-find:8000/api/v1.0/documents/index/" + +# Search endpoint. Uses the OIDC token for authentication +SEARCH_INDEXER_QUERY_URL="http://app-find:8000/api/v1.0/documents/search/" +``` + +We also need to enable the **OIDC Token** refresh or the authentication will fail quickly. + +```shell +# Store OIDC tokens in the session +OIDC_STORE_ACCESS_TOKEN = True # Store the access token in the session +OIDC_STORE_REFRESH_TOKEN = True # Store the encrypted refresh token in the session +OIDC_STORE_REFRESH_TOKEN_KEY = "your-32-byte-encryption-key==" # Must be a valid Fernet key (32 url-safe base64-encoded bytes) +``` diff --git a/docs/setup-indexer.md b/docs/setup-indexer.md new file mode 100644 index 0000000..93ddc41 --- /dev/null +++ b/docs/setup-indexer.md @@ -0,0 +1,87 @@ +# Using the Find indexer + +This guide explains how to setup the Find service which provide an API for indexation and fulltext search of +documents from various sources in a secure way : only the documents within the scope of the user's OIDC token +are visible. + +## Setup Opensearch + +Add the following settings to your Django settings for the Find application. + +```shell +#ย Login for opensearch +OPENSEARCH_USER=opensearch-user +OPENSEARCH_PASSWORD=your-opensearch-password + +# Host configuration +OPENSEARCH_HOST=opensearch +OPENSEARCH_PORT=9200 + +# Enable SSL for opensearch connection (False in dev mode) +OPENSEARCH_USE_SSL=True +``` + +## Setup indexation API + +Other applications can index their files through the **`/index/`** endpoint with a simple token authentication. + +For each application a new **Service** must be created through the admin interface +(see http://localhost:9071/admin/core/service/add/) + +| Field | Description | +|-----------------------------|----------------------------------------------------| +| Name | Name of the service and also the name of the index in Opensearch database | +| Is active | Toggle service availability | +| Client id | Calling service client_id (e.g `impress` for docs) | +| Allowed services for search | List of sub-services. Will add the results from all these index
to the search results. | +| Token (_read-only_) | Random token for calling service authentication | + +And add the key in the calling application Django settings. + +**Development Mode (Docs + Find)** + +The command `make demo` will create a working service configuration for `docs` with the following secret key + +```python +SEARCH_INDEXER_SECRET="find-api-key-for-docs-with-exactly-50-chars-length" +``` + +## Setup search API + +The **`/search/`** endpoint is an OIDC ResourceServer view and needs extra Django settings (see [lasuite](https://github.com/suitenumerique/django-lasuite/blob/main/documentation/how-to-use-oidc-resource-server-backend.md) for details) + +```shell +OIDC_OP_JWKS_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/certs +OIDC_OP_AUTHORIZATION_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/auth +OIDC_OP_TOKEN_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/token +OIDC_OP_USER_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/userinfo + +# To run Find in development mode along other projects like docs/impress +# we should to use OIDC endpoints on a common keycloak realm. e.g : +# OIDC_OP_URL = http://nginx:8083/realms/impress +# +# This will cause a conflict with the 'iss' claim validation rule because the docs realm +# gives {'iss': 'http://localhost:8083/realms/impress'} so it must be the same +OIDC_OP_URL=http://localhost:8083/realms/impress + +# Introspection endpoint is needed to get the "audience" and "sub" from the user token +OIDC_OP_INTROSPECTION_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/token/introspect + +# In development, the resource server use insecure settings +# OIDC_VERIFY_SSL=False + +# Resource server +OIDC_RS_SCOPES="openid" +OIDC_RS_SIGN_ALGO=RS256 + +# This backend allows authentication without any model in database. +OIDC_RS_BACKEND_CLASS="core.authentication.FinderResourceServerBackend" +``` + +**Development mode (Docs + Find)** + +Docs and Find projects stacks can be run together but must have the same keycloak server. + +So the endpoints of Docs on the 'nginx' domain for ResourceServer authentication and introspection are also used for Find. + +**IMPORTANT:** Keep OIDC_OP_URL on 'localhost' or it will break the OIDC token claims validation : the `iss` claim from the token of the 'impress' users are 'localhost' and not 'nginx'. diff --git a/docs/system-requirements.md b/docs/system-requirements.md new file mode 100644 index 0000000..589a201 --- /dev/null +++ b/docs/system-requirements.md @@ -0,0 +1,12 @@ +# La Suite Find โ€“ System & Requirements + +## Ports (dev defaults) + +| Port | Service | +| --------- | --------------------- | +| 8081 | Django (main) | +| 8083 | Nginx proxy | +| 25432 | PostgreSQL (main) | +| 9200 | Opensearch | +| 9600 | Opensearch admin | +| 5601 | Opensearch dashboard |