Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Fabre Florian
|
09c687878d | ||
|
Fabre Florian
|
43265acf36 | ||
|
Fabre Florian
|
79208765c9 | ||
|
Fabre Florian
|
15f58e68c8 |
@@ -1,3 +1,5 @@
|
||||
name: find
|
||||
|
||||
services:
|
||||
postgresql:
|
||||
image: postgres:15
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
## Architecture
|
||||
|
||||
### Global system architecture
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Docs -- REST API --> Back("Backend (Django)")
|
||||
Back --> DB("Database (PostgreSQL)")
|
||||
Back -- REST API --> Opensearch
|
||||
Back <--> Celery --> DB
|
||||
User -- HTTP --> Dashboard --> Opensearch
|
||||
```
|
||||
+106
@@ -0,0 +1,106 @@
|
||||
# Find variables
|
||||
|
||||
Here we describe all environment variables that can be set for the find application.
|
||||
|
||||
## find-backend container
|
||||
|
||||
These are the environment variables you can set for the `find-backend` container.
|
||||
|
||||
| Option | Description | default |
|
||||
|-------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------|
|
||||
| API_USERS_LIST_LIMIT | Limit on API users | 5 |
|
||||
| API_USERS_LIST_THROTTLE_RATE_BURST | Throttle rate for api on burst | 30/minute |
|
||||
| API_USERS_LIST_THROTTLE_RATE_SUSTAINED | Throttle rate for api | 180/hour |
|
||||
| CACHES_DEFAULT_TIMEOUT | Cache default timeout | 30 |
|
||||
| CACHES_KEY_PREFIX | The prefix used to every cache keys. | docs |
|
||||
| DB_ENGINE | Engine to use for database connections | django.db.backends.postgresql_psycopg2 |
|
||||
| DB_HOST | Host of the database | localhost |
|
||||
| DB_NAME | Name of the database | impress |
|
||||
| DB_PASSWORD | Password to authenticate with | pass |
|
||||
| DB_PORT | Port of the database | 5432 |
|
||||
| DB_USER | User to authenticate with | dinum |
|
||||
| DJANGO_ALLOWED_HOSTS | Allowed hosts | [] |
|
||||
| DJANGO_CELERY_BROKER_TRANSPORT_OPTIONS | Celery broker transport options | {} |
|
||||
| DJANGO_CELERY_BROKER_URL | Celery broker url | redis://redis:6379/0 |
|
||||
| DJANGO_CORS_ALLOW_ALL_ORIGINS | Allow all CORS origins | false |
|
||||
| DJANGO_CORS_ALLOWED_ORIGIN_REGEXES | List of origins allowed for CORS using regulair expressions | [] |
|
||||
| DJANGO_CORS_ALLOWED_ORIGINS | List of origins allowed for CORS | [] |
|
||||
| DJANGO_CSRF_TRUSTED_ORIGINS | CSRF trusted origins | [] |
|
||||
| DJANGO_EMAIL_BACKEND | Email backend library | django.core.mail.backends.smtp.EmailBackend |
|
||||
| DJANGO_EMAIL_BRAND_NAME | Brand name for email | |
|
||||
| DJANGO_EMAIL_FROM | Email address used as sender | from@example.com |
|
||||
| DJANGO_EMAIL_HOST | Hostname of email | |
|
||||
| DJANGO_EMAIL_HOST_PASSWORD | Password to authenticate with on the email host | |
|
||||
| DJANGO_EMAIL_HOST_USER | User to authenticate with on the email host | |
|
||||
| DJANGO_EMAIL_LOGO_IMG | Logo for the email | |
|
||||
| DJANGO_EMAIL_PORT | Port used to connect to email host | |
|
||||
| DJANGO_EMAIL_USE_SSL | Use ssl for email host connection | false |
|
||||
| DJANGO_EMAIL_USE_TLS | Use tls for email host connection | false |
|
||||
| DJANGO_SECRET_KEY | Secret key | |
|
||||
| DJANGO_SERVER_TO_SERVER_API_TOKENS | | [] |
|
||||
| DOCUMENT_IMAGE_MAX_SIZE | Maximum size of document in bytes | 10485760 |
|
||||
| FRONTEND_CSS_URL | To add a external css file to the app | |
|
||||
| FRONTEND_HOMEPAGE_FEATURE_ENABLED | Frontend feature flag to display the homepage | false |
|
||||
| FRONTEND_THEME | Frontend theme to use | |
|
||||
| LANGUAGE_CODE | Default language | en-us |
|
||||
| LOGGING_LEVEL_LOGGERS_APP | Application logging level. options are "DEBUG", "INFO", "WARN", "ERROR", "CRITICAL" | INFO |
|
||||
| LOGGING_LEVEL_LOGGERS_ROOT | Default logging level. options are "DEBUG", "INFO", "WARN", "ERROR", "CRITICAL" | INFO |
|
||||
| LOGIN_REDIRECT_URL | Login redirect url | |
|
||||
| LOGIN_REDIRECT_URL_FAILURE | Login redirect url on failure | |
|
||||
| LOGOUT_REDIRECT_URL | Logout redirect url | |
|
||||
| MALWARE_DETECTION_BACKEND | The malware detection backend use from the django-lasuite package | lasuite.malware_detection.backends.dummy.DummyBackend |
|
||||
| MALWARE_DETECTION_PARAMETERS | A dict containing all the parameters to initiate the malware detection backend | {"callback_path": "core.malware_detection.malware_detection_callback",} |
|
||||
| MEDIA_BASE_URL | | |
|
||||
| NO_WEBSOCKET_CACHE_TIMEOUT | Cache used to store current editor session key when only users without websocket are editing a document | 120 |
|
||||
| OIDC_ALLOW_DUPLICATE_EMAILS | Allow duplicate emails | false |
|
||||
| OIDC_AUTH_REQUEST_EXTRA_PARAMS | OIDC extra auth parameters | {} |
|
||||
| OIDC_CREATE_USER | Create used on OIDC | false |
|
||||
| OIDC_DRF_AUTH_BACKEND | DRF Authentication backend class for OIDC | lasuite.oidc_login.backends.OIDCAuthenticationBackend |
|
||||
| OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION | Fallback to email for identification | true |
|
||||
| OIDC_OP_AUTHORIZATION_ENDPOINT | Authorization endpoint for OIDC | |
|
||||
| OIDC_OP_INTROSPECTION_ENDPOINT | Introspection endpoint for OIDC | |
|
||||
| OIDC_OP_JWKS_ENDPOINT | JWKS endpoint for OIDC | |
|
||||
| OIDC_OP_LOGOUT_ENDPOINT | Logout endpoint for OIDC | |
|
||||
| OIDC_OP_TOKEN_ENDPOINT | Token endpoint for OIDC | |
|
||||
| OIDC_OP_USER_ENDPOINT | User endpoint for OIDC | |
|
||||
| OIDC_REDIRECT_ALLOWED_HOSTS | Allowed hosts for OIDC redirect url | [] |
|
||||
| OIDC_REDIRECT_REQUIRE_HTTPS | Require https for OIDC redirect url | false |
|
||||
| OIDC_RP_CLIENT_ID | Client id used for OIDC | impress |
|
||||
| OIDC_RP_CLIENT_SECRET | Client secret used for OIDC | |
|
||||
| OIDC_RP_SCOPES | Scopes requested for OIDC | openid email |
|
||||
| OIDC_RP_SIGN_ALGO | verification algorithm used OIDC tokens | RS256 |
|
||||
| OIDC_RS_BACKEND_CLASS | Resource server backend class for OIDC | core.authentication.FinderResourceServerBackend |
|
||||
| OIDC_RS_AUDIENCE_CLAIM | The claim used to identify the audience | client_id |
|
||||
| OIDC_RS_CLIENT_ID | Client id used for OIDC resource server | |
|
||||
| OIDC_RS_CLIENT_SECRET | Client secret used for OIDC resource server | |
|
||||
| OIDC_RS_SIGNING_ALGO | Signing algorithm | ES256 |
|
||||
| OIDC_RS_SCOPES | Required scopes for authentication | ["openid"] |
|
||||
| OIDC_RS_PRIVATE_KEY_STR | Private key for encryption/decryption | |
|
||||
| OIDC_RS_ENCRYPTION_KEY_TYPE | Encryption key type (RSA, EC, etc.) | RSA |
|
||||
| OIDC_RS_ENCRYPTION_ALGO | Encryption algorithm | RSA-OAEP |
|
||||
| OIDC_RS_ENCRYPTION_ENCODING | Encryption encoding algorithm | A256GCM |
|
||||
| OIDC_RS_VERIFY_CLAIMS | Enable OICD token claims validation | true |
|
||||
| OIDC_STORE_ID_TOKEN | Store OIDC token | true |
|
||||
| OIDC_USE_NONCE | Use nonce for OIDC | true |
|
||||
| OIDC_USERINFO_FULLNAME_FIELDS | OIDC token claims to create full name | ["first_name", "last_name"] |
|
||||
| OIDC_USERINFO_SHORTNAME_FIELD | OIDC token claims to create shortname | first_name |
|
||||
| OIDC_VERIFY_SSL | Enable SSL validation for OIDC | true |
|
||||
| OIDC_TIMEOUT | Timeout delay for OID requests | |
|
||||
| OIDC_PROXY | Defines a proxy for all requests to the OpenID Connect provider | |
|
||||
| OPENSEARCH_HOST | Opensearch database url | default |
|
||||
| OPENSEARCH_PORT | Port of Opensearch database | 9200 |
|
||||
| OPENSEARCH_USER | Opensearch database user | admin |
|
||||
| OPENSEARCH_PASSWORD | Opensearch database user password | |
|
||||
| OPENSEARCH_USE_SSL | Enable SSL connection for Opensearch database | true |
|
||||
| POSTHOG_KEY | Posthog key for analytics | |
|
||||
| REDIS_URL | Cache url | redis://redis:6379/1 |
|
||||
| SENTRY_DSN | Sentry host | |
|
||||
| SESSION_COOKIE_AGE | duration of the cookie session | 60*60*12 |
|
||||
| SPECTACULAR_SETTINGS_ENABLE_DJANGO_DEPLOY_CHECK | | false |
|
||||
| STORAGES_STATICFILES_BACKEND | | whitenoise.storage.CompressedManifestStaticFilesStorage |
|
||||
| THEME_CUSTOMIZATION_CACHE_TIMEOUT | Cache duration for the customization settings | 86400 |
|
||||
| THEME_CUSTOMIZATION_FILE_PATH | Full path to the file customizing the theme. An example is provided in src/backend/impress/configuration/theme/default.json | BASE_DIR/impress/configuration/theme/default.json |
|
||||
| TRASHBIN_CUTOFF_DAYS | Trashbin cutoff | 30 |
|
||||
| USER_OIDC_ESSENTIAL_CLAIMS | Essential claims in OIDC token | [] |
|
||||
| Y_PROVIDER_API_BASE_URL | Y Provider url | |
|
||||
| Y_PROVIDER_API_KEY | Y provider API key | |
|
||||
@@ -0,0 +1,58 @@
|
||||
# Releasing a new version
|
||||
|
||||
Whenever we are cooking a new release (e.g. `0.0.1`) we should follow a standard procedure described below:
|
||||
|
||||
1. Create a new branch named: `release/0.0.1`.
|
||||
|
||||
2. Bump the release number for backend project and Helm files:
|
||||
|
||||
- for backend, update the version number by hand in `pyproject.toml`,
|
||||
- for Helm, update Docker image tag in files located at `src/helm/env.d` for both `preprod` and `production` environments:
|
||||
|
||||
```yaml
|
||||
image:
|
||||
repository: lasuite/find
|
||||
pullPolicy: Always
|
||||
tag: "v0.0.1" # Replace with your new version number, without forgetting the "v" prefix
|
||||
...
|
||||
|
||||
The new images don't exist _yet_: they will be created automatically later in the process.
|
||||
|
||||
3. Update the project's `Changelog` following the [keepachangelog](https://keepachangelog.com/en/0.3.0/) recommendations
|
||||
|
||||
4. Commit your changes with the following format: the 🔖 release emoji, the type of release (patch/minor/patch) and the release version:
|
||||
|
||||
```text
|
||||
🔖(minor) bump release to 0.0.1
|
||||
```
|
||||
|
||||
5. Open a pull request, wait for an approval from your peers and merge it.
|
||||
6. Checkout and pull changes from the `main` branch to ensure you have the latest updates.
|
||||
7. Tag and push your commit:
|
||||
|
||||
```bash
|
||||
git tag v0.0.1 && git push origin tag v0.0.1
|
||||
```
|
||||
|
||||
Doing this triggers the CI and tells it to build the new Docker image versions that you targeted earlier in the Helm files.
|
||||
|
||||
8. Ensure the new [backend](https://hub.docker.com/r/lasuite/find/tags) image tags are on Docker Hub.
|
||||
9. The release is now done!
|
||||
|
||||
# Deploying
|
||||
|
||||
> [!TIP]
|
||||
> The `staging` platform is deployed automatically with every update of the `main` branch.
|
||||
|
||||
Making a new release doesn't publish it automatically in production.
|
||||
|
||||
Deployment is done by ArgoCD. ArgoCD checks for the `production` tag and automatically deploys the production platform with the targeted commit.
|
||||
|
||||
To publish, we mark the commit we want with the `production` tag. ArgoCD is then notified that the tag has changed. It then deploys the Docker image tags specified in the Helm files of the targeted commit.
|
||||
|
||||
To publish the release you just made:
|
||||
|
||||
```bash
|
||||
git tag --force production v0.0.1
|
||||
git push --force origin production
|
||||
```
|
||||
@@ -0,0 +1,40 @@
|
||||
# Setup the Find search for Impress
|
||||
|
||||
This configuration will enable the fulltext search feature for Docs :
|
||||
- Each save on **core.Document** or **core.DocumentAccess** will trigger the indexer
|
||||
- The `api/v1.0/documents/search/` will work as a proxy with the Find API for fulltext search.
|
||||
|
||||
## Create an index service for Docs
|
||||
|
||||
Configure a **Service** for Docs application with these settings
|
||||
|
||||
- **Name**: `docs`<br>_request.auth.name of the Docs application._
|
||||
- **Client id**: `impress`<br>_Name of the token audience or client_id of the Docs application._
|
||||
|
||||
See [how-to-use-indexer.md](how-to-use-indexer.md) for details.
|
||||
|
||||
## Configure settings of Docs
|
||||
|
||||
Add those Django settings the Docs application to enable the feature.
|
||||
|
||||
```shell
|
||||
SEARCH_INDEXER_CLASS="core.services.search_indexers.FindDocumentIndexer"
|
||||
SEARCH_INDEXER_COUNTDOWN=10 # Debounce delay in seconds for the indexer calls.
|
||||
|
||||
# Indexation endpoint.
|
||||
SEARCH_INDEXER_SECRET=my-token-from-the-find-impress-service
|
||||
# The token from service "docs" of Find application.
|
||||
SEARCH_INDEXER_URL="http://app-find:8000/api/v1.0/documents/index/"
|
||||
|
||||
# Search endpoint. Uses the OIDC token for authentication
|
||||
SEARCH_INDEXER_QUERY_URL="http://app-find:8000/api/v1.0/documents/search/"
|
||||
```
|
||||
|
||||
We also need to enable the **OIDC Token** refresh or the authentication will fail quickly.
|
||||
|
||||
```shell
|
||||
# Store OIDC tokens in the session
|
||||
OIDC_STORE_ACCESS_TOKEN = True # Store the access token in the session
|
||||
OIDC_STORE_REFRESH_TOKEN = True # Store the encrypted refresh token in the session
|
||||
OIDC_STORE_REFRESH_TOKEN_KEY = "your-32-byte-encryption-key==" # Must be a valid Fernet key (32 url-safe base64-encoded bytes)
|
||||
```
|
||||
@@ -0,0 +1,83 @@
|
||||
# Using the Find indexer
|
||||
|
||||
This guide explains how to setup the Find service which provide an API for indexation and fulltext search of
|
||||
documents from various sources in a secure way : only the documents within the scope of the user's OIDC token
|
||||
are visible.
|
||||
|
||||
## Setup Opensearch
|
||||
|
||||
Add the following settings to your Django settings for the Find application.
|
||||
|
||||
```shell
|
||||
# Login for opensearch
|
||||
OPENSEARCH_USER=opensearch-user
|
||||
OPENSEARCH_PASSWORD=your-opensearch-password
|
||||
|
||||
# Host configuration
|
||||
OPENSEARCH_HOST=opensearch
|
||||
OPENSEARCH_PORT=9200
|
||||
|
||||
# Enable SSL for opensearch connection (False in dev mode)
|
||||
OPENSEARCH_USE_SSL=True
|
||||
```
|
||||
|
||||
## Setup indexation API
|
||||
|
||||
Other applications can index their files through the **`/index/`** endpoint with a simple token authentication.
|
||||
|
||||
For each application a new **Service** must be created through the admin interface
|
||||
(see http://localhost:9071/admin/core/service/add/)
|
||||
|
||||
| Field | Description |
|
||||
|-----------------------------|----------------------------------------------------|
|
||||
| Name | Name of the service and also the name of the index in Opensearch database |
|
||||
| Is active | Toggle service availability |
|
||||
| Client id | Calling service client_id (e.g `impress` for docs) |
|
||||
| Allowed services for search | List of sub-services. Will add the results from all these index<br>to the search results. |
|
||||
| Token (_read-only_) | Random token for calling service authentication |
|
||||
|
||||
And add the key in the calling application Django settings.
|
||||
|
||||
**Development Mode (Docs + Find)**
|
||||
|
||||
The command `make demo` will create a working service configuration for `docs` with the following secret key
|
||||
|
||||
```python
|
||||
SEARCH_INDEXER_SECRET="find-api-key-for-docs-with-exactly-50-chars-length"
|
||||
```
|
||||
|
||||
## Setup search API
|
||||
|
||||
The **`/search/`** endpoint is an OIDC ResourceServer view and needs extra Django settings (see [lasuite](https://github.com/suitenumerique/django-lasuite/blob/main/documentation/how-to-use-oidc-resource-server-backend.md) for details)
|
||||
|
||||
```shell
|
||||
OIDC_OP_JWKS_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/certs
|
||||
OIDC_OP_AUTHORIZATION_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/auth
|
||||
OIDC_OP_TOKEN_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/token
|
||||
OIDC_OP_USER_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/userinfo
|
||||
OIDC_OP_URL=http://nginx:8083/realms/impress
|
||||
|
||||
# Introspection endpoint is needed to get the "audience" and "sub" from the user token
|
||||
OIDC_OP_INTROSPECTION_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/token/introspect
|
||||
|
||||
# In development, the resource server use insecure settings
|
||||
# OIDC_VERIFY_SSL=False
|
||||
# OIDC_RS_VERIFY_CLAIMS=False
|
||||
|
||||
# Resource server
|
||||
OIDC_RS_SCOPES="openid"
|
||||
OIDC_RS_SIGN_ALGO=RS256
|
||||
|
||||
# This backend allows authentication without any model in database.
|
||||
OIDC_RS_BACKEND_CLASS="core.authentication.FinderResourceServerBackend"
|
||||
```
|
||||
|
||||
**Development mode (Docs + Find)**
|
||||
|
||||
Docs and Find projects stacks can be run together but must have the same keycloak server.
|
||||
|
||||
So the endpoints of Docs on the 'nginx' domain for ResourceServer authentication and introspection are also used for Find.
|
||||
This configuration breaks the OIDC token claims validation : the `iss` claim from the token of the 'impress' users are 'localhost' and not 'nginx'.
|
||||
|
||||
The setting `OIDC_RS_VERIFY_CLAIMS=False` allows to ignore this step and make things works.
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
# La Suite Find – System & Requirements
|
||||
|
||||
## Ports (dev defaults)
|
||||
|
||||
| Port | Service |
|
||||
| --------- | --------------------- |
|
||||
| 8081 | Django (main) |
|
||||
| 8083 | Nginx proxy |
|
||||
| 25432 | PostgreSQL (main) |
|
||||
| 9200 | Opensearch |
|
||||
| 9600 | Opensearch admin |
|
||||
| 5601 | Opensearch dashboard |
|
||||
@@ -17,12 +17,12 @@ OPENSEARCH_PASSWORD=find.PASS123
|
||||
OPENSEARCH_USE_SSL=false
|
||||
|
||||
# OIDC
|
||||
OIDC_OP_JWKS_ENDPOINT=http://nginx:8083/realms/find/protocol/openid-connect/certs
|
||||
OIDC_OP_AUTHORIZATION_ENDPOINT=http://localhost:8083/realms/find/protocol/openid-connect/auth
|
||||
OIDC_OP_TOKEN_ENDPOINT=http://nginx:8083/realms/find/protocol/openid-connect/token
|
||||
OIDC_OP_USER_ENDPOINT=http://nginx:8083/realms/find/protocol/openid-connect/userinfo
|
||||
OIDC_OP_URL=http://nginx:8083/realms/find
|
||||
OIDC_OP_INTROSPECTION_ENDPOINT=http://nginx:8083/realms/find/protocol/openid-connect/token/introspect
|
||||
OIDC_OP_JWKS_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/certs
|
||||
OIDC_OP_AUTHORIZATION_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/auth
|
||||
OIDC_OP_TOKEN_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/token
|
||||
OIDC_OP_USER_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/userinfo
|
||||
OIDC_OP_URL=http://nginx:8083/realms/impress
|
||||
OIDC_OP_INTROSPECTION_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/token/introspect
|
||||
|
||||
OIDC_RP_CLIENT_ID=find
|
||||
OIDC_RP_CLIENT_SECRET=ThisIsAnExampleKeyForDevPurposeOnly
|
||||
@@ -39,6 +39,7 @@ OIDC_RS_SCOPES="openid"
|
||||
OIDC_RS_CLIENT_ID=impress
|
||||
OIDC_RS_CLIENT_SECRET=ThisIsAnExampleKeyForDevPurposeOnly
|
||||
OIDC_RS_SIGN_ALGO=RS256
|
||||
OIDC_RS_VERIFY_CLAIMS=False
|
||||
|
||||
OIDC_RS_BACKEND_CLASS="core.authentication.FinderResourceServerBackend"
|
||||
OIDC_RS_ENCRYPTION_KEY_TYPE="RSA"
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import logging
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import AnonymousUser
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
|
||||
@@ -67,5 +68,33 @@ class FinderResourceServerBackend(ResourceServerBackend):
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
try:
|
||||
super().__init__()
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
raise
|
||||
|
||||
self.UserModel = ResourceUser
|
||||
|
||||
def _verify_claims(self, token):
|
||||
"""
|
||||
Verify the claims of the token to ensure authentication security.
|
||||
|
||||
By verifying these claims, we ensure that the token was issued by a
|
||||
trusted authorization server and is intended for this specific
|
||||
resource server. This prevents various types of attacks, such as
|
||||
token substitution or misuse of tokens issued for different clients.
|
||||
"""
|
||||
|
||||
# To run Find in development mode along other projects like docs/impress
|
||||
# we have to use OIDC endpoints on a common keycloak realm. e.g :
|
||||
# OIDC_OP_URL = http://nginx:8083/realms/impress
|
||||
#
|
||||
# This will cause a conflict with the 'iss' claim validation rule because the docs realm
|
||||
# gives {'iss': 'http://localhost:8083/realms/impress'} and it must be OIDC_OP_URL
|
||||
#
|
||||
# In order to make it work anyway, this flag allows to disable the claims validation.
|
||||
if settings.OIDC_RS_VERIFY_CLAIMS:
|
||||
return super()._verify_claims(token)
|
||||
|
||||
return token.claims
|
||||
|
||||
@@ -52,7 +52,7 @@ def test_api_documents_index_bulk_success():
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == 207
|
||||
assert response.status_code == 201
|
||||
responses = response.json()
|
||||
assert [d["status"] for d in responses] == ["success"] * 3
|
||||
|
||||
@@ -75,7 +75,7 @@ def test_api_documents_index_bulk_ensure_index():
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == 207
|
||||
assert response.status_code == 201
|
||||
responses = response.json()
|
||||
assert len(responses) == 3
|
||||
assert [d["status"] for d in responses] == ["success"] * 3
|
||||
@@ -284,7 +284,7 @@ def test_api_documents_index_bulk_default(field, default_value):
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == 207
|
||||
assert response.status_code == 201
|
||||
responses = response.json()
|
||||
assert [d["status"] for d in responses] == ["success"] * 3
|
||||
|
||||
@@ -409,7 +409,7 @@ def test_api_documents_index_opensearch_errors():
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == 207
|
||||
assert response.status_code == 201
|
||||
responses = response.json()
|
||||
assert responses == [
|
||||
{
|
||||
|
||||
@@ -67,6 +67,62 @@ def test_api_documents_search_query_unknown_user(settings):
|
||||
assert response.json() == {"detail": "Login failed"}
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_query_iss_not_valid(settings):
|
||||
"""Searching a document with a conflicting iss in token should result in a 401 error"""
|
||||
settings.OIDC_RS_VERIFY_CLAIMS = True
|
||||
|
||||
setup_oicd_resource_server(
|
||||
responses,
|
||||
settings,
|
||||
sub="user_sub",
|
||||
iss="http://other.com",
|
||||
)
|
||||
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
prepare_index(service.name, [])
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "a quick fox"},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 401
|
||||
assert response.json() == {"detail": "Login failed"}
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_query_iss_not_valid_ignored(settings):
|
||||
"""Searching a document with a conflicting iss in token should result in a 401 error"""
|
||||
settings.OIDC_RS_VERIFY_CLAIMS = False
|
||||
|
||||
setup_oicd_resource_server(
|
||||
responses,
|
||||
settings,
|
||||
sub="user_sub",
|
||||
iss="http://other.com",
|
||||
)
|
||||
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
prepare_index(service.name, [])
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "a quick fox"},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == []
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_services_invalid_parameters(settings):
|
||||
"""Invalid pagination parameters should result in a 400 error"""
|
||||
@@ -266,6 +322,87 @@ def test_api_documents_search_query_content(settings):
|
||||
assert other_fox_data["fields"] == {"number_of_users": [3], "number_of_groups": [3]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query,expected",
|
||||
[
|
||||
# Search for the start of the word "brown"
|
||||
(
|
||||
"bro",
|
||||
[
|
||||
("doc-1", "3.00"), # "bro" found in title & content
|
||||
("doc-3", "2.00"), # "bro" found in content
|
||||
],
|
||||
),
|
||||
# Search for the start of the word "wolf"
|
||||
(
|
||||
"wol",
|
||||
[
|
||||
("doc-2", "2.00"), # "wol" found in title
|
||||
("doc-3", "2.00"), # "wol" found in content
|
||||
],
|
||||
),
|
||||
# Search for the start of the word "wolf" and "brown"
|
||||
(
|
||||
"brown wol",
|
||||
[
|
||||
("doc-1", "2.85"), # "brown" found in title & content
|
||||
("doc-3", "2.45"), # "brown wol" found in content
|
||||
("doc-2", "2.00"), # "wol" found in title
|
||||
],
|
||||
),
|
||||
# Search for the start of the words "brown" & "wolf"
|
||||
(
|
||||
"bro wol",
|
||||
[
|
||||
("doc-2", "2.00"), # "wolf" found in title
|
||||
("doc-3", "2.00"), # "brown" found in content
|
||||
],
|
||||
),
|
||||
],
|
||||
)
|
||||
@responses.activate
|
||||
def test_api_documents_search_query_content_word_parts(settings, query, expected):
|
||||
"""Searching a document by its content should work as expected"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
document = factories.DocumentSchemaFactory.build(
|
||||
id="doc-1",
|
||||
title="the brown goat",
|
||||
content="The quick brown fox",
|
||||
reach=random.choice(["public", "authenticated"]),
|
||||
)
|
||||
other_fox_document = factories.DocumentSchemaFactory.build(
|
||||
id="doc-2",
|
||||
title="the wolf",
|
||||
content="The blue fox",
|
||||
reach=random.choice(["public", "authenticated"]),
|
||||
)
|
||||
no_fox_document = factories.DocumentSchemaFactory.build(
|
||||
id="doc-3",
|
||||
title="the fox",
|
||||
content="The brown wolf",
|
||||
reach=random.choice(["public", "authenticated"]),
|
||||
)
|
||||
|
||||
documents = [document, other_fox_document, no_fox_document]
|
||||
prepare_index(service.name, documents)
|
||||
|
||||
# Search for the start of the word "brown" in the body
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": query, "visited": [doc["id"] for doc in documents]},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
brown_results = [(d["_id"], f"{d['_score']:.2f}") for d in response.json()]
|
||||
assert brown_results == expected
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_ordering_by_fields(settings):
|
||||
"""It should be possible to order by several fields"""
|
||||
|
||||
@@ -170,13 +170,14 @@ def setup_oicd_jwt_resource_server(
|
||||
)
|
||||
|
||||
|
||||
def setup_oicd_resource_server(
|
||||
def setup_oicd_resource_server( # noqa : PLR0913
|
||||
responses,
|
||||
settings,
|
||||
sub="some_sub",
|
||||
audience="some_client_id",
|
||||
iss="https://oidc.example.com",
|
||||
introspect=None,
|
||||
): # pylint: disable=too-many-arguments
|
||||
): # pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
"""
|
||||
Setup settings for a resource server.
|
||||
Simulate a token introspection.
|
||||
@@ -184,7 +185,7 @@ def setup_oicd_resource_server(
|
||||
"""
|
||||
token_data = {
|
||||
"sub": sub,
|
||||
"iss": "https://oidc.example.com",
|
||||
"iss": iss,
|
||||
"aud": audience,
|
||||
"client_id": audience,
|
||||
"scope": "docs",
|
||||
|
||||
+27
-13
@@ -120,7 +120,7 @@ class IndexDocumentView(views.APIView):
|
||||
else:
|
||||
results[i]["status"] = "success"
|
||||
|
||||
return Response(results, status=status.HTTP_207_MULTI_STATUS)
|
||||
return Response(results, status=status.HTTP_201_CREATED)
|
||||
|
||||
# Indexing a single document
|
||||
document = schemas.DocumentSchema(**request.data)
|
||||
@@ -168,6 +168,29 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
|
||||
|
||||
return allowed_services
|
||||
|
||||
def _fulltext_search_query(self, text):
|
||||
if text == "*":
|
||||
return {"match_all": {}}
|
||||
|
||||
return {
|
||||
"bool": {
|
||||
"should": [
|
||||
{
|
||||
"multi_match": {
|
||||
"query": text,
|
||||
# Give title more importance over content by a power of 3
|
||||
"fields": ["title.text^3", "content"],
|
||||
# Add extra score when multiple keywords are in the field
|
||||
"tie_breaker": 0.3,
|
||||
},
|
||||
},
|
||||
{"match_bool_prefix": {"title.text": text}},
|
||||
{"match_bool_prefix": {"content": text}},
|
||||
],
|
||||
"minimum_should_match": 1,
|
||||
}
|
||||
}
|
||||
|
||||
def post(self, request, *args, **kwargs):
|
||||
"""
|
||||
Handle POST requests to perform a search on indexed documents with optional filtering
|
||||
@@ -245,18 +268,9 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
|
||||
}
|
||||
|
||||
# Adding the text query
|
||||
if params.q == "*":
|
||||
search_body["query"]["bool"]["must"].append({"match_all": {}})
|
||||
else:
|
||||
search_body["query"]["bool"]["must"].append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": params.q,
|
||||
# Give title more importance over content by a power of 3
|
||||
"fields": ["title.text^3", "content"],
|
||||
}
|
||||
}
|
||||
)
|
||||
search_body["query"]["bool"]["must"].append(
|
||||
self._fulltext_search_query(params.q)
|
||||
)
|
||||
|
||||
# Add sorting logic based on relevance or specified field
|
||||
if params.order_by == enums.RELEVANCE:
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
"""Parameters that define how the demo site will be built."""
|
||||
|
||||
NB_OBJECTS = {
|
||||
"documents": 1000,
|
||||
"services": 5,
|
||||
}
|
||||
NB_OBJECTS = {"documents": 1000, "services": 5}
|
||||
|
||||
DEV_SERVICES = (
|
||||
{
|
||||
"name": "docs",
|
||||
"client_id": "impress",
|
||||
"token": "find-api-key-for-docs-with-exactly-50-chars-length",
|
||||
},
|
||||
)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# ruff: noqa: S311, S106
|
||||
# ruff: noqa: S311
|
||||
"""create_demo management command"""
|
||||
|
||||
import logging
|
||||
@@ -147,6 +147,7 @@ def create_demo(stdout):
|
||||
services = factories.ServiceFactory.create_batch(
|
||||
defaults.NB_OBJECTS["services"]
|
||||
)
|
||||
|
||||
for service in services:
|
||||
opensearch.ensure_index_exists(service.name)
|
||||
opensearch.client.indices.refresh(index=service.name)
|
||||
@@ -159,6 +160,12 @@ def create_demo(stdout):
|
||||
actions.push(service.name, uuid4(), document)
|
||||
actions.flush()
|
||||
|
||||
with Timeit(stdout, "Creating dev services"):
|
||||
for conf in defaults.DEV_SERVICES:
|
||||
service = factories.ServiceFactory(**conf)
|
||||
opensearch.ensure_index_exists(service.name)
|
||||
opensearch.client.indices.refresh(index=service.name)
|
||||
|
||||
# Check and report on indexed documents
|
||||
total_indexed = 0
|
||||
for service in services:
|
||||
|
||||
@@ -25,5 +25,8 @@ def test_commands_create_demo():
|
||||
"""The create_demo management command should create objects as expected."""
|
||||
call_command("create_demo")
|
||||
|
||||
assert models.Service.objects.count() == 2
|
||||
assert models.Service.objects.exclude(name="docs").count() == 2
|
||||
assert opensearch.client.count()["count"] == 4
|
||||
|
||||
docs = models.Service.objects.get(name="docs")
|
||||
assert docs.client_id == "impress"
|
||||
|
||||
@@ -373,6 +373,10 @@ class Base(Configuration):
|
||||
environ_name="OIDC_RS_ENCRYPTION_ENCODING",
|
||||
environ_prefix=None,
|
||||
)
|
||||
# In production mode we must check the token claims
|
||||
OIDC_RS_VERIFY_CLAIMS = values.BooleanValue(
|
||||
default=True, environ_name="OIDC_RS_VERIFY_CLAIMS", environ_prefix=None
|
||||
)
|
||||
|
||||
OIDC_VERIFY_SSL = values.BooleanValue(
|
||||
True, environ_name="OIDC_VERIFY_SSL", environ_prefix=None
|
||||
@@ -404,7 +408,6 @@ class Base(Configuration):
|
||||
ALLOW_LOGOUT_GET_METHOD = values.BooleanValue(
|
||||
default=True, environ_name="ALLOW_LOGOUT_GET_METHOD", environ_prefix=None
|
||||
)
|
||||
|
||||
# Logging
|
||||
# We want to make it easy to log to console but by default we log production
|
||||
# to Sentry and don't want to log to console.
|
||||
@@ -595,6 +598,9 @@ class Production(Base):
|
||||
},
|
||||
}
|
||||
|
||||
# In production mode we must check the token claims
|
||||
OIDC_RS_VERIFY_CLAIMS = True
|
||||
|
||||
|
||||
class Feature(Production):
|
||||
"""
|
||||
|
||||
+21
-21
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "find"
|
||||
version = "1.1.0"
|
||||
version = "0.0.1"
|
||||
authors = [{ "name" = "DINUM", "email" = "dev@mail.numerique.gouv.fr" }]
|
||||
classifiers = [
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
@@ -25,24 +25,24 @@ license = { file = "LICENSE" }
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"celery[redis]==5.4.0",
|
||||
"celery[redis]==5.5.3",
|
||||
"django-configurations==2.5.1",
|
||||
"django-cors-headers==4.6.0",
|
||||
"django-cors-headers==4.7.0",
|
||||
"redis==5.2.1",
|
||||
"django-redis==5.4.0",
|
||||
"django==5.1.4",
|
||||
"django-lasuite[all]==0.0.11",
|
||||
"djangorestframework==3.15.2",
|
||||
"django-redis==6.0.0",
|
||||
"django==5.2.6",
|
||||
"django-lasuite[all]==0.0.14",
|
||||
"djangorestframework==3.16.0",
|
||||
"drf_spectacular==0.28.0",
|
||||
"dockerflow==2024.4.2",
|
||||
"factory_boy==3.3.1",
|
||||
"gunicorn==23.0.0",
|
||||
"mozilla-django-oidc==4.0.1",
|
||||
"psycopg[binary]==3.2.3",
|
||||
"psycopg[binary]==3.2.9",
|
||||
"pydantic==2.10.5",
|
||||
"pyjwt==2.10.1",
|
||||
"requests==2.32.3",
|
||||
"sentry-sdk==2.19.2",
|
||||
"requests==2.32.4",
|
||||
"sentry-sdk==2.32.0",
|
||||
"url-normalize==1.4.3",
|
||||
"opensearch-py==2.8.0",
|
||||
"whitenoise==6.8.2",
|
||||
@@ -56,22 +56,22 @@ dependencies = [
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"django-extensions==3.2.3",
|
||||
"drf-spectacular-sidecar==2024.12.1",
|
||||
"django-extensions==4.1",
|
||||
"drf-spectacular-sidecar==2025.7.1",
|
||||
"faker==33.3.0",
|
||||
"ipdb==0.13.13",
|
||||
"ipython==8.31.0",
|
||||
"pyfakefs==5.7.3",
|
||||
"pyfakefs==5.9.1",
|
||||
"pylint-django==2.6.1",
|
||||
"pylint==3.3.3",
|
||||
"pytest-cov==6.0.0",
|
||||
"pytest-django==4.9.0",
|
||||
"pytest==8.3.4",
|
||||
"pylint==3.3.7",
|
||||
"pytest-cov==6.2.1",
|
||||
"pytest-django==4.11.1",
|
||||
"pytest==8.4.1",
|
||||
"pytest-icdiff==0.9",
|
||||
"pytest-xdist==3.6.1",
|
||||
"responses==0.25.3",
|
||||
"ruff==0.9.0",
|
||||
"types-requests==2.32.0.20241016",
|
||||
"pytest-xdist==3.8.0",
|
||||
"responses==0.25.7",
|
||||
"ruff==0.12.2",
|
||||
"types-requests==2.32.4.20250611",
|
||||
]
|
||||
|
||||
[tool.setuptools]
|
||||
|
||||
Reference in New Issue
Block a user