Compare commits
19 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 09c687878d | |||
| 43265acf36 | |||
| 79208765c9 | |||
| 15f58e68c8 | |||
| 0dd513a4d3 | |||
| 7c2a60eb6f | |||
| 23c4412114 | |||
| fec8c375a2 | |||
| 3cb3bf3d8f | |||
| b8b335d724 | |||
| 89923e694b | |||
| 6b80aa280a | |||
| c8d1af667c | |||
| 5242417738 | |||
| 7eaa284357 | |||
| d48c837c3e | |||
| 51dee6475b | |||
| d8050bf63d | |||
| e31cf57bcd |
@@ -12,3 +12,10 @@ and this project adheres to
|
||||
|
||||
- backend application
|
||||
- helm chart
|
||||
- 🐛(backend) fix missing index creation in 'index/' view
|
||||
- ✨(backend) allow indexation of documents with either empty content or title.
|
||||
- ✨(api) new fulltext 'search/' view with OIDC resource server authentication
|
||||
- ✨(backend) limit access to documents : public & authenticated with a
|
||||
linkreach & owned ones
|
||||
- ✨(backend) limit search to the calling app (audience) and a configured
|
||||
list of services
|
||||
|
||||
+30
-7
@@ -1,10 +1,12 @@
|
||||
name: find
|
||||
|
||||
services:
|
||||
postgresql:
|
||||
image: postgres:15
|
||||
env_file:
|
||||
- env.d/development/postgresql
|
||||
ports:
|
||||
- "15432:5432"
|
||||
- "25432:5432"
|
||||
|
||||
opensearch:
|
||||
image: opensearchproject/opensearch:latest
|
||||
@@ -22,6 +24,12 @@ services:
|
||||
ports:
|
||||
- "9200:9200"
|
||||
- "9600:9600"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--head", "-fsS", "http://localhost:9200/"]
|
||||
start_period: 5s
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 10
|
||||
|
||||
opensearch-dashboards:
|
||||
image: opensearchproject/opensearch-dashboards:latest
|
||||
@@ -31,7 +39,9 @@ services:
|
||||
OPENSEARCH_HOSTS: '["http://opensearch:9200"]'
|
||||
DISABLE_SECURITY_DASHBOARDS_PLUGIN: "true"
|
||||
depends_on:
|
||||
- opensearch
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
restart: true
|
||||
|
||||
redis:
|
||||
image: redis:5
|
||||
@@ -51,15 +61,23 @@ services:
|
||||
- env.d/development/common
|
||||
- env.d/development/postgresql
|
||||
ports:
|
||||
- "8071:8000"
|
||||
- "8081:8000"
|
||||
networks:
|
||||
default: {}
|
||||
lasuite-net:
|
||||
aliases:
|
||||
- find
|
||||
volumes:
|
||||
- ./src/backend:/app
|
||||
- ./data/static:/data/static
|
||||
depends_on:
|
||||
- postgresql
|
||||
- opensearch
|
||||
- redis
|
||||
|
||||
postgresql:
|
||||
condition: service_started
|
||||
redis:
|
||||
condition: service_started
|
||||
opensearch:
|
||||
condition: service_healthy
|
||||
|
||||
celery:
|
||||
user: ${DOCKER_USER:-1000}
|
||||
image: find:backend-development
|
||||
@@ -94,3 +112,8 @@ services:
|
||||
HOME: /tmp
|
||||
volumes:
|
||||
- ".:/app"
|
||||
|
||||
networks:
|
||||
lasuite-net:
|
||||
name: lasuite-net
|
||||
driver: bridge
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
## Architecture
|
||||
|
||||
### Global system architecture
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Docs -- REST API --> Back("Backend (Django)")
|
||||
Back --> DB("Database (PostgreSQL)")
|
||||
Back -- REST API --> Opensearch
|
||||
Back <--> Celery --> DB
|
||||
User -- HTTP --> Dashboard --> Opensearch
|
||||
```
|
||||
+106
@@ -0,0 +1,106 @@
|
||||
# Find variables
|
||||
|
||||
Here we describe all environment variables that can be set for the find application.
|
||||
|
||||
## find-backend container
|
||||
|
||||
These are the environment variables you can set for the `find-backend` container.
|
||||
|
||||
| Option | Description | default |
|
||||
|-------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------|
|
||||
| API_USERS_LIST_LIMIT | Limit on API users | 5 |
|
||||
| API_USERS_LIST_THROTTLE_RATE_BURST | Throttle rate for api on burst | 30/minute |
|
||||
| API_USERS_LIST_THROTTLE_RATE_SUSTAINED | Throttle rate for api | 180/hour |
|
||||
| CACHES_DEFAULT_TIMEOUT | Cache default timeout | 30 |
|
||||
| CACHES_KEY_PREFIX | The prefix used to every cache keys. | docs |
|
||||
| DB_ENGINE | Engine to use for database connections | django.db.backends.postgresql_psycopg2 |
|
||||
| DB_HOST | Host of the database | localhost |
|
||||
| DB_NAME | Name of the database | impress |
|
||||
| DB_PASSWORD | Password to authenticate with | pass |
|
||||
| DB_PORT | Port of the database | 5432 |
|
||||
| DB_USER | User to authenticate with | dinum |
|
||||
| DJANGO_ALLOWED_HOSTS | Allowed hosts | [] |
|
||||
| DJANGO_CELERY_BROKER_TRANSPORT_OPTIONS | Celery broker transport options | {} |
|
||||
| DJANGO_CELERY_BROKER_URL | Celery broker url | redis://redis:6379/0 |
|
||||
| DJANGO_CORS_ALLOW_ALL_ORIGINS | Allow all CORS origins | false |
|
||||
| DJANGO_CORS_ALLOWED_ORIGIN_REGEXES | List of origins allowed for CORS using regulair expressions | [] |
|
||||
| DJANGO_CORS_ALLOWED_ORIGINS | List of origins allowed for CORS | [] |
|
||||
| DJANGO_CSRF_TRUSTED_ORIGINS | CSRF trusted origins | [] |
|
||||
| DJANGO_EMAIL_BACKEND | Email backend library | django.core.mail.backends.smtp.EmailBackend |
|
||||
| DJANGO_EMAIL_BRAND_NAME | Brand name for email | |
|
||||
| DJANGO_EMAIL_FROM | Email address used as sender | from@example.com |
|
||||
| DJANGO_EMAIL_HOST | Hostname of email | |
|
||||
| DJANGO_EMAIL_HOST_PASSWORD | Password to authenticate with on the email host | |
|
||||
| DJANGO_EMAIL_HOST_USER | User to authenticate with on the email host | |
|
||||
| DJANGO_EMAIL_LOGO_IMG | Logo for the email | |
|
||||
| DJANGO_EMAIL_PORT | Port used to connect to email host | |
|
||||
| DJANGO_EMAIL_USE_SSL | Use ssl for email host connection | false |
|
||||
| DJANGO_EMAIL_USE_TLS | Use tls for email host connection | false |
|
||||
| DJANGO_SECRET_KEY | Secret key | |
|
||||
| DJANGO_SERVER_TO_SERVER_API_TOKENS | | [] |
|
||||
| DOCUMENT_IMAGE_MAX_SIZE | Maximum size of document in bytes | 10485760 |
|
||||
| FRONTEND_CSS_URL | To add a external css file to the app | |
|
||||
| FRONTEND_HOMEPAGE_FEATURE_ENABLED | Frontend feature flag to display the homepage | false |
|
||||
| FRONTEND_THEME | Frontend theme to use | |
|
||||
| LANGUAGE_CODE | Default language | en-us |
|
||||
| LOGGING_LEVEL_LOGGERS_APP | Application logging level. options are "DEBUG", "INFO", "WARN", "ERROR", "CRITICAL" | INFO |
|
||||
| LOGGING_LEVEL_LOGGERS_ROOT | Default logging level. options are "DEBUG", "INFO", "WARN", "ERROR", "CRITICAL" | INFO |
|
||||
| LOGIN_REDIRECT_URL | Login redirect url | |
|
||||
| LOGIN_REDIRECT_URL_FAILURE | Login redirect url on failure | |
|
||||
| LOGOUT_REDIRECT_URL | Logout redirect url | |
|
||||
| MALWARE_DETECTION_BACKEND | The malware detection backend use from the django-lasuite package | lasuite.malware_detection.backends.dummy.DummyBackend |
|
||||
| MALWARE_DETECTION_PARAMETERS | A dict containing all the parameters to initiate the malware detection backend | {"callback_path": "core.malware_detection.malware_detection_callback",} |
|
||||
| MEDIA_BASE_URL | | |
|
||||
| NO_WEBSOCKET_CACHE_TIMEOUT | Cache used to store current editor session key when only users without websocket are editing a document | 120 |
|
||||
| OIDC_ALLOW_DUPLICATE_EMAILS | Allow duplicate emails | false |
|
||||
| OIDC_AUTH_REQUEST_EXTRA_PARAMS | OIDC extra auth parameters | {} |
|
||||
| OIDC_CREATE_USER | Create used on OIDC | false |
|
||||
| OIDC_DRF_AUTH_BACKEND | DRF Authentication backend class for OIDC | lasuite.oidc_login.backends.OIDCAuthenticationBackend |
|
||||
| OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION | Fallback to email for identification | true |
|
||||
| OIDC_OP_AUTHORIZATION_ENDPOINT | Authorization endpoint for OIDC | |
|
||||
| OIDC_OP_INTROSPECTION_ENDPOINT | Introspection endpoint for OIDC | |
|
||||
| OIDC_OP_JWKS_ENDPOINT | JWKS endpoint for OIDC | |
|
||||
| OIDC_OP_LOGOUT_ENDPOINT | Logout endpoint for OIDC | |
|
||||
| OIDC_OP_TOKEN_ENDPOINT | Token endpoint for OIDC | |
|
||||
| OIDC_OP_USER_ENDPOINT | User endpoint for OIDC | |
|
||||
| OIDC_REDIRECT_ALLOWED_HOSTS | Allowed hosts for OIDC redirect url | [] |
|
||||
| OIDC_REDIRECT_REQUIRE_HTTPS | Require https for OIDC redirect url | false |
|
||||
| OIDC_RP_CLIENT_ID | Client id used for OIDC | impress |
|
||||
| OIDC_RP_CLIENT_SECRET | Client secret used for OIDC | |
|
||||
| OIDC_RP_SCOPES | Scopes requested for OIDC | openid email |
|
||||
| OIDC_RP_SIGN_ALGO | verification algorithm used OIDC tokens | RS256 |
|
||||
| OIDC_RS_BACKEND_CLASS | Resource server backend class for OIDC | core.authentication.FinderResourceServerBackend |
|
||||
| OIDC_RS_AUDIENCE_CLAIM | The claim used to identify the audience | client_id |
|
||||
| OIDC_RS_CLIENT_ID | Client id used for OIDC resource server | |
|
||||
| OIDC_RS_CLIENT_SECRET | Client secret used for OIDC resource server | |
|
||||
| OIDC_RS_SIGNING_ALGO | Signing algorithm | ES256 |
|
||||
| OIDC_RS_SCOPES | Required scopes for authentication | ["openid"] |
|
||||
| OIDC_RS_PRIVATE_KEY_STR | Private key for encryption/decryption | |
|
||||
| OIDC_RS_ENCRYPTION_KEY_TYPE | Encryption key type (RSA, EC, etc.) | RSA |
|
||||
| OIDC_RS_ENCRYPTION_ALGO | Encryption algorithm | RSA-OAEP |
|
||||
| OIDC_RS_ENCRYPTION_ENCODING | Encryption encoding algorithm | A256GCM |
|
||||
| OIDC_RS_VERIFY_CLAIMS | Enable OICD token claims validation | true |
|
||||
| OIDC_STORE_ID_TOKEN | Store OIDC token | true |
|
||||
| OIDC_USE_NONCE | Use nonce for OIDC | true |
|
||||
| OIDC_USERINFO_FULLNAME_FIELDS | OIDC token claims to create full name | ["first_name", "last_name"] |
|
||||
| OIDC_USERINFO_SHORTNAME_FIELD | OIDC token claims to create shortname | first_name |
|
||||
| OIDC_VERIFY_SSL | Enable SSL validation for OIDC | true |
|
||||
| OIDC_TIMEOUT | Timeout delay for OID requests | |
|
||||
| OIDC_PROXY | Defines a proxy for all requests to the OpenID Connect provider | |
|
||||
| OPENSEARCH_HOST | Opensearch database url | default |
|
||||
| OPENSEARCH_PORT | Port of Opensearch database | 9200 |
|
||||
| OPENSEARCH_USER | Opensearch database user | admin |
|
||||
| OPENSEARCH_PASSWORD | Opensearch database user password | |
|
||||
| OPENSEARCH_USE_SSL | Enable SSL connection for Opensearch database | true |
|
||||
| POSTHOG_KEY | Posthog key for analytics | |
|
||||
| REDIS_URL | Cache url | redis://redis:6379/1 |
|
||||
| SENTRY_DSN | Sentry host | |
|
||||
| SESSION_COOKIE_AGE | duration of the cookie session | 60*60*12 |
|
||||
| SPECTACULAR_SETTINGS_ENABLE_DJANGO_DEPLOY_CHECK | | false |
|
||||
| STORAGES_STATICFILES_BACKEND | | whitenoise.storage.CompressedManifestStaticFilesStorage |
|
||||
| THEME_CUSTOMIZATION_CACHE_TIMEOUT | Cache duration for the customization settings | 86400 |
|
||||
| THEME_CUSTOMIZATION_FILE_PATH | Full path to the file customizing the theme. An example is provided in src/backend/impress/configuration/theme/default.json | BASE_DIR/impress/configuration/theme/default.json |
|
||||
| TRASHBIN_CUTOFF_DAYS | Trashbin cutoff | 30 |
|
||||
| USER_OIDC_ESSENTIAL_CLAIMS | Essential claims in OIDC token | [] |
|
||||
| Y_PROVIDER_API_BASE_URL | Y Provider url | |
|
||||
| Y_PROVIDER_API_KEY | Y provider API key | |
|
||||
@@ -0,0 +1,58 @@
|
||||
# Releasing a new version
|
||||
|
||||
Whenever we are cooking a new release (e.g. `0.0.1`) we should follow a standard procedure described below:
|
||||
|
||||
1. Create a new branch named: `release/0.0.1`.
|
||||
|
||||
2. Bump the release number for backend project and Helm files:
|
||||
|
||||
- for backend, update the version number by hand in `pyproject.toml`,
|
||||
- for Helm, update Docker image tag in files located at `src/helm/env.d` for both `preprod` and `production` environments:
|
||||
|
||||
```yaml
|
||||
image:
|
||||
repository: lasuite/find
|
||||
pullPolicy: Always
|
||||
tag: "v0.0.1" # Replace with your new version number, without forgetting the "v" prefix
|
||||
...
|
||||
|
||||
The new images don't exist _yet_: they will be created automatically later in the process.
|
||||
|
||||
3. Update the project's `Changelog` following the [keepachangelog](https://keepachangelog.com/en/0.3.0/) recommendations
|
||||
|
||||
4. Commit your changes with the following format: the 🔖 release emoji, the type of release (patch/minor/patch) and the release version:
|
||||
|
||||
```text
|
||||
🔖(minor) bump release to 0.0.1
|
||||
```
|
||||
|
||||
5. Open a pull request, wait for an approval from your peers and merge it.
|
||||
6. Checkout and pull changes from the `main` branch to ensure you have the latest updates.
|
||||
7. Tag and push your commit:
|
||||
|
||||
```bash
|
||||
git tag v0.0.1 && git push origin tag v0.0.1
|
||||
```
|
||||
|
||||
Doing this triggers the CI and tells it to build the new Docker image versions that you targeted earlier in the Helm files.
|
||||
|
||||
8. Ensure the new [backend](https://hub.docker.com/r/lasuite/find/tags) image tags are on Docker Hub.
|
||||
9. The release is now done!
|
||||
|
||||
# Deploying
|
||||
|
||||
> [!TIP]
|
||||
> The `staging` platform is deployed automatically with every update of the `main` branch.
|
||||
|
||||
Making a new release doesn't publish it automatically in production.
|
||||
|
||||
Deployment is done by ArgoCD. ArgoCD checks for the `production` tag and automatically deploys the production platform with the targeted commit.
|
||||
|
||||
To publish, we mark the commit we want with the `production` tag. ArgoCD is then notified that the tag has changed. It then deploys the Docker image tags specified in the Helm files of the targeted commit.
|
||||
|
||||
To publish the release you just made:
|
||||
|
||||
```bash
|
||||
git tag --force production v0.0.1
|
||||
git push --force origin production
|
||||
```
|
||||
@@ -0,0 +1,40 @@
|
||||
# Setup the Find search for Impress
|
||||
|
||||
This configuration will enable the fulltext search feature for Docs :
|
||||
- Each save on **core.Document** or **core.DocumentAccess** will trigger the indexer
|
||||
- The `api/v1.0/documents/search/` will work as a proxy with the Find API for fulltext search.
|
||||
|
||||
## Create an index service for Docs
|
||||
|
||||
Configure a **Service** for Docs application with these settings
|
||||
|
||||
- **Name**: `docs`<br>_request.auth.name of the Docs application._
|
||||
- **Client id**: `impress`<br>_Name of the token audience or client_id of the Docs application._
|
||||
|
||||
See [how-to-use-indexer.md](how-to-use-indexer.md) for details.
|
||||
|
||||
## Configure settings of Docs
|
||||
|
||||
Add those Django settings the Docs application to enable the feature.
|
||||
|
||||
```shell
|
||||
SEARCH_INDEXER_CLASS="core.services.search_indexers.FindDocumentIndexer"
|
||||
SEARCH_INDEXER_COUNTDOWN=10 # Debounce delay in seconds for the indexer calls.
|
||||
|
||||
# Indexation endpoint.
|
||||
SEARCH_INDEXER_SECRET=my-token-from-the-find-impress-service
|
||||
# The token from service "docs" of Find application.
|
||||
SEARCH_INDEXER_URL="http://app-find:8000/api/v1.0/documents/index/"
|
||||
|
||||
# Search endpoint. Uses the OIDC token for authentication
|
||||
SEARCH_INDEXER_QUERY_URL="http://app-find:8000/api/v1.0/documents/search/"
|
||||
```
|
||||
|
||||
We also need to enable the **OIDC Token** refresh or the authentication will fail quickly.
|
||||
|
||||
```shell
|
||||
# Store OIDC tokens in the session
|
||||
OIDC_STORE_ACCESS_TOKEN = True # Store the access token in the session
|
||||
OIDC_STORE_REFRESH_TOKEN = True # Store the encrypted refresh token in the session
|
||||
OIDC_STORE_REFRESH_TOKEN_KEY = "your-32-byte-encryption-key==" # Must be a valid Fernet key (32 url-safe base64-encoded bytes)
|
||||
```
|
||||
@@ -0,0 +1,83 @@
|
||||
# Using the Find indexer
|
||||
|
||||
This guide explains how to setup the Find service which provide an API for indexation and fulltext search of
|
||||
documents from various sources in a secure way : only the documents within the scope of the user's OIDC token
|
||||
are visible.
|
||||
|
||||
## Setup Opensearch
|
||||
|
||||
Add the following settings to your Django settings for the Find application.
|
||||
|
||||
```shell
|
||||
# Login for opensearch
|
||||
OPENSEARCH_USER=opensearch-user
|
||||
OPENSEARCH_PASSWORD=your-opensearch-password
|
||||
|
||||
# Host configuration
|
||||
OPENSEARCH_HOST=opensearch
|
||||
OPENSEARCH_PORT=9200
|
||||
|
||||
# Enable SSL for opensearch connection (False in dev mode)
|
||||
OPENSEARCH_USE_SSL=True
|
||||
```
|
||||
|
||||
## Setup indexation API
|
||||
|
||||
Other applications can index their files through the **`/index/`** endpoint with a simple token authentication.
|
||||
|
||||
For each application a new **Service** must be created through the admin interface
|
||||
(see http://localhost:9071/admin/core/service/add/)
|
||||
|
||||
| Field | Description |
|
||||
|-----------------------------|----------------------------------------------------|
|
||||
| Name | Name of the service and also the name of the index in Opensearch database |
|
||||
| Is active | Toggle service availability |
|
||||
| Client id | Calling service client_id (e.g `impress` for docs) |
|
||||
| Allowed services for search | List of sub-services. Will add the results from all these index<br>to the search results. |
|
||||
| Token (_read-only_) | Random token for calling service authentication |
|
||||
|
||||
And add the key in the calling application Django settings.
|
||||
|
||||
**Development Mode (Docs + Find)**
|
||||
|
||||
The command `make demo` will create a working service configuration for `docs` with the following secret key
|
||||
|
||||
```python
|
||||
SEARCH_INDEXER_SECRET="find-api-key-for-docs-with-exactly-50-chars-length"
|
||||
```
|
||||
|
||||
## Setup search API
|
||||
|
||||
The **`/search/`** endpoint is an OIDC ResourceServer view and needs extra Django settings (see [lasuite](https://github.com/suitenumerique/django-lasuite/blob/main/documentation/how-to-use-oidc-resource-server-backend.md) for details)
|
||||
|
||||
```shell
|
||||
OIDC_OP_JWKS_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/certs
|
||||
OIDC_OP_AUTHORIZATION_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/auth
|
||||
OIDC_OP_TOKEN_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/token
|
||||
OIDC_OP_USER_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/userinfo
|
||||
OIDC_OP_URL=http://nginx:8083/realms/impress
|
||||
|
||||
# Introspection endpoint is needed to get the "audience" and "sub" from the user token
|
||||
OIDC_OP_INTROSPECTION_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/token/introspect
|
||||
|
||||
# In development, the resource server use insecure settings
|
||||
# OIDC_VERIFY_SSL=False
|
||||
# OIDC_RS_VERIFY_CLAIMS=False
|
||||
|
||||
# Resource server
|
||||
OIDC_RS_SCOPES="openid"
|
||||
OIDC_RS_SIGN_ALGO=RS256
|
||||
|
||||
# This backend allows authentication without any model in database.
|
||||
OIDC_RS_BACKEND_CLASS="core.authentication.FinderResourceServerBackend"
|
||||
```
|
||||
|
||||
**Development mode (Docs + Find)**
|
||||
|
||||
Docs and Find projects stacks can be run together but must have the same keycloak server.
|
||||
|
||||
So the endpoints of Docs on the 'nginx' domain for ResourceServer authentication and introspection are also used for Find.
|
||||
This configuration breaks the OIDC token claims validation : the `iss` claim from the token of the 'impress' users are 'localhost' and not 'nginx'.
|
||||
|
||||
The setting `OIDC_RS_VERIFY_CLAIMS=False` allows to ignore this step and make things works.
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
# La Suite Find – System & Requirements
|
||||
|
||||
## Ports (dev defaults)
|
||||
|
||||
| Port | Service |
|
||||
| --------- | --------------------- |
|
||||
| 8081 | Django (main) |
|
||||
| 8083 | Nginx proxy |
|
||||
| 25432 | PostgreSQL (main) |
|
||||
| 9200 | Opensearch |
|
||||
| 9600 | Opensearch admin |
|
||||
| 5601 | Opensearch dashboard |
|
||||
@@ -17,10 +17,12 @@ OPENSEARCH_PASSWORD=find.PASS123
|
||||
OPENSEARCH_USE_SSL=false
|
||||
|
||||
# OIDC
|
||||
OIDC_OP_JWKS_ENDPOINT=http://nginx:8083/realms/find/protocol/openid-connect/certs
|
||||
OIDC_OP_AUTHORIZATION_ENDPOINT=http://localhost:8083/realms/find/protocol/openid-connect/auth
|
||||
OIDC_OP_TOKEN_ENDPOINT=http://nginx:8083/realms/find/protocol/openid-connect/token
|
||||
OIDC_OP_USER_ENDPOINT=http://nginx:8083/realms/find/protocol/openid-connect/userinfo
|
||||
OIDC_OP_JWKS_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/certs
|
||||
OIDC_OP_AUTHORIZATION_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/auth
|
||||
OIDC_OP_TOKEN_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/token
|
||||
OIDC_OP_USER_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/userinfo
|
||||
OIDC_OP_URL=http://nginx:8083/realms/impress
|
||||
OIDC_OP_INTROSPECTION_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/token/introspect
|
||||
|
||||
OIDC_RP_CLIENT_ID=find
|
||||
OIDC_RP_CLIENT_SECRET=ThisIsAnExampleKeyForDevPurposeOnly
|
||||
@@ -29,3 +31,15 @@ OIDC_RP_SCOPES="openid email"
|
||||
|
||||
OIDC_REDIRECT_ALLOWED_HOSTS=["http://localhost:8083", "http://localhost:3000"]
|
||||
OIDC_AUTH_REQUEST_EXTRA_PARAMS={"acr_values": "eidas1"}
|
||||
|
||||
# OIDC Resource server
|
||||
OIDC_DRF_AUTH_BACKEND="lasuite.oidc_login.backends.OIDCAuthenticationBackend"
|
||||
|
||||
OIDC_RS_SCOPES="openid"
|
||||
OIDC_RS_CLIENT_ID=impress
|
||||
OIDC_RS_CLIENT_SECRET=ThisIsAnExampleKeyForDevPurposeOnly
|
||||
OIDC_RS_SIGN_ALGO=RS256
|
||||
OIDC_RS_VERIFY_CLAIMS=False
|
||||
|
||||
OIDC_RS_BACKEND_CLASS="core.authentication.FinderResourceServerBackend"
|
||||
OIDC_RS_ENCRYPTION_KEY_TYPE="RSA"
|
||||
|
||||
@@ -1,11 +1,18 @@
|
||||
"""Token authentication."""
|
||||
|
||||
from django.contrib.auth.models import AnonymousUser
|
||||
import logging
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import AnonymousUser
|
||||
from django.core.exceptions import ObjectDoesNotExist
|
||||
|
||||
from lasuite.oidc_resource_server.backend import ResourceServerBackend
|
||||
from rest_framework import authentication, exceptions
|
||||
|
||||
from .models import Service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ServiceTokenAuthentication(authentication.BaseAuthentication):
|
||||
"""A custom authentication looking for valid tokens among registered services"""
|
||||
@@ -25,11 +32,69 @@ class ServiceTokenAuthentication(authentication.BaseAuthentication):
|
||||
def authenticate_credentials(self, token):
|
||||
"""Check that the token is registered and valid."""
|
||||
try:
|
||||
service_name = (
|
||||
self.model.objects.only("name").get(token=token, is_active=True).name
|
||||
)
|
||||
service = self.model.objects.only("name").get(token=token, is_active=True)
|
||||
except self.model.DoesNotExist as excpt:
|
||||
raise exceptions.AuthenticationFailed("Invalid token.") from excpt
|
||||
|
||||
# We don't associate tokens with a user
|
||||
return AnonymousUser(), service_name
|
||||
return AnonymousUser(), service
|
||||
|
||||
|
||||
class ResourceUserManager:
|
||||
"""Fake manager that simply returns an instance of user object with the right sub"""
|
||||
|
||||
def get(self, sub):
|
||||
"""Returns a ResourceUser for this sub"""
|
||||
return ResourceUser(sub=sub)
|
||||
|
||||
|
||||
class ResourceUser:
|
||||
"""Fake user model for the ResourceServerBackend.get_object() method"""
|
||||
|
||||
DoesNotExist = ObjectDoesNotExist
|
||||
|
||||
objects = ResourceUserManager()
|
||||
|
||||
def __init__(self, sub: str):
|
||||
self.sub = sub
|
||||
self.pk = None
|
||||
self.is_authenticated = True
|
||||
|
||||
|
||||
class FinderResourceServerBackend(ResourceServerBackend):
|
||||
"""
|
||||
Custom resource server that uses a ResourceUser object instead of
|
||||
a user model from the database
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
try:
|
||||
super().__init__()
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
raise
|
||||
|
||||
self.UserModel = ResourceUser
|
||||
|
||||
def _verify_claims(self, token):
|
||||
"""
|
||||
Verify the claims of the token to ensure authentication security.
|
||||
|
||||
By verifying these claims, we ensure that the token was issued by a
|
||||
trusted authorization server and is intended for this specific
|
||||
resource server. This prevents various types of attacks, such as
|
||||
token substitution or misuse of tokens issued for different clients.
|
||||
"""
|
||||
|
||||
# To run Find in development mode along other projects like docs/impress
|
||||
# we have to use OIDC endpoints on a common keycloak realm. e.g :
|
||||
# OIDC_OP_URL = http://nginx:8083/realms/impress
|
||||
#
|
||||
# This will cause a conflict with the 'iss' claim validation rule because the docs realm
|
||||
# gives {'iss': 'http://localhost:8083/realms/impress'} and it must be OIDC_OP_URL
|
||||
#
|
||||
# In order to make it work anyway, this flag allows to disable the claims validation.
|
||||
if settings.OIDC_RS_VERIFY_CLAIMS:
|
||||
return super()._verify_claims(token)
|
||||
|
||||
return token.claims
|
||||
|
||||
@@ -1,12 +1,32 @@
|
||||
"""Enums for find's core app."""
|
||||
|
||||
RELEVANCE = "relevance"
|
||||
from enum import Enum
|
||||
|
||||
# Reach
|
||||
|
||||
|
||||
class ReachEnum(str, Enum):
|
||||
"""Publication options for indexed documents"""
|
||||
|
||||
PUBLIC = "public"
|
||||
AUTHENTICATED = "authenticated"
|
||||
RESTRICTED = "restricted"
|
||||
|
||||
|
||||
# Fields
|
||||
|
||||
CREATED_AT = "created_at"
|
||||
IS_PUBLIC = "is_public"
|
||||
DEPTH = "depth"
|
||||
PATH = "path"
|
||||
NUMCHILD = "numchild"
|
||||
REACH = "reach"
|
||||
SIZE = "size"
|
||||
TITLE = "title"
|
||||
UPDATED_AT = "updated_at"
|
||||
USERS = "users"
|
||||
GROUPS = "groups"
|
||||
|
||||
ORDER_BY_OPTIONS = (RELEVANCE, TITLE, CREATED_AT, UPDATED_AT, SIZE, IS_PUBLIC)
|
||||
SOURCE_FIELDS = (TITLE, SIZE, CREATED_AT, UPDATED_AT, IS_PUBLIC)
|
||||
RELEVANCE = "relevance"
|
||||
|
||||
ORDER_BY_OPTIONS = (RELEVANCE, TITLE, CREATED_AT, UPDATED_AT, SIZE, REACH)
|
||||
SOURCE_FIELDS = (TITLE, SIZE, DEPTH, PATH, NUMCHILD, CREATED_AT, UPDATED_AT, REACH)
|
||||
|
||||
@@ -8,7 +8,7 @@ from django.utils.text import slugify
|
||||
import factory
|
||||
from faker import Faker
|
||||
|
||||
from .models import Service
|
||||
from . import enums, models
|
||||
|
||||
fake = Faker()
|
||||
|
||||
@@ -19,16 +19,20 @@ class DocumentSchemaFactory(factory.DictFactory):
|
||||
indexation for testing and development purposes.
|
||||
"""
|
||||
|
||||
id = factory.LazyFunction(uuid4)
|
||||
id = factory.LazyFunction(lambda: str(uuid4()))
|
||||
title = factory.Sequence(lambda n: f"Test title {n!s}")
|
||||
path = factory.Sequence(lambda n: f"000{n}")
|
||||
content = factory.Sequence(lambda n: f"Test content {n!s}")
|
||||
created_at = factory.LazyFunction(
|
||||
lambda: fake.date_time_this_decade(tzinfo=timezone.get_current_timezone())
|
||||
)
|
||||
size = factory.LazyFunction(lambda: fake.random_int(min=0, max=1024**2))
|
||||
users = factory.LazyFunction(lambda: [uuid4() for _ in range(3)])
|
||||
users = factory.LazyFunction(lambda: [str(uuid4()) for _ in range(3)])
|
||||
groups = factory.LazyFunction(lambda: [slugify(fake.word()) for _ in range(3)])
|
||||
is_public = factory.Faker("boolean")
|
||||
reach = factory.Iterator(list(enums.ReachEnum))
|
||||
depth = 1
|
||||
numchild = 0
|
||||
is_active = True
|
||||
|
||||
@factory.lazy_attribute
|
||||
def updated_at(self):
|
||||
@@ -48,6 +52,7 @@ class ServiceFactory(factory.django.DjangoModelFactory):
|
||||
name = factory.Sequence(lambda n: f"test-index-{n!s}")
|
||||
created_at = factory.Faker("date_time_this_year", tzinfo=None)
|
||||
is_active = True
|
||||
client_id = "some_client_id"
|
||||
|
||||
class Meta:
|
||||
model = Service
|
||||
model = models.Service
|
||||
|
||||
@@ -59,6 +59,6 @@ class Migration(migrations.Migration):
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='service',
|
||||
constraint=models.CheckConstraint(check=models.Q(('token__length', 50)), name='token_length_exact_50'),
|
||||
constraint=models.CheckConstraint(condition=models.Q(('token__length', 50)), name='token_length_exact_50'),
|
||||
),
|
||||
]
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# Generated by Django 5.1.4 on 2025-09-05 04:41
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('core', '0001_initial'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='service',
|
||||
name='client_id',
|
||||
field=models.CharField(blank=True, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='service',
|
||||
name='services',
|
||||
field=models.ManyToManyField(blank=True, to='core.service', verbose_name='Allowed services for search'),
|
||||
),
|
||||
]
|
||||
@@ -24,6 +24,12 @@ class Service(models.Model):
|
||||
token = models.CharField(max_length=TOKEN_LENGTH)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
is_active = models.BooleanField(default=True)
|
||||
client_id = models.CharField(blank=True, null=True)
|
||||
services = models.ManyToManyField(
|
||||
"self",
|
||||
verbose_name=_("Allowed services for search"),
|
||||
blank=True,
|
||||
)
|
||||
|
||||
class Meta:
|
||||
db_table = "find_service"
|
||||
@@ -32,7 +38,8 @@ class Service(models.Model):
|
||||
ordering = ["-is_active", "-created_at"]
|
||||
constraints = [
|
||||
models.CheckConstraint(
|
||||
check=models.Q(token__length=TOKEN_LENGTH), name="token_length_exact_50"
|
||||
condition=models.Q(token__length=TOKEN_LENGTH),
|
||||
name="token_length_exact_50",
|
||||
),
|
||||
]
|
||||
|
||||
@@ -49,11 +56,8 @@ class Service(models.Model):
|
||||
@staticmethod
|
||||
def generate_secure_token():
|
||||
"""Generate a secure token with with Python secret module"""
|
||||
characters = string.ascii_letters + string.digits + string.punctuation
|
||||
characters = (
|
||||
string.ascii_letters + string.digits + r"""!#%&'()*+,-./:;<=>?@[\]^_`{|}~"""
|
||||
)
|
||||
token = "".join(secrets.choice(characters) for _ in range(TOKEN_LENGTH))
|
||||
return token
|
||||
|
||||
@property
|
||||
def index_name(self):
|
||||
"""Compute index name from service name"""
|
||||
return f"find-{self.name:s}"
|
||||
|
||||
@@ -25,21 +25,29 @@ def ensure_index_exists(index_name):
|
||||
"mappings": {
|
||||
"dynamic": "strict",
|
||||
"properties": {
|
||||
"id": {"type": "keyword"},
|
||||
"title": {
|
||||
"type": "keyword", # Primary field for exact matches and sorting
|
||||
"fields": {
|
||||
"text": {
|
||||
"type": "text" # Sub-field for full-text search
|
||||
}
|
||||
"type": "text"
|
||||
} # Sub-field for full-text search
|
||||
},
|
||||
},
|
||||
"depth": {"type": "integer"},
|
||||
"path": {
|
||||
"type": "keyword",
|
||||
"fields": {"text": {"type": "text"}},
|
||||
},
|
||||
"numchild": {"type": "integer"},
|
||||
"content": {"type": "text"},
|
||||
"created_at": {"type": "date"},
|
||||
"updated_at": {"type": "date"},
|
||||
"size": {"type": "long"},
|
||||
"users": {"type": "keyword"},
|
||||
"groups": {"type": "keyword"},
|
||||
"is_public": {"type": "boolean"},
|
||||
"reach": {"type": "keyword"},
|
||||
"is_active": {"type": "boolean"},
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
+39
-16
@@ -9,6 +9,7 @@ from pydantic import (
|
||||
UUID4,
|
||||
AwareDatetime,
|
||||
BaseModel,
|
||||
BeforeValidator,
|
||||
ConfigDict,
|
||||
Field,
|
||||
conint,
|
||||
@@ -23,16 +24,20 @@ class DocumentSchema(BaseModel):
|
||||
"""Schema for validating the documents submitted to our API for indexing"""
|
||||
|
||||
id: UUID4
|
||||
title: Annotated[str, Field(max_length=300)]
|
||||
content: str
|
||||
title: Annotated[str, Field(max_length=300, min_length=0)]
|
||||
depth: Annotated[int, Field(ge=0)]
|
||||
path: Annotated[str, Field(max_length=300)]
|
||||
numchild: Annotated[int, Field(ge=0)]
|
||||
content: Annotated[str, Field(min_length=0)]
|
||||
created_at: AwareDatetime
|
||||
updated_at: AwareDatetime
|
||||
size: Annotated[int, Field(ge=0, le=100 * 1024**3)] # File size limited to 100GB
|
||||
users: List[UUID4] = Field(default_factory=list)
|
||||
users: List[Annotated[str, Field(max_length=50)]] = Field(default_factory=list)
|
||||
groups: List[Annotated[str, Field(pattern=r"^[a-z0-9]+(?:-[a-z0-9]+)*$")]] = Field(
|
||||
default_factory=list
|
||||
)
|
||||
is_public: bool = Field(default=False)
|
||||
reach: Optional[enums.ReachEnum] = Field(default=enums.ReachEnum.RESTRICTED)
|
||||
is_active: bool
|
||||
|
||||
model_config = ConfigDict(
|
||||
str_min_length=1, str_strip_whitespace=True, use_enum_values=True
|
||||
@@ -52,6 +57,13 @@ class DocumentSchema(BaseModel):
|
||||
raise ValueError(f"{info.field_name} must be earlier than now")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_empty_content(self):
|
||||
"""Validate that either `title` or `content` are not empty."""
|
||||
if not self.title and not self.content:
|
||||
raise ValueError("Either title or content should have at least 1 character")
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_update_at_after_created_at(self):
|
||||
"""Date and time of last modification should be later than date and time of creation"""
|
||||
@@ -74,22 +86,33 @@ class DocumentSchema(BaseModel):
|
||||
return validated_groups
|
||||
|
||||
|
||||
def cleanlist(value):
|
||||
"""Build a list of strings from a string, None (empty list) or a list of objects."""
|
||||
if isinstance(value, str):
|
||||
# Convert comma-separated strings to list
|
||||
return [s.strip() for s in value.split(",") if s.strip()]
|
||||
|
||||
if isinstance(value, list):
|
||||
# Clean up list of strings
|
||||
return [str(s).strip() for s in value if s is not None and str(s).strip()]
|
||||
|
||||
if value is None:
|
||||
return []
|
||||
|
||||
raise ValueError()
|
||||
|
||||
|
||||
StringListParameter = Annotated[List[str], BeforeValidator(cleanlist)]
|
||||
|
||||
|
||||
class SearchQueryParametersSchema(BaseModel):
|
||||
"""Schema for validating the querystring on the search API endpoint"""
|
||||
|
||||
q: str
|
||||
is_public: Optional[bool] = None
|
||||
order_by: Optional[Literal[enums.ORDER_BY_OPTIONS]] = Field(default="relevance")
|
||||
services: StringListParameter = Field(default_factory=list)
|
||||
visited: StringListParameter = Field(default_factory=list)
|
||||
reach: Optional[enums.ReachEnum] = None
|
||||
order_by: Optional[Literal[enums.ORDER_BY_OPTIONS]] = Field(default=enums.RELEVANCE)
|
||||
order_direction: Optional[Literal["asc", "desc"]] = Field(default="desc")
|
||||
page_number: Optional[conint(ge=1)] = Field(default=1)
|
||||
page_size: Optional[conint(ge=1, le=100)] = Field(default=50)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@staticmethod
|
||||
def handle_lists(values):
|
||||
"""Make sure we get strings and ignore multiple values."""
|
||||
for key, value in values.items():
|
||||
if isinstance(value, list):
|
||||
# Take the first item if it's a list
|
||||
values[key] = value[0] if value else None
|
||||
return values
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Fixtures for tests in the find core application"""
|
||||
|
||||
import pytest
|
||||
from lasuite.oidc_resource_server.authentication import (
|
||||
get_resource_server_backend,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(name="jwt_rs_backend")
|
||||
def jwt_resource_server_backend_fixture(settings):
|
||||
"""Fixture to switch the backend to the JWTResourceServerBackend."""
|
||||
_original_backend = str(settings.OIDC_RS_BACKEND_CLASS)
|
||||
|
||||
settings.OIDC_RS_BACKEND_CLASS = (
|
||||
"lasuite.oidc_resource_server.backend.JWTResourceServerBackend"
|
||||
)
|
||||
get_resource_server_backend.cache_clear()
|
||||
|
||||
yield
|
||||
|
||||
settings.OIDC_RS_BACKEND_CLASS = _original_backend
|
||||
get_resource_server_backend.cache_clear()
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Tests indexing documents in OpenSearch over the API"""
|
||||
|
||||
import datetime
|
||||
from unittest import mock
|
||||
|
||||
from django.utils import timezone
|
||||
|
||||
@@ -16,7 +17,7 @@ def test_api_documents_index_bulk_anonymous():
|
||||
"""Anonymous requests should not be allowed to index documents in bulk."""
|
||||
documents = factories.DocumentSchemaFactory.build_batch(3)
|
||||
|
||||
response = APIClient().post("/api/v1.0/documents/", documents, format="json")
|
||||
response = APIClient().post("/api/v1.0/documents/index/", documents, format="json")
|
||||
|
||||
assert response.status_code == 403
|
||||
assert response.json() == {
|
||||
@@ -29,7 +30,7 @@ def test_api_documents_index_bulk_invalid_token():
|
||||
documents = factories.DocumentSchemaFactory.build_batch(3)
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/",
|
||||
"/api/v1.0/documents/index/",
|
||||
documents,
|
||||
HTTP_AUTHORIZATION="Bearer invalid",
|
||||
format="json",
|
||||
@@ -45,17 +46,42 @@ def test_api_documents_index_bulk_success():
|
||||
documents = factories.DocumentSchemaFactory.build_batch(3)
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/",
|
||||
"/api/v1.0/documents/index/",
|
||||
documents,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == 207
|
||||
assert response.status_code == 201
|
||||
responses = response.json()
|
||||
assert [d["status"] for d in responses] == ["success"] * 3
|
||||
|
||||
|
||||
def test_api_documents_index_bulk_ensure_index():
|
||||
"""A registered service should be create the opensearch index if need."""
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
documents = factories.DocumentSchemaFactory.build_batch(3)
|
||||
|
||||
# Delete the index
|
||||
opensearch.client.indices.delete(index="*test*")
|
||||
|
||||
with pytest.raises(opensearch.NotFoundError):
|
||||
opensearch.client.indices.get(index="test-service")
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/index/",
|
||||
documents,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == 201
|
||||
responses = response.json()
|
||||
assert len(responses) == 3
|
||||
for result in response.json():
|
||||
assert result["status"] == "success"
|
||||
assert [d["status"] for d in responses] == ["success"] * 3
|
||||
|
||||
# The index has been rebuilt
|
||||
opensearch.client.indices.get(index="test-service")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -77,6 +103,36 @@ def test_api_documents_index_bulk_success():
|
||||
"string_too_long",
|
||||
"String should have at most 300 characters",
|
||||
),
|
||||
(
|
||||
"depth",
|
||||
-1,
|
||||
"greater_than_equal",
|
||||
"Input should be greater than or equal to 0",
|
||||
),
|
||||
(
|
||||
"depth",
|
||||
"a",
|
||||
"int_parsing",
|
||||
"Input should be a valid integer, unable to parse string as an integer",
|
||||
),
|
||||
(
|
||||
"path",
|
||||
"a" * 301,
|
||||
"string_too_long",
|
||||
"String should have at most 300 characters",
|
||||
),
|
||||
(
|
||||
"numchild",
|
||||
-1,
|
||||
"greater_than_equal",
|
||||
"Input should be greater than or equal to 0",
|
||||
),
|
||||
(
|
||||
"numchild",
|
||||
"a",
|
||||
"int_parsing",
|
||||
"Input should be a valid integer, unable to parse string as an integer",
|
||||
),
|
||||
("content", 1, "string_type", "Input should be a valid string"),
|
||||
(
|
||||
"created_at",
|
||||
@@ -110,12 +166,9 @@ def test_api_documents_index_bulk_success():
|
||||
),
|
||||
(
|
||||
"users",
|
||||
["33052c8b-3181-4420-aede-f8396fc0f9az"], # invalid UUID b/c contains a z
|
||||
"uuid_parsing",
|
||||
(
|
||||
"Input should be a valid UUID, invalid character: expected an optional "
|
||||
"prefix of `urn:uuid:` followed by [0-9a-fA-F-], found `z` at 36"
|
||||
),
|
||||
["a" * 51],
|
||||
"string_too_long",
|
||||
"String should have at most 50 characters",
|
||||
),
|
||||
(
|
||||
"groups",
|
||||
@@ -130,8 +183,14 @@ def test_api_documents_index_bulk_success():
|
||||
"Input should be a valid list",
|
||||
),
|
||||
(
|
||||
"is_public",
|
||||
"invalid_boolean",
|
||||
"reach",
|
||||
"invalid",
|
||||
"enum",
|
||||
"Input should be 'public', 'authenticated' or 'restricted'",
|
||||
),
|
||||
(
|
||||
"is_active",
|
||||
"invalid",
|
||||
"bool_parsing",
|
||||
"Input should be a valid boolean, unable to interpret input",
|
||||
),
|
||||
@@ -148,7 +207,7 @@ def test_api_documents_index_bulk_invalid_document(
|
||||
documents[0][field] = invalid_value
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/",
|
||||
"/api/v1.0/documents/index/",
|
||||
documents,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
@@ -156,18 +215,28 @@ def test_api_documents_index_bulk_invalid_document(
|
||||
|
||||
assert response.status_code == 400
|
||||
responses = response.json()
|
||||
assert [d["status"] for d in responses] == ["error", "valid", "valid"]
|
||||
|
||||
assert responses[0]["status"] == "error"
|
||||
assert len(responses[0]["errors"]) == 1
|
||||
assert responses[0]["errors"][0]["msg"] == error_message
|
||||
assert responses[0]["errors"][0]["type"] == error_type
|
||||
|
||||
for i in [1, 2]:
|
||||
assert responses[i]["status"] == "valid"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field", ["id", "title", "content", "size", "created_at", "updated_at"]
|
||||
"field",
|
||||
[
|
||||
"id",
|
||||
"title",
|
||||
"depth",
|
||||
"path",
|
||||
"numchild",
|
||||
"content",
|
||||
"size",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"is_active",
|
||||
],
|
||||
)
|
||||
def test_api_documents_index_bulk_required(field):
|
||||
"""Test bulk document indexing with a required field missing."""
|
||||
@@ -177,7 +246,7 @@ def test_api_documents_index_bulk_required(field):
|
||||
del documents[0][field]
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/",
|
||||
"/api/v1.0/documents/index/",
|
||||
documents,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
@@ -185,21 +254,20 @@ def test_api_documents_index_bulk_required(field):
|
||||
|
||||
assert response.status_code == 400
|
||||
responses = response.json()
|
||||
assert [d["status"] for d in responses] == ["error", "valid", "valid"]
|
||||
|
||||
assert responses[0]["status"] == "error"
|
||||
assert len(responses[0]["errors"]) == 1
|
||||
assert responses[0]["errors"][0]["msg"] == "Field required"
|
||||
assert responses[0]["errors"][0]["type"] == "missing"
|
||||
|
||||
for i in [1, 2]:
|
||||
assert responses[i]["status"] == "valid"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,default_value",
|
||||
[
|
||||
("users", []),
|
||||
("groups", []),
|
||||
("is_public", False),
|
||||
("reach", "restricted"),
|
||||
],
|
||||
)
|
||||
def test_api_documents_index_bulk_default(field, default_value):
|
||||
@@ -210,20 +278,18 @@ def test_api_documents_index_bulk_default(field, default_value):
|
||||
del documents[0][field]
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/",
|
||||
"/api/v1.0/documents/index/",
|
||||
documents,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == 207
|
||||
assert response.status_code == 201
|
||||
responses = response.json()
|
||||
assert len(responses) == 3
|
||||
for result in response.json():
|
||||
assert result["status"] == "success"
|
||||
assert [d["status"] for d in responses] == ["success"] * 3
|
||||
|
||||
indexed_document = opensearch.client.get(
|
||||
index=service.index_name, id=responses[0]["_id"]
|
||||
index=service.name, id=responses[0]["_id"]
|
||||
)["_source"]
|
||||
assert indexed_document[field] == default_value
|
||||
|
||||
@@ -238,7 +304,7 @@ def test_api_documents_index_bulk_updated_at_before_created_at():
|
||||
)
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/",
|
||||
"/api/v1.0/documents/index/",
|
||||
documents,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
@@ -246,6 +312,9 @@ def test_api_documents_index_bulk_updated_at_before_created_at():
|
||||
|
||||
assert response.status_code == 400
|
||||
responses = response.json()
|
||||
|
||||
assert [d["status"] for d in responses] == ["error", "valid", "valid"]
|
||||
|
||||
assert responses[0]["status"] == "error"
|
||||
assert len(responses[0]["errors"]) == 1
|
||||
assert (
|
||||
@@ -254,9 +323,6 @@ def test_api_documents_index_bulk_updated_at_before_created_at():
|
||||
)
|
||||
assert responses[0]["errors"][0]["type"] == "value_error"
|
||||
|
||||
for i in [1, 2]:
|
||||
assert responses[i]["status"] == "valid"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field",
|
||||
@@ -271,7 +337,7 @@ def test_api_documents_index_bulk_datetime_future(field):
|
||||
documents[0][field] = now + datetime.timedelta(seconds=3)
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/",
|
||||
"/api/v1.0/documents/index/",
|
||||
documents,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
@@ -279,6 +345,8 @@ def test_api_documents_index_bulk_datetime_future(field):
|
||||
|
||||
assert response.status_code == 400
|
||||
responses = response.json()
|
||||
assert [d["status"] for d in responses] == ["error", "valid", "valid"]
|
||||
|
||||
assert responses[0]["status"] == "error"
|
||||
assert len(responses[0]["errors"]) == 1
|
||||
assert (
|
||||
@@ -287,5 +355,78 @@ def test_api_documents_index_bulk_datetime_future(field):
|
||||
)
|
||||
assert responses[0]["errors"][0]["type"] == "value_error"
|
||||
|
||||
for i in [1, 2]:
|
||||
assert responses[i]["status"] == "valid"
|
||||
|
||||
def test_api_documents_index_empty_content_check():
|
||||
"""Test bulk document indexing with both empty title & content."""
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
documents = factories.DocumentSchemaFactory.build_batch(3)
|
||||
|
||||
documents[0]["content"] = ""
|
||||
documents[0]["title"] = ""
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/index/",
|
||||
documents,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
responses = response.json()
|
||||
assert [d["status"] for d in responses] == ["error", "valid", "valid"]
|
||||
|
||||
assert responses[0]["status"] == "error"
|
||||
assert len(responses[0]["errors"]) == 1
|
||||
assert (
|
||||
responses[0]["errors"][0]["msg"]
|
||||
== "Value error, Either title or content should have at least 1 character"
|
||||
)
|
||||
assert responses[0]["errors"][0]["type"] == "value_error"
|
||||
|
||||
|
||||
def test_api_documents_index_opensearch_errors():
|
||||
"""Test bulk document indexing errors"""
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
documents = factories.DocumentSchemaFactory.build_batch(3)
|
||||
|
||||
with mock.patch.object(opensearch.client, "bulk") as mock_bulk:
|
||||
mock_bulk.return_value = {
|
||||
"items": [
|
||||
{"index": {"status": 201}},
|
||||
{
|
||||
"index": {
|
||||
"status": 400,
|
||||
}
|
||||
},
|
||||
{"index": {"status": 403, "error": {"reason": "This is forbidden"}}},
|
||||
]
|
||||
}
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/index/",
|
||||
documents,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == 201
|
||||
responses = response.json()
|
||||
assert responses == [
|
||||
{
|
||||
"_id": documents[0]["id"],
|
||||
"index": 0,
|
||||
"status": "success",
|
||||
},
|
||||
{
|
||||
"_id": documents[1]["id"],
|
||||
"index": 1,
|
||||
"status": "error",
|
||||
"message": "Unknown error",
|
||||
},
|
||||
{
|
||||
"_id": documents[2]["id"],
|
||||
"index": 2,
|
||||
"status": "error",
|
||||
"message": "This is forbidden",
|
||||
},
|
||||
]
|
||||
|
||||
@@ -16,7 +16,7 @@ def test_api_documents_index_single_anonymous():
|
||||
"""Anonymous requests should not be allowed to index documents."""
|
||||
document = factories.DocumentSchemaFactory.build()
|
||||
|
||||
response = APIClient().post("/api/v1.0/documents/", document, format="json")
|
||||
response = APIClient().post("/api/v1.0/documents/index/", document, format="json")
|
||||
|
||||
assert response.status_code == 403
|
||||
assert response.json() == {
|
||||
@@ -29,7 +29,7 @@ def test_api_documents_index_single_invalid_token():
|
||||
document = factories.DocumentSchemaFactory.build()
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/",
|
||||
"/api/v1.0/documents/index/",
|
||||
document,
|
||||
HTTP_AUTHORIZATION="Bearer invalid",
|
||||
format="json",
|
||||
@@ -45,7 +45,7 @@ def test_api_documents_index_single_success():
|
||||
document = factories.DocumentSchemaFactory.build()
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/",
|
||||
"/api/v1.0/documents/index/",
|
||||
document,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
@@ -55,6 +55,58 @@ def test_api_documents_index_single_success():
|
||||
assert response.json()["_id"] == str(document["id"])
|
||||
|
||||
|
||||
def test_api_documents_index_bulk_ensure_index():
|
||||
"""A registered service should be create the opensearch index if need."""
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
document = factories.DocumentSchemaFactory.build()
|
||||
|
||||
# Delete the index
|
||||
opensearch.client.indices.delete(index="*test*")
|
||||
|
||||
with pytest.raises(opensearch.NotFoundError):
|
||||
opensearch.client.indices.get(index="test-service")
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/index/",
|
||||
document,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == 201
|
||||
assert response.json()["_id"] == str(document["id"])
|
||||
|
||||
# The index has been rebuilt
|
||||
data = opensearch.client.indices.get(index="test-service")
|
||||
|
||||
assert data["test-service"]["mappings"] == {
|
||||
"dynamic": "strict",
|
||||
"properties": {
|
||||
"id": {"type": "keyword"},
|
||||
"title": {
|
||||
"type": "keyword", # Primary field for exact matches and sorting
|
||||
"fields": {
|
||||
"text": {"type": "text"} # Sub-field for full-text search
|
||||
},
|
||||
},
|
||||
"depth": {"type": "integer"},
|
||||
"path": {
|
||||
"type": "keyword",
|
||||
"fields": {"text": {"type": "text"}},
|
||||
},
|
||||
"numchild": {"type": "integer"},
|
||||
"content": {"type": "text"},
|
||||
"created_at": {"type": "date"},
|
||||
"updated_at": {"type": "date"},
|
||||
"size": {"type": "long"},
|
||||
"users": {"type": "keyword"},
|
||||
"groups": {"type": "keyword"},
|
||||
"reach": {"type": "keyword"},
|
||||
"is_active": {"type": "boolean"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field, invalid_value, error_type, error_message",
|
||||
[
|
||||
@@ -74,6 +126,36 @@ def test_api_documents_index_single_success():
|
||||
"string_too_long",
|
||||
"String should have at most 300 characters",
|
||||
),
|
||||
(
|
||||
"depth",
|
||||
-1,
|
||||
"greater_than_equal",
|
||||
"Input should be greater than or equal to 0",
|
||||
),
|
||||
(
|
||||
"depth",
|
||||
"a",
|
||||
"int_parsing",
|
||||
"Input should be a valid integer, unable to parse string as an integer",
|
||||
),
|
||||
(
|
||||
"path",
|
||||
"a" * 301,
|
||||
"string_too_long",
|
||||
"String should have at most 300 characters",
|
||||
),
|
||||
(
|
||||
"numchild",
|
||||
-1,
|
||||
"greater_than_equal",
|
||||
"Input should be greater than or equal to 0",
|
||||
),
|
||||
(
|
||||
"numchild",
|
||||
"a",
|
||||
"int_parsing",
|
||||
"Input should be a valid integer, unable to parse string as an integer",
|
||||
),
|
||||
("content", 1, "string_type", "Input should be a valid string"),
|
||||
(
|
||||
"created_at",
|
||||
@@ -107,12 +189,9 @@ def test_api_documents_index_single_success():
|
||||
),
|
||||
(
|
||||
"users",
|
||||
["33052c8b-3181-4420-aede-f8396fc0f9az"], # invalid UUID b/c contains a z*
|
||||
"uuid_parsing",
|
||||
(
|
||||
"Input should be a valid UUID, invalid character: expected an optional "
|
||||
"prefix of `urn:uuid:` followed by [0-9a-fA-F-], found `z` at 36"
|
||||
),
|
||||
["a" * 51],
|
||||
"string_too_long",
|
||||
"String should have at most 50 characters",
|
||||
),
|
||||
(
|
||||
"groups",
|
||||
@@ -127,8 +206,14 @@ def test_api_documents_index_single_success():
|
||||
("Input should be a valid list"),
|
||||
),
|
||||
(
|
||||
"is_public",
|
||||
"invalid_boolean",
|
||||
"reach",
|
||||
"invalid",
|
||||
"enum",
|
||||
"Input should be 'public', 'authenticated' or 'restricted'",
|
||||
),
|
||||
(
|
||||
"is_active",
|
||||
"invalid",
|
||||
"bool_parsing",
|
||||
"Input should be a valid boolean, unable to interpret input",
|
||||
),
|
||||
@@ -145,7 +230,7 @@ def test_api_documents_index_single_invalid_document(
|
||||
document[field] = invalid_value
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/",
|
||||
"/api/v1.0/documents/index/",
|
||||
document,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
@@ -157,7 +242,19 @@ def test_api_documents_index_single_invalid_document(
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field", ["id", "title", "content", "size", "created_at", "updated_at"]
|
||||
"field",
|
||||
[
|
||||
"id",
|
||||
"title",
|
||||
"depth",
|
||||
"path",
|
||||
"numchild",
|
||||
"content",
|
||||
"size",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"is_active",
|
||||
],
|
||||
)
|
||||
def test_api_documents_index_single_required(field):
|
||||
"""Test document indexing with a required field missing."""
|
||||
@@ -167,7 +264,7 @@ def test_api_documents_index_single_required(field):
|
||||
del document[field]
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/",
|
||||
"/api/v1.0/documents/index/",
|
||||
document,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
@@ -183,7 +280,7 @@ def test_api_documents_index_single_required(field):
|
||||
[
|
||||
("users", []),
|
||||
("groups", []),
|
||||
("is_public", False),
|
||||
("reach", "restricted"),
|
||||
],
|
||||
)
|
||||
def test_api_documents_index_single_default(field, default_value):
|
||||
@@ -194,7 +291,7 @@ def test_api_documents_index_single_default(field, default_value):
|
||||
del document[field]
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/",
|
||||
"/api/v1.0/documents/index/",
|
||||
document,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
@@ -204,7 +301,7 @@ def test_api_documents_index_single_default(field, default_value):
|
||||
assert response.json()["_id"] == str(document["id"])
|
||||
|
||||
indexed_document = opensearch.client.get(
|
||||
index=service.index_name, id=str(document["id"])
|
||||
index=service.name, id=str(document["id"])
|
||||
)["_source"]
|
||||
assert indexed_document[field] == default_value
|
||||
|
||||
@@ -217,7 +314,7 @@ def test_api_documents_index_single_udpated_at_before_created():
|
||||
document["updated_at"] = document["created_at"] - datetime.timedelta(seconds=1)
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/",
|
||||
"/api/v1.0/documents/index/",
|
||||
document,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
@@ -244,7 +341,7 @@ def test_api_documents_index_single_datetime_future(field):
|
||||
document[field] = now + datetime.timedelta(seconds=3)
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/",
|
||||
"/api/v1.0/documents/index/",
|
||||
document,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
@@ -253,3 +350,26 @@ def test_api_documents_index_single_datetime_future(field):
|
||||
assert response.status_code == 400
|
||||
assert response.data[0]["msg"] == f"Value error, {field:s} must be earlier than now"
|
||||
assert response.data[0]["type"] == "value_error"
|
||||
|
||||
|
||||
def test_api_documents_index_empty_content_check():
|
||||
"""Test document indexing with both empty title & content."""
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
document = factories.DocumentSchemaFactory.build()
|
||||
|
||||
document["content"] = ""
|
||||
document["title"] = ""
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/index/",
|
||||
document,
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert (
|
||||
response.data[0]["msg"]
|
||||
== "Value error, Either title or content should have at least 1 character"
|
||||
)
|
||||
assert response.data[0]["type"] == "value_error"
|
||||
|
||||
@@ -6,65 +6,205 @@ of documents is slow and better done only once.
|
||||
"""
|
||||
|
||||
import operator
|
||||
import random
|
||||
|
||||
import pytest
|
||||
from opensearchpy.helpers import bulk
|
||||
import responses
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from core import enums, factories, opensearch
|
||||
from core import enums, factories
|
||||
|
||||
from .utils import build_authorization_bearer, prepare_index, setup_oicd_resource_server
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
def prepare_index(index_name, documents):
|
||||
"""Prepare the search index before testing a query on it."""
|
||||
@responses.activate
|
||||
def test_api_documents_search_auth_invalid_parameters(settings):
|
||||
"""Invalid service parameters should result in a 401 error"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
|
||||
documents = documents if isinstance(documents, list) else [documents]
|
||||
settings.OIDC_RS_CLIENT_ID = None
|
||||
settings.OIDC_RS_CLIENT_SECRET = None
|
||||
|
||||
# Ensure existence of an empty index
|
||||
opensearch.client.indices.delete(index=index_name)
|
||||
opensearch.ensure_index_exists(index_name)
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
prepare_index(service.name, [])
|
||||
|
||||
# Bulk index documents
|
||||
actions = []
|
||||
for document in documents:
|
||||
_source = document.copy()
|
||||
_id = _source.pop("id")
|
||||
actions.append(
|
||||
{
|
||||
"_op_type": "index",
|
||||
"_index": index_name,
|
||||
"_id": _id,
|
||||
"_source": _source,
|
||||
}
|
||||
)
|
||||
bulk(opensearch.client, actions)
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "a quick fox"},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION="Bearer unknown",
|
||||
)
|
||||
|
||||
opensearch.client.indices.refresh(index=index_name)
|
||||
assert opensearch.client.count(index=index_name)["count"] == len(documents)
|
||||
assert response.status_code == 401
|
||||
assert response.json() == {"detail": "Resource Server is improperly configured"}
|
||||
|
||||
|
||||
def test_api_documents_search_query_title():
|
||||
@responses.activate
|
||||
def test_api_documents_search_query_unknown_user(settings):
|
||||
"""Searching a document without an existing user should result in a 401 error"""
|
||||
setup_oicd_resource_server(
|
||||
responses,
|
||||
settings,
|
||||
sub="unknown",
|
||||
introspect=lambda request, user_info: (404, {}, ""),
|
||||
)
|
||||
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
prepare_index(service.name, [])
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "a quick fox"},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 401
|
||||
assert response.json() == {"detail": "Login failed"}
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_query_iss_not_valid(settings):
|
||||
"""Searching a document with a conflicting iss in token should result in a 401 error"""
|
||||
settings.OIDC_RS_VERIFY_CLAIMS = True
|
||||
|
||||
setup_oicd_resource_server(
|
||||
responses,
|
||||
settings,
|
||||
sub="user_sub",
|
||||
iss="http://other.com",
|
||||
)
|
||||
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
prepare_index(service.name, [])
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "a quick fox"},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 401
|
||||
assert response.json() == {"detail": "Login failed"}
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_query_iss_not_valid_ignored(settings):
|
||||
"""Searching a document with a conflicting iss in token should result in a 401 error"""
|
||||
settings.OIDC_RS_VERIFY_CLAIMS = False
|
||||
|
||||
setup_oicd_resource_server(
|
||||
responses,
|
||||
settings,
|
||||
sub="user_sub",
|
||||
iss="http://other.com",
|
||||
)
|
||||
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
prepare_index(service.name, [])
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "a quick fox"},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == []
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_services_invalid_parameters(settings):
|
||||
"""Invalid pagination parameters should result in a 400 error"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
factories.ServiceFactory(name="test-service")
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "a quick fox", "services": {}},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json() == [
|
||||
{
|
||||
"loc": ["services"],
|
||||
"msg": "Value error, ",
|
||||
"type": "value_error",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_reached_docs_invalid_parameters(settings):
|
||||
"""Invalid pagination parameters should result in a 400 error"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
factories.ServiceFactory(name="test-service")
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "a quick fox", "visited": {}},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json() == [
|
||||
{
|
||||
"loc": ["visited"],
|
||||
"msg": "Value error, ",
|
||||
"type": "value_error",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_query_title(settings):
|
||||
"""Searching a document by its title should work as expected"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
document = factories.DocumentSchemaFactory.build(
|
||||
title="The quick brown fox", content="the wolf"
|
||||
title="The quick brown fox",
|
||||
content="the wolf",
|
||||
reach=random.choice(["public", "authenticated"]),
|
||||
)
|
||||
|
||||
# Add other documents
|
||||
other_fox_document = factories.DocumentSchemaFactory.build(
|
||||
title="The blue fox", content="the wolf"
|
||||
title="The blue fox",
|
||||
content="the wolf",
|
||||
reach=random.choice(["public", "authenticated"]),
|
||||
)
|
||||
no_fox_document = factories.DocumentSchemaFactory.build(
|
||||
title="The brown goat", content="the wolf"
|
||||
title="The brown goat",
|
||||
content="the wolf",
|
||||
reach=random.choice(["public", "authenticated"]),
|
||||
)
|
||||
documents = [document, other_fox_document, no_fox_document]
|
||||
prepare_index(service.index_name, documents)
|
||||
prepare_index(service.name, documents)
|
||||
|
||||
response = APIClient().get(
|
||||
"/api/v1.0/documents/",
|
||||
{"q": "a quick fox"},
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "a quick fox", "visited": [doc["id"] for doc in documents]},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -74,10 +214,13 @@ def test_api_documents_search_query_title():
|
||||
assert list(fox_data.keys()) == ["_index", "_id", "_score", "_source", "fields"]
|
||||
assert fox_data["_id"] == str(document["id"])
|
||||
assert fox_data["_source"] == {
|
||||
"depth": 1,
|
||||
"numchild": 0,
|
||||
"path": document["path"],
|
||||
"size": document["size"],
|
||||
"created_at": document["created_at"].isoformat(),
|
||||
"updated_at": document["updated_at"].isoformat(),
|
||||
"is_public": document["is_public"],
|
||||
"reach": document["reach"],
|
||||
"title": "The quick brown fox",
|
||||
}
|
||||
assert fox_data["fields"] == {"number_of_users": [3], "number_of_groups": [3]}
|
||||
@@ -92,35 +235,51 @@ def test_api_documents_search_query_title():
|
||||
]
|
||||
assert other_fox_data["_id"] == str(other_fox_document["id"])
|
||||
assert other_fox_data["_source"] == {
|
||||
"depth": 1,
|
||||
"numchild": 0,
|
||||
"path": other_fox_document["path"],
|
||||
"size": other_fox_document["size"],
|
||||
"created_at": other_fox_document["created_at"].isoformat(),
|
||||
"updated_at": other_fox_document["updated_at"].isoformat(),
|
||||
"is_public": other_fox_document["is_public"],
|
||||
"reach": other_fox_document["reach"],
|
||||
"title": "The blue fox",
|
||||
}
|
||||
assert other_fox_data["fields"] == {"number_of_users": [3], "number_of_groups": [3]}
|
||||
|
||||
|
||||
def test_api_documents_search_query_content():
|
||||
@responses.activate
|
||||
def test_api_documents_search_query_content(settings):
|
||||
"""Searching a document by its content should work as expected"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
document = factories.DocumentSchemaFactory.build(
|
||||
title="the wolf", content="The quick brown fox"
|
||||
title="the wolf",
|
||||
content="The quick brown fox",
|
||||
reach=random.choice(["public", "authenticated"]),
|
||||
)
|
||||
|
||||
# Add other documents
|
||||
other_fox_document = factories.DocumentSchemaFactory.build(
|
||||
title="the wolf", content="The blue fox"
|
||||
title="the wolf",
|
||||
content="The blue fox",
|
||||
reach=random.choice(["public", "authenticated"]),
|
||||
)
|
||||
no_fox_document = factories.DocumentSchemaFactory.build(
|
||||
title="the wolf", content="The brown goat"
|
||||
title="the wolf",
|
||||
content="The brown goat",
|
||||
reach=random.choice(["public", "authenticated"]),
|
||||
)
|
||||
prepare_index(service.index_name, [document, other_fox_document, no_fox_document])
|
||||
|
||||
response = APIClient().get(
|
||||
"/api/v1.0/documents/",
|
||||
{"q": "a quick fox"},
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
documents = [document, other_fox_document, no_fox_document]
|
||||
prepare_index(service.name, documents)
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "a quick fox", "visited": [doc["id"] for doc in documents]},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -130,10 +289,13 @@ def test_api_documents_search_query_content():
|
||||
assert list(fox_data.keys()) == ["_index", "_id", "_score", "_source", "fields"]
|
||||
assert fox_data["_id"] == str(document["id"])
|
||||
assert fox_data["_source"] == {
|
||||
"depth": 1,
|
||||
"numchild": 0,
|
||||
"path": document["path"],
|
||||
"size": document["size"],
|
||||
"created_at": document["created_at"].isoformat(),
|
||||
"updated_at": document["updated_at"].isoformat(),
|
||||
"is_public": document["is_public"],
|
||||
"reach": document["reach"],
|
||||
"title": document["title"],
|
||||
}
|
||||
assert fox_data["fields"] == {"number_of_users": [3], "number_of_groups": [3]}
|
||||
@@ -148,20 +310,110 @@ def test_api_documents_search_query_content():
|
||||
]
|
||||
assert other_fox_data["_id"] == str(other_fox_document["id"])
|
||||
assert other_fox_data["_source"] == {
|
||||
"depth": 1,
|
||||
"numchild": 0,
|
||||
"path": other_fox_document["path"],
|
||||
"size": other_fox_document["size"],
|
||||
"created_at": other_fox_document["created_at"].isoformat(),
|
||||
"updated_at": other_fox_document["updated_at"].isoformat(),
|
||||
"is_public": other_fox_document["is_public"],
|
||||
"reach": other_fox_document["reach"],
|
||||
"title": other_fox_document["title"],
|
||||
}
|
||||
assert other_fox_data["fields"] == {"number_of_users": [3], "number_of_groups": [3]}
|
||||
|
||||
|
||||
def test_api_documents_search_ordering_by_fields():
|
||||
"""It should be possible to order by several fields"""
|
||||
@pytest.mark.parametrize(
|
||||
"query,expected",
|
||||
[
|
||||
# Search for the start of the word "brown"
|
||||
(
|
||||
"bro",
|
||||
[
|
||||
("doc-1", "3.00"), # "bro" found in title & content
|
||||
("doc-3", "2.00"), # "bro" found in content
|
||||
],
|
||||
),
|
||||
# Search for the start of the word "wolf"
|
||||
(
|
||||
"wol",
|
||||
[
|
||||
("doc-2", "2.00"), # "wol" found in title
|
||||
("doc-3", "2.00"), # "wol" found in content
|
||||
],
|
||||
),
|
||||
# Search for the start of the word "wolf" and "brown"
|
||||
(
|
||||
"brown wol",
|
||||
[
|
||||
("doc-1", "2.85"), # "brown" found in title & content
|
||||
("doc-3", "2.45"), # "brown wol" found in content
|
||||
("doc-2", "2.00"), # "wol" found in title
|
||||
],
|
||||
),
|
||||
# Search for the start of the words "brown" & "wolf"
|
||||
(
|
||||
"bro wol",
|
||||
[
|
||||
("doc-2", "2.00"), # "wolf" found in title
|
||||
("doc-3", "2.00"), # "brown" found in content
|
||||
],
|
||||
),
|
||||
],
|
||||
)
|
||||
@responses.activate
|
||||
def test_api_documents_search_query_content_word_parts(settings, query, expected):
|
||||
"""Searching a document by its content should work as expected"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
documents = factories.DocumentSchemaFactory.build_batch(4)
|
||||
prepare_index(service.index_name, documents)
|
||||
document = factories.DocumentSchemaFactory.build(
|
||||
id="doc-1",
|
||||
title="the brown goat",
|
||||
content="The quick brown fox",
|
||||
reach=random.choice(["public", "authenticated"]),
|
||||
)
|
||||
other_fox_document = factories.DocumentSchemaFactory.build(
|
||||
id="doc-2",
|
||||
title="the wolf",
|
||||
content="The blue fox",
|
||||
reach=random.choice(["public", "authenticated"]),
|
||||
)
|
||||
no_fox_document = factories.DocumentSchemaFactory.build(
|
||||
id="doc-3",
|
||||
title="the fox",
|
||||
content="The brown wolf",
|
||||
reach=random.choice(["public", "authenticated"]),
|
||||
)
|
||||
|
||||
documents = [document, other_fox_document, no_fox_document]
|
||||
prepare_index(service.name, documents)
|
||||
|
||||
# Search for the start of the word "brown" in the body
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": query, "visited": [doc["id"] for doc in documents]},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
brown_results = [(d["_id"], f"{d['_score']:.2f}") for d in response.json()]
|
||||
assert brown_results == expected
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_ordering_by_fields(settings):
|
||||
"""It should be possible to order by several fields"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
documents = factories.DocumentSchemaFactory.build_batch(
|
||||
4, reach=random.choice(["public", "authenticated"])
|
||||
)
|
||||
prepare_index(service.name, documents)
|
||||
|
||||
parameters = [
|
||||
(enums.TITLE, "asc"),
|
||||
@@ -172,66 +424,96 @@ def test_api_documents_search_ordering_by_fields():
|
||||
(enums.UPDATED_AT, "desc"),
|
||||
(enums.SIZE, "asc"),
|
||||
(enums.SIZE, "desc"),
|
||||
(enums.IS_PUBLIC, "asc"),
|
||||
(enums.IS_PUBLIC, "desc"),
|
||||
(enums.REACH, "asc"),
|
||||
(enums.REACH, "desc"),
|
||||
]
|
||||
|
||||
for field, direction in parameters:
|
||||
response = APIClient().get(
|
||||
f"/api/v1.0/documents/?q=*&order_by={field}&order_direction={direction}",
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token}",
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{
|
||||
"q": "*",
|
||||
"order_by": field,
|
||||
"order_direction": direction,
|
||||
"visited": [doc["id"] for doc in documents],
|
||||
},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
responses = response.json()
|
||||
assert len(responses) == 4
|
||||
data = response.json()
|
||||
assert len(data) == 4
|
||||
|
||||
# Check that results are sorted by the field as expected
|
||||
compare = operator.le if direction == "asc" else operator.ge
|
||||
for i in range(len(responses) - 1):
|
||||
assert compare(
|
||||
responses[i]["_source"][field], responses[i + 1]["_source"][field]
|
||||
)
|
||||
for i in range(len(data) - 1):
|
||||
assert compare(data[i]["_source"][field], data[i + 1]["_source"][field])
|
||||
|
||||
|
||||
def test_api_documents_search_ordering_by_relevance():
|
||||
@responses.activate
|
||||
def test_api_documents_search_ordering_by_relevance(settings):
|
||||
"""It should be possible to order by relevance (score)"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
documents = factories.DocumentSchemaFactory.build_batch(4)
|
||||
prepare_index(service.index_name, documents)
|
||||
documents = factories.DocumentSchemaFactory.build_batch(
|
||||
4, reach=random.choice(["public", "authenticated"])
|
||||
)
|
||||
prepare_index(service.name, documents)
|
||||
|
||||
for direction in ["asc", "desc"]:
|
||||
response = APIClient().get(
|
||||
f"/api/v1.0/documents/?q=*&order_by=relevance&order_direction={direction}",
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token}",
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{
|
||||
"q": "*",
|
||||
"order_by": "relevance",
|
||||
"order_direction": direction,
|
||||
"visited": [doc["id"] for doc in documents],
|
||||
},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
responses = response.json()
|
||||
assert len(responses) == 4
|
||||
data = response.json()
|
||||
assert len(data) == 4
|
||||
|
||||
# Check that results are sorted by score as expected
|
||||
compare = operator.le if direction == "asc" else operator.ge
|
||||
for i in range(len(responses) - 1):
|
||||
assert compare(responses[i]["_score"], responses[i + 1]["_score"])
|
||||
for i in range(len(data) - 1):
|
||||
assert compare(data[i]["_score"], data[i + 1]["_score"])
|
||||
|
||||
|
||||
def test_api_documents_search_ordering_by_unknown_field():
|
||||
@responses.activate
|
||||
def test_api_documents_search_ordering_by_unknown_field(settings):
|
||||
"""Trying to sort by an unknown field should return a 400 error"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
# Setup: Initialize the service and documents only once
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
documents = factories.DocumentSchemaFactory.build_batch(2)
|
||||
prepare_index(service.index_name, documents)
|
||||
documents = factories.DocumentSchemaFactory.build_batch(
|
||||
2, reach=random.choice(["public", "authenticated"])
|
||||
)
|
||||
prepare_index(service.name, documents)
|
||||
|
||||
# Define the parameters manually
|
||||
directions = ["asc", "desc"]
|
||||
|
||||
# Perform the parameterized tests
|
||||
for direction in directions:
|
||||
response = APIClient().get(
|
||||
f"/api/v1.0/documents/?q=*&order_by=unknown&order_direction={direction}",
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token}",
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{
|
||||
"q": "*",
|
||||
"order_by": "unknown",
|
||||
"order_direction": direction,
|
||||
"visited": [doc["id"] for doc in documents],
|
||||
},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
@@ -240,23 +522,36 @@ def test_api_documents_search_ordering_by_unknown_field():
|
||||
"loc": ["order_by"],
|
||||
"msg": (
|
||||
"Input should be 'relevance', 'title', 'created_at', "
|
||||
"'updated_at', 'size' or 'is_public'"
|
||||
"'updated_at', 'size' or 'reach'"
|
||||
),
|
||||
"type": "literal_error",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_api_documents_search_ordering_by_unknown_direction():
|
||||
@responses.activate
|
||||
def test_api_documents_search_ordering_by_unknown_direction(settings):
|
||||
"""Trying to sort with an unknown direction should return a 400 error"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
documents = factories.DocumentSchemaFactory.build_batch(2)
|
||||
prepare_index(service.index_name, documents)
|
||||
documents = factories.DocumentSchemaFactory.build_batch(
|
||||
2, reach=random.choice(["public", "authenticated"])
|
||||
)
|
||||
prepare_index(service.name, documents)
|
||||
|
||||
for field in enums.ORDER_BY_OPTIONS:
|
||||
response = APIClient().get(
|
||||
f"/api/v1.0/documents/?q=*&order_by={field}&order_direction=unknown",
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token}",
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{
|
||||
"q": "*",
|
||||
"order_by": field,
|
||||
"order_direction": "unknown",
|
||||
"visited": [doc["id"] for doc in documents],
|
||||
},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
@@ -269,87 +564,131 @@ def test_api_documents_search_ordering_by_unknown_direction():
|
||||
]
|
||||
|
||||
|
||||
def test_api_documents_search_filtering_by_is_public():
|
||||
"""It should be possible to filter results by their publication status"""
|
||||
@responses.activate
|
||||
def test_api_documents_search_filtering_by_reach(settings):
|
||||
"""It should be possible to filter results by their reach"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
documents = factories.DocumentSchemaFactory.build_batch(4)
|
||||
prepare_index(service.index_name, documents)
|
||||
documents = factories.DocumentSchemaFactory.build_batch(
|
||||
4, reach=random.choice(["public", "authenticated"])
|
||||
)
|
||||
prepare_index(service.name, documents)
|
||||
|
||||
parameters = [
|
||||
[True, "True"],
|
||||
[False, "False"],
|
||||
[True, "true"],
|
||||
[False, "false"],
|
||||
[True, "1"],
|
||||
[False, "0"],
|
||||
]
|
||||
|
||||
for public_boolean, public_string in parameters:
|
||||
response = APIClient().get(
|
||||
f"/api/v1.0/documents/?q=*&is_public={public_string}",
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token}",
|
||||
for reach in enums.ReachEnum:
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{
|
||||
"q": "*",
|
||||
"reach": reach.value,
|
||||
"visited": [doc["id"] for doc in documents],
|
||||
},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
responses = response.json()
|
||||
data = response.json()
|
||||
|
||||
for result in responses:
|
||||
assert public_boolean == result["_source"]["is_public"]
|
||||
for result in data:
|
||||
assert reach == result["_source"]["reach"]
|
||||
|
||||
|
||||
# Pagination
|
||||
|
||||
|
||||
def test_api_documents_search_pagination_basic():
|
||||
@responses.activate
|
||||
def test_api_documents_search_pagination_basic(settings):
|
||||
"""Pagination should correctly return documents for the specified page and page size"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
documents = factories.DocumentSchemaFactory.build_batch(9)
|
||||
documents = factories.DocumentSchemaFactory.build_batch(
|
||||
9, reach=random.choice(["public", "authenticated"])
|
||||
)
|
||||
ids = [str(doc["id"]) for doc in documents]
|
||||
prepare_index(service.index_name, documents)
|
||||
prepare_index(service.name, documents)
|
||||
|
||||
# Request the first page with a page size of 3
|
||||
response = APIClient().get(
|
||||
"/api/v1.0/documents/?q=*&page_number=1&page_size=3",
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{
|
||||
"q": "*",
|
||||
"page_number": 1,
|
||||
"page_size": 3,
|
||||
"visited": [doc["id"] for doc in documents],
|
||||
},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
responses = response.json()
|
||||
assert len(responses) == 3 # Page size is 3
|
||||
assert [r["_id"] for r in responses] == ids[0:3]
|
||||
data = response.json()
|
||||
assert len(data) == 3 # Page size is 3
|
||||
assert [r["_id"] for r in data] == ids[0:3]
|
||||
|
||||
# Request the second page with a page size of 3
|
||||
response = APIClient().get(
|
||||
"/api/v1.0/documents/?q=*&page_number=2&page_size=3",
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{
|
||||
"q": "*",
|
||||
"page_number": 2,
|
||||
"page_size": 3,
|
||||
"visited": [doc["id"] for doc in documents],
|
||||
},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
responses = response.json()
|
||||
assert len(responses) == 3
|
||||
assert [r["_id"] for r in responses] == ids[3:6]
|
||||
data = response.json()
|
||||
assert len(data) == 3
|
||||
assert [r["_id"] for r in data] == ids[3:6]
|
||||
|
||||
# Request the third page with a page size of 5 (should contain the remaining 3 documents)
|
||||
response = APIClient().get(
|
||||
"/api/v1.0/documents/?q=*&page_number=3&page_size=3",
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{
|
||||
"q": "*",
|
||||
"page_number": 3,
|
||||
"page_size": 3,
|
||||
"visited": [doc["id"] for doc in documents],
|
||||
},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
responses = response.json()
|
||||
assert len(responses) == 3
|
||||
assert [r["_id"] for r in responses] == ids[6:9]
|
||||
data = response.json()
|
||||
assert len(data) == 3
|
||||
assert [r["_id"] for r in data] == ids[6:9]
|
||||
|
||||
|
||||
def test_api_documents_search_pagination_last_page_edge_case():
|
||||
@responses.activate
|
||||
def test_api_documents_search_pagination_last_page_edge_case(settings):
|
||||
"""Requesting the last page should return the correct number of remaining documents"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
documents = factories.DocumentSchemaFactory.build_batch(8)
|
||||
documents = factories.DocumentSchemaFactory.build_batch(
|
||||
8, reach=random.choice(["public", "authenticated"])
|
||||
)
|
||||
ids = [str(doc["id"]) for doc in documents]
|
||||
prepare_index(service.index_name, documents)
|
||||
prepare_index(service.name, documents)
|
||||
|
||||
# Request the first page with a page size of 3
|
||||
response = APIClient().get(
|
||||
"/api/v1.0/documents/?q=*&page_number=1&page_size=3",
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{
|
||||
"q": "*",
|
||||
"page_number": 1,
|
||||
"page_size": 3,
|
||||
"visited": [doc["id"] for doc in documents],
|
||||
},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -357,9 +696,16 @@ def test_api_documents_search_pagination_last_page_edge_case():
|
||||
assert [r["_id"] for r in response.json()] == ids[0:3]
|
||||
|
||||
# Request the third page with a page size of 3 (should contain the last 1 document)
|
||||
response = APIClient().get(
|
||||
"/api/v1.0/documents/?q=*&page_number=3&page_size=3",
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{
|
||||
"q": "*",
|
||||
"page_number": 3,
|
||||
"page_size": 3,
|
||||
"visited": [doc["id"] for doc in documents],
|
||||
},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -367,29 +713,48 @@ def test_api_documents_search_pagination_last_page_edge_case():
|
||||
assert [r["_id"] for r in response.json()] == ids[6:]
|
||||
|
||||
|
||||
def test_api_documents_search_pagination_out_of_bounds():
|
||||
@responses.activate
|
||||
def test_api_documents_search_pagination_out_of_bounds(settings):
|
||||
"""
|
||||
Requesting a page number that exceeds the total number of pages should return an empty list
|
||||
"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
documents = factories.DocumentSchemaFactory.build_batch(4)
|
||||
prepare_index(service.index_name, documents)
|
||||
documents = factories.DocumentSchemaFactory.build_batch(
|
||||
4, reach=random.choice(["public", "authenticated"])
|
||||
)
|
||||
prepare_index(service.name, documents)
|
||||
|
||||
# Request the fourth page with a page size of 2 (there are only 2 pages)
|
||||
response = APIClient().get(
|
||||
"/api/v1.0/documents/?q=*&page_number=4&page_size=2",
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{
|
||||
"q": "*",
|
||||
"page_number": 4,
|
||||
"page_size": 2,
|
||||
"visited": [doc["id"] for doc in documents],
|
||||
},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert len(response.json()) == 0 # No documents should be returned
|
||||
|
||||
|
||||
def test_api_documents_search_pagination_invalid_parameters():
|
||||
@responses.activate
|
||||
def test_api_documents_search_pagination_invalid_parameters(settings):
|
||||
"""Invalid pagination parameters should result in a 400 error"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
documents = factories.DocumentSchemaFactory.build_batch(4)
|
||||
prepare_index(service.index_name, documents)
|
||||
documents = factories.DocumentSchemaFactory.build_batch(
|
||||
4, reach=random.choice(["public", "authenticated"])
|
||||
)
|
||||
prepare_index(service.name, documents)
|
||||
|
||||
parameters = [
|
||||
(
|
||||
@@ -411,9 +776,11 @@ def test_api_documents_search_pagination_invalid_parameters():
|
||||
]
|
||||
|
||||
for page_number, page_size, error_type, error_message in parameters:
|
||||
response = APIClient().get(
|
||||
f"/api/v1.0/documents/?q=*&page_number={page_number}&page_size={page_size}",
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token}",
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "*", "page_number": page_number, "page_size": page_size},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
@@ -421,27 +788,49 @@ def test_api_documents_search_pagination_invalid_parameters():
|
||||
assert response.data[0]["type"] == error_type
|
||||
|
||||
|
||||
def test_api_documents_search_pagination_with_filtering():
|
||||
"""Pagination should work correctly when combined with filtering by is_public"""
|
||||
@responses.activate
|
||||
def test_api_documents_search_pagination_with_filtering(settings):
|
||||
"""Pagination should work correctly when combined with filtering by reach"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
public_documents = factories.DocumentSchemaFactory.build_batch(3, is_public=True)
|
||||
public_documents = factories.DocumentSchemaFactory.build_batch(3, reach="public")
|
||||
public_ids = [str(doc["id"]) for doc in public_documents]
|
||||
private_documents = factories.DocumentSchemaFactory.build_batch(2, is_public=False)
|
||||
prepare_index(service.index_name, public_documents + private_documents)
|
||||
private_documents = factories.DocumentSchemaFactory.build_batch(
|
||||
2, reach="authenticated"
|
||||
)
|
||||
prepare_index(service.name, public_documents + private_documents)
|
||||
|
||||
# Filter by public documents, request first page
|
||||
response = APIClient().get(
|
||||
"/api/v1.0/documents/?q=*&is_public=true&page_number=1&page_size=2",
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{
|
||||
"q": "*",
|
||||
"reach": "public",
|
||||
"page_number": 1,
|
||||
"page_size": 2,
|
||||
"visited": public_ids,
|
||||
},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert len(response.json()) == 2
|
||||
assert [r["_id"] for r in response.json()] == public_ids[0:2]
|
||||
|
||||
# Request second page for public documents (remaining 1 document)
|
||||
response = APIClient().get(
|
||||
"/api/v1.0/documents/?q=*&is_public=true&page_number=2&page_size=2",
|
||||
HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{
|
||||
"q": "*",
|
||||
"reach": "public",
|
||||
"page_number": 2,
|
||||
"page_size": 2,
|
||||
"visited": public_ids,
|
||||
},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
@@ -0,0 +1,438 @@
|
||||
"""
|
||||
Test suite for access control when searching documents over the API.
|
||||
|
||||
Don't use pytest parametrized tests because batch generation and indexing
|
||||
of documents is slow and better done only once.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import responses
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from core import enums, factories
|
||||
|
||||
from .utils import (
|
||||
build_authorization_bearer,
|
||||
delete_test_indices,
|
||||
prepare_index,
|
||||
setup_oicd_resource_server,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
def test_api_documents_search_access_control_anonymous():
|
||||
"""Anonymous users should not be allowed to search documents even public."""
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
documents = []
|
||||
for reach in enums.ReachEnum:
|
||||
documents.extend(factories.DocumentSchemaFactory.build_batch(3, reach=reach))
|
||||
prepare_index(service.name, documents)
|
||||
|
||||
response = APIClient().post("/api/v1.0/documents/search/?q=*")
|
||||
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_access_control(settings):
|
||||
"""
|
||||
Authenticated users should only see documents:
|
||||
- for which they are listed in the "users" field
|
||||
- that have a reach set to "authenticated" or "public"
|
||||
- only configured services providers are allowed (e.g docs)
|
||||
(groups is not yet implemnted)
|
||||
"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service")
|
||||
documents_reach = factories.DocumentSchemaFactory.build_batch(6)
|
||||
documents_open = [
|
||||
doc for doc in documents_reach if doc["reach"] in ["authenticated", "public"]
|
||||
]
|
||||
documents_user = factories.DocumentSchemaFactory.build_batch(
|
||||
6, users=["user_sub", "user_sub2"]
|
||||
)
|
||||
expected_ids = [doc["id"] for doc in documents_open + documents_user]
|
||||
|
||||
prepare_index(service.name, documents_user + documents_reach)
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "*", "visited": [d["id"] for d in documents_open]},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert sorted([d["_id"] for d in response.json()]) == sorted(expected_ids)
|
||||
|
||||
|
||||
@responses.activate
|
||||
@pytest.mark.parametrize(
|
||||
"doc_ids,visited,expected",
|
||||
[
|
||||
(["a", "b"], [], []),
|
||||
(["a", "b"], "", []),
|
||||
(["a", "b"], None, []),
|
||||
(["a", "b"], ["other"], []),
|
||||
([], ["a"], []),
|
||||
(["a", "b"], ["a"], ["a"]),
|
||||
(["a", "b"], ["a", "b", "c"], ["a", "b"]),
|
||||
(["a", "b"], "a,b,c", ["a", "b"]),
|
||||
],
|
||||
)
|
||||
def test_api_documents_search_access__only_visited_public(
|
||||
doc_ids, visited, expected, settings
|
||||
):
|
||||
"""
|
||||
Authenticated users should only see documents with reach="public"
|
||||
that are in "visited" list.
|
||||
"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub", audience="docs")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service", client_id="docs")
|
||||
|
||||
docs = [
|
||||
factories.DocumentSchemaFactory(
|
||||
reach=[enums.ReachEnum.PUBLIC, enums.ReachEnum.AUTHENTICATED], id=doc_id
|
||||
)
|
||||
for doc_id in doc_ids
|
||||
]
|
||||
|
||||
prepare_index(service.name, docs)
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "*", "visited": visited},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.json()
|
||||
assert sorted([d["_id"] for d in response.json()]) == sorted(expected)
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_access__any_owner_public(settings):
|
||||
"""
|
||||
Authenticated users should only see documents with reach="public"
|
||||
that are in "visited" list.
|
||||
"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub", audience="docs")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service", client_id="docs")
|
||||
|
||||
docs = factories.DocumentSchemaFactory.build_batch(
|
||||
6,
|
||||
reach=enums.ReachEnum.PUBLIC,
|
||||
users=["user_sub"],
|
||||
)
|
||||
|
||||
other_docs = factories.DocumentSchemaFactory.build_batch(
|
||||
6,
|
||||
reach=enums.ReachEnum.PUBLIC,
|
||||
users=["other_sub"],
|
||||
)
|
||||
|
||||
prepare_index(service.name, docs + other_docs)
|
||||
|
||||
expected = [d["id"] for d in docs]
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "*", "visited": []},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.json()
|
||||
assert sorted([d["_id"] for d in response.json()]) == sorted(expected)
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_access__services(settings):
|
||||
"""
|
||||
Authenticated users should only see documents of audience
|
||||
service providers (e.g docs)
|
||||
"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub", audience="a-client")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service_a = factories.ServiceFactory(name="test-index-a", client_id="a-client")
|
||||
service_b = factories.ServiceFactory(name="test-index-b", client_id="b-client")
|
||||
|
||||
service_a_docs = factories.DocumentSchemaFactory.build_batch(
|
||||
3, reach=enums.ReachEnum.AUTHENTICATED, users=["user_sub"]
|
||||
)
|
||||
service_b_docs = factories.DocumentSchemaFactory.build_batch(
|
||||
3, reach=enums.ReachEnum.AUTHENTICATED, users=["user_sub"]
|
||||
)
|
||||
|
||||
expected_ids = [doc["id"] for doc in service_a_docs]
|
||||
|
||||
prepare_index(service_a.name, service_a_docs)
|
||||
prepare_index(service_b.name, service_b_docs, cleanup=False)
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "*"},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert sorted([d["_id"] for d in response.json()]) == sorted(expected_ids)
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_access__missing_index(settings):
|
||||
"""
|
||||
When the service has no opensearch index, returns an empty list.
|
||||
"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub", audience="a-client")
|
||||
token = build_authorization_bearer()
|
||||
factories.ServiceFactory(name="test-index-a", client_id="a-client")
|
||||
|
||||
delete_test_indices()
|
||||
|
||||
# a-client has no index. ignore it.
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "*"},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == []
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_access__related_services(settings):
|
||||
"""
|
||||
Authenticated users should only see documents of audience
|
||||
service providers and its related services (e.g drive + docs)
|
||||
"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub", audience="c-client")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service_a = factories.ServiceFactory(name="test-index-a", client_id="a-client")
|
||||
service_b = factories.ServiceFactory(name="test-index-b", client_id="b-client")
|
||||
service_c = factories.ServiceFactory(name="test-index-c", client_id="c-client")
|
||||
service_c.services.set([service_a])
|
||||
service_c.save()
|
||||
|
||||
service_a_docs = factories.DocumentSchemaFactory.build_batch(
|
||||
3, reach=enums.ReachEnum.AUTHENTICATED, users=["user_sub"]
|
||||
)
|
||||
service_b_docs = factories.DocumentSchemaFactory.build_batch(
|
||||
3, reach=enums.ReachEnum.AUTHENTICATED, users=["user_sub"]
|
||||
)
|
||||
service_c_docs = factories.DocumentSchemaFactory.build_batch(
|
||||
3, reach=enums.ReachEnum.AUTHENTICATED, users=["user_sub"]
|
||||
)
|
||||
|
||||
expected_ids = [doc["id"] for doc in service_a_docs + service_c_docs]
|
||||
|
||||
prepare_index(service_a.name, service_a_docs)
|
||||
prepare_index(service_b.name, service_b_docs, cleanup=False)
|
||||
prepare_index(service_c.name, service_c_docs, cleanup=False)
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "*"},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert sorted([d["_id"] for d in response.json()]) == sorted(expected_ids)
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_access__related_missing_index(settings):
|
||||
"""
|
||||
When the service has no opensearch index, returns the related services data.
|
||||
"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub", audience="a-client")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service_a = factories.ServiceFactory(name="test-index-a", client_id="a-client")
|
||||
service_b = factories.ServiceFactory(name="test-index-b", client_id="b-client")
|
||||
service_c = factories.ServiceFactory(name="test-index-c", client_id="c-client")
|
||||
service_c.services.set([service_a])
|
||||
service_c.save()
|
||||
|
||||
service_b_docs = factories.DocumentSchemaFactory.build_batch(
|
||||
3, reach=enums.ReachEnum.AUTHENTICATED, users=["user_sub"]
|
||||
)
|
||||
service_c_docs = factories.DocumentSchemaFactory.build_batch(
|
||||
3, reach=enums.ReachEnum.AUTHENTICATED, users=["user_sub"]
|
||||
)
|
||||
|
||||
expected_ids = [doc["id"] for doc in service_c_docs]
|
||||
|
||||
prepare_index(service_b.name, service_b_docs)
|
||||
prepare_index(service_c.name, service_c_docs, cleanup=False)
|
||||
|
||||
# a-client has no index. ignore it.
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "*"},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert sorted([d["_id"] for d in response.json()]) == sorted(expected_ids)
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_access__request_services(settings):
|
||||
"""
|
||||
Authenticated users should only see documents of audience
|
||||
from requested services : 'services' parameter.
|
||||
Raise 400 error if not all requested services are authorized.
|
||||
"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub", audience="c-client")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service_a = factories.ServiceFactory(name="test-index-a", client_id="a-client")
|
||||
service_b = factories.ServiceFactory(name="test-index-b", client_id="b-client")
|
||||
service_c = factories.ServiceFactory(name="test-index-c", client_id="c-client")
|
||||
|
||||
service_a_docs = factories.DocumentSchemaFactory.build_batch(
|
||||
3, reach=enums.ReachEnum.AUTHENTICATED, users=["user_sub"]
|
||||
)
|
||||
service_b_docs = factories.DocumentSchemaFactory.build_batch(
|
||||
3, reach=enums.ReachEnum.AUTHENTICATED, users=["user_sub"]
|
||||
)
|
||||
service_c_docs = factories.DocumentSchemaFactory.build_batch(
|
||||
3, reach=enums.ReachEnum.AUTHENTICATED, users=["user_sub"]
|
||||
)
|
||||
|
||||
expected_ids = [doc["id"] for doc in service_c_docs]
|
||||
|
||||
prepare_index(service_a.name, service_a_docs)
|
||||
prepare_index(service_b.name, service_b_docs, cleanup=False)
|
||||
prepare_index(service_c.name, service_c_docs, cleanup=False)
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "*", "services": ["test-index-c"]},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert sorted([d["_id"] for d in response.json()]) == sorted(expected_ids)
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "*", "services": ["test-index-c", "test-index-b"]},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json() == {"detail": "Some requested services are not available"}
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_access__request_inactive_services(settings):
|
||||
"""
|
||||
Authenticated users should only see documents of audience
|
||||
from requested services : 'services' parameter.
|
||||
Raise 400 error if not all requested services are active.
|
||||
"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub", audience="client")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
factories.ServiceFactory(name="test-index", client_id="client", is_active=False)
|
||||
factories.ServiceFactory(name="test-index-b", client_id="b-client")
|
||||
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "*", "services": ["test-index"]},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json() == {"detail": "Service is not available"}
|
||||
|
||||
# Event without explicit argument, the client service from the request is not active
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "*"},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json() == {"detail": "Service is not available"}
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_access__authenticated(settings):
|
||||
"""
|
||||
Authenticated users should only see documents
|
||||
- for which they are listed in the "users" field
|
||||
- that have a reach set to "authenticated" or "public" AND in visited list
|
||||
- only configured services providers are allowed (e.g docs)
|
||||
(groups is not yet implemnted)
|
||||
"""
|
||||
setup_oicd_resource_server(responses, settings, sub="user_sub", audience="docs")
|
||||
token = build_authorization_bearer()
|
||||
|
||||
service = factories.ServiceFactory(name="test-service", client_id="docs")
|
||||
|
||||
documents_open = factories.DocumentSchemaFactory.build_batch(
|
||||
2, reach=enums.ReachEnum.PUBLIC
|
||||
) + factories.DocumentSchemaFactory.build_batch(
|
||||
2, reach=enums.ReachEnum.AUTHENTICATED
|
||||
)
|
||||
|
||||
documents_restricted = factories.DocumentSchemaFactory.build_batch(
|
||||
2, reach=enums.ReachEnum.RESTRICTED
|
||||
)
|
||||
|
||||
documents_user = factories.DocumentSchemaFactory.build_batch(
|
||||
6, users=["user_sub", "user_sub2"]
|
||||
)
|
||||
documents = documents_user + documents_open + documents_restricted
|
||||
|
||||
prepare_index(service.name, documents_user + documents_open + documents_restricted)
|
||||
|
||||
# Only owned documents (reach is ignored)
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "*"},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
assert sorted([d["_id"] for d in response.json()]) == sorted(
|
||||
[doc["id"] for doc in documents_user]
|
||||
)
|
||||
|
||||
# Owned documents and visited public/authenticated ones.
|
||||
# Restricted ones from another owner are filtered (even if given as visited ones)
|
||||
response = APIClient().post(
|
||||
"/api/v1.0/documents/search/",
|
||||
{"q": "*", "visited": [d["id"] for d in documents]},
|
||||
format="json",
|
||||
HTTP_AUTHORIZATION=f"Bearer {token}",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
assert sorted([d["_id"] for d in response.json()]) == sorted(
|
||||
[doc["id"] for doc in documents_user + documents_open]
|
||||
)
|
||||
@@ -23,12 +23,6 @@ def test_models_services_name_slugified():
|
||||
assert service.name == "my-service-name"
|
||||
|
||||
|
||||
def test_models_services_index_name():
|
||||
"""The index name should be computed as a property from the service name."""
|
||||
service = factories.ServiceFactory(name="My service name")
|
||||
assert service.index_name == "find-my-service-name"
|
||||
|
||||
|
||||
def test_models_services_token_50_characters_exact():
|
||||
"""The token field should be 50 characters long."""
|
||||
service = factories.ServiceFactory()
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Test pydantic models & helpers"""
|
||||
|
||||
import pytest
|
||||
|
||||
from core.schemas import cleanlist
|
||||
|
||||
|
||||
def test_cleanlist_empty():
|
||||
"""Empty data should return an empty list"""
|
||||
assert cleanlist(None) == []
|
||||
assert cleanlist([]) == []
|
||||
assert cleanlist("") == []
|
||||
|
||||
|
||||
def test_cleanlist_error():
|
||||
"""Invalid data should raise"""
|
||||
with pytest.raises(ValueError):
|
||||
cleanlist(123)
|
||||
|
||||
|
||||
def test_cleanlist():
|
||||
"""Should return a list of stripped strings and remove the empty items"""
|
||||
assert cleanlist([1, 2, 3]) == ["1", "2", "3"]
|
||||
assert cleanlist(" 1, 2,3 ") == ["1", "2", "3"]
|
||||
assert cleanlist(["1 ", " 2", "3 "]) == ["1", "2", "3"]
|
||||
assert cleanlist([None, 2, 3, ""]) == ["2", "3"]
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Tests Service model for find's core app."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
from functools import partial
|
||||
from typing import List
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from joserfc import jwe as jose_jwe
|
||||
from joserfc import jwt as jose_jwt
|
||||
from joserfc.jwk import RSAKey
|
||||
from jwt.utils import to_base64url_uint
|
||||
from opensearchpy.helpers import bulk
|
||||
|
||||
from core import opensearch
|
||||
|
||||
|
||||
def delete_test_indices():
|
||||
"""Drop all search index containing the 'test' word"""
|
||||
opensearch.client.indices.delete(index="*test*")
|
||||
|
||||
|
||||
def prepare_index(index_name, documents: List, cleanup=True):
|
||||
"""Prepare the search index before testing a query on it."""
|
||||
if cleanup:
|
||||
delete_test_indices()
|
||||
|
||||
opensearch.ensure_index_exists(index_name)
|
||||
|
||||
# Index new documents
|
||||
actions = [
|
||||
{
|
||||
"_op_type": "index",
|
||||
"_index": index_name,
|
||||
"_id": doc["id"],
|
||||
"_source": {k: v for k, v in doc.items() if k != "id"},
|
||||
}
|
||||
for doc in documents
|
||||
]
|
||||
bulk(opensearch.client, actions)
|
||||
|
||||
# Force refresh again so all changes are visible to search
|
||||
opensearch.client.indices.refresh(index=index_name)
|
||||
|
||||
count = opensearch.client.count(index=index_name)["count"]
|
||||
assert count == len(documents), f"Expected {len(documents)}, got {count}"
|
||||
|
||||
|
||||
def build_authorization_bearer(token="some_token"):
|
||||
"""
|
||||
Build an Authorization Bearer header value from a token.
|
||||
|
||||
This can be used like this:
|
||||
client.post(
|
||||
...
|
||||
HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer('some_token')}",
|
||||
)
|
||||
"""
|
||||
return base64.b64encode(token.encode("utf-8")).decode("utf-8")
|
||||
|
||||
|
||||
def setup_oicd_jwt_resource_server(
|
||||
responses,
|
||||
settings,
|
||||
sub="some_sub",
|
||||
audience="some_client_id",
|
||||
):
|
||||
"""
|
||||
Setup settings for a resource server with JWT backend.
|
||||
Simulate an encrypted token introspection.
|
||||
NOTE : Use it with @responses.activate or the fake introspection view will not work.
|
||||
"""
|
||||
token_data = {
|
||||
"sub": sub,
|
||||
"iss": "https://oidc.example.com",
|
||||
"aud": audience,
|
||||
"client_id": "some_service_provider",
|
||||
"scope": "docs",
|
||||
"active": True,
|
||||
}
|
||||
|
||||
private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
|
||||
unencrypted_pem_private_key = private_key.private_bytes(
|
||||
encoding=serialization.Encoding.PEM,
|
||||
format=serialization.PrivateFormat.TraditionalOpenSSL,
|
||||
encryption_algorithm=serialization.NoEncryption(),
|
||||
)
|
||||
|
||||
pem_public_key = private_key.public_key().public_bytes(
|
||||
encoding=serialization.Encoding.PEM,
|
||||
format=serialization.PublicFormat.SubjectPublicKeyInfo,
|
||||
)
|
||||
|
||||
settings.OIDC_RS_PRIVATE_KEY_STR = unencrypted_pem_private_key.decode("utf-8")
|
||||
settings.OIDC_RS_ENCRYPTION_KEY_TYPE = "RSA"
|
||||
settings.OIDC_RS_ENCRYPTION_ENCODING = "A256GCM"
|
||||
settings.OIDC_RS_ENCRYPTION_ALGO = "RSA-OAEP"
|
||||
settings.OIDC_RS_SIGNING_ALGO = "RS256"
|
||||
settings.OIDC_RS_CLIENT_ID = audience
|
||||
settings.OIDC_RS_CLIENT_SECRET = "some_client_secret"
|
||||
settings.OIDC_RS_SCOPES = ["openid", "docs", "email"]
|
||||
|
||||
settings.OIDC_OP_URL = "https://oidc.example.com"
|
||||
settings.OIDC_OP_JWKS_ENDPOINT = "https://oidc.example.com/jwks"
|
||||
settings.OIDC_OP_INTROSPECTION_ENDPOINT = "https://oidc.example.com/introspect"
|
||||
|
||||
settings.OIDC_VERIFY_SSL = False
|
||||
settings.OIDC_TIMEOUT = 5
|
||||
settings.OIDC_PROXY = None
|
||||
settings.OIDC_CREATE_USER = False
|
||||
|
||||
# Mock the JWKS endpoint
|
||||
public_numbers = private_key.public_key().public_numbers()
|
||||
responses.add(
|
||||
responses.GET,
|
||||
settings.OIDC_OP_JWKS_ENDPOINT,
|
||||
body=json.dumps(
|
||||
{
|
||||
"keys": [
|
||||
{
|
||||
"kty": settings.OIDC_RS_ENCRYPTION_KEY_TYPE,
|
||||
"alg": settings.OIDC_RS_SIGNING_ALGO,
|
||||
"use": "sig",
|
||||
"kid": "1234567890",
|
||||
"n": to_base64url_uint(public_numbers.n).decode("ascii"),
|
||||
"e": to_base64url_uint(public_numbers.e).decode("ascii"),
|
||||
}
|
||||
]
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
def encrypt_jwt(json_data):
|
||||
"""Encrypt the JWT token for the backend to decrypt."""
|
||||
token = jose_jwt.encode(
|
||||
{
|
||||
"kid": "1234567890",
|
||||
"alg": settings.OIDC_RS_SIGNING_ALGO,
|
||||
},
|
||||
json_data,
|
||||
RSAKey.import_key(unencrypted_pem_private_key),
|
||||
algorithms=[settings.OIDC_RS_SIGNING_ALGO],
|
||||
)
|
||||
|
||||
return jose_jwe.encrypt_compact(
|
||||
protected={
|
||||
"alg": settings.OIDC_RS_ENCRYPTION_ALGO,
|
||||
"enc": settings.OIDC_RS_ENCRYPTION_ENCODING,
|
||||
},
|
||||
plaintext=token,
|
||||
public_key=RSAKey.import_key(pem_public_key),
|
||||
algorithms=[
|
||||
settings.OIDC_RS_ENCRYPTION_ALGO,
|
||||
settings.OIDC_RS_ENCRYPTION_ENCODING,
|
||||
],
|
||||
)
|
||||
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"https://oidc.example.com/introspect",
|
||||
body=encrypt_jwt(
|
||||
{
|
||||
"iss": "https://oidc.example.com",
|
||||
"aud": audience, # settings.OIDC_RS_CLIENT_ID
|
||||
"token_introspection": token_data,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def setup_oicd_resource_server( # noqa : PLR0913
|
||||
responses,
|
||||
settings,
|
||||
sub="some_sub",
|
||||
audience="some_client_id",
|
||||
iss="https://oidc.example.com",
|
||||
introspect=None,
|
||||
): # pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
"""
|
||||
Setup settings for a resource server.
|
||||
Simulate a token introspection.
|
||||
NOTE : Use it with @responses.activate or the fake introspection view will not work.
|
||||
"""
|
||||
token_data = {
|
||||
"sub": sub,
|
||||
"iss": iss,
|
||||
"aud": audience,
|
||||
"client_id": audience,
|
||||
"scope": "docs",
|
||||
"active": True,
|
||||
}
|
||||
|
||||
settings.OIDC_RS_ENCRYPTION_KEY_TYPE = "RSA"
|
||||
settings.OIDC_RS_ENCRYPTION_ENCODING = "A256GCM"
|
||||
settings.OIDC_RS_ENCRYPTION_ALGO = "RSA-OAEP"
|
||||
settings.OIDC_RS_SIGNING_ALGO = "RS256"
|
||||
settings.OIDC_RS_CLIENT_ID = audience
|
||||
settings.OIDC_RS_CLIENT_SECRET = "some_client_secret"
|
||||
settings.OIDC_RS_SCOPES = ["openid", "docs", "email"]
|
||||
|
||||
settings.OIDC_OP_URL = "https://oidc.example.com"
|
||||
settings.OIDC_OP_INTROSPECTION_ENDPOINT = "https://oidc.example.com/introspect"
|
||||
|
||||
settings.OIDC_VERIFY_SSL = False
|
||||
settings.OIDC_TIMEOUT = 5
|
||||
settings.OIDC_PROXY = None
|
||||
settings.OIDC_CREATE_USER = False
|
||||
|
||||
if callable(introspect):
|
||||
responses.add_callback(
|
||||
responses.POST,
|
||||
"https://oidc.example.com/introspect",
|
||||
callback=partial(introspect, user_info=token_data),
|
||||
)
|
||||
else:
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"https://oidc.example.com/introspect",
|
||||
body=json.dumps(token_data),
|
||||
)
|
||||
@@ -1,9 +1,11 @@
|
||||
"""URL configuration for find's core app."""
|
||||
|
||||
from django.urls import path
|
||||
from django.urls import include, path
|
||||
|
||||
from .views import DocumentView
|
||||
from .views import IndexDocumentView, SearchDocumentView
|
||||
|
||||
urlpatterns = [
|
||||
path("documents/", DocumentView.as_view(), name="document"),
|
||||
path("documents/index/", IndexDocumentView.as_view(), name="document"),
|
||||
path("documents/search/", SearchDocumentView.as_view(), name="document"),
|
||||
path("", include("lasuite.oidc_resource_server.urls")),
|
||||
]
|
||||
|
||||
+151
-62
@@ -1,43 +1,35 @@
|
||||
"""Views for find's core app."""
|
||||
|
||||
import logging
|
||||
|
||||
from django.core.exceptions import SuspiciousOperation
|
||||
|
||||
from lasuite.oidc_resource_server.authentication import ResourceServerAuthentication
|
||||
from lasuite.oidc_resource_server.mixins import ResourceServerMixin
|
||||
from pydantic import ValidationError as PydanticValidationError
|
||||
from rest_framework import status, views
|
||||
from rest_framework.response import Response
|
||||
from urllib3.exceptions import ReadTimeoutError
|
||||
|
||||
from . import enums, schemas
|
||||
from .authentication import ServiceTokenAuthentication
|
||||
from .models import Service
|
||||
from .opensearch import client, ensure_index_exists
|
||||
from .permissions import IsAuthAuthenticated
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class DocumentView(views.APIView):
|
||||
|
||||
class IndexDocumentView(views.APIView):
|
||||
"""
|
||||
API view for managing documents in an OpenSearch index.
|
||||
|
||||
This view provides functionality for both indexing and searching documents
|
||||
within an OpenSearch index dedicated to the authenticated service. The class
|
||||
supports the following operations:
|
||||
|
||||
1. **Document Indexing (POST)**:
|
||||
API view for indexing documents in OpenSearch.
|
||||
- Handles both single document and bulk document indexing.
|
||||
- The index is dynamically determined based on the service authentication token,
|
||||
ensuring that each service has its own isolated index.
|
||||
|
||||
2. **Document Search (GET)**:
|
||||
- Enables searching through indexed documents with support for various filters
|
||||
and sorting options.
|
||||
- The search results can be sorted or filtered via querystring parameters.
|
||||
ensuring that each service has its own isolated index.
|
||||
"""
|
||||
|
||||
authentication_classes = [ServiceTokenAuthentication]
|
||||
permission_classes = [IsAuthAuthenticated]
|
||||
|
||||
@property
|
||||
def index_name(self):
|
||||
"""Compute index name from the service name extracted during authentication"""
|
||||
return f"find-{self.request.auth}"
|
||||
|
||||
# pylint: disable=too-many-locals
|
||||
def post(self, request, *args, **kwargs):
|
||||
"""
|
||||
@@ -87,6 +79,7 @@ class DocumentView(views.APIView):
|
||||
- Returns a list of results for all documents, with details of success and indexing
|
||||
errors.
|
||||
"""
|
||||
index_name = request.auth.name
|
||||
|
||||
if isinstance(request.data, list):
|
||||
# Bulk indexing several documents
|
||||
@@ -110,11 +103,14 @@ class DocumentView(views.APIView):
|
||||
actions.append({"index": {"_id": _id}})
|
||||
actions.append(document_dict)
|
||||
results.append({"index": i, "_id": _id, "status": "valid"})
|
||||
|
||||
if has_errors:
|
||||
return Response(results, status=status.HTTP_400_BAD_REQUEST)
|
||||
|
||||
ensure_index_exists(self.index_name)
|
||||
response = client.bulk(index=self.index_name, body=actions)
|
||||
# Build index if needed.
|
||||
ensure_index_exists(index_name)
|
||||
|
||||
response = client.bulk(index=index_name, body=actions)
|
||||
for i, item in enumerate(response["items"]):
|
||||
if item["index"]["status"] != 201:
|
||||
results[i]["status"] = "error"
|
||||
@@ -124,39 +120,94 @@ class DocumentView(views.APIView):
|
||||
else:
|
||||
results[i]["status"] = "success"
|
||||
|
||||
return Response(results, status=status.HTTP_207_MULTI_STATUS)
|
||||
return Response(results, status=status.HTTP_201_CREATED)
|
||||
|
||||
# Indexing a single document
|
||||
document = schemas.DocumentSchema(**request.data)
|
||||
document_dict = document.model_dump()
|
||||
_id = document_dict.pop("id")
|
||||
try:
|
||||
client.index(index=self.index_name, body=document_dict, id=_id)
|
||||
except ReadTimeoutError:
|
||||
ensure_index_exists(self.index_name)
|
||||
client.index(index=self.index_name, body=document_dict, id=_id)
|
||||
|
||||
# Build index if needed.
|
||||
ensure_index_exists(index_name)
|
||||
|
||||
client.index(index=index_name, body=document_dict, id=_id)
|
||||
|
||||
return Response(
|
||||
{"status": "created", "_id": _id}, status=status.HTTP_201_CREATED
|
||||
)
|
||||
|
||||
def get(self, request, *args, **kwargs):
|
||||
|
||||
class SearchDocumentView(ResourceServerMixin, views.APIView):
|
||||
"""
|
||||
API view for searching documents in OpenSearch.
|
||||
- Enables searching through indexed documents with support for various filters
|
||||
and sorting options.
|
||||
- The search results can be sorted or filtered via querystring parameters.
|
||||
"""
|
||||
|
||||
authentication_classes = [ResourceServerAuthentication]
|
||||
permission_classes = [IsAuthAuthenticated]
|
||||
|
||||
def _get_opensearch_indices(self, audience, services):
|
||||
# Get request user service
|
||||
try:
|
||||
user_service = Service.objects.get(client_id=audience, is_active=True)
|
||||
except Service.DoesNotExist as e:
|
||||
logger.warning("Login failed: No service %s found", audience)
|
||||
raise SuspiciousOperation("Service is not available") from e
|
||||
|
||||
# Find allowed sub-services for this service
|
||||
allowed_services = set(user_service.services.values_list("name", flat=True))
|
||||
allowed_services.add(user_service.name)
|
||||
|
||||
if services:
|
||||
available = set(services).intersection(allowed_services)
|
||||
|
||||
if len(available) < len(services):
|
||||
raise SuspiciousOperation("Some requested services are not available")
|
||||
|
||||
return allowed_services
|
||||
|
||||
def _fulltext_search_query(self, text):
|
||||
if text == "*":
|
||||
return {"match_all": {}}
|
||||
|
||||
return {
|
||||
"bool": {
|
||||
"should": [
|
||||
{
|
||||
"multi_match": {
|
||||
"query": text,
|
||||
# Give title more importance over content by a power of 3
|
||||
"fields": ["title.text^3", "content"],
|
||||
# Add extra score when multiple keywords are in the field
|
||||
"tie_breaker": 0.3,
|
||||
},
|
||||
},
|
||||
{"match_bool_prefix": {"title.text": text}},
|
||||
{"match_bool_prefix": {"content": text}},
|
||||
],
|
||||
"minimum_should_match": 1,
|
||||
}
|
||||
}
|
||||
|
||||
def post(self, request, *args, **kwargs):
|
||||
"""
|
||||
Handle GET requests to perform a search on indexed documents with optional filtering
|
||||
Handle POST requests to perform a search on indexed documents with optional filtering
|
||||
and ordering.
|
||||
|
||||
The search query should be provided as a query parameter 'q'. The method constructs a
|
||||
The search query should be provided as a "q" parameter. The method constructs a
|
||||
search request to OpenSearch using the specified query, with the option to filter by
|
||||
'is_public' and order by 'relevance', 'created_at', 'updated_at', or 'size'.
|
||||
'reach' and order by 'relevance', 'created_at', 'updated_at', or 'size'.
|
||||
The results are further filtered by 'users' and 'groups' based on the authentication
|
||||
header.
|
||||
|
||||
Query Parameters:
|
||||
Body Parameters:
|
||||
---------------
|
||||
q : str
|
||||
The search query string. This is a required parameter.
|
||||
is_public : bool, optional
|
||||
Filter results based on the 'is_public' field.
|
||||
reach : str, optional
|
||||
Filter results based on the 'reach' field.
|
||||
order_by : str, optional
|
||||
Order results by 'relevance', 'created_at', 'updated_at', or 'size'.
|
||||
Defaults to 'relevance' if not specified.
|
||||
@@ -169,6 +220,12 @@ class DocumentView(views.APIView):
|
||||
page_size : int, optional
|
||||
The number of results to return per page.
|
||||
Defaults to 50 if not specified.
|
||||
services: List[str], optional
|
||||
List of services on which we intend to run the query (current service if left empty)
|
||||
visited: List[sub], optional
|
||||
List of public/authenticated documents the user has visited to limit
|
||||
the document returned to the ones the current user has seen.
|
||||
Built from linkreach list of a document in docs app.
|
||||
|
||||
Returns:
|
||||
--------
|
||||
@@ -176,13 +233,27 @@ class DocumentView(views.APIView):
|
||||
- 200 OK: Returns a list of search results matching the query.
|
||||
- 400 Bad Request: If the query parameter 'q' is not provided or invalid.
|
||||
"""
|
||||
# Get list of groups related to the user from SCIM provider (consider caching result)
|
||||
audience = self._get_service_provider_audience()
|
||||
user_sub = self.request.user.sub
|
||||
groups = []
|
||||
# //////////////////////////////////////////////////
|
||||
|
||||
# Extract and validate query parameters using Pydantic schema
|
||||
params = schemas.SearchQueryParametersSchema(**request.GET)
|
||||
params = schemas.SearchQueryParametersSchema(**request.data)
|
||||
|
||||
# Compute pagination parameters
|
||||
from_value = (params.page_number - 1) * params.page_size
|
||||
size_value = params.page_size
|
||||
|
||||
# Get index list for search query
|
||||
try:
|
||||
search_indices = self._get_opensearch_indices(
|
||||
audience, services=params.services
|
||||
)
|
||||
except SuspiciousOperation as e:
|
||||
return Response({"detail": str(e)}, status=status.HTTP_400_BAD_REQUEST)
|
||||
|
||||
# Prepare the search query
|
||||
search_body = {
|
||||
"_source": enums.SOURCE_FIELDS, # limit the fields to return
|
||||
@@ -197,18 +268,9 @@ class DocumentView(views.APIView):
|
||||
}
|
||||
|
||||
# Adding the text query
|
||||
if params.q == "*":
|
||||
search_body["query"]["bool"]["must"].append({"match_all": {}})
|
||||
else:
|
||||
search_body["query"]["bool"]["must"].append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": params.q,
|
||||
# Give title more importance over content by a power of 3
|
||||
"fields": ["title.text^3", "content"],
|
||||
}
|
||||
}
|
||||
)
|
||||
search_body["query"]["bool"]["must"].append(
|
||||
self._fulltext_search_query(params.q)
|
||||
)
|
||||
|
||||
# Add sorting logic based on relevance or specified field
|
||||
if params.order_by == enums.RELEVANCE:
|
||||
@@ -218,23 +280,50 @@ class DocumentView(views.APIView):
|
||||
{params.order_by: {"order": params.order_direction}}
|
||||
)
|
||||
|
||||
# Filter by is_public if provided
|
||||
if params.is_public is not None:
|
||||
# Apply access control based on documents reach
|
||||
search_body["query"]["bool"]["must"].append(
|
||||
{
|
||||
"bool": {
|
||||
"should": [
|
||||
# Access control on public & authenticated reach
|
||||
{
|
||||
"bool": {
|
||||
"must_not": {
|
||||
"term": {enums.REACH: enums.ReachEnum.RESTRICTED},
|
||||
},
|
||||
# Limit search to already visited documents.
|
||||
"must": {
|
||||
"terms": {
|
||||
"_id": sorted(params.visited),
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
# Access control on restricted search : either user or group should match
|
||||
{"term": {enums.USERS: user_sub}},
|
||||
{"terms": {enums.GROUPS: groups}},
|
||||
],
|
||||
# At least one of the 2 optional should clauses must apply
|
||||
"minimum_should_match": 1,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
# Optional filter by reach if explicitly provided in the query
|
||||
if params.reach is not None:
|
||||
search_body["query"]["bool"]["filter"].append(
|
||||
{"term": {enums.IS_PUBLIC: params.is_public}}
|
||||
{"term": {enums.REACH: params.reach}}
|
||||
)
|
||||
|
||||
# Filter by users and groups based on authentication$
|
||||
# user = request.user
|
||||
# groups = user.get_teams()
|
||||
# Always filter out inactive documents
|
||||
search_body["query"]["bool"]["filter"].append({"term": {"is_active": True}})
|
||||
|
||||
# search_body['query']['bool']['filter'].append({
|
||||
# "terms": {"users": [user.sub]}
|
||||
# })
|
||||
response = client.search( # pylint: disable=unexpected-keyword-arg
|
||||
index=",".join(search_indices),
|
||||
body=search_body,
|
||||
# Argument added by the query_params() decorator of opensearch and
|
||||
# not in the method declaration.
|
||||
ignore_unavailable=True,
|
||||
)
|
||||
|
||||
# search_body['query']['bool']['filter'].append({
|
||||
# "terms": {"groups": list(groups)}
|
||||
# })
|
||||
|
||||
response = client.search(index=self.index_name, body=search_body)
|
||||
return Response(response["hits"]["hits"], status=status.HTTP_200_OK)
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
"""Parameters that define how the demo site will be built."""
|
||||
|
||||
NB_OBJECTS = {
|
||||
"documents": 1000,
|
||||
"services": 5,
|
||||
}
|
||||
NB_OBJECTS = {"documents": 1000, "services": 5}
|
||||
|
||||
DEV_SERVICES = (
|
||||
{
|
||||
"name": "docs",
|
||||
"client_id": "impress",
|
||||
"token": "find-api-key-for-docs-with-exactly-50-chars-length",
|
||||
},
|
||||
)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# ruff: noqa: S311, S106
|
||||
# ruff: noqa: S311
|
||||
"""create_demo management command"""
|
||||
|
||||
import logging
|
||||
@@ -14,7 +14,7 @@ from django.utils.text import slugify
|
||||
from faker import Faker
|
||||
from opensearchpy.helpers import bulk
|
||||
|
||||
from core import factories, opensearch
|
||||
from core import enums, factories, opensearch
|
||||
|
||||
from demo import defaults
|
||||
|
||||
@@ -133,7 +133,7 @@ def generate_document():
|
||||
"size": random.randint(0, 100 * 1024**2),
|
||||
"users": [str(uuid4()) for _ in range(3)],
|
||||
"groups": [slugify(fake.word()) for _ in range(3)],
|
||||
"is_public": fake.boolean(),
|
||||
"reach": random.choice(list(enums.ReachEnum)).value,
|
||||
}
|
||||
|
||||
|
||||
@@ -147,24 +147,31 @@ def create_demo(stdout):
|
||||
services = factories.ServiceFactory.create_batch(
|
||||
defaults.NB_OBJECTS["services"]
|
||||
)
|
||||
|
||||
for service in services:
|
||||
opensearch.ensure_index_exists(service.index_name)
|
||||
opensearch.client.indices.refresh(index=service.index_name)
|
||||
opensearch.ensure_index_exists(service.name)
|
||||
opensearch.client.indices.refresh(index=service.name)
|
||||
|
||||
with Timeit(stdout, "Creating documents"):
|
||||
actions = BulkIndexing(stdout)
|
||||
for _ in range(defaults.NB_OBJECTS["documents"]):
|
||||
service = random.choice(services)
|
||||
document = generate_document()
|
||||
actions.push(service.index_name, uuid4(), document)
|
||||
actions.push(service.name, uuid4(), document)
|
||||
actions.flush()
|
||||
|
||||
with Timeit(stdout, "Creating dev services"):
|
||||
for conf in defaults.DEV_SERVICES:
|
||||
service = factories.ServiceFactory(**conf)
|
||||
opensearch.ensure_index_exists(service.name)
|
||||
opensearch.client.indices.refresh(index=service.name)
|
||||
|
||||
# Check and report on indexed documents
|
||||
total_indexed = 0
|
||||
for service in services:
|
||||
opensearch.client.indices.refresh(index=service.index_name)
|
||||
indexed = opensearch.client.count(index=service.index_name)["count"]
|
||||
stdout.write(f" - {service.index_name:s}: {indexed:d} documents")
|
||||
opensearch.client.indices.refresh(index=service.name)
|
||||
indexed = opensearch.client.count(index=service.name)["count"]
|
||||
stdout.write(f" - {service.name:s}: {indexed:d} documents")
|
||||
total_indexed += indexed
|
||||
|
||||
stdout.write(f" TOTAL: {total_indexed:d} documents")
|
||||
|
||||
@@ -25,5 +25,8 @@ def test_commands_create_demo():
|
||||
"""The create_demo management command should create objects as expected."""
|
||||
call_command("create_demo")
|
||||
|
||||
assert models.Service.objects.count() == 2
|
||||
assert models.Service.objects.exclude(name="docs").count() == 2
|
||||
assert opensearch.client.count()["count"] == 4
|
||||
|
||||
docs = models.Service.objects.get(name="docs")
|
||||
assert docs.client_id == "impress"
|
||||
|
||||
+106
-35
@@ -296,9 +296,13 @@ class Base(Configuration):
|
||||
environ_name="OIDC_RP_CLIENT_SECRET",
|
||||
environ_prefix=None,
|
||||
)
|
||||
OIDC_OP_URL = values.Value(None, environ_name="OIDC_OP_URL", environ_prefix=None)
|
||||
OIDC_OP_JWKS_ENDPOINT = values.Value(
|
||||
environ_name="OIDC_OP_JWKS_ENDPOINT", environ_prefix=None
|
||||
)
|
||||
OIDC_OP_INTROSPECTION_ENDPOINT = values.Value(
|
||||
environ_name="OIDC_OP_INTROSPECTION_ENDPOINT", environ_prefix=None
|
||||
)
|
||||
OIDC_OP_AUTHORIZATION_ENDPOINT = values.Value(
|
||||
environ_name="OIDC_OP_AUTHORIZATION_ENDPOINT", environ_prefix=None
|
||||
)
|
||||
@@ -317,6 +321,69 @@ class Base(Configuration):
|
||||
OIDC_RP_SCOPES = values.Value(
|
||||
"openid email", environ_name="OIDC_RP_SCOPES", environ_prefix=None
|
||||
)
|
||||
|
||||
# OIDC - Resource server
|
||||
OIDC_DRF_AUTH_BACKEND = values.Value(
|
||||
"lasuite.oidc_login.backends.OIDCAuthenticationBackend",
|
||||
environ_name="OIDC_DRF_AUTH_BACKEND",
|
||||
environ_prefix=None,
|
||||
)
|
||||
|
||||
OIDC_RS_BACKEND_CLASS = values.Value(
|
||||
"core.authentication.FinderResourceServerBackend",
|
||||
environ_name="OIDC_RS_BACKEND_CLASS",
|
||||
environ_prefix=None,
|
||||
)
|
||||
OIDC_RS_AUDIENCE_CLAIM = values.Value(
|
||||
"client_id",
|
||||
environ_name="OIDC_RS_AUDIENCE_CLAIM",
|
||||
environ_prefix=None,
|
||||
)
|
||||
OIDC_RS_CLIENT_ID = values.Value(
|
||||
None, environ_name="OIDC_RS_CLIENT_ID", environ_prefix=None
|
||||
)
|
||||
OIDC_RS_CLIENT_SECRET = values.Value(
|
||||
None,
|
||||
environ_name="OIDC_RS_CLIENT_SECRET",
|
||||
environ_prefix=None,
|
||||
)
|
||||
OIDC_RS_SIGNING_ALGO = values.Value(
|
||||
default="ES256", environ_name="OIDC_RS_SIGNING_ALGO", environ_prefix=None
|
||||
)
|
||||
OIDC_RS_SCOPES = values.ListValue(
|
||||
["openid"], environ_name="OIDC_RS_SCOPES", environ_prefix=None
|
||||
)
|
||||
OIDC_RS_PRIVATE_KEY_STR = values.Value(
|
||||
default=None,
|
||||
environ_name="OIDC_RS_PRIVATE_KEY_STR",
|
||||
environ_prefix=None,
|
||||
)
|
||||
OIDC_RS_ENCRYPTION_KEY_TYPE = values.Value(
|
||||
default="RSA",
|
||||
environ_name="OIDC_RS_ENCRYPTION_KEY_TYPE",
|
||||
environ_prefix=None,
|
||||
)
|
||||
OIDC_RS_ENCRYPTION_ALGO = values.Value(
|
||||
default="RSA-OAEP",
|
||||
environ_name="OIDC_RS_ENCRYPTION_ALGO",
|
||||
environ_prefix=None,
|
||||
)
|
||||
OIDC_RS_ENCRYPTION_ENCODING = values.Value(
|
||||
default="A256GCM",
|
||||
environ_name="OIDC_RS_ENCRYPTION_ENCODING",
|
||||
environ_prefix=None,
|
||||
)
|
||||
# In production mode we must check the token claims
|
||||
OIDC_RS_VERIFY_CLAIMS = values.BooleanValue(
|
||||
default=True, environ_name="OIDC_RS_VERIFY_CLAIMS", environ_prefix=None
|
||||
)
|
||||
|
||||
OIDC_VERIFY_SSL = values.BooleanValue(
|
||||
True, environ_name="OIDC_VERIFY_SSL", environ_prefix=None
|
||||
)
|
||||
OIDC_TIMEOUT = values.Value(None, environ_name="OIDC_TIMEOUT", environ_prefix=None)
|
||||
OIDC_PROXY = values.Value(None, environ_name="OIDC_PROXY", environ_prefix=None)
|
||||
|
||||
LOGIN_REDIRECT_URL = values.Value(
|
||||
None, environ_name="LOGIN_REDIRECT_URL", environ_prefix=None
|
||||
)
|
||||
@@ -341,6 +408,43 @@ class Base(Configuration):
|
||||
ALLOW_LOGOUT_GET_METHOD = values.BooleanValue(
|
||||
default=True, environ_name="ALLOW_LOGOUT_GET_METHOD", environ_prefix=None
|
||||
)
|
||||
# Logging
|
||||
# We want to make it easy to log to console but by default we log production
|
||||
# to Sentry and don't want to log to console.
|
||||
LOGGING = {
|
||||
"version": 1,
|
||||
"disable_existing_loggers": False,
|
||||
"formatters": {
|
||||
"simple": {
|
||||
"format": "{asctime} {name} {levelname} {message}",
|
||||
"style": "{",
|
||||
},
|
||||
},
|
||||
"handlers": {
|
||||
"console": {
|
||||
"class": "logging.StreamHandler",
|
||||
"formatter": "simple",
|
||||
},
|
||||
},
|
||||
# Override root logger to send it to console
|
||||
"root": {
|
||||
"handlers": ["console"],
|
||||
"level": values.Value(
|
||||
"INFO", environ_name="LOGGING_LEVEL_LOGGERS_ROOT", environ_prefix=None
|
||||
),
|
||||
},
|
||||
"loggers": {
|
||||
"find": {
|
||||
"handlers": ["console"],
|
||||
"level": values.Value(
|
||||
"INFO",
|
||||
environ_name="LOGGING_LEVEL_LOGGERS_APP",
|
||||
environ_prefix=None,
|
||||
),
|
||||
"propagate": True,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
# pylint: disable=invalid-name
|
||||
@property
|
||||
@@ -423,23 +527,6 @@ class Development(Base):
|
||||
class Test(Base):
|
||||
"""Test environment settings"""
|
||||
|
||||
LOGGING = values.DictValue(
|
||||
{
|
||||
"version": 1,
|
||||
"disable_existing_loggers": False,
|
||||
"handlers": {
|
||||
"console": {
|
||||
"class": "logging.StreamHandler",
|
||||
},
|
||||
},
|
||||
"loggers": {
|
||||
"find": {
|
||||
"handlers": ["console"],
|
||||
"level": "DEBUG",
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
PASSWORD_HASHERS = [
|
||||
"django.contrib.auth.hashers.MD5PasswordHasher",
|
||||
]
|
||||
@@ -511,24 +598,8 @@ class Production(Base):
|
||||
},
|
||||
}
|
||||
|
||||
LOGGING = values.DictValue(
|
||||
{
|
||||
"version": 1,
|
||||
"disable_existing_loggers": False,
|
||||
"handlers": {
|
||||
"console": {
|
||||
"class": "logging.StreamHandler",
|
||||
},
|
||||
},
|
||||
"loggers": {
|
||||
"find": {
|
||||
"handlers": ["console"],
|
||||
"level": "INFO",
|
||||
"propagate": True,
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
# In production mode we must check the token claims
|
||||
OIDC_RS_VERIFY_CLAIMS = True
|
||||
|
||||
|
||||
class Feature(Production):
|
||||
|
||||
+21
-20
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "find"
|
||||
version = "1.1.0"
|
||||
version = "0.0.1"
|
||||
authors = [{ "name" = "DINUM", "email" = "dev@mail.numerique.gouv.fr" }]
|
||||
classifiers = [
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
@@ -25,23 +25,24 @@ license = { file = "LICENSE" }
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"celery[redis]==5.4.0",
|
||||
"celery[redis]==5.5.3",
|
||||
"django-configurations==2.5.1",
|
||||
"django-cors-headers==4.6.0",
|
||||
"django-cors-headers==4.7.0",
|
||||
"redis==5.2.1",
|
||||
"django-redis==5.4.0",
|
||||
"django==5.1.4",
|
||||
"djangorestframework==3.15.2",
|
||||
"django-redis==6.0.0",
|
||||
"django==5.2.6",
|
||||
"django-lasuite[all]==0.0.14",
|
||||
"djangorestframework==3.16.0",
|
||||
"drf_spectacular==0.28.0",
|
||||
"dockerflow==2024.4.2",
|
||||
"factory_boy==3.3.1",
|
||||
"gunicorn==23.0.0",
|
||||
"mozilla-django-oidc==4.0.1",
|
||||
"psycopg[binary]==3.2.3",
|
||||
"psycopg[binary]==3.2.9",
|
||||
"pydantic==2.10.5",
|
||||
"pyjwt==2.10.1",
|
||||
"requests==2.32.3",
|
||||
"sentry-sdk==2.19.2",
|
||||
"requests==2.32.4",
|
||||
"sentry-sdk==2.32.0",
|
||||
"url-normalize==1.4.3",
|
||||
"opensearch-py==2.8.0",
|
||||
"whitenoise==6.8.2",
|
||||
@@ -55,22 +56,22 @@ dependencies = [
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"django-extensions==3.2.3",
|
||||
"drf-spectacular-sidecar==2024.12.1",
|
||||
"django-extensions==4.1",
|
||||
"drf-spectacular-sidecar==2025.7.1",
|
||||
"faker==33.3.0",
|
||||
"ipdb==0.13.13",
|
||||
"ipython==8.31.0",
|
||||
"pyfakefs==5.7.3",
|
||||
"pyfakefs==5.9.1",
|
||||
"pylint-django==2.6.1",
|
||||
"pylint==3.3.3",
|
||||
"pytest-cov==6.0.0",
|
||||
"pytest-django==4.9.0",
|
||||
"pytest==8.3.4",
|
||||
"pylint==3.3.7",
|
||||
"pytest-cov==6.2.1",
|
||||
"pytest-django==4.11.1",
|
||||
"pytest==8.4.1",
|
||||
"pytest-icdiff==0.9",
|
||||
"pytest-xdist==3.6.1",
|
||||
"responses==0.25.3",
|
||||
"ruff==0.9.0",
|
||||
"types-requests==2.32.0.20241016",
|
||||
"pytest-xdist==3.8.0",
|
||||
"responses==0.25.7",
|
||||
"ruff==0.12.2",
|
||||
"types-requests==2.32.4.20250611",
|
||||
]
|
||||
|
||||
[tool.setuptools]
|
||||
|
||||
Reference in New Issue
Block a user