From b533b93169b60899286645b8e0997dd397571c59 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Tue, 24 Jun 2025 15:26:28 +0200 Subject: [PATCH] save work --- .github/workflows/impress.yml | 3 + src/backend/core/api/viewsets.py | 29 ++++++++- src/backend/core/cache.py | 10 +++ src/backend/core/enums.py | 5 ++ src/backend/core/middleware.py | 21 +++++++ src/backend/core/tests/conftest.py | 5 +- src/backend/impress/settings.py | 63 +++++++++++++++++-- src/frontend/servers/y-provider/package.json | 1 + .../servers/y-provider/src/api/getDoc.ts | 2 +- src/frontend/servers/y-provider/src/env.ts | 2 + .../src/servers/hocusPocusServer.ts | 62 +++++++++++++++++- src/frontend/servers/y-provider/src/utils.ts | 16 ++++- src/frontend/yarn.lock | 43 +++++++++++++ 13 files changed, 250 insertions(+), 12 deletions(-) create mode 100644 src/backend/core/cache.py create mode 100644 src/backend/core/middleware.py diff --git a/.github/workflows/impress.yml b/.github/workflows/impress.yml index 02b4ce88..0eee2852 100644 --- a/.github/workflows/impress.yml +++ b/.github/workflows/impress.yml @@ -123,6 +123,9 @@ jobs: # needed because the postgres container does not provide a healthcheck options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + redis: + image: redis:5 + env: DJANGO_CONFIGURATION: Test DJANGO_SETTINGS_MODULE: impress.settings diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 2250c91a..f582d59b 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -10,7 +10,7 @@ from django.conf import settings from django.contrib.postgres.aggregates import ArrayAgg from django.contrib.postgres.fields import ArrayField from django.contrib.postgres.search import TrigramSimilarity -from django.core.cache import cache +from django.core.cache import cache, caches from django.core.exceptions import ValidationError from django.core.files.storage import default_storage from django.db import connection, transaction @@ -631,6 +631,33 @@ class DocumentViewSet( """Override to implement a soft delete instead of dumping the record in database.""" instance.soft_delete() + def perform_update(self, serializer): + """Check rules about collaboration.""" + + shared_cache = caches["shared"] + cache_key = f"docs:state:{serializer.instance.id}" + doc_state = shared_cache.get(cache_key, enums.DEFAULT_DOCS_STATE.copy()) + + session_key = self.request.session.session_key + + if doc_state["wsUsers"] and not session_key in doc_state["wsUsers"]: + raise drf.exceptions.PermissionDenied( + "You are not allowed to edit this document." + ) + + if doc_state["httpUser"] and doc_state["httpUser"] != session_key: + raise drf.exceptions.PermissionDenied( + "You are not allowed to edit this document." + ) + + if doc_state["httpUser"] is None: + doc_state["httpUser"] = session_key + shared_cache.set(cache_key, doc_state) + + shared_cache.touch(cache_key) + + return super().perform_update(serializer) + @drf.decorators.action( detail=False, methods=["get"], diff --git a/src/backend/core/cache.py b/src/backend/core/cache.py new file mode 100644 index 00000000..18b5365b --- /dev/null +++ b/src/backend/core/cache.py @@ -0,0 +1,10 @@ +"""Cache utilities""" + + +# pylint: disable=unused-argument +def shared_key_func(key: str, key_prefix: str, version: int = 1) -> str: + """ + Compute key for shared cache. In order to be compatiable with other system, + only the key is used. + """ + return key diff --git a/src/backend/core/enums.py b/src/backend/core/enums.py index 46e62b2c..8a34534d 100644 --- a/src/backend/core/enums.py +++ b/src/backend/core/enums.py @@ -22,6 +22,11 @@ MEDIA_STORAGE_URL_EXTRACT = re.compile( f"{settings.MEDIA_URL:s}({UUID_REGEX}/{ATTACHMENTS_FOLDER}/{UUID_REGEX}{FILE_EXT_REGEX})" ) +DEFAULT_DOCS_STATE = { + "httpUser": None, + "wsUsers": [], +} + # In Django's code base, `LANGUAGES` is set by default with all supported languages. # We can use it for the choice of languages which should not be limited to the few languages diff --git a/src/backend/core/middleware.py b/src/backend/core/middleware.py new file mode 100644 index 00000000..a46cb3eb --- /dev/null +++ b/src/backend/core/middleware.py @@ -0,0 +1,21 @@ +"""Force session creation for all requests.""" + + +class ForceSessionMiddleware: + """ + Force session creation for unauthenticated users. + Must be used after Authentication middleware. + """ + + def __init__(self, get_response): + """Initialize the middleware.""" + self.get_response = get_response + + def __call__(self, request): + """Force session creation for unauthenticated users.""" + + if not request.user.is_authenticated and request.session.session_key is None: + request.session.save() + + response = self.get_response(request) + return response diff --git a/src/backend/core/tests/conftest.py b/src/backend/core/tests/conftest.py index 00e830e1..8c3e6d2a 100644 --- a/src/backend/core/tests/conftest.py +++ b/src/backend/core/tests/conftest.py @@ -2,7 +2,7 @@ from unittest import mock -from django.core.cache import cache +from django.core.cache import caches import pytest @@ -14,7 +14,8 @@ VIA = [USER, TEAM] @pytest.fixture(autouse=True) def clear_cache(): """Fixture to clear the cache before each test.""" - cache.clear() + for cache in caches.all(): + cache.clear() @pytest.fixture diff --git a/src/backend/impress/settings.py b/src/backend/impress/settings.py index 5d16e165..9f375f3b 100755 --- a/src/backend/impress/settings.py +++ b/src/backend/impress/settings.py @@ -9,6 +9,7 @@ https://docs.djangoproject.com/en/3.1/topics/settings/ For the full list of settings and their values, see https://docs.djangoproject.com/en/3.1/ref/settings/ """ +# pylint: disable=too-many-lines import os import tomllib @@ -283,6 +284,7 @@ class Base(Configuration): "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", + "core.middleware.ForceSessionMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "dockerflow.django.middleware.DockerflowMiddleware", ] @@ -323,6 +325,24 @@ class Base(Configuration): # Cache CACHES = { "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}, + "shared": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": values.Value( + "redis://redis:6379/0", + environ_name="REDIS_URL", + environ_prefix=None, + ), + "TIMEOUT": values.IntegerValue( + 120, # timeout in seconds + environ_name="SHARED_CACHE_TIMEOUT", + environ_prefix=None, + ), + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + "SERIALIZER": "django_redis.serializers.json.JSONSerializer", + }, + "KEY_FUNCTION": "core.cache.shared_key_func", + }, } REST_FRAMEWORK = { @@ -470,6 +490,7 @@ class Base(Configuration): SESSION_COOKIE_AGE = values.PositiveIntegerValue( default=60 * 60 * 12, environ_name="SESSION_COOKIE_AGE", environ_prefix=None ) + SESSION_COOKIE_NAME = "docs_sessionid" # OIDC - Authorization Code Flow OIDC_CREATE_USER = values.BooleanValue( @@ -811,8 +832,6 @@ class Development(Base): CSRF_TRUSTED_ORIGINS = ["http://localhost:8072", "http://localhost:3000"] DEBUG = True - SESSION_COOKIE_NAME = "impress_sessionid" - USE_SWAGGER = True SESSION_CACHE_ALIAS = "session" CACHES = { @@ -822,7 +841,7 @@ class Development(Base): "session": { "BACKEND": "django_redis.cache.RedisCache", "LOCATION": values.Value( - "redis://redis:6379/2", + "redis://redis:6379/0", environ_name="REDIS_URL", environ_prefix=None, ), @@ -835,6 +854,24 @@ class Development(Base): "CLIENT_CLASS": "django_redis.client.DefaultClient", }, }, + "shared": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": values.Value( + "redis://redis:6379/0", + environ_name="REDIS_URL", + environ_prefix=None, + ), + "TIMEOUT": values.IntegerValue( + 120, # timeout in seconds + environ_name="SHARED_CACHE_TIMEOUT", + environ_prefix=None, + ), + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + "SERIALIZER": "django_redis.serializers.json.JSONSerializer", + }, + "KEY_FUNCTION": "core.cache.shared_key_func", + }, } def __init__(self): @@ -920,7 +957,7 @@ class Production(Base): "default": { "BACKEND": "django_redis.cache.RedisCache", "LOCATION": values.Value( - "redis://redis:6379/1", + "redis://redis:6379/0", environ_name="REDIS_URL", environ_prefix=None, ), @@ -938,6 +975,24 @@ class Production(Base): environ_prefix=None, ), }, + "shared": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": values.Value( + "redis://redis:6379/0", + environ_name="REDIS_URL", + environ_prefix=None, + ), + "TIMEOUT": values.IntegerValue( + 120, # timeout in seconds + environ_name="SHARED_CACHE_TIMEOUT", + environ_prefix=None, + ), + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + "SERIALIZER": "django_redis.serializers.json.JSONSerializer", + }, + "KEY_FUNCTION": "core.cache.shared_key_func", + }, } diff --git a/src/frontend/servers/y-provider/package.json b/src/frontend/servers/y-provider/package.json index 719417aa..a9d93c94 100644 --- a/src/frontend/servers/y-provider/package.json +++ b/src/frontend/servers/y-provider/package.json @@ -24,6 +24,7 @@ "cors": "2.8.5", "express": "5.1.0", "express-ws": "5.0.2", + "redis": "5.5.6", "uuid": "11.1.0", "y-protocols": "1.0.6", "yjs": "*" diff --git a/src/frontend/servers/y-provider/src/api/getDoc.ts b/src/frontend/servers/y-provider/src/api/getDoc.ts index 6c712f6d..93e45a29 100644 --- a/src/frontend/servers/y-provider/src/api/getDoc.ts +++ b/src/frontend/servers/y-provider/src/api/getDoc.ts @@ -17,7 +17,7 @@ enum LinkRole { type Base64 = string; -interface Doc { +export interface Doc { id: string; title?: string; content: Base64; diff --git a/src/frontend/servers/y-provider/src/env.ts b/src/frontend/servers/y-provider/src/env.ts index fe281930..98ca0ace 100644 --- a/src/frontend/servers/y-provider/src/env.ts +++ b/src/frontend/servers/y-provider/src/env.ts @@ -10,3 +10,5 @@ export const PORT = Number(process.env.PORT || 4444); export const SENTRY_DSN = process.env.SENTRY_DSN || ''; export const COLLABORATION_BACKEND_BASE_URL = process.env.COLLABORATION_BACKEND_BASE_URL || 'http://app-dev:8000'; +export const REDIS_URL = process.env.REDIS_URL || 'redis://redis:6379/0'; +export const CACHES_KEY_PREFIX = process.env.CACHES_KEY_PREFIX || 'docs'; \ No newline at end of file diff --git a/src/frontend/servers/y-provider/src/servers/hocusPocusServer.ts b/src/frontend/servers/y-provider/src/servers/hocusPocusServer.ts index 0fdefa7e..2c235738 100644 --- a/src/frontend/servers/y-provider/src/servers/hocusPocusServer.ts +++ b/src/frontend/servers/y-provider/src/servers/hocusPocusServer.ts @@ -1,9 +1,9 @@ import { Server } from '@hocuspocus/server'; import { validate as uuidValidate, version as uuidVersion } from 'uuid'; -import { fetchDocument } from '@/api/getDoc'; +import { fetchDocument, Doc } from '@/api/getDoc'; import { getMe } from '@/api/getMe'; -import { logger } from '@/utils'; +import { logger, getRedisClient } from '@/utils'; export const hocusPocusServer = Server.configure({ name: 'docs-collaboration', @@ -38,9 +38,10 @@ export const hocusPocusServer = Server.configure({ } let can_edit = false; + let document: Doc; try { - const document = await fetchDocument(documentName, requestHeaders); + document = await fetchDocument(documentName, requestHeaders); if (!document.abilities.retrieve) { logger( @@ -61,6 +62,34 @@ export const hocusPocusServer = Server.configure({ connection.readOnly = !can_edit; + const session = requestHeaders['cookie']?.split('; ').find(cookie => cookie.startsWith('docs_sessionid=')); + if (session) { + const sessionKey = session.split('=')[1]; + const redis = await getRedisClient(); + const redisKey = `docs:state:${document.id}`; + + const rawDocsState = await redis.get(redisKey); + + const docsState = rawDocsState ? JSON.parse(rawDocsState): { + httpUser: null, + wsUsers: [] + }; + context.sessionKey = sessionKey; + if (!docsState.wsUsers.includes(sessionKey)) { + await redis.set(redisKey, JSON.stringify({ + httpUser: null, + wsUsers: [ + ...(docsState?.wsUsers || []), + sessionKey + ], + }), + { + EX: 120, // 2 minutes + } + ); + } + } + /* * Unauthenticated users can be allowed to connect * so we flag only authenticated users @@ -79,4 +108,31 @@ export const hocusPocusServer = Server.configure({ ); return Promise.resolve(); }, + async onDisconnect({ + documentName, + context, + }) { + const sessionKey = context.sessionKey; + if (sessionKey) { + const redis = await getRedisClient(); + const redisKey = `docs:state:${documentName}`; + + const rawDocsState = await redis.get(redisKey); + + const docsState = rawDocsState ? JSON.parse(rawDocsState): { + httpUser: null, + wsUsers: [] + }; + + if (docsState.wsUsers.includes(sessionKey)) { + const index = docsState.wsUsers.indexOf(sessionKey); + docsState.wsUsers.splice(index, 1); + await redis.set(redisKey, JSON.stringify(docsState), + { + EX: 120, // 2 minutes + } + ); + } + } + } }); diff --git a/src/frontend/servers/y-provider/src/utils.ts b/src/frontend/servers/y-provider/src/utils.ts index 18d7671f..b851704f 100644 --- a/src/frontend/servers/y-provider/src/utils.ts +++ b/src/frontend/servers/y-provider/src/utils.ts @@ -1,4 +1,5 @@ -import { COLLABORATION_LOGGING } from './env'; +import { COLLABORATION_LOGGING, REDIS_URL } from './env'; +import { createClient } from 'redis'; // eslint-disable-next-line @typescript-eslint/no-explicit-any export function logger(...args: any[]) { @@ -11,3 +12,16 @@ export function logger(...args: any[]) { export const toBase64 = function (str: Uint8Array) { return Buffer.from(str).toString('base64'); }; + + +const redisClient = createClient({ + url: REDIS_URL, +}); + +export const getRedisClient = async () => { + if (!redisClient.isOpen) { + await redisClient.connect(); + } + + return redisClient; +} \ No newline at end of file diff --git a/src/frontend/yarn.lock b/src/frontend/yarn.lock index f2757f1a..b569146a 100644 --- a/src/frontend/yarn.lock +++ b/src/frontend/yarn.lock @@ -5378,6 +5378,33 @@ "@react-types/overlays" "^3.8.15" "@react-types/shared" "^3.29.1" +"@redis/bloom@5.5.6": + version "5.5.6" + resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-5.5.6.tgz#84c8cdf4fcf84f03ce2cee0082ad3e8b22b5a798" + integrity sha512-bNR3mxkwtfuCxNOzfV8B3R5zA1LiN57EH6zK4jVBIgzMzliNuReZXBFGnXvsi80/SYohajn78YdpYI+XNpqL+A== + +"@redis/client@5.5.6": + version "5.5.6" + resolved "https://registry.yarnpkg.com/@redis/client/-/client-5.5.6.tgz#a3ddb41c558dc98f5a8cacff56b3fa4efc6e3ef1" + integrity sha512-M3Svdwt6oSfyfQdqEr0L2HOJH2vK7GgCFx1NfAQvpWAT4+ljoT1L5S5cKT3dA9NJrxrOPDkdoTPWJnIrGCOcmw== + dependencies: + cluster-key-slot "1.1.2" + +"@redis/json@5.5.6": + version "5.5.6" + resolved "https://registry.yarnpkg.com/@redis/json/-/json-5.5.6.tgz#68b6c48f2c876b9a41fd88f12cc7dab1a6621fc1" + integrity sha512-AIsoe3SsGQagqAmSQHaqxEinm5oCWr7zxPWL90kKaEdLJ+zw8KBznf2i9oK0WUFP5pFssSQUXqnscQKe2amfDQ== + +"@redis/search@5.5.6": + version "5.5.6" + resolved "https://registry.yarnpkg.com/@redis/search/-/search-5.5.6.tgz#edf42e52531671ec8924865907791de9ab4c91a8" + integrity sha512-JSqasYqO0mVcHL7oxvbySRBBZYRYhFl3W7f0Da7BW8M/r0Z9wCiVrdjnN4/mKBpWZkoJT/iuisLUdPGhpKxBew== + +"@redis/time-series@5.5.6": + version "5.5.6" + resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-5.5.6.tgz#f04af4719638a852ff09267b43a8b1c250a5bb4c" + integrity sha512-jkpcgq3NOI3TX7xEAJ3JgesJTxAx7k0m6lNxNsYdEM8KOl+xj7GaB/0CbLkoricZDmFSEAz7ClA1iK9XkGHf+Q== + "@remirror/core-constants@3.0.0": version "3.0.0" resolved "https://registry.yarnpkg.com/@remirror/core-constants/-/core-constants-3.0.0.tgz#96fdb89d25c62e7b6a5d08caf0ce5114370e3b8f" @@ -7922,6 +7949,11 @@ clsx@2.1.1, clsx@^2.0.0, clsx@^2.1.1: resolved "https://registry.yarnpkg.com/clsx/-/clsx-2.1.1.tgz#eed397c9fd8bd882bfb18deab7102049a2f32999" integrity sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA== +cluster-key-slot@1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" + integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== + cmdk@1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/cmdk/-/cmdk-1.0.4.tgz#cbddef6f5ade2378f85c80a0b9ad9a8a712779b5" @@ -13725,6 +13757,17 @@ redent@^3.0.0: indent-string "^4.0.0" strip-indent "^3.0.0" +redis@5.5.6: + version "5.5.6" + resolved "https://registry.yarnpkg.com/redis/-/redis-5.5.6.tgz#56251a639aeb8e6d3751335c9b0324b4edcf47e5" + integrity sha512-hbpqBfcuhWHOS9YLNcXcJ4akNr7HFX61Dq3JuFZ9S7uU7C7kvnzuH2PDIXOP62A3eevvACoG8UacuXP3N07xdg== + dependencies: + "@redis/bloom" "5.5.6" + "@redis/client" "5.5.6" + "@redis/json" "5.5.6" + "@redis/search" "5.5.6" + "@redis/time-series" "5.5.6" + redux@^4.1.1: version "4.2.1" resolved "https://registry.yarnpkg.com/redux/-/redux-4.2.1.tgz#c08f4306826c49b5e9dc901dee0452ea8fce6197"