From ce4bfe074a3f175eb906717ae744850bae71db29 Mon Sep 17 00:00:00 2001 From: daedalus <44623501+ComfortablyCoding@users.noreply.github.com> Date: Mon, 2 Dec 2024 11:48:51 -0500 Subject: [PATCH] Add support for activity/revisions/logs retention (#24058) * Add retention support for `directus_activity` and `directus_revisions` (#23410) * add retention support * add changeset * add docs * rework imports * skip processing for no records * improve retention tests * improve tests * add error handling * remove redundant while condition in retention * improve error message * increase retention to 90 days * add lock * Use the action constant * Skip the full table count * Add false to disable in docs * Reduce batch to 500 for db compat * Add timeout to lock * skip revisions if activities is more frequent * Update docs/self-hosted/config-options.md Co-authored-by: Kevin Lewis * Update docs/self-hosted/config-options.md Co-authored-by: Kevin Lewis * RETENTION_TASKS are no longer constant * fix formatting for retentions enabled config option * reword RETENTION_SCHEDULE config option * add explicit return types and update to async for consistency * fix schedules tests to be async * update jsdoc to correctly indicate schedule job * add correct default value for retention schedules --------- Co-authored-by: ian Co-authored-by: Kevin Lewis * Removed outdated logic for versioning (#23749) * Extract comments to a separate table (#22295) * Add directus_comments migration * Add comments controller and service * Remove from activity * Update system-data and types * Refactor app with new endpoints * Expose service * Update app minimal permissions * Add collection translation * Define relations * Allow comment creation only if there's item read access * Patch for MSSQL double constraints issue * Fix users service test * Add sdk support * Update specs * Fix formatting * Fix specs error * Patch whoopsie * Remove obsolete GraphQL mutations * Update required fields * Remove unused vars * Allow edit and delete of legacy activity comments * Remove legacy comments from SDK * Add changeset * Batch upwards migration * Update SDK to use keysOrQuery Co-authored-by: Brainslug * Update implementation for keysOrQuery * Remove singleton check * Update SDK to use keysOrQuery 2 Co-authored-by: Brainslug * Update keysOrQuery typedoc * Fix import * Update migration timestamp * fixed import * Update api/src/utils/get-service.ts * utilize chunk processing in migration * formatting * only services extended from itemservice should be added * remove redundant checks from comment header * update comment service to v11 permission format * specify missing required fields * Mock comments in users test * Simplify migration and update date * WIP legacy access * Optimise imports * WIP app cleanup * Update loadUserPreviews typing Co-authored-by: Daniel Biegler * Read legacy comments * Parse using comments service * Perform migration directly * Fix legacy app sort query which uses id * Migrate legacy comments in mutations * Reduce api semver * Update app recommended permissions * Recommend updating of comment only * replace hardcoded type with existing one * Allow users to update or delete their own comments Co-authored-by: daedalus <44623501+ComfortablyCoding@users.noreply.github.com> * Skip further access validation for non-existent collections * Check if collection exists before the admin check Co-authored-by: daedalus <44623501+ComfortablyCoding@users.noreply.github.com> * Fix incorrect legacy check Co-authored-by: Daniel Biegler * Fix merging of count when db returns count as string type * Remove unused import --------- Co-authored-by: Brainslug Co-authored-by: Pascal Jufer Co-authored-by: Brainslug Co-authored-by: daedalus <44623501+ComfortablyCoding@users.noreply.github.com> Co-authored-by: Daniel Biegler * Consolidate content versioning (#22413) * Add migration * Use the new delta field * Add cast-json flag * Fix typing * Fetch existing deltas if version created during migration * Add changeset * Add version delta field into sdk schema * Update migration timestamp * Update versions.save() to return finalVersionDelta Co-authored-by: Pascal Jufer * Sort on DB level * Update migration date * Disallow passing delta via create/update * Update docs & specs * Fix save response * Remove unnecessary access check Already checked by the subsequent itemsService.readOne call * Update changeset * Don't require update perms on versions for save * Optimize validateCreateData * update to new validateAccess * Update docs/reference/system/versions.md * Remove migration of delta * Rename to legacy * Add missed changes for Remove migration of delta in 2e2f50fa67bb821304b829f21ba200fccbf98cee * Update docs/reference/system/versions.md --------- Co-authored-by: Pascal Jufer Co-authored-by: daedalus <44623501+ComfortablyCoding@users.noreply.github.com> * Update migration dates * api change should be major for versioning * Remove comment paths from activity reference * Added comments reference * Added directus_comments to table of system collections * The linter demands newline * Revert function renaming for patch semver * Use transaction in down migration for comments (#23715) * Remove outdated logic for versioning * Fix migration * Update api/src/database/migrations/20240924B-populate-versioning-deltas.ts * add changeset * reword changeset --------- Co-authored-by: Brainslug Co-authored-by: Pascal Jufer Co-authored-by: Brainslug Co-authored-by: daedalus <44623501+ComfortablyCoding@users.noreply.github.com> Co-authored-by: Daniel Biegler Co-authored-by: Kevin Lewis Co-authored-by: Rijk van Zanten * Removed outdated logic for comments (#23748) * Extract comments to a separate table (#22295) * Add directus_comments migration * Add comments controller and service * Remove from activity * Update system-data and types * Refactor app with new endpoints * Expose service * Update app minimal permissions * Add collection translation * Define relations * Allow comment creation only if there's item read access * Patch for MSSQL double constraints issue * Fix users service test * Add sdk support * Update specs * Fix formatting * Fix specs error * Patch whoopsie * Remove obsolete GraphQL mutations * Update required fields * Remove unused vars * Allow edit and delete of legacy activity comments * Remove legacy comments from SDK * Add changeset * Batch upwards migration * Update SDK to use keysOrQuery Co-authored-by: Brainslug * Update implementation for keysOrQuery * Remove singleton check * Update SDK to use keysOrQuery 2 Co-authored-by: Brainslug * Update keysOrQuery typedoc * Fix import * Update migration timestamp * fixed import * Update api/src/utils/get-service.ts * utilize chunk processing in migration * formatting * only services extended from itemservice should be added * remove redundant checks from comment header * update comment service to v11 permission format * specify missing required fields * Mock comments in users test * Simplify migration and update date * WIP legacy access * Optimise imports * WIP app cleanup * Update loadUserPreviews typing Co-authored-by: Daniel Biegler * Read legacy comments * Parse using comments service * Perform migration directly * Fix legacy app sort query which uses id * Migrate legacy comments in mutations * Reduce api semver * Update app recommended permissions * Recommend updating of comment only * replace hardcoded type with existing one * Allow users to update or delete their own comments Co-authored-by: daedalus <44623501+ComfortablyCoding@users.noreply.github.com> * Skip further access validation for non-existent collections * Check if collection exists before the admin check Co-authored-by: daedalus <44623501+ComfortablyCoding@users.noreply.github.com> * Fix incorrect legacy check Co-authored-by: Daniel Biegler * Fix merging of count when db returns count as string type * Remove unused import --------- Co-authored-by: Brainslug Co-authored-by: Pascal Jufer Co-authored-by: Brainslug Co-authored-by: daedalus <44623501+ComfortablyCoding@users.noreply.github.com> Co-authored-by: Daniel Biegler * Consolidate content versioning (#22413) * Add migration * Use the new delta field * Add cast-json flag * Fix typing * Fetch existing deltas if version created during migration * Add changeset * Add version delta field into sdk schema * Update migration timestamp * Update versions.save() to return finalVersionDelta Co-authored-by: Pascal Jufer * Sort on DB level * Update migration date * Disallow passing delta via create/update * Update docs & specs * Fix save response * Remove unnecessary access check Already checked by the subsequent itemsService.readOne call * Update changeset * Don't require update perms on versions for save * Optimize validateCreateData * update to new validateAccess * Update docs/reference/system/versions.md * Remove migration of delta * Rename to legacy * Add missed changes for Remove migration of delta in 2e2f50fa67bb821304b829f21ba200fccbf98cee * Update docs/reference/system/versions.md --------- Co-authored-by: Pascal Jufer Co-authored-by: daedalus <44623501+ComfortablyCoding@users.noreply.github.com> * Update migration dates * api change should be major for versioning * Remove comment paths from activity reference * Added comments reference * Added directus_comments to table of system collections * The linter demands newline * Revert function renaming for patch semver * Use transaction in down migration for comments (#23715) * Remove outdated logic for comments * remove duplicate app access permission * remove unused import/params in gql * remove remaining comment code in activity * remove remaining activity logic in comment * add changeset * reword changeset --------- Co-authored-by: Brainslug Co-authored-by: Pascal Jufer Co-authored-by: Brainslug Co-authored-by: daedalus <44623501+ComfortablyCoding@users.noreply.github.com> Co-authored-by: Daniel Biegler Co-authored-by: Kevin Lewis Co-authored-by: Rijk van Zanten * Update config-options.md * add missing docs changeset for retentions * ensure selected id is not ambiguous for joins * Restrict comment create, update and delete to authenticated users (#23996) * Extract comments to a separate table (#22295) * Add directus_comments migration * Add comments controller and service * Remove from activity * Update system-data and types * Refactor app with new endpoints * Expose service * Update app minimal permissions * Add collection translation * Define relations * Allow comment creation only if there's item read access * Patch for MSSQL double constraints issue * Fix users service test * Add sdk support * Update specs * Fix formatting * Fix specs error * Patch whoopsie * Remove obsolete GraphQL mutations * Update required fields * Remove unused vars * Allow edit and delete of legacy activity comments * Remove legacy comments from SDK * Add changeset * Batch upwards migration * Update SDK to use keysOrQuery Co-authored-by: Brainslug * Update implementation for keysOrQuery * Remove singleton check * Update SDK to use keysOrQuery 2 Co-authored-by: Brainslug * Update keysOrQuery typedoc * Fix import * Update migration timestamp * fixed import * Update api/src/utils/get-service.ts * utilize chunk processing in migration * formatting * only services extended from itemservice should be added * remove redundant checks from comment header * update comment service to v11 permission format * specify missing required fields * Mock comments in users test * Simplify migration and update date * WIP legacy access * Optimise imports * WIP app cleanup * Update loadUserPreviews typing Co-authored-by: Daniel Biegler * Read legacy comments * Parse using comments service * Perform migration directly * Fix legacy app sort query which uses id * Migrate legacy comments in mutations * Reduce api semver * Update app recommended permissions * Recommend updating of comment only * replace hardcoded type with existing one * Allow users to update or delete their own comments Co-authored-by: daedalus <44623501+ComfortablyCoding@users.noreply.github.com> * Skip further access validation for non-existent collections * Check if collection exists before the admin check Co-authored-by: daedalus <44623501+ComfortablyCoding@users.noreply.github.com> * Fix incorrect legacy check Co-authored-by: Daniel Biegler * Fix merging of count when db returns count as string type * Remove unused import --------- Co-authored-by: Brainslug Co-authored-by: Pascal Jufer Co-authored-by: Brainslug Co-authored-by: daedalus <44623501+ComfortablyCoding@users.noreply.github.com> Co-authored-by: Daniel Biegler * Consolidate content versioning (#22413) * Add migration * Use the new delta field * Add cast-json flag * Fix typing * Fetch existing deltas if version created during migration * Add changeset * Add version delta field into sdk schema * Update migration timestamp * Update versions.save() to return finalVersionDelta Co-authored-by: Pascal Jufer * Sort on DB level * Update migration date * Disallow passing delta via create/update * Update docs & specs * Fix save response * Remove unnecessary access check Already checked by the subsequent itemsService.readOne call * Update changeset * Don't require update perms on versions for save * Optimize validateCreateData * update to new validateAccess * Update docs/reference/system/versions.md * Remove migration of delta * Rename to legacy * Add missed changes for Remove migration of delta in 2e2f50fa67bb821304b829f21ba200fccbf98cee * Update docs/reference/system/versions.md --------- Co-authored-by: Pascal Jufer Co-authored-by: daedalus <44623501+ComfortablyCoding@users.noreply.github.com> * Update migration dates * api change should be major for versioning * Remove comment paths from activity reference * Added comments reference * Added directus_comments to table of system collections * The linter demands newline * Revert function renaming for patch semver * Use transaction in down migration for comments (#23715) * Remove outdated logic for comments * remove duplicate app access permission * remove unused import/params in gql * remove remaining comment code in activity * remove remaining activity logic in comment * add changeset * reword changeset * restrict comment CUD to authenticated users * update docs * add changeset * skip mention processing if none are present * update restriction wording in the docs --------- Co-authored-by: ian Co-authored-by: Brainslug Co-authored-by: Pascal Jufer Co-authored-by: Brainslug Co-authored-by: Daniel Biegler Co-authored-by: Kevin Lewis Co-authored-by: Rijk van Zanten * remove foreign key constraint on comment `collection` column * update migration comment wording * Update api/src/database/migrations/20240924A-migrate-legacy-comments.ts Co-authored-by: ian --------- Co-authored-by: ian Co-authored-by: Kevin Lewis Co-authored-by: Brainslug Co-authored-by: Pascal Jufer Co-authored-by: Brainslug Co-authored-by: Daniel Biegler Co-authored-by: Rijk van Zanten --- .changeset/metal-rules-pretend.md | 5 + .changeset/perfect-pens-laugh.md | 7 + .changeset/silver-comics-arrive.md | 7 + .changeset/thick-timers-cough.md | 6 + api/src/app.ts | 11 +- api/src/controllers/activity.ts | 129 +------ api/src/controllers/comments.ts | 7 - api/src/controllers/tus.ts | 19 - api/src/controllers/versions.ts | 10 +- .../migrations/20240909A-separate-comments.ts | 7 +- .../20240924A-migrate-legacy-comments.ts | 69 ++++ .../20240924B-populate-versioning-deltas.ts | 40 +++ api/src/schedules/retention.test.ts | 52 +++ api/src/schedules/retention.ts | 127 +++++++ .../telemetry.test.ts} | 27 +- .../telemetry.ts} | 12 +- api/src/schedules/tus.test.ts | 34 ++ api/src/schedules/tus.ts | 27 ++ api/src/services/comments.ts | 332 ++---------------- api/src/services/graphql/index.ts | 97 ----- api/src/services/versions.ts | 48 +-- api/src/telemetry/index.ts | 1 - api/src/utils/get-service.ts | 2 +- docs/reference/system/comments.md | 12 +- docs/self-hosted/config-options.md | 11 + packages/env/src/constants/defaults.ts | 7 + .../app-access-permissions.yaml | 6 - packages/system-data/src/fields/activity.yaml | 6 - sdk/src/schema/activity.ts | 1 - 29 files changed, 461 insertions(+), 658 deletions(-) create mode 100644 .changeset/metal-rules-pretend.md create mode 100644 .changeset/perfect-pens-laugh.md create mode 100644 .changeset/silver-comics-arrive.md create mode 100644 .changeset/thick-timers-cough.md create mode 100644 api/src/database/migrations/20240924A-migrate-legacy-comments.ts create mode 100644 api/src/database/migrations/20240924B-populate-versioning-deltas.ts create mode 100644 api/src/schedules/retention.test.ts create mode 100644 api/src/schedules/retention.ts rename api/src/{telemetry/lib/init-telemetry.test.ts => schedules/telemetry.test.ts} (76%) rename api/src/{telemetry/lib/init-telemetry.ts => schedules/telemetry.ts} (69%) create mode 100644 api/src/schedules/tus.test.ts create mode 100644 api/src/schedules/tus.ts diff --git a/.changeset/metal-rules-pretend.md b/.changeset/metal-rules-pretend.md new file mode 100644 index 0000000000000..acc16a8ce05cd --- /dev/null +++ b/.changeset/metal-rules-pretend.md @@ -0,0 +1,5 @@ +--- +'@directus/api': patch +--- + +Removed outdated logic for the versioning service diff --git a/.changeset/perfect-pens-laugh.md b/.changeset/perfect-pens-laugh.md new file mode 100644 index 0000000000000..155ad4f1a7159 --- /dev/null +++ b/.changeset/perfect-pens-laugh.md @@ -0,0 +1,7 @@ +--- +'@directus/env': minor +'@directus/api': minor +'docs': patch +--- + +Added retention support for `directus_activity` and `directus_revisions` diff --git a/.changeset/silver-comics-arrive.md b/.changeset/silver-comics-arrive.md new file mode 100644 index 0000000000000..b936220a24330 --- /dev/null +++ b/.changeset/silver-comics-arrive.md @@ -0,0 +1,7 @@ +--- +'@directus/system-data': patch +'@directus/api': patch +'@directus/sdk': patch +--- + +Removed outdated logic for the comment service diff --git a/.changeset/thick-timers-cough.md b/.changeset/thick-timers-cough.md new file mode 100644 index 0000000000000..e5aaf89bbf118 --- /dev/null +++ b/.changeset/thick-timers-cough.md @@ -0,0 +1,6 @@ +--- +'docs': patch +'@directus/api': patch +--- + +Restricted comment create, update and delete to authenticated users diff --git a/api/src/app.ts b/api/src/app.ts index f9a8ce227785e..50fee9ba9e562 100644 --- a/api/src/app.ts +++ b/api/src/app.ts @@ -40,11 +40,14 @@ import serverRouter from './controllers/server.js'; import settingsRouter from './controllers/settings.js'; import sharesRouter from './controllers/shares.js'; import translationsRouter from './controllers/translations.js'; -import { default as tusRouter, scheduleTusCleanup } from './controllers/tus.js'; +import tusRouter from './controllers/tus.js'; import usersRouter from './controllers/users.js'; import utilsRouter from './controllers/utils.js'; import versionsRouter from './controllers/versions.js'; import webhooksRouter from './controllers/webhooks.js'; +import retentionSchedule from './schedules/retention.js'; +import telemetrySchedule from './schedules/telemetry.js'; +import tusSchedule from './schedules/tus.js'; import { isInstalled, validateDatabaseConnection, @@ -64,7 +67,6 @@ import rateLimiterGlobal from './middleware/rate-limiter-global.js'; import rateLimiter from './middleware/rate-limiter-ip.js'; import sanitizeQuery from './middleware/sanitize-query.js'; import schema from './middleware/schema.js'; -import { initTelemetry } from './telemetry/index.js'; import { getConfigFromEnv } from './utils/get-config-from-env.js'; import { Url } from './utils/url.js'; import { validateStorage } from './utils/validate-storage.js'; @@ -321,8 +323,9 @@ export default async function createApp(): Promise { await emitter.emitInit('routes.after', { app }); - initTelemetry(); - scheduleTusCleanup(); + await retentionSchedule(); + await telemetrySchedule(); + await tusSchedule(); await emitter.emitInit('app.after', { app }); diff --git a/api/src/controllers/activity.ts b/api/src/controllers/activity.ts index eaec4df7ca94b..72bee78071de7 100644 --- a/api/src/controllers/activity.ts +++ b/api/src/controllers/activity.ts @@ -1,11 +1,8 @@ -import { ErrorCode, InvalidPayloadError, isDirectusError } from '@directus/errors'; import express from 'express'; -import Joi from 'joi'; import { respond } from '../middleware/respond.js'; import useCollection from '../middleware/use-collection.js'; import { validateBatch } from '../middleware/validate-batch.js'; import { ActivityService } from '../services/activity.js'; -import { CommentsService } from '../services/comments.js'; import { MetaService } from '../services/meta.js'; import asyncHandler from '../utils/async-handler.js'; @@ -25,40 +22,16 @@ const readHandler = asyncHandler(async (req, res, next) => { }); let result; - let isComment; if (req.singleton) { result = await service.readSingleton(req.sanitizedQuery); } else if (req.body.keys) { result = await service.readMany(req.body.keys, req.sanitizedQuery); } else { - const sanitizedFilter = req.sanitizedQuery.filter; - - if ( - sanitizedFilter && - '_and' in sanitizedFilter && - Array.isArray(sanitizedFilter['_and']) && - sanitizedFilter['_and'].find( - (andItem) => 'action' in andItem && '_eq' in andItem['action'] && andItem['action']['_eq'] === 'comment', - ) - ) { - const commentsService = new CommentsService({ - accountability: req.accountability, - schema: req.schema, - serviceOrigin: 'activity', - }); - - result = await commentsService.readByQuery(req.sanitizedQuery); - isComment = true; - } else { - result = await service.readByQuery(req.sanitizedQuery); - } + result = await service.readByQuery(req.sanitizedQuery); } - const meta = await metaService.getMetaForQuery( - isComment ? 'directus_comments' : 'directus_activity', - req.sanitizedQuery, - ); + const meta = await metaService.getMetaForQuery('directus_activity', req.sanitizedQuery); res.locals['payload'] = { data: result, @@ -90,102 +63,4 @@ router.get( respond, ); -const createCommentSchema = Joi.object({ - comment: Joi.string().required(), - collection: Joi.string().required(), - item: [Joi.number().required(), Joi.string().required()], -}); - -router.post( - '/comment', - asyncHandler(async (req, res, next) => { - const service = new CommentsService({ - accountability: req.accountability, - schema: req.schema, - serviceOrigin: 'activity', - }); - - const { error } = createCommentSchema.validate(req.body); - - if (error) { - throw new InvalidPayloadError({ reason: error.message }); - } - - const primaryKey = await service.createOne(req.body); - - try { - const record = await service.readOne(primaryKey, req.sanitizedQuery); - - res.locals['payload'] = { - data: record || null, - }; - } catch (error: any) { - if (isDirectusError(error, ErrorCode.Forbidden)) { - return next(); - } - - throw error; - } - - return next(); - }), - respond, -); - -const updateCommentSchema = Joi.object({ - comment: Joi.string().required(), -}); - -router.patch( - '/comment/:pk', - asyncHandler(async (req, res, next) => { - const commentsService = new CommentsService({ - accountability: req.accountability, - schema: req.schema, - serviceOrigin: 'activity', - }); - - const { error } = updateCommentSchema.validate(req.body); - - if (error) { - throw new InvalidPayloadError({ reason: error.message }); - } - - const primaryKey = await commentsService.updateOne(req.params['pk']!, req.body); - - try { - const record = await commentsService.readOne(primaryKey, req.sanitizedQuery); - - res.locals['payload'] = { - data: record || null, - }; - } catch (error: any) { - if (isDirectusError(error, ErrorCode.Forbidden)) { - return next(); - } - - throw error; - } - - return next(); - }), - respond, -); - -router.delete( - '/comment/:pk', - asyncHandler(async (req, _res, next) => { - const commentsService = new CommentsService({ - accountability: req.accountability, - schema: req.schema, - serviceOrigin: 'activity', - }); - - await commentsService.deleteOne(req.params['pk']!); - - return next(); - }), - respond, -); - export default router; diff --git a/api/src/controllers/comments.ts b/api/src/controllers/comments.ts index d6fc29a7e14aa..fdb5ee3871e27 100644 --- a/api/src/controllers/comments.ts +++ b/api/src/controllers/comments.ts @@ -19,7 +19,6 @@ router.post( const service = new CommentsService({ accountability: req.accountability, schema: req.schema, - serviceOrigin: 'comments', }); const savedKeys: PrimaryKey[] = []; @@ -57,7 +56,6 @@ const readHandler = asyncHandler(async (req, res, next) => { const service = new CommentsService({ accountability: req.accountability, schema: req.schema, - serviceOrigin: 'comments', }); const metaService = new MetaService({ @@ -88,7 +86,6 @@ router.get( const service = new CommentsService({ accountability: req.accountability, schema: req.schema, - serviceOrigin: 'comments', }); const record = await service.readOne(req.params['pk']!, req.sanitizedQuery); @@ -106,7 +103,6 @@ router.patch( const service = new CommentsService({ accountability: req.accountability, schema: req.schema, - serviceOrigin: 'comments', }); let keys: PrimaryKey[] = []; @@ -142,7 +138,6 @@ router.patch( const service = new CommentsService({ accountability: req.accountability, schema: req.schema, - serviceOrigin: 'comments', }); const primaryKey = await service.updateOne(req.params['pk']!, req.body); @@ -170,7 +165,6 @@ router.delete( const service = new CommentsService({ accountability: req.accountability, schema: req.schema, - serviceOrigin: 'comments', }); if (Array.isArray(req.body)) { @@ -193,7 +187,6 @@ router.delete( const service = new CommentsService({ accountability: req.accountability, schema: req.schema, - serviceOrigin: 'comments', }); await service.deleteOne(req.params['pk']!); diff --git a/api/src/controllers/tus.ts b/api/src/controllers/tus.ts index 613ecc1613e07..801288efb5c68 100644 --- a/api/src/controllers/tus.ts +++ b/api/src/controllers/tus.ts @@ -1,12 +1,9 @@ import type { PermissionsAction } from '@directus/types'; import { Router } from 'express'; -import { RESUMABLE_UPLOADS } from '../constants.js'; import getDatabase from '../database/index.js'; import { validateAccess } from '../permissions/modules/validate-access/validate-access.js'; import { createTusServer } from '../services/tus/index.js'; import asyncHandler from '../utils/async-handler.js'; -import { getSchema } from '../utils/get-schema.js'; -import { scheduleSynchronizedJob, validateCron } from '../utils/schedule.js'; const mapAction = (method: string): PermissionsAction => { switch (method) { @@ -52,22 +49,6 @@ const handler = asyncHandler(async (req, res) => { cleanupServer(); }); -export function scheduleTusCleanup() { - if (!RESUMABLE_UPLOADS.ENABLED) return; - - if (validateCron(RESUMABLE_UPLOADS.SCHEDULE)) { - scheduleSynchronizedJob('tus-cleanup', RESUMABLE_UPLOADS.SCHEDULE, async () => { - const [tusServer, cleanupServer] = await createTusServer({ - schema: await getSchema(), - }); - - await tusServer.cleanUpExpiredUploads(); - - cleanupServer(); - }); - } -} - const router = Router(); router.post('/', checkFileAccess, handler); diff --git a/api/src/controllers/versions.ts b/api/src/controllers/versions.ts index c73604c8dc22f..7deb2a1768ffd 100644 --- a/api/src/controllers/versions.ts +++ b/api/src/controllers/versions.ts @@ -211,15 +211,7 @@ router.get( const { outdated, mainHash } = await service.verifyHash(version['collection'], version['item'], version['hash']); - let current; - - if (version['delta']) { - current = version['delta']; - } else { - const saves = await service.getVersionSavesById(version['id']); - - current = assign({}, ...saves); - } + const current = assign({}, version['delta']); const main = await service.getMainItem(version['collection'], version['item']); diff --git a/api/src/database/migrations/20240909A-separate-comments.ts b/api/src/database/migrations/20240909A-separate-comments.ts index 6c1d52da3d306..d9749fec94985 100644 --- a/api/src/database/migrations/20240909A-separate-comments.ts +++ b/api/src/database/migrations/20240909A-separate-comments.ts @@ -5,12 +5,7 @@ export async function up(knex: Knex): Promise { await knex.schema.createTable('directus_comments', (table) => { table.uuid('id').primary().notNullable(); - table - .string('collection', 64) - .notNullable() - .references('collection') - .inTable('directus_collections') - .onDelete('CASCADE'); + table.string('collection', 64).notNullable(); table.string('item').notNullable(); table.text('comment').notNullable(); diff --git a/api/src/database/migrations/20240924A-migrate-legacy-comments.ts b/api/src/database/migrations/20240924A-migrate-legacy-comments.ts new file mode 100644 index 0000000000000..e08c56cc64bdc --- /dev/null +++ b/api/src/database/migrations/20240924A-migrate-legacy-comments.ts @@ -0,0 +1,69 @@ +import { Action } from '@directus/constants'; +import type { Knex } from 'knex'; +import { randomUUID } from 'node:crypto'; + +export async function up(knex: Knex): Promise { + // remove foreign key constraint for projects already migrated to retentions-p1 + try { + await knex.schema.alterTable('directus_comments', (table) => { + table.dropForeign('collection'); + }); + } catch { + // ignore + } + + const rowsLimit = 50; + let hasMore = true; + + while (hasMore) { + const legacyComments = await knex + .select('*') + .from('directus_activity') + .where('action', '=', Action.COMMENT) + .limit(rowsLimit); + + if (legacyComments.length === 0) { + hasMore = false; + break; + } + + await knex.transaction(async (trx) => { + for (const legacyComment of legacyComments) { + let primaryKey; + + // Migrate legacy comment + if (legacyComment['action'] === Action.COMMENT) { + primaryKey = randomUUID(); + + await trx('directus_comments').insert({ + id: primaryKey, + collection: legacyComment.collection, + item: legacyComment.item, + comment: legacyComment.comment, + user_created: legacyComment.user, + date_created: legacyComment.timestamp, + }); + + await trx('directus_activity') + .update({ + action: Action.CREATE, + collection: 'directus_comments', + item: primaryKey, + comment: null, + }) + .where('id', '=', legacyComment.id); + } + } + }); + } + + await knex.schema.alterTable('directus_activity', (table) => { + table.dropColumn('comment'); + }); +} + +export async function down(knex: Knex): Promise { + await knex.schema.alterTable('directus_activity', (table) => { + table.text('comment'); + }); +} diff --git a/api/src/database/migrations/20240924B-populate-versioning-deltas.ts b/api/src/database/migrations/20240924B-populate-versioning-deltas.ts new file mode 100644 index 0000000000000..041b6aba36fef --- /dev/null +++ b/api/src/database/migrations/20240924B-populate-versioning-deltas.ts @@ -0,0 +1,40 @@ +import { parseJSON } from '@directus/utils'; +import type { Knex } from 'knex'; +import { assign } from 'lodash-es'; + +export async function up(knex: Knex): Promise { + const rowsLimit = 50; + let hasMore = true; + + while (hasMore) { + const missingDeltaVersions = await knex.select('id').from('directus_versions').whereNull('delta').limit(rowsLimit); + + if (missingDeltaVersions.length === 0) { + hasMore = false; + break; + } + + await knex.transaction(async (trx) => { + for (const missingDeltaVersion of missingDeltaVersions) { + const revisions = await trx + .select('delta') + .from('directus_revisions') + .where('version', '=', missingDeltaVersion.id) + .orderBy('id'); + + const deltas = revisions.map((revision) => parseJSON(revision.delta)); + const consolidatedDelta = assign({}, ...deltas); + + await trx('directus_versions') + .update({ + delta: JSON.stringify(consolidatedDelta), + }) + .where('id', '=', missingDeltaVersion.id); + } + }); + } +} + +export async function down(): Promise { + // No down migration required +} diff --git a/api/src/schedules/retention.test.ts b/api/src/schedules/retention.test.ts new file mode 100644 index 0000000000000..266135711d794 --- /dev/null +++ b/api/src/schedules/retention.test.ts @@ -0,0 +1,52 @@ +import { useEnv } from '@directus/env'; +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; +import * as schedule from '../utils/schedule.js'; +import { handleRetentionJob, default as retentionSchedule } from './retention.js'; + +vi.mock('@directus/env', () => ({ + useEnv: vi.fn().mockReturnValue({}), +})); + +vi.spyOn(schedule, 'scheduleSynchronizedJob'); +vi.spyOn(schedule, 'validateCron'); + +beforeEach(() => { + vi.mocked(useEnv).mockReturnValue({ RETENTION_ENABLED: true, RETENTION_SCHEDULE: '0 0 * * *' }); +}); + +afterEach(() => { + vi.clearAllMocks(); +}); + +describe('retention', () => { + test('Returns early when retention is disabled', async () => { + vi.mocked(useEnv).mockReturnValue({ RETENTION_ENABLED: false, RETENTION_SCHEDULE: '0 0 * * *' }); + + const res = await retentionSchedule(); + + expect(schedule.validateCron).not.toHaveBeenCalled(); + expect(res).toBe(false); + }); + + test('Returns early for invalid retention schedule', async () => { + vi.mocked(useEnv).mockReturnValue({ RETENTION_ENABLED: true, RETENTION_SCHEDULE: '#' }); + + const res = await retentionSchedule(); + + expect(schedule.validateCron).toHaveBeenCalledWith('#'); + expect(res).toBe(false); + }); + + test('Schedules synchronized job', async () => { + await retentionSchedule(); + + expect(schedule.validateCron).toHaveBeenCalledWith('0 0 * * *'); + expect(schedule.scheduleSynchronizedJob).toHaveBeenCalledWith('retention', '0 0 * * *', handleRetentionJob); + }); + + test('Returns true on successful init', async () => { + const res = await retentionSchedule(); + + expect(res).toBe(true); + }); +}); diff --git a/api/src/schedules/retention.ts b/api/src/schedules/retention.ts new file mode 100644 index 0000000000000..fcf651b4e29db --- /dev/null +++ b/api/src/schedules/retention.ts @@ -0,0 +1,127 @@ +import { Action } from '@directus/constants'; +import { useEnv } from '@directus/env'; +import { toBoolean } from '@directus/utils'; +import type { Knex } from 'knex'; +import getDatabase from '../database/index.js'; +import { useLock } from '../lock/index.js'; +import { useLogger } from '../logger/index.js'; +import { getMilliseconds } from '../utils/get-milliseconds.js'; +import { scheduleSynchronizedJob, validateCron } from '../utils/schedule.js'; + +export interface RetentionTask { + collection: string; + where?: readonly [string, string, Knex.Value | null]; + join?: readonly [string, string, string]; + timeframe: number | undefined; +} + +const env = useEnv(); + +const retentionLockKey = 'schedule--data-retention'; +const retentionLockTimeout = 10 * 60 * 1000; // 10 mins + +const ACTIVITY_RETENTION_TIMEFRAME = getMilliseconds(env['ACTIVITY_RETENTION']); +const FLOW_LOGS_RETENTION_TIMEFRAME = getMilliseconds(env['FLOW_LOGS_RETENTION']); +const REVISIONS_RETENTION_TIMEFRAME = getMilliseconds(env['REVISIONS_RETENTION']); + +const retentionTasks: RetentionTask[] = [ + { + collection: 'directus_activity', + where: ['action', '!=', Action.RUN], + timeframe: ACTIVITY_RETENTION_TIMEFRAME, + }, + { + collection: 'directus_activity', + where: ['action', '=', Action.RUN], + timeframe: FLOW_LOGS_RETENTION_TIMEFRAME, + }, +]; + +export async function handleRetentionJob() { + const database = getDatabase(); + const logger = useLogger(); + const lock = useLock(); + const batch = Number(env['RETENTION_BATCH']); + const lockTime = await lock.get(retentionLockKey); + const now = Date.now(); + + if (lockTime && Number(lockTime) > now - retentionLockTimeout) { + // ensure only one connected process + return; + } + + await lock.set(retentionLockKey, Date.now()); + + for (const task of retentionTasks) { + let count = 0; + + if (task.timeframe === undefined) { + // skip disabled tasks + continue; + } + + do { + const subquery = database + .queryBuilder() + .select(`${task.collection}.id`) + .from(task.collection) + .where('timestamp', '<', Date.now() - task.timeframe) + .limit(batch); + + if (task.where) { + subquery.where(...task.where); + } + + if (task.join) { + subquery.join(...task.join); + } + + try { + count = await database(task.collection).where('id', 'in', subquery).delete(); + } catch (error) { + logger.error(error, `Retention failed for Collection ${task.collection}`); + + break; + } + + // Update lock time to prevent concurrent runs + await lock.set(retentionLockKey, Date.now()); + } while (count >= batch); + } + + await lock.delete(retentionLockKey); +} + +/** + * Schedule the retention tracking + * + * @returns Whether or not retention has been initialized + */ +export default async function schedule(): Promise { + const env = useEnv(); + + if (!toBoolean(env['RETENTION_ENABLED'])) { + return false; + } + + if (!validateCron(String(env['RETENTION_SCHEDULE']))) { + return false; + } + + if ( + !ACTIVITY_RETENTION_TIMEFRAME || + (ACTIVITY_RETENTION_TIMEFRAME && + REVISIONS_RETENTION_TIMEFRAME && + ACTIVITY_RETENTION_TIMEFRAME > REVISIONS_RETENTION_TIMEFRAME) + ) { + retentionTasks.push({ + collection: 'directus_revisions', + join: ['directus_activity', 'directus_revisions.activity', 'directus_activity.id'], + timeframe: REVISIONS_RETENTION_TIMEFRAME, + }); + } + + scheduleSynchronizedJob('retention', String(env['RETENTION_SCHEDULE']), handleRetentionJob); + + return true; +} diff --git a/api/src/telemetry/lib/init-telemetry.test.ts b/api/src/schedules/telemetry.test.ts similarity index 76% rename from api/src/telemetry/lib/init-telemetry.test.ts rename to api/src/schedules/telemetry.test.ts index 8caefb04583ba..5161b967770f6 100644 --- a/api/src/telemetry/lib/init-telemetry.test.ts +++ b/api/src/schedules/telemetry.test.ts @@ -1,12 +1,12 @@ import { useEnv } from '@directus/env'; import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; -import { getCache } from '../../cache.js'; -import { scheduleSynchronizedJob } from '../../utils/schedule.js'; -import { initTelemetry, jobCallback } from './init-telemetry.js'; -import { track } from './track.js'; +import { getCache } from '../cache.js'; +import { scheduleSynchronizedJob } from '../utils/schedule.js'; +import { default as telemetrySchedule, jobCallback } from './telemetry.js'; +import { track } from '../telemetry/index.js'; -vi.mock('./track.js'); -vi.mock('../../cache.js'); +vi.mock('../telemetry/index.js'); +vi.mock('../cache.js'); // This is required because logger uses global env which is imported before the tests run. Can be // reduce to just mock the file when logger is also using useLogger everywhere @TODO @@ -16,7 +16,7 @@ vi.mock('@directus/env', () => ({ }), })); -vi.mock('../../utils/schedule.js'); +vi.mock('../utils/schedule.js'); let mockCache: ReturnType; @@ -31,31 +31,33 @@ afterEach(() => { vi.clearAllMocks(); }); -describe('initTelemetry', () => { +describe('telemetry', () => { test('Returns early when telemetry is disabled', async () => { vi.mocked(useEnv).mockReturnValue({ TELEMETRY: false }); - const res = await initTelemetry(); + const res = await telemetrySchedule(); expect(res).toBe(false); }); test('Schedules synchronized job', async () => { - await initTelemetry(); + await telemetrySchedule(); + expect(scheduleSynchronizedJob).toHaveBeenCalledWith('telemetry', '0 */6 * * *', jobCallback); }); test('Sets lock and calls track without waiting if lock is not set yet', async () => { vi.mocked(mockCache.lockCache.get).mockResolvedValue(null as any); - await initTelemetry(); + await telemetrySchedule(); expect(mockCache.lockCache.set).toHaveBeenCalledWith('telemetry-lock', true, 30000); expect(track).toHaveBeenCalledWith({ wait: false }); }); test('Returns true on successful init', async () => { - const res = await initTelemetry(); + const res = await telemetrySchedule(); + expect(res).toBe(true); }); }); @@ -63,6 +65,7 @@ describe('initTelemetry', () => { describe('jobCallback', () => { test('Calls track', () => { jobCallback(); + expect(track).toHaveBeenCalledWith(); }); }); diff --git a/api/src/telemetry/lib/init-telemetry.ts b/api/src/schedules/telemetry.ts similarity index 69% rename from api/src/telemetry/lib/init-telemetry.ts rename to api/src/schedules/telemetry.ts index 6b5e9eec401f2..804d571e14dcf 100644 --- a/api/src/telemetry/lib/init-telemetry.ts +++ b/api/src/schedules/telemetry.ts @@ -1,8 +1,8 @@ import { useEnv } from '@directus/env'; import { toBoolean } from '@directus/utils'; -import { getCache } from '../../cache.js'; -import { scheduleSynchronizedJob } from '../../utils/schedule.js'; -import { track } from './track.js'; +import { getCache } from '../cache.js'; +import { scheduleSynchronizedJob } from '../utils/schedule.js'; +import { track } from '../telemetry/index.js'; /** * Exported to be able to test the anonymous callback function @@ -12,12 +12,12 @@ export const jobCallback = () => { }; /** - * Initialize the telemetry tracking. Will generate a report on start, and set a schedule to report + * Schedule the telemetry tracking. Will generate a report on start, and set a schedule to report * every 6 hours * * @returns Whether or not telemetry has been initialized */ -export const initTelemetry = async () => { +export default async function schedule(): Promise { const env = useEnv(); if (toBoolean(env['TELEMETRY']) === false) return false; @@ -35,4 +35,4 @@ export const initTelemetry = async () => { } return true; -}; +} diff --git a/api/src/schedules/tus.test.ts b/api/src/schedules/tus.test.ts new file mode 100644 index 0000000000000..6d8967a9dc286 --- /dev/null +++ b/api/src/schedules/tus.test.ts @@ -0,0 +1,34 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import * as schedule from '../utils/schedule.js'; +import tusSchedule from './tus.js'; + +// This is required because logger uses global env which is imported before the tests run. Can be +// reduce to just mock the file when logger is also using useLogger everywhere @TODO +vi.mock('@directus/env', () => ({ + useEnv: vi.fn().mockReturnValue({ + EMAIL_TEMPLATES_PATH: './templates', + TUS_ENABLED: true, + TUS_CLEANUP_SCHEDULE: '0 */6 * * *', + }), +})); + +vi.spyOn(schedule, 'scheduleSynchronizedJob'); +vi.spyOn(schedule, 'validateCron'); + +afterEach(() => { + vi.clearAllMocks(); +}); + +describe('tus', () => { + test('Schedules synchronized job', async () => { + await tusSchedule(); + + expect(schedule.scheduleSynchronizedJob).toHaveBeenCalled(); + }); + + test('Returns true on successful init', async () => { + const res = await tusSchedule(); + + expect(res).toBe(true); + }); +}); diff --git a/api/src/schedules/tus.ts b/api/src/schedules/tus.ts new file mode 100644 index 0000000000000..4d846434506af --- /dev/null +++ b/api/src/schedules/tus.ts @@ -0,0 +1,27 @@ +import { RESUMABLE_UPLOADS } from '../constants.js'; +import { getSchema } from '../utils/get-schema.js'; +import { createTusServer } from '../services/tus/index.js'; +import { scheduleSynchronizedJob, validateCron } from '../utils/schedule.js'; + +/** + * Schedule the tus cleanup + * + * @returns Whether or not tus cleanup has been initialized + */ +export default async function schedule(): Promise { + if (!RESUMABLE_UPLOADS.ENABLED) return false; + + if (validateCron(RESUMABLE_UPLOADS.SCHEDULE)) { + scheduleSynchronizedJob('tus-cleanup', RESUMABLE_UPLOADS.SCHEDULE, async () => { + const [tusServer, cleanupServer] = await createTusServer({ + schema: await getSchema(), + }); + + await tusServer.cleanUpExpiredUploads(); + + cleanupServer(); + }); + } + + return true; +} diff --git a/api/src/services/comments.ts b/api/src/services/comments.ts index 12d4df5bbf558..4da0c7b2f6122 100644 --- a/api/src/services/comments.ts +++ b/api/src/services/comments.ts @@ -1,114 +1,35 @@ -import { Action } from '@directus/constants'; import { useEnv } from '@directus/env'; import { ErrorCode, ForbiddenError, InvalidPayloadError, isDirectusError } from '@directus/errors'; -import type { Filter } from '@directus/system-data'; -import type { Accountability, Comment, Item, PrimaryKey, Query } from '@directus/types'; -import { cloneDeep, mergeWith, uniq } from 'lodash-es'; -import { randomUUID } from 'node:crypto'; +import type { Accountability, Comment, PrimaryKey } from '@directus/types'; +import { uniq } from 'lodash-es'; import { useLogger } from '../logger/index.js'; import { fetchRolesTree } from '../permissions/lib/fetch-roles-tree.js'; import { fetchGlobalAccess } from '../permissions/modules/fetch-global-access/fetch-global-access.js'; import { validateAccess } from '../permissions/modules/validate-access/validate-access.js'; import type { AbstractServiceOptions, MutationOptions } from '../types/index.js'; import { isValidUuid } from '../utils/is-valid-uuid.js'; -import { transaction } from '../utils/transaction.js'; import { Url } from '../utils/url.js'; import { userName } from '../utils/user-name.js'; -import { ActivityService } from './activity.js'; -import { ItemsService, type QueryOptions } from './items.js'; +import { ItemsService } from './items.js'; import { NotificationsService } from './notifications.js'; import { UsersService } from './users.js'; const env = useEnv(); const logger = useLogger(); -type serviceOrigin = 'activity' | 'comments'; - -// TODO: Remove legacy comments logic export class CommentsService extends ItemsService { - activityService: ActivityService; notificationsService: NotificationsService; usersService: UsersService; - serviceOrigin: serviceOrigin; - constructor( - options: AbstractServiceOptions & { serviceOrigin: serviceOrigin }, // TODO: Remove serviceOrigin when legacy comments are migrated - ) { + constructor(options: AbstractServiceOptions) { super('directus_comments', options); - this.activityService = new ActivityService(options); this.notificationsService = new NotificationsService({ schema: this.schema }); this.usersService = new UsersService({ schema: this.schema }); - this.serviceOrigin = options.serviceOrigin ?? 'comments'; - } - - override readOne(key: PrimaryKey, query?: Query, opts?: QueryOptions): Promise { - const isLegacyComment = isNaN(Number(key)); - - let result; - - if (isLegacyComment) { - const activityQuery = this.serviceOrigin === 'activity' ? query : this.generateQuery('activity', query || {}); - result = this.activityService.readOne(key, activityQuery, opts); - } else { - const commentsQuery = this.serviceOrigin === 'comments' ? query : this.generateQuery('comments', query || {}); - result = super.readOne(key, commentsQuery, opts); - } - - return result; - } - - override async readByQuery(query: Query, opts?: QueryOptions): Promise { - const activityQuery = this.serviceOrigin === 'activity' ? query : this.generateQuery('activity', query); - const commentsQuery = this.serviceOrigin === 'comments' ? query : this.generateQuery('comments', query); - const activityResult = await this.activityService.readByQuery(activityQuery, opts); - const commentsResult = await super.readByQuery(commentsQuery, opts); - - if (query.aggregate) { - // Merging the first result only as the app does not utilise group - return [ - mergeWith({}, activityResult[0], commentsResult[0], (a: any, b: any) => { - const numA = Number(a); - const numB = Number(b); - - if (!isNaN(numA) && !isNaN(numB)) { - return numA + numB; - } - - return; - }), - ]; - } else if (query.sort) { - return this.sortLegacyResults([...activityResult, ...commentsResult], query.sort); - } else { - return [...activityResult, ...commentsResult]; - } - } - - override async readMany(keys: PrimaryKey[], query?: Query, opts?: QueryOptions): Promise { - const commentsKeys = []; - const activityKeys = []; - - for (const key of keys) { - if (isNaN(Number(key))) { - commentsKeys.push(key); - } else { - activityKeys.push(key); - } - } - - const activityQuery = this.serviceOrigin === 'activity' ? query : this.generateQuery('activity', query || {}); - const commentsQuery = this.serviceOrigin === 'comments' ? query : this.generateQuery('comments', query || {}); - const activityResult = await this.activityService.readMany(activityKeys, activityQuery, opts); - const commentsResult = await super.readMany(commentsKeys, commentsQuery, opts); - - if (query?.sort) { - return this.sortLegacyResults([...activityResult, ...commentsResult], query.sort); - } else { - return [...activityResult, ...commentsResult]; - } } override async createOne(data: Partial, opts?: MutationOptions): Promise { + if (!this.accountability?.user) throw new ForbiddenError(); + if (!data['comment']) { throw new InvalidPayloadError({ reason: `"comment" is required` }); } @@ -136,11 +57,17 @@ export class CommentsService extends ItemsService { ); } + const result = await super.createOne(data, opts); + const usersRegExp = new RegExp(/@[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}/gi); const mentions = uniq(data['comment'].match(usersRegExp) ?? []); - const sender = await this.usersService.readOne(this.accountability!.user!, { + if (mentions.length === 0) { + return result; + } + + const sender = await this.usersService.readOne(this.accountability.user, { fields: ['id', 'first_name', 'last_name', 'email'], }); @@ -236,237 +163,18 @@ ${comment} } } - return super.createOne(data, opts); - } - - override async updateByQuery(query: Query, data: Partial, opts?: MutationOptions): Promise { - const keys = await this.getKeysByQuery(query); - const migratedKeys = await this.processPrimaryKeys(keys); - - return super.updateMany(migratedKeys, data, opts); - } - - override async updateMany(keys: PrimaryKey[], data: Partial, opts?: MutationOptions): Promise { - const migratedKeys = await this.processPrimaryKeys(keys); - - return super.updateMany(migratedKeys, data, opts); - } - - override async updateOne(key: PrimaryKey, data: Partial, opts?: MutationOptions): Promise { - const migratedKey = await this.migrateLegacyComment(key); - - return super.updateOne(migratedKey, data, opts); - } - - override async deleteByQuery(query: Query, opts?: MutationOptions): Promise { - const keys = await this.getKeysByQuery(query); - const migratedKeys = await this.processPrimaryKeys(keys); - - return super.deleteMany(migratedKeys, opts); - } - - override async deleteMany(keys: PrimaryKey[], opts?: MutationOptions): Promise { - const migratedKeys = await this.processPrimaryKeys(keys); - - return super.deleteMany(migratedKeys, opts); - } - - override async deleteOne(key: PrimaryKey, opts?: MutationOptions): Promise { - const migratedKey = await this.migrateLegacyComment(key); - - return super.deleteOne(migratedKey, opts); - } - - private async processPrimaryKeys(keys: PrimaryKey[]) { - const migratedKeys = []; - - for (const key of keys) { - if (isNaN(Number(key))) { - migratedKeys.push(key); - continue; - } - - migratedKeys.push(await this.migrateLegacyComment(key)); - } - - return migratedKeys; - } - - async migrateLegacyComment(activityPk: PrimaryKey): Promise { - // Skip migration if not a legacy comment - if (isNaN(Number(activityPk))) { - return activityPk; - } - - return transaction(this.knex, async (trx) => { - let primaryKey; - const legacyComment = await trx('directus_activity').select('*').where('id', '=', activityPk).first(); - - // Legacy comment - if (legacyComment['action'] === Action.COMMENT) { - primaryKey = randomUUID(); - - await trx('directus_comments').insert({ - id: primaryKey, - collection: legacyComment.collection, - item: legacyComment.item, - comment: legacyComment.comment, - user_created: legacyComment.user, - date_created: legacyComment.timestamp, - }); - - await trx('directus_activity') - .update({ - action: Action.CREATE, - collection: 'directus_comments', - item: primaryKey, - comment: null, - }) - .where('id', '=', activityPk); - } - // Migrated comment - else if (legacyComment.collection === 'directus_comment' && legacyComment.action === Action.CREATE) { - primaryKey = legacyComment.item; - } - - if (!primaryKey) { - throw new ForbiddenError(); - } - - return primaryKey; - }); + return result; } - generateQuery(type: serviceOrigin, originalQuery: Query): Query { - const query: Query = cloneDeep(originalQuery); - const defaultActivityCommentFilter = { action: { _eq: Action.COMMENT } }; - - const commentsToActivityFieldMap: Record = { - id: 'id', - comment: 'comment', - item: 'item', - collection: 'collection', - user_created: 'user', - date_created: 'timestamp', - }; - - const activityToCommentsFieldMap: Record = { - id: 'id', - comment: 'comment', - item: 'item', - collection: 'collection', - user: 'user_created', - timestamp: 'date_created', - }; - - const targetFieldMap = type === 'activity' ? commentsToActivityFieldMap : activityToCommentsFieldMap; - - for (const key of Object.keys(originalQuery)) { - switch (key as keyof Filter) { - case 'fields': - if (!originalQuery.fields) break; - - query.fields = []; - - for (const field of originalQuery.fields) { - if (field === '*') { - query.fields = ['*']; - break; - } - - const parts = field.split('.'); - const firstPart = parts[0]; - - if (firstPart && targetFieldMap[firstPart]) { - query.fields.push(field); - - if (firstPart !== targetFieldMap[firstPart]) { - (query.alias = query.alias || {})[firstPart] = targetFieldMap[firstPart]!; - } - } - } - - break; - case 'filter': - if (!originalQuery.filter) break; - - if (type === 'activity') { - query.filter = { _and: [defaultActivityCommentFilter, originalQuery.filter] }; - } - - if (type === 'comments' && this.serviceOrigin === 'activity') { - if ('_and' in originalQuery.filter && Array.isArray(originalQuery.filter['_and'])) { - query.filter = { - _and: originalQuery.filter['_and'].filter( - (andItem) => - !('action' in andItem && '_eq' in andItem['action'] && andItem['action']['_eq'] === 'comment'), - ), - }; - } else { - query.filter = originalQuery.filter; - } - } - - break; - case 'aggregate': - if (originalQuery.aggregate) { - query.aggregate = originalQuery.aggregate; - } - - break; - case 'sort': - if (!originalQuery.sort) break; - - query.sort = []; - - for (const sort of originalQuery.sort) { - const isDescending = sort.startsWith('-'); - const field = isDescending ? sort.slice(1) : sort; - - if (field && targetFieldMap[field]) { - query.sort.push(`${isDescending ? '-' : ''}${targetFieldMap[field]}`); - } - } - - break; - } - } - - if (type === 'activity' && !query.filter) { - query.filter = defaultActivityCommentFilter; - } + override updateOne(key: PrimaryKey, data: Partial, opts?: MutationOptions): Promise { + if (!this.accountability?.user) throw new ForbiddenError(); - return query; + return super.updateOne(key, data, opts); } - private sortLegacyResults(results: Item[], sort: Query['sort']) { - if (!sort) return results; - - let sortKeys = sort; + override deleteOne(key: PrimaryKey, opts?: MutationOptions): Promise { + if (!this.accountability?.user) throw new ForbiddenError(); - // Fix legacy app sort query which uses id - if (sortKeys.length === 1 && sortKeys[0]?.endsWith('id') && results[0]?.['timestamp']) { - sortKeys = [`${sortKeys[0].startsWith('-') ? '-' : ''}timestamp`]; - } - - return results.sort((a, b) => { - for (const key of sortKeys) { - const isDescending = key.startsWith('-'); - const actualKey = isDescending ? key.substring(1) : key; - - let aValue = a[actualKey]; - let bValue = b[actualKey]; - - if (actualKey === 'date_created' || actualKey === 'timestamp') { - aValue = new Date(aValue); - bValue = new Date(bValue); - } - - if (aValue < bValue) return isDescending ? 1 : -1; - if (aValue > bValue) return isDescending ? -1 : 1; - } - - return 0; - }); + return super.deleteOne(key, opts); } } diff --git a/api/src/services/graphql/index.ts b/api/src/services/graphql/index.ts index 951d204c6264d..1ed48d87f751b 100644 --- a/api/src/services/graphql/index.ts +++ b/api/src/services/graphql/index.ts @@ -83,7 +83,6 @@ import { sanitizeQuery } from '../../utils/sanitize-query.js'; import { validateQuery } from '../../utils/validate-query.js'; import { AuthenticationService } from '../authentication.js'; import { CollectionsService } from '../collections.js'; -import { CommentsService } from '../comments.js'; import { ExtensionsService } from '../extensions.js'; import { FieldsService } from '../fields.js'; import { FilesService } from '../files.js'; @@ -325,7 +324,6 @@ export class GraphQLService { CreateCollectionTypes, ReadCollectionTypes, UpdateCollectionTypes, - DeleteCollectionTypes, }, schema, ); @@ -2094,12 +2092,10 @@ export class GraphQLService { CreateCollectionTypes, ReadCollectionTypes, UpdateCollectionTypes, - DeleteCollectionTypes, }: { CreateCollectionTypes: Record>; ReadCollectionTypes: Record>; UpdateCollectionTypes: Record>; - DeleteCollectionTypes: Record>; }, schema: { create: SchemaOverview; @@ -3356,99 +3352,6 @@ export class GraphQLService { }); } - if ('directus_activity' in schema.create.collections) { - schemaComposer.Mutation.addFields({ - create_comment: { - type: ReadCollectionTypes['directus_activity'] ?? GraphQLBoolean, - args: { - collection: new GraphQLNonNull(GraphQLString), - item: new GraphQLNonNull(GraphQLID), - comment: new GraphQLNonNull(GraphQLString), - }, - resolve: async (_, args, __, info) => { - const service = new CommentsService({ - accountability: this.accountability, - schema: this.schema, - serviceOrigin: 'activity', - }); - - const primaryKey = await service.createOne({ - ...args, - }); - - if ('directus_activity' in ReadCollectionTypes) { - const selections = this.replaceFragmentsInSelections( - info.fieldNodes[0]?.selectionSet?.selections, - info.fragments, - ); - - const query = this.getQuery(args, selections || [], info.variableValues); - - return await service.readOne(primaryKey, query); - } - - return true; - }, - }, - }); - } - - if ('directus_activity' in schema.update.collections) { - schemaComposer.Mutation.addFields({ - update_comment: { - type: ReadCollectionTypes['directus_activity'] ?? GraphQLBoolean, - args: { - id: new GraphQLNonNull(GraphQLID), - comment: new GraphQLNonNull(GraphQLString), - }, - resolve: async (_, args, __, info) => { - const commentsService = new CommentsService({ - accountability: this.accountability, - schema: this.schema, - serviceOrigin: 'activity', - }); - - const primaryKey = await commentsService.updateOne(args['id'], { comment: args['comment'] }); - - if ('directus_activity' in ReadCollectionTypes) { - const selections = this.replaceFragmentsInSelections( - info.fieldNodes[0]?.selectionSet?.selections, - info.fragments, - ); - - const query = this.getQuery(args, selections || [], info.variableValues); - - return { ...(await commentsService.readOne(primaryKey, query)), id: args['id'] }; - } - - return true; - }, - }, - }); - } - - if ('directus_activity' in schema.delete.collections) { - schemaComposer.Mutation.addFields({ - delete_comment: { - type: DeleteCollectionTypes['one']!, - args: { - id: new GraphQLNonNull(GraphQLID), - }, - resolve: async (_, args) => { - const commentsService = new CommentsService({ - accountability: this.accountability, - schema: this.schema, - serviceOrigin: 'activity', - }); - - await commentsService.deleteOne(args['id']); - - return { id: args['id'] }; - }, - }, - }); - } - if ('directus_files' in schema.create.collections) { schemaComposer.Mutation.addFields({ import_file: { diff --git a/api/src/services/versions.ts b/api/src/services/versions.ts index 5278eda85c516..f8614cfb606ec 100644 --- a/api/src/services/versions.ts +++ b/api/src/services/versions.ts @@ -106,20 +106,6 @@ export class VersionsService extends ItemsService { return { outdated: hash !== mainHash, mainHash }; } - async getVersionSavesById(id: PrimaryKey): Promise[]> { - const revisionsService = new RevisionsService({ - knex: this.knex, - schema: this.schema, - }); - - const result = await revisionsService.readByQuery({ - filter: { version: { _eq: id } }, - }); - - return result.map((revision) => revision['delta']); - } - - // TODO: Remove legacy need to return a version array in subsequent release async getVersionSaves(key: string, collection: string, item: string | undefined): Promise[] | null> { const filter: Filter = { key: { _eq: key }, @@ -138,9 +124,7 @@ export class VersionsService extends ItemsService { return [versions[0]['delta']]; } - const saves = await this.getVersionSavesById(versions[0]['id']); - - return saves; + return null; } override async createOne(data: Partial, opts?: MutationOptions): Promise { @@ -262,15 +246,7 @@ export class VersionsService extends ItemsService { delta: revisionDelta, }); - let existingDelta = version['delta']; - - if (!existingDelta) { - const saves = await this.getVersionSavesById(key); - - existingDelta = assign({}, ...saves); - } - - const finalVersionDelta = assign({}, existingDelta, revisionDelta ? JSON.parse(revisionDelta) : null); + const finalVersionDelta = assign({}, version['delta'], revisionDelta ? JSON.parse(revisionDelta) : null); const sudoService = new ItemsService(this.collection, { knex: this.knex, @@ -289,7 +265,7 @@ export class VersionsService extends ItemsService { } async promote(version: PrimaryKey, mainHash: string, fields?: string[]) { - const { id, collection, item, delta } = (await this.readOne(version)) as ContentVersion; + const { collection, item, delta } = (await this.readOne(version)) as ContentVersion; // will throw an error if the accountability does not have permission to update the item if (this.accountability) { @@ -307,6 +283,12 @@ export class VersionsService extends ItemsService { ); } + if (!delta) { + throw new UnprocessableContentError({ + reason: `No changes to promote`, + }); + } + const { outdated } = await this.verifyHash(collection, item, mainHash); if (outdated) { @@ -315,17 +297,7 @@ export class VersionsService extends ItemsService { }); } - let versionResult; - - if (delta) { - versionResult = delta; - } else { - const saves = await this.getVersionSavesById(id); - - versionResult = assign({}, ...saves); - } - - const payloadToUpdate = fields ? pick(versionResult, fields) : versionResult; + const payloadToUpdate = fields ? pick(delta, fields) : delta; const itemsService = new ItemsService(collection, { accountability: this.accountability, diff --git a/api/src/telemetry/index.ts b/api/src/telemetry/index.ts index d7c9c410e6934..8a10809006794 100644 --- a/api/src/telemetry/index.ts +++ b/api/src/telemetry/index.ts @@ -1,4 +1,3 @@ export * from './lib/get-report.js'; -export * from './lib/init-telemetry.js'; export * from './lib/send-report.js'; export * from './lib/track.js'; diff --git a/api/src/utils/get-service.ts b/api/src/utils/get-service.ts index 6188a9dba0385..8c1aca1493535 100644 --- a/api/src/utils/get-service.ts +++ b/api/src/utils/get-service.ts @@ -36,7 +36,7 @@ export function getService(collection: string, opts: AbstractServiceOptions): It case 'directus_activity': return new ActivityService(opts); case 'directus_comments': - return new CommentsService({ ...opts, serviceOrigin: 'comments' }); + return new CommentsService(opts); case 'directus_dashboards': return new DashboardsService(opts); case 'directus_files': diff --git a/docs/reference/system/comments.md b/docs/reference/system/comments.md index d6f9efe024a48..ae6b46a21a8c0 100644 --- a/docs/reference/system/comments.md +++ b/docs/reference/system/comments.md @@ -143,7 +143,7 @@ Returns a [comment object](#the-comment-object) if a valid identifier was provid ## Create a Comment -Create a new comment. +Create a new comment. This action is only available to authenticated users. ### Request @@ -193,7 +193,7 @@ Returns the [comment object](#the-comment-object) of the comment that was create ## Create Multiple Comments -Create multiple new comments. +Create multiple new comments. This action is only available to authenticated users. ### Request @@ -243,7 +243,7 @@ Returns an array of [comment objects](#the-comment-object) of the comments that ## Update a Comment -Update an existing comment. +Update an existing comment. This action is only available to authenticated users. ### Request @@ -293,7 +293,7 @@ Returns the [comment object](#the-comment-object) of the comment that was update ## Update Multiple Comments -Update multiple existing comments. +Update multiple existing comments. This action is only available to authenticated users. ### Request @@ -352,7 +352,7 @@ Returns the [comment objects](#the-comment-object) of the comments that were upd ## Delete a Comment -Delete an existing comment. +Delete an existing comment. This action is only available to authenticated users. ### Request @@ -392,7 +392,7 @@ Empty body. ## Delete Multiple Comments -Delete multiple existing comments. +Delete multiple existing comments. This action is only available to authenticated users. ### Request diff --git a/docs/self-hosted/config-options.md b/docs/self-hosted/config-options.md index 8a344429f5b5d..9df252f945c70 100644 --- a/docs/self-hosted/config-options.md +++ b/docs/self-hosted/config-options.md @@ -357,6 +357,17 @@ This includes: ::: +## Data Retention + +| Variable | Description | Default Value | +| --------------------- | ---------------------------------------------------------------------------------------------------------------- | ------------- | +| `RETENTION_ENABLED` | Whether or not to enable custom data retention settings. `false` will not delete data. | `false` | +| `RETENTION_SCHEDULE` | The cron schedule at which to check for removable records, the default is once a day at 00:00. | `0 0 * * *` | +| `RETENTION_BATCH` | The maximum number of records to delete in a single query. | `500` | +| `ACTIVITY_RETENTION` | The maximum amount of time to retain `directus_activity` records or `false` to disable. This excludes flow logs. | `90d` | +| `REVISIONS_RETENTION` | The maximum amount of time to retain `directus_revisions` records or `false` to disable. | `90d` | +| `FLOW_LOGS_RETENTION` | The maximum amount of time to retain flow logs or `false` to disable. | `90d` | + ## Security | Variable | Description | Default Value | diff --git a/packages/env/src/constants/defaults.ts b/packages/env/src/constants/defaults.ts index c533bbaf8d1b4..3d703505f5492 100644 --- a/packages/env/src/constants/defaults.ts +++ b/packages/env/src/constants/defaults.ts @@ -131,6 +131,13 @@ export const DEFAULTS = { TUS_UPLOAD_EXPIRATION: '10m', TUS_CLEANUP_SCHEDULE: '0 * * * *', // every hour + RETENTION_ENABLED: false, + RETENTION_BATCH: 500, + RETENTION_SCHEDULE: '0 0 * * *', // once a day at 12AM + ACTIVITY_RETENTION: '90d', + REVISIONS_RETENTION: '90d', + FLOW_LOGS_RETENTION: '90d', + GRAPHQL_INTROSPECTION: true, GRAPHQL_QUERY_TOKEN_LIMIT: 5000, diff --git a/packages/system-data/src/app-access-permissions/app-access-permissions.yaml b/packages/system-data/src/app-access-permissions/app-access-permissions.yaml index f3877f03f160a..675c2fee1524e 100644 --- a/packages/system-data/src/app-access-permissions/app-access-permissions.yaml +++ b/packages/system-data/src/app-access-permissions/app-access-permissions.yaml @@ -7,12 +7,6 @@ user: _eq: $CURRENT_USER -- collection: directus_activity - action: create - validation: - comment: - _nnull: true - - collection: directus_comments action: read permissions: diff --git a/packages/system-data/src/fields/activity.yaml b/packages/system-data/src/fields/activity.yaml index 820b4d9734667..64cfd25d249ff 100644 --- a/packages/system-data/src/fields/activity.yaml +++ b/packages/system-data/src/fields/activity.yaml @@ -48,12 +48,6 @@ fields: display: user width: half - - field: comment - display: formatted-value - display_options: - color: 'var(--theme--foreground-subdued)' - width: half - - field: user_agent display: formatted-value display_options: diff --git a/sdk/src/schema/activity.ts b/sdk/src/schema/activity.ts index 810aebf71e242..1474637ce46f3 100644 --- a/sdk/src/schema/activity.ts +++ b/sdk/src/schema/activity.ts @@ -14,7 +14,6 @@ export type DirectusActivity = MergeCoreCollection< user_agent: string | null; collection: string; item: string; - comment: string | null; origin: string | null; revisions: DirectusRevision[] | number[] | null; }