feat: added dynamo db

package.json

@@ -39,6 +39,7 @@
     "@semantic-release/git": "10.0.1",
     "@semantic-release/npm": "12.0.1",
     "@types/aws-lambda": "8.10.138",
+    "@types/express": "5.0.0",
     "@types/jest": "29.5.14",
     "@types/lodash": "4.17.13",
     "@types/node": "20.12.11",
@@ -61,6 +62,7 @@
   "dependencies": {
     "@aws-sdk/client-cloudformation": "3.590.0",
     "@aws-sdk/client-cloudfront": "3.590.0",
+    "@aws-sdk/client-dynamodb": "3.709.0",
     "@aws-sdk/client-elastic-beanstalk": "3.590.0",
     "@aws-sdk/client-s3": "3.591.0",
     "@aws-sdk/client-sqs": "3.682.0",
@@ -70,9 +72,11 @@
     "@dbbs/next-cache-handler-core": "1.3.0",
     "aws-cdk-lib": "2.144.0",
     "aws-sdk": "2.1635.0",
+    "body-parser": "^1.20.3",
     "cdk-assets": "2.144.0",
     "constructs": "10.3.0",
     "esbuild": "0.21.4",
+    "express": "4.21.2",
     "lodash": "4.17.21",
     "yargs": "17.7.2"
   },

src/build/cache/revalidateServer.ts

@@ -0,0 +1,135 @@
+import express from 'express'
+import { json } from 'body-parser'
+import { S3 } from '@aws-sdk/client-s3'
+import { DynamoDB, type AttributeValue } from '@aws-sdk/client-dynamodb'
+import http from 'http'
+import { chunkArray } from '../../common/array'
+
+const port = parseInt(process.env.PORT || '', 10) || 3000
+const nextServerPort = 3001
+const nextServerHostname = process.env.HOSTNAME || '0.0.0.0'
+
+const PAGE_CACHE_EXTENSIONS = ['json', 'html', 'rsc']
+const CHUNK_LIMIT = 1000
+const DYNAMODB_BATCH_LIMIT = 25
+
+interface RevalidateBody {
+  paths: string[]
+  cacheSegment?: string
+}
+
+const s3 = new S3({ region: process.env.AWS_REGION })
+const dynamoDB = new DynamoDB({ region: process.env.AWS_REGION })
+
+async function deleteS3Objects(bucketName: string, keys: string[]) {
+  if (!keys.length) return
+
+  // Delete objects in chunks to stay within AWS limits
+  await Promise.allSettled(
+    chunkArray(keys, CHUNK_LIMIT).map((chunk) => {
+      return s3.deleteObjects({
+        Bucket: bucketName,
+        Delete: { Objects: chunk.map((Key) => ({ Key })) }
+      })
+    })
+  )
+}
+
+async function batchDeleteFromDynamoDB(tableName: string, items: Record<string, AttributeValue>[]) {
+  if (!items.length) return
+
+  // Split items into chunks of 25 (DynamoDB batch limit)
+  const chunks = chunkArray(items, DYNAMODB_BATCH_LIMIT)
+
+  await Promise.all(
+    chunks.map(async (chunk) => {
+      const deleteRequests = chunk.map((item) => ({
+        DeleteRequest: {
+          Key: item
+        }
+      }))
+
+      try {
+        await dynamoDB.batchWriteItem({
+          RequestItems: {
+            [tableName]: deleteRequests
+          }
+        })
+      } catch (error) {
+        console.error('Error in batch delete:', error)
+        // Handle unprocessed items if needed
+        throw error
+      }
+    })
+  )
+}
+
+const app = express()
+
+app.use(json())
+
+app.post('/api/revalidate-pages', async (req, res) => {
+  try {
+    const { paths, cacheSegment } = req.body as RevalidateBody
+
+    if (!paths.length) {
+      res.status(400).json({ Message: 'paths is required.' }).end()
+    } else {
+      const attributeValues: Record<string, AttributeValue> = {}
+      const keyConditionExpression =
+        paths.length === 1 ? 'pageKey = :path0' : 'pageKey IN (' + paths.map((_, i) => `:path${i}`).join(',') + ')'
+
+      paths.forEach((path, index) => {
+        attributeValues[`:path${index}`] = { S: path.substring(1) }
+      })
+
+      if (cacheSegment) {
+        attributeValues[':segment'] = { S: cacheSegment }
+      }
+
+      const result = await dynamoDB.query({
+        TableName: process.env.DYNAMODB_CACHE_TABLE!,
+        IndexName: 'cacheKey-index',
+        KeyConditionExpression: keyConditionExpression,
+        FilterExpression: cacheSegment ? 'cacheKey = :segment' : undefined,
+        ExpressionAttributeValues: attributeValues
+      })
+
+      if (result?.Items?.length) {
+        const s3KeysToDelete = result.Items.flatMap((item) => {
+          return PAGE_CACHE_EXTENSIONS.map((ext) => `${item.s3Key.S}.${ext}`)
+        })
+        await deleteS3Objects(process.env.STATIC_BUCKET_NAME!, s3KeysToDelete)
+        await batchDeleteFromDynamoDB(process.env.DYNAMODB_CACHE_TABLE!, result.Items)
+      }
+
+      await Promise.all(
+        paths.map((path) =>
+          http.get({
+            hostname: nextServerHostname,
+            port: nextServerPort,
+            path
+          })
+        )
+      )
+    }
+
+    res.status(200).json({ Message: 'Revalidated.' })
+  } catch (err) {
+    res.status(400).json({ Message: err })
+  }
+})
+
+app.use((_req, res) => {
+  res.status(404).json({ error: 'Not found' })
+})
+
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+app.use((err: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
+  console.error('Server error:', err)
+  res.status(500).json({ error: 'Internal server error' })
+})
+
+app.listen(port, () => {
+  console.log(`> Revalidation server ready on port ${port}`)
+})

src/build/next.ts

@@ -3,13 +3,9 @@ import fs from 'fs/promises'
 import path from 'node:path'
 import type { PrerenderManifest, RoutesManifest } from 'next/dist/build'
 import { type ProjectPackager, type ProjectSettings } from '../common/project'
-import appRouterRevalidateTemplate from './cache/handlers/appRouterRevalidate'
 
 interface BuildOptions {
   packager: ProjectPackager
-  nextConfigPath: string
-  isAppDir: boolean
-  projectPath: string
 }
 
 interface BuildAppOptions {
@@ -23,27 +19,11 @@ const setNextEnvs = () => {
   process.env.NEXT_SERVERLESS_DEPLOYING_PHASE = 'true'
 }
 
-const appendRevalidateApi = async (projectPath: string, isAppDir: boolean): Promise<string> => {
-  const routeFolderPath = path.join(projectPath, isAppDir ? 'src/app' : 'src', 'api', 'revalidate')
-  const routePath = path.join(routeFolderPath, 'route.ts')
-
-  await fs.mkdir(routeFolderPath, { recursive: true })
-  await fs.writeFile(routePath, appRouterRevalidateTemplate, 'utf-8')
-
-  return routePath
-}
-
-export const buildNext = async (options: BuildOptions): Promise<() => Promise<void>> => {
-  const { packager, projectPath, isAppDir } = options
+export const buildNext = async (options: BuildOptions) => {
+  const { packager } = options
 
   setNextEnvs()
-  const revalidateRoutePath = await appendRevalidateApi(projectPath, isAppDir)
   childProcess.execSync(packager.buildCommand, { stdio: 'inherit' })
-
-  // Reverts changes to the next project
-  return async () => {
-    await fs.rm(revalidateRoutePath)
-  }
 }
 
 const copyAssets = async (outputPath: string, appPath: string, appRelativePath: string) => {
@@ -85,19 +65,16 @@ export const getNextCachedRoutesMatchers = async (outputPath: string, appRelativ
 export const buildApp = async (options: BuildAppOptions) => {
   const { projectSettings, outputPath } = options
 
-  const { packager, nextConfigPath, projectPath, isAppDir, root, isMonorepo } = projectSettings
+  const { packager, projectPath, root, isMonorepo } = projectSettings
 
-  const cleanNextApp = await buildNext({
-    packager,
-    nextConfigPath,
-    isAppDir,
-    projectPath
+  await buildNext({
+    packager
   })
 
   const appRelativePath = isMonorepo ? path.relative(root, projectPath) : ''
 
   await copyAssets(outputPath, projectPath, appRelativePath)
   const nextCachedRoutesMatchers = await getNextCachedRoutesMatchers(outputPath, appRelativePath)
 
-  return { cleanNextApp, nextCachedRoutesMatchers }
+  return { nextCachedRoutesMatchers }
 }

src/cacheHandler/strategy/s3.spec.ts

@@ -165,7 +165,7 @@ describe('S3Cache', () => {
 
     expect(await s3Cache.get(cacheKey, cacheKey)).toEqual(mockCacheEntryWithTags.value.pageData)
 
-    await s3Cache.revalidateTag(cacheKey, [])
+    await s3Cache.revalidateTag(cacheKey)
 
     expect(await s3Cache.get(cacheKey, cacheKey)).toBeNull()
   })

src/cacheHandler/strategy/s3.ts

@@ -1,6 +1,6 @@
 import { NEXT_CACHE_TAGS_HEADER } from 'next/dist/lib/constants'
-import { ListObjectsV2CommandOutput, S3 } from '@aws-sdk/client-s3'
-import { PutObjectCommandInput } from '@aws-sdk/client-s3/dist-types/commands/PutObjectCommand'
+import { type ListObjectsV2CommandOutput, type PutObjectCommandInput, S3 } from '@aws-sdk/client-s3'
+import { DynamoDB } from '@aws-sdk/client-dynamodb'
 import { chunkArray } from '../../common/array'
 import type { CacheEntry, CacheStrategy, CacheContext } from '@dbbs/next-cache-handler-core'
 
@@ -13,16 +13,17 @@ enum CacheExtension {
 }
 const PAGE_CACHE_EXTENSIONS = Object.values(CacheExtension)
 const CHUNK_LIMIT = 1000
-const EXT_REGEX = new RegExp(`.(${PAGE_CACHE_EXTENSIONS.join('|')})$`)
 
 export class S3Cache implements CacheStrategy {
   public readonly client: S3
   public readonly bucketName: string
+  #dynamoDBClient: DynamoDB
 
   constructor(bucketName: string) {
     const region = process.env.AWS_REGION
     this.client = new S3({ region })
     this.bucketName = bucketName
+    this.#dynamoDBClient = new DynamoDB({ region })
   }
 
   buildTagKeys(tags?: string | string[]) {
@@ -69,12 +70,25 @@ export class S3Cache implements CacheStrategy {
       Metadata: {
         'Cache-Fragment-Key': cacheKey
       },
-      ...(data.revalidate ? { CacheControl: `max-age=${data.revalidate}` } : undefined)
+      ...(data.revalidate ? { CacheControl: `smax-age=${data.revalidate}, stale-while-revalidate` } : undefined)
     }
 
     if (data.value?.kind === 'PAGE' || data.value?.kind === 'ROUTE') {
       const headersTags = this.buildTagKeys(data.value.headers?.[NEXT_CACHE_TAGS_HEADER]?.toString())
-      const input: PutObjectCommandInput = { ...baseInput, ...(headersTags ? { Tagging: headersTags } : {}) }
+      const input: PutObjectCommandInput = { ...baseInput }
+
+      promises.push(
+        this.#dynamoDBClient.putItem({
+          TableName: process.env.DYNAMODB_CACHE_TABLE!,
+          Item: {
+            pageKey: { S: pageKey },
+            cacheKey: { S: cacheKey },
+            s3Key: { S: baseInput.Key! },
+            tags: { S: [headersTags, this.buildTagKeys(data.tags)].filter(Boolean).join('&') },
+            createdAt: { S: new Date().toISOString() }
+          }
+        })
+      )
 
       if (data.value?.kind === 'PAGE') {
         promises.push(
@@ -119,18 +133,32 @@ export class S3Cache implements CacheStrategy {
           ...baseInput,
           Key: `${baseInput.Key}.${CacheExtension.JSON}`,
           Body: JSON.stringify(data),
-          ContentType: 'application/json',
-          ...(data.tags?.length ? { Tagging: `${this.buildTagKeys(data.tags)}` } : {})
+          ContentType: 'application/json'
+          // ...(data.tags?.length ? { Tagging: `${this.buildTagKeys(data.tags)}` } : {})
         })
       )
     }
 
     await Promise.all(promises)
   }
 
-  async revalidateTag(tag: string, allowCacheKeys: string[]): Promise<void> {
+  async revalidateTag(tag: string): Promise<void> {
     const keysToDelete: string[] = []
     let nextContinuationToken: string | undefined = undefined
+
+    const result = await this.#dynamoDBClient.query({
+      TableName: process.env.DYNAMODB_CACHE_TABLE!,
+      KeyConditionExpression: '#field = :value',
+      ExpressionAttributeNames: {
+        '#field': 'tags'
+      },
+      ExpressionAttributeValues: {
+        ':value': { S: tag }
+      }
+    })
+
+    console.log('HERE_IS_RESULT', result)
+    console.log('HERE_IS_RESULT_ITEMS', result.Items)
     do {
       const { Contents: contents = [], NextContinuationToken: token }: ListObjectsV2CommandOutput =
         await this.client.listObjectsV2({
@@ -141,11 +169,9 @@ export class S3Cache implements CacheStrategy {
 
       keysToDelete.push(
         ...(await contents.reduce<Promise<string[]>>(async (acc, { Key: key }) => {
-          if (
-            !key ||
-            (allowCacheKeys.length && !allowCacheKeys.some((allowKey) => key.replace(EXT_REGEX, '').endsWith(allowKey)))
-          )
+          if (!key) {
             return acc
+          }
 
           const { TagSet = [] } = await this.client.getObjectTagging({ Bucket: this.bucketName, Key: key })
           const tags = TagSet.filter(({ Key: key }) => key?.startsWith(TAG_PREFIX)).map(({ Value: tags }) => tags || '')
@@ -163,6 +189,7 @@ export class S3Cache implements CacheStrategy {
   }
 
   async delete(pageKey: string, cacheKey: string): Promise<void> {
+    console.log('HERE_IS_CALL_DELETE')
     await this.client.deleteObjects({
       Bucket: this.bucketName,
       Delete: { Objects: PAGE_CACHE_EXTENSIONS.map((ext) => ({ Key: `${pageKey}/${cacheKey}.${ext}` })) }
@@ -172,6 +199,17 @@ export class S3Cache implements CacheStrategy {
   async deleteAllByKeyMatch(pageKey: string, cacheKey: string): Promise<void> {
     if (cacheKey) {
       await this.deleteObjects(PAGE_CACHE_EXTENSIONS.map((ext) => `${pageKey}/${cacheKey}.${ext}`))
+      await this.#dynamoDBClient.deleteItem({
+        TableName: process.env.DYNAMODB_CACHE_TABLE!,
+        Key: {
+          pageKey: {
+            S: pageKey
+          },
+          cacheKey: {
+            S: cacheKey
+          }
+        }
+      })
       return
     }
     const keysToDelete: string[] = []