feat: add KV storage adapters (#9913)

Adds KV storage support to Payload Local API with an adapter pattern,
includes 3 adapters. We'll use it for the Realtime API.
You can access it via `payload.kv`:

```ts
interface KVAdapter {
  /**
   * Clears all entries in the store.
   * @returns A promise that resolves once the store is cleared.
   */
  clear(): Promise<void>

  /**
   * Deletes a value from the store by its key.
   * @param key - The key to delete.
   * @returns A promise that resolves once the key is deleted.
   */
  delete(key: string): Promise<void>

  /**
   * Retrieves a value from the store by its key.
   * @param key - The key to look up.
   * @returns A promise that resolves to the value, or `null` if not found.
   */
  get(key: string): Promise<KVStoreValue | null>

  /**
   * Checks if a key exists in the store.
   * @param key - The key to check.
   * @returns A promise that resolves to `true` if the key exists, otherwise `false`.
   */
  has(key: string): Promise<boolean>

  /**
   * Retrieves all the keys in the store.
   * @returns A promise that resolves to an array of keys.
   */
  keys(): Promise<string[]>

  /**
   * Sets a value in the store with the given key.
   * @param key - The key to associate with the value.
   * @param value - The value to store.
   * @returns A promise that resolves once the value is stored.
   */
  set(key: string, value: KVStoreValue): Promise<void>
}
```

To configure the adapter you can use the `kv` property of the root
config.
```ts
buildConfig({
  kv: adapter()
})
```

#### Database KV adapter (default)
No need to configure, as Payload uses it by default. It generates new
collection `payload-kv` and uses the current database adapter to access
it. The collection is hidden in the admin panel and access locked.
If you want to  override the generated collection:
```ts
import { databaseKVAdapter } from 'payload'

buildConfig({
  kv: databaseKVAdapter({
    kvCollectionOverrides: {
      slug: 'custom-kv',
      ...(process.env.DEBUG === 'true' && {
        admin: { hidden: false },
        access: {},
      }),
    },
  }),
})
```

#### In Memory KV adapter
Simple and very fast storage using memory. Don't use it on Vercel /
multiple instances or if you need data persistence.
```ts
import { inMemoryKVAdapter } from 'payload'

buildConfig({
  kv: inMemoryKVAdapter(),
})
```

#### Redis KV Adapter
Uses Redis. Probably the best option as it's faster than database,
persistent and works with Vercel / multiple instances, but requires
additional setup
```sh
pnpm add @payloadcms/kv-redis
```

```ts
import { redisKVAdapter } from '@payloadcms/kv-redis'

buildConfig({
  kv: redisKVAdapter({
    keyPrefix: "custom-prefix:", // defaults to 'payload-kv:'
    redisURL: "redis://127.0.0.1:6379" // defaults to process.env.REDIS_URL (Vercel generates this variable for you if you connect a project to Redis)
  }),
})
```

Author: r1tsuu

.github/workflows/main.yml

@@ -185,6 +185,13 @@ jobs:
         # needed because the postgres container does not provide a healthcheck
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
+      redis:
+        image: redis:latest
+        ports:
+          - 6379:6379
+
+        options: --health-cmd "redis-cli ping" --health-timeout 30s --health-retries 3
+
     steps:
       - uses: actions/checkout@v5
 
@@ -232,6 +239,10 @@ jobs:
           echo "POSTGRES_URL=postgresql://postgres:[email protected]:54322/postgres" >> $GITHUB_ENV
         if: matrix.database == 'supabase'
 
+      - name: Configure Redis
+        run: |
+          echo  "REDIS_URL=redis://127.0.0.1:6379" >> $GITHUB_ENV
+
       - name: Integration Tests
         run: pnpm test:int
         env:

package.json

@@ -31,6 +31,7 @@
     "build:essentials:force": "pnpm clean:build && turbo build --filter=\"payload...\" --filter=\"@payloadcms/ui\" --filter=\"@payloadcms/next\" --filter=\"@payloadcms/db-mongodb\" --filter=\"@payloadcms/db-postgres\" --filter=\"@payloadcms/richtext-lexical\" --filter=\"@payloadcms/translations\" --filter=\"@payloadcms/plugin-cloud\" --filter=\"@payloadcms/graphql\" --no-cache --force",
     "build:force": "pnpm run build:core:force",
     "build:graphql": "turbo build --filter \"@payloadcms/graphql\"",
+    "build:kv-redis": "turbo build --filter \"@payloadcms/kv-redis\"",
     "build:live-preview": "turbo build --filter \"@payloadcms/live-preview\"",
     "build:live-preview-react": "turbo build --filter \"@payloadcms/live-preview-react\"",
     "build:live-preview-vue": "turbo build --filter \"@payloadcms/live-preview-vue\"",

packages/kv-redis/.prettierignore

@@ -0,0 +1,10 @@
+.tmp
+**/.git
+**/.hg
+**/.pnp.*
+**/.svn
+**/.yarn/**
+**/build
+**/dist/**
+**/node_modules
+**/temp

packages/kv-redis/.swcrc

@@ -0,0 +1,15 @@
+{
+  "$schema": "https://json.schemastore.org/swcrc",
+  "sourceMaps": true,
+  "jsc": {
+    "target": "esnext",
+    "parser": {
+      "syntax": "typescript",
+      "tsx": true,
+      "dts": true
+    }
+  },
+  "module": {
+    "type": "es6"
+  }
+}

packages/kv-redis/LICENSE.md

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2018-2024 Payload CMS, Inc. <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

packages/kv-redis/README.md

@@ -0,0 +1,33 @@
+# Redis KV Adapter for Payload (beta)
+
+This package provides a way to use [Redis](https://redis.io) as a KV adapter with Payload.
+
+## Installation
+
+```sh
+pnpm add @payloadcms/kv-redis
+```
+
+## Usage
+
+```ts
+import { redisKVAdapter } from '@payloadcms/kv-redis'
+
+export default buildConfig({
+  collections: [Media],
+  kv: redisKVAdapter({
+    // Redis connection URL. Defaults to process.env.REDIS_URL
+    redisURL: 'redis://localhost:6379',
+    // Optional prefix for Redis keys to isolate the store. Defaults to 'payload-kv'
+    keyPrefix: 'kv-storage',
+  }),
+})
+```
+
+Then you can access the KV storage using `payload.kv`:
+
+```ts
+await payload.kv.set('key', { value: 1 })
+const data = await payload.kv.get('key')
+payload.logger.info(data)
+```

packages/kv-redis/eslint.config.js

@@ -0,0 +1,18 @@
+import { rootEslintConfig, rootParserOptions } from '../../eslint.config.js'
+
+/** @typedef {import('eslint').Linter.Config} Config */
+
+/** @type {Config[]} */
+export const index = [
+  ...rootEslintConfig,
+  {
+    languageOptions: {
+      parserOptions: {
+        ...rootParserOptions,
+        tsconfigRootDir: import.meta.dirname,
+      },
+    },
+  },
+]
+
+export default index

packages/kv-redis/package.json

@@ -0,0 +1,66 @@
+{
+  "name": "@payloadcms/kv-redis",
+  "version": "3.6.0",
+  "description": "Redis KV adapter for Payload",
+  "homepage": "https://payloadcms.com",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/payloadcms/payload.git",
+    "directory": "packages/kv-redis"
+  },
+  "license": "MIT",
+  "author": "Payload <[email protected]> (https://payloadcms.com)",
+  "maintainers": [
+    {
+      "name": "Payload",
+      "email": "[email protected]",
+      "url": "https://payloadcms.com"
+    }
+  ],
+  "type": "module",
+  "exports": {
+    ".": {
+      "import": "./src/index.ts",
+      "types": "./src/index.ts",
+      "default": "./src/index.ts"
+    }
+  },
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "pnpm build:types && pnpm build:swc",
+    "build:clean": "find . \\( -type d \\( -name build -o -name dist -o -name .cache \\) -o -type f -name tsconfig.tsbuildinfo \\) -exec rm -rf {} + && pnpm build",
+    "build:swc": "swc ./src -d ./dist --config-file .swcrc --strip-leading-paths",
+    "build:types": "tsc --emitDeclarationOnly --outDir dist",
+    "clean": "rimraf {dist,*.tsbuildinfo}",
+    "lint": "eslint .",
+    "lint:fix": "eslint . --fix",
+    "prepublishOnly": "pnpm clean && pnpm turbo build"
+  },
+  "dependencies": {
+    "ioredis": "^5.4.1"
+  },
+  "devDependencies": {
+    "payload": "workspace:*"
+  },
+  "peerDependencies": {
+    "payload": "workspace:*"
+  },
+  "engines": {
+    "node": "^18.20.2 || >=20.9.0"
+  },
+  "publishConfig": {
+    "exports": {
+      ".": {
+        "import": "./dist/index.js",
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      }
+    },
+    "main": "./dist/index.js",
+    "types": "./dist/index.d.ts"
+  }
+}

packages/kv-redis/src/index.ts

@@ -0,0 +1,79 @@
+import type { KVAdapter, KVAdapterResult, KVStoreValue } from 'payload'
+
+import { Redis } from 'ioredis'
+
+export class RedisKVAdapter implements KVAdapter {
+  redisClient: Redis
+
+  constructor(
+    readonly keyPrefix: string,
+    redisURL: string,
+  ) {
+    this.redisClient = new Redis(redisURL)
+  }
+
+  async clear(): Promise<void> {
+    const keys = await this.redisClient.keys(`${this.keyPrefix}*`)
+
+    if (keys.length > 0) {
+      await this.redisClient.del(keys)
+    }
+  }
+
+  async delete(key: string): Promise<void> {
+    await this.redisClient.del(`${this.keyPrefix}${key}`)
+  }
+
+  async get(key: string): Promise<KVStoreValue | null> {
+    const data = await this.redisClient.get(`${this.keyPrefix}${key}`)
+
+    if (data === null) {
+      return data
+    }
+
+    return JSON.parse(data)
+  }
+
+  async has(key: string): Promise<boolean> {
+    const exists = await this.redisClient.exists(`${this.keyPrefix}${key}`)
+    return exists === 1
+  }
+
+  async keys(): Promise<string[]> {
+    const prefixedKeys = await this.redisClient.keys(`${this.keyPrefix}*`)
+
+    if (this.keyPrefix) {
+      return prefixedKeys.map((key) => key.replace(this.keyPrefix, ''))
+    }
+
+    return prefixedKeys
+  }
+
+  async set(key: string, data: KVStoreValue): Promise<void> {
+    await this.redisClient.set(`${this.keyPrefix}${key}`, JSON.stringify(data))
+  }
+}
+
+export type RedisKVAdapterOptions = {
+  /**
+   * Optional prefix for Redis keys to isolate the store
+   *
+   * @default 'payload-kv:'
+   */
+  keyPrefix?: string
+  /** Redis connection URL (e.g., 'redis://localhost:6379'). Defaults to process.env.REDIS_URL */
+  redisURL?: string
+}
+
+export const redisKVAdapter = (options: RedisKVAdapterOptions = {}): KVAdapterResult => {
+  const keyPrefix = options.keyPrefix ?? 'payload-kv:'
+  const redisURL = options.redisURL ?? process.env.REDIS_URL
+
+  if (!redisURL) {
+    throw new Error('redisURL or REDIS_URL env variable is required')
+  }
+
+  return {
+    init: () => new RedisKVAdapter(keyPrefix, redisURL),
+  }
+}

packages/kv-redis/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "composite": true, // Make sure typescript knows that this module depends on their references
+    "noEmit": false /* Do not emit outputs. */,
+    "emitDeclarationOnly": true,
+    "outDir": "./dist" /* Specify an output folder for all emitted files. */,
+    "rootDir": "./src" /* Specify the root folder within your source files. */,
+    "strict": true
+  },
+  "exclude": ["dist", "node_modules"],
+  "include": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.d.ts", "src/**/*.json"],
+  "references": [{ "path": "../payload" }]
+}