From fc277cace051b9523cceaedef22eee06ada567f9 Mon Sep 17 00:00:00 2001 From: Michael Dodsworth Date: Wed, 29 Jan 2025 13:18:18 -0800 Subject: [PATCH 1/2] feat: adding valkey container, heavily based on the existing redis container --- docs/modules/valkey.md | 41 ++++++ mkdocs.yml | 1 + package-lock.json | 131 ++++++++++++++++++ packages/modules/valkey/jest.config.ts | 11 ++ packages/modules/valkey/package.json | 38 +++++ packages/modules/valkey/src/import.sh | 4 + packages/modules/valkey/src/index.ts | 1 + packages/modules/valkey/src/initData.valkey | 2 + .../valkey/src/valkey-container.test.ts | 104 ++++++++++++++ .../modules/valkey/src/valkey-container.ts | 103 ++++++++++++++ packages/modules/valkey/tsconfig.build.json | 9 ++ packages/modules/valkey/tsconfig.json | 16 +++ 12 files changed, 461 insertions(+) create mode 100644 docs/modules/valkey.md create mode 100644 packages/modules/valkey/jest.config.ts create mode 100644 packages/modules/valkey/package.json create mode 100644 packages/modules/valkey/src/import.sh create mode 100644 packages/modules/valkey/src/index.ts create mode 100644 packages/modules/valkey/src/initData.valkey create mode 100644 packages/modules/valkey/src/valkey-container.test.ts create mode 100644 packages/modules/valkey/src/valkey-container.ts create mode 100644 packages/modules/valkey/tsconfig.build.json create mode 100644 packages/modules/valkey/tsconfig.json diff --git a/docs/modules/valkey.md b/docs/modules/valkey.md new file mode 100644 index 000000000..5c49c80ee --- /dev/null +++ b/docs/modules/valkey.md @@ -0,0 +1,41 @@ +# Valkey Module + +[Valkey](https://valkey.io/) is a distributed, in-memory, key-value store. + +## Install + +```bash +npm install @testcontainers/valkey --save-dev +``` + +## Examples + + + +[Start container:](../../packages/modules/valkey/src/valkey-container.test.ts) inside_block:startContainer + + + + + +[Connect valkey client to container:](../../packages/modules/valkey/src/valkey-container.test.ts) inside_block:simpleConnect + + + + + +[Start container with password authentication:](../../packages/modules/valkey/src/valkey-container.test.ts) inside_block:startWithCredentials + + + + + +[Define volume for persistent/predefined data:](../../packages/modules/valkey/src/valkey-container.test.ts) inside_block:persistentData + + + + + +[Execute a command inside the container:](../../packages/modules/valkey/src/valkey-container.test.ts) inside_block:executeCommand + + diff --git a/mkdocs.yml b/mkdocs.yml index 92091d744..d9372ea17 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -71,5 +71,6 @@ nav: - ScyllaDB: modules/scylladb.md - Selenium: modules/selenium.md - ToxiProxy: modules/toxiproxy.md + - Valkey: modules/valkey.md - Weaviate: modules/weaviate.md - Configuration: configuration.md diff --git a/package-lock.json b/package-lock.json index b5a7de8ec..a34f1fc7e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5854,6 +5854,10 @@ "resolved": "packages/modules/toxiproxy", "link": true }, + "node_modules/@testcontainers/valkey": { + "resolved": "packages/modules/valkey", + "link": true + }, "node_modules/@testcontainers/weaviate": { "resolved": "packages/modules/weaviate", "link": true @@ -20490,6 +20494,120 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/valkey": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/valkey/-/valkey-0.0.3.tgz", + "integrity": "sha512-rQu/DB1pFoOpm2qoPSNhDGsAcA7TmjHskhK+m05IwHqlvGAxAoqeGD0N62wsS3rm+Uf2EqGG4F0EjJWYwmOJNw==", + "dev": true, + "license": "MIT", + "workspaces": [ + "./packages/*" + ], + "dependencies": { + "valkey-bloom": "0.0.1", + "valkey-client": "0.0.1", + "valkey-graph": "0.0.1", + "valkey-json": "0.0.1", + "valkey-search": "0.0.1", + "valkey-time-series": "0.0.1" + } + }, + "node_modules/valkey-bloom": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/valkey-bloom/-/valkey-bloom-0.0.1.tgz", + "integrity": "sha512-O7QrCQ92KYm6nSCxmAAfsnQWskVzbLZVFuW6hWacFWBfJilBd3JVpGDx4W9CIFHSM6RlfKV7p/YWoZvsZVhEPg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "valkey-client": "^1.0.0" + } + }, + "node_modules/valkey-client": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/valkey-client/-/valkey-client-1.0.0.tgz", + "integrity": "sha512-QqzWDZiNMpPON/iRl4QGQQ30Pq4/GWIfxbQ9qJJSYer3fbcarsFB0wxW/t4erzdAFYU/NIfHOprRL+eoyy9LVQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/valkey-client/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true, + "license": "ISC", + "peer": true + }, + "node_modules/valkey-graph": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/valkey-graph/-/valkey-graph-0.0.1.tgz", + "integrity": "sha512-ajPqDh3vp14NX1zaR1MX/R/8x6alfpJJ1l+kOF1kD4SqEwse9JDGNum6B2ajiuPJl6Wgu+9eyw0kcFHGsMjYBw==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "valkey-client": "^1.0.0" + } + }, + "node_modules/valkey-json": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/valkey-json/-/valkey-json-0.0.1.tgz", + "integrity": "sha512-PgipELi66j8lbXgYlet2L40HHmdx9LZSt7O55GJWUFXok5Lb/hYhCveX8Y4fHDFrIoxb8SdIDAnJvRWhvUiUNQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "valkey-client": "^1.0.0" + } + }, + "node_modules/valkey-search": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/valkey-search/-/valkey-search-0.0.1.tgz", + "integrity": "sha512-6KLxQtObxJUSuPIR+kMDB2U8PN5pX0ARnMRnjDzm2Mh5I1Yz2OZ1tMbRXEkj5EF3UcRmkL9hgt7Ws4T73VxTmA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "valkey-client": "^1.0.0" + } + }, + "node_modules/valkey-time-series": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/valkey-time-series/-/valkey-time-series-0.0.1.tgz", + "integrity": "sha512-NvlSgQyxF+TtCGyctbUPMssZYHJ4w34a02XcYENAtrAJrJw11Qx4wBqtXE2us1I+qfNtuLBEMJqam/72JMRrIg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "valkey-client": "^1.0.0" + } + }, + "node_modules/valkey/node_modules/valkey-client": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/valkey-client/-/valkey-client-0.0.1.tgz", + "integrity": "sha512-nhiKE3GHpZ68N7Dhze23KXzqL6wdgalcaHTk7O0dUUnlRstyXwtywFzsI5ugPeyF5H1NmkCEpMFhbXHQ7r0j3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/valkey/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true, + "license": "ISC" + }, "node_modules/walk-up-path": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/walk-up-path/-/walk-up-path-3.0.1.tgz", @@ -21479,6 +21597,19 @@ "redis": "^4.7.0" } }, + "packages/modules/valkey": { + "name": "@testcontainers/valkey", + "version": "10.17.1", + "license": "MIT", + "dependencies": { + "testcontainers": "^10.17.1" + }, + "devDependencies": { + "@types/redis": "^4.0.11", + "redis": "^4.6.15", + "valkey": "^0.0.3" + } + }, "packages/modules/weaviate": { "name": "@testcontainers/weaviate", "version": "10.17.2", diff --git a/packages/modules/valkey/jest.config.ts b/packages/modules/valkey/jest.config.ts new file mode 100644 index 000000000..1f677baaf --- /dev/null +++ b/packages/modules/valkey/jest.config.ts @@ -0,0 +1,11 @@ +import type { Config } from "jest"; +import * as path from "path"; + +const config: Config = { + preset: "ts-jest", + moduleNameMapper: { + "^testcontainers$": path.resolve(__dirname, "../../testcontainers/src"), + }, +}; + +export default config; diff --git a/packages/modules/valkey/package.json b/packages/modules/valkey/package.json new file mode 100644 index 000000000..886122e76 --- /dev/null +++ b/packages/modules/valkey/package.json @@ -0,0 +1,38 @@ +{ + "name": "@testcontainers/valkey", + "version": "10.17.1", + "license": "MIT", + "keywords": [ + "valkey", + "testing", + "docker", + "testcontainers" + ], + "description": "Valkey module for Testcontainers", + "homepage": "https://github.com/testcontainers/testcontainers-node#readme", + "repository": { + "type": "git", + "url": "https://github.com/testcontainers/testcontainers-node" + }, + "bugs": { + "url": "https://github.com/testcontainers/testcontainers-node/issues" + }, + "main": "build/index.js", + "files": [ + "build" + ], + "publishConfig": { + "access": "public" + }, + "scripts": { + "prepack": "shx cp ../../../README.md . && shx cp ../../../LICENSE .", + "build": "tsc --project tsconfig.build.json" + }, + "devDependencies": { + "@types/redis": "^4.0.11", + "redis": "^4.6.15" + }, + "dependencies": { + "testcontainers": "^10.17.1" + } +} diff --git a/packages/modules/valkey/src/import.sh b/packages/modules/valkey/src/import.sh new file mode 100644 index 000000000..83c070b90 --- /dev/null +++ b/packages/modules/valkey/src/import.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -e +valkey-cli $([[ -n "$1" ]] && echo "-a $1") < "/tmp/import.valkey" +echo "Imported" \ No newline at end of file diff --git a/packages/modules/valkey/src/index.ts b/packages/modules/valkey/src/index.ts new file mode 100644 index 000000000..e9cb9c23e --- /dev/null +++ b/packages/modules/valkey/src/index.ts @@ -0,0 +1 @@ +export { ValkeyContainer, StartedValkeyContainer } from "./valkey-container"; diff --git a/packages/modules/valkey/src/initData.valkey b/packages/modules/valkey/src/initData.valkey new file mode 100644 index 000000000..fa89e52ed --- /dev/null +++ b/packages/modules/valkey/src/initData.valkey @@ -0,0 +1,2 @@ +SET "user:001" '{"first_name":"John","last_name":"Doe","dob":"12-JUN-1970"}' +SET "user:002" '{"first_name":"David","last_name":"Bloom","dob":"03-MAR-1981"}' \ No newline at end of file diff --git a/packages/modules/valkey/src/valkey-container.test.ts b/packages/modules/valkey/src/valkey-container.test.ts new file mode 100644 index 000000000..3f8702e25 --- /dev/null +++ b/packages/modules/valkey/src/valkey-container.test.ts @@ -0,0 +1,104 @@ +import { createClient } from "redis"; +import { ValkeyContainer, StartedValkeyContainer } from "./valkey-container"; +import * as os from "os"; +import * as path from "path"; +import * as fs from "fs"; + +describe("ValkeyContainer", () => { + jest.setTimeout(240_000); + + it("should connect and execute set-get", async () => { + const container = await new ValkeyContainer().start(); + + const client = await connectTo(container); + + await client.set("key", "val"); + expect(await client.get("key")).toBe("val"); + + await client.disconnect(); + await container.stop(); + }); + + it("should connect with password and execute set-get", async () => { + const container = await new ValkeyContainer().withPassword("test").start(); + + const client = await connectTo(container); + + await client.set("key", "val"); + expect(await client.get("key")).toBe("val"); + + await client.disconnect(); + await container.stop(); + }); + + it("should reconnect with volume and persistence data", async () => { + const sourcePath = fs.mkdtempSync(path.join(os.tmpdir(), "valkey-")); + const container = await new ValkeyContainer().withPassword("test").withPersistence(sourcePath).start(); + let client = await connectTo(container); + + await client.set("key", "val"); + await client.disconnect(); + await container.restart(); + client = await connectTo(container); + expect(await client.get("key")).toBe("val"); + + await client.disconnect(); + await container.stop(); + try { + fs.rmSync(sourcePath, { force: true, recursive: true }); + } catch (e) { + //Ignore clean up, when have no access on fs. + console.log(e); + } + }); + + it("should load initial data and can read it", async () => { + const container = await new ValkeyContainer() + .withPassword("test") + .withInitialData(path.join(__dirname, "initData.valkey")) + .start(); + const client = await connectTo(container); + const user = { + first_name: "David", + last_name: "Bloom", + dob: "03-MAR-1981", + }; + expect(await client.get("user:002")).toBe(JSON.stringify(user)); + + await client.disconnect(); + await container.stop(); + }); + + it("should start with credentials and login", async () => { + const password = "testPassword"; + + const container = await new ValkeyContainer().withPassword(password).start(); + expect(container.getConnectionUrl()).toEqual(`redis://:${password}@${container.getHost()}:${container.getPort()}`); + + const client = await connectTo(container); + + await client.set("key", "val"); + expect(await client.get("key")).toBe("val"); + + await client.disconnect(); + await container.stop(); + }); + + it("should execute container cmd and return the result", async () => { + const container = await new ValkeyContainer().start(); + + const queryResult = await container.executeCliCmd("info", ["clients"]); + expect(queryResult).toEqual(expect.stringContaining("connected_clients:1")); + + await container.stop(); + }); + + async function connectTo(container: StartedValkeyContainer) { + const client = createClient({ + url: container.getConnectionUrl(), + }); + await client.connect(); + expect(client.isOpen).toBeTruthy(); + return client; + } +}); diff --git a/packages/modules/valkey/src/valkey-container.ts b/packages/modules/valkey/src/valkey-container.ts new file mode 100644 index 000000000..5d879a2e9 --- /dev/null +++ b/packages/modules/valkey/src/valkey-container.ts @@ -0,0 +1,103 @@ +import { AbstractStartedContainer, GenericContainer, StartedTestContainer, Wait } from "testcontainers"; +import path from "path"; + +const VALKEY_PORT = 6379; + +export class ValkeyContainer extends GenericContainer { + private readonly importFilePath = "/tmp/import.valkey"; + private password? = ""; + private persistenceVolume? = ""; + private initialImportScriptFile? = ""; + + constructor(image = "valkey/valkey:8.0") { + super(image); + this.withExposedPorts(VALKEY_PORT) + .withStartupTimeout(120_000) + .withWaitStrategy(Wait.forLogMessage("Ready to accept connections")); + } + + public withPassword(password: string): this { + this.password = password; + return this; + } + + public withPersistence(sourcePath: string): this { + this.persistenceVolume = sourcePath; + return this; + } + + public withInitialData(importScriptFile: string): this { + this.initialImportScriptFile = importScriptFile; + return this; + } + + public override async start(): Promise { + this.withCommand([ + "valkey-server", + ...(this.password ? [`--requirepass "${this.password}"`] : []), + ...(this.persistenceVolume ? ["--save 1 1 ", "--appendonly yes"] : []), + ]); + if (this.persistenceVolume) { + this.withBindMounts([{ mode: "rw", source: this.persistenceVolume, target: "/data" }]); + } + if (this.initialImportScriptFile) { + this.withCopyFilesToContainer([ + { + mode: 666, + source: this.initialImportScriptFile, + target: this.importFilePath, + }, + { + mode: 777, + source: path.join(__dirname, "import.sh"), + target: "/tmp/import.sh", + }, + ]); + } + const startedContainer = new StartedValkeyContainer(await super.start(), this.password); + if (this.initialImportScriptFile) await this.importInitialData(startedContainer); + return startedContainer; + } + + private async importInitialData(container: StartedValkeyContainer) { + const re = await container.exec(`/tmp/import.sh ${this.password || ""}`); + if (re.exitCode !== 0 || re.output.includes("ERR")) { + throw Error(`Could not import initial data from ${this.initialImportScriptFile}: ${re.output}`); + } + } +} + +export class StartedValkeyContainer extends AbstractStartedContainer { + constructor(startedTestContainer: StartedTestContainer, private readonly password?: string) { + super(startedTestContainer); + } + + public getPort(): number { + return this.getMappedPort(VALKEY_PORT); + } + + public getPassword(): string { + return this.password ? this.password.toString() : ""; + } + + public getConnectionUrl(): string { + const url = new URL("", "redis://"); + url.hostname = this.getHost(); + url.port = this.getPort().toString(); + url.password = this.getPassword(); + return url.toString(); + } + + public async executeCliCmd(cmd: string, additionalFlags: string[] = []): Promise { + const result = await this.startedTestContainer.exec([ + "redis-cli", + ...(this.password != "" ? [`-a ${this.password}`] : []), + `${cmd}`, + ...additionalFlags, + ]); + if (result.exitCode !== 0) { + throw new Error(`executeQuery failed with exit code ${result.exitCode} for query: ${cmd}. ${result.output}`); + } + return result.output; + } +} diff --git a/packages/modules/valkey/tsconfig.build.json b/packages/modules/valkey/tsconfig.build.json new file mode 100644 index 000000000..82b71d92e --- /dev/null +++ b/packages/modules/valkey/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["build", "jest.config.ts", "src/**/*.test.ts"], + "references": [ + { + "path": "../../testcontainers" + } + ] +} diff --git a/packages/modules/valkey/tsconfig.json b/packages/modules/valkey/tsconfig.json new file mode 100644 index 000000000..9ac528252 --- /dev/null +++ b/packages/modules/valkey/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "build", + "paths": { + "testcontainers": ["../../testcontainers/src"] + } + }, + "exclude": ["build", "jest.config.ts"], + "references": [ + { + "path": "../../testcontainers" + } + ] +} From 840aac39092ed3f3f22a148589aa370836d27c35 Mon Sep 17 00:00:00 2001 From: Michael Dodsworth Date: Sun, 2 Feb 2025 15:55:47 -0800 Subject: [PATCH 2/2] feat: using containerStarted lifecycle function to import data in valkey and redis containers --- packages/modules/redis/src/redis-container.ts | 8 +++++++- packages/modules/valkey/src/valkey-container.ts | 13 +++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/modules/redis/src/redis-container.ts b/packages/modules/redis/src/redis-container.ts index f04f124e0..a45fe4b62 100644 --- a/packages/modules/redis/src/redis-container.ts +++ b/packages/modules/redis/src/redis-container.ts @@ -32,6 +32,12 @@ export class RedisContainer extends GenericContainer { return this; } + protected override async containerStarted(container: StartedTestContainer): Promise { + if (this.initialImportScriptFile) { + await this.importInitialData(container); + } + } + public override async start(): Promise { this.withCommand([ "redis-server", @@ -60,7 +66,7 @@ export class RedisContainer extends GenericContainer { return startedRedisContainer; } - private async importInitialData(container: StartedRedisContainer) { + private async importInitialData(container: StartedTestContainer) { const re = await container.exec(`/tmp/import.sh ${this.password}`); if (re.exitCode != 0 || re.output.includes("ERR")) throw Error(`Could not import initial data from ${this.initialImportScriptFile}: ${re.output}`); diff --git a/packages/modules/valkey/src/valkey-container.ts b/packages/modules/valkey/src/valkey-container.ts index 5d879a2e9..fdb14318d 100644 --- a/packages/modules/valkey/src/valkey-container.ts +++ b/packages/modules/valkey/src/valkey-container.ts @@ -31,6 +31,12 @@ export class ValkeyContainer extends GenericContainer { return this; } + protected override async containerStarted(container: StartedTestContainer): Promise { + if (this.initialImportScriptFile) { + await this.importInitialData(container); + } + } + public override async start(): Promise { this.withCommand([ "valkey-server", @@ -54,12 +60,11 @@ export class ValkeyContainer extends GenericContainer { }, ]); } - const startedContainer = new StartedValkeyContainer(await super.start(), this.password); - if (this.initialImportScriptFile) await this.importInitialData(startedContainer); - return startedContainer; + + return new StartedValkeyContainer(await super.start(), this.password); } - private async importInitialData(container: StartedValkeyContainer) { + private async importInitialData(container: StartedTestContainer) { const re = await container.exec(`/tmp/import.sh ${this.password || ""}`); if (re.exitCode !== 0 || re.output.includes("ERR")) { throw Error(`Could not import initial data from ${this.initialImportScriptFile}: ${re.output}`);