From 23fc5c3550fd240d08a6f882ae09f051098e3d00 Mon Sep 17 00:00:00 2001 From: Victorlouisdg Date: Wed, 9 Apr 2025 18:11:55 +0200 Subject: [PATCH 1/5] feat: Add ClickHouse module initial implementation --- docs/modules/clickhouse.md | 48 ++++++ mkdocs.yml | 1 + package-lock.json | 35 ++++ packages/modules/clickhouse/package.json | 37 ++++ .../src/clickhouse-container.test.ts | 158 ++++++++++++++++++ .../clickhouse/src/clickhouse-container.ts | 106 ++++++++++++ packages/modules/clickhouse/src/index.ts | 1 + .../modules/clickhouse/tsconfig.build.json | 12 ++ packages/modules/clickhouse/tsconfig.json | 8 + 9 files changed, 406 insertions(+) create mode 100644 docs/modules/clickhouse.md create mode 100644 packages/modules/clickhouse/package.json create mode 100644 packages/modules/clickhouse/src/clickhouse-container.test.ts create mode 100644 packages/modules/clickhouse/src/clickhouse-container.ts create mode 100644 packages/modules/clickhouse/src/index.ts create mode 100644 packages/modules/clickhouse/tsconfig.build.json create mode 100644 packages/modules/clickhouse/tsconfig.json diff --git a/docs/modules/clickhouse.md b/docs/modules/clickhouse.md new file mode 100644 index 000000000..133ee3531 --- /dev/null +++ b/docs/modules/clickhouse.md @@ -0,0 +1,48 @@ +# ClickHouse Module + +[ClickHouse](https://clickhouse.com/) is a column-oriented database management system for online analytical processing (OLAP) that allows users to generate analytical reports using SQL queries in real-time. + +## Install + +```bash +npm install @testcontainers/clickhouse --save-dev +``` + +## Examples + + + +[Connect and execute query:](../../packages/modules/clickhouse/src/clickhouse-container.test.ts) inside_block:connect + + + + + +[Connect and execute query using HTTP interface:](../../packages/modules/clickhouse/src/clickhouse-container.test.ts) inside_block:httpConnect + + + + + +[Set database:](../../packages/modules/clickhouse/src/clickhouse-container.test.ts) inside_block:setDatabase + + + + + +[Set username:](../../packages/modules/clickhouse/src/clickhouse-container.test.ts) inside_block:setUsername + + + +### Connection URIs + +The module provides two methods to get connection URIs: + +1. `getConnectionUri()` - Returns a URI in the form of `clickhouse://[username[:password]@][host[:port],]/database` +2. `getHttpConnectionUri()` - Returns a URI in the form of `http://[username[:password]@][host[:port]]` + +These URIs can be used with the `@clickhouse/client` package or any other ClickHouse client that supports connection URIs. + +!!!tip +The HTTP interface (port 8123) is often easier to use for simple queries and is more widely supported by various clients. +The native interface (port 9000) offers better performance for high-throughput scenarios. diff --git a/mkdocs.yml b/mkdocs.yml index be01dd2ca..60b544f3b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -51,6 +51,7 @@ nav: - Azurite: modules/azurite.md - Cassandra: modules/cassandra.md - ChromaDB: modules/chromadb.md + - ClickHouse: modules/clickhouse.md - CosmosDB: modules/cosmosdb.md - Couchbase: modules/couchbase.md - CockroachDB: modules/cockroachdb.md diff --git a/package-lock.json b/package-lock.json index bcd5713b4..6cf5bdf92 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1552,6 +1552,26 @@ "tough-cookie": "^4.1.4" } }, + "node_modules/@clickhouse/client": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@clickhouse/client/-/client-1.11.0.tgz", + "integrity": "sha512-VYTQfR0y/BtrIDEjuSce1zv85OvHak5sUhZVyNYJzbAgWHy3jFf8Os7FdUSeqyKav0xGGy+2X+dRanTFjI5Oug==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@clickhouse/client-common": "1.11.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@clickhouse/client-common": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@clickhouse/client-common/-/client-common-1.11.0.tgz", + "integrity": "sha512-O0xbwv7HiMXayokrf5dYIBpjBnYekcOXWz60T1cXLmiZ8vgrfNRCiOpybJkrMXKnw9D0mWCgPUu/rgMY7U1f4g==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/@colors/colors": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", @@ -5077,6 +5097,10 @@ "resolved": "packages/modules/chromadb", "link": true }, + "node_modules/@testcontainers/clickhouse": { + "resolved": "packages/modules/clickhouse", + "link": true + }, "node_modules/@testcontainers/cockroachdb": { "resolved": "packages/modules/cockroachdb", "link": true @@ -19816,6 +19840,17 @@ } } }, + "packages/modules/clickhouse": { + "name": "@testcontainers/clickhouse", + "version": "10.24.1", + "license": "MIT", + "dependencies": { + "testcontainers": "^10.24.1" + }, + "devDependencies": { + "@clickhouse/client": "^1.11.0" + } + }, "packages/modules/cockroachdb": { "name": "@testcontainers/cockroachdb", "version": "10.24.1", diff --git a/packages/modules/clickhouse/package.json b/packages/modules/clickhouse/package.json new file mode 100644 index 000000000..ed8eb91ba --- /dev/null +++ b/packages/modules/clickhouse/package.json @@ -0,0 +1,37 @@ +{ + "name": "@testcontainers/clickhouse", + "version": "10.24.1", + "license": "MIT", + "keywords": [ + "clickhouse", + "testing", + "docker", + "testcontainers" + ], + "description": "ClickHouse module for Testcontainers", + "homepage": "https://github.com/testcontainers/testcontainers-node#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/testcontainers/testcontainers-node.git" + }, + "bugs": { + "url": "https://github.com/testcontainers/testcontainers-node/issues" + }, + "main": "build/index.js", + "files": [ + "build" + ], + "publishConfig": { + "access": "public" + }, + "scripts": { + "prepack": "shx cp ../../../README.md . && shx cp ../../../LICENSE .", + "build": "tsc --project tsconfig.build.json" + }, + "devDependencies": { + "@clickhouse/client": "^1.11.0" + }, + "dependencies": { + "testcontainers": "^10.24.1" + } +} \ No newline at end of file diff --git a/packages/modules/clickhouse/src/clickhouse-container.test.ts b/packages/modules/clickhouse/src/clickhouse-container.test.ts new file mode 100644 index 000000000..c3bd93b11 --- /dev/null +++ b/packages/modules/clickhouse/src/clickhouse-container.test.ts @@ -0,0 +1,158 @@ +import { ClickHouseClient, createClient } from "@clickhouse/client"; +import { ClickHouseContainer } from "./clickhouse-container"; + +describe("ClickHouseContainer", { timeout: 180_000 }, () => { + const sleep = async (waitTime: number) => new Promise((resolve) => setTimeout(resolve, waitTime)); + + // httpConnect { + it("should connect and return a query result", async () => { + const container = await new ClickHouseContainer().start(); + let client: ClickHouseClient | undefined; + console.log("container.getConnectionUri()", container.getConnectionUri()); + // console.log("container.getHttpConnectionUri()", container.getHttpConnectionUri()); + try { + // client = createClient({ + // url: container.getHttpConnectionUri(), + // database: container.getDatabase(), + // username: container.getUsername(), + // password: container.getPassword(), + // request_timeout: 100000, + // }); + + client = createClient({ + url: container.getConnectionUri(), + }); + + console.error("Created client"); + + const result = await client.query({ + query: "SELECT 1 AS value", + format: "JSON", + }); + const data = (await result.json()) as any; + expect(data?.data?.[0]?.value).toBe(1); + } finally { + await client?.close(); + await container.stop(); + } + }); + // } + + // // connect { + // it("should work with database URI", async () => { + // const container = await new ClickHouseContainer().start(); + // let client: ClickHouseClient | undefined; + + // try { + // client = createClient({ + // url: container.getConnectionUri(), + // }); + + // await client.ping(); + + // const result = await client.query({ + // query: "SELECT 1 AS value", + // format: "JSON", + // }); + + // const data = (await result.json()) as any; + // expect(data?.data?.[0]?.value).toBe(1); + // } finally { + // await client?.close(); + // await container.stop(); + // } + // }); + // // } + + // // setDatabase { + // it("should set database", async () => { + // const customDatabase = "customDatabase"; + // const container = await new ClickHouseContainer().withDatabase(customDatabase).start(); + + // let client: ClickHouseClient | undefined; + + // try { + // client = createClient({ + // url: container.getConnectionUri(), + // }); + + // await client.ping(); + + // const result = await client.query({ + // query: "SELECT currentDatabase() AS current_database", + // format: "JSON", + // }); + + // const data = (await result.json()) as any; + // expect(data?.data?.[0]?.current_database).toBe(customDatabase); + // } finally { + // await client?.close(); + // await container.stop(); + // } + // }); + // // } + + // // setUsername { + // it("should set username", async () => { + // const customUsername = "customUsername"; + // const container = await new ClickHouseContainer().withUsername(customUsername).start(); + + // let client: ClickHouseClient | undefined; + + // try { + // client = createClient({ + // url: container.getConnectionUri(), + // }); + + // await client.ping(); + + // const result = await client.query({ + // query: "SELECT currentUser() AS current_user", + // format: "JSON", + // }); + + // const data = (await result.json()) as any; + // expect(data?.data?.[0]?.current_user).toBe(customUsername); + // } finally { + // await client?.close(); + // await container.stop(); + // } + // }); + // // } + + // it("should work with restarted container", async () => { + // const container = await new ClickHouseContainer().start(); + // await container.restart(); + + // let client: ClickHouseClient | undefined; + // try { + // client = createClient({ + // url: container.getConnectionUri(), + // }); + + // await client.ping(); + + // const result = await client.query({ + // query: "SELECT 1 AS value", + // format: "JSON", + // }); + + // const data = (await result.json()) as any; + // expect(data?.data?.[0]?.value).toBe(1); + // } finally { + // await client?.close(); + // await container.stop(); + // } + // }); + + // it("should allow custom healthcheck", async () => { + // const container = new ClickHouseContainer().withHealthCheck({ + // test: ["CMD-SHELL", "exit 1"], + // interval: 100, + // retries: 0, + // timeout: 100, + // }); + + // await expect(() => container.start()).rejects.toThrow(); + // }); +}); diff --git a/packages/modules/clickhouse/src/clickhouse-container.ts b/packages/modules/clickhouse/src/clickhouse-container.ts new file mode 100644 index 000000000..133951110 --- /dev/null +++ b/packages/modules/clickhouse/src/clickhouse-container.ts @@ -0,0 +1,106 @@ +import { AbstractStartedContainer, GenericContainer, StartedTestContainer, Wait } from "testcontainers"; + +const CLICKHOUSE_PORT = 9000; +const CLICKHOUSE_HTTP_PORT = 8123; + +export class ClickHouseContainer extends GenericContainer { + private username = "test"; + private password = "test"; + private database = "test"; + + constructor(image = "clickhouse/clickhouse-server:24") { + super(image); + this.withExposedPorts(CLICKHOUSE_PORT, CLICKHOUSE_HTTP_PORT); + this.withWaitStrategy(Wait.forHealthCheck()); + this.withStartupTimeout(120_000); + this.withDatabase("test"); + this.withUlimits({ + nofile: { + hard: 262144, + soft: 262144, + }, + }); + } + + public withDatabase(database: string): this { + this.database = database; + return this; + } + + public withUsername(username: string): this { + this.username = username; + return this; + } + + public withPassword(password: string): this { + this.password = password; + return this; + } + + public override async start(): Promise { + this.withEnvironment({ + CLICKHOUSE_USER: this.username, + CLICKHOUSE_PASSWORD: this.password, + CLICKHOUSE_DB: this.database, + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1", + }); + + if (!this.healthCheck) { + this.withHealthCheck({ + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://127.0.0.1:8123/ping || exit 1"], + interval: 250, + timeout: 1000, + retries: 1000, + }); + } + + return new StartedClickHouseContainer(await super.start(), this.database, this.username, this.password); + } +} + +export class StartedClickHouseContainer extends AbstractStartedContainer { + constructor( + startedTestContainer: StartedTestContainer, + private readonly database: string, + private readonly username: string, + private readonly password: string + ) { + super(startedTestContainer); + } + + public getPort(): number { + return super.getMappedPort(CLICKHOUSE_PORT); + } + + public getHttpPort(): number { + return super.getMappedPort(CLICKHOUSE_HTTP_PORT); + } + + public getDatabase(): string { + return this.database; + } + + public getUsername(): string { + return this.username; + } + + public getPassword(): string { + return this.password; + } + + public getConnectionUri(): string { + const protocol = "http"; + const host = this.getHost(); + const port = this.getHttpPort(); + + return `${protocol}://${host}:${port}`; + } + + // /** + // * @returns A connection URI for HTTP interface in the form of `http://[username[:password]@][host[:port]]` + // */ + // public getHttpConnectionUri(): string { + // const url = new URL(`http://127.0.0.1:${this.getHttpPort()}`); + // return url.toString(); + // } +} diff --git a/packages/modules/clickhouse/src/index.ts b/packages/modules/clickhouse/src/index.ts new file mode 100644 index 000000000..1882737bc --- /dev/null +++ b/packages/modules/clickhouse/src/index.ts @@ -0,0 +1 @@ +export * from "./clickhouse-container"; diff --git a/packages/modules/clickhouse/tsconfig.build.json b/packages/modules/clickhouse/tsconfig.build.json new file mode 100644 index 000000000..e5121688c --- /dev/null +++ b/packages/modules/clickhouse/tsconfig.build.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "exclude": [ + "build", + "src/**/*.test.ts" + ], + "references": [ + { + "path": "../../testcontainers" + } + ] +} \ No newline at end of file diff --git a/packages/modules/clickhouse/tsconfig.json b/packages/modules/clickhouse/tsconfig.json new file mode 100644 index 000000000..1c0258e13 --- /dev/null +++ b/packages/modules/clickhouse/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "build", + "rootDir": "src" + }, + "include": ["src/**/*"] +} \ No newline at end of file From 6c7986c51adf53055b29b81f23bbddcaf4f3e30d Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Wed, 9 Apr 2025 18:27:22 +0100 Subject: [PATCH 2/5] Use "/" path instead of ping + use HTTP wait strategy --- .../modules/clickhouse/src/clickhouse-container.ts | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/packages/modules/clickhouse/src/clickhouse-container.ts b/packages/modules/clickhouse/src/clickhouse-container.ts index 133951110..06a1c929b 100644 --- a/packages/modules/clickhouse/src/clickhouse-container.ts +++ b/packages/modules/clickhouse/src/clickhouse-container.ts @@ -11,7 +11,9 @@ export class ClickHouseContainer extends GenericContainer { constructor(image = "clickhouse/clickhouse-server:24") { super(image); this.withExposedPorts(CLICKHOUSE_PORT, CLICKHOUSE_HTTP_PORT); - this.withWaitStrategy(Wait.forHealthCheck()); + this.withWaitStrategy( + Wait.forHttp("/", CLICKHOUSE_HTTP_PORT).forResponsePredicate((response) => response === "Ok.\n") + ); this.withStartupTimeout(120_000); this.withDatabase("test"); this.withUlimits({ @@ -45,15 +47,6 @@ export class ClickHouseContainer extends GenericContainer { CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1", }); - if (!this.healthCheck) { - this.withHealthCheck({ - test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://127.0.0.1:8123/ping || exit 1"], - interval: 250, - timeout: 1000, - retries: 1000, - }); - } - return new StartedClickHouseContainer(await super.start(), this.database, this.username, this.password); } } From fb21ab6d77491c14cb4641ed489a0a0d735c66ee Mon Sep 17 00:00:00 2001 From: Victorlouisdg Date: Thu, 10 Apr 2025 10:23:11 +0200 Subject: [PATCH 3/5] feat(clickhouse): improve helper methods to align with @clickhouse/client and fix tests --- docs/modules/clickhouse.md | 10 +- .../src/clickhouse-container.test.ts | 290 ++++++++++-------- .../clickhouse/src/clickhouse-container.ts | 61 +++- 3 files changed, 212 insertions(+), 149 deletions(-) diff --git a/docs/modules/clickhouse.md b/docs/modules/clickhouse.md index 133ee3531..0e9afc16b 100644 --- a/docs/modules/clickhouse.md +++ b/docs/modules/clickhouse.md @@ -12,13 +12,19 @@ npm install @testcontainers/clickhouse --save-dev -[Connect and execute query:](../../packages/modules/clickhouse/src/clickhouse-container.test.ts) inside_block:connect +[Connect and execute query:](../../packages/modules/clickhouse/src/clickhouse-container.test.ts) inside_block:connectWithOptions -[Connect and execute query using HTTP interface:](../../packages/modules/clickhouse/src/clickhouse-container.test.ts) inside_block:httpConnect +[Connect using URL and execute query:](../../packages/modules/clickhouse/src/clickhouse-container.test.ts) inside_block:connectWithUrl + + + + + +[Connect with username and password and execute query:](../../packages/modules/clickhouse/src/clickhouse-container.test.ts) inside_block:connectWithUsernameAndPassword diff --git a/packages/modules/clickhouse/src/clickhouse-container.test.ts b/packages/modules/clickhouse/src/clickhouse-container.test.ts index c3bd93b11..4b0bacdf4 100644 --- a/packages/modules/clickhouse/src/clickhouse-container.test.ts +++ b/packages/modules/clickhouse/src/clickhouse-container.test.ts @@ -1,35 +1,90 @@ import { ClickHouseClient, createClient } from "@clickhouse/client"; +import { Wait } from "testcontainers"; import { ClickHouseContainer } from "./clickhouse-container"; +interface ClickHouseQueryResponse { + data: T[]; +} + describe("ClickHouseContainer", { timeout: 180_000 }, () => { - const sleep = async (waitTime: number) => new Promise((resolve) => setTimeout(resolve, waitTime)); + it("should start successfully with default settings", async () => { + const container = new ClickHouseContainer(); + const startedContainer = await container.start(); + expect(startedContainer.getHost()).toBeDefined(); + expect(startedContainer.getHttpPort()).toBeDefined(); + expect(startedContainer.getDatabase()).toBeDefined(); + expect(startedContainer.getUsername()).toBeDefined(); + expect(startedContainer.getPassword()).toBeDefined(); + await startedContainer.stop(); + }); - // httpConnect { - it("should connect and return a query result", async () => { + // connectWithOptions { + it("should connect using the client options object", async () => { const container = await new ClickHouseContainer().start(); let client: ClickHouseClient | undefined; - console.log("container.getConnectionUri()", container.getConnectionUri()); - // console.log("container.getHttpConnectionUri()", container.getHttpConnectionUri()); + try { - // client = createClient({ - // url: container.getHttpConnectionUri(), - // database: container.getDatabase(), - // username: container.getUsername(), - // password: container.getPassword(), - // request_timeout: 100000, - // }); + client = createClient(container.getClientOptions()); + const result = await client.query({ + query: "SELECT 1 AS value", + format: "JSON", + }); + const data = (await result.json()) as ClickHouseQueryResponse<{ value: number }>; + expect(data?.data?.[0]?.value).toBe(1); + } finally { + await client?.close(); + await container.stop(); + } + }); + // } + + // connectWithUrl { + it("should connect using the URL", async () => { + const container = await new ClickHouseContainer().start(); + let client: ClickHouseClient | undefined; + + try { client = createClient({ - url: container.getConnectionUri(), + url: container.getConnectionUrl(), }); - console.error("Created client"); + const result = await client.query({ + query: "SELECT 1 AS value", + format: "JSON", + }); + + const data = (await result.json()) as ClickHouseQueryResponse<{ value: number }>; + expect(data?.data?.[0]?.value).toBe(1); + } finally { + await client?.close(); + await container.stop(); + } + }); + // } + + // connectWithUsernameAndPassword { + it("should connect using the username and password", async () => { + const container = await new ClickHouseContainer() + .withUsername("customUsername") + .withPassword("customPassword") + .start(); + + let client: ClickHouseClient | undefined; + + try { + client = createClient({ + url: container.getHttpUrl(), + username: container.getUsername(), + password: container.getPassword(), + }); const result = await client.query({ query: "SELECT 1 AS value", format: "JSON", }); - const data = (await result.json()) as any; + + const data = (await result.json()) as ClickHouseQueryResponse<{ value: number }>; expect(data?.data?.[0]?.value).toBe(1); } finally { await client?.close(); @@ -38,121 +93,92 @@ describe("ClickHouseContainer", { timeout: 180_000 }, () => { }); // } - // // connect { - // it("should work with database URI", async () => { - // const container = await new ClickHouseContainer().start(); - // let client: ClickHouseClient | undefined; - - // try { - // client = createClient({ - // url: container.getConnectionUri(), - // }); - - // await client.ping(); - - // const result = await client.query({ - // query: "SELECT 1 AS value", - // format: "JSON", - // }); - - // const data = (await result.json()) as any; - // expect(data?.data?.[0]?.value).toBe(1); - // } finally { - // await client?.close(); - // await container.stop(); - // } - // }); - // // } - - // // setDatabase { - // it("should set database", async () => { - // const customDatabase = "customDatabase"; - // const container = await new ClickHouseContainer().withDatabase(customDatabase).start(); - - // let client: ClickHouseClient | undefined; - - // try { - // client = createClient({ - // url: container.getConnectionUri(), - // }); - - // await client.ping(); - - // const result = await client.query({ - // query: "SELECT currentDatabase() AS current_database", - // format: "JSON", - // }); - - // const data = (await result.json()) as any; - // expect(data?.data?.[0]?.current_database).toBe(customDatabase); - // } finally { - // await client?.close(); - // await container.stop(); - // } - // }); - // // } - - // // setUsername { - // it("should set username", async () => { - // const customUsername = "customUsername"; - // const container = await new ClickHouseContainer().withUsername(customUsername).start(); - - // let client: ClickHouseClient | undefined; - - // try { - // client = createClient({ - // url: container.getConnectionUri(), - // }); - - // await client.ping(); - - // const result = await client.query({ - // query: "SELECT currentUser() AS current_user", - // format: "JSON", - // }); - - // const data = (await result.json()) as any; - // expect(data?.data?.[0]?.current_user).toBe(customUsername); - // } finally { - // await client?.close(); - // await container.stop(); - // } - // }); - // // } - - // it("should work with restarted container", async () => { - // const container = await new ClickHouseContainer().start(); - // await container.restart(); - - // let client: ClickHouseClient | undefined; - // try { - // client = createClient({ - // url: container.getConnectionUri(), - // }); - - // await client.ping(); - - // const result = await client.query({ - // query: "SELECT 1 AS value", - // format: "JSON", - // }); - - // const data = (await result.json()) as any; - // expect(data?.data?.[0]?.value).toBe(1); - // } finally { - // await client?.close(); - // await container.stop(); - // } - // }); - - // it("should allow custom healthcheck", async () => { - // const container = new ClickHouseContainer().withHealthCheck({ - // test: ["CMD-SHELL", "exit 1"], - // interval: 100, - // retries: 0, - // timeout: 100, - // }); - - // await expect(() => container.start()).rejects.toThrow(); - // }); + // setDatabase { + it("should set database", async () => { + const customDatabase = "customDatabase"; + const container = await new ClickHouseContainer().withDatabase(customDatabase).start(); + + let client: ClickHouseClient | undefined; + + try { + client = createClient(container.getClientOptions()); + + const result = await client.query({ + query: "SELECT currentDatabase() AS current_database", + format: "JSON", + }); + + const data = (await result.json()) as ClickHouseQueryResponse<{ current_database: string }>; + expect(data?.data?.[0]?.current_database).toBe(customDatabase); + } finally { + await client?.close(); + await container.stop(); + } + }); + // } + + // setUsername { + it("should set username", async () => { + const customUsername = "customUsername"; + const container = await new ClickHouseContainer().withUsername(customUsername).start(); + + let client: ClickHouseClient | undefined; + + try { + client = createClient(container.getClientOptions()); + + const result = await client.query({ + query: "SELECT currentUser() AS current_user", + format: "JSON", + }); + + const data = (await result.json()) as ClickHouseQueryResponse<{ current_user: string }>; + expect(data?.data?.[0]?.current_user).toBe(customUsername); + } finally { + await client?.close(); + await container.stop(); + } + }); + // } + + it("should work with restarted container", async () => { + const container = await new ClickHouseContainer().start(); + await container.restart(); + + let client: ClickHouseClient | undefined; + try { + client = createClient(container.getClientOptions()); + + const result = await client.query({ + query: "SELECT 1 AS value", + format: "JSON", + }); + + const data = (await result.json()) as ClickHouseQueryResponse<{ value: number }>; + expect(data?.data?.[0]?.value).toBe(1); + } finally { + await client?.close(); + await container.stop(); + } + }); + + /** + * Verifies that a custom Docker health check that fails immediately + * causes the container startup process (`container.start()`) to reject with an error. + * + * Note: This test pattern was adapted from a similar test case used for + * PostgreSQL containers to ensure custom failing health checks are handled correctly. + */ + it("should allow custom healthcheck", async () => { + const container = new ClickHouseContainer() + .withHealthCheck({ + test: ["CMD-SHELL", "exit 1"], + interval: 100, + retries: 0, + timeout: 100, + }) + .withWaitStrategy(Wait.forHealthCheck()); + + await expect(() => container.start()).rejects.toThrow(); + }); }); diff --git a/packages/modules/clickhouse/src/clickhouse-container.ts b/packages/modules/clickhouse/src/clickhouse-container.ts index 06a1c929b..361f52212 100644 --- a/packages/modules/clickhouse/src/clickhouse-container.ts +++ b/packages/modules/clickhouse/src/clickhouse-container.ts @@ -15,7 +15,9 @@ export class ClickHouseContainer extends GenericContainer { Wait.forHttp("/", CLICKHOUSE_HTTP_PORT).forResponsePredicate((response) => response === "Ok.\n") ); this.withStartupTimeout(120_000); - this.withDatabase("test"); + + // Setting this high ulimits value proactively prevents the "Too many open files" error, + // especially under potentially heavy load during testing. this.withUlimits({ nofile: { hard: 262144, @@ -44,7 +46,6 @@ export class ClickHouseContainer extends GenericContainer { CLICKHOUSE_USER: this.username, CLICKHOUSE_PASSWORD: this.password, CLICKHOUSE_DB: this.database, - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1", }); return new StartedClickHouseContainer(await super.start(), this.database, this.username, this.password); @@ -69,10 +70,6 @@ export class StartedClickHouseContainer extends AbstractStartedContainer { return super.getMappedPort(CLICKHOUSE_HTTP_PORT); } - public getDatabase(): string { - return this.database; - } - public getUsername(): string { return this.username; } @@ -81,19 +78,53 @@ export class StartedClickHouseContainer extends AbstractStartedContainer { return this.password; } - public getConnectionUri(): string { + public getDatabase(): string { + return this.database; + } + + /** + * Gets the base HTTP URL (protocol, host and mapped port) for the ClickHouse container's HTTP interface. + * Example: `http://localhost:32768` + */ + public getHttpUrl(): string { const protocol = "http"; const host = this.getHost(); const port = this.getHttpPort(); - return `${protocol}://${host}:${port}`; } - // /** - // * @returns A connection URI for HTTP interface in the form of `http://[username[:password]@][host[:port]]` - // */ - // public getHttpConnectionUri(): string { - // const url = new URL(`http://127.0.0.1:${this.getHttpPort()}`); - // return url.toString(); - // } + /** + * Gets configuration options suitable for passing directly to `createClient({...})` + * from `@clickhouse/client`. Uses the HTTP interface. + */ + public getClientOptions(): { + url?: string; + username: string; + password: string; + database: string; + } { + return { + url: this.getHttpUrl(), + username: this.getUsername(), + password: this.getPassword(), + database: this.getDatabase(), + }; + } + + /** + * Gets a ClickHouse connection URL for the HTTP interface with format: + * http://username:password@hostname:port/database + * @returns The ClickHouse HTTP URL string. + */ + public getConnectionUrl(): string { + const url = new URL(this.getHttpUrl()); + + url.username = this.getUsername(); + url.password = this.getPassword(); + + const dbName = this.getDatabase(); + url.pathname = dbName.startsWith("/") ? dbName : `/${dbName}`; + + return url.toString(); + } } From 4eefcf176d9b6d62a2beba456020f1d3ffd6e4b9 Mon Sep 17 00:00:00 2001 From: Victorlouisdg Date: Thu, 10 Apr 2025 10:48:28 +0200 Subject: [PATCH 4/5] feat(clickhouse): update docs --- docs/modules/clickhouse.md | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/docs/modules/clickhouse.md b/docs/modules/clickhouse.md index 0e9afc16b..aa1b72b56 100644 --- a/docs/modules/clickhouse.md +++ b/docs/modules/clickhouse.md @@ -40,15 +40,30 @@ npm install @testcontainers/clickhouse --save-dev -### Connection URIs +### Connection Methods -The module provides two methods to get connection URIs: +The module provides several methods to connect to the ClickHouse container: -1. `getConnectionUri()` - Returns a URI in the form of `clickhouse://[username[:password]@][host[:port],]/database` -2. `getHttpConnectionUri()` - Returns a URI in the form of `http://[username[:password]@][host[:port]]` +1. `getClientOptions()` - Returns a configuration object suitable for `@clickhouse/client`: -These URIs can be used with the `@clickhouse/client` package or any other ClickHouse client that supports connection URIs. + ```typescript + { + url: string; // HTTP URL with host and port + username: string; // Container username + password: string; // Container password + database: string; // Container database + } + ``` -!!!tip -The HTTP interface (port 8123) is often easier to use for simple queries and is more widely supported by various clients. -The native interface (port 9000) offers better performance for high-throughput scenarios. +2. `getConnectionUrl()` - Returns a complete HTTP URL including credentials and database: + + ``` + http://[username[:password]@][host[:port]]/database + ``` + +3. `getHttpUrl()` - Returns the base HTTP URL without credentials: + ``` + http://[host[:port]] + ``` + +These methods can be used with the `@clickhouse/client` package or any other ClickHouse client that supports HTTP connections. From 4bc1a5717a1eaa3afeddfb53de820794b33201cb Mon Sep 17 00:00:00 2001 From: Victorlouisdg Date: Thu, 10 Apr 2025 21:54:41 +0200 Subject: [PATCH 5/5] fix(clickhouse): change image version, streamline test cases, and fix docs inconsistencies --- docs/modules/clickhouse.md | 16 +- .../src/clickhouse-container.test.ts | 197 +++++++----------- .../clickhouse/src/clickhouse-container.ts | 2 +- 3 files changed, 73 insertions(+), 142 deletions(-) diff --git a/docs/modules/clickhouse.md b/docs/modules/clickhouse.md index aa1b72b56..f69752717 100644 --- a/docs/modules/clickhouse.md +++ b/docs/modules/clickhouse.md @@ -11,33 +11,23 @@ npm install @testcontainers/clickhouse --save-dev ## Examples - [Connect and execute query:](../../packages/modules/clickhouse/src/clickhouse-container.test.ts) inside_block:connectWithOptions - - [Connect using URL and execute query:](../../packages/modules/clickhouse/src/clickhouse-container.test.ts) inside_block:connectWithUrl - - [Connect with username and password and execute query:](../../packages/modules/clickhouse/src/clickhouse-container.test.ts) inside_block:connectWithUsernameAndPassword - - [Set database:](../../packages/modules/clickhouse/src/clickhouse-container.test.ts) inside_block:setDatabase - - [Set username:](../../packages/modules/clickhouse/src/clickhouse-container.test.ts) inside_block:setUsername - ### Connection Methods @@ -45,7 +35,6 @@ npm install @testcontainers/clickhouse --save-dev The module provides several methods to connect to the ClickHouse container: 1. `getClientOptions()` - Returns a configuration object suitable for `@clickhouse/client`: - ```typescript { url: string; // HTTP URL with host and port @@ -54,16 +43,13 @@ The module provides several methods to connect to the ClickHouse container: database: string; // Container database } ``` - 2. `getConnectionUrl()` - Returns a complete HTTP URL including credentials and database: - ``` http://[username[:password]@][host[:port]]/database ``` - 3. `getHttpUrl()` - Returns the base HTTP URL without credentials: ``` http://[host[:port]] ``` -These methods can be used with the `@clickhouse/client` package or any other ClickHouse client that supports HTTP connections. +These methods can be used with the `@clickhouse/client` package or any other ClickHouse client. diff --git a/packages/modules/clickhouse/src/clickhouse-container.test.ts b/packages/modules/clickhouse/src/clickhouse-container.test.ts index 4b0bacdf4..d96864d01 100644 --- a/packages/modules/clickhouse/src/clickhouse-container.test.ts +++ b/packages/modules/clickhouse/src/clickhouse-container.test.ts @@ -1,5 +1,4 @@ -import { ClickHouseClient, createClient } from "@clickhouse/client"; -import { Wait } from "testcontainers"; +import { createClient } from "@clickhouse/client"; import { ClickHouseContainer } from "./clickhouse-container"; interface ClickHouseQueryResponse { @@ -7,59 +6,40 @@ interface ClickHouseQueryResponse { } describe("ClickHouseContainer", { timeout: 180_000 }, () => { - it("should start successfully with default settings", async () => { - const container = new ClickHouseContainer(); - const startedContainer = await container.start(); - expect(startedContainer.getHost()).toBeDefined(); - expect(startedContainer.getHttpPort()).toBeDefined(); - expect(startedContainer.getDatabase()).toBeDefined(); - expect(startedContainer.getUsername()).toBeDefined(); - expect(startedContainer.getPassword()).toBeDefined(); - await startedContainer.stop(); - }); - // connectWithOptions { it("should connect using the client options object", async () => { const container = await new ClickHouseContainer().start(); - let client: ClickHouseClient | undefined; - - try { - client = createClient(container.getClientOptions()); - - const result = await client.query({ - query: "SELECT 1 AS value", - format: "JSON", - }); - const data = (await result.json()) as ClickHouseQueryResponse<{ value: number }>; - expect(data?.data?.[0]?.value).toBe(1); - } finally { - await client?.close(); - await container.stop(); - } + const client = createClient(container.getClientOptions()); + + const result = await client.query({ + query: "SELECT 1 AS value", + format: "JSON", + }); + const data = (await result.json()) as ClickHouseQueryResponse<{ value: number }>; + expect(data?.data?.[0]?.value).toBe(1); + + await client.close(); + await container.stop(); }); // } // connectWithUrl { it("should connect using the URL", async () => { const container = await new ClickHouseContainer().start(); - let client: ClickHouseClient | undefined; - - try { - client = createClient({ - url: container.getConnectionUrl(), - }); - - const result = await client.query({ - query: "SELECT 1 AS value", - format: "JSON", - }); - - const data = (await result.json()) as ClickHouseQueryResponse<{ value: number }>; - expect(data?.data?.[0]?.value).toBe(1); - } finally { - await client?.close(); - await container.stop(); - } + const client = createClient({ + url: container.getConnectionUrl(), + }); + + const result = await client.query({ + query: "SELECT 1 AS value", + format: "JSON", + }); + + const data = (await result.json()) as ClickHouseQueryResponse<{ value: number }>; + expect(data?.data?.[0]?.value).toBe(1); + + await client.close(); + await container.stop(); }); // } @@ -70,26 +50,22 @@ describe("ClickHouseContainer", { timeout: 180_000 }, () => { .withPassword("customPassword") .start(); - let client: ClickHouseClient | undefined; - - try { - client = createClient({ - url: container.getHttpUrl(), - username: container.getUsername(), - password: container.getPassword(), - }); - - const result = await client.query({ - query: "SELECT 1 AS value", - format: "JSON", - }); - - const data = (await result.json()) as ClickHouseQueryResponse<{ value: number }>; - expect(data?.data?.[0]?.value).toBe(1); - } finally { - await client?.close(); - await container.stop(); - } + const client = createClient({ + url: container.getHttpUrl(), + username: container.getUsername(), + password: container.getPassword(), + }); + + const result = await client.query({ + query: "SELECT 1 AS value", + format: "JSON", + }); + + const data = (await result.json()) as ClickHouseQueryResponse<{ value: number }>; + expect(data?.data?.[0]?.value).toBe(1); + + await client.close(); + await container.stop(); }); // } @@ -98,22 +74,18 @@ describe("ClickHouseContainer", { timeout: 180_000 }, () => { const customDatabase = "customDatabase"; const container = await new ClickHouseContainer().withDatabase(customDatabase).start(); - let client: ClickHouseClient | undefined; + const client = createClient(container.getClientOptions()); - try { - client = createClient(container.getClientOptions()); + const result = await client.query({ + query: "SELECT currentDatabase() AS current_database", + format: "JSON", + }); - const result = await client.query({ - query: "SELECT currentDatabase() AS current_database", - format: "JSON", - }); + const data = (await result.json()) as ClickHouseQueryResponse<{ current_database: string }>; + expect(data?.data?.[0]?.current_database).toBe(customDatabase); - const data = (await result.json()) as ClickHouseQueryResponse<{ current_database: string }>; - expect(data?.data?.[0]?.current_database).toBe(customDatabase); - } finally { - await client?.close(); - await container.stop(); - } + await client.close(); + await container.stop(); }); // } @@ -122,22 +94,18 @@ describe("ClickHouseContainer", { timeout: 180_000 }, () => { const customUsername = "customUsername"; const container = await new ClickHouseContainer().withUsername(customUsername).start(); - let client: ClickHouseClient | undefined; + const client = createClient(container.getClientOptions()); - try { - client = createClient(container.getClientOptions()); + const result = await client.query({ + query: "SELECT currentUser() AS current_user", + format: "JSON", + }); - const result = await client.query({ - query: "SELECT currentUser() AS current_user", - format: "JSON", - }); + const data = (await result.json()) as ClickHouseQueryResponse<{ current_user: string }>; + expect(data?.data?.[0]?.current_user).toBe(customUsername); - const data = (await result.json()) as ClickHouseQueryResponse<{ current_user: string }>; - expect(data?.data?.[0]?.current_user).toBe(customUsername); - } finally { - await client?.close(); - await container.stop(); - } + await client.close(); + await container.stop(); }); // } @@ -145,40 +113,17 @@ describe("ClickHouseContainer", { timeout: 180_000 }, () => { const container = await new ClickHouseContainer().start(); await container.restart(); - let client: ClickHouseClient | undefined; - try { - client = createClient(container.getClientOptions()); - - const result = await client.query({ - query: "SELECT 1 AS value", - format: "JSON", - }); - - const data = (await result.json()) as ClickHouseQueryResponse<{ value: number }>; - expect(data?.data?.[0]?.value).toBe(1); - } finally { - await client?.close(); - await container.stop(); - } - }); + const client = createClient(container.getClientOptions()); + + const result = await client.query({ + query: "SELECT 1 AS value", + format: "JSON", + }); + + const data = (await result.json()) as ClickHouseQueryResponse<{ value: number }>; + expect(data?.data?.[0]?.value).toBe(1); - /** - * Verifies that a custom Docker health check that fails immediately - * causes the container startup process (`container.start()`) to reject with an error. - * - * Note: This test pattern was adapted from a similar test case used for - * PostgreSQL containers to ensure custom failing health checks are handled correctly. - */ - it("should allow custom healthcheck", async () => { - const container = new ClickHouseContainer() - .withHealthCheck({ - test: ["CMD-SHELL", "exit 1"], - interval: 100, - retries: 0, - timeout: 100, - }) - .withWaitStrategy(Wait.forHealthCheck()); - - await expect(() => container.start()).rejects.toThrow(); + await client.close(); + await container.stop(); }); }); diff --git a/packages/modules/clickhouse/src/clickhouse-container.ts b/packages/modules/clickhouse/src/clickhouse-container.ts index 361f52212..8386420be 100644 --- a/packages/modules/clickhouse/src/clickhouse-container.ts +++ b/packages/modules/clickhouse/src/clickhouse-container.ts @@ -8,7 +8,7 @@ export class ClickHouseContainer extends GenericContainer { private password = "test"; private database = "test"; - constructor(image = "clickhouse/clickhouse-server:24") { + constructor(image = "clickhouse/clickhouse-server:25.3-alpine") { super(image); this.withExposedPorts(CLICKHOUSE_PORT, CLICKHOUSE_HTTP_PORT); this.withWaitStrategy(