chore(ci): Enable smoke testing for Oracle (#6196)

Author: ovr

.github/actions/smoke.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+set -eo pipefail
+
+# Debug log for test containers
+export DEBUG=testcontainers
+
+echo "::group::Oracle"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:oracle
+echo "::endgroup::"
+
+echo "::group::Postgres"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:postgres
+echo "::endgroup::"
+
+echo "::group::QuestDB"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:questdb
+echo "::endgroup::"
+
+echo "::group::Crate"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:crate
+echo "::endgroup::"
+
+echo "::group::Lambda"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:lambda
+echo "::endgroup::"
+
+echo "::group::Materialize"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:materialize
+echo "::endgroup::"
+
+echo "::group::Multidb"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:multidb
+echo "::endgroup::"
+
+echo "::group::Prestodb"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:prestodb
+echo "::endgroup::"
+
+echo "::group::Trino"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:trino
+echo "::endgroup::"
+
+echo "::group::MS SQL"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:mssql
+echo "::endgroup::"

.github/workflows/push.yml

@@ -494,19 +494,13 @@ jobs:
         with:
           command: build
           args: --manifest-path rust/cubestore/Cargo.toml -j 4
-      - name: Run Integration smoke tests
-        timeout-minutes: 30
-        env:
-          CUBEJS_AWS_KEY: ${{ secrets.CUBEJS_AWS_KEY }}
-          CUBEJS_AWS_SECRET: ${{ secrets.CUBEJS_AWS_SECRET }}
-          CUBEJS_AWS_REGION: us-east-1
-          CUBEJS_AWS_S3_OUTPUT_LOCATION: s3://cubejs-opensource/testing/output
-          CUBEJS_DB_EXPORT_BUCKET: s3://cubejs-opensource/testing/export
+      - name: Copy Cube Store
         run: |
           mkdir -p rust/cubestore/downloaded/latest/bin
           cp rust/cubestore/target/debug/cubestored rust/cubestore/downloaded/latest/bin/cubestored
-          cd packages/cubejs-testing/
-          yarn run smoke
+      - name: Run Integration smoke tests
+        timeout-minutes: 30
+        run: ./.github/actions/smoke.sh
 
   docker-image-latest-set-tag:
     # At least git should be completed pushed up until this moment

packages/cubejs-backend-shared/src/process.ts

@@ -20,3 +20,11 @@ let i = 0;
 export function getNext(): number {
   return i++;
 }
+
+export function isCI(): boolean {
+  if (process.env.GITHUB_EVENT_NAME) {
+    return true;
+  }
+
+  return false;
+}

packages/cubejs-testing-shared/src/db/oracle.ts

@@ -1,4 +1,6 @@
-import { GenericContainer } from 'testcontainers';
+import { GenericContainer, Wait } from 'testcontainers';
+import { isCI } from '@cubejs-backend/shared';
+
 import { DbRunnerAbstract, DBRunnerContainerOptions } from './db-runner.abstract';
 
 type OracleStartOptions = DBRunnerContainerOptions & {
@@ -11,8 +13,15 @@ export class OracleDBRunner extends DbRunnerAbstract {
 
     const container = new GenericContainer(`gvenzl/oracle-xe:${version}`)
       .withEnv('ORACLE_PASSWORD', 'test')
-      .withExposedPorts(1521)
-      .withStartupTimeout(30 * 1000);
+      .withHealthCheck({
+        test: 'healthcheck.sh',
+        interval: 2 * 1000,
+        timeout: 5 * 1000,
+        retries: 5,
+        startPeriod: (isCI() ? 45 : 15) * 1000
+      })
+      .withWaitStrategy(Wait.forHealthCheck())
+      .withExposedPorts(1521);
 
     if (options.volumes) {
       // eslint-disable-next-line no-restricted-syntax

packages/cubejs-testing/package.json

@@ -48,7 +48,6 @@
     "driver:databricks:snap": "jest --verbose --updateSnapshot -i dist/test/driver-databricks.test.js",
     "integration:cubestore": "jest --verbose --updateSnapshot -i dist/test/driver-cubestore.test.js",
     "rest:postgres": "yarn tsc && clear && jest --verbose -i dist/test/rest-postgres.test.js",
-    "smoke": "jest --verbose -i 'dist/test/smoke-(?!.*?(redshift|athena|bigquery|firebolt|cubesql|oracle))'",
     "smoke:athena": "jest --verbose -i dist/test/smoke-athena.test.js",
     "smoke:athena:snapshot": "jest --verbose --updateSnapshot -i dist/test/smoke-athena.test.js",
     "smoke:bigquery": "jest --verbose -i dist/test/smoke-bigquery.test.js",

packages/cubejs-testing/test/smoke-athena.test.ts

@@ -3,7 +3,7 @@ import cubejs, { CubejsApi } from '@cubejs-client/core';
 // eslint-disable-next-line import/no-extraneous-dependencies
 import { afterAll, beforeAll, jest, expect } from '@jest/globals';
 import { BirdBox, getBirdbox } from '../src';
-import { DEFAULT_CONFIG, testQueryMeasure } from './smoke-tests';
+import { DEFAULT_API_TOKEN, DEFAULT_CONFIG, testQueryMeasure } from './smoke-tests';
 
 const delay = (t: number) => new Promise(resolve => setTimeout(() => resolve(null), t));
 
@@ -25,7 +25,7 @@ describe('athena', () => {
         schemaDir: 'postgresql/schema',
       }
     );
-    client = cubejs(async () => 'test', {
+    client = cubejs(async () => DEFAULT_API_TOKEN, {
       apiUrl: birdbox.configuration.apiUrl,
     });
   });

packages/cubejs-testing/test/smoke-bigquery.test.ts

@@ -2,7 +2,7 @@ import cubejs, { CubejsApi } from '@cubejs-client/core';
 // eslint-disable-next-line import/no-extraneous-dependencies
 import { afterAll, beforeAll, jest } from '@jest/globals';
 import { BirdBox, getBirdbox } from '../src';
-import { DEFAULT_CONFIG, testQueryMeasure } from './smoke-tests';
+import { DEFAULT_API_TOKEN, DEFAULT_CONFIG, testQueryMeasure } from './smoke-tests';
 
 describe('bigquery', () => {
   jest.setTimeout(60 * 5 * 1000);
@@ -21,7 +21,7 @@ describe('bigquery', () => {
         schemaDir: 'postgresql/schema',
       }
     );
-    client = cubejs(async () => 'test', {
+    client = cubejs(async () => DEFAULT_API_TOKEN, {
       apiUrl: birdbox.configuration.apiUrl,
     });
   });

packages/cubejs-testing/test/smoke-crate.test.ts

@@ -4,7 +4,7 @@ import { afterAll, beforeAll, jest } from '@jest/globals';
 import { StartedTestContainer } from 'testcontainers';
 import { CrateDBRunner } from '@cubejs-backend/testing-shared';
 import { BirdBox, getBirdbox } from '../src';
-import { DEFAULT_CONFIG, testQueryMeasure } from './smoke-tests';
+import { DEFAULT_API_TOKEN, DEFAULT_CONFIG, testQueryMeasure } from './smoke-tests';
 
 describe('crate', () => {
   jest.setTimeout(60 * 5 * 1000);
@@ -30,7 +30,7 @@ describe('crate', () => {
         schemaDir: 'postgresql/schema',
       }
     );
-    client = cubejs(async () => 'test', {
+    client = cubejs(async () => DEFAULT_API_TOKEN, {
       apiUrl: birdbox.configuration.apiUrl,
     });
   });

packages/cubejs-testing/test/smoke-firebolt.test.ts

@@ -2,7 +2,7 @@ import cubejs, { CubejsApi } from '@cubejs-client/core';
 // eslint-disable-next-line import/no-extraneous-dependencies
 import { afterAll, beforeAll, jest, expect } from '@jest/globals';
 import { BirdBox, getBirdbox } from '../src';
-import { DEFAULT_CONFIG, testQueryMeasure } from './smoke-tests';
+import { DEFAULT_API_TOKEN, DEFAULT_CONFIG, testQueryMeasure } from './smoke-tests';
 
 describe('firebolt', () => {
   jest.setTimeout(60 * 5 * 1000);
@@ -20,7 +20,7 @@ describe('firebolt', () => {
         schemaDir: 'firebolt/schema',
       }
     );
-    client = cubejs(async () => 'test', {
+    client = cubejs(async () => DEFAULT_API_TOKEN, {
       apiUrl: birdbox.configuration.apiUrl,
     });
   });

packages/cubejs-testing/test/smoke-lambda.test.ts

@@ -7,7 +7,7 @@ import cubejs, { CubejsApi, Query } from '@cubejs-client/core';
 // eslint-disable-next-line import/no-extraneous-dependencies
 import { afterAll, beforeAll, expect, jest } from '@jest/globals';
 import { BirdBox, getBirdbox } from '../src';
-import { DEFAULT_CONFIG } from './smoke-tests';
+import { DEFAULT_API_TOKEN, DEFAULT_CONFIG } from './smoke-tests';
 
 const CubeStoreDriver = require('@cubejs-backend/cubestore-driver');
 const PostgresDriver = require('@cubejs-backend/postgres-driver');
@@ -85,7 +85,7 @@ describe('lambda', () => {
         cubejsConfig: 'lambda/cube.js',
       }
     );
-    client = cubejs(async () => 'test', {
+    client = cubejs(async () => DEFAULT_API_TOKEN, {
       apiUrl: birdbox.configuration.apiUrl,
     });
     postgres = new PostgresDriver({