UBERF-11178 Migrate datalake workspace id (#9092)

Signed-off-by: Alexander Onnikov <[email protected]>

Author: aonnikov

services/datalake/pod-datalake/src/datalake/db.ts

@@ -572,7 +572,7 @@ export function escape (value: any): string {
 }
 
 function getMigrations (): [string, string][] {
-  return [migrationV1()]
+  return [migrationV1(), migrationV2()]
 }
 
 function migrationV1 (): [string, string] {
@@ -611,3 +611,22 @@ function migrationV1 (): [string, string] {
   `
   return ['init_tables_01', sql]
 }
+
+function migrationV2 (): [string, string] {
+  const sql = `
+    ALTER TABLE blob.meta DROP CONSTRAINT IF EXISTS fk_blob;
+
+    UPDATE blob.blob
+    SET workspace = w.uuid
+    FROM global_account.workspace w
+    WHERE workspace = w.data_id;
+
+    UPDATE blob.meta
+    SET workspace = w.uuid
+    FROM global_account.workspace w
+    WHERE workspace = w.data_id;
+
+    ALTER TABLE blob.meta ADD CONSTRAINT fk_blob_meta FOREIGN KEY (workspace, name) REFERENCES blob.blob (workspace, name);
+  `
+  return ['migrate_workspaces_02', sql]
+}

services/datalake/pod-datalake/src/server.ts

@@ -14,6 +14,7 @@
 //
 
 import { Analytics } from '@hcengineering/analytics'
+import { getClient } from '@hcengineering/account-client'
 import { MeasureContext, Tx, metricsAggregate } from '@hcengineering/core'
 import { PlatformQueue, QueueTopic, getCPUInfo, getMemoryInfo } from '@hcengineering/server-core'
 import { decodeToken, TokenError } from '@hcengineering/server-token'
@@ -115,6 +116,8 @@ export async function createServer (
   queue: PlatformQueue,
   config: Config
 ): Promise<{ app: Express, close: () => void }> {
+  await ensureAccountReady(ctx, config)
+
   const buckets: Array<{ location: Location, bucket: S3Bucket }> = []
   for (const bucket of config.Buckets) {
     const location = bucket.location as Location
@@ -348,3 +351,13 @@ export function listen (e: Express, port: number, host?: string): Server {
 
   return server
 }
+
+async function ensureAccountReady (ctx: MeasureContext, config: Config): Promise<void> {
+  const client = getClient(config.AccountsUrl)
+  try {
+    await client.getRegionInfo()
+  } catch (err: any) {
+    ctx.error('Accounts service not ready', { err })
+    throw new Error('Accounts service not ready')
+  }
+}

tests/.env

@@ -1,4 +1,5 @@
-STORAGE_CONFIG="datalake|http://datalake:4031"
+STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin"
+DATALAKE_STORAGE_CONFIG="datalake|http://datalake:4031"
 BACKUP_STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin"
 BACKUP_BUCKET_NAME=dev-backups
 

tests/docker-compose.override.yaml

@@ -2,13 +2,49 @@ services:
   account:
     environment:
       - DB_URL=${DB_PG_URL}
+      - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG}
       - PROCEED_V7_MONGO=false
   transactor:
     environment:
       - DB_URL=${DB_PG_URL}
+      - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG}
   workspace:
     environment:
       - DB_URL=${DB_PG_URL}
+      - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG}
   fulltext:
     environment:
       - DB_URL=${DB_PG_URL}
+      - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG}
+  front:
+    environment:
+      - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG}
+      - FILES_URL=http://localhost:4031/blob/:workspace/:blobId/:filename
+      - UPLOAD_URL=http://localhost:4031/upload/form-data/:workspace
+      - PREVIEW_CONFIG=http://localhost:4031/image/fit=cover,width=:width,height=:height,dpr=:dpr/:workspace/:blobId
+  collaborator:
+    environment:
+      - STORAGE_CONFIG=${DATALAKE_STORAGE_CONFIG}
+  datalake:
+    image: hardcoreeng/datalake
+    depends_on:
+      minio:
+        condition: service_healthy
+      cockroach:
+        condition: service_started
+      stats:
+        condition: service_started
+      account:
+        condition: service_started
+    ports:
+      - 4031:4031
+    environment:
+      - PORT=4031
+      - SECRET=secret
+      - ACCOUNTS_URL=http://account:3003
+      - STATS_URL=http://stats:4901
+      - STREAM_URL=http://localhost:1081/recording
+      - DB_URL=${DB_PG_URL}
+      - BUCKETS=blobs,eu|http://minio:9000?accessKey=minioadmin&secretKey=minioadmin
+      - QUEUE_CONFIG=${QUEUE_CONFIG}
+    restart: unless-stopped

tests/docker-compose.yaml

@@ -174,9 +174,7 @@ services:
       - SERVER_SECRET=secret
       - ACCOUNTS_URL_INTERNAL=http://account:3003
       - ACCOUNTS_URL=http://localhost:3003
-      - FILES_URL=http://localhost:4031/blob/:workspace/:blobId/:filename
-      - UPLOAD_URL=http://localhost:4031/upload/form-data/:workspace
-      - PREVIEW_CONFIG=http://localhost:4031/image/fit=cover,width=:width,height=:height,dpr=:dpr/:workspace/:blobId
+      - UPLOAD_URL=/files
       - ELASTIC_URL=http://elastic:9200
       - GMAIL_URL=http://localhost:8088
       - CALENDAR_URL=http://localhost:8095
@@ -278,28 +276,6 @@ services:
       - REKONI_URL=http://rekoni:4007
       - ACCOUNTS_URL=http://account:3003
       - STATS_URL=http://stats:4901
-  datalake:
-    image: hardcoreeng/datalake
-    depends_on:
-      minio:
-        condition: service_healthy
-      cockroach:
-        condition: service_started
-      stats:
-        condition: service_started
-      account:
-        condition: service_started
-    ports:
-      - 4031:4031
-    environment:
-      - PORT=4031
-      - SECRET=secret
-      - ACCOUNTS_URL=http://account:3003
-      - STATS_URL=http://stats:4901
-      - STREAM_URL=http://localhost:1081/recording
-      - DB_URL=${DB_PG_URL}
-      - BUCKETS=blobs,eu|http://minio:9000?accessKey=minioadmin&secretKey=minioadmin
-    restart: unless-stopped
   stats:
     image: hardcoreeng/stats
     ports:

tests/tool.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 export MODEL_VERSION=$(node ../common/scripts/show_version.js)
-export STORAGE_CONFIG="datalake|http://localhost:4031"
+export STORAGE_CONFIG="minio|localhost:9002?accessKey=minioadmin&secretKey=minioadmin"
 export ACCOUNTS_URL=http://localhost:3003
 export TRANSACTOR_URL=ws://localhost:3334
 export ACCOUNT_DB_URL=mongodb://localhost:27018

ws-tests/.env

@@ -1,7 +1,7 @@
-STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin"
+STORAGE_CONFIG="datalake|http://huly.local:4031"
 BACKUP_STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin"
 BACKUP_BUCKET_NAME=ws-dev-backups
 DB_URL=mongodb://huly.local:27018
 MONGO_URL=mongodb://huly.local:27018
-DB_EU_URL=postgresql://root@huly.local:26258/defaultdb?sslmode=disable
+DB_EU_URL=postgresql://root@cockroach:26257/defaultdb?sslmode=disable
 QUEUE_CONFIG=redpanda:9093

ws-tests/docker-compose.yaml

@@ -5,11 +5,9 @@ services:
     extra_hosts:
       - 'huly.local:host-gateway'
     environment:
-      - STREAM_ENDPOINT_URL=s3://huly.local:9000
+      - STREAM_ENDPOINT_URL=datalake://huly.local:4031
       - STREAM_INSECURE=true
       - STREAM_SERVER_SECRET=secret
-      - AWS_ACCESS_KEY_ID=minioadmin
-      - AWS_SECRET_ACCESS_KEY=minioadmin
     ports:
       - 1081:1080
     restart: unless-stopped
@@ -95,11 +93,38 @@ services:
     extra_hosts:
       - 'huly.local:host-gateway'
     expose:
-      - 9002
-      - 9003
+      - 9000
+      - 9001
     ports:
       - 9002:9000
       - 9003:9001
+    healthcheck:
+      test: ['CMD', 'mc', 'ready', 'local']
+      interval: 5s
+      retries: 10
+  datalake:
+    image: hardcoreeng/datalake
+    depends_on:
+      minio:
+        condition: service_healthy
+      cockroach:
+        condition: service_started
+      stats:
+        condition: service_started
+      account:
+        condition: service_started
+    ports:
+      - 4031:4031
+    environment:
+      - PORT=4031
+      - SECRET=secret
+      - ACCOUNTS_URL=http://account:3003
+      - STATS_URL=http://stats:4901
+      - STREAM_URL=http://localhost:1081/recording
+      - DB_URL=${DB_EU_URL}
+      - BUCKETS=blobs,eu|http://minio:9000?accessKey=minioadmin&secretKey=minioadmin
+      - QUEUE_CONFIG=${QUEUE_CONFIG}
+    restart: unless-stopped
   elastic:
     image: 'elasticsearch:7.14.2'
     expose: