UBERF-12323: Include accounts info into backup (#9659)

* uberf-12323: include accounts data into backups
Signed-off-by: Alexey Zinoviev <[email protected]>

Signed-off-by: Alexey Zinoviev <[email protected]>

* uberf-12323: support full recheck
Signed-off-by: Alexey Zinoviev <[email protected]>

Signed-off-by: Alexey Zinoviev <[email protected]>

* uberf-12323: bump forced full recheck
Signed-off-by: Alexey Zinoviev <[email protected]>

Signed-off-by: Alexey Zinoviev <[email protected]>

* uberf-12323: remove excess logs
Signed-off-by: Alexey Zinoviev <[email protected]>

Signed-off-by: Alexey Zinoviev <[email protected]>

* uberf-12323: fix tests
Signed-off-by: Alexey Zinoviev <[email protected]>

Signed-off-by: Alexey Zinoviev <[email protected]>

* uberf-12323: update local dev setup
Signed-off-by: Alexey Zinoviev <[email protected]>

---------

Signed-off-by: Alexey Zinoviev <[email protected]>

Author: lexiv0re

dev/docker-compose.min.yaml

@@ -143,6 +143,7 @@ services:
       - STORAGE_CONFIG=${STORAGE_CONFIG}
       - MODEL_ENABLED=*
       - ACCOUNTS_URL=http://huly.local:3000
+      - ACCOUNTS_DB_URL=${DB_CR_URL}
       - BRANDING_PATH=/var/cfg/branding.json
       - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG}
       - BACKUP_BUCKET=${BACKUP_BUCKET_NAME}

dev/docker-compose.yaml

@@ -197,6 +197,7 @@ services:
       - STATS_URL=http://huly.local:4900
       - STORAGE_CONFIG=${STORAGE_CONFIG}
       - ACCOUNTS_URL=http://huly.local:3000
+      - ACCOUNTS_DB_URL=${DB_CR_URL}
       - BRANDING_PATH=/var/cfg/branding.json
       - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG}
       - BACKUP_BUCKET=${BACKUP_BUCKET_NAME}
@@ -536,6 +537,7 @@ services:
     environment:
       - SECRET=secret
       - ACCOUNTS_URL=http://huly.local:3000
+      - ACCOUNTS_DB_URL=${DB_CR_URL}
       - STATS_URL=http://huly.local:4900
       - DB_URL=${DB_CR_URL}
       - REGION=cockroach

dev/tool/src/index.ts

@@ -916,7 +916,7 @@ export function devTool (
 
   program
     .command('backup <dirName> <workspace>')
-    .description('dump workspace transactions and minio resources')
+    .description('dump workspace transactions, blobs and accounts')
     .option('-i, --include <include>', 'A list of ; separated domain names to include during backup', '*')
     .option('-s, --skip <skip>', 'A list of ; separated domain names to skip during backup', '')
     .option('--full', 'Full recheck', false)
@@ -929,6 +929,7 @@ export function devTool (
     .option('-f, --force', 'Force backup', false)
     .option('-t, --timeout <timeout>', 'Connect timeout in seconds', '30')
     .option('-k, --keepSnapshots <keepSnapshots>', 'Keep snapshots for days', '14')
+    .option('-fv, --fullVerify', 'Full verification', false)
     .action(
       async (
         dirName: string,
@@ -942,6 +943,7 @@ export function devTool (
           contentTypes: string
           full: boolean
           keepSnapshots: string
+          fullVerify: boolean
         }
       ) => {
         const storage = await createFileBackupStorage(dirName)
@@ -980,7 +982,7 @@ export function devTool (
               return
             }
 
-            await backup(toolCtx, pipeline, wsIds, storage, {
+            await backup(toolCtx, pipeline, wsIds, storage, db, {
               force: cmd.force,
               include: cmd.include === '*' ? undefined : new Set(cmd.include.split(';').map((it) => it.trim())),
               skipDomains: (cmd.skip ?? '').split(';').map((it) => it.trim()),
@@ -991,7 +993,8 @@ export function devTool (
                 .split(';')
                 .map((it) => it.trim())
                 .filter((it) => it.length > 0),
-              keepSnapshots: parseInt(cmd.keepSnapshots)
+              keepSnapshots: parseInt(cmd.keepSnapshots),
+              fullVerify: cmd.fullVerify
             })
           } catch (err: any) {
             toolCtx.error('Failed to backup workspace', { err, workspace })

qms-tests/docker-compose.yaml

@@ -122,6 +122,7 @@ services:
       - TRANSACTOR_URL=ws://transactor:3334;ws://huly.local:3334
       - STORAGE_CONFIG=${STORAGE_CONFIG}
       - ACCOUNTS_URL=http://account:3003
+      - ACCOUNTS_DB_URL=${DB_PG_URL}
       - BRANDING_PATH=/var/cfg/branding.json
     restart: unless-stopped
   front:

server/backup-service/src/config.ts

@@ -17,6 +17,8 @@ import { type BackupConfig } from '@hcengineering/server-backup'
 
 interface Config extends Omit<BackupConfig, 'Token'> {
   AccountsURL: string
+  AccountsDbURL: string
+  AccountsDbNS: string
   ServiceID: string
   Secret: string
 
@@ -36,6 +38,8 @@ interface Config extends Omit<BackupConfig, 'Token'> {
 
 const envMap: { [key in keyof Config]: string } = {
   AccountsURL: 'ACCOUNTS_URL',
+  AccountsDbURL: 'ACCOUNTS_DB_URL',
+  AccountsDbNS: 'ACCOUNTS_NS',
   ServiceID: 'SERVICE_ID',
   Secret: 'SECRET',
   BucketName: 'BUCKET_NAME',
@@ -53,6 +57,7 @@ const envMap: { [key in keyof Config]: string } = {
 
 const required: Array<keyof Config> = [
   'AccountsURL',
+  'AccountsDbURL',
   'Secret',
   'ServiceID',
   'BucketName',
@@ -64,6 +69,8 @@ const required: Array<keyof Config> = [
 export const config: () => Config = () => {
   const params: Partial<Config> = {
     AccountsURL: process.env[envMap.AccountsURL],
+    AccountsDbURL: process.env[envMap.AccountsDbURL],
+    AccountsDbNS: process.env[envMap.AccountsDbNS],
     Secret: process.env[envMap.Secret],
     BucketName: process.env[envMap.BucketName] ?? 'backups',
     ServiceID: process.env[envMap.ServiceID] ?? 'backup-service',

server/backup/package.json

@@ -42,7 +42,9 @@
   "dependencies": {
     "@hcengineering/platform": "^0.6.11",
     "@hcengineering/core": "^0.6.32",
+    "@hcengineering/account": "^0.6.0",
     "@hcengineering/contact": "^0.6.24",
+    "@hcengineering/model-contact": "^0.6.1",
     "@hcengineering/client-resources": "^0.6.27",
     "@hcengineering/client": "^0.6.18",
     "@hcengineering/model": "^0.6.11",