Fix init workspace in dev (#8799)

Signed-off-by: Anna Khismatullina <[email protected]>
Signed-off-by: Alexey Zinoviev <[email protected]>
Co-authored-by: Alexey Zinoviev <[email protected]>

Author: akhismat

.gitignore

@@ -4,6 +4,7 @@ _api-extractor-temp/
 temp/
 .idea
 pods/workspace/init/
+pods/workspace/init-scripts/
 
 # Logs
 *.log

.vscode/launch.json

@@ -276,12 +276,14 @@
         "MINIO_ACCESS_KEY": "minioadmin",
         "MINIO_SECRET_KEY": "minioadmin",
         "MINIO_ENDPOINT": "localhost",
-        "MODEL_VERSION": "v0.7.1",
+        "MODEL_VERSION": "v0.7.75",
         "WS_OPERATION": "all+backup",
         "BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin",
-        "BACKUP_BUCKET": "dev-backups"
-        // "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init",
-        // "INIT_WORKSPACE": "staging-dev"
+        "BACKUP_BUCKET": "dev-backups",
+        "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init",
+        "INIT_WORKSPACE": "staging-dev",
+        "QUEUE_CONFIG": "huly.local:19092",
+        "QUEUE_REGION": ""
       },
       "runtimeVersion": "20",
       "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
@@ -309,10 +311,11 @@
         "MINIO_ACCESS_KEY": "minioadmin",
         "MINIO_SECRET_KEY": "minioadmin",
         "MINIO_ENDPOINT": "localhost",
-        "MODEL_VERSION": "0.7.1",
+        "MODEL_VERSION": "0.7.75",
         "WS_OPERATION": "all+backup",
         "BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin",
-        "BACKUP_BUCKET": "dev-backups"
+        "BACKUP_BUCKET": "dev-backups",
+        "QUEUE_CONFIG": "huly.local:19092"
         // "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init",
         // "INIT_WORKSPACE": "staging-dev"
       },

dev/docker-compose.min.yaml

@@ -53,6 +53,10 @@ services:
       - 9001:9001
     volumes:
       - files:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 5s
+      retries: 10
     restart: unless-stopped
   elastic:
     image: 'elasticsearch:7.14.2'
@@ -143,7 +147,7 @@ services:
       - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG}
       - BACKUP_BUCKET=${BACKUP_BUCKET_NAME}
       - MAIL_URL=
-      # - INIT_WORKSPACE=staging-dev
+      - INIT_WORKSPACE=staging-dev
     restart: unless-stopped
   collaborator:
     image: hardcoreeng/collaborator
@@ -255,6 +259,30 @@ services:
       - STATS_URL=http://huly.local:4900
       - REKONI_URL=http://huly.local:4004
       - ACCOUNTS_URL=http://huly.local:3000
+  datalake:
+    image: hardcoreeng/datalake
+    extra_hosts:
+      - 'huly.local:host-gateway'
+    depends_on:
+      minio:
+        condition: service_healthy
+      cockroach:
+        condition: service_started
+      stats:
+        condition: service_started
+      account:
+        condition: service_started
+    ports:
+      - 4030:4030
+    environment:
+      - PORT=4030
+      - SECRET=secret
+      - ACCOUNTS_URL=http://huly.local:3000
+      - STATS_URL=http://huly.local:4900
+      - STREAM_URL=http://huly.local:1080/recording
+      - DB_URL=${DB_CR_URL}
+      - BUCKETS=blobs,eu|http://minio:9000?accessKey=minioadmin&secretKey=minioadmin
+    restart: unless-stopped
 volumes:
   db:
   dbpg:

models/card/src/migration.ts

@@ -96,10 +96,11 @@ async function setParentInfo (client: MigrationClient): Promise<void> {
   )
 }
 
-function extractObjectProps<T extends Doc> (doc: T): Data<T> {
+function extractObjectData<T extends Doc> (doc: T): Data<T> {
+  const dataKeys = ['_id', 'space', 'modifiedOn', 'modifiedBy', 'createdBy', 'createdOn']
   const data: any = {}
   for (const key in doc) {
-    if (key === '_id') {
+    if (dataKeys.includes(key)) {
       continue
     }
     data[key] = doc[key]
@@ -114,8 +115,8 @@ async function migrateViewlets (client: Client): Promise<void> {
   const currentViewlets = await client.findAll(view.class.Viewlet, { attachTo: { $in: masterTags.map((p) => p._id) } })
   for (const masterTag of masterTags) {
     for (const viewlet of viewlets) {
-      const base = extractObjectProps(viewlet)
-      const resConfig = base.config
+      const base = extractObjectData(viewlet)
+      const resConfig = [...base.config]
       let index = -1
       if (viewlet.descriptor === view.viewlet.List) {
         index = viewlet.config.findIndex((p) => typeof p !== 'string' && p.displayProps?.grow === true)

models/contact/package.json

@@ -43,6 +43,7 @@
     "@hcengineering/model-presentation": "^0.6.0",
     "@hcengineering/model-view": "^0.6.0",
     "@hcengineering/model-workbench": "^0.6.1",
+    "@hcengineering/model-card": "^0.6.0",
     "@hcengineering/notification": "^0.6.23",
     "@hcengineering/platform": "^0.6.11",
     "@hcengineering/setting": "^0.6.17",

models/contact/src/index.ts

@@ -15,7 +15,7 @@
 //
 
 import activity from '@hcengineering/activity'
-import card, { type Role, type Card } from '@hcengineering/card'
+import { type Role, type Card } from '@hcengineering/card'
 import {
   AvatarType,
   type UserRole,
@@ -36,7 +36,6 @@ import {
 } from '@hcengineering/contact'
 import {
   AccountRole,
-  ClassifierKind,
   DOMAIN_MODEL,
   DateRangeMode,
   IndexKind,
@@ -52,6 +51,7 @@ import {
   type SocialIdType,
   type Timestamp
 } from '@hcengineering/core'
+import { createSystemType } from '@hcengineering/model-card'
 import {
   Collection as CollectionType,
   Hidden,
@@ -280,55 +280,6 @@ export class TUserRole extends TDoc implements UserRole {
   role!: Ref<Role>
 }
 
-function createUserProfileTag (builder: Builder): void {
-  builder.createDoc(
-    card.class.MasterTag,
-    core.space.Model,
-    {
-      extends: card.class.Card,
-      label: contact.string.UserProfile,
-      kind: ClassifierKind.CLASS,
-      icon: contact.icon.Person
-    },
-    contact.class.UserProfile
-  )
-
-  builder.createDoc(core.class.Attribute, core.space.Model, {
-    attributeOf: contact.class.UserProfile,
-    name: 'person',
-    label: contact.string.Person,
-    icon: contact.icon.Person,
-    type: TypeRef(contact.class.Person),
-    readonly: true
-  })
-
-  builder.mixin(contact.class.UserProfile, core.class.Mixin, setting.mixin.Editable, {
-    value: false
-  })
-  builder.mixin(contact.class.UserProfile, core.class.Mixin, setting.mixin.UserMixin, {})
-}
-
-function createUserProfileViewlet (builder: Builder): void {
-  builder.createDoc<Viewlet>(
-    view.class.Viewlet,
-    core.space.Model,
-    {
-      attachTo: contact.class.UserProfile,
-      descriptor: view.viewlet.Table,
-      config: [
-        '',
-        { key: 'modifiedOn', displayProps: { key: 'modified', fixed: 'right' } },
-        { key: 'createdBy', displayProps: { fixed: 'left', key: 'app' } }
-      ],
-      configOptions: {
-        hiddenKeys: ['name'],
-        sortable: true
-      }
-    },
-    contact.viewlet.TableUserProfile
-  )
-}
-
 export function createModel (builder: Builder): void {
   builder.createModel(
     TAvatarProvider,
@@ -1319,6 +1270,14 @@ export function createModel (builder: Builder): void {
   })
 
   createAttributePresenter(builder, contact.component.SpaceMembersEditor, core.class.Space, 'members', 'array')
-  createUserProfileTag(builder)
-  createUserProfileViewlet(builder)
+  createSystemType(builder, contact.class.UserProfile, contact.icon.Person, contact.string.UserProfile)
+  builder.createDoc(core.class.Attribute, core.space.Model, {
+    attributeOf: contact.class.UserProfile,
+    name: 'person',
+    label: contact.string.Person,
+    icon: contact.icon.Person,
+    type: TypeRef(contact.class.Person),
+    isCustom: true,
+    readonly: true
+  })
 }

packages/importer/src/huly/cards.ts

@@ -32,11 +32,10 @@ import * as yaml from 'js-yaml'
 import { contentType } from 'mime-types'
 import * as path from 'path'
 import { IntlString } from '../../../platform/types'
-import { Props, UnifiedDoc, UnifiedUpdate, UnifiedFile, UnifiedMixin } from '../types'
-import { MetadataRegistry, AssociationMetadata } from './registry'
-import { readMarkdownContent, readYamlHeader } from './parsing'
 import { Logger } from '../importer/logger'
-import { validateSchema } from './validation'
+import { Props, UnifiedDoc, UnifiedFile, UnifiedMixin, UnifiedUpdate } from '../types'
+import { UnifiedFormatParser } from './parser'
+import { AssociationMetadata, MetadataRegistry } from './registry'
 import {
   AssociationSchema,
   BaseFieldType,
@@ -51,6 +50,7 @@ import {
   StringFieldType,
   TagSchema
 } from './schema'
+import { validateSchema } from './validation'
 
 export interface UnifiedDocProcessResult {
   docs: Map<string, Array<UnifiedDoc<Doc>>>
@@ -62,6 +62,7 @@ export interface UnifiedDocProcessResult {
 export class CardsProcessor {
   constructor (
     private readonly metadataRegistry: MetadataRegistry,
+    private readonly parser: UnifiedFormatParser,
     private readonly logger: Logger
   ) {}
 
@@ -199,7 +200,7 @@ export class CardsProcessor {
     for (const entry of entries) {
       if (entry.isFile() && entry.name.endsWith('.md')) {
         const cardPath = path.join(currentPath, entry.name)
-        const { class: cardType, ...cardProps } = await readYamlHeader(cardPath)
+        const { class: cardType, ...cardProps } = this.parser.readYamlHeader(cardPath)
 
         if (masterTagId !== undefined) {
           await this.processCard(result, cardPath, cardProps, masterTagId, masterTagAssociaions, masterTagAttributes)
@@ -231,7 +232,7 @@ export class CardsProcessor {
     for (const entry of entries) {
       if (entry.isFile() && entry.name.endsWith('.md')) {
         const cardPath = path.join(currentDir, entry.name)
-        const { class: cardType, ...cardProps } = await readYamlHeader(cardPath)
+        const { class: cardType, ...cardProps } = this.parser.readYamlHeader(cardPath)
 
         if (cardType !== undefined && cardType.startsWith('card:types:') === false) {
           throw new Error('Unsupported card type: ' + cardType + ' in ' + cardPath)
@@ -310,7 +311,7 @@ export class CardsProcessor {
 
     for (const entry of entries) {
       const childCardPath = path.join(cardDir, entry.name)
-      const { class: cardClass, ...cardProps } = await readYamlHeader(childCardPath)
+      const { class: cardClass, ...cardProps } = this.parser.readYamlHeader(childCardPath)
       await this.processCard(
         result,
         childCardPath,
@@ -594,7 +595,7 @@ export class CardsProcessor {
       {
         _class: masterTagId,
         collabField: 'content',
-        contentProvider: () => readMarkdownContent(cardPath),
+        contentProvider: () => Promise.resolve(this.parser.readMarkdownContent(cardPath)),
         props: cardProps as Props<Card>
       },
       ...relations