working state after refactoring

Signed-off-by: Anna Khismatullina <[email protected]>

Author: akhismat

packages/importer/src/huly/huly.ts

@@ -27,6 +27,7 @@ import core, {
   Attribute,
   type Class,
   type Doc,
+  isId,
   generateId,
   PersonId,
   type Ref,
@@ -63,6 +64,8 @@ import { type Logger } from '../importer/logger'
 import { BaseMarkdownPreprocessor } from '../importer/preprocessor'
 import { type FileUploader } from '../importer/uploader'
 import { Props, UnifiedDoc } from '../types'
+import { readMarkdownContent, readYamlHeader } from './parsing'
+import { UnifiedDocProcessor } from './unified'
 
 export interface HulyComment {
   author: string
@@ -350,13 +353,16 @@ export class HulyFormatImporter {
 
   private readonly personIdByEmail = new Map<string, PersonId>()
 
+  private readonly unifiedDocImporter: UnifiedDocProcessor
+
   constructor (
     private readonly client: TxOperations,
     private readonly fileUploader: FileUploader,
     private readonly logger: Logger,
     private readonly importerSocialId?: PersonId,
     private readonly importerPerson?: Ref<Person>
   ) {
+    this.unifiedDocImporter = new UnifiedDocProcessor(this.client, this.logger)
   }
 
   private async initCaches (): Promise<void> {
@@ -482,6 +488,29 @@ export class HulyFormatImporter {
       }
     }
 
+    // Импортируем UnifiedDoc сущности
+    const unifiedResult = await this.unifiedDocImporter.importFromDirectory(folderPath)
+
+    // Разбираем и добавляем в билдер по классу
+    for (const [path, docs] of unifiedResult.entries()) {
+      for (const doc of docs) {
+        switch (doc._class) {
+          case card.class.MasterTag:
+            builder.addMasterTag(path, doc as UnifiedDoc<MasterTag>)
+            break
+          case core.class.Attribute:
+            builder.addMasterTagAttributes(path, [doc as UnifiedDoc<Attribute<MasterTag>>])
+            break
+          default:
+            if (isId(doc._class)) {
+              builder.addCard(path, doc as UnifiedDoc<Card>)
+            } else {
+              this.logger.error(`Unknown doc class ${String(doc._class)} for path ${path}`)
+            }
+        }
+      }
+    }
+
     // Process all yaml files first
     const yamlFiles = fs.readdirSync(folderPath).filter((f) => f.endsWith('.yaml') && f !== 'settings.yaml')
 
@@ -528,22 +557,7 @@ export class HulyFormatImporter {
           }
 
           case card.class.MasterTag: {
-            const masterTag = await this.processMasterTag(spaceConfig)
-            const masterTagId = masterTag.props._id as Ref<MasterTag>
-            if (masterTagId === undefined) {
-              throw new Error('Master tag ID is undefined')
-            }
-            builder.addMasterTag(spacePath, masterTag)
-
-            const attributesByLabel = await this.processMasterTagAttributes(spaceConfig, masterTagId)
-            builder.addMasterTagAttributes(spacePath, Array.from(attributesByLabel.values()))
-
-            if (fs.existsSync(spacePath) && fs.statSync(spacePath).isDirectory()) {
-              // Сначала обрабатываем дочерние мастер-теги
-              await this.processSubTagsRecursively(builder, spacePath, spacePath, masterTagId, attributesByLabel)
-              // Затем обрабатываем карточки
-              await this.processCardsRecursively(builder, spacePath, spacePath, masterTagId, attributesByLabel)
-            }
+            this.logger.log(`Skipping ${spaceName}: master tag already processed`)
             break
           }
 
@@ -669,7 +683,7 @@ export class HulyFormatImporter {
 
     for (const cardFile of cardFiles) {
       const cardPath = path.join(currentPath, cardFile)
-      const cardHeader = (await this.readYamlHeader(cardPath)) as Record<string, any>
+      const cardHeader = await readYamlHeader(cardPath)
 
       if (cardHeader.class === undefined) { // means it's a card of class MasterTag
         const card = await this.processCard(cardHeader, cardPath, masterTagId, attributesByLabel)
@@ -710,7 +724,7 @@ export class HulyFormatImporter {
     return {
       _class: masterTagId,
       collabField: 'content',
-      contentProvider: () => this.readMarkdownContent(cardPath),
+      contentProvider: () => readMarkdownContent(cardPath),
       props: props as Props<Card> // todo: what is the correct props type?
     }
   }
@@ -726,7 +740,7 @@ export class HulyFormatImporter {
 
     for (const issueFile of issueFiles) {
       const issuePath = path.join(currentPath, issueFile)
-      const issueHeader = (await this.readYamlHeader(issuePath)) as HulyIssueHeader
+      const issueHeader = (await readYamlHeader(issuePath)) as HulyIssueHeader
 
       if (issueHeader.class === undefined) {
         this.logger.error(`Skipping ${issueFile}: not an issue`)
@@ -750,7 +764,7 @@ export class HulyFormatImporter {
           class: tracker.class.Issue,
           title: issueHeader.title,
           number: parseInt(issueNumber ?? 'NaN'),
-          descrProvider: async () => await this.readMarkdownContent(issuePath),
+          descrProvider: async () => await readMarkdownContent(issuePath),
           status: { name: issueHeader.status },
           priority: issueHeader.priority,
           estimation: issueHeader.estimation,
@@ -838,7 +852,7 @@ export class HulyFormatImporter {
 
     for (const docFile of docFiles) {
       const docPath = path.join(currentPath, docFile)
-      const docHeader = (await this.readYamlHeader(docPath)) as HulyDocumentHeader
+      const docHeader = (await readYamlHeader(docPath)) as HulyDocumentHeader
 
       if (docHeader.class === undefined) {
         this.logger.error(`Skipping ${docFile}: not a document`)
@@ -859,7 +873,7 @@ export class HulyFormatImporter {
           id: docMeta.id as Ref<Document>,
           class: document.class.Document,
           title: docHeader.title,
-          descrProvider: async () => await this.readMarkdownContent(docPath),
+          descrProvider: async () => await readMarkdownContent(docPath),
           subdocs: [] // Will be added via builder
         }
 
@@ -886,7 +900,7 @@ export class HulyFormatImporter {
 
     for (const docFile of docFiles) {
       const docPath = path.join(currentPath, docFile)
-      const docHeader = (await this.readYamlHeader(docPath)) as
+      const docHeader = (await readYamlHeader(docPath)) as
         | HulyControlledDocumentHeader
         | HulyDocumentTemplateHeader
 
@@ -1087,7 +1101,7 @@ export class HulyFormatImporter {
       reviewers: header.reviewers?.map((email) => this.findEmployeeByName(email)) ?? [],
       approvers: header.approvers?.map((email) => this.findEmployeeByName(email)) ?? [],
       coAuthors: header.coAuthors?.map((email) => this.findEmployeeByName(email)) ?? [],
-      descrProvider: async () => await this.readMarkdownContent(docPath),
+      descrProvider: async () => await readMarkdownContent(docPath),
       ccReason: header.changeControl?.reason,
       ccImpact: header.changeControl?.impact,
       ccDescription: header.changeControl?.description,
@@ -1125,30 +1139,14 @@ export class HulyFormatImporter {
       reviewers: header.reviewers?.map((email) => this.findEmployeeByName(email)) ?? [],
       approvers: header.approvers?.map((email) => this.findEmployeeByName(email)) ?? [],
       coAuthors: header.coAuthors?.map((email) => this.findEmployeeByName(email)) ?? [],
-      descrProvider: async () => await this.readMarkdownContent(docPath),
+      descrProvider: async () => await readMarkdownContent(docPath),
       ccReason: header.changeControl?.reason,
       ccImpact: header.changeControl?.impact,
       ccDescription: header.changeControl?.description,
       subdocs: []
     }
   }
 
-  private async readYamlHeader (filePath: string): Promise<any> {
-    this.logger.log('Read YAML header from: ' + filePath)
-    const content = fs.readFileSync(filePath, 'utf8')
-    const match = content.match(/^---\n([\s\S]*?)\n---/)
-    if (match != null) {
-      return yaml.load(match[1])
-    }
-    return {}
-  }
-
-  private async readMarkdownContent (filePath: string): Promise<string> {
-    const content = fs.readFileSync(filePath, 'utf8')
-    const match = content.match(/^---\n[\s\S]*?\n---\n(.*)$/s)
-    return match != null ? match[1] : content
-  }
-
   private async cacheAccountsByEmails (): Promise<void> {
     const employees = await this.client.findAll(
       contact.mixin.Employee,

packages/importer/src/huly/parsing.ts

@@ -0,0 +1,17 @@
+import * as fs from 'fs'
+import * as yaml from 'js-yaml'
+
+export async function readYamlHeader (filePath: string): Promise<any> {
+  const content = fs.readFileSync(filePath, 'utf8')
+  const match = content.match(/^---\n([\s\S]*?)\n---/)
+  if (match != null) {
+    return yaml.load(match[1])
+  }
+  return {}
+}
+
+export async function readMarkdownContent (filePath: string): Promise<string> {
+  const content = fs.readFileSync(filePath, 'utf8')
+  const match = content.match(/^---\n[\s\S]*?\n---\n(.*)$/s)
+  return match != null ? match[1] : content
+}

packages/importer/src/huly/unified.ts

@@ -0,0 +1,169 @@
+// unified.ts
+import { UnifiedDoc, Props } from '../types'
+import card, { Card, MasterTag } from '@hcengineering/card'
+import core, {
+  Attribute,
+  type Doc,
+  generateId,
+  type Ref,
+  type TxOperations
+} from '@hcengineering/core'
+import * as fs from 'fs'
+import * as path from 'path'
+import * as yaml from 'js-yaml'
+import { Logger } from '../importer/logger'
+import { IntlString } from '../../../platform/types'
+import { readMarkdownContent, readYamlHeader } from './parsing'
+
+export type UnifiedDocProcessResult = Map<string, Array<UnifiedDoc<Doc>>>
+
+export class UnifiedDocProcessor {
+  constructor (
+    private readonly client: TxOperations,
+    private readonly logger: Logger
+  ) {}
+
+  async importFromDirectory (directoryPath: string): Promise<UnifiedDocProcessResult> {
+    const unifiedDocs: UnifiedDocProcessResult = new Map()
+    await this.processDirectory(directoryPath, unifiedDocs)
+    return unifiedDocs
+  }
+
+  private async processDirectory (
+    currentPath: string,
+    result: UnifiedDocProcessResult,
+    parentMasterTagId?: Ref<MasterTag>,
+    parentAttributesByLabel?: Map<string, UnifiedDoc<Attribute<MasterTag>>>
+  ): Promise<void> {
+    const entries = fs.readdirSync(currentPath, { withFileTypes: true })
+
+    // Сначала обрабатываем YAML файлы (потенциальные мастер-теги)
+    for (const entry of entries) {
+      if (!entry.isFile() || !entry.name.endsWith('.yaml')) continue // todo: filter entries by extension
+
+      const yamlPath = path.join(currentPath, entry.name)
+      const yamlConfig = yaml.load(fs.readFileSync(yamlPath, 'utf8')) as Record<string, any>
+
+      if (yamlConfig?.class === card.class.MasterTag) {
+        const masterTag = await this.createMasterTag(yamlConfig, parentMasterTagId)
+        const masterTagId = masterTag.props._id as Ref<MasterTag>
+        const attributesByLabel = await this.createMasterTagAttributes(yamlConfig, masterTagId)
+
+        // Добавляем мастер-тег и его атрибуты
+        const docs = result.get(yamlPath) ?? []
+        docs.push(
+          masterTag,
+          ...Array.from(attributesByLabel.values())
+        )
+        result.set(yamlPath, docs)
+
+        // Рекурсивно обрабатываем содержимое директории мастер-тега
+        const tagDir = path.join(currentPath, path.basename(yamlPath, '.yaml'))
+        if (fs.existsSync(tagDir) && fs.statSync(tagDir).isDirectory()) {
+          await this.processDirectory(tagDir, result, masterTagId, attributesByLabel)
+        }
+      }
+    }
+
+    if (parentMasterTagId === undefined || parentAttributesByLabel === undefined) {
+      // means we are in the root directory
+      return
+    }
+
+    // Затем обрабатываем markdown файлы (карточки)
+    for (const entry of entries) {
+      if (!entry.isFile() || !entry.name.endsWith('.md')) continue
+
+      const cardPath = path.join(currentPath, entry.name)
+      const cardHeader = await readYamlHeader(cardPath)
+      const unifiedDoc = await this.createCard(cardHeader, cardPath, parentMasterTagId, parentAttributesByLabel)
+
+      if (unifiedDoc != null) {
+        const docs = result.get(cardPath) ?? []
+        docs.push(unifiedDoc)
+        result.set(cardPath, docs)
+      }
+    }
+  }
+
+  private async createMasterTag (
+    data: Record<string, any>,
+    parentMasterTagId?: Ref<MasterTag>
+  ): Promise<UnifiedDoc<MasterTag>> {
+    const { class: _class, title } = data
+    if (_class !== card.class.MasterTag) {
+      throw new Error('Invalid master tag data')
+    }
+
+    return {
+      _class: card.class.MasterTag,
+      props: {
+        _id: generateId<MasterTag>(),
+        space: core.space.Model,
+        extends: parentMasterTagId ?? card.class.Card,
+        label: 'embedded:embedded:' + title as IntlString, // todo: check if it's correct
+        kind: 0,
+        icon: card.icon.MasterTag
+      }
+    }
+  }
+
+  private async createMasterTagAttributes (
+    data: Record<string, any>,
+    masterTagId: Ref<MasterTag>
+  ): Promise<Map<string, UnifiedDoc<Attribute<MasterTag>>>> {
+    if (data.properties === undefined) {
+      return new Map()
+    }
+
+    const attributesByLabel = new Map<string, UnifiedDoc<Attribute<MasterTag>>>()
+    for (const property of data.properties) {
+      const attr: UnifiedDoc<Attribute<MasterTag>> = {
+        _class: core.class.Attribute,
+        props: {
+          space: core.space.Model,
+          attributeOf: masterTagId,
+          name: generateId<Attribute<MasterTag>>(),
+          label: 'embedded:embedded:' + property.label as IntlString, // todo: check if it's correct
+          isCustom: true,
+          type: {
+            _class: 'core:class:' + property.type
+          },
+          defaultValue: property.defaultValue ?? null
+        }
+      }
+      attributesByLabel.set(property.label, attr)
+    }
+    return attributesByLabel
+  }
+
+  private async createCard (
+    cardHeader: Record<string, any>,
+    cardPath: string,
+    masterTagId: Ref<MasterTag>,
+    attributesByLabel: Map<string, UnifiedDoc<Attribute<MasterTag>>>
+  ): Promise<UnifiedDoc<Card>> {
+    const { _class, title, ...customProperties } = cardHeader
+
+    const props: Record<string, any> = {
+      _id: generateId(),
+      space: core.space.Workspace,
+      title
+    }
+
+    for (const [key, value] of Object.entries(customProperties)) {
+      const attributeName = attributesByLabel.get(key)?.props.name
+      if (attributeName === undefined) {
+        throw new Error(`Attribute not found: ${key}`) // todo: keep the error till builder validation
+      }
+      props[attributeName] = value
+    }
+
+    return {
+      _class: masterTagId,
+      collabField: 'content',
+      contentProvider: () => readMarkdownContent(cardPath),
+      props: props as Props<Card> // todo: what is the correct props type?
+    }
+  }
+}