support attachments

Signed-off-by: Anna Khismatullina <[email protected]>

Author: akhismat

.vscode/launch.json

@@ -839,7 +839,7 @@
         "-pw",
         "1234",
         "-ws",
-        "ws4"
+        "ws1"
       ],
       "env": {
         "FRONT_URL": "http://localhost:8087"

dev/import-tool/docs/huly/example-workspace/SlaveCard.yaml

@@ -1,5 +1,5 @@
 class: card:class:MasterTag
-title: Slave Card 11
+title: Slave Card 1
 properties:
   - label: sex
     type: TypeBoolean

dev/import-tool/docs/huly/example-workspace/SlaveCard/Frodo.md

@@ -9,6 +9,11 @@ color: pink
 x: from familiar
 y: from minion
 multy-enum: [Beta]
+attachments: 
+    - ../../CARDS_INSTRUCTIONS.md
+    - ../files/screenshot.png
+# blobs:
+#     - ../README.md
 ---
 
 A brave hobbit guided by Gandalf.

packages/importer/src/huly/huly.ts

@@ -68,6 +68,7 @@ import { type FileUploader } from '../importer/uploader'
 import { UnifiedDoc } from '../types'
 import { readMarkdownContent, readYamlHeader } from './parsing'
 import { UnifiedDocProcessor } from './unified'
+import attachment from '@hcengineering/model-attachment'
 
 export interface HulyComment {
   author: string
@@ -490,7 +491,7 @@ export class HulyFormatImporter {
     }
 
     // Импортируем UnifiedDoc сущности
-    const { docs: unifiedDocs, mixins: unifiedMixins } = await this.unifiedDocImporter.importFromDirectory(folderPath)
+    const { docs: unifiedDocs, mixins: unifiedMixins, files } = await this.unifiedDocImporter.importFromDirectory(folderPath)
 
     // Разбираем и добавляем в билдер по классу
     for (const [path, docs] of unifiedDocs.entries()) {
@@ -514,6 +515,9 @@ export class HulyFormatImporter {
           case core.class.Enum:
             builder.addEnum(path, doc as UnifiedDoc<Enum>)
             break
+          case attachment.class.Attachment:
+            builder.addAttachment(path, doc as UnifiedDoc<Attachment>)
+            break
           default:
             if (isId(doc._class) || (doc._class as string).startsWith('card:types:')) { // todo: fix system cards validation
               builder.addCard(path, doc as UnifiedDoc<Card>)
@@ -532,6 +536,10 @@ export class HulyFormatImporter {
       }
     }
 
+    for (const [path, file] of files.entries()) {
+      builder.addFile(path, file)
+    }
+
     // Process all yaml files first
     const yamlFiles = fs.readdirSync(folderPath).filter((f) => f.endsWith('.yaml') && f !== 'settings.yaml')
 

packages/importer/src/huly/metadata.ts

@@ -17,7 +17,7 @@ export interface TagMetadata {
 }
 
 export class MetadataStorage {
-  private readonly pathToRef = new Map<string, Ref<Doc>>()
+  private readonly pathToRef = new Map<string, Ref<Doc>>() // todo: attachments to a separate map?
   private readonly pathToMetadata = new Map<string, TagMetadata>()
 
   public getIdByFullPath (path: string): Ref<Doc> {

packages/importer/src/huly/unified.ts

@@ -1,8 +1,11 @@
 // unified.ts
+import { Attachment } from '@hcengineering/attachment'
 import card, { Card, MasterTag, Tag } from '@hcengineering/card'
 import core, {
   Association,
   Attribute,
+  Blob as PlatformBlob,
+  Class,
   Doc,
   Enum,
   generateId,
@@ -11,15 +14,17 @@ import core, {
 } from '@hcengineering/core'
 import * as fs from 'fs'
 import * as yaml from 'js-yaml'
+import { contentType } from 'mime-types'
 import * as path from 'path'
 import { IntlString } from '../../../platform/types'
-import { Props, UnifiedDoc, UnifiedMixin } from '../types'
+import { Props, UnifiedDoc, UnifiedFile, UnifiedMixin } from '../types'
 import { MetadataStorage, RelationMetadata } from './metadata'
 import { readMarkdownContent, readYamlHeader } from './parsing'
 
 export interface UnifiedDocProcessResult {
   docs: Map<string, Array<UnifiedDoc<Doc>>>
   mixins: Map<string, Array<UnifiedMixin<Doc, Doc>>>
+  files: Map<string, UnifiedFile>
 }
 
 export class UnifiedDocProcessor {
@@ -28,7 +33,8 @@ export class UnifiedDocProcessor {
   async importFromDirectory (directoryPath: string): Promise<UnifiedDocProcessResult> {
     const result: UnifiedDocProcessResult = {
       docs: new Map(),
-      mixins: new Map()
+      mixins: new Map(),
+      files: new Map()
     }
     // Первый проход - собираем метаданные
     await this.processMetadata(directoryPath, result)
@@ -137,68 +143,6 @@ export class UnifiedDocProcessor {
     }
   }
 
-  // private async processDirectory (
-  //   currentPath: string,
-  //   result: UnifiedDocProcessResult,
-  //   parentMasterTagId?: Ref<MasterTag>,
-  //   parentMasterTagAttrs?: Map<string, UnifiedDoc<Attribute<MasterTag>>>
-  // ): Promise<void> {
-  //   const entries = fs.readdirSync(currentPath, { withFileTypes: true })
-
-  //   // Сначала обрабатываем YAML файлы (потенциальные мастер-теги)
-  //   for (const entry of entries) {
-  //     if (!entry.isFile() || !entry.name.endsWith('.yaml')) continue // todo: filter entries by extension
-
-  //     const yamlPath = path.resolve(currentPath, entry.name)
-  //     const yamlConfig = yaml.load(fs.readFileSync(yamlPath, 'utf8')) as Record<string, any>
-
-  //     switch (yamlConfig?.class) {
-  //       case card.class.MasterTag: {
-  //         const masterTagId = this.metadataStorage.getIdByFullPath(yamlPath) as Ref<MasterTag>
-  //         const masterTag = await this.createMasterTag(yamlConfig, masterTagId, parentMasterTagId)
-
-  //         const masterTagAttrs = await this.createAttributes(yamlConfig, masterTagId)
-  //         this.metadataStorage.setAttributes(yamlPath, masterTagAttrs)
-
-  //         const docs = result.docs.get(yamlPath) ?? []
-  //         docs.push(
-  //           masterTag,
-  //           ...Array.from(masterTagAttrs.values())
-  //         )
-  //         result.docs.set(yamlPath, docs)
-
-  //         const masterTagDir = path.join(currentPath, path.basename(yamlPath, '.yaml'))
-  //         if (fs.existsSync(masterTagDir) && fs.statSync(masterTagDir).isDirectory()) {
-  //           await this.processDirectory(masterTagDir, result, masterTagId, masterTagAttrs)
-  //         }
-  //         break
-  //       }
-  //       case card.class.Tag: {
-  //         if (parentMasterTagId === undefined) {
-  //           throw new Error('Tag should be inside master tag folder: ' + currentPath)
-  //         }
-
-  //         await this.processTag(yamlPath, yamlConfig, result, parentMasterTagId)
-  //         break
-  //       }
-  //       case core.class.Association: {
-  //         const association = await this.createAssociation(yamlPath, yamlConfig)
-  //         result.docs.set(yamlPath, [association])
-  //         break
-  //       }
-  //       default:
-  //         throw new Error('Unsupported class: ' + yamlConfig?.class) // todo: handle default case just convert to UnifiedDoc
-  //     }
-  //   }
-
-  //   if (parentMasterTagId === undefined || parentMasterTagAttrs === undefined) {
-  //     await this.processSystemCards(currentPath, result)
-  //   } else {
-  //     // await this.processCardDirectory(result, currentPath, parentMasterTagId, parentMasterTagAttrs) // todo: handle parent master tag attrs
-  //     await this.processCardDirectory(result, currentPath, parentMasterTagId)
-  //   }
-  // }
-
   private async processSystemCards (
     currentDir: string,
     result: UnifiedDocProcessResult,
@@ -239,6 +183,9 @@ export class UnifiedDocProcessor {
       const card = cardWithRelations[0] as UnifiedDoc<Card>
       await this.applyTags(card, cardProps, cardPath, result)
 
+      const attachments = cardProps.attachments ?? []
+      await this.processAttachments(attachments, cardPath, card, result)
+
       // Проверяем наличие дочерних карточек
       const cardDir = path.join(path.dirname(cardPath), path.basename(cardPath, '.md'))
       if (fs.existsSync(cardDir) && fs.statSync(cardDir).isDirectory()) {
@@ -563,6 +510,50 @@ export class UnifiedDocProcessor {
     }
   }
 
+  private async processAttachments (
+    attachments: string[],
+    cardPath: string,
+    card: UnifiedDoc<Card>,
+    result: UnifiedDocProcessResult
+  ): Promise<void> {
+    for (const attachment of attachments) {
+      const attachmentPath = path.resolve(path.dirname(cardPath), attachment)
+      const attachmentName = path.basename(attachmentPath)
+      const fileId = this.metadataStorage.getIdByFullPath(attachmentPath) as Ref<PlatformBlob>
+      const type = contentType(attachmentPath)
+      const size = fs.statSync(attachmentPath).size
+
+      const file: UnifiedFile = {
+        _id: fileId, // id for datastore
+        name: attachmentName,
+        blobProvider: async () => {
+          const data = fs.readFileSync(attachmentPath)
+          const props = type !== false ? { type } : undefined
+          return new Blob([data], props)
+        }
+      }
+      result.files.set(attachmentPath, file)
+
+      const attachmentId = this.metadataStorage.getIdByFullPath(attachmentPath) as Ref<Attachment>
+      const attachmentDoc: UnifiedDoc<Attachment> = {
+        _class: 'attachment:class:Attachment' as Ref<Class<Attachment>>,
+        props: {
+          _id: attachmentId, // id for attachment doc
+          space: core.space.Workspace,
+          attachedTo: card.props._id as Ref<Card>,
+          attachedToClass: card._class,
+          file: fileId,
+          name: attachmentName,
+          collection: 'attachments',
+          lastModified: Date.now(),
+          type: type !== false ? type : 'application/octet-stream',
+          size
+        }
+      }
+      result.docs.set(attachmentPath, [attachmentDoc])
+    }
+  }
+
   private async createAssociation (
     yamlPath: string,
     yamlConfig: Record<string, any>

packages/importer/src/importer/builder.ts

@@ -29,7 +29,8 @@ import {
   type ImportTeamspace,
   type ImportWorkspace
 } from './importer'
-import { UnifiedDoc, UnifiedMixin } from '../types'
+import { UnifiedDoc, UnifiedFile, UnifiedMixin } from '../types'
+import { Attachment } from '@hcengineering/attachment'
 
 export interface ValidationError {
   path: string
@@ -65,7 +66,9 @@ export class ImportWorkspaceBuilder {
   private readonly associations = new Map<string, UnifiedDoc<Association>>()
   private readonly relations = new Map<string, UnifiedDoc<Relation>>()
   private readonly enums = new Map<string, UnifiedDoc<Enum>>()
+  private readonly attachments = new Map<string, UnifiedDoc<Attachment>>()
   private readonly mixins = new Map<string, UnifiedMixin<Doc, Doc>>()
+  private readonly files = new Map<string, UnifiedFile>()
 
   private readonly projectTypes = new Map<string, ImportProjectType>()
   private readonly issueStatusCache = new Map<string, Ref<IssueStatus>>()
@@ -270,11 +273,21 @@ export class ImportWorkspaceBuilder {
     return this
   }
 
+  addAttachment (path: string, attachment: UnifiedDoc<Attachment>): this {
+    this.validateAndAdd('attachment', path, attachment, (a) => this.validateAttachment(a), this.attachments, path)
+    return this
+  }
+
   addTagMixin (path: string, mixin: UnifiedMixin<Doc, Doc>): this {
     this.validateAndAdd('tagMixin', path, mixin, (m) => this.validateTagMixin(m), this.mixins, path + '/' + mixin.mixin) // todo: fix mixin key
     return this
   }
 
+  addFile (path: string, file: UnifiedFile): this {
+    this.validateAndAdd('file', path, file, (f) => this.validateFile(f), this.files, path)
+    return this
+  }
+
   validate (): ValidationResult {
     // Perform cross-entity validation
     this.validateSpacesReferences()
@@ -352,9 +365,11 @@ export class ImportWorkspaceBuilder {
         ...Array.from(this.cards.values()),
         ...Array.from(this.associations.values()),
         ...Array.from(this.relations.values()),
-        ...Array.from(this.enums.values())
+        ...Array.from(this.enums.values()),
+        ...Array.from(this.attachments.values())
       ],
       mixins: Array.from(this.mixins.values()),
+      files: Array.from(this.files.values()),
       attachments: []
     }
   }
@@ -929,6 +944,14 @@ export class ImportWorkspaceBuilder {
     return errors
   }
 
+  private validateAttachment (attachment: UnifiedDoc<Attachment>): string[] {
+    const errors: string[] = []
+
+    // todo: validate attachment
+
+    return errors
+  }
+
   private validateCard (card: UnifiedDoc<Card>): string[] {
     const errors: string[] = []
 
@@ -1001,6 +1024,14 @@ export class ImportWorkspaceBuilder {
     return errors
   }
 
+  private validateFile (file: UnifiedFile): string[] {
+    const errors: string[] = []
+
+    // todo: validate file
+
+    return errors
+  }
+
   private validateOrgSpace (space: ImportOrgSpace): string[] {
     const errors: string[] = []