support blobs

Signed-off-by: Anna Khismatullina <[email protected]>

Author: akhismat

dev/import-tool/docs/huly/example-workspace/FileImport.md

@@ -0,0 +1,9 @@
+---
+class: card:types:File
+title: File with blob
+blobs:
+    - ./files/screenshot.png
+---
+
+*DESCR)*
+[](./files/screenshot.png)

dev/import-tool/docs/huly/example-workspace/SlaveCard/Frodo.md

@@ -9,10 +9,10 @@ color: pink
 x: from familiar
 y: from minion
 multy-enum: [Beta]
-attachments: 
-    - ../../CARDS_INSTRUCTIONS.md
+# attachments: 
+#     - ../../CARDS_INSTRUCTIONS.md
+blobs:
     - ../files/screenshot.png
-# blobs:
 #     - ../README.md
 ---
 

packages/importer/src/huly/metadata.ts

@@ -1,7 +1,7 @@
 import { Tag } from '@hcengineering/card'
-import { Association, Attribute, Doc, generateId, Ref } from '@hcengineering/core'
+import { Association, Attribute, Doc, generateId, Ref, Blob as PlatformBlob } from '@hcengineering/core'
 import path from 'path'
-import { UnifiedDoc } from '../types'
+import { UnifiedDoc, UnifiedFile } from '../types'
 
 export interface RelationMetadata { // todo: rename
   association: Ref<Association>
@@ -10,6 +10,7 @@ export interface RelationMetadata { // todo: rename
 }
 export type MapAttributeToUnifiedDoc = Map<string, UnifiedDoc<Attribute<Tag>>>
 export type MapNameToIsMany = Map<string, RelationMetadata> // todo: rename
+
 export interface TagMetadata {
   _id: string
   attributes: MapAttributeToUnifiedDoc // title -> attribute id
@@ -19,6 +20,7 @@ export interface TagMetadata {
 export class MetadataStorage {
   private readonly pathToRef = new Map<string, Ref<Doc>>() // todo: attachments to a separate map?
   private readonly pathToMetadata = new Map<string, TagMetadata>()
+  private readonly pathToBlobUuid = new Map<string, Ref<PlatformBlob>>() // todo: blobs to a separate map?
 
   public getIdByAbsolutePath (path: string): Ref<Doc> {
     let id = this.pathToRef.get(path)
@@ -29,11 +31,6 @@ export class MetadataStorage {
     return id
   }
 
-  public getIdByRelativePath (currentPath: string, relativePath: string): Ref<Doc> {
-    const fullPath = path.resolve(currentPath, relativePath)
-    return this.getIdByAbsolutePath(fullPath)
-  }
-
   public hasMetadata (path: string): boolean {
     return this.pathToMetadata.has(path)
   }

packages/importer/src/huly/unified.ts

@@ -4,6 +4,7 @@ import card, { Card, MasterTag, Tag } from '@hcengineering/card'
 import core, {
   Association,
   Attribute,
+  BlobType,
   Class,
   Doc,
   Enum,
@@ -29,7 +30,6 @@ export interface UnifiedDocProcessResult {
 
 export class UnifiedDocProcessor {
   private readonly metadataStorage = new MetadataStorage()
-
   async importFromDirectory (directoryPath: string): Promise<UnifiedDocProcessResult> {
     const result: UnifiedDocProcessResult = {
       docs: new Map(),
@@ -173,7 +173,11 @@ export class UnifiedDocProcessor {
     masterTagAttrs: Map<string, UnifiedDoc<Attribute<MasterTag>>>,
     parentCardId?: Ref<Card>
   ): Promise<void> {
-    const cardWithRelations = await this.createCardWithRelations(cardProps, cardPath, masterTagId, masterTagRelations, masterTagAttrs, parentCardId, blobs)
+    if (cardProps.blobs !== undefined) {
+      await this.createBlobs(cardProps.blobs, cardPath, result)
+    }
+
+    const cardWithRelations = await this.createCardWithRelations(cardProps, cardPath, masterTagId, masterTagRelations, masterTagAttrs, result.files, parentCardId)
 
     if (cardWithRelations.length > 0) {
       const docs = result.docs.get(cardPath) ?? []
@@ -184,7 +188,7 @@ export class UnifiedDocProcessor {
       await this.applyTags(card, cardProps, cardPath, result)
 
       if (cardProps.attachments !== undefined) {
-        await this.processAttachments(cardProps.attachments, cardPath, card, result)
+        await this.createAttachments(cardProps.attachments, cardPath, card, result)
       }
 
       const cardDir = path.join(path.dirname(cardPath), path.basename(cardPath, '.md'))
@@ -387,10 +391,12 @@ export class UnifiedDocProcessor {
     masterTagId: Ref<MasterTag>,
     masterTagRelations: Map<string, RelationMetadata>, // todo: rename to masterTagsAssociations
     masterTagAttrs: Map<string, UnifiedDoc<Attribute<MasterTag>>>,
+    blobFiles: Map<string, UnifiedFile>,
     parentCardId?: Ref<Card>
   ): Promise<UnifiedDoc<Doc>[]> {
-    const { _class, title, tags: rawTags, ...customProperties } = cardHeader
+    const { _class, title, blobs: rawBlobs, tags: rawTags, ...customProperties } = cardHeader
     const tags = rawTags !== undefined ? (Array.isArray(rawTags) ? rawTags : [rawTags]) : []
+    const blobs = rawBlobs !== undefined ? (Array.isArray(rawBlobs) ? rawBlobs : [rawBlobs]) : []
 
     const cardId = this.metadataStorage.getIdByAbsolutePath(cardPath) as Ref<Card>
     const cardProps: Record<string, any> = {
@@ -400,6 +406,24 @@ export class UnifiedDocProcessor {
       parent: parentCardId
     }
 
+    if (blobs.length > 0) {
+      const blobProps: Record<string, BlobType> = {}
+      for (const blob of blobs) {
+        const blobPath = path.resolve(path.dirname(cardPath), blob)
+        const blobFile = blobFiles.get(blobPath)
+        if (blobFile === undefined) {
+          throw new Error('Blob file not found: ' + blobPath + ' from:' + cardPath)
+        }
+        blobProps[blobFile._id] = {
+          file: blobFile._id,
+          type: blobFile.type,
+          name: blobFile.name,
+          metadata: {} // todo: blobFile.metadata
+        }
+      }
+      cardProps.blobs = blobProps
+    }
+
     const tagAssociations = new Map<string, RelationMetadata>()
     for (const tag of tags) {
       const tagPath = path.resolve(path.dirname(cardPath), tag)
@@ -510,28 +534,15 @@ export class UnifiedDocProcessor {
     }
   }
 
-  private async processAttachments (
+  private async createAttachments (
     attachments: string[],
     cardPath: string,
     card: UnifiedDoc<Card>,
     result: UnifiedDocProcessResult
   ): Promise<void> {
     for (const attachment of attachments) {
       const attachmentPath = path.resolve(path.dirname(cardPath), attachment)
-      const attachmentName = path.basename(attachmentPath)
-      const fileId = this.metadataStorage.getIdByAbsolutePath(attachmentPath) as Ref<PlatformBlob>
-      const type = contentType(attachmentPath)
-      const size = fs.statSync(attachmentPath).size
-
-      const file: UnifiedFile = {
-        _id: fileId, // id for datastore
-        name: attachmentName,
-        blobProvider: async () => {
-          const data = fs.readFileSync(attachmentPath)
-          const props = type !== false ? { type } : undefined
-          return new Blob([data], props)
-        }
-      }
+      const file = await this.createFile(attachmentPath)
       result.files.set(attachmentPath, file)
 
       const attachmentId = this.metadataStorage.getIdByAbsolutePath(attachmentPath) as Ref<Attachment>
@@ -542,18 +553,53 @@ export class UnifiedDocProcessor {
           space: core.space.Workspace,
           attachedTo: card.props._id as Ref<Card>,
           attachedToClass: card._class,
-          file: fileId,
-          name: attachmentName,
+          file: file._id,
+          name: file.name,
           collection: 'attachments',
           lastModified: Date.now(),
-          type: type !== false ? type : 'application/octet-stream',
-          size
+          type: file.type,
+          size: file.size
         }
       }
       result.docs.set(attachmentPath, [attachmentDoc])
     }
   }
 
+  private async createBlobs (
+    blobs: string[],
+    cardPath: string,
+    result: UnifiedDocProcessResult
+  ): Promise<void> {
+    for (const blob of blobs) {
+      const blobPath = path.resolve(path.dirname(cardPath), blob)
+      const file = await this.createFile(blobPath)
+      result.files.set(blobPath, file)
+    }
+  }
+
+  private async createFile (
+    fileAbsPath: string
+  ): Promise<UnifiedFile> {
+    // const fileAbsPath = path.resolve(path.dirname(currentPath), filePath)
+    const fileName = path.basename(fileAbsPath)
+    const fileId = this.metadataStorage.getIdByAbsolutePath(fileAbsPath) as Ref<PlatformBlob>
+    const type = contentType(fileName)
+    const size = fs.statSync(fileAbsPath).size
+
+    const file: UnifiedFile = {
+      _id: fileId, // id for datastore
+      name: fileName,
+      type: type !== false ? type : 'application/octet-stream',
+      size, // todo: make sure this one is needed
+      blobProvider: async () => {
+        const data = fs.readFileSync(fileAbsPath)
+        const props = type !== false ? { type } : undefined
+        return new Blob([data], props)
+      }
+    }
+    return file
+  }
+
   private async createAssociation (
     yamlPath: string,
     yamlConfig: Record<string, any>

packages/importer/src/types.ts

@@ -17,6 +17,8 @@ export interface UnifiedMixin<T extends Doc, M extends Doc> { // todo: extends T
 export interface UnifiedFile {
   _id: Ref<PlatformBlob>
   name: string
+  type: string
+  size: number
   blobProvider: blobProvider
 }