[TM-2769] add bulk upload media

apps/entity-service/src/entities/dto/media-bulk-response.dto.ts

@@ -0,0 +1,19 @@
+import { IsString } from "class-validator";
+import { ApiProperty } from "@nestjs/swagger";
+import { JsonApiDto } from "@terramatch-microservices/common/decorators";
+
+@JsonApiDto({ type: "mediaBulkResponse" })
+export class MediaBulkResponseDto {
+  constructor(index: number, error: string) {
+    this.index = index;
+    this.error = error;
+  }
+
+  @IsString()
+  @ApiProperty({ description: "The index of the media" })
+  index: number;
+
+  @IsString()
+  @ApiProperty({ description: "The error message" })
+  error: string;
+}

apps/entity-service/src/entities/dto/media-request-bulk.dto.ts

@@ -0,0 +1,37 @@
+import { ApiProperty } from "@nestjs/swagger";
+import { CreateDataDto, JsonApiBulkBodyDto } from "@terramatch-microservices/common/util/json-api-update-dto";
+import { IsBoolean, IsNumber, IsOptional, IsString, IsUrl } from "class-validator";
+
+export class MediaRequestBulkAttributes {
+  @IsString()
+  @IsUrl()
+  @ApiProperty({ required: true, description: "The URL of the media" })
+  downloadUrl: string;
+
+  @IsBoolean()
+  @ApiProperty({ description: "Whether the media is public" })
+  isPublic: boolean;
+
+  @IsNumber()
+  @IsOptional()
+  @ApiProperty({ type: Number, nullable: true, description: "The latitude of the media" })
+  lat: number | null;
+
+  @IsNumber()
+  @IsOptional()
+  @ApiProperty({ type: Number, nullable: true, description: "The longitude of the media" })
+  lng: number | null;
+}
+
+export class MediaRequestBulkBody extends JsonApiBulkBodyDto(
+  class MediaRequestBulkData extends CreateDataDto("media", MediaRequestBulkAttributes) {},
+  {
+    minSize: 1,
+    minSizeMessage: "At least one media must be provided",
+    description: "Array of media to create",
+    example: [
+      { type: "media", attributes: { isPublic: true, downloadUrl: "https://example.com/image.jpg" } },
+      { type: "media", attributes: { isPublic: false, downloadUrl: "https://example.com/image.jpg" } }
+    ]
+  }
+) {}

apps/entity-service/src/entities/dto/site-media-bulk-upload.dto.ts

@@ -0,0 +1,8 @@
+import { ApiProperty } from "@nestjs/swagger";
+import { IsUUID } from "class-validator";
+
+export class SiteMediaBulkUploadDto {
+  @IsUUID()
+  @ApiProperty({ description: "Site UUID to upload media to" })
+  siteUuid: string;
+}

apps/entity-service/src/entities/files.controller.ts

@@ -32,6 +32,11 @@ import { getBaseEntityByLaravelTypeAndId } from "./processors/media-owner-proces
 import { MediaUpdateBody } from "@terramatch-microservices/common/dto/media-update.dto";
 import { SingleMediaDto } from "./dto/media-query.dto";
 import { EntityType } from "@terramatch-microservices/database/constants/entities";
+import { SiteMediaBulkUploadDto } from "./dto/site-media-bulk-upload.dto";
+import { Site } from "@terramatch-microservices/database/entities/site.entity";
+import { MediaRequestBulkBody } from "./dto/media-request-bulk.dto";
+import { MediaBulkResponseDto } from "./dto/media-bulk-response.dto";
+import { Media } from "@terramatch-microservices/database/entities/media.entity";
 
 @Controller("entities/v3/files")
 export class FilesController {
@@ -59,6 +64,75 @@ export class FilesController {
     return this.entitiesService.mediaDto(media, { entityType: media.modelType as EntityType, entityUuid: model.uuid });
   }
 
+  @Post("/site/:siteUuid/bulkUpload")
+  @ApiOperation({
+    operationId: "siteMediaBulkUpload",
+    summary: "Upload multiple files to a site photos collection",
+    description: "Upload multiple files to a site photos collection"
+  })
+  @ExceptionResponse(UnauthorizedException, {
+    description: "Authentication failed, or site unavailable to current user."
+  })
+  @ExceptionResponse(NotFoundException, { description: "Site not found." })
+  @ExceptionResponse(BadRequestException, { description: "Invalid request." })
+  @JsonApiResponse([MediaDto, MediaBulkResponseDto])
+  async siteMediaBulkUpload(@Param() { siteUuid }: SiteMediaBulkUploadDto, @Body() payload: MediaRequestBulkBody) {
+    const site = await Site.findOne({ where: { uuid: siteUuid }, attributes: ["id", "frameworkKey", "projectId"] });
+    if (site == null) {
+      throw new NotFoundException(`Site with UUID ${siteUuid} not found`);
+    }
+    await this.policyService.authorize("uploadFiles", site);
+    const errors: MediaBulkResponseDto[] = [];
+    const createdMedias: Media[] = [];
+    await this.fileUploadService.transaction(async transaction => {
+      for (const [index, payloadData] of payload.data.entries()) {
+        let file: Express.Multer.File;
+        try {
+          file = await this.fileUploadService.fetchDataFromUrlAsMulterFile(payloadData.attributes.downloadUrl);
+          const media = await this.fileUploadService.uploadFile(
+            site,
+            "sites",
+            "photos",
+            file,
+            payloadData.attributes,
+            transaction
+          );
+          createdMedias.push(media);
+        } catch (error) {
+          errors.push(new MediaBulkResponseDto(index, error.message));
+        }
+      }
+      if (errors.length > 0) {
+        // clear s3 files
+        for (const media of createdMedias) {
+          await this.mediaService.deleteMediaFromS3(media);
+        }
+        throw new BadRequestException("Failed to upload some files");
+      }
+    });
+    let document;
+    if (errors.length > 0) {
+      document = buildJsonApi(MediaBulkResponseDto);
+      for (const error of errors) {
+        document.addData(error.index.toString(), new MediaBulkResponseDto(error.index, error.error));
+      }
+    } else {
+      document = buildJsonApi(MediaDto);
+      for (const media of createdMedias) {
+        document.addData(
+          media.uuid,
+          new MediaDto(media, {
+            url: this.mediaService.getUrl(media),
+            thumbUrl: this.mediaService.getUrl(media, "thumbnail"),
+            entityType: "sites",
+            entityUuid: site.uuid
+          })
+        );
+      }
+    }
+    return document;
+  }
+
   @Post("/:entity/:uuid/:collection")
   @ApiOperation({
     operationId: "uploadFile",

apps/entity-service/src/file/file-upload.service.spec.ts

@@ -8,10 +8,25 @@ import {
   MEDIA_OWNER_MODELS,
   MediaConfiguration,
   MediaOwnerModel,
-  MediaOwnerType
+  MediaOwnerType,
+  ValidationKey
 } from "@terramatch-microservices/database/constants/media-owners";
 import { MediaRequestAttributes } from "../entities/dto/media-request.dto";
 import { TranslatableException } from "@terramatch-microservices/common/exceptions/translatable.exception";
+import { Media } from "@terramatch-microservices/database/entities";
+import { Readable } from "stream";
+import path from "path";
+
+jest.mock("sharp", () => {
+  return jest.fn(() => ({
+    rotate: jest.fn().mockReturnThis(),
+    keepExif: jest.fn().mockReturnThis(),
+    toBuffer: jest.fn().mockResolvedValue(Buffer.from("mocked-image")),
+    resize: jest.fn().mockReturnThis()
+  }));
+});
+
+import sharp from "sharp";
 
 describe("FileUploadService", () => {
   let mediaService: jest.Mocked<MediaService>;
@@ -31,6 +46,94 @@ describe("FileUploadService", () => {
     validateFile(file: Express.Multer.File, config: MediaConfiguration): boolean | undefined;
   };
 
+  describe("fetchDataFromUrlAsMulterFile", () => {
+    const mockFetch = jest.fn();
+
+    beforeAll(() => {
+      global.fetch = mockFetch;
+    });
+
+    beforeEach(() => {
+      jest.clearAllMocks();
+    });
+
+    it("should fetch data from url as multer file", async () => {
+      const url = "https://example.com/image.png";
+      const buffer = Buffer.from("fake-image-data");
+
+      mockFetch.mockResolvedValue({
+        ok: true,
+        statusText: "OK",
+        headers: {
+          get: (key: string) => {
+            if (key === "content-type") return "image/png";
+            if (key === "content-length") return buffer.length.toString();
+            return null;
+          }
+        },
+        arrayBuffer: async () => buffer
+      });
+
+      const result = await service.fetchDataFromUrlAsMulterFile(url);
+
+      expect(result).toEqual(
+        expect.objectContaining({
+          fieldname: "uploadFile",
+          originalname: "image.png",
+          mimetype: "image/png",
+          size: buffer.length,
+          buffer
+        })
+      );
+
+      expect(result.stream).toBeInstanceOf(Readable);
+      expect(result.filename).toBe(path.basename(new URL(url).pathname));
+    });
+
+    it("should throw BadRequestException if fetch fails", async () => {
+      const url = "https://example.com/image.png";
+
+      mockFetch.mockRejectedValue(new Error("Network error"));
+
+      await expect(service.fetchDataFromUrlAsMulterFile(url)).rejects.toThrow(
+        new BadRequestException(`Failed to download file from URL ${url}: Network error`)
+      );
+    });
+    it("should throw BadRequestException if response is not ok", async () => {
+      const url = "https://example.com/image.png";
+
+      mockFetch.mockResolvedValue({
+        ok: false,
+        statusText: "404 Not Found"
+      });
+
+      await expect(service.fetchDataFromUrlAsMulterFile(url)).rejects.toThrow(
+        new BadRequestException(`Failed to download file from URL ${url}: 404 Not Found`)
+      );
+    });
+
+    it("should throw BadRequestException for invalid mime type", async () => {
+      const url = "https://example.com/file.txt";
+      const buffer = Buffer.from("text-data");
+
+      mockFetch.mockResolvedValue({
+        ok: true,
+        statusText: "OK",
+        headers: {
+          get: (key: string) => {
+            if (key === "content-type") return "text/plain";
+            return null;
+          }
+        },
+        arrayBuffer: async () => buffer
+      });
+
+      await expect(service.fetchDataFromUrlAsMulterFile(url)).rejects.toThrow(
+        new BadRequestException("Invalid file type")
+      );
+    });
+  });
+
   describe("getMediaType", () => {
     it('should return "documents" for a PDF mimetype', () => {
       const svc = service as unknown as PrivateFileUploadService;
@@ -48,6 +151,13 @@ describe("FileUploadService", () => {
   describe("validateFile", () => {
     const generalConfig: MediaConfiguration = { multiple: false, validation: "general-documents" };
 
+    it("should return false if configuration.validation is not set", () => {
+      const svc = service as unknown as PrivateFileUploadService;
+      const cfg: MediaConfiguration = { multiple: true, validation: null as unknown as ValidationKey };
+      const file = { mimetype: "any/type", size: 0 } as Express.Multer.File;
+      expect(svc.validateFile(file, cfg)).toBe(false);
+    });
+
     it("should do nothing if configuration.validation is not set", () => {
       const svc = service as unknown as PrivateFileUploadService;
       const cfg: MediaConfiguration = { multiple: true, validation: "documents" };
@@ -129,5 +239,52 @@ describe("FileUploadService", () => {
       expect(result.modelId).toBe(model.id);
       expect(result.createdBy).toBe(entitiesService.userId);
     });
+
+    it("should create thumbnail if file is an image", async () => {
+      const buffer = Buffer.from("mocked-image");
+      const file = {
+        originalname: "test.png",
+        mimetype: "image/png",
+        size: 123,
+        buffer: buffer
+      } as Express.Multer.File;
+      const result = await service.uploadFile(model, "sites", "photos", file, attributes);
+      expect(mediaService.uploadFile).toHaveBeenCalledWith(file.buffer, `${result.id}/test.png`, "image/png");
+      expect(mediaService.uploadFile).toHaveBeenCalledWith(
+        file.buffer,
+        `${result.id}/conversions/test-thumbnail.png`,
+        "image/png"
+      );
+      expect(result.generatedConversions).toEqual({ thumbnail: true });
+      expect(result.customProperties).toEqual({ custom_headers: { ACL: "public-read" }, thumbnailExtension: ".png" });
+    });
+
+    it("should throw an error when creating thumbnail fails", async () => {
+      (sharp as unknown as jest.Mock).mockImplementationOnce(() => {
+        throw new Error("sharp failed");
+      });
+
+      const buffer = Buffer.from("mocked-image");
+      const file = {
+        originalname: "test.png",
+        mimetype: "image/png",
+        size: 123,
+        buffer: buffer
+      } as Express.Multer.File;
+
+      await expect(service.uploadFile(model, "sites", "photos", file, attributes)).rejects.toThrow(Error);
+    });
+  });
+
+  describe("transaction", () => {
+    it("should commit a transaction on success", async () => {
+      const commit = jest.fn();
+      const mockTransaction = { commit, rollback: jest.fn() };
+      // @ts-expect-error incomplete mock
+      jest.spyOn(Media.sequelize, "transaction").mockResolvedValue(mockTransaction);
+      const result = await service.transaction(async () => "success");
+      expect(result).toBe("success");
+      expect(commit).toHaveBeenCalled();
+    });
   });
 });