SIMSBIOHUB-833: Bulk Download Based on Query (#355)

* feat(database): add search_filters column to download table

* feat(download): path, service, repo adds

* feat(download): Frontend — Download All button

* fix(download): Anonymous user

* refactor(download): cleanup interfaces

* fix(download): Lint

* fix(download): small fix

* fix(download): addressed reviewer's comments

* fix(download): fixed a few tests

---------

Co-authored-by: Macgregor Aubertin-Young <108430771+mauberti-bc@users.noreply.github.com>

Author: dwhogan

api/src/__integration__/db/download-service.integration.ts

@@ -60,7 +60,11 @@ describe('DownloadPipelineService (integration)', function () {
       const featureId1 = await createTestFeature(connection, submissionId, 'dataset', { name: 'Dataset A' });
       const featureId2 = await createTestFeature(connection, submissionId, 'dataset', { name: 'Dataset B' });
       // Step 2: Create download request through the service (no team for test)
-      const result = await service.createDownloadRequest(null, [featureId1, featureId2]);
+      const result = await service.createDownloadRequest({
+        systemUserId: null,
+        teamId: null,
+        submissionFeatureIds: [featureId1, featureId2]
+      });
 
       // Step 3: Verify download record was created with correct initial state
       const download = await crudService.findDownloadById(result.download_id);
@@ -94,7 +98,7 @@ describe('DownloadPipelineService (integration)', function () {
 
       // Step 3: Attempt to link a non-existent submission_feature_id
       try {
-        await service.createDownloadRequest(null, [999999]);
+        await service.createDownloadRequest({ systemUserId: null, teamId: null, submissionFeatureIds: [999999] });
         expect.fail('Should have thrown a foreign key violation');
       } catch (error) {
         // Expected: FK constraint violation on download_feature.submission_feature_id
@@ -116,7 +120,11 @@ describe('DownloadPipelineService (integration)', function () {
       // Step 1: Create a download
       const submissionId = await createTestSubmission(connection);
       const featureId = await createTestFeature(connection, submissionId, 'dataset', { name: 'Test' });
-      const { download_id } = await service.createDownloadRequest(null, [featureId]);
+      const { download_id } = await service.createDownloadRequest({
+        systemUserId: null,
+        teamId: null,
+        submissionFeatureIds: [featureId]
+      });
 
       // Step 2: Transition to processing
       await service.updateDownloadStatus(download_id, DownloadStatusEnum.PROCESSING);
@@ -143,6 +151,7 @@ describe('DownloadPipelineService (integration)', function () {
   describe('full status lifecycle', () => {
     it('should transition pending → processing → ready and track all timestamps', async () => {
       // Step 1: Create a download with features
+      const apiUserId = connection.systemUserId();
       const submissionId = await createTestSubmission(connection);
       const featureId1 = await createTestFeature(connection, submissionId, 'species_observation', {
         taxon_id: 180703,
@@ -154,7 +163,11 @@ describe('DownloadPipelineService (integration)', function () {
         count: 12,
         timestamp: '2024-01-16T14:30:00Z'
       });
-      const { download_id } = await service.createDownloadRequest(null, [featureId1, featureId2]);
+      const { download_id } = await service.createDownloadRequest({
+        systemUserId: apiUserId,
+        teamId: null,
+        submissionFeatureIds: [featureId1, featureId2]
+      });
 
       // Step 2: Verify initial state
       const initial = await crudService.findDownloadById(download_id);
@@ -191,7 +204,11 @@ describe('DownloadPipelineService (integration)', function () {
       const submissionId = await createTestSubmission(connection);
       const featureId = await createTestFeature(connection, submissionId, 'dataset', { name: 'Size Test' });
 
-      const { download_id } = await service.createDownloadRequest(null, [featureId]);
+      const { download_id } = await service.createDownloadRequest({
+        systemUserId: null,
+        teamId: null,
+        submissionFeatureIds: [featureId]
+      });
 
       const sizeData = await crudService.getDownloadFeatureSummaries(download_id, null);
 
@@ -209,7 +226,11 @@ describe('DownloadPipelineService (integration)', function () {
       const securedFeatureId = await createTestFeature(connection, submissionId, 'dataset', { name: 'Secured' });
       await secureFeature(securedFeatureId);
 
-      const { download_id } = await service.createDownloadRequest(null, [openFeatureId, securedFeatureId]);
+      const { download_id } = await service.createDownloadRequest({
+        systemUserId: null,
+        teamId: null,
+        submissionFeatureIds: [openFeatureId, securedFeatureId]
+      });
 
       const sizeData = await crudService.getDownloadFeatureSummaries(download_id, null);
 
@@ -235,7 +256,11 @@ describe('DownloadPipelineService (integration)', function () {
       );
 
       // Step 2: Create download and plan fragments
-      const { download_id } = await service.createDownloadRequest(null, [childFeatureId]);
+      const { download_id } = await service.createDownloadRequest({
+        systemUserId: null,
+        teamId: null,
+        submissionFeatureIds: [childFeatureId]
+      });
       const sizeEstimate = await service.estimateDownloadSize(download_id, null);
       await service.planFragments(download_id, sizeEstimate);
 
@@ -275,7 +300,11 @@ describe('DownloadPipelineService (integration)', function () {
       });
 
       // Step 2: Create download and plan fragments
-      const { download_id } = await service.createDownloadRequest(null, [rootFeatureId]);
+      const { download_id } = await service.createDownloadRequest({
+        systemUserId: null,
+        teamId: null,
+        submissionFeatureIds: [rootFeatureId]
+      });
       const sizeEstimate = await service.estimateDownloadSize(download_id, null);
       await service.planFragments(download_id, sizeEstimate);
 
@@ -386,7 +415,11 @@ describe('DownloadPipelineService (integration)', function () {
   async function createAnonymousDownload(): Promise<string> {
     const submissionId = await createTestSubmission(connection);
     const featureId = await createTestFeature(connection, submissionId, 'dataset', { name: 'Anon' });
-    const { download_id } = await service.createDownloadRequest(null, [featureId]);
+    const { download_id } = await service.createDownloadRequest({
+      systemUserId: null,
+      teamId: null,
+      submissionFeatureIds: [featureId]
+    });
     // createDownloadRequest sets system_user_id from connection.systemUserId(),
     // but the API user connection returns a real user id. Clear it for anonymous:
     await connection.sql(SQL`
@@ -438,7 +471,12 @@ describe('DownloadPipelineService (integration)', function () {
       // Create a download with team_id set
       const submissionId = await createTestSubmission(connection);
       const featureId = await createTestFeature(connection, submissionId, 'dataset', { name: 'Secured' });
-      const { download_id } = await service.createDownloadRequest(teamId, [featureId], dataRequestId);
+      const { download_id } = await service.createDownloadRequest({
+        systemUserId: null,
+        teamId,
+        submissionFeatureIds: [featureId],
+        dataRequestId
+      });
 
       // Clear system_user_id but keep team_id
       await connection.sql(SQL`
@@ -459,20 +497,27 @@ describe('DownloadPipelineService (integration)', function () {
 
   describe('getAuthorizedDownload', () => {
     it('should authorize owner (system_user_id path)', async () => {
+      const apiUserId = connection.systemUserId();
       const submissionId = await createTestSubmission(connection);
       const featureId = await createTestFeature(connection, submissionId, 'dataset', { name: 'Owned' });
-      const { download_id } = await service.createDownloadRequest(null, [featureId]);
-
-      // API user is the owner (createDownloadRequest sets system_user_id)
-      const apiUserId = connection.systemUserId();
+      const { download_id } = await service.createDownloadRequest({
+        systemUserId: apiUserId,
+        teamId: null,
+        submissionFeatureIds: [featureId]
+      });
       await service.getAuthorizedDownload(download_id, apiUserId);
       // No error thrown — authorized
     });
 
     it('should throw HTTP403 for wrong user (not owner, not shared, not team member)', async () => {
+      const apiUserId = connection.systemUserId();
       const submissionId = await createTestSubmission(connection);
       const featureId = await createTestFeature(connection, submissionId, 'dataset', { name: 'Owned' });
-      const { download_id } = await service.createDownloadRequest(null, [featureId]);
+      const { download_id } = await service.createDownloadRequest({
+        systemUserId: apiUserId,
+        teamId: null,
+        submissionFeatureIds: [featureId]
+      });
 
       const otherUserId = await createOtherUser();
       try {
@@ -486,7 +531,11 @@ describe('DownloadPipelineService (integration)', function () {
     it('should authorize via download_share (shared path)', async () => {
       const submissionId = await createTestSubmission(connection);
       const featureId = await createTestFeature(connection, submissionId, 'dataset', { name: 'Shared' });
-      const { download_id } = await service.createDownloadRequest(null, [featureId]);
+      const { download_id } = await service.createDownloadRequest({
+        systemUserId: null,
+        teamId: null,
+        submissionFeatureIds: [featureId]
+      });
 
       // Share with another user
       const otherUserId = await createOtherUser();
@@ -505,7 +554,12 @@ describe('DownloadPipelineService (integration)', function () {
       // Create download linked to data request
       const submissionId = await createTestSubmission(connection);
       const featureId = await createTestFeature(connection, submissionId, 'dataset', { name: 'Request' });
-      const { download_id } = await service.createDownloadRequest(teamId, [featureId], dataRequestId);
+      const { download_id } = await service.createDownloadRequest({
+        systemUserId: null,
+        teamId,
+        submissionFeatureIds: [featureId],
+        dataRequestId
+      });
 
       await service.getAuthorizedDownload(download_id, otherUserId);
       // No error thrown — authorized
@@ -520,7 +574,12 @@ describe('DownloadPipelineService (integration)', function () {
 
       const submissionId = await createTestSubmission(connection);
       const featureId = await createTestFeature(connection, submissionId, 'dataset', { name: 'Locked' });
-      const { download_id } = await service.createDownloadRequest(teamId, [featureId], dataRequestId);
+      const { download_id } = await service.createDownloadRequest({
+        systemUserId: null,
+        teamId,
+        submissionFeatureIds: [featureId],
+        dataRequestId
+      });
 
       try {
         await service.getAuthorizedDownload(download_id, outsider);
@@ -535,11 +594,14 @@ describe('DownloadPipelineService (integration)', function () {
 
   describe('getDownloadsByTeamMembership', () => {
     it('should return owned downloads', async () => {
+      const apiUserId = connection.systemUserId();
       const submissionId = await createTestSubmission(connection);
       const featureId = await createTestFeature(connection, submissionId, 'dataset', { name: 'Mine' });
-      const { download_id } = await service.createDownloadRequest(null, [featureId]);
-
-      const apiUserId = connection.systemUserId();
+      const { download_id } = await service.createDownloadRequest({
+        systemUserId: apiUserId,
+        teamId: null,
+        submissionFeatureIds: [featureId]
+      });
       const downloads = await crudService.getDownloadsByTeamMembership(apiUserId);
       const ids = downloads.map((d) => d.download_id);
       expect(ids).to.include(download_id);
@@ -548,7 +610,11 @@ describe('DownloadPipelineService (integration)', function () {
     it('should return shared downloads', async () => {
       const submissionId = await createTestSubmission(connection);
       const featureId = await createTestFeature(connection, submissionId, 'dataset', { name: 'Shared List' });
-      const { download_id } = await service.createDownloadRequest(null, [featureId]);
+      const { download_id } = await service.createDownloadRequest({
+        systemUserId: null,
+        teamId: null,
+        submissionFeatureIds: [featureId]
+      });
 
       const otherUserId = await createOtherUser();
       await shareDownload(download_id, otherUserId);
@@ -566,7 +632,12 @@ describe('DownloadPipelineService (integration)', function () {
 
       const submissionId = await createTestSubmission(connection);
       const featureId = await createTestFeature(connection, submissionId, 'dataset', { name: 'Team DL' });
-      const { download_id } = await service.createDownloadRequest(teamId, [featureId], dataRequestId);
+      const { download_id } = await service.createDownloadRequest({
+        systemUserId: null,
+        teamId,
+        submissionFeatureIds: [featureId],
+        dataRequestId
+      });
 
       const downloads = await crudService.getDownloadsByTeamMembership(otherUserId);
       const ids = downloads.map((d) => d.download_id);
@@ -576,7 +647,11 @@ describe('DownloadPipelineService (integration)', function () {
     it('should not return downloads the user has no access to', async () => {
       const submissionId = await createTestSubmission(connection);
       const featureId = await createTestFeature(connection, submissionId, 'dataset', { name: 'Private' });
-      const { download_id } = await service.createDownloadRequest(null, [featureId]);
+      const { download_id } = await service.createDownloadRequest({
+        systemUserId: null,
+        teamId: null,
+        submissionFeatureIds: [featureId]
+      });
 
       const otherUserId = await createOtherUser();
       const downloads = await crudService.getDownloadsByTeamMembership(otherUserId);

api/src/__integration__/system/download-system.integration.ts

@@ -544,7 +544,11 @@ describe('DownloadPipelineService download pipeline (system)', function () {
    * Helper: run the full download pipeline and return the resulting zip.
    */
   async function executeAndGetZip(featureIds: number[]): Promise<{ zip: AdmZip; downloadId: string }> {
-    const { download_id } = await service.createDownloadRequest(null, featureIds);
+    const { download_id } = await service.createDownloadRequest({
+      systemUserId: null,
+      teamId: null,
+      submissionFeatureIds: featureIds
+    });
 
     // Run the three-phase pipeline within the test transaction
     await service.planDownloadIfNeeded(download_id);

api/src/models/download.ts

@@ -1,4 +1,5 @@
 import { z } from 'zod';
+import { SearchFeatureFiltersSchema } from '../services/search-feature-service.interface';
 import { DownloadStatusZod } from './download-status';
 
 export const DownloadRecord = z.object({
@@ -53,3 +54,35 @@ export interface DownloadSizeEstimate {
   /** The features included in this download with per-feature estimated_byte_size. */
   features: DownloadFeatureSummary[];
 }
+
+/**
+ * Payload for creating a new download record.
+ *
+ * Bundles the identifiers and optional configuration that flow from the
+ * pipeline service through to the repository INSERT.
+ */
+export const CreateDownload = z.object({
+  teamId: z.string().nullable(),
+  dataRequestId: z.string().nullable(),
+  fragmentSizeBytes: z.number().optional(),
+  systemUserId: z.number().nullable().optional(),
+  filters: SearchFeatureFiltersSchema.optional()
+});
+export type CreateDownload = z.infer<typeof CreateDownload>;
+
+/**
+ * Payload for creating a download request through the pipeline.
+ *
+ * Accepted by the pipeline service's public entry point. Includes the feature
+ * selection and optional fragment sizing before the request is persisted and
+ * features are linked.
+ */
+export const CreateDownloadRequest = z.object({
+  systemUserId: z.number().nullable(),
+  teamId: z.string().nullable(),
+  submissionFeatureIds: z.array(z.number()),
+  dataRequestId: z.string().optional(),
+  fragmentSizeMb: z.number().optional(),
+  filters: SearchFeatureFiltersSchema.optional()
+});
+export type CreateDownloadRequest = z.infer<typeof CreateDownloadRequest>;