Csv upload (#77)

* fix: remove console.log

* feat: export large compressed csv

* fix: expected lines

* fix: id is not 1 as db is not reset

Author: jean-michelet

package.json

@@ -51,6 +51,7 @@
     "@fastify/under-pressure": "^9.0.1",
     "@sinclair/typebox": "^0.34.11",
     "concurrently": "^9.0.1",
+    "csv-stringify": "^6.5.2",
     "fastify": "^5.0.0",
     "fastify-cli": "^7.0.0",
     "fastify-plugin": "^5.0.1",

src/routes/api/tasks/index.ts

@@ -12,8 +12,10 @@ import {
   TaskPaginationResultSchema
 } from '../../../schemas/tasks.js'
 import path from 'node:path'
-import { pipeline } from 'node:stream/promises'
 import fs from 'node:fs'
+import { pipeline } from 'node:stream/promises'
+import { createGzip } from 'node:zlib'
+import { stringify } from 'csv-stringify'
 
 const plugin: FastifyPluginAsyncTypebox = async (fastify) => {
   fastify.get(
@@ -355,6 +357,35 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => {
       })
     }
   )
+
+  fastify.get(
+    '/download/csv',
+    {
+      schema: {
+        response: {
+          200: { type: 'string', contentMediaType: 'application/gzip' },
+          400: Type.Object({ message: Type.String() })
+        },
+        tags: ['Tasks']
+      }
+    },
+    async function (request, reply) {
+      const queryStream = fastify.knex.select('*').from('tasks').stream()
+
+      const csvTransform = stringify({
+        header: true,
+        columns: undefined
+      })
+
+      reply.header('Content-Type', 'application/gzip')
+      reply.header(
+        'Content-Disposition',
+      `attachment; filename="${encodeURIComponent('tasks.csv.gz')}"`
+      )
+
+      return queryStream.pipe(csvTransform).pipe(createGzip())
+    }
+  )
 }
 
 function isErrnoException (error: unknown): error is NodeJS.ErrnoException {

test/routes/api/tasks/tasks.test.ts

@@ -13,6 +13,7 @@ import { pipeline } from 'node:stream/promises'
 import path from 'node:path'
 import FormData from 'form-data'
 import os from 'os'
+import { gunzipSync } from 'node:zlib'
 
 async function createUser (
   app: FastifyInstance,
@@ -28,8 +29,16 @@ async function createTask (app: FastifyInstance, taskData: Partial<Task>) {
   return id
 }
 
-async function uploadImageForTask (app: FastifyInstance, taskId: number, filePath: string, uploadDir: string) {
-  await app.knex<Task>('tasks').where({ id: taskId }).update({ filename: `${taskId}_short-logo.png` })
+async function uploadImageForTask (
+  app: FastifyInstance,
+  taskId: number,
+  filePath: string,
+  uploadDir: string
+) {
+  await app
+    .knex<Task>('tasks')
+    .where({ id: taskId })
+    .update({ filename: `${taskId}_short-logo.png` })
 
   const file = fs.createReadStream(filePath)
   const filename = `${taskId}_short-logo.png`
@@ -803,4 +812,48 @@ describe('Tasks api (logged user only)', () => {
       })
     })
   })
+
+  describe('GET /api/tasks/download/csv', () => {
+    before(async () => {
+      const app = await build()
+      await app.knex('tasks').del()
+      await app.close()
+    })
+
+    it('should stream a gzipped CSV file', async (t) => {
+      const app = await build(t)
+
+      const tasks = []
+      for (let i = 0; i < 1000; i++) {
+        tasks.push({
+          name: `Task ${i + 1}`,
+          author_id: 1,
+          assigned_user_id: 2,
+          filename: 'task.png',
+          status: TaskStatusEnum.InProgress
+        })
+      }
+
+      await app.knex('tasks').insert(tasks)
+
+      const res = await app.injectWithLogin('basic', {
+        method: 'GET',
+        url: '/api/tasks/download/csv'
+      })
+
+      assert.strictEqual(res.statusCode, 200)
+      assert.strictEqual(res.headers['content-type'], 'application/gzip')
+      assert.strictEqual(
+        res.headers['content-disposition'],
+        'attachment; filename="tasks.csv.gz"'
+      )
+
+      const decompressed = gunzipSync(res.rawPayload).toString('utf-8')
+      const lines = decompressed.split('\n')
+      assert.equal(lines.length - 1, 1001)
+
+      assert.ok(lines[1].includes('Task 1,1,2,task.png,in-progress'))
+      assert.equal(lines[0], 'id,name,author_id,assigned_user_id,filename,status,created_at,updated_at')
+    })
+  })
 })