Basic implementation of apollo export gff3 (#511)

* Basic implementation of `apollo export gff3`

To be fixed?: Output gff3 includes fasta sequence *only* if the assembly is
editable (see tests).

* Handle invalid assembly

* Optional export of fasta in `apollo export gff3`

For UI, always include fasta when downloading gff3

* Export fasta by streaing original file

Also re-enable cypress test to download annotation

* Fix lint

* Fix import

* Include comments here #511 (review)

Use includeFASTA instead of withFasta and --include-fasta instead of
--with-fasta in cli

* Tiny docs fix


skip-checks:true

---------

Co-authored-by: Garrett Stevens <stevens.garrett.j@gmail.com>

Author: dariober

.github/workflows/pull_request.yml

@@ -42,7 +42,7 @@ jobs:
           yarn --cwd packages/apollo-shared start &
           ALLOW_ROOT_USER=true ROOT_USER_PASSWORD=pass yarn --cwd packages/apollo-collaboration-server start &
       - name: Run CLI tests
-        run: yarn tsx src/test/test.ts
+        run: yarn test:cli
         working-directory: packages/apollo-cli
       # - name: Run docker tests
       #   working-directory: packages/apollo-cli

packages/apollo-cli/README.md

@@ -37,6 +37,7 @@ USAGE
 - [`apollo assembly sequence`](#apollo-assembly-sequence)
 - [`apollo change get`](#apollo-change-get)
 - [`apollo config [KEY] [VALUE]`](#apollo-config-key-value)
+- [`apollo export gff3 ASSEMBLY`](#apollo-export-gff3-assembly)
 - [`apollo feature add-child`](#apollo-feature-add-child)
 - [`apollo feature check`](#apollo-feature-check)
 - [`apollo feature copy`](#apollo-feature-copy)
@@ -354,6 +355,34 @@ EXAMPLES
 _See code:
 [src/commands/config.ts](https://github.com/GMOD/Apollo3/blob/v0.3.1/packages/apollo-cli/src/commands/config.ts)_
 
+## `apollo export gff3 ASSEMBLY`
+
+Export the annotations for an assembly to stdout as gff3
+
+```
+USAGE
+  $ apollo export gff3 ASSEMBLY [--profile <value>] [--config-file <value>] [--include-fasta]
+
+ARGUMENTS
+  ASSEMBLY  Export annotations for this assembly name or id
+
+FLAGS
+  --config-file=<value>  Use this config file (mostly for testing)
+  --include-fasta        Include fasta sequence in output
+  --profile=<value>      Use credentials from this profile
+
+DESCRIPTION
+  Export the annotations for an assembly to stdout as gff3
+
+EXAMPLES
+  Export annotations for myAssembly:
+
+    $ apollo export gff3 myAssembly > out.gff3
+```
+
+_See code:
+[src/commands/export/gff3.ts](https://github.com/GMOD/Apollo3/blob/v0.3.1/packages/apollo-cli/src/commands/export/gff3.ts)_
+
 ## `apollo feature add-child`
 
 Add a child feature (e.g. add an exon to an mRNA)

packages/apollo-cli/package.json

@@ -29,6 +29,7 @@
     "test": "mocha --require src/test/fixtures.ts 'src/**/*.test.ts'",
     "posttest": "yarn lint",
     "test:ci": "nyc mocha 'src/**/*.test.ts'",
+    "test:cli": "yarn tsx src/test/test.ts",
     "version": "oclif readme --multi --dir ../website/docs/admin/cli/ && oclif readme && git add README.md"
   },
   "oclif": {
@@ -51,6 +52,9 @@
       "change": {
         "description": "Commands to manage the change log"
       },
+      "export": {
+        "description": "Commands to export data"
+      },
       "feature": {
         "description": "Commands to manage features"
       },

packages/apollo-cli/src/commands/export/gff3.ts

@@ -0,0 +1,103 @@
+import { Args, Flags } from '@oclif/core'
+import { fetch } from 'undici'
+
+import { BaseCommand } from '../../baseCommand.js'
+import {
+  convertAssemblyNameToId,
+  createFetchErrorMessage,
+  idReader,
+  localhostToAddress,
+} from '../../utils.js'
+
+import { Readable } from 'node:stream'
+
+export default class Get extends BaseCommand<typeof Get> {
+  static description =
+    'Export the annotations for an assembly to stdout as gff3'
+
+  static examples = [
+    {
+      description: 'Export annotations for myAssembly:',
+      command: '<%= config.bin %> <%= command.id %> myAssembly > out.gff3',
+    },
+  ]
+
+  static args = {
+    assembly: Args.string({
+      description: 'Export annotations for this assembly name or id',
+      required: true,
+    }),
+  }
+
+  static flags = {
+    'include-fasta': Flags.boolean({
+      description: 'Include fasta sequence in output',
+      default: false,
+    }),
+  }
+
+  public async run(): Promise<void> {
+    const { args } = await this.parse(Get)
+
+    const access = await this.getAccess()
+
+    const assembly = await idReader([args.assembly])
+    const [assemblyId] = await convertAssemblyNameToId(
+      access.address,
+      access.accessToken,
+      assembly,
+    )
+    if (!assemblyId) {
+      this.error(`Invalid assembly name or id: ${args.assembly}`)
+    }
+
+    const url = new URL(localhostToAddress(`${access.address}/export/getID`))
+    const searchParams = new URLSearchParams({
+      assembly: assemblyId,
+    })
+    url.search = searchParams.toString()
+    const uri = url.toString()
+    const auth = {
+      headers: {
+        authorization: `Bearer ${access.accessToken}`,
+      },
+    }
+    const response = await fetch(uri, auth)
+    if (!response.ok) {
+      const newErrorMessage = await createFetchErrorMessage(
+        response,
+        'Error when exporting ID',
+      )
+      throw new Error(newErrorMessage)
+    }
+
+    const { exportID } = (await response.json()) as { exportID: string }
+
+    const exportURL = new URL(localhostToAddress(`${access.address}/export`))
+
+    const params: Record<string, string> = {
+      exportID,
+      assemblyId,
+      includeFASTA: this.flags['include-fasta'] ? 'true' : 'false',
+    }
+    const exportSearchParams = new URLSearchParams(params)
+    exportURL.search = exportSearchParams.toString()
+    const exportUri = exportURL.toString()
+
+    const responseExport = await fetch(exportUri, auth)
+    if (!responseExport.ok) {
+      const newErrorMessage = await createFetchErrorMessage(
+        responseExport,
+        'Error when exporting gff',
+      )
+      throw new Error(newErrorMessage)
+    }
+    const { body } = responseExport
+    if (body) {
+      const readable = Readable.from(body)
+      readable.pipe(process.stdout)
+    } else {
+      this.error('Failed to export gff3')
+    }
+  }
+}

packages/apollo-cli/src/test/test.ts

@@ -9,11 +9,11 @@
  * USAGE
  * From package root directory (`packages/apollo-cli`). Run all tests:
  *
- * yarn tsx src/test/test.ts
+ * yarn test:cli
  *
  * Run only matching pattern:
  *
- * yarn tsx --test-name-pattern='Print help|Feature get' src/test/test.ts
+ * yarn test:cli --test-name-pattern='Print help|Feature get'
  */
 
 import assert from 'node:assert'
@@ -1388,4 +1388,68 @@ void describe('Test CLI', () => {
     out = JSON.parse(p.stdout)
     assert.strictEqual(out.length, 0)
   })
+
+  void globalThis.itName('Export gff3 from editable assembly', () => {
+    new Shell(
+      `${apollo} assembly add-from-fasta ${P} test_data/tiny.fasta.gz -a vv1 -f --editable`,
+    )
+    new Shell(`${apollo} feature import ${P} test_data/tiny.fasta.gff3 -a vv1`)
+    let p = new Shell(`${apollo} export gff3 ${P} vv1 --include-fasta`)
+    let gff = p.stdout
+    assert.ok(gff.startsWith('##gff-version 3'))
+    assert.ok(gff.includes('multivalue=val1,val2,val3'))
+    assert.ok(gff.includes('##FASTA\n'))
+    assert.deepStrictEqual(gff.slice(-6, gff.length), 'taccc\n')
+
+    p = new Shell(`${apollo} export gff3 ${P} vv1`)
+    gff = p.stdout
+    assert.ok(gff.startsWith('##gff-version 3'))
+    assert.ok(gff.includes('multivalue=val1,val2,val3'))
+    assert.ok(!gff.includes('##FASTA\n'))
+
+    // Invalid assembly
+    p = new Shell(`${apollo} export gff3 ${P} foobar`, false)
+    assert.ok(p.returncode != 0)
+    assert.ok(p.stderr.includes('foobar'))
+  })
+
+  void globalThis.itName('Export gff3 from non-editable assembly', () => {
+    new Shell(
+      `${apollo} assembly add-from-fasta ${P} test_data/tiny.fasta.gz -a vv1 -f`,
+    )
+    new Shell(`${apollo} feature import ${P} test_data/tiny.fasta.gff3 -a vv1`)
+    let p = new Shell(`${apollo} export gff3 ${P} vv1 --include-fasta`)
+    let gff = p.stdout
+    assert.ok(gff.startsWith('##gff-version 3'))
+    assert.ok(gff.includes('multivalue=val1,val2,val3'))
+    assert.ok(gff.includes('##FASTA\n'))
+    // We end with two newlines because the test data does have an extra newline at the end.
+    assert.deepStrictEqual(gff.slice(-7, gff.length), 'taccc\n\n')
+
+    p = new Shell(`${apollo} export gff3 ${P} vv1`)
+    gff = p.stdout
+    assert.ok(gff.startsWith('##gff-version 3'))
+    assert.ok(gff.includes('multivalue=val1,val2,val3'))
+    assert.ok(!gff.includes('##FASTA\n'))
+  })
+
+  void globalThis.itName('Export gff3 from external assembly', () => {
+    new Shell(
+      `${apollo} assembly add-from-fasta ${P} https://raw.githubusercontent.com/GMOD/Apollo3/refs/heads/main/packages/apollo-cli/test_data/tiny.fasta.gz -a vv1 -f`,
+    )
+    new Shell(`${apollo} feature import ${P} test_data/tiny.fasta.gff3 -a vv1`)
+    let p = new Shell(`${apollo} export gff3 ${P} vv1 --include-fasta`)
+    let gff = p.stdout
+    assert.ok(gff.startsWith('##gff-version 3'))
+    assert.ok(gff.includes('multivalue=val1,val2,val3'))
+    assert.ok(gff.includes('##FASTA\n'))
+    // We end with two newlines because the test data does have an extra newline at the end.
+    assert.deepStrictEqual(gff.slice(-7, gff.length), 'taccc\n\n')
+
+    p = new Shell(`${apollo} export gff3 ${P} vv1`)
+    gff = p.stdout
+    assert.ok(gff.startsWith('##gff-version 3'))
+    assert.ok(gff.includes('multivalue=val1,val2,val3'))
+    assert.ok(!gff.includes('##FASTA\n'))
+  })
 })

packages/apollo-collaboration-server/src/export/export.controller.ts

@@ -1,8 +1,11 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 import {
   Controller,
+  DefaultValuePipe,
   Get,
   Logger,
+  ParseBoolPipe,
+  ParseIntPipe,
   Query,
   Response,
   StreamableFile,
@@ -41,12 +44,17 @@ export class ExportController {
   @Validations(Role.None)
   @Get()
   async exportGFF3(
-    @Query() request: { exportID: string; fastaWidth?: number },
+    // @Query()
+    @Query('exportID') exportID: string,
+    @Query('includeFASTA', new DefaultValuePipe(false), ParseBoolPipe)
+    includeFASTA: boolean,
+    @Query('fastaWidth', new DefaultValuePipe(80), ParseIntPipe)
+    fastaWidth: number,
     @Response({ passthrough: true }) res: ExpressResponse,
   ) {
-    const { exportID, ...rest } = request
     const [stream, assembly] = await this.exportService.exportGFF3(exportID, {
-      ...rest,
+      includeFASTA,
+      fastaWidth,
     })
     const assemblyName = await this.exportService.getAssemblyName(assembly)
     res.set({

packages/apollo-collaboration-server/src/export/export.module.ts

@@ -8,11 +8,13 @@ import { RefSeqChunksModule } from '../refSeqChunks/refSeqChunks.module'
 import { RefSeqsModule } from '../refSeqs/refSeqs.module'
 import { ExportController } from './export.controller'
 import { ExportService } from './export.service'
+import { FilesModule } from '../files/files.module'
 
 @Module({
   imports: [
     AssembliesModule,
     FeaturesModule,
+    FilesModule,
     MongooseModule.forFeature([{ name: Export.name, schema: ExportSchema }]),
     RefSeqsModule,
     RefSeqChunksModule,