Merge pull request #363 from CanDIG/stable-candidate

v5.0.0: File Download, Sequencing Support

Author: OrdiNeu

.github/workflows/test.yml

@@ -14,7 +14,8 @@ jobs:
       DB_PATH: localhost
       SERVER_LOCAL_DATA: ${{github.workspace}}/data
       PGPASSWORD: OG0pjmnQDWUTvLotYrxPrg
-      INDEXING_PATH: ${{github.workspace}}/tmp
+      INDEXING_PATH: ${{github.workspace}}/tmp/indexer
+      SEARCH_PATH: ${{github.workspace}}/tmp/search
       INDEXING_SWITCH_FILE: ${{github.workspace}}/indexer_on
       TESTENV_URL: http://localhost:3000
       AGGREGATE_COUNT_THRESHOLD: 5
@@ -52,12 +53,15 @@ jobs:
         pip install -r requirements.txt
         bash create_db.sh
         mkdir -p ${{ env.INDEXING_PATH }}
+        mkdir -p ${{ env.SEARCH_PATH }}/results
+        mkdir -p ${{ env.SEARCH_PATH }}/to_search
         touch ${{ env.INDEXING_SWITCH_FILE }}
         sed -i s@\<AGGREGATE_COUNT_THRESHOLD\>@${{env.AGGREGATE_COUNT_THRESHOLD}}@ config.ini
         sed -i s@\<POSTGRES_USERNAME\>@${{env.POSTGRES_USERNAME}}@ config.ini
     - name: Test
       run: |
         python htsget_server/server.py &
         python htsget_server/indexing.py &
+        python htsget_server/search.py &
         sleep 5
         pytest tests/test_htsget_server.py

create_db.sh

@@ -43,4 +43,5 @@ psql --quiet -h "$db" -U $PGUSER -d genomic -a -f data/pr_288.sql >>setup_out.tx
 psql --quiet -h "$db" -U $PGUSER -d genomic -a -f data/pr_315.sql >>setup_out.txt
 psql --quiet -h "$db" -U $PGUSER -d genomic -a -f data/pr_339.sql >>setup_out.txt
 psql --quiet -h "$db" -U $PGUSER -d genomic -a -f data/pr_341.sql >>setup_out.txt
+psql --quiet -h "$db" -U $PGUSER -d genomic -a -f data/pr_352.sql >>setup_out.txt
 echo "...done"

data/files.sql

@@ -17,6 +17,7 @@ CREATE TABLE drs_object (
         description VARCHAR,
         aliases VARCHAR,
         program_id VARCHAR,
+        meta_data JSONB,
         PRIMARY KEY (id),
         FOREIGN KEY(program_id) REFERENCES program (id)
 );

data/files/counts.tsv

@@ -0,0 +1,3 @@
+gene_id	transcript_id(s)	length	effective_length	expected_count	TPM	FPKM
+ENSG00000000003.15	ENST00000373020.8,ENST00000494424.1,ENST00000496771.5,ENST00000612152.4,ENST00000614008.4	2000.22	1008.6	150.00	6.24	7.32
+ENSG00000000003.16	ENST00000373020.8,ENST00000494424.1,ENST00000496771.5,ENST00000612152.4,ENST00000614008.4	2000.22	1008.6	150.00	6.24	9.32

data/files/counts2.tsv

@@ -0,0 +1,3 @@
+gene_id	abundance	counts	length	countsFromAbundance
+ENSG00000000003	41.459314	1984.82025	3312.70606925252	no
+ENSG00000000005	10.857373	13.2324342	1049.23423423242	no

data/pr_352.sql

@@ -0,0 +1,7 @@
+-- add the metadata column if it doesn't already exist
+DO
+$$
+    BEGIN
+        ALTER TABLE drs_object ADD COLUMN IF NOT EXISTS meta_data jsonb;
+    END;
+$$;

entrypoint.sh

@@ -12,7 +12,9 @@ if [[ -f "initial_setup" ]]; then
     sed -i s@\<POSTGRES_USERNAME\>@$POSTGRES_USERNAME@ config.ini
 
     bash create_db.sh
-    mkdir $INDEXING_PATH
+    mkdir -p $INDEXING_PATH
+    mkdir -p $SEARCH_PATH/results
+    mkdir -p $SEARCH_PATH/to_search
     touch $INDEXING_SWITCH_FILE
     rm initial_setup
 fi
@@ -24,6 +26,7 @@ candigv2_logging.logging.initialize()"
 #python3 htsget_server/server.py
 
 bash htsget_server/indexing.sh &
+bash htsget_server/search.sh &
 
 # use the following instead for production deployment
 cd htsget_server

htsget_server/beacon_openapi.yaml

@@ -109,6 +109,25 @@ paths:
           description: Successful operation
       tags:
         - Informational endpoints
+  /result/{queue_id}:
+    parameters:
+      - in: path
+        required: true
+        name: queue_id
+        schema:
+          type: string
+    get:
+      description: Get full Beacon result for a search
+      operationId: beacon_operations.get_full_result
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BeaconResultsetsResponseBody"
+        '404':
+          description: Not Found
   /g_variants:
     get:
       description: Search for variants
@@ -262,7 +281,7 @@ components:
             oneOf:
               - $ref: "#/components/schemas/BeaconBooleanResponseBody"
               - $ref: "#/components/schemas/BeaconCountResponseBody"
-              - $ref: "#/components/schemas/BeaconResultsetsResponseBody"
+              - $ref: "#/components/schemas/BeaconSearchResponseBody"
       description: Successful operation.
     BeaconErrorResponse:
       description: An unsuccessful operation.
@@ -325,6 +344,22 @@ components:
         - meta
         - responseSummary
       type: object
+    BeaconSearchResponseBody:
+      description: Beacon quick search result
+      properties:
+        info:
+          $ref: '#/components/schemas/Info'
+        meta:
+          $ref: '#/components/schemas/BeaconResponseMeta'
+        response:
+          $ref: '#/components/schemas/QuickSearchResult'
+        responseSummary:
+          $ref: '#/components/schemas/BeaconSummaryResponseSection'
+      required:
+        - meta
+        - responseSummary
+        - response
+      type: object
     BeaconResultsetsResponseBody:
       description: Beacon response that includes record level details, grouped in Resultsets.
       properties:
@@ -448,7 +483,7 @@ components:
       type: object
     HandoverType:
       type: object
-      description: in our context, only htsget urls should be returned
+      description: in our context, only htsget or download urls should be returned
       properties:
         id:
           type: string
@@ -458,6 +493,7 @@ components:
           type: string
           enum:
             - HTSGET
+            - DOWNLOAD
     IncludeResultsetResponses:
       default: HIT
       description: Indicator of whether responses from every Resultset should be included in the response to this request or just the ones with positive, negative results or no details at all. If null (not specified), the default value of 'HIT' is assumed. This parameter allows for returning boolean/counting results although the Beacon instance is capable to return record level details.
@@ -603,6 +639,23 @@ components:
         - resultSets
       title: Beacon ResultSet
       type: object
+    QuickSearchResult:
+      description: Result with initial estimated data and a link to full Beacon result
+      type: object
+      properties:
+        beaconResultUrl:
+          type: string
+        estimatedResults:
+          type: array
+          items:
+            type: object
+            properties:
+              analysisId:
+                type: string
+              programId:
+                type: string
+              variantCount:
+                type: integer
     BeaconCountResponseSection:
       description: Payload definition for the "count" response.
       properties:
@@ -884,6 +937,7 @@ components:
                 type: string
               genotype:
                 type: object
+                description: based on suggested modification at https://github.com/ga4gh-beacon/beacon-v2/issues/57#issuecomment-1452561660
                 properties:
                   zygosity:
                     $ref: '#/components/schemas/OntologyTerm'