New classifiers with scicrunch RRIDs (⚠️ devops) (#2045)

* Create submodule with logic to communicate with scicrunch.org

* Added pg tables for scicrunch resources

* split submodule for scicrunch

* Extends groups api in webserver

* Added handlers in groups (w/o implementation) and added scicrunch setup

* Adds tests fixture

* Improves OAS descriptions

* Deduces links from settings and cleanup

* Defined classifiers tree with Odei

* Adds migration script for scircunrch resources

* Adds column in group_classifiers to use scicrunch alternative

* updates config

* Adds tests for actual scicrunch service. Intended for manual exploratory testing of the actual service API

* pytest-simcore: adds fixture to mock .env-devel

* Implements and cleanup group/*/classifiers handlers

* Fixes conflicting migration steps after master rebase

* Fixes environ key name

* Renamed column to uses_scicrunch

* Fixes while manual testing

* Moved scicrunch into a folder

* reviews get_group_classifiers

* renames scicrunch_api -> scicrunch.service_client

* Removed testing in this PR (next!)

Author: pcrespov

.env-devel

@@ -37,6 +37,9 @@ S3_ENDPOINT=172.17.0.1:9001
 S3_SECRET_KEY=12345678
 S3_SECURE=0
 
+SCICRUNCH_API_BASE_URL=https://scicrunch.org/api/1
+SCICRUNCH_API_KEY=REPLACE_ME_with_valid_api_key
+
 SMTP_HOST=mail.speag.com
 SMTP_PORT=25
 

api/specs/webserver/openapi-groups.yaml

@@ -211,23 +211,81 @@ paths:
           $ref: "#/components/responses/DefaultErrorResponse"
 
   /groups/{gid}/classifiers:
+    get:
+      parameters:
+        - name: gid
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: tree_view
+          in: query
+          description: "Some classifiers (e.g. sparc's) offer different tree views"
+          schema:
+            type: string
+            enum: [std, sort]
+            default: std
+      tags:
+        - group
+      summary: Gets classifiers bundle for this group
+      operationId: get_group_classifiers
+      responses:
+        "200":
+          description: got a bundle with all information about classifiers
+        default:
+          $ref: "#/components/responses/DefaultErrorResponse"
+
+  /groups/sparc/classifiers/scicrunch-resources/{rrid}:
     parameters:
-      - name: gid
+      - name: rrid
         in: path
         required: true
         schema:
           type: string
     get:
       tags:
         - group
-      summary: Gets classifiers bundle for this group
-      operationId: get_group_classifiers
+      summary: "Returns information on a valid RRID (https://www.force11.org/group/resource-identification-initiative)"
+      operationId: get_scicrunch_resource
       responses:
         "200":
-          description: got a bundle with all information about classifiers
+          description: Got information of a valid RRID
+        "404":
+          description: Cannot find a valid research resource for the provided RRID
+        "503":
+          description: Underline validation service is not reachable
+        default:
+          $ref: "#/components/responses/DefaultErrorResponse"
+    post:
+      tags:
+        - group
+      summary: "Adds new RRID to classifiers"
+      operationId: add_scicrunch_resource
+      responses:
+        "200":
+          description: Got information of a valid RRID
+        "422":
+          description: Invalid RRID
         default:
           $ref: "#/components/responses/DefaultErrorResponse"
 
+  /groups/sparc/classifiers/scicrunch-resources:search:
+     get:
+      parameters:
+        - name: guess_name
+          in: query
+          required: true
+          schema:
+            type: string
+      tags:
+        - group
+      summary: "Returns a list of related resource provided a search name"
+      operationId: search_scicrunch_resources
+      responses:
+        "200":
+          description: Got information of a valid RRID
+        default:
+          $ref: "#/components/responses/DefaultErrorResponse"
 components:
   responses:
     DefaultErrorResponse:

api/specs/webserver/openapi.yaml

@@ -119,6 +119,13 @@ paths:
   /groups/{gid}/classifiers:
     $ref: "./openapi-groups.yaml#/paths/~1groups~1{gid}~1classifiers"
 
+  /groups/sparc/classifiers/scicrunch-resources/{rrid}:
+    $ref: "./openapi-groups.yaml#/paths/~1groups~1sparc~1classifiers~1scicrunch-resources~1{rrid}"
+
+  /groups/sparc/classifiers/scicrunch-resources:search:
+    $ref: "./openapi-groups.yaml#/paths/~1groups~1sparc~1classifiers~1scicrunch-resources:search"
+
+
   # DATA STORAGE SERVICES ----------------------------------------------------------
 
   /storage/locations:

packages/postgres-database/Makefile

@@ -11,7 +11,7 @@ requirements: ## compiles pip requirements (.in -> .txt)
 
 
 .PHONY: install-dev install-prod install-ci
-install-dev install-prod install-ci: requirements _check_venv_active ## install app in development/production or CI mode
+install-dev install-prod install-ci: _check_venv_active ## install app in development/production or CI mode
 	# installing in $(subst install-,,$@) mode
 	pip-sync requirements/$(subst install-,,$@).txt
 
@@ -67,6 +67,7 @@ setup-prod: guard-POSTGRES_DATA_VOLUME install-dev up-prod ## sets up a database
 
 .PHONY: setup-commit
 setup-commit: install-dev up-pg ## sets up a database to create a new commit into migration history
+	sleep 2
 	# discovering
 	sc-pg --help
 	sc-pg discover -u test -p test -d test

packages/postgres-database/doc/img/postgres-database-models.svg

@@ -0,0 +1,45 @@
+"""Adds table for scicrunch rrids
+
+Revision ID: 39fa67f45cc0
+Revises: 3452ca7b13e9
+Create Date: 2020-12-15 18:16:03.581479+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "39fa67f45cc0"
+down_revision = "3452ca7b13e9"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "scicrunch_resources",
+        sa.Column("rrid", sa.String(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("description", sa.String(), nullable=True),
+        sa.Column(
+            "creation_date",
+            sa.DateTime(),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "last_change_date",
+            sa.DateTime(),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("rrid"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("scicrunch_resources")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/migration/versions/39fa67f45cc0_adds_table_for_scicrunch_rrids.py

@@ -0,0 +1,37 @@
+"""Adds column to use scicrunch alternative
+
+Revision ID: b60363fe438f
+Revises: 39fa67f45cc0
+Create Date: 2020-12-15 18:26:25.552123+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "b60363fe438f"
+down_revision = "39fa67f45cc0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "group_classifiers",
+        sa.Column("uses_scicrunch", sa.Boolean(), nullable=False, default=False),
+    )
+    # ### end Alembic commands ###
+
+    # Applies the default to all
+    query = 'UPDATE "group_classifiers" SET uses_scicrunch=false;'
+    op.execute(query)
+    # makes non nullable
+    query = 'ALTER TABLE "group_classifiers" ALTER "uses_scicrunch" SET NOT NULL;'
+    op.execute(query)
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("group_classifiers", "uses_scicrunch")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/migration/versions/b60363fe438f_adds_column_to_use_scicrunch_alternative.py

@@ -17,7 +17,6 @@
     "group_classifiers",
     metadata,
     sa.Column("id", sa.BigInteger, nullable=False),
-    # TODO: add jsonschema validtion
     sa.Column("bundle", JSONB, nullable=False),
     sa.Column("created", sa.DateTime(), nullable=False, server_default=func.now()),
     sa.Column(
@@ -38,5 +37,7 @@
         ),
         unique=True,  # Every Group can ONLY have one set of classifiers
     ),
+    # uses scicrunch service to acccess curated classifiers instead of static bundle
+    sa.Column("uses_scicrunch", sa.Boolean, nullable=False, default=False),
     sa.PrimaryKeyConstraint("id", name="group_classifiers_pkey"),
 )

packages/postgres-database/src/simcore_postgres_database/models/classifiers.py

@@ -0,0 +1,25 @@
+""" Stores SOME of the information associated to Research Resource Identifiers (RRIDs) as defined in https://scicrunch.org/resources
+"""
+
+import sqlalchemy as sa
+from sqlalchemy.sql import func
+
+from .base import metadata
+
+scicrunch_resources = sa.Table(
+    "scicrunch_resources",
+    metadata,
+    sa.Column("rrid", sa.String, nullable=False, primary_key=True),
+    sa.Column("name", sa.String, nullable=False),
+    sa.Column("description", sa.String, nullable=True),
+    sa.Column(
+        "creation_date", sa.DateTime(), nullable=False, server_default=func.now()
+    ),
+    sa.Column(
+        "last_change_date",
+        sa.DateTime(),
+        nullable=False,
+        server_default=func.now(),
+        onupdate=func.now(),  # this will auto-update on modification
+    ),
+)

packages/postgres-database/src/simcore_postgres_database/models/scicrunch_resources.py

@@ -6,12 +6,13 @@
 """
 from .models.api_keys import api_keys
 from .models.classifiers import group_classifiers
-from .models.comp_pipeline import comp_pipeline, StateType
-from .models.comp_tasks import DB_CHANNEL_NAME, comp_tasks, NodeClass
+from .models.comp_pipeline import StateType, comp_pipeline
+from .models.comp_tasks import DB_CHANNEL_NAME, NodeClass, comp_tasks
 from .models.confirmations import ConfirmationAction, confirmations
 from .models.groups import GroupType, groups, user_to_groups
 from .models.products import products
 from .models.projects import ProjectType, projects
+from .models.scicrunch_resources import scicrunch_resources
 from .models.tags import study_tags, tags
 from .models.tokens import tokens
 from .models.users import UserRole, UserStatus, users
@@ -25,6 +26,7 @@
     "UserStatus",
     "projects",
     "ProjectType",
+    "scicrunch_resources",
     "confirmations",
     "ConfirmationAction",
     "tokens",