Merge pull request #6 from deepesdl/tejas-xxx-align-with-latest-schema

align all the records generated with latest schema

Author: TejasMorbagal

.github/workflows/unittest-workflow.yaml

@@ -21,7 +21,12 @@ jobs:
         shell: bash -l {0}
         run: |
           cd /home/runner/work/deep-code/deep-code
-          pip install .[dev]
+          pip install -e ".[dev]" 
+
+      - name: Lint with ruff
+        run: |
+            pip install ruff
+            ruff check
 
       - name: Run unit tests
         shell: bash -l {0}

README.md

@@ -56,25 +56,26 @@ pytest --cov-report html --cov=deep-code
 providing different utility functions.
 Use the --help option with these subcommands to get more details on usage.
 
-The CLI retrieves the Git username and personal access token from a hidden file named .gitaccess. Ensure this file is located in the same directory where you execute the CLI
+The CLI retrieves the Git username and personal access token from a hidden file named 
+.gitaccess. Ensure this file is located in the same directory where you execute the CLI
 command.
 
-###  deep-code publish-product
-
-Publish a dataset which is a result of an experiment to the EarthCODE 
-open-science catalog.
-
-```commandline
- deep-code publish-dataset /path/to/dataset-config.yaml
- ```
-
 #### .gitaccess example
 
 ```
 github-username: your-git-user
 github-token: personal access token
 ```
 
+###  deep-code publish
+
+Publish the experiment, workflow and dataset which is a result of an experiment to 
+the EarthCODE open-science catalog.
+
+```commandline
+ deep-code publish /path/to/dataset-config.yaml /path/to/workflow-config.yaml
+ ```
+
 #### dataset-config.yaml example
 
 ```
@@ -92,42 +93,26 @@ cf_parameter:
   - name: hydrology
 ```
 
-dataset-id has to be a valid dataset-id from `deep-esdl-public` s3 or your team bucket.
-
-### deep-code publish-workflow
+dataset-id has to be a valid dataset-id from `deep-esdl-public` s3 bucket or your team 
+bucket.
 
-Publish a workflow/experiment to the EarthCODE open-science catalog.
-
-```commandline
-deep-code publish-workflow /path/to/workflow-config.yaml
- ```
 #### workflow-config.yaml example
 
 ```
-workflow_id: "4D Med hydrology cube generation"
+workflow_id: "esa-cci-permafrost"
 properties:
-  title: "Hydrology cube generation recipe"
-  description: "4D Med cube generation"
+  title: "ESA CCI permafrost"
+  description: "cube generation workflow for esa-cci-permafrost"
   keywords:
     - Earth Science
   themes:
-      - Atmosphere
-      - Ocean
-      - Evaporation
+      - cryosphere
   license: proprietary
   jupyter_kernel_info:
-    name: deepesdl-xcube-1.7.1
+    name: deepesdl-xcube-1.8.3
     python_version: 3.11
-    env_file: https://git/env.yml
-links:
-  - rel: "documentation"
-    type: "application/json"
-    title: "4DMed Hydrology Cube Generation Recipe"
-    href: "https://github.com/deepesdl/cube-gen/tree/main/hydrology/README.md"
-  - rel: "jupyter-notebook"
-    type: "application/json"
-    title: "Workflow Jupyter Notebook"
-    href: "https://github.com/deepesdl/cube-gen/blob/main/hydrology/notebooks/reading_hydrology.ipynb"
+    env_file: "https://github.com/deepesdl/cube-gen/blob/main/Permafrost/environment.yml"
+jupyter_notebook_url: "https://github.com/deepesdl/cube-gen/blob/main/Permafrost/Create-CCI-Permafrost-cube-EarthCODE.ipynb"
 contact:
   - name: Tejas Morbagal Harish
     organization: Brockmann Consult GmbH

deep_code/cli/main.py

@@ -6,7 +6,7 @@
 
 import click
 
-from deep_code.cli.publish import publish_dataset, publish_workflow
+from deep_code.cli.publish import publish
 
 
 @click.group()
@@ -15,8 +15,7 @@ def main():
     pass
 
 
-main.add_command(publish_dataset)
-main.add_command(publish_workflow)
+main.add_command(publish)
 
 if __name__ == "__main__":
     main()

deep_code/cli/publish.py

@@ -6,21 +6,16 @@
 
 import click
 
-from deep_code.tools.publish import DatasetPublisher, WorkflowPublisher
+from deep_code.tools.publish import Publisher
 
 
-@click.command(name="publish-dataset")
+@click.command(name="publish")
 @click.argument("dataset_config", type=click.Path(exists=True))
-def publish_dataset(dataset_config):
+@click.argument("workflow_config", type=click.Path(exists=True))
+def publish(dataset_config, workflow_config):
     """Request publishing a dataset to the open science catalogue.
     """
-    publisher = DatasetPublisher()
-    publisher.publish_dataset(dataset_config_path=dataset_config)
-
-
-@click.command(name="publish-workflow")
-@click.argument("workflow_metadata", type=click.Path(exists=True))
-def publish_workflow(workflow_metadata):
-
-    workflow_publisher = WorkflowPublisher()
-    workflow_publisher.publish_workflow(workflow_config_path=workflow_metadata)
+    publisher = Publisher(
+        dataset_config_path=dataset_config, workflow_config_path=workflow_config
+    )
+    publisher.publish_all()

deep_code/constants.py

@@ -4,8 +4,10 @@
 # Permissions are hereby granted under the terms of the MIT License:
 # https://opensource.org/licenses/MIT.
 
-OSC_SCHEMA_URI = "https://stac-extensions.github.io/osc/v1.0.0-rc.3/schema.json"
+OSC_SCHEMA_URI = "https://stac-extensions.github.io/osc/v1.0.0/schema.json"
 CF_SCHEMA_URI = "https://stac-extensions.github.io/cf/v0.2.0/schema.json"
+THEMES_SCHEMA_URI = "https://stac-extensions.github.io/themes/v1.0.0/schema.json"
+OSC_THEME_SCHEME = "https://github.com/stac-extensions/osc#theme"
 OSC_REPO_OWNER = "ESA-EarthCODE"
 OSC_REPO_NAME = "open-science-catalog-metadata-testing"
 OSC_BRANCH_NAME = "add-new-collection"
@@ -14,3 +16,16 @@
 )
 OGC_API_RECORD_SPEC = "http://www.opengis.net/spec/ogcapi-records-1/1.0/req/record-core"
 WF_BRANCH_NAME = "add-new-workflow-from-deepesdl"
+VARIABLE_BASE_CATALOG_SELF_HREF = "https://esa-earthcode.github.io/open-science-catalog-metadata/variables/catalog.json"
+PRODUCT_BASE_CATALOG_SELF_HREF = "https://esa-earthcode.github.io/open-science-catalog-metadata/products/catalog.json"
+DEEPESDL_COLLECTION_SELF_HREF = (
+    "https://esa-earthcode.github.io/open-science-catalog-metadata/projects/deepesdl"
+    "/collection.json"
+)
+BASE_URL_OSC = "https://esa-earthcode.github.io/open-science-catalog-metadata"
+EXPERIMENT_BASE_CATALOG_SELF_HREF = "https://esa-earthcode.github.io/open-science-catalog-metadata/experiments/catalog.json"
+WORKFLOW_BASE_CATALOG_SELF_HREF = (
+    "https://esa-earthcode.github.io/open-science-catalog-metadata/workflows/catalog"
+    ".json"
+)
+PROJECT_COLLECTION_NAME = "deep-earth-system-data-lab"