added remove function

Author: emptymalei

dataherb/cmd/configs.py

@@ -6,7 +6,8 @@
 logger.remove()
 logger.add(sys.stderr, level="INFO", enqueue=True)
 
-def load_dataherb_config(config_path=None):
+
+def load_dataherb_config(config_path=None, no_config_error=True):
     """Loads the dataherb config file.
 
     Load the content from the specified file as the config. The config file has to be json.
@@ -17,10 +18,14 @@ def load_dataherb_config(config_path=None):
         config_path = home / ".dataherb/config.json"
 
     if not config_path.exists():
-        logger.error(
-            f"Config file {config_path} does not exist.\n"
-            f"If this is the first time you use dataherb, please run `dataherb configure` to config dataherb."
-        )
+        if no_config_error:
+            logger.error(
+                f"Config file {config_path} does not exist.\n"
+                f"If this is the first time you use dataherb, please run `dataherb configure` to config dataherb."
+            )
+        else:
+            pass
+        return {}
 
     logger.debug(f"Using {config_path} as config file for dataherb")
     try:

dataherb/cmd/create.py

@@ -8,6 +8,7 @@
 logger.remove()
 logger.add(sys.stderr, level="INFO", enqueue=True)
 
+
 def describe_file(file):
     """
     describe_file [summary]
@@ -89,6 +90,7 @@ def describe_dataset():
     meta = {
         "source": answers.get("source"),
         "name": answers.get("name", ""),
+        "id": answers["id"],
         "description": answers.get("description", ""),
         "uri": answers.get("uri", ""),
         "metadata_uri": metadata_uri,

dataherb/command.py

@@ -44,9 +44,14 @@ def configure():
     config_path = home / ".dataherb/config.json"
 
     if config_path.exists():
-        is_overwite = click.confirm(f"Config file ({config_path}) already exists. Overwrite?", default=False)
+        is_overwite = click.confirm(
+            click.style(
+                f"Config file ({config_path}) already exists. Overwrite?", fg="red"
+            ),
+            default=False,
+        )
         if is_overwite:
-            click.echo("Overwriting config file...")
+            click.echo(click.style("Overwriting config file...", fg="red"))
         else:
             click.echo("Skipping...")
             sys.exit(0)
@@ -62,35 +67,36 @@ def configure():
             "workdir",
             message="Where should I put all the datasets and flora database? An empty folder is recommended.",
             # path_type=inquirer.Path.DIRECTORY,
-            normalize_to_absolute_path=True
+            normalize_to_absolute_path=True,
         ),
         inquirer.Text(
             "default_flora",
             message="How would you name the default flora? Please keep the default value if this is not clear to you.",
-            default="flora"
-        )
+            default="flora",
+        ),
     ]
 
     answers = inquirer.prompt(questions)
 
     config = {
         "workdir": answers.get("workdir"),
-        "default": {
-            "flora": answers.get("default_flora")
-        }
+        "default": {"flora": answers.get("default_flora")},
     }
 
     logger.debug(f"config: {config}")
 
     with open(config_path, "w") as f:
         json.dump(config, f, indent=4)
 
-    click.echo(f"The dataherb config has been saved to {config_path}!")
-
+    click.echo(
+        click.style(f"The dataherb config has been saved to {config_path}!", fg="green")
+    )
 
 
-CONFIG = load_dataherb_config()
+CONFIG = load_dataherb_config(no_config_error=False)
 logger.debug(CONFIG)
+if not CONFIG:
+    click.echo("No config file found. Please run 'dataherb configure' to create one.")
 WD = CONFIG.get("workdir", ".")
 which_flora = CONFIG.get("default", {}).get("flora")
 if which_flora:
@@ -100,8 +106,6 @@ def configure():
         raise Exception(f"flora config {which_flora} does not exist")
 
 
-
-
 @dataherb.command()
 @click.argument("keywords", required=False)
 @click.option("--id", "-i", required=False)
@@ -138,21 +142,24 @@ def search(flora, id=None, keywords=None):
 @click.option("--flora", "-f", default=which_flora)
 @click.option("--workdir", "-w", default=WD, required=True)
 @click.option("--dev_addr", "-a", metavar="<IP:PORT>")
-def serve(flora, workdir, dev_addr):
+@click.option("--recreate", "-r", default=False, required=False)
+def serve(flora, workdir, dev_addr, recreate):
     fl = Flora(flora=flora)
     mk = SaveMkDocs(flora=fl, workdir=workdir)
-    mk.save_all()
+    mk.save_all(recreate=recreate)
 
-    mkdocs_config = str(Path(WD) / "serve" / "mkdocs.yml")
+    mkdocs_config = str(Path(workdir) / "serve" / "mkdocs.yml")
 
     click.echo("Open http://localhost:8000")
+    click.launch("http://localhost:8000")
     _serve(config_file=mkdocs_config, dev_addr=dev_addr)
 
 
 @dataherb.command()
 @click.argument("id", required=True)
 @click.option("--flora", "-f", default=which_flora)
-def download(id, flora):
+@click.option("--workdir", "-w", default=WD, required=True)
+def download(id, flora, workdir):
     """
     download dataset using id
     """
@@ -167,7 +174,7 @@ def download(id, flora):
         click.echo(f'Downloading DataHerb ID: {result_metadata.get("id")}')
         result_uri = result_metadata.get("uri")
         result_id = result_metadata.get("id")
-        dest_folder = str(Path(WD) / result_id)
+        dest_folder = str(Path(workdir) / result_id)
         if os.path.exists(dest_folder):
             click.echo(f"Can not download dataset to {dest_folder}: folder exists.\n")
 
@@ -225,18 +232,53 @@ def create(flora):
 
         md.metadata.update(pkg_descriptor)
 
-        md.create()
-
-        click.echo(
-            "The dataherb.json file has been created inside \n"
-            f"{__CWD__}\n"
-            "Please review the dataherb.json file and update other necessary fields."
-        )
+        if (Path(__CWD__) / "dataherb.json").exists():
+            is_overwrite = click.confirm("Replace the current dataherb.json file?", default=False)
+            if is_overwrite:
+                md.create(overwrite=is_overwrite)
 
-    hb = Herb(md.metadata)
+                click.echo(
+                    f"The dataherb.json file in folder {__CWD__} has been replaced. \n"
+                    "Please review the dataherb.json file and update other necessary fields."
+                )
+            else:
+                click.echo("We did nothing.")
+                sys.exit()
+        else:
+            md.create()
+            click.echo(
+                "The dataherb.json file has been created inside \n"
+                f"{__CWD__}\n"
+                "Please review the dataherb.json file and update other necessary fields."
+            )
+
+    hb = Herb(md.metadata, with_resources=False)
     fl.add(hb)
 
-    click.echo(f"Added {hb.metadata['id']} into the flora.")
+    click.echo(f"Added {hb.id} into the flora.")
+
+
+@dataherb.command()
+@click.option("--flora", "-f", default=which_flora)
+@click.argument("herb_id", required=True)
+def remove(flora, herb_id):
+    """
+    remove herb from flora
+    """
+
+    fl = Flora(flora=flora)
+    herb = fl.herb(herb_id)
+    if not herb:
+        click.echo(click.style(f"Could not find herb with id {herb_id}", fg="red"))
+        click.echo("We did nothing.")
+        sys.exit()
+
+    to_remove = click.confirm(f"Remove {herb_id} from the flora?", default=False)
+    if to_remove:
+        fl.remove(herb_id)
+        click.echo(f"Removed {herb_id} into the flora.")
+    else:
+        click.echo("We did nothing.")
 
 
 @dataherb.command()
@@ -329,10 +371,6 @@ def echo_summary(key, value_dict, bg=None, fg=None):
     )
 
 
-
-
-
-
 if __name__ == "__main__":
     fl = Flora()
     pass

dataherb/core/base.py

@@ -21,7 +21,7 @@ class Herb(object):
     Herb is a collection of the dataset.
     """
 
-    def __init__(self, meta_dict, base_path=None):
+    def __init__(self, meta_dict, base_path=None, with_resources=True):
         """
         :param herb_meta_json: the dictionary that specifies the herb
         :type herb_meta_json: dict
@@ -34,6 +34,8 @@ def __init__(self, meta_dict, base_path=None):
         if isinstance(self.base_path, str):
             self.base_path = Path(self.base_path)
 
+        self.with_resources = with_resources
+
         if isinstance(meta_dict, dict):
             self.herb_meta_json = meta_dict
         elif isinstance(meta_dict, MetaData):
@@ -69,10 +71,11 @@ def _from_meta_dict(self, meta_dict):
         if not self.datapackage:
             self.update_datapackage()
 
-        self.resources = [
-            self.get_resource(i, source_only=False)
-            for i in range(len(self.datapackage.resources))
-        ]
+        if self.with_resources:
+            self.resources = [
+                self.get_resource(i, source_only=False)
+                for i in range(len(self.datapackage.resources))
+            ]
 
     def get_resource(self, idx=None, path=None, name=None, source_only=True):
         if idx is None:
@@ -98,7 +101,7 @@ def get_resource(self, idx=None, path=None, name=None, source_only=True):
                 )
 
         if self.is_local:
-            logger.info(
+            logger.debug(
                 f"Using local dataset for {self.id}, sync it if you need the updated version."
             )
             r = self.datapackage.resources[idx]

dataherb/flora.py

@@ -5,10 +5,10 @@
 from loguru import logger
 
 from dataherb.core.base import Herb
-from dataherb.core.search import \
-    search_by_ids_in_flora as _search_by_ids_in_flora
-from dataherb.core.search import \
-    search_by_keywords_in_flora as _search_by_keywords_in_flora
+from dataherb.core.search import search_by_ids_in_flora as _search_by_ids_in_flora
+from dataherb.core.search import (
+    search_by_keywords_in_flora as _search_by_keywords_in_flora,
+)
 from dataherb.fetch.remote import get_data_from_url as _get_data_from_url
 from dataherb.parse.model_json import MetaData
 
@@ -87,6 +87,19 @@ def add(self, herb):
         self.flora.append(herb)
         self.save()
 
+    def remove(self, herb_id):
+        """
+        add add a herb to the flora
+        """
+
+        for id in [i.id for i in self.flora]:
+            if id == herb_id:
+                logger.debug(f"found herb id = {herb_id}")
+
+        self.flora = [h for h in self.flora if h.id != herb_id]
+
+        self.save()
+
     def save(self, path=None):
         """save flora metadata to json file"""
 

dataherb/parse/model_json.py

@@ -46,7 +46,7 @@ def load(self):
 
         logger.debug(f"Loaded metadata: {self.metadata}")
 
-    def create(self):
+    def create(self, overwrite=False):
         """creates dataherb.json file"""
         # create .dataherb folder
         dataherb_folder = self.dataherb_folder
@@ -62,11 +62,15 @@ def create(self):
 
         metadata_file = self.metadata_file
 
+
         if os.path.isfile(os.path.join(dataherb_folder, metadata_file)):
-            logger.error(
-                f"File {os.path.join(dataherb_folder, metadata_file)} already exists!"
-            )
-            raise SystemExit
+            if not overwrite:
+                logger.error(
+                    f"File {os.path.join(dataherb_folder, metadata_file)} already exists!"
+                )
+                raise SystemExit
+            else:
+                logger.debug(f"Will overwrite {os.path.join(dataherb_folder, metadata_file)}")
 
         with open(os.path.join(dataherb_folder, metadata_file), "w") as fp:
             documents = json.dump(self.metadata, fp, indent=4)

dataherb/serve/mkdocs_template/herbs/index.md

@@ -0,0 +1,5 @@
+---
+title: Home
+---
+
+This is the landing page of DataHerb. Please use the search of the sidebar to find view the datasets.

dataherb/serve/mkdocs_template/mkdocs.yml

@@ -0,0 +1,40 @@
+site_name: My DataHerb
+
+docs_dir: herbs
+
+theme:
+  name: "material"
+  custom_dir: overrides
+  include_search_page: false
+  search_index_only: true
+
+  language: en
+  features:
+    - navigation.sections
+    - navigation.instant
+  palette:
+    - scheme: default
+      primary: indigo
+      accent: indigo
+      toggle:
+        icon: material/toggle-switch-off-outline
+        name: Switch to dark mode
+    - scheme: slate
+      primary: red
+      accent: red
+      toggle:
+        icon: material/toggle-switch
+        name: Switch to light mode
+
+
+
+
+markdown_extensions:
+  - meta
+  - admonition
+
+plugins:
+  - search
+  - macros
+
+