added parser for csv

Author: emptymalei

dataherb/command.py

@@ -2,19 +2,105 @@
 import os
 
 import click
+import inquirer
 
 from dataherb.flora import Flora
-from dataherb.parse.model import MetaData
+from dataherb.parse.model import MetaData, IGNORED_FOLDERS_AND_FILES
 
 __CWD__ = os.getcwd()
 
 logging.basicConfig()
 logger = logging.getLogger("dataherb.command")
 
+def describe_file(file):
+    questions = [
+            inquirer.Text(
+                'name',
+                message=f"How would you like to name the file: {file}?"
+            ),
+            inquirer.Text(
+                'description',
+                message=f"What is {file} about?"
+            ),
+            inquirer.Text(
+                'updated_at',
+                message=f"When was {file} last updated? In ISO date format such as 2020-02-17."
+            )
+        ]
 
-_FLORA = Flora()
+    answers = inquirer.prompt(questions)
+    meta = {
+        "name": answers.get('name'),
+        "description": answers.get("description"),
+        "updated_at": answers.get("updated_at")
+    }
 
-_FLORA.herb("geonames_timezone").leaves.get("dataset/geonames_timezone.csv").data
+    return meta
+
+def describe_dataset():
+    """
+    describe_dataset asks the user to specify some basic info about the dataset
+    """
+    questions = [
+            inquirer.Text(
+                'name',
+                message="How would you like to name the dataset?"
+            ),
+            inquirer.Text(
+                'description',
+                message="What is the dataset about? This will be the description of the dataset."
+            )
+        ]
+
+    answers = inquirer.prompt(questions)
+    meta = {
+        "name": answers.get('name', ""),
+        "description": answers.get('description', "")
+    }
+
+    return meta
+
+
+def where_is_dataset():
+    """
+    where_is_dataset asks the user where the dataset is located.
+    """
+    try:
+        folders = []
+        for root, dirs, files in os.walk(__CWD__):
+            for d in dirs:
+                if d not in IGNORED_FOLDERS_AND_FILES:
+                    folders.append(
+                        os.path.relpath(os.path.join(root, d), ".")
+                    )
+    except Exception as e:
+        logger.error("Can not get a list of folders in current directory.")
+        folders = []
+
+    if folders:
+        questions = [
+            inquirer.List(
+                'dataset_folder',
+                message="Which folder contains the data file?",
+                choices=folders
+            )
+        ]
+    else:
+        questions = [
+            inquirer.Path(
+                'dataset_folder',
+                message="Which folder will you place the data files?",
+                path_type=inquirer.Path.DIRECTORY,
+            )
+        ]
+
+    answers = inquirer.prompt(questions)
+    dataset_folder = answers.get('dataset_folder')
+
+    return dataset_folder
+
+
+# _FLORA.herb("geonames_timezone").leaves.get("dataset/geonames_timezone.csv").data
 
 @click.group()
 def dataherb():
@@ -23,8 +109,9 @@ def dataherb():
 
 @dataherb.command()
 def search(keywords, ids):
+    fl = Flora()
     click.echo('Search Herbs in DataHerb Flora ...')
-    _FLORA.search()
+    fl.search()
 
 @dataherb.command()
 @click.confirmation_option(
@@ -35,13 +122,33 @@ def search(keywords, ids):
 def create():
 
     md = MetaData()
+
+    dataset_basics = describe_dataset()
+    print(dataset_basics)
+    md.template.update(dataset_basics)
+
+    dataset_folder = where_is_dataset()
+    print(
+        f"Looking into the folder {dataset_folder} for data files..."
+    )
+
+    dataset_files = md.parse_structure(dataset_folder)
+    print(
+        f"found {dataset_files}"
+    )
+
+    for file in dataset_files:
+        file_meta = describe_file(file)
+        md.append_leaf(file, file_meta)
+
     md.create()
 
     click.echo(
         "The .dataherb folder and metadata.yml file has been created inside \n"
-        f"{__CWD__}"
+        f"{__CWD__}\n"
+        "Please review the metadata.yml file and update other necessary fields of your desire."
     )
 
 if __name__ == "__main__":
     fl = Flora()
-    pass
+    pass

dataherb/parse/model.py

@@ -1,14 +1,33 @@
-import os
+import csv
 import logging
-import yaml
+import os
+from collections import OrderedDict
+from pathlib import Path
+
+import ruamel.yaml
+from ruamel.yaml.representer import RoundTripRepresenter
 
 logging.basicConfig()
 logger = logging.getLogger("dataherb.parse.model")
 
+IGNORED_FOLDERS_AND_FILES = ['.git', '.dataherb', '.vscode']
+
+# Add representer to ruamel.yaml for OrderedDict
+class MyRepresenter(RoundTripRepresenter):
+    pass
+
+ruamel.yaml.add_representer(
+    OrderedDict, MyRepresenter.represent_dict, representer=MyRepresenter
+)
+yaml = ruamel.yaml.YAML()
+yaml.Representer = MyRepresenter
+
 
 class MetaData(object):
     def __init__(self):
-        self.template = {
+        self.dataherb_folder = '.dataherb'
+        self.metadata_file = 'metadata.yml'
+        self.template = OrderedDict({
             "name": "",
             "description": "",
             "contributors": [
@@ -17,44 +36,123 @@ def __init__(self):
                     "github": ""
                 }
             ],
-            "data": [
+            "data": [],
+            "references": [
                 {
                     "name": "",
-                    "path": "",
-                    "format": "",
-                    "size": "",
-                    "updated_at": "",
-                    "fields": [
-                        {
-                            "name": "",
-                            "description": ""
-                        },
-                        {
-                            "name": "",
-                            "description": ""
-                        }
-                    ]
+                    "link": ""
                 }
-            ],
-            "references": [
+            ]
+        })
+
+    def parse_structure(self, folder=None):
+
+        if folder is None:
+            folder = '.'
+
+        tree_f = []
+        tree_d = []
+        for root, dirs, files in os.walk(folder):
+            for d in dirs:
+                if d not in IGNORED_FOLDERS_AND_FILES:
+                    tree_d.append(
+                        os.path.relpath(os.path.join(root, d), folder)
+                    )
+            for f in files:
+                tree_f.append(
+                    os.path.relpath(os.path.join(root, f), folder)
+                )
+
+        self.tree = tree_f
+
+        return self.tree
+
+    def parse_csv(self, csv_file):
+        """
+        parse_csv parses the csv files for metadata generation
+        """
+
+        with open(csv_file, "r") as f:
+            reader = csv.reader(f)
+            columns = next(reader)
+
+        fields = []
+        for col in columns:
+            fields.append({
+                "name": col,
+                "description": ""
+            })
+
+        return fields
+
+    def _generate_leaf(self, path, meta_input):
+
+        name = meta_input.get("name", "")
+        description = meta_input.get("description", "")
+        updated_at = meta_input.get("updated_at", "")
+
+        file_format = path.split(".")[-1]
+        if len(file_format) >= 10:
+            logger.error(f"The format of file {path} could not be determined!")
+            file_format = ""
+
+        file_size = os.stat(path).st_size
+
+        if file_format == "csv":
+            fields = self.parse_csv(path)
+        else:
+            fields = [
                 {
                     "name": "",
-                    "link": ""
+                    "description": ""
+                },
+                {
+                    "name": "",
+                    "description": ""
                 }
             ]
+
+        res = {
+            "name": name,
+            "description": description,
+            "path": path,
+            "format": file_format,
+            "size": file_size,
+            "updated_at": updated_at,
+            "fields": fields
         }
 
+        return res
+
+    def append_leaf(self, dataset_file, meta_input):
+
+        existing_leaves = self.template["data"]
+        existing_leaves.append(
+            self._generate_leaf(dataset_file, meta_input)
+        )
+        self.template.update(data=existing_leaves)
+
     def create(self):
 
         # create .dataherb folder
-        dataherb_folder = '.dataherb'
+        dataherb_folder = self.dataherb_folder
         try:
             os.mkdir(dataherb_folder)
             logger.info("Created ", dataherb_folder)
         except FileExistsError:
-            logger.info(dataherb_folder,  " already exists!")
+            logger.info(
+                dataherb_folder,
+                " already exists! Creating metadata.yml file inside."
+            )
+            pass
+
+        metadata_file = self.metadata_file
 
-        metadata_file = 'metadata.yml'
+        if os.path.isfile(os.path.join(dataherb_folder, metadata_file)):
+            logger.error(
+                f'File {os.path.join(dataherb_folder, metadata_file)} already exists!'
+            )
+            raise SystemExit
 
         with open(os.path.join(dataherb_folder, metadata_file), 'w') as fp:
             documents = yaml.dump(self.template, fp)

docs/source/HISTORY.rst

@@ -6,4 +6,12 @@ This is the CHANGELOG of the package.
 [0.0.1] - 2020-02-13
 -------------------------
 
-First version
+First version
+
+[0.0.3] - 2020-02-23
+-------------------------
+
+Added
+
+1. Data columns autogenerator for csv files.
+

environment.yml

@@ -6,9 +6,10 @@ dependencies:
   - python=3.7
   - pandas>=0.23
   - requests>=2.22.0
-  - pyyaml>=5.3
   - click==7.0
   - pip
   - pip:
     - fuzzywuzzy>=0.18.0
-    - python-Levenshtein>=0.12
+    - python-Levenshtein>=0.12
+    - ruamel.yaml>=0.16.10
+    - inquirer>=2.6.3

requirements.txt

@@ -1,6 +1,7 @@
 pandas>=0.23
 requests>=2.22.0
 fuzzywuzzy>=0.18.0
-pyyaml>=1.18
+ruamel.yaml>=0.16.10
 python-Levenshtein>=0.12
-click==7.0
+click==7.0
+inquirer>=2.6.3