hxl2example: now example also allow expose API via web (uses hug)

Author: fititnt

README.md

@@ -63,6 +63,34 @@ license etc.
 
 What it does: `hxl2example` accepts one HXLated dataset and save as .CSV.
 
+**Quick examples**
+
+```bash
+### Basic examples
+
+# This will output a local file to stdout (tip: you can disable local files)
+hxl2example tests/files/iris_hxlated-csv.csv
+
+# This will save to a local file
+hxl2example tests/files/iris_hxlated-csv.csv my-local-file.example
+
+# Since we use the libhxl-python, remote HXLated remote urls works too!
+hxl2example https://docs.google.com/spreadsheets/d/1En9FlmM8PrbTWgl3UHPF_MXnJ6ziVZFhBbojSJzBdLI/edit#gid=319251406
+
+### Advanced usage (if you need to share work with others)
+
+## Quick ad-hoc web proxy, local usage
+# @see https://github.com/hugapi/hug
+
+hug -f bin/hxl2example
+# http://localhost:8000/ will how an JSON documentation of hug endpoints. TL;DR:
+# http://localhost:8000/hxl2example.csv?source_url=http://example.com/remote-file.csv
+
+## Expose local web proxy to others
+# @see https://ngrok.com/
+ngrok http 8000
+```
+
 ##### 1.2.2 `hxl2tab`: tab format, focused for compatibility with Orange Data Mining 
 - Main issue: <https://github.com/EticaAI/HXL-Data-Science-file-formats/issues/2>
 - Orange File Specification: <https://orange-data-mining-library.readthedocs.io/en/latest/reference/data.io.html>

bin/hxl2example

@@ -5,17 +5,26 @@
 #
 #         USAGE:  hxl2example hxlated-data.hxl my-exported-file.example
 #                 cat hxlated-data.hxl | hxl2example > my-exported-file.example
+#                 # Via web, in two different terminals, do it
+#                     hug -f bin/hxl2example
+#                     ngrok http 8000
 #
 #   DESCRIPTION:  hxl2example is an example script to create other scripts with
 #                 some bare minimum command line interface that could work.
 #                 With exception of external libraries, the hxl2example is
 #                 meant to be somewhat self-contained one-file executable ready
-#                 to just be added to the path
+#                 to just be added to the path.
+#
+#                 Hug API can be used to create an ad-hoc web interface to your
+#                 script. This can be both useful if you are using an software
+#                 that accepts an URL as data source and you don't want to use
+#                 this script to save a file locally.
 #
 #       OPTIONS:  ---
 #
 #  REQUIREMENTS:  - python3
 #                     - libhxl (@see https://pypi.org/project/libhxl/)
+#                     - hug (https://github.com/hugapi/hug/)
 #                     - your-extra-python-lib-here
 #                 - your-non-python-dependency-here
 #          BUGS:  ---
@@ -33,11 +42,19 @@ import os
 import logging
 import argparse
 
+# @see https://github.com/HXLStandard/libhxl-python
+#    pip3 install libhxl --upgrade
 # Do not import hxl, to avoid circular imports
 import hxl.converters
 import hxl.filters
 import hxl.io
 
+import tempfile
+
+# @see https://github.com/hugapi/hug
+#     pip3 install hug --upgrade
+import hug
+
 # In Python2, sys.stdin is a byte stream; in Python3, it's a text stream
 STDIN = sys.stdin.buffer
 
@@ -72,10 +89,10 @@ class HXL2Example:
         self.args = parser.parse_args()
         return self.args
 
-    def do_example_output(self, args,
-                          stdin=STDIN, stdout=sys.stdout, stderr=sys.stderr):
+    def execute_cli(self, args,
+                    stdin=STDIN, stdout=sys.stdout, stderr=sys.stderr):
         """
-        The do_example_output is the main entrypoint of HXL2Example. When
+        The execute_cli is the main entrypoint of HXL2Example. When
         called will convert the HXL source to example format.
         """
 
@@ -88,6 +105,49 @@ class HXL2Example:
 
         return self.EXIT_OK
 
+    def execute_web(self, source_url, stdin=STDIN, stdout=sys.stdout,
+                    stderr=sys.stderr, hxlmeta=False):
+        """
+        The execute_web is the main entrypoint of HXL2Tab when this class is
+        called outside command line interface, like the build in HTTP use with
+        hug
+        """
+
+        # TODO: the execute_web needs to output the tabfile with correct
+        #       mimetype, compression, etc
+        #       (fititnt, 2021-02-07 15:59 UTC)
+
+        self.hxlhelper = HXLUtils()
+
+        try:
+            temp_input = tempfile.NamedTemporaryFile('w')
+            temp_output = tempfile.NamedTemporaryFile('w')
+
+            webargs = type('obj', (object,), {
+                "infile": source_url,
+                "sheet_index": None,
+                "selector": None,
+                'sheet': None,
+                'http_header': None,
+                'ignore_certs': False
+            })
+
+            with self.hxlhelper.make_source(webargs, stdin) as source:
+                for line in source.gen_csv(True, True):
+                    temp_input.write(line)
+
+                temp_input.seek(0)
+                # self.hxl2tab(temp_input.name, temp_output.name, False)
+
+                result_file = open(temp_input.name, 'r')
+                return result_file.read()
+
+        finally:
+            temp_input.close()
+            temp_output.close()
+
+        return self.EXIT_OK
+
 
 class HXLUtils:
     """
@@ -224,7 +284,7 @@ class HXLUtils:
             url_or_filename or stdin,
             sheet_index=sheet_index,
             selector=selector,
-            allow_local=True,
+            allow_local=True,  # TODO: consider change this for execute_web
             http_headers=http_headers,
             verify_ssl=(not args.ignore_certs)
         )
@@ -297,4 +357,30 @@ if __name__ == "__main__":
     hxl2example = HXL2Example()
     args = hxl2example.make_args_hxl2example()
 
-    hxl2example.do_example_output(args)
+    hxl2example.execute_cli(args)
+
+
+@hug.format.content_type('text/csv')
+def output_csv(data, response):
+    if isinstance(data, dict) and 'errors' in data:
+        response.content_type = 'application/json'
+        return hug.output_format.json(data)
+    response.content_type = 'text/csv'
+    if hasattr(data, "read"):
+        return data
+
+    return str(data).encode("utf8")
+
+
+@hug.get('/hxl2example.csv', output=output_csv)
+def api_hxl2tab(source_url):
+    """hxl2example (@see https://github.com/EticaAI/HXL-Data-Science-file-formats)
+
+    Example:
+    http://localhost:8000/hxl2tab.tab?source_url=https://docs.google.com/spreadsheets/u/1/d/1l7POf1WPfzgJb-ks4JM86akFSvaZOhAUWqafSJsm3Y4/edit#gid=634938833
+
+    """
+
+    hxl2example = HXL2Example()
+
+    return hxl2example.execute_web(source_url)

bin/hxl2tab

@@ -172,6 +172,7 @@ class HXL2Tab:
             temp_input = tempfile.NamedTemporaryFile('w')
             temp_output = tempfile.NamedTemporaryFile('w')
 
+            # TODO: implement other options beyond source_url
             webargs = type('obj', (object,), {
                 "infile": source_url,
                 "sheet_index": None,

tests/manual-tests.sh

@@ -10,6 +10,22 @@ sudo snap install ngrok
 
 #### The tests _________________________________________________________________
 
+### hxl2example ----------------------------------------------------------------
+
+hxl2example tests/files/iris_hxlated-csv.csv | head
+hxl2example tests/files/iris_hxlated-csv.csv temp/iris.tab
+hxl2example https://docs.google.com/spreadsheets/u/1/d/1l7POf1WPfzgJb-ks4JM86akFSvaZOhAUWqafSJsm3Y4/edit#gid=634938833 | head
+hxl2example https://docs.google.com/spreadsheets/u/1/d/1l7POf1WPfzgJb-ks4JM86akFSvaZOhAUWqafSJsm3Y4/edit#gid=634938833 data-mining-projects/output/HXL-CPLP-Exemplar_iris.tab
+
+## hug -f bin/hxl2example ......................................................
+#@see https://hugapi.github.io/hug/
+#@see https://github.com/hugapi/hug/
+hug -f bin/hxl2example
+
+curl --silent http://localhost:8000/hxl2example.csv?source_url=https://docs.google.com/spreadsheets/u/1/d/1l7POf1WPfzgJb-ks4JM86akFSvaZOhAUWqafSJsm3Y4/edit#gid=634938833 | head
+# HXLStandard_HXLCoreSchema_CoreHashtags
+curl --silent http://localhost:8000/hxl2example.csv?source_url=https://docs.google.com/spreadsheets/d/1En9FlmM8PrbTWgl3UHPF_MXnJ6ziVZFhBbojSJzBdLI/edit#gid=319251406 | head
+
 ### hxl2tab --------------------------------------------------------------------
 
 hxl2tab tests/files/iris_hxlated-csv.csv temp/iris.tab
@@ -22,7 +38,7 @@ hxl2tab https://docs.google.com/spreadsheets/d/1Vqv6-EAdSHMSZvZtE426aXkDiwP8Mdrp
 hxl2tab https://docs.google.com/spreadsheets/d/1Vqv6-EAdSHMSZvZtE426aXkDiwP8Mdrpft3tiGQ1RH0/edit#gid=0 temp/example-ebola-dataset-1_HXLated+tab_hxltabv15_b.tab
 hxl2tab https://docs.google.com/spreadsheets/d/1Vqv6-EAdSHMSZvZtE426aXkDiwP8Mdrpft3tiGQ1RH0/edit#gid=0 temp/example-ebola-dataset-1_HXLated+tab_hxltabv16.tab
 
-### hug -f bin/hxl2tab ---------------------------------------------------------
+## hug -f bin/hxl2tab ..........................................................
 #@see https://hugapi.github.io/hug/
 #@see https://github.com/hugapi/hug/
 hug -f bin/hxl2tab