1509 update gh docs

.circleci/config.yml

@@ -55,7 +55,6 @@ jobs:
 #      - slack/status:
 #          fail_only: false
 
-# cli build
   cli-build:
     executor: docker/docker
     steps:
@@ -75,30 +74,55 @@ jobs:
       - slack/status:
           fail_only: false
 
-# clients build
-  client-build:
+  onestop-s3-handler-build:
     executor: docker/docker
     steps:
       - setup_remote_docker
       - checkout
       - docker/check
       - docker/build:
-          path: onestop-python-client
+          path: scripts/sqs-to-registry
+          image: cedardevs/onestop-s3-handler
+          tag: latest
+      - docker/push:
+          image: cedardevs/onestop-s3-handler
+          tag: latest
+      - slack/status:
+          fail_only: false
+
+  onestop-sme-build:
+    executor: docker/docker
+    steps:
+      - setup_remote_docker
+      - checkout
+      - docker/check
+      - docker/build:
+          path: scripts/sme/
+          image: cedardevs/onestop-sme
+          tag: latest
+      - docker/push:
+          image: cedardevs/onestop-sme
+          tag: latest
+      - slack/status:
+          fail_only: false
+
+  onestop-python-client-build:
+    executor: docker/docker
+    steps:
+      - setup_remote_docker
+      - checkout
+      - docker/check
+      - docker/build:
+          path: ./
           image: cedardevs/onestop-python-client
-          tag: ${CIRCLE_BRANCH}-SNAPSHOT
-      - run:
-          name: "What branch am I on now?"
-          command: echo $CIRCLE_BRANCH
-#no need to push this image yet
+          tag: latest
       - docker/push:
           image: cedardevs/onestop-python-client
-          tag: ${CIRCLE_BRANCH}-SNAPSHOT
+          tag: latest
       - slack/status:
           fail_only: false
-  # Base test configuration for Go library tests Each distinct version should
-  # inherit this base, and override (at least) the container image used.
 
-  python-client-test: &python-client-test
+  onestop-python-client-test: &python-client-test
     executor: python/default
     steps: &steps
       - checkout
@@ -107,10 +131,16 @@ jobs:
           app-dir: ./onestop-python-client
           pkg-manager: pip
       - run:
-          name: "Run util tests"
+          name: "Run unit tests"
           command: >
             cd onestop-python-client/;
-            python -m unittest tests/util/*.py
+            python -m unittest discover -s test/unit
+#          This is commented out only because the OneStop we have running on cedardevs doesn't have its registry exposed. You can only reach it via sshing to another machine.
+#      - run:
+#          name: "Run integration tests"
+#          command: >
+#            cd onestop-python-client/;
+#            python -m unittest discover -s test/integration
 
 orbs:
   slack: circleci/[email protected]
@@ -121,9 +151,15 @@ version: 2.1
 workflows:
   main:
     jobs:
-#      - "latest"
 #      - cli-test
 #      - cli-build
-#      - client-build
-       - python-client-test
-
+       - onestop-python-client-test
+       - onestop-python-client-build:
+           requires:
+             - onestop-python-client-test
+       - onestop-sme-build:
+           requires:
+             - onestop-python-client-build
+       - onestop-s3-handler-build:
+           requires:
+             - onestop-python-client-build

Dockerfile

@@ -1,10 +1,15 @@
 FROM python:3.8
+
 COPY ./onestop-python-client /onestop-python-client
 COPY ./scripts /scripts
+
 RUN apt-get update
 RUN pip install --upgrade pip
-RUN pip install ./onestop-python-client
 RUN pip install -r ./onestop-python-client/requirements.txt
 
+# Needed for scripts - do here since directory out of scope when in scripts/* dockerfiles.
+# Unsure if possible this isn't latest build, like doing pip install before this is built.
+RUN pip install ./onestop-python-client
+
 #Base image stays up for dev access
 CMD tail -f /dev/null

docs/README.md

@@ -17,13 +17,42 @@
 </p>
 
 ## Table of contents
+* [onestop-python-client](#onestop-python-client)
+* [Python Scripts](#python-scripts)
+* [Helm](#helm)
+* [CLI](#cli)
+* [Build Pipeline and Test Execution](build-pipeline)
 
-- [Project Overview](#project-overview)
-- [Navigating The Documentation](#navigating-the-documentation)
-  - [By User Type](#by-user-type)
-  - [By Project Component](#by-project-component)
-- [External Documentation](#docs/cli/developer/quickstart.md)
+This OneStop-clients project is a collection of clients to aid in communicating with OneStop and directly with the cloud.
 
-## Project Overview
-OneStop-clients is an open-sourced commandline interface and subject matter consumer clients ... 
+## [onestop-python-client](onestop-python-client)
+The onestop-python-client is a tool for subject matter experts (SME) to publish and consume metadata to and from OneStop as well as directly to the cloud.
+This would enable someone to feed data into OneStop, have OneStop digest it, and then read it out via a python script.
 
+[onestop-python-client](onestop-python-client) - More details.
+
+Additional information:
+* [onestop-test-data repository readme](https://github.com/cedardevs/onestop-test-data/blob/master/README.md) - loading test data into OneStop.
+* [OneStop documentation](https://cedardevs.github.io/onestop/) - OneStop documentation. 
+
+## [Python Scripts](scripts)
+There are some sample python scripts that use the onestop-python-client in the scripts directory.
+
+[python scripts](scripts) - More details.
+
+## [Helm](helm)
+There is a helm directory full of helm charts to create different kubernetes containers which each contain from this repository the onestop-python-client code and the scripts directory.
+ They have python installed so that a SME user could execute scripts from within. 
+
+[Helm](helm) - More details.
+
+## [CLI](cli)
+The CLI is an open-sourced commandline interface for OneStop's search API.
+
+* [Developer Quickstart](cli/developer-quickstart)
+
+* [Public User Quickstart](cli/public-user-quickstart)
+
+* [SCDR Files](cli/scdr-files)
+
+[CLI](cli) - More details.

docs/build-pipeline.md

@@ -0,0 +1,42 @@
+<div align="center"><a href="/onestop-clients/">Documentation Home</a></div>
+<hr>
+
+# Build Pipeline and Test Execution
+
+## Table of Contents
+* [CircleCI](#circleci)
+* [Building Manually](#building-manually)
+* [Test Execution](#test-execution)
+
+## CircleCI
+Currently, this project uses CircleCI to build the multiple images needed. If you example the circleci configuration file you will see what tests it executes and images it builds with what tags.
+
+## Building Manually
+* If you change the onestop-python-client code then run this, from the project root:
+
+```
+docker build . -t cedardevs/onestop-python-client:latest
+```
+
+* If you modify just the scripts then run this (only need to do the one relevant for your script), from the project root:
+
+```
+docker build ./scripts/sqs-to-registry -t cedardevs/onestop-s3-handler:latest
+```
+
+```
+docker build ./scripts/sme/ -t cedardevs/onestop-sme:latest
+```
+
+## Test Execution
+To execute the onestop-python-client tests via python's unittest execute this from the onestop-python-client directory:
+
+```
+python3 -m unittest discover
+```
+
+If you wish to run a specific test file, here's an example:
+
+```
+python -m unittest test/unit/util/test_S3MessageAdapter.py 
+```

docs/cli/README.md

@@ -0,0 +1,7 @@
+<div align="center"><a href="/onestop-clients/">Documentation Home</a></div>
+<hr>
+
+## Table of contents
+* [Public User Quickstart](public-user-quickstart)
+* [SCDR Files](scdr-files)
+* [Developer](developer-quickstart)

docs/cli/developer/quickstart.md → docs/cli/developer-quickstart.md

@@ -1,4 +1,9 @@
-# OneStop CLI tool for developers
+<div align="center"><a href="/onestop-clients/">Documentation Home</a></div>
+<hr>
+# OneStop CLI tool for Developers
+
+## Table of Contents
+
 
 The `onestop-cli` tool provides a convenient command line interface for the OneStop search API. This tool is partly generated from the OpenAPI spec in the search module. We have added custom middleware for convenient syntax for frequently used filters and queries.
 

docs/cli/public-user/quickstart.md → docs/cli/public-user-quickstart.md

@@ -1,13 +1,35 @@
+<div align="center"><a href="/onestop-clients/">Documentation Home</a></div>
+<hr>
 # OneStop CLI tool
 
+## Table of Contents
+* [OneStop Info](#onestop-info)
+* [Installation](#installation)
+    * [Requirements](#requirements)
+    * [Install and run using a docker container (golang not required)](#install-and-run-using-a-docker-container-golang-not-required)
+    * [Download and use as Go package](#download-and-use-as-go-package)
+* [Configuration](#configuration)
+* [Usage](#usage)
+    * [Verbose](#verbose)
+    * [Run against a test or local API](#run-against-a-test-or-local-api)
+    * [Get](#get)
+    * [Search](#search)
+        * [by identifier](#by-identifier)
+        * [by parent identifier](#by-parent-identifier)
+        * [by date](#by-date)
+        * [by geometry](#by-geometry)
+        * [combinations](#combinations)
+
 The `onestop-cli` tool provides a convenient command line interface for the OneStop search API.
+There are three primary methods to search the data contained within onestop via the command line interface (CLI) tool.
+If you're new to the OneStop ecosystem, you should interact with one or both of the running instances of the UI to get some familiarity with basic search features. Then if you choose to be more of a power user, we encourage you to explore the ins and outs of our API and CLI.
 
-Read the [OneStop OpenAPI spec 2.0.0](https://app.swaggerhub.com/apis/cedardevs/one-stop_search_api/2.0.0).
-Or check the [OneStop OpenAPI spec 2.4.0](https://app.swaggerhub.com/apis/cedarbot/OneStop/2.4.0).
+## OneStop Info
+* [OneStop OpenAPI spec 2.0.0](https://app.swaggerhub.com/apis/cedardevs/one-stop_search_api/2.0.0)
+* [OneStop OpenAPI spec 2.4.0](https://app.swaggerhub.com/apis/cedarbot/OneStop/2.4.0)
 
 ## Installation
-
-### Requirements -
+### Requirements
 
 Either golang, or docker. Direct downloads of binaries will be available in the future.
 
@@ -157,6 +179,3 @@ Longhand query, including the `--verbose` flag to provide more logging:
 For complex query and filter structure, refer to the [short hand documentation](https://github.com/danielgtaylor/openapi-cli-generator/tree/master/shorthand).
 
 Note: As it is now, you cannot combine the flags with json shorthand. e.g. This will not work - `onestop searchcollection --area="POLYGON(( 22.686768 34.051522, 30.606537 34.051522, 30.606537 41.280903,  22.686768 41.280903, 22.686768 34.051522 ))" --query="satellite" filters[]{ type:datetime, after:2017-01-01T00:00:00Z, before:2017-02-01T00:00:00Z} `
-
-<hr>
-<div align="center"><a href="/onestop/public-user">Previous</a> | <a href="#">Top of Page</a> | <a href="/onestop/public-user/cli/scdr-files">Next</a></div>

docs/cli/public-user/scdr-files.md → docs/cli/scdr-files.md

@@ -4,8 +4,6 @@
 
 ## scdr-files configurations
 
-Config locations -
-
 Users can supply a configuration to map scdr-file type short names to OneStop IDs. See default config in [default config](cli/scdr-files-config.yaml). File can be yaml or json, but must be named "scdr-files-config" and placed in one of the following locations-  project directory, current working directory, /etc/scdr-files/, or $HOME/.scdr-files.
 
 

docs/helm.md

@@ -0,0 +1,43 @@
+<div align="center"><a href="/onestop-clients/">Documentation Home</a></div>
+<hr>
+
+# Helm
+
+## Table of Contents
+* [Intro](#intro)
+* [Helm Configuration](#helm-configuration)
+* [Create and Start the Script Container](#create-and-start-the-script-container)
+
+## Intro
+This project has a helm directory which is set up to pull a onestop-python-client image (specified in the image section in `helm/<chart directory>/values.yml`) and create a kubernetes container with that image inside. The container should be able to communicate to the configured OneStop stack (specified in the conf section in `helm/<chart directory>/values.yml`).
+ It also copies the onestop-python-client and scripts directories into the container.
+
+## Helm Configuration
+The helm charts are setup to create a configuration file from the template at `helm/<chart directory>/values.yml` and copy it to `/etc/config/config.yml` within the container. You don't have to use this file but most likely one will be necessary in a location where the scripts can access it.
+
+Please see the [onestop-python-client configuration](onestop-python-client#configuration) section for configuration information.
+
+Please see the [scripts](scripts) documentation for information on how to pass in a configuration file via CLI and execute the scripts.
+
+## Create and Start the Script Container
+The helm install command, done from the root of this repository, will use the charts in the helm directory to create the specified container.
+
+In this example we will create the `sme` using the helm charts and configuration information in this repo from `helm/onestop-sqs-consumer`
+1. cd to the root of this project
+1. `helm uninstall sme`
+1. `helm install sme helm/onestop-sqs-consumer`
+
+To check the container status execute `kubectl get pods` and look for the pod with the expected name, as defined by the `name` field in the `helm/<chart directory>/Chart.yaml`:
+
+```
+(base) ~/repo/onestop-clients 07:00 PM$ kubectl get pods
+NAME                                        READY   STATUS    RESTARTS   AGE
+sme-onestop-sqs-consumer-5c678675f7-q2s7h   0/1     Pending   0          26s
+```
+If it isn't in a 'Running' state within about 10 seconds then something is probably wrong. If it hasn't crashed yet (indicated by a STATUS of CrashBackLoop) then one possibility is a connection timeout trying to connect to a resource.
+
+Once the container is running you can exec into the container (much like "sshing") via this command, use the NAME from the `kubectl get pods` command:
+
+```
+kubectl exec --stdin --tty sme-onestop-sqs-consumer-5c678675f7-q2s7h -- /bin/bash
+```