Merge branch 'master' into v3.0

In support of the "Aloe" update to the jobs service APIs.

Author: johnfonner

.gitignore

@@ -3,4 +3,5 @@
 
 # test data
 tests/*.log
-tests/agave_mock_server/*.egg-info
+tests/agave_mock_server/*.egg-info
+._DS_Store

CHANGELOG.md

@@ -1,10 +1,20 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## 3.0.1 - 2019-04-23
+### ADDED
+- Updated documentation to reflect transition to api.tacc.utexas.edu
+
+### CHANGED
+- Revised references to agaveapi.co domains
+
+### REMOVED
+- Support for requestbin and json-mirror
+
 ## 3.0.0 - 2018-12-13
 ### ADDED
-- Documentationi.
-- TACC/agavepy as dependency.
+- Updated documentation.
+- TACC/agavepy as dependency
 
 ### CHANGED
 - Rewrite authentication-related scripts.
@@ -63,7 +73,7 @@ All notable changes to this project will be documented in this file.
 
 ## 2.1.8.1 - 2016-07-13
 ### ADDED
-- `Dockerfile` to build a minimal image with embedded webhook server written in Golang and `ngrok.com` reverse tunnel for local webhook inspection behind a proxy. 
+- `Dockerfile` to build a minimal image with embedded webhook server written in Golang and `ngrok.com` reverse tunnel for local webhook inspection behind a proxy.
 
 ### FIXED
 - Fixed a bug in `systems-roles-addupdate` where the curl statement was misprinting.
@@ -77,7 +87,7 @@ All notable changes to this project will be documented in this file.
 - `jobs-kick` command to roll a job back to its previous status and retry processing from there.
 - Added search support to the `monitors-checks-list` command.
 - Added support for specify custom notification delivery retry policy in the `notifications-addupdate` script.
-- Adding support for strict validation of inputs and paramters in the `jobs-resubmit` script. You can now use the `-I` and `--strictinputs` to enforce strict reproducibility on the submissi    on syntax and `-P` and  `--strictparams` to enforce strict reproducibility on the input syntax. Thes allow you to shield yourself against changes to the app descriptions that would otherwise silently go through. 
+- Adding support for strict validation of inputs and paramters in the `jobs-resubmit` script. You can now use the `-I` and `--strictinputs` to enforce strict reproducibility on the submissi    on syntax and `-P` and  `--strictparams` to enforce strict reproducibility on the input syntax. Thes allow you to shield yourself against changes to the app descriptions that would otherwise silently go through.
 
 ### FIXED
 - Fixed a bug in `monitors-addupdate` preventing json descriptions from being read from stdin.
@@ -116,7 +126,6 @@ All notable changes to this project will be documented in this file.
 ## 2.1.0 - 2015-02-23
 ### ADDED
 - `*-addupdate` Adding support for reading from stdin to all the scripts that previously accepted only files by replacing the file name with `-`
-- `jobs-template` adding webhook url with [Agave RequestBin](http://requestbin.agaveapi.co/) valid for 24 hours created on each request.
 - `jobs-template` added support for parsing most enum values, properly creating arrays vs primary types based on min and max cardinality, and the ability to populate with random default values, including inputs
 
 ### FIXED

Dockerfile.public

@@ -0,0 +1,74 @@
+FROM ubuntu:cosmic
+
+LABEL org.label-schema.vendor = "Texas Advanced Computing Center"
+LABEL org.label-schema.name = "TACC Tapis API CLI"
+LABEL org.label-schema.vcs-url = "https://github.com/TACC-Cloud/home"
+LABEL org.label-schema.vcs-ref = "master"
+LABEL org.label-schema.version = "3.0.1"
+LABEL org.label-schema.organization=tacc.cloud
+LABEL cloud.tacc.project="Tapis API CLI"
+
+ARG AGAVEPY_BRANCH=v0.8.0
+ARG PYTHON_PIP_VERSION=9.0.3
+
+#### This is a temporary fix for an issue with OverlayFS on TACC systems ###
+RUN mkdir -p /work && chown root:root /work
+RUN mkdir -p /gpfs && chown root:root /gpfs
+RUN mkdir -p /data && chown root:root /data
+RUN mkdir -p /scratch && chown root:root /scratch
+
+# In-container volume mounts
+# (For containers launched using TACC.cloud SSO identity)
+ARG CORRAL=/corral
+ARG STOCKYARD=/work/projects
+
+# Basics
+RUN apt-get -y update && \
+    apt-get -y install --no-install-recommends \
+    apt-utils bash curl dialog git vim rsync && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# jq for parsing JSON
+RUN curl -L -sk -o /usr/local/bin/jq "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" \
+    && chmod a+x /usr/local/bin/jq
+
+RUN apt-get -y update && \
+    apt-get -y install --no-install-recommends \
+    python3 \
+    python3-pip \
+    python3-setuptools && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Make python3 the default user python
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+RUN pip3 install --upgrade pip==${PYTHON_PIP_VERSION}
+RUN pip3 install --upgrade virtualenv==16.0.0
+
+# Make pip behave
+RUN curl -skL -o /etc/pip.conf \
+    https://raw.githubusercontent.com/SD2E/base-images/master/languages/python3/files/pip.conf
+
+# AgavePy
+WORKDIR /tmp
+RUN git clone https://github.com/TACC/agavepy \
+    && cd agavepy \
+    && git fetch --all && \
+    git checkout ${AGAVEPY_BRANCH} && \
+    python setup.py install --quiet
+
+RUN mkdir -p /home
+WORKDIR /home
+
+COPY bin /home/bin
+COPY completion/agave-cli /etc/bash_completion.d/agave-cli
+COPY etc /home/etc
+ENV PATH $PATH:/home/bin
+ENV HOME /home
+ENV AGAVE_CACHE_DIR=/home/.agave
+
+# Set command prompt
+RUN echo 'source /home/etc/.dockerprompt' >> /home/.bashrc
+
+CMD ["/bin/bash"]

Makefile

@@ -40,3 +40,6 @@ shell: build
 
 clean:
 	make -C docs clean
+
+public-image:
+	docker build $(DOCKER_BUILD_ARGS)  -f Dockerfile.public -t tacc/tapis-cli:latest .

README.md

@@ -6,22 +6,22 @@ Agave is an open platform-as-a-service built by TACC designed for scalable and
 reproducible research that integrates HPC resources which can be managed
 through a single API.
 Agave forms part of a thriving community that empowers users to run code in a
-reproducible manner, manage data efficiently, collaborate meaningfully, and 
-integrate easily with third-party applications. 
-TACC operates a professionally managed and supported instance of Agave as part 
+reproducible manner, manage data efficiently, collaborate meaningfully, and
+integrate easily with third-party applications.
+TACC operates a professionally managed and supported instance of Agave as part
 of its TACC.cloud Platform.
 
 
 ## What is the Agave CLI?
 
 The Agave command line interface (CLI) is a client for interacting with the
-Agave platform. 
-The CLI empowers you to streamline common interactions with the API and 
-automating repetitive and/or background tasks. Many developers and analysts 
-use it as their primary interface to TACC and other computing environments. 
-By default, this distribution of the CLI is configured to work with 
-TACC-hosted instances of Agave, but it can easily be adapted to support 
-on-premises or other hosted Agave installations. 
+Agave platform.
+The CLI empowers you to streamline common interactions with the API and
+automating repetitive and/or background tasks. Many developers and analysts
+use it as their primary interface to TACC and other computing environments.
+By default, this distribution of the CLI is configured to work with
+TACC-hosted instances of Agave, but it can easily be adapted to support
+on-premises or other hosted Agave installations.
 
 
 ## Installation
@@ -30,7 +30,7 @@ on-premises or other hosted Agave installations.
 tooling,bug fixes, and rewrites are done in Python 3. The Python scripts make
 use of [TACC/agavepy](https://github.com/TACC/agavepy).
 
-To use `TACC/agave-cli` you'll need the following dependencies. 
+To use `TACC/agave-cli` you'll need the following dependencies.
 
 	* Bash 3.2.50+
 	* curl 7.2+ with TLS support
@@ -71,29 +71,28 @@ echo "export PATH=$PATH:$PWD/agave-cli/bin" >> ~/.bash_profile
 
 ## Getting started
 
-To get more details 
+To get more details
 [see the documentation for the latest changes](docs/docsite).
-For more oficial documentation see
+For more official documentation see
 [Agave](https://tacc-cloud.readthedocs.io/projects/agave/en/latest/).
 
 The first time you use the CLI, you will need to create a session.
 You will need to create credentials to interact with a tenant.
 The credentials will, by default, be store in `~/.agave/`.
 
 ```
-$ auth-session-init 
-ID                   NAME                                     URL                                               
-3dem                 3dem Tenant                              https://api.3dem.org/                             
-agave.prod           Agave Public Tenant                      https://public.agaveapi.co/                       
-araport.org          Araport                                  https://api.araport.org/                          
-designsafe           DesignSafe                               https://agave.designsafe-ci.org/                  
-iplantc.org          CyVerse Science APIs                     https://agave.iplantc.org/                        
-irec                 iReceptor                                https://irec.tenants.prod.tacc.cloud/             
-portals              Portals Tenant                           https://portals-api.tacc.utexas.edu/              
-sd2e                 SD2E Tenant                              https://api.sd2e.org/                             
-sgci                 Science Gateways Community Institute     https://sgci.tacc.cloud/                          
-tacc.prod            TACC                                     https://api.tacc.utexas.edu/                      
-vdjserver.org        VDJ Server                               https://vdj-agave-api.tacc.utexas.edu/            
+$ auth-session-init
+ID                   NAME                                     URL
+3dem                 3dem Tenant                              https://api.3dem.org/
+araport.org          Araport                                  https://api.araport.org/
+designsafe           DesignSafe                               https://agave.designsafe-ci.org/
+iplantc.org          CyVerse Science APIs                     https://agave.iplantc.org/
+irec                 iReceptor                                https://irec.tenants.prod.tacc.cloud/
+portals              Portals Tenant                           https://portals-api.tacc.utexas.edu/
+sd2e                 SD2E Tenant                              https://api.sd2e.org/
+sgci                 Science Gateways Community Institute     https://sgci.tacc.cloud/
+tacc.prod            TACC                                     https://api.tacc.utexas.edu/
+vdjserver.org        VDJ Server                               https://vdj-agave-api.tacc.utexas.edu/
 
 Please specify the ID for the tenant you wish to interact with:
 ```

VERSION

@@ -1 +1 @@
-2.3.0
+3.0.1

bin/auth-session-init

@@ -2,7 +2,7 @@
 """
     auth-session-init [OPTION]
 
-Creates a new API client application and generates a set of API keys for the user.
+Creates a new API client application and generates a set of API keys.
 
  Options:
     -c, --cachedir     Directory to save confiurations in.

bin/jobs-stop

@@ -54,13 +54,14 @@ main() {
 	if [ -z "$args" ]; then
 		err "Please specify a job id to stop"
 	else
-		cmd="curl -sk -H \"${authheader}\" -X POST -d \"action=stop\" '$hosturl$args?pretty=true'"
+		cmd="curl -sk -H \"${authheader}\" -X POST  -H \"Content-Type: application/json\" --data '{\"action\":\"stop\"}' '$hosturl$args?pretty=true'"
+
 
 		if ((veryverbose)); then
 			[ "$piped" -eq 0 ] && log "Calling $cmd"
 		fi
 
-		response=`curl -sk -H "${authheader}" -X POST -d "action=stop" "$hosturl$args?pretty=true"`
+		response=`curl -sk -H "${authheader}" -X POST  -H "Content-Type: application/json" --data '{"action":"stop"}' "$hosturl$args?pretty=true"`
 
 		if [[ $(jsonquery "$response" "status") = 'success' ]]; then
 			result=$(format_api_json "$response")

bin/jobs-submit

@@ -87,13 +87,13 @@ main() {
     # standard file upload
     else
 
-      cmd="curl -sk -H \"${authheader}\" -X POST -d \"fileToUpload=@$filetoupload\" '$hosturl?pretty=true'"
+      cmd="curl -sk -H \"${authheader}\" -H \"Content-Type: application/json\" -X POST -d \"@$filetoupload\" '$hosturl?pretty=true'"
 
       if ((veryverbose)); then
         [ "$piped" -eq 0 ] && log "Calling $cmd"
       fi
 
-      response=`curl -sk -H "${authheader}" -X POST -F "fileToUpload=@$filetoupload" "$hosturl?pretty=true"`
+      response=`curl -sk -H "${authheader}" -H "Content-Type: application/json" -X POST -d "@$filetoupload" "$hosturl?pretty=true"`
 
     fi
 

bin/jobs-template

@@ -465,7 +465,7 @@ main() {
       if ((verbose)); then
         [ "$piped" -eq 0 ] && log "Creating a new webhook url for your job..."
       fi
-      requestbin=$(requestbin-create -v)
+      equestbin=$(requestbin-create -v)
       requestbin_id=$(jsonquery "$requestbin" "name")
       webhookurl="https://requestbin.agaveapi.co/${requestbin_id}?job_id=\${JOB_ID}&status=\${JOB_STATUS}"
 
@@ -494,7 +494,7 @@ main() {
     if ((allfields)); then
       printf "{\n  \"name\":\"%s\",\n  \"appId\": \"%s\",\n  \"batchQueue\": \"%s\",\n  \"executionSystem\": \"%s\",\n  \"maxRunTime\": \"%s\",\n  \"memoryPerNode\": \"%sGB\",\n  \"nodeCount\": %s,\n  \"processorsPerNode\": %s,\n  \"archive\": %s,\n  \"archiveSystem\": \"%s\",\n  \"archivePath\": %s,\n  \"inputs\": {%s\n  },\n  \"parameters\": {%s\n  },\n  \"notifications\": [\n%s\n  ]\n}\n" "$job_name" "$args" "$queue" "$execution_system_id" "$maxRunTime" "$memoryPerNode" "$nodeCount" "$processorsPerNode" "$archive" "$archiveSystem" "$archivePath" "$inputs" "$parameters" "$notifications"
     else
-      printf "{\n  \"name\": \"%s\",\n  \"appId\": \"%s\",\n  \"archive\": %s,\n  \"inputs\": {%s\n  },\n  \"parameter\": {%s\n  }\n}\n" "$job_name" "$args" "$archive" "$inputs" "$parameters"
+      printf "{\n  \"name\": \"%s\",\n  \"appId\": \"%s\",\n  \"archive\": %s,\n  \"inputs\": {%s\n  },\n  \"parameters\": {%s\n  }\n}\n" "$job_name" "$args" "$archive" "$inputs" "$parameters"
     fi
 
     if ((verbose)); then