Merge pull request #373 from common-workflow-language/staging

2023  release

Author: mr-c

.github/workflows/ci-tests.yml

@@ -0,0 +1,44 @@
+name: CI Tests
+
+# Migrated from .travis.yml, based on cwltool's ci-tests.yml.
+# See .travis.yml in git history for previous changes and configurations.
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+
+concurrency:
+  group: build-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+
+  ci_tests:
+    name: CI Tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        py-ver-major: [ 3 ]
+        py-ver-minor: [ 9, 10, 11 ]
+
+    env:
+      py-semver: ${{ format('{0}.{1}', matrix.py-ver-major, matrix.py-ver-minor) }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.py-semver }}
+          cache: pip
+          cache-dependency-path: |
+            **/setup.cfg
+
+      - run: pip install -U pip setuptools wheel typing
+      - run: pip install -e .[all]
+
+      - run: make RUNNER=cwltool unittest-examples

.github/workflows/codeql.yml

@@ -0,0 +1,42 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main", "gh-pages", "release" ]
+  pull_request:
+    branches: [ "main" ]
+  schedule:
+    - cron: "59 14 * * 5"
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ javascript, python ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          queries: +security-and-quality
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+        if: ${{ matrix.language == 'javascript' || matrix.language == 'python' }}
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "/language:${{ matrix.language }}"

.travis.yml

@@ -12,7 +12,7 @@
             "name": "Andrew Jesaitis"
 	},
 	{
-            "name": "bamiwoaluko"
+            "name": "Bamiwo Aluko"
 	},
 	{
             "orcid": "https://orcid.org/0000-0002-6206-4638",

.zenodo.json

@@ -1,7 +1,7 @@
 Thank you to the following contributors (in alphabetical order by user name):
 * Alain Domissy (@alaindomissy)
 * Andrew Jesaitis (@andrewjesaitis)
-* @bamiwoaluko
+* Bamiwo Aluko (@bamiwoaluko)
 * Benjamin Carr (@BenjaminHCCarr) <https://orcid.org/0000-0002-6206-4638>
 * Michele Mastropietro (@elehcim) <https://orcid.org/0000-0002-6324-5713>
 * Emannuel Fernandes de Oliveira Carvalho (@emannuelOC)

AUTHORS.md

@@ -106,25 +106,29 @@ The `dot` program from Graphviz is needed to render some of the diagrams.
 
 * For Debian/Ubuntu users:
 ```bash
-sudo apt get install graphviz
+sudo apt-get install graphviz
 ```
 * For non-Debian/Ubuntu users, follow the directions at [the GraphViz download site](https://graphviz.org/download).
 ```bash
 # Create and activate a virtual environment
-python -m venv venv
+python3 -m venv venv
 source venv/bin/activate
 # update the version of pip, setuptools, and wheel
 (venv) pip install -U pip setuptools wheel
 # Install all the dependencies in your virtual environment.
 (venv) pip install ".[all]"
 # Create the HTML to visualize locally
 (venv) make html
-(venv) open _build/index.html
+(venv) open _build/html/index.html
 # Or you can start a serve that watches for local file changes
 (venv) make watch
 # Open <http://localhost:8000/> in your browser
 ```
 
+You can use `udocker` or any other Docker compatible engine to build the
+user guide, by setting the following environment variable:
+`CWLTOOL_OPTIONS=--udocker`, or `CWLTOOL_OPTIONS=--podman` for Podman, or `CWLTOOL_OPTIONS=--singularity` for Singularity/Apptainer.
+
 > NOTE: When you modify the packages installed with apt or pip, please verify
 > if the change needs to be applied to either or to both of CI and readthedocs.
 > ReadTheDocs builds and deploys previews. CI builds and deploys production
@@ -148,7 +152,10 @@ are still working after the change.
 Use “tool description” not “tool wrapper” for describing the first argument
 given to the `cwl-runner` or `cwltool` commands.
 
-### Code examples
+
+### Code Examples
+
+All CWL documents should be version 1.2 unless there is a very good reason otherwise.
 
 To include code into a Markdown file you have two options. For external files use
 the following command:
@@ -182,6 +189,34 @@ If you are having trouble with links to sections or code blocks, it might
 be due to duplicated sections, or to spaces or other characters. To
 preview the generated links, use the `myst-anchors` tool.
 
+### Troubleshooting
+
+General advice for troubleshooting is to increase the verbosity of Sphinx
+logs. When using `make`, change your command to the following pattern:
+`make SPHINXOPTS="-vvv" html`. Or if you are using `sphinx-build`, then
+use the following: `sphinx-build -vvv -b html src build/`.
+
+It is also worth removing build directories and files such as `_build`, `build`,
+and [`~/.cache/salad`](https://github.com/common-workflow-language/user_guide/issues/268#issuecomment-1276068865).
+You may want to start afresh with a new Python virtual environment,
+installing the dependencies from scratch.
+
+If running `make html` or `make watch` appears to run fine but [gets stuck
+](https://github.com/common-workflow-language/user_guide/issues/268) at some
+part of the process (`10%`, `33%`, `57%`, etc.) for longer than 2-5 minutes
+you may have to try the following:
+
+1. Check if the Python dependencies installed in your Python virtual environment
+   have updates available (you can re-create the virtual environment, use `--upgrade`
+   for `pip`, or manually verify in PyPI and `pip`).
+2. Use `top` or `htop` to find out if there is any `cwltool` command running. Then
+   try running that command individually to see what is the error message.
+3. Upgrade your version of `docker` if you have [an error message](
+   https://github.com/docker/compose/issues/8121#issuecomment-806055733) similar to
+   `docker: Failed to create the container ID file: open /tmp/....cid: no such file or directory.`.
+4. Try using `udocker` instead of `docker`, by using `pip` to install it, and setting
+   `CWLTOOL_OPTIONS` to `--udocker`.
+
 ## Other Resources
 
 General discussion of [Common Workflow Language][cwl-site] project

CONTRIBUTING.md

@@ -16,9 +16,9 @@ You are free:
 for any purpose, even commercially.
 
 The licensor cannot revoke these freedoms as long as you follow the
-license terms.
+license terms:
+
 
-Under the following terms:
 
 * **Attribution**---You must give appropriate credit (mentioning that
   your work is derived from work that is Copyright © the Common Workflow
@@ -28,16 +28,14 @@ Under the following terms:
   so in any reasonable manner, but not in any way that suggests the
   licensor endorses you or your use.
 
-**No additional restrictions**---You may not apply legal terms or
+* **No additional restrictions**---You may not apply legal terms or
 technological measures that legally restrict others from doing
 anything the license permits.  With the understanding that:
 
-Notices:
-
-* You do not have to comply with the license for elements of the
+  * You do not have to comply with the license for elements of the
   material in the public domain or where your use is permitted by an
   applicable exception or limitation.
-* No warranties are given. The license may not give you all of the
+  * No warranties are given. The license may not give you all of the
   permissions necessary for your intended use. For example, other
   rights such as publicity, privacy, or moral rights may limit how you
   use the material.
@@ -55,7 +53,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
-## Trademark
+
 
 
 [cc-by-human]: https://creativecommons.org/licenses/by/4.0/

LICENSE.md

@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line, and also
 # from the environment for the first two.
-SPHINXOPTS		= "-W"
+#SPHINXOPTS		=   # was "-W" (turn warnings into errors)
 SPHINXBUILD		= sphinx-build
 SOURCEDIR		= src
 BUILDDIR		= _build
@@ -25,15 +25,17 @@ watch: clean
 	@echo
 	@echo "Building and watching for changes in the documentation."
 	sphinx-autobuild "$(SOURCEDIR)" "$(BUILDDIR)" \
+			-b html \
 			--ignore='**venv' \
 			--ignore='**.github' \
 			--ignore='*.egg-info' \
 			--ignore='**_includes/**/*.txt' \
-			--watch='cwl'
+			--watch='cwl' \
+			$(SPHINXOPTS) $(O)
 
 ## unittest-examples	:
 unittest-examples:
-	cd src/_includes/cwl; cwltest --test=conformance-test.yml --tool=${RUNNER}
+	cd src/_includes/cwl; cwltest --test=cwl_tests.yml --tool=${RUNNER}
 
 ## check-json			:
 check-json:
@@ -42,6 +44,10 @@ check-json:
 container-pull:
 	for container in $$(git grep dockerPull $$(git ls-files *.cwl) | awk '-F: ' '{print $$3}'); do docker pull $${container}; done
 
+update_translations: gettext
+	sphinx-intl update -p _build/gettext
+
+
 .PHONY: help clean watch unittest-examples check-json Makefile
 
 # Catch-all target		: route all unknown targets to Sphinx using the new

Makefile

@@ -1,6 +1,8 @@
 [![DOI for the latest version](https://zenodo.org/badge/89621457.svg)](https://zenodo.org/badge/latestdoi/89621457)
 
-[![Syntax Check](https://app.travis-ci.com/common-workflow-language/user_guide.svg?branch=main)](https://app.travis-ci.com/common-workflow-language/user_guide)
+[![CI Tests](https://github.com/common-workflow-language/user_guide/actions/workflows/ci-tests.yml/badge.svg?branch=main)](https://github.com/common-workflow-language/user_guide/actions/workflows/ci-tests.yml)
+
+[![Translation status](https://hosted.weblate.org/widgets/commonwl/-/user-guide/svg-badge.svg)](https://hosted.weblate.org/engage/commonwl/)
 
 This is the source of the official user guide for the Common Workflow Language standards.
 

README.md

@@ -40,10 +40,13 @@ class Singleton(_Singleton("SingletonMeta", (object,), {})):
 
 class CMDCache(Singleton):
     cache = {}
+    exclude_cache_cmd = {hash("cat output.txt")}
 
     def get(self, cmd, working_directory):
         h = hash(cmd)
-        if h in self.cache:
+        if h in self.exclude_cache_cmd:
+            return run_command(cmd, working_directory)
+        elif h in self.cache:
             return self.cache[h]
         else:
             result = run_command(cmd, working_directory)