Merge branch 'main' into fix-env-type

Author: andre-merzky

.github/workflows/build-website.yml

@@ -2,7 +2,7 @@ name: build-website
 
 on:
   push:
-    branches: [ main, web_site ]
+    branches: [ main, web_site, 'releases/**' ]
 
 jobs:
   publish:

Makefile

@@ -79,10 +79,3 @@ install:
 .PHONY: develop
 develop:
 	$(PYTHON) setup.py develop
-
-
-.PHONY: tag-and-release
-tag-and-release: ## create a tag in git. to run, do a 'make VERSION="version string" tag-and-release
-	./tag_and_release.sh create_tag $(VERSION)
-	./tag_and_release.sh package
-	./tag_and_release.sh release

RELEASE

@@ -1 +1 @@
-0.9.0
+0.9.3

docs/_static/extras.js

@@ -67,7 +67,6 @@ function detectAll(selectorType) {
     // simple_value_2. However, if either "B" or "C" are selected in either selector,
     // we want to change both.
 
-    console.log("type:" + selectorType);
     $("p." + selectorType + "-selector").addClass(selectorType + "-item");
     var prevSpans = [];
     $("span").each(function() {
@@ -91,6 +90,28 @@ function detectAll(selectorType) {
                 $(this).text("PROJECT_NAME");
             }
         }
+        if (text == '_get_executor_instance') {
+            // replace _get_executor_instance(execparams, job) with selector value
+            // the spans are: "_get_executor_instance", "(", "execparams",   ",", "job",    ")"
+            // we need:       "JobExecutor",            ".", "get_instance", "(", selector, ")"
+            // the classes happen to match, so just replace the text
+            // also, remove the space after the comma, which is rendered by sphinx as a text
+            // node rather than span like everything else
+            var crt = $(this);
+            var space = $(this).next().next().next().get(0).nextSibling;
+            if (space.nodeType == 3) {
+                space.nodeValue = "";
+            }
+            else {
+                console.log("Expected a text node: ", space);
+            }
+            var newText = ["JobExecutor", ".", "get_instance", "("];
+            for (var i = 0; i < 4; i++) {
+                crt.text(newText[i]);
+                crt = crt.next();
+            }
+            crt.addClass(selectorType + "-item").addClass("psij-selector-value");
+        }
         prevSpans.push($(this));
         if (prevSpans.length > 2) {
             prevSpans.shift();

docs/user_guide.rst

@@ -254,7 +254,7 @@ processes on each node, for a total of 8 processes:
 .. literalinclude:: ../tests/user_guide/test_resourcespec.py
     :language: python
     :dedent: 4
-    :lines: 9-17
+    :lines: 8-16
 
 .. note::
     All processes of a job will share at most one MPI communicator
@@ -332,7 +332,7 @@ and an instance of the
 .. literalinclude:: ../tests/user_guide/test_scheduling_information.py
     :language: python
     :dedent: 4
-    :lines: 11-19
+    :lines: 10-18,21-22
 
 where `QUEUE_NAME` is the LRM queue where the job should be sent and
 `PROJECT_NAME` is a project/account that may need to be specified for

psij-ci-setup

@@ -57,7 +57,17 @@ cron_install() {
     echo "The following line will be installed in your crontab:"
     echo "$LINE"
     echo "================================================================"
-    { crontab_list && echo "$LINE"; } | crontab -
+
+    CRT_CRONTAB=`crontab -l 2>&1`
+    if [ "$?" != "0" ]; then
+        if echo "$CRT_CRONTAB" | grep "no crontab for" >/dev/null 2>&1 ; then
+            CRT_CRONTAB=""
+        else
+            echo "Error updating crontab: $CRT_CRONTAB"
+            exit 2
+        fi
+    fi
+    { echo "$CRT_CRONTAB"; echo "$LINE"; } | crontab -
 
     declare -x | egrep -v "^declare -x (BASH_VERSINFO|DISPLAY|EUID|GROUPS|SHELLOPTS|TERM|UID|_)=" >psij-ci-env
 }
@@ -249,6 +259,7 @@ check_key() {
                     read -n1 RESPONSE
                     RESPONSE=${RESPONSE^}
                 done
+                mkdir -p ~/.psij
                 echo "$KEY" > ~/.psij/key
                 chmod 600 ~/.psij/key
                 DONE=1

release.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+# This script does the following:
+# - does some sanity checks
+# - updates RELEASE and src/psij/version.py
+# - commits and pushes the version files
+# - creates a release tag
+# - builds packages
+# - pushes said packages to PyPi
+# - triggers github workflows to rebuild the web page (including documentation)
+
+set -e
+
+error() {
+    echo $@
+    exit 1
+}
+
+if [ "$1" == "" ]; then
+    error "Missing required version argument"
+else
+    TARGET_VERSION="$1"
+fi
+
+# Ensure that we are on the main branch
+CRT_BRANCH=`git rev-parse --abbrev-ref HEAD`
+
+if [ "$CRT_BRANCH" != "main" ]; then
+    error "Must be on the main branch to make a release."
+fi
+
+STATUS=`git status --untracked-files=no --porcelain`
+
+# Ensure current dir is clean
+if [ ! -z "$STATUS" ]; then
+    error "Working directory is not clean. Please run this script on a clean checkout."
+    echo
+fi
+
+# Make sure version in newer than other releases
+TAGS=`git tag`
+if echo "$TAGS" | grep "$TARGET_VERSION" 2>&1 >/dev/null; then
+    error "Version $TARGET_VERSION is already tagged."
+fi
+
+LAST=`echo -e "$TAGS\n$TARGET_VERSION" | sed '/^\s*$/d' | sort -V | tail -n 1`
+if [ "$LAST" != "$TARGET_VERSION" ]; then
+    error "Version $TARGET_VERSION is lower than the latest tagged version ($LAST)."
+fi
+
+echo "This will tag and release psij-python to version $TARGET_VERSION."
+echo -n "Type 'yes' if you want to continue: "
+read REPLY
+
+if [ "$REPLY" != "yes" ]; then
+    error "Release canceled."
+fi
+
+# There are mutable operations from here on
+
+echo "Creating release branch"
+git checkout -b "releases/$TARGET_VERSION"
+
+# Update the two version files and push them
+echo "Updating version and tagging..."
+
+echo -n "$TARGET_VERSION" > RELEASE
+cat <<EOF >src/psij/version.py
+"""This module stores the current version of this library."""
+
+# Do not change this file manually. It is updated automatically by the release script.
+VERSION = '$TARGET_VERSION'
+EOF
+
+git commit -m "Updated version files to $TARGET_VERSION." RELEASE src/psij/version.py
+git push --set-upstream origin releases/$TARGET_VERSION
+
+git tag -a "$TARGET_VERSION" -m "Tagging $TARGET_VERSION"
+git push origin --tags
+
+echo "Building packages..."
+python3 setup.py sdist
+python3 setup.py bdist_wheel
+
+echo "Releasing PyPi package..."
+twine upload dist/*
+
+echo "Triggering web docs build..."
+git commit --allow-empty -m "Trigger web build for $TARGET_VERSION"
+git push
+
+echo "All done!"

src/psij/executors/batch/batch_scheduler_executor.py

@@ -9,7 +9,6 @@
 from threading import Thread, RLock
 from typing import Optional, List, Dict, Collection, cast, Union, IO
 
-from .escape_functions import bash_escape
 from psij.launchers.script_based_launcher import ScriptBasedLauncher
 
 from psij import JobExecutor, JobExecutorConfig, Launcher, Job, SubmitException, \
@@ -68,7 +67,7 @@ def _env_to_mustache(job: Job) -> List[Dict[str, str]]:
 
     r = []
     for k, v in job.spec.environment.items():
-        r.append({'name': k, 'value': bash_escape(v)})
+        r.append({'name': k, 'value': v})
     return r
 
 

src/psij/version.py

@@ -1,5 +1,4 @@
 """This module stores the current version of this library."""
 
-# Use PEP 440 for this
-#: This is a global constant with the PSI/J Python version number
-VERSION = '0.9.0'
+# Do not change this file manually. It is updated automatically by the release script.
+VERSION = '0.9.3'