Merge pull request #555 from MetRonnie/8.0.x-deconflict

8.0.x to master

Author: oliver-sanders

.github/actions/install-dependencies/action.yml

@@ -1,5 +1,5 @@
 name: Install dependencies
-description: apt-get dependencies
+description: Install dependencies and setup + activate conda env
 
 runs:
   using: composite
@@ -9,17 +9,30 @@ runs:
         echo "[command]sudo apt-get update"
         sudo apt-get update
 
+    # Explanation of these dependencies:
+    # - https://github.com/ImageMagick/ImageMagick/issues/374#issuecomment-279252866
+    # - aspell-en for spellchecking
     - shell: bash
       run: |
         echo "[command]sudo apt-get install -y ..."
         sudo apt-get install -y \
           librsvg2-bin \
           ghostscript \
-          graphviz \
           pkg-config \
           libgraphviz-dev \
           gsfonts \
           libfreetype6-dev \
-          libfontconfig1-dev
-      # Explanation of some of these dependencies:
-      # https://github.com/ImageMagick/ImageMagick/issues/374#issuecomment-279252866
+          libfontconfig1-dev \
+          aspell-en
+
+    - uses: conda-incubator/setup-miniconda@35d1405e78aa3f784fe3ce9a2eb378d5eeb62169 # v2.1.1
+      with:
+        channels: conda-forge
+        python-version: ${{ matrix.python-version || '' }}
+        miniforge-version: latest
+
+    # use login shell for conda activation
+    - shell: bash -leo pipefail {0}
+      run: |
+        echo "CONDA_PREFIX: ${CONDA_PREFIX}"
+        conda install -y graphviz

.github/workflows/deploy.yml

@@ -57,8 +57,12 @@ concurrency:
 
 jobs:
   deploy:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     timeout-minutes: 20
+    defaults:
+      run:
+        # use login shell for conda activation
+        shell: bash -leo pipefail {0}
     steps:
       - name: configure python
         uses: actions/setup-python@v4
@@ -120,7 +124,8 @@ jobs:
             html \
             slides \
             linkcheck \
-            SPHINXOPTS='-Wn' \
+            SPHINXOPTS='-Wn --keep-going' \
+            FORCE_COLOR=true \
             STABLE=${{ inputs.set_stable }} \
             LATEST=${{ inputs.set_latest }}
 

.github/workflows/nightly.yml

@@ -31,8 +31,12 @@ concurrency:
 
 jobs:
   deploy:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     timeout-minutes: 10
+    defaults:
+      run:
+        # use login shell for conda activation
+        shell: bash -leo pipefail {0}
     steps:
       - name: configure python
         uses: actions/setup-python@v4
@@ -105,10 +109,11 @@ jobs:
             html \
             slides \
             linkcheck \
-            SPHINXOPTS="-Wn -A sidebar_version_name=${version}" \
+            SPHINXOPTS="-Wn --keep-going -A sidebar_version_name=${version}" \
             STABLE=false \
             LATEST=false \
-            BUILDDIR='doc/nightly'
+            BUILDDIR='doc/nightly' \
+            FORCE_COLOR=true
           git -C gh-pages add 'nightly' 'versions.json'
 
       - name: push changes

.github/workflows/test.yml

@@ -25,8 +25,12 @@ concurrency:
 
 jobs:
   test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     timeout-minutes: 10
+    defaults:
+      run:
+        # use login shell for conda activation
+        shell: bash -leo pipefail {0}
     steps:
       - name: configure python
         uses: actions/setup-python@v4
@@ -47,7 +51,18 @@ jobs:
       - name: install cylc-doc
         run: pip install .[all]
 
-      - name: install libs to document
+      - name: install libs to document (8.0.x)
+        if: github.base_ref == '8.0.x'
+        uses: ./.github/actions/install-libs
+        with:
+          # NOTE: these need to be manually updated as necessary:
+          cylc-flow-tag: '8.0.x'
+          cylc-rose-tag: '1.1.x'
+          cylc-uis-tag: '1.1.x'
+          metomi-rose-tag: '2.0.x'
+
+      - name: install libs to document (other branches)
+        if: github.base_ref != '8.0.x'
         uses: ./.github/actions/install-libs
 
       - name: install eslint
@@ -59,11 +74,21 @@ jobs:
           flake8
           eslint .
 
-      - name: build
+      - name: (debug only) list language dictionaries
+        if: runner.debug
+        shell: python
+        env:
+          PYENCHANT_VERBOSE_FIND: true
+        run: |
+          import enchant
+          print(enchant.list_dicts())
+
+      - name: build & test
         run: |
-          make html slides spelling linkcheck doctest SPHINXOPTS='-Wn'
+          make html slides spelling linkcheck doctest \
+            SPHINXOPTS='-Wn --keep-going' FORCE_COLOR=true
 
-      - name: debug
+      - name: debug sphinx failure
         if: failure()
         run: |
           cat /tmp/sphinx-err* || true  # sphinx traceback

.github/workflows/undeploy.yml

@@ -25,15 +25,15 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: 'Tag present on both cylc-flow and cylc-docs.'
+        description: 'Tag present on both cylc-flow and cylc-doc'
         required: true
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
 
 jobs:
   undeploy:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     env:
       TAG: ${{ inputs.tag }}

src/7-to-8/caveats.rst

@@ -58,7 +58,7 @@ Multiple Selection
    * https://github.com/cylc/cylc-ui/issues/434
 Installing Workflows
    At present there is no way to view or install
-   :term:`source workflows <source directory>`) in the UI.
+   :term:`source workflows <source directory>` in the UI.
 Rose Edit
    Rose Edit is awaiting reimplementation in the UI.
 Trigger Edit

src/7-to-8/major-changes/cylc-install.rst

@@ -89,7 +89,7 @@ Remote Installation
 
 Remote file installation does not occur until running the workflow.
 When the first task runs on a remote platform, Cylc will transfer files from
-the :term:`run directory` to the install target.
+the :term:`run directory` to the :term:`install target`.
 
 If you have used Rose 2019, you may be used to all files and directories in
 the run directory being included.

src/7-to-8/major-changes/platforms.rst

@@ -6,46 +6,11 @@ Platforms
 .. admonition:: Does This Change Affect Me?
    :class: tip
 
-   .. cylc-scope:: flow.cylc
+   Cylc platforms are a new feature which replace the task ``job`` and
+   ``remote`` configuration sections:
 
-   Platforms replace the deprecated :cylc:conf:`[runtime][<namespace>][job]`
-   and :cylc:conf:`[runtime][<namespace>][remote]`
-   sections:
-
-   .. code-block:: cylc
-
-      [runtime]
-          [[foo]]
-              [[[job]]]
-                  batch system = slurm
-              [[[remote]]]
-                  host = my_supercomputer
-
-   .. cylc-scope::
-
-   Read this section if your workflow's jobs run on a remote computer or if
-   you see the following warning on running ``cylc validate``:
-
-   .. code-block:: console
-
-      WARNING - deprecated settings found (please replace with [runtime][foo]platform):
-          [runtime][foo][remote]host
-          [runtime][foo][job]batch system
-
-   If you currently use the ``rose host-select`` utility or a similar host
-   selection or load balancing utility the intelligent host selection
-   functionality of Cylc 8 may be used instead:
-
-   .. code-block:: cylc
-
-      [runtime]
-          [[task1]]
-              [[[remote]]]
-                  host = $(rose host-select my-computer)
-          [[task2]]
-              # An example of a home-rolled host selector
-              [[[remote]]]
-                  host = $(test $((RANDOM%2)) -eq 0 && echo "host_a" || echo "host_b")
+   * :cylc:conf:`[runtime][<namespace>][job]`
+   * :cylc:conf:`[runtime][<namespace>][remote]`
 
 
 Overview
@@ -57,23 +22,18 @@ Overview
    - The terms :term:`job runner` (in Cylc 8 configurations) and batch system
      (in Cylc 7 configurations) are equivalent.
 
-Cylc 7 defines settings for remote :term:`jobs <job>` in each
-:term:`task's <task>` definition.
+Submitting a job to a :term:`job runner` may require configuration.
 
-Cylc 8 allows site administrators (and users) to configure
-:term:`platforms <platform>` in ``global.cylc``. A platform can have
-multiple hosts with associated platform-specific settings. Users only need to
-select the platform for their task jobs.
+In Cylc 7 this configuration must be provided for each task in the workflow
+configuration (``suite.rc``).
 
-Platforms also define how hosts are selected from each platform:
-
-- Randomly (default)
-- By definition order
+In Cylc 8 "platforms" can be defined in the global configuration
+(:cylc:conf:`global.cylc`) so that this configuration doesn't have to be
+repeated for each task in each workflow.
 
 There may be cases where sets of platforms (for example a group of
 standalone compute servers, or a pair of mirrored HPC's) might be equally
-suitable for a task, but not share files systems to allow them to constitute
-a single platform. Such platforms can be set up to be ``platform groups``
+suitable for a task. Such platforms can be set up to be ``platform groups``
 
 .. seealso::
 
@@ -92,6 +52,20 @@ a single platform. Such platforms can be set up to be ``platform groups``
    Deprecated settings will be removed in a later release of Cylc.
 
 
+What is a Platform?
+-------------------
+
+A "platform" represents one or more hosts from which jobs can be submitted to or
+polled from a common job submission system.
+
+If a platform has multiple hosts Cylc will automatically select a host when
+needed and will fallback to other hosts if it is not contactable.
+
+A "platform group" represents a collection of independent platforms. Cylc will
+automatically select a platform and will fallback to other platforms in the
+group (for appropriate operations) if the platform is not contactable.
+
+
 Examples
 --------
 
@@ -194,7 +168,7 @@ At Cylc 8 the equivalent might be:
 
        [[mytask_login_to_hpc_and_submit]]
            # Recommended:
-           platform = just_run_it
+           platform = slurm_supercomputer
            # ...but This is still legal:
            #platform = $(selector-script)
 
@@ -210,13 +184,25 @@ And the platform settings for these examples might be:
            # A computer with PBS, that takes local job submissions
            job runner = pbs
            hosts = localhost
+           install target = localhost
 
        [[slurm_supercomputer]]
            # This computer with Slurm requires you to use a login node.
            hosts = login_node01, login_node02  # Cylc will pick a host.
            job runner = slurm
 
 
+Note that in these examples, it is assumed that ``linuxboxNN``, ``pbs_local`` and
+``slurm_supercomputer`` have distinct file systems.
+Sets of platforms which share a file system must specify
+a single :ref:`install target <Install Targets>`.
+
+.. note::
+   If an install target is not set, a platform will use its own platform name
+   as the install target name. If multiple platforms share a file system
+   but have separate :ref:`install targets <Install Targets>` task initialization
+   will fail.
+
 .. _host-to-platform-logic:
 
 How Cylc 8 handles host-to-platform upgrades
@@ -243,6 +229,7 @@ platforms section:
        [[supercomputer_A]]
            hosts = localhost
            job runner = slurm
+           install target = localhost
        [[supercomputer_B]]
            hosts = tigger, wol, eeyore
            job runner = pbs
@@ -257,7 +244,7 @@ And you have a workflow runtime configuration:
                batch system = slurm
        [[task2]]
            [[[remote]]]
-               hosts = eeyore
+               host = eeyore
            [[[job]]]
                batch system = pbs
 

src/7-to-8/major-changes/task-job-states.rst

@@ -14,11 +14,11 @@ the submission stage, real job scripts). A task can have multiple jobs, by
 automatic retries and manual re-triggering.
 
 Cylc 7 had 13 task/job states. The GUI only showed tasks, with job data
-from the latest task job.
+from the latest job.
 
 Cylc 8 has only 8 task/job states. The Cylc 8 UI shows both task and jobs.
 Task icons are monochrome circles; job icons are coloured squares. The running
 task icon incorporates a radial progress indicator.
 
 .. image:: ../../img/task-job.png
-   :align: center
+   :align: center

src/7-to-8/summary.rst

@@ -97,7 +97,7 @@ Task/Job States
 
 :term:`Tasks <task>` are nodes in the abstract workflow graph, representing
 applications to run at the appropriate point in the workflow. A :term:`job <job>`
-(or a *task job*) is the script (and subsequent process) submitted by Cylc to
+is the script (and subsequent process) submitted by Cylc to
 actually run the application. A task can have multiple jobs as the result of
 automatic retries or manual re-triggering.
 
@@ -146,7 +146,7 @@ Cylc 8 is aware of sets of host settings called
 
 Hosts of a platform must share a file system and :term:`job runner`:
 If one host is unavailable Cylc 8 can use other hosts
-on the same platform to interact with task jobs.
+on the same platform to interact with jobs.
 
 The same hosts can belong to multiple platforms, for example
 you might be able to use the same host to launch both background and Slurm
@@ -250,7 +250,7 @@ Queues
 Time Zones
    :cylc:conf:`[scheduler]cycle point time zone` now defaults to UTC, unless you
    are working in :ref:`cylc_7_compat_mode`.
-Task Job Scripts
+Job Scripts
    All user-defined task scripting now runs in a subshell, so you can safely
    switch Python environments inside tasks without affecting Cylc.
    Further information is available in the User Guide: :ref:`JobScripts`.