Add upstream CI build (#882)

Author: jrbourbeau

.github/workflows/upstream.yml

@@ -0,0 +1,124 @@
+name: Upstream
+
+on:
+  schedule:
+    - cron: "0 1 * * *"
+  push:
+  pull_request:
+
+jobs:
+
+  check:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || github.event_name == 'pull_request'
+    outputs:
+      test-upstream: ${{ steps.detect-trigger.outputs.trigger-found }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - uses: xarray-contrib/ci-trigger@v1
+        id: detect-trigger
+        with:
+          keyword: "test-upstream"
+
+  build:
+    needs: check
+    runs-on: ubuntu-latest
+    if: |
+      always()
+      && (
+          needs.check.outputs.test-upstream == 'true'
+          || (github.repository == 'dask/dask-ml' && github.event_name != 'pull_request')
+      )
+
+    env:
+      COVERAGE: "true"
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2
+
+      - name: Setup Conda Environment
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+          channel-priority: strict
+          python-version: "3.9"
+          environment-file: ci/environment-3.9.yaml
+          activate-environment: test-environment
+          auto-activate-base: false
+
+      - name: Install
+        shell: bash -l {0}
+        env:
+          UPSTREAM_DEV: 1
+        run: source ci/install.sh
+
+      - name: Run tests
+        shell: bash -l {0}
+        run: pytest -v
+
+  report:
+    name: report
+    needs: build
+    if: |
+      always()
+      && github.event_name != 'pull_request'
+      && github.repository == 'dask/dask-ml'
+      && needs.build.result == 'failure'
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v2
+      - name: Report failures
+        uses: actions/github-script@v3
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const title = "⚠️ Upstream CI failed ⚠️"
+            const workflow_url = `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
+            const issue_body = `[Workflow Run URL](${workflow_url})`
+            // Run GraphQL query against GitHub API to find the most recent open issue used for reporting failures
+            const query = `query($owner:String!, $name:String!, $creator:String!, $label:String!){
+              repository(owner: $owner, name: $name) {
+                issues(first: 1, states: OPEN, filterBy: {createdBy: $creator, labels: [$label]}, orderBy: {field: CREATED_AT, direction: DESC}) {
+                  edges {
+                    node {
+                      body
+                      id
+                      number
+                    }
+                  }
+                }
+              }
+            }`;
+            const variables = {
+                owner: context.repo.owner,
+                name: context.repo.repo,
+                label: 'upstream',
+                creator: "github-actions[bot]"
+            }
+            const result = await github.graphql(query, variables)
+            // If no issue is open, create a new issue,
+            // else update the body of the existing issue.
+            if (result.repository.issues.edges.length === 0) {
+                github.issues.create({
+                    owner: variables.owner,
+                    repo: variables.name,
+                    body: issue_body,
+                    title: title,
+                    labels: [variables.label]
+                })
+            } else {
+                github.issues.update({
+                    owner: variables.owner,
+                    repo: variables.name,
+                    issue_number: result.repository.issues.edges[0].node.number,
+                    body: issue_body
+                })
+            }

ci/install.sh

@@ -1,3 +1,21 @@
+
+# Optionally, install development versions of dependenies
+if [[ ${UPSTREAM_DEV} ]]; then
+    # FIXME https://github.com/mamba-org/mamba/issues/412
+    # mamba uninstall --force dask distributed scikit-learn
+    conda uninstall --force dask distributed scikit-learn
+
+    python -m pip install --no-deps --pre \
+        -i https://pypi.anaconda.org/scipy-wheels-nightly/simple \
+        scikit-learn
+
+    python -m pip install \
+        --upgrade \
+        git+https://github.com/dask/dask \
+        git+https://github.com/dask/distributed
+fi
+
+# Install dask-ml
 python -m pip install --quiet --no-deps -e .
 
 echo mamba list