Merge branch 'TechEmpower:master' into master

Author: jkuhn1

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,7 @@
 <!--
 Thank you for submitting to the TechEmpower Framework Benchmarks!
 
-If you are submitting a new framework, please make sure that you add the appropriate line in the `.github/workflows/build.yml` file for proper integration testing. Also please make sure that an appropriate `README.md` is added in your framework directory with information about the framework and a link to its homepage and documentation.
+If you are submitting a new framework, please make sure that an appropriate `README.md` is added in your framework directory with information about the framework and a link to its homepage and documentation.
 
 For new frameworks, please do not include source code that isn't required for the benchmarks.
 

.github/workflows/build.yml

@@ -2,7 +2,7 @@ name: build
 on: [ push, pull_request ]
 jobs:
   setup:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       # Required for workflow triggers like the auto-label for failing PRs
       - name: Save PR number
@@ -20,7 +20,7 @@ jobs:
       #
       # We need to fetch more than one commit to be able to access HEAD^2 in case
       # of a pull request
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 10
       # In case of a push event, the commit we care about is simply HEAD.
@@ -46,9 +46,9 @@ jobs:
           echo "$(git log --format=%B -n 1 HEAD^2)" >> $GITHUB_ENV
           echo "EOF" >> $GITHUB_ENV
           echo "PREVIOUS_COMMIT=$(git log --format=%H -n 1 HEAD^2~1)" >> $GITHUB_ENV
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
-          python-version: '2.7'
+          python-version: '3.10'
           architecture: 'x64'
       - name: Get all changes vs master
         env:
@@ -65,14 +65,16 @@ jobs:
       - id: event_out
         name: Write event outputs
         run: |
-          # Escape the multiline string for Github Actions, see https://github.community/t/set-output-truncates-multiline-strings/16852/3
-          COMMIT_MESSAGE="${COMMIT_MESSAGE//'%'/'%25'}"
-          COMMIT_MESSAGE="${COMMIT_MESSAGE//$'\n'/'%0A'}"
-          COMMIT_MESSAGE="${COMMIT_MESSAGE//$'\r'/'%0D'}"
-          echo "::set-output name=commit_message::$COMMIT_MESSAGE"
-          echo "::set-output name=branch_name::$BRANCH_NAME"
-          echo "::set-output name=target_branch_name::$TARGET_BRANCH_NAME"
-          echo "::set-output name=previous_commit::$PREVIOUS_COMMIT"
+          # Escape the multiline string for Github Actions, see https://lab.amalitsky.com/posts/2022/github-actions-set-output-migration/#migration-from-set-output-to-github-output
+          delimiter="$(openssl rand -hex 8)"
+              {
+                echo "commit_message<<${delimiter}"
+                echo $COMMIT_MESSAGE
+                echo "${delimiter}"
+              } >> $GITHUB_OUTPUT
+          echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT
+          echo "target_branch_name=$TARGET_BRANCH_NAME" >> $GITHUB_OUTPUT
+          echo "previous_commit=$PREVIOUS_COMMIT" >> $GITHUB_OUTPUT
       - id: verify_out
         name: Write verify job matrix
         run: |
@@ -82,7 +84,7 @@ jobs:
           #   - TESTLANG: {lang}
           # with a TESTLANG object in the include array for each language under frameworks
           VERIFY_MATRIX=$(ls -1 frameworks | jq -Rc '.+"/" | select(inside(env.RUN_TESTS))  | rtrimstr("/")' | jq -sc '{include: map({TESTLANG: .})}')
-          echo "::set-output name=verify_matrix::$VERIFY_MATRIX"
+          echo "verify_matrix=$VERIFY_MATRIX" >> $GITHUB_OUTPUT
     outputs:
       commit_message: ${{ steps.event_out.outputs.commit_message }}
       branch_name: ${{ steps.event_out.outputs.branch_name }}
@@ -93,7 +95,7 @@ jobs:
     needs: setup
     # The matrix check is necessary because an empty job matrix is otherwise considered a workflow failure
     if: ${{ !contains(needs.setup.outputs.commit_message, '[ci skip]') && contains(needs.setup.outputs.verify_matrix, 'TESTLANG') }}
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix: ${{ fromJSON(needs.setup.outputs.verify_matrix) }}
       # Disable fail-fast to allow all failing frameworks/etc to fail in a
@@ -108,12 +110,12 @@ jobs:
       PREVIOUS_COMMIT: ${{ needs.setup.outputs.previous_commit }}
       PR_NUMBER: ${{ github.event.pull_request.number }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 10
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
-          python-version: '2.7'
+          python-version: '3.10'
           architecture: 'x64'
       - name: Get all changes vs master
         # Runs github_actions_diff, with the the output accessible in later steps
@@ -138,7 +140,10 @@ jobs:
           if [ "$RUN_TESTS" ]; then echo "Proceeding to run tests."; else echo 'Skipping test verification.'; fi
       - name: Build tfb dockerfile
         if: ${{ env.RUN_TESTS }}
-        run: docker build -t techempower/tfb - < ./Dockerfile;
+        uses: mattes/cached-docker-build-action@v1
+        with:
+          args: " --file ./Dockerfile --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) --tag techempower/tfb ."
+          cache_key: "${{ hashFiles('./Dockerfile') }}"
       - name: Stop services
         # Stop services that would claim ports we may need
         run: |
@@ -150,4 +155,22 @@ jobs:
           # run-ci.py runs the diffing to see if github actions needs to test this framework. Ideally/eventually,
           # we'd like to try and do the diffing before github_actions_clean & setup.
           # This will run the tests exactly as you would in your own vm:
-          docker network create tfb > /dev/null 2>&1 && docker run --network=tfb -v /var/run/docker.sock:/var/run/docker.sock --mount type=bind,source=`pwd`,target=/FrameworkBenchmarks techempower/tfb --mode verify --test-dir $RUN_TESTS --results-environment Github-Actions;
+          docker network create tfb > /dev/null 2>&1 && docker run --network=tfb -e USER_ID=$(id -u) -v /var/run/docker.sock:/var/run/docker.sock --mount type=bind,source=`pwd`,target=/FrameworkBenchmarks techempower/tfb --mode verify --test-dir $RUN_TESTS --results-environment Github-Actions;
+  dependabot:
+    needs: verify
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+      contents: write
+    if: ${{ github.actor == 'dependabot[bot]' }}
+    steps:
+      - name: Dependabot metadata
+        id: metadata
+        uses: dependabot/[email protected]
+        with:
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+      - name: Enable auto-merge for Dependabot PRs
+        run: gh pr merge --auto --merge "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/get-maintainers.yml

@@ -0,0 +1,37 @@
+name: get-maintainers
+on: 
+  pull_request:
+    types: [opened, reopened]
+permissions:
+  pull-requests: write
+jobs:
+  get_maintainers:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 10
+      - name: Get commit branch and commit message from PR
+        run: |
+          echo "BRANCH_NAME=$GITHUB_HEAD_REF" >> $GITHUB_ENV
+          echo "TARGET_BRANCH_NAME=$(echo ${GITHUB_BASE_REF##*/})" >> $GITHUB_ENV
+          echo "COMMIT_MESSAGE<<EOF" >> $GITHUB_ENV
+          echo "$(git log --format=%B -n 1 HEAD^2)" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+          echo "PREVIOUS_COMMIT=$(git log --format=%H -n 1 HEAD^2~1)" >> $GITHUB_ENV
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          architecture: 'x64'
+      - name: Save PR number
+        run: |
+          mkdir -p ./maintainers
+          echo ${{ github.event.number }} > ./maintainers/NR
+      - name: Get Maintainers
+        run: |
+          python ./toolset/github_actions/get_maintainers.py > ./maintainers/maintainers.md
+      - name: Save Maintainers
+        uses: actions/upload-artifact@v3
+        with:
+          name: maintainers
+          path: maintainers/

.github/workflows/label-failing-pr.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   apply_label:
     if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'failure' }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - name: 'Download artifact'
         uses: actions/[email protected]

.github/workflows/ping-maintainers.yml

@@ -0,0 +1,49 @@
+name: Ping Maintainers
+on:
+  workflow_run:
+    workflows: [ "get-maintainers"]
+    types:
+      - completed
+permissions:
+  pull-requests: write
+jobs:
+  ping_maintainers:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: 'Download maintainers artifact'
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: ${{github.event.workflow_run.id }},
+            });
+            let matchArtifact = artifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "maintainers"
+            })[0];
+            let download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            let fs = require('fs');
+            fs.writeFileSync('${{github.workspace}}/maintainers.zip', Buffer.from(download.data));
+      - run: unzip maintainers.zip
+      - name: Ping maintainers
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            let fs = require('fs');
+            let issue_number = Number(fs.readFileSync('./NR'));
+            let maintainers_comment = fs.readFileSync('./maintainers.md', 'utf8');
+            if (maintainers_comment) {
+              await github.rest.issues.createComment({
+                issue_number: issue_number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: maintainers_comment
+              });
+            }

.gitignore

@@ -1,6 +1,4 @@
-.DS_Store
 installs
-*.log
 node_modules/
 .travis.bak
 
@@ -35,6 +33,7 @@ build/
 *.patch
 */bin/
 
+
 # intellij
 *.iml
 *.ipr
@@ -59,9 +58,14 @@ benchmark.cfg
 
 # Visual Studio Code
 .vscode
+.history/
+.devcontainer
 
 # vim
 .*.sw[a-p]
 
 # merge tooling
-*.orig
+*.orig
+
+# python
+.venv/

Dockerfile

@@ -1,26 +1,60 @@
-FROM buildpack-deps:bionic
+FROM ubuntu:22.04
 
+ARG DEBIAN_FRONTEND=noninteractive
+# WARNING: DON'T PUT A SPACE AFTER ANY BACKSLASH OR APT WILL BREAK
 # One -q produces output suitable for logging (mostly hides
 # progress indicators)
-RUN apt-get -yqq update
+RUN apt-get -yqq update && \
+    apt-get -yqq install \
+      -o Dpkg::Options::="--force-confdef" \
+      -o Dpkg::Options::="--force-confold" \
+      cloc \
+      curl \
+      gcc \
+      git-core \
+      gosu \
+      # Needed for mysqlclient
+      libmysqlclient-dev \
+      libpq-dev \
+      pkg-config \
+      python3 \
+      python3-dev \
+      python3-pip \
+      siege \
+      software-properties-common
 
-# WARNING: DONT PUT A SPACE AFTER ANY BACKSLASH OR APT WILL BREAK
-RUN apt-get -yqq install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \
-  git-core \
-  cloc dstat                    `# Collect resource usage statistics` \
-  python-dev \
-  python-pip \
-  software-properties-common \
-  libmysqlclient-dev            `# Needed for MySQL-python`
+RUN pip3 install \
+      colorama==0.3.1 \
+      docker==4.0.2 \
+      mysqlclient \
+      psutil \
+      psycopg2-binary \
+      pymongo==3.13.0 \
+      # urllib3 incompatibility:
+      # https://github.com/docker/docker-py/issues/3113#issuecomment-1525500104
+      requests==2.28.1
 
-RUN pip install colorama==0.3.1 requests MySQL-python psycopg2-binary pymongo docker==4.0.2 psutil
+# Collect resource usage statistics
+ARG DOOL_VERSION=v1.2.0
 
-RUN apt-get install -yqq siege
+WORKDIR /tmp
+RUN curl -LSs "https://github.com/scottchiefbaker/dool/archive/${DOOL_VERSION}.tar.gz" | \
+      tar --strip-components=1 -xz && \
+    ./install.py
 
-# Fix for docker-py trying to import one package from the wrong location
-RUN cp -r /usr/local/lib/python2.7/dist-packages/backports/ssl_match_hostname/ /usr/lib/python2.7/dist-packages/backports
+# Check if the group ID is already created
+ARG GROUP_ID
+RUN if ! getent group "$GROUP_ID"; then \
+      addgroup --gid "$GROUP_ID" user; \
+    fi
 
-ENV PYTHONPATH /FrameworkBenchmarks
-ENV FWROOT /FrameworkBenchmarks
+# Check if the user ID is already created
+ARG USER_ID
+RUN if ! getent passwd "$USER_ID"; then \
+      adduser --disabled-password --gecos '' --gid "$GROUP_ID" --uid "$USER_ID" user; \
+    fi
 
-ENTRYPOINT ["python", "/FrameworkBenchmarks/toolset/run-tests.py"]
+ENV FWROOT=/FrameworkBenchmarks USER_ID="$USER_ID"
+ENV PYTHONPATH="$FWROOT"
+
+ENTRYPOINT ["/FrameworkBenchmarks/entrypoint.sh"]

README.md

@@ -89,7 +89,8 @@ Results of continuous benchmarking runs are available in real time [here](https:
 
 ### Data Visualization
 
-If you have a `results.json` file that you would like to visualize, you can [do that here](https://tfb-status.techempower.com/share). You can also attach a `runid` parameter to that url where `runid` is a run listed on [tfb-status](https://tfb-status.techempower.com) like so: https://www.techempower.com/benchmarks/#section=test&runid=fd07b64e-47ce-411e-8b9b-b13368e988c6
+If you have a `results.json` file that you would like to visualize, you can [do that here](https://tfb-status.techempower.com/share). You can also attach a `runid` parameter to that url where `runid` is a run listed on [tfb-status](https://tfb-status.techempower.com) like so: https://www.techempower.com/benchmarks/#section=test&runid=fd07b64e-47ce-411e-8b9b-b13368e988c6.
+If you want to visualize them or compare different results files on bash, here is an unofficial [plaintext results parser](https://github.com/joeyleeeeeee97/PlainTextResultsParser)
 
 ## Contributing
 

entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -eo pipefail -u
+
+chown -LR "$USER_ID" /var/run
+# Drop permissions of user to match those of the host system
+gosu "$USER_ID" python3 "${FWROOT}/toolset/run-tests.py" "$@"

frameworks/C++/cpoll_cppsp/benchmark_config.json

@@ -38,7 +38,8 @@
       "database_os": "Linux",
       "display_name": "cpoll-cppsp-raw",
       "notes": "",
-      "versus": "cpoll_cppsp"
+      "versus": "cpoll_cppsp",
+      "tags": ["broken"]
     },
     "postgres-raw": {
       "db_url": "/db_pg_async",