add a single sirun test (#1272)

* example usage of sirun

* put sirun output data in artifacts

* add script to get sirun results per commit

* explicitly use env var in sirun startup test

* make CIRCLE_TOKEN optional in get-results.js

Author: bengl

.circleci/config.yml

@@ -69,6 +69,31 @@ node-bench-e2e-base: &node-bench-e2e-base
         name: E2E Benchmark
         command: yarn bench:e2e
 
+node-bench-sirun-base: &node-bench-sirun-base
+  resource_class: small
+  working_directory: ~/dd-trace-js
+  steps:
+    - checkout
+    - *yarn-versions
+    - *restore-yarn-cache
+    - *yarn-install
+    - *save-yarn-cache
+    - run:
+        name: Install system deps
+        command: apt-get update && apt-get install -y valgrind wget
+    - run:
+        name: Install sirun
+        command: wget https://github.com/DataDog/sirun/releases/download/v0.1.3/sirun-v0.1.3-x86_64-unknown-linux-gnu.tar.gz && tar zxf sirun-v0.1.3-x86_64-unknown-linux-gnu.tar.gz && mv sirun /usr/bin
+    - run:
+        name: Run sirun
+        command: |
+          cd $SIRUN_TEST_DIR
+          node /root/dd-trace-js/benchmark/sirun/run-all-variants.js | tee /tmp/sirun-output.ndjson
+    - store_artifacts:
+        path: /tmp/sirun-output.ndjson
+        destination: sirun-output.ndjson
+
+
 node-integration-base: &node-integration-base
   resource_class: small
   working_directory: ~/dd-trace-js
@@ -253,6 +278,13 @@ jobs:
           - PLUGINS=mongodb-core
       - image: circleci/mongo
 
+  node-bench-sirun-startup-latest:
+    <<: *node-bench-sirun-base
+    docker:
+      - image: node
+        environment:
+          - SIRUN_TEST_DIR=/root/dd-trace-js/benchmark/sirun/startup/
+
   # Core tests
 
   node-core-8:
@@ -1092,6 +1124,7 @@ workflows:
       - node-bench-latest
       - node-bench-profiler-latest
       - node-bench-e2e-latest
+      - node-bench-sirun-startup-latest
       - node-amqplib
       - node-amqp10
       - node-aws-sdk
@@ -1342,6 +1375,7 @@ workflows:
       - node-bench-latest
       - node-bench-profiler-latest
       - node-bench-e2e-latest
+      - node-bench-sirun-startup-latest
       - node-amqplib
       - node-amqp10
       - node-aws-sdk

benchmark/sirun/get-results.js

@@ -0,0 +1,73 @@
+'use strict'
+
+const https = require('https')
+const { execSync } = require('child_process')
+
+const { CIRCLE_TOKEN } = process.env
+
+const ref = process.argv.length > 2 ? process.arv[2] : 'HEAD'
+const gitCommit = execSync(`git rev-parse ${ref}`).toString().trim()
+
+const circleHeaders = CIRCLE_TOKEN ? {
+  'circle-token': CIRCLE_TOKEN
+} : {}
+
+const statusUrl = ref =>
+  `https://api.github.com/repos/DataDog/dd-trace-js/commits/${ref}/statuses?per_page=100`
+const artifactsUrl = num =>
+  `https://circleci.com/api/v1.1/project/github/DataDog/dd-trace-js/${num}/artifacts`
+
+function get (url, headers) {
+  return new Promise((resolve, reject) => {
+    https.get(url, { headers: Object.assign({
+      'user-agent': 'dd-results-retriever',
+      accept: 'application/json'
+    }, headers) }, async res => {
+      if (res.statusCode >= 300 && res.statusCode < 400) {
+        resolve(get(res.headers.location))
+        return
+      }
+      res.on('error', reject)
+      let payload = ''
+      for await (const chunk of res) {
+        payload += chunk
+      }
+      resolve(payload)
+    }).on('error', reject)
+  })
+}
+
+async function getBuildNumsFromGithub (ref) {
+  const results = JSON.parse(await get(statusUrl(ref)))
+  const namesAndNums = {}
+  for (const build of results.filter(s => s.context.includes('-sirun-'))) {
+    const url = new URL(build.target_url)
+    namesAndNums[build.context.replace('ci/circleci: ', '')] = url.pathname.split('/').pop()
+  }
+  return namesAndNums
+}
+
+async function main () {
+  const builds = await getBuildNumsFromGithub(gitCommit)
+  const buildData = {}
+  for (const name in builds) {
+    const artifacts = JSON.parse(await get(artifactsUrl(builds[name]), circleHeaders))
+    const artifactUrl = artifacts.find(a => a.path === 'sirun-output.ndjson').url
+    const testResults = (await get(artifactUrl, circleHeaders))
+      .trim().split('\n').map(x => JSON.parse(x))
+    for (const result of testResults) {
+      const name = result.name
+      const variant = result.variant
+      if (!buildData[name]) {
+        buildData[name] = {}
+      }
+      delete result.name
+      delete result.variant
+      buildData[name][variant] = result
+    }
+  }
+  // eslint-disable-next-line no-console
+  console.log(require('util').inspect(buildData, false, null, true))
+}
+
+main()

benchmark/sirun/run-all-variants.js

@@ -0,0 +1,18 @@
+'use strict'
+
+const childProcess = require('child_process')
+const path = require('path')
+
+const metaJson = require(path.join(process.cwd(), 'meta.json'))
+
+const env = Object.assign({}, process.env, { SIRUN_NO_STDIO: '1' })
+
+if (metaJson.variants) {
+  const variants = metaJson.variants
+  for (const variant in variants) {
+    const variantEnv = Object.assign({}, env, { SIRUN_VARIANT: variant })
+    childProcess.execSync(`sirun meta.json`, { env: variantEnv, stdio: 'inherit' })
+  }
+} else {
+  childProcess.exec(`sirun meta.json`, { env, stdio: 'inherit' })
+}

benchmark/sirun/startup/meta.json

@@ -0,0 +1,18 @@
+{
+  "name": "startup",
+  "run": "node startup-test.js",
+  "cachegrind": true,
+  "iterations": 10,
+  "variants": {
+    "control": {
+      "env": {
+        "USE_TRACER": "0"
+      }
+    },
+    "with-tracer": {
+      "env": {
+        "USE_TRACER": "1"
+      }
+    }
+  }
+}

benchmark/sirun/startup/startup-test.js

@@ -0,0 +1,5 @@
+'use strict'
+
+if (Number(process.env.USE_TRACER)) {
+  require('../../..').init()
+}