fix(ci): Integration test improvements (#41)

* Support for running against local copy of YAML tests

* Add a skipped test

* Use correct file path 🤦

Author: JoshMock

scripts/download-artifacts.js

@@ -26,7 +26,7 @@ const fetch = require('node-fetch')
 const crossZip = require('cross-zip')
 const ora = require('ora')
 
-const { mkdir } = promises
+const { mkdir, cp } = promises
 const pipeline = promisify(stream.pipeline)
 const unzip = promisify(crossZip.unzip)
 
@@ -35,7 +35,7 @@ const zipFile = join(__dirname, '..', 'serverless-clients-tests.zip')
 
 const specFolder = join(__dirname, '..', 'rest-api-spec')
 
-async function downloadArtifacts () {
+async function downloadArtifacts (localTests) {
   const log = ora('Checking out spec and test').start()
 
   const { GITHUB_TOKEN } = process.env
@@ -46,31 +46,36 @@ async function downloadArtifacts () {
 
   log.text = 'Fetching test YAML files'
 
-  if (!GITHUB_TOKEN) {
-    log.fail("Missing required environment variable 'GITHUB_TOKEN'")
-    process.exit(1)
-  }
-
-  let response = await fetch('https://api.github.com/repos/elastic/serverless-clients-tests/zipball/main', {
-    headers: {
-      Authorization: `Bearer ${GITHUB_TOKEN}`,
-      Accept: "application/vnd.github+json",
+  if (localTests) {
+    log.text = `Copying local tests from ${localTests}`
+    await cp(localTests, testYamlFolder, { recursive: true })
+  } else {
+    if (!GITHUB_TOKEN) {
+      log.fail("Missing required environment variable 'GITHUB_TOKEN'")
+      process.exit(1)
     }
-  })
 
-  if (!response.ok) {
-    log.fail(`unexpected response ${response.statusText}`)
-    process.exit(1)
-  }
+    let response = await fetch('https://api.github.com/repos/elastic/serverless-clients-tests/zipball/main', {
+      headers: {
+        Authorization: `Bearer ${GITHUB_TOKEN}`,
+        Accept: "application/vnd.github+json",
+      }
+    })
 
-  log.text = 'Downloading tests zipball'
-  await pipeline(response.body, createWriteStream(zipFile))
+    if (!response.ok) {
+      log.fail(`unexpected response ${response.statusText}`)
+      process.exit(1)
+    }
 
-  log.text = 'Unzipping tests'
-  await unzip(zipFile, testYamlFolder)
+    log.text = 'Downloading tests zipball'
+    await pipeline(response.body, createWriteStream(zipFile))
 
-  log.text = 'Cleanup'
-  await rimraf(zipFile)
+    log.text = 'Unzipping tests'
+    await unzip(zipFile, testYamlFolder)
+
+    log.text = 'Cleanup'
+    await rimraf(zipFile)
+  }
 
   log.text = 'Fetching Elasticsearch spec info'
   await rimraf(specFolder)

test/integration/index.js

@@ -42,7 +42,7 @@ const MAX_TEST_TIME = 1000 * 60
 
 const options = minimist(process.argv.slice(2), {
   boolean: ['bail'],
-  string: ['suite', 'test'],
+  string: ['suite', 'test', 'local'],
 })
 
 const skips = {
@@ -54,7 +54,10 @@ const skips = {
   'transform/10_basic.yml': ['*'],
   // TODO: scripts_painless_execute expects {"result":"0.1"}, gets {"result":"0"}
   // body sent as Buffer, unsure if related
-  'script/10_basic.yml': ['*']
+  'script/10_basic.yml': ['*'],
+  // TODO: expects {"outlier_detection.auc_roc.value":0.99995}, gets {"outlier_detection.auc_roc.value":0.5}
+  // remove if/when https://github.com/elastic/serverless-clients-tests/issues/37 is resolved
+  'machine_learning/data_frame_evaluate.yml': ['*'],
 }
 
 const shouldSkip = (file, name) => {
@@ -105,7 +108,7 @@ function runner (opts = {}) {
 
 async function start ({ client }) {
   log(`Downloading YAML test artifacts...`)
-  await downloadArtifacts()
+  await downloadArtifacts(options.local)
 
   log(`Testing serverless API...`)
   const junit = createJunitReporter()