downloader: extend it to optionally download the artifacts from Azure Pipelines

In addition to determining the latest successful build, we actually may
want to download the artifact. We now return also a function to do that.

The separation between determining the ID and the actual download is
required because we will later on want to cache the artifact, in which
case we need an ID to look up whether it has already been cached, and
only download it if it was not found in the cache.

In that vein, we also no longer only return the build ID, but prefix it
to identify the actual artifact flavor (so that we would not mistake a
cached "makepkg-git" SDK for a "build-installer" one, for example.

Signed-off-by: Johannes Schindelin <[email protected]>

Author: dscho

.vscode/settings.json

@@ -6,6 +6,7 @@
     ],
     "cSpell.words": [
         "Pacman",
+        "autodrain",
         "unzipper",
         "vercel"
     ]

__tests__/downloader.test.ts

@@ -76,5 +76,5 @@ test('can obtain build ID', async () => {
     }
   )
   const {id} = await get('minimal', 'x86_64')
-  expect(id).toEqual(71000)
+  expect(id).toEqual('git-sdk-64-minimal-71000')
 })

src/downloader.ts

@@ -1,4 +1,6 @@
+import fs from 'fs'
 import https from 'https'
+import unzipper from 'unzipper'
 
 async function fetchJSONFromURL<T>(url: string): Promise<T> {
   return new Promise<T>((resolve, reject) => {
@@ -29,32 +31,104 @@ async function fetchJSONFromURL<T>(url: string): Promise<T> {
   })
 }
 
+function mkdirp(directoryPath: string): void {
+  try {
+    const stat = fs.statSync(directoryPath)
+    if (stat.isDirectory()) {
+      return
+    }
+    throw new Error(`${directoryPath} exists, but is not a directory`)
+  } catch (e) {
+    if (!e || e.code !== 'ENOENT') {
+      throw e
+    }
+  }
+  fs.mkdirSync(directoryPath, {recursive: true})
+}
+
+async function unzip(
+  url: string,
+  stripPrefix: string,
+  outputDirectory: string,
+  verbose: boolean | number
+): Promise<void> {
+  let progress =
+    verbose === false
+      ? (): void => {}
+      : (path: string): void => {
+          path === undefined || process.stderr.write(`${path}\n`)
+        }
+  if (typeof verbose === 'number') {
+    let counter = 0
+    progress = (path?: string): void => {
+      if (path === undefined || ++counter % verbose === 0) {
+        process.stderr.write(`${counter} items extracted\n`)
+      }
+    }
+  }
+  mkdirp(outputDirectory)
+  return new Promise<void>((resolve, reject) => {
+    https.get(url, res =>
+      res
+        .pipe(unzipper.Parse())
+        .on('entry', entry => {
+          if (!entry.path.startsWith(stripPrefix)) {
+            process.stderr.write(
+              `warning: skipping ${entry.path} because it does not start with ${stripPrefix}\n`
+            )
+          }
+          const entryPath = `${outputDirectory}/${entry.path.substring(
+            stripPrefix.length
+          )}`
+          progress(entryPath)
+          if (entryPath.endsWith('/')) {
+            mkdirp(entryPath.replace(/\/$/, ''))
+            entry.autodrain()
+          } else {
+            entry.pipe(fs.createWriteStream(`${entryPath}`))
+          }
+        })
+        .on('error', reject)
+        .on('finish', progress)
+        .on('finish', resolve)
+    )
+  })
+}
+
 export async function get(
   flavor: string,
   architecture: string
 ): Promise<{
   id: string
+  download: (
+    outputDirectory: string,
+    verbose?: number | boolean
+  ) => Promise<void>
 }> {
   if (!['x86_64', 'i686'].includes(architecture)) {
     throw new Error(`Unsupported architecture: ${architecture}`)
   }
 
   let definitionId: number
+  let artifactName: string
   switch (flavor) {
     case 'minimal':
       if (architecture === 'i686') {
         throw new Error(`Flavor "minimal" is only available for x86_64`)
       }
       definitionId = 22
+      artifactName = 'git-sdk-64-minimal'
       break
     case 'makepkg-git':
       if (architecture === 'i686') {
         throw new Error(`Flavor "makepkg-git" is only available for x86_64`)
       }
       definitionId = 29
+      artifactName = 'git-sdk-64-makepkg-git'
       break
     case 'build-installers':
       definitionId = architecture === 'i686' ? 30 : 29
+      artifactName = `git-sdk-${architecture === 'i686' ? 32 : 64}-${flavor}`
       break
     default:
       throw new Error(`Unknown flavor: '${flavor}`)
@@ -63,12 +137,30 @@ export async function get(
   const baseURL = 'https://dev.azure.com/git-for-windows/git/_apis/build/builds'
   const data = await fetchJSONFromURL<{
     count: number
-    value: [{id: string}]
+    value: [{id: string; downloadURL: string}]
   }>(
     `${baseURL}?definitions=${definitionId}&statusFilter=completed&resultFilter=succeeded&$top=1`
   )
   if (data.count !== 1) {
     throw new Error(`Unexpected number of builds: ${data.count}`)
   }
-  return {id: data.value[0].id}
+  const id = `${artifactName}-${data.value[0].id}`
+  const download = async (
+    outputDirectory: string,
+    verbose: number | boolean = false
+  ): Promise<void> => {
+    const data2 = await fetchJSONFromURL<{
+      count: number
+      value: [{name: string; resource: {downloadUrl: string}}]
+    }>(`${baseURL}/${data.value[0].id}/artifacts`)
+    const filtered = data2.value.filter(e => e.name === artifactName)
+    if (filtered.length !== 1) {
+      throw new Error(
+        `Could not find ${artifactName} in ${JSON.stringify(data2, null, 4)}`
+      )
+    }
+    const url = filtered[0].resource.downloadUrl
+    await unzip(url, `${artifactName}/`, outputDirectory, verbose)
+  }
+  return {download, id}
 }