Skip to content

Commit 18233c6

Browse files
dschodscho
committed
Do not use any Azure Pipelines artifact anymore
We already avoid doing that for all flavors except the `minimal` one. Now, we can do the same for the `minimal`, too, because we just taught `getViaGit()` to look for the latest successful `ci-artifacts` run and use the corresponding `git-sdk-64` commit. Signed-off-by: Johannes Schindelin <[email protected]>
1 parent 7317e58 commit 18233c6

File tree

3 files changed

+2
-331
lines changed

3 files changed

+2
-331
lines changed

‎__tests__/downloader.test.ts

Lines changed: 0 additions & 55 deletions
This file was deleted.

‎main.ts

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import * as core from '@actions/core'
2-
import {get, mkdirp} from './src/downloader'
2+
import {mkdirp} from './src/downloader'
33
import {restoreCache, saveCache} from '@actions/cache'
44
import process from 'process'
55
import {spawnSync} from 'child_process'
@@ -18,10 +18,7 @@ async function run(): Promise<void> {
1818
const verbose = core.getInput('verbose')
1919
const msysMode = core.getInput('msys') === 'true'
2020

21-
const {artifactName, download, id} =
22-
flavor === 'minimal'
23-
? await get(flavor, architecture)
24-
: await getViaGit(flavor, architecture)
21+
const {artifactName, download, id} = await getViaGit(flavor, architecture)
2522
const outputDirectory = core.getInput('path') || `C:/${artifactName}`
2623
let useCache: boolean
2724
switch (core.getInput('cache')) {

‎src/downloader.ts

Lines changed: 0 additions & 271 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,4 @@
1-
import {Readable} from 'stream'
2-
import {delimiter} from 'path'
3-
import fetch from '@adobe/node-fetch-retry'
41
import fs from 'fs'
5-
import https from 'https'
6-
import {spawn} from 'child_process'
7-
import unzipper from 'unzipper'
8-
9-
const gitForWindowsUsrBinPath = 'C:/Program Files/Git/usr/bin'
10-
const gitForWindowsMINGW64BinPath = 'C:/Program Files/Git/mingw64/bin'
11-
12-
async function fetchJSONFromURL<T>(url: string): Promise<T> {
13-
const res = await fetch(url)
14-
if (res.status !== 200) {
15-
throw new Error(
16-
`Got code ${res.status}, URL: ${url}, message: ${res.statusText}`
17-
)
18-
}
19-
return (await res.json()) as T
20-
}
212

223
export function mkdirp(directoryPath: string): void {
234
try {
@@ -33,255 +14,3 @@ export function mkdirp(directoryPath: string): void {
3314
}
3415
fs.mkdirSync(directoryPath, {recursive: true})
3516
}
36-
37-
async function unzip(
38-
url: string,
39-
bytesToExtract: number,
40-
stripPrefix: string,
41-
outputDirectory: string,
42-
verbose: boolean | number,
43-
downloader?: (
44-
_url: string,
45-
directory: string,
46-
_verbose: boolean | number
47-
) => Promise<void>
48-
): Promise<void> {
49-
let progress =
50-
verbose === false
51-
? (): void => {}
52-
: (path: string): void => {
53-
path === undefined || process.stderr.write(`${path}\n`)
54-
}
55-
if (typeof verbose === 'number') {
56-
let counter = 0
57-
progress = (path?: string): void => {
58-
if (path === undefined || ++counter % verbose === 0) {
59-
process.stderr.write(`${counter} items extracted\n`)
60-
}
61-
}
62-
}
63-
mkdirp(outputDirectory)
64-
65-
if (downloader) {
66-
// `https.get()` seems to have performance problems that cause frequent
67-
// ECONNRESET problems with larger payloads. Let's (ab-)use Git for Windows'
68-
// `curl.exe` to do the downloading for us in that case.
69-
return await downloader(url, outputDirectory, verbose)
70-
}
71-
72-
return new Promise<void>((resolve, reject) => {
73-
https
74-
.get(url, (res: Readable): void => {
75-
res
76-
.on('error', reject)
77-
.pipe(unzipper.Parse())
78-
.on('entry', entry => {
79-
if (!entry.path.startsWith(stripPrefix)) {
80-
process.stderr.write(
81-
`warning: skipping ${entry.path} because it does not start with ${stripPrefix}\n`
82-
)
83-
}
84-
const entryPath = `${outputDirectory}/${entry.path.substring(
85-
stripPrefix.length
86-
)}`
87-
progress(entryPath)
88-
if (entryPath.endsWith('/')) {
89-
mkdirp(entryPath.replace(/\/$/, ''))
90-
entry.autodrain()
91-
} else {
92-
entry
93-
.pipe(fs.createWriteStream(`${entryPath}`))
94-
.on('finish', () => {
95-
bytesToExtract -= fs.statSync(entryPath).size
96-
})
97-
}
98-
})
99-
.on('error', reject)
100-
.on('finish', progress)
101-
.on('finish', () => {
102-
bytesToExtract === 0
103-
? resolve()
104-
: // eslint-disable-next-line prefer-promise-reject-errors
105-
reject(`${bytesToExtract} bytes left to extract`)
106-
})
107-
})
108-
.on('error', reject)
109-
})
110-
}
111-
112-
/* We're (ab-)using Git for Windows' `tar.exe` and `xz.exe` to do the job */
113-
async function unpackTarXZInZipFromURL(
114-
url: string,
115-
outputDirectory: string,
116-
verbose: boolean | number = false
117-
): Promise<void> {
118-
const tmp = await fs.promises.mkdtemp(`${outputDirectory}/tmp`)
119-
const zipPath = `${tmp}/artifacts.zip`
120-
const curl = spawn(
121-
`${gitForWindowsMINGW64BinPath}/curl.exe`,
122-
[
123-
'--retry',
124-
'16',
125-
'--retry-all-errors',
126-
'--retry-connrefused',
127-
'-o',
128-
zipPath,
129-
url
130-
],
131-
{stdio: [undefined, 'inherit', 'inherit']}
132-
)
133-
await new Promise<void>((resolve, reject) => {
134-
curl
135-
.on('close', code =>
136-
code === 0 ? resolve() : reject(new Error(`${code}`))
137-
)
138-
.on('error', e => reject(new Error(`${e}`)))
139-
})
140-
141-
const zipContents = (await unzipper.Open.file(zipPath)).files.filter(
142-
e => !e.path.endsWith('/')
143-
)
144-
if (zipContents.length !== 1) {
145-
throw new Error(
146-
`${zipPath} does not contain exactly one file (${zipContents.map(
147-
e => e.path
148-
)})`
149-
)
150-
}
151-
152-
// eslint-disable-next-line no-console
153-
console.log(`unzipping ${zipPath}\n`)
154-
const tarXZ = spawn(
155-
`${gitForWindowsUsrBinPath}/bash.exe`,
156-
[
157-
'-lc',
158-
`unzip -p "${zipPath}" ${zipContents[0].path} | tar ${
159-
verbose === true ? 'xJvf' : 'xJf'
160-
} -`
161-
],
162-
{
163-
cwd: outputDirectory,
164-
env: {
165-
CHERE_INVOKING: '1',
166-
MSYSTEM: 'MINGW64',
167-
PATH: `${gitForWindowsUsrBinPath}${delimiter}${process.env.PATH}`
168-
},
169-
stdio: [undefined, 'inherit', 'inherit']
170-
}
171-
)
172-
await new Promise<void>((resolve, reject) => {
173-
tarXZ.on('close', code => {
174-
if (code === 0) {
175-
resolve()
176-
} else {
177-
reject(new Error(`tar: exited with code ${code}`))
178-
}
179-
})
180-
})
181-
await fs.promises.unlink(zipPath)
182-
await fs.promises.rmdir(tmp)
183-
}
184-
185-
export async function get(
186-
flavor: string,
187-
architecture: string
188-
): Promise<{
189-
artifactName: string
190-
id: string
191-
download: (
192-
outputDirectory: string,
193-
verbose?: number | boolean
194-
) => Promise<void>
195-
}> {
196-
if (!['x86_64', 'i686'].includes(architecture)) {
197-
throw new Error(`Unsupported architecture: ${architecture}`)
198-
}
199-
200-
let definitionId: number
201-
let artifactName: string
202-
switch (flavor) {
203-
case 'minimal':
204-
if (architecture === 'i686') {
205-
throw new Error(`Flavor "minimal" is only available for x86_64`)
206-
}
207-
definitionId = 22
208-
artifactName = 'git-sdk-64-minimal'
209-
break
210-
case 'makepkg-git':
211-
if (architecture === 'i686') {
212-
throw new Error(`Flavor "makepkg-git" is only available for x86_64`)
213-
}
214-
definitionId = 29
215-
artifactName = 'git-sdk-64-makepkg-git'
216-
break
217-
case 'build-installers':
218-
case 'full':
219-
definitionId = architecture === 'i686' ? 30 : 29
220-
artifactName = `git-sdk-${architecture === 'i686' ? 32 : 64}-${
221-
flavor === 'full' ? 'full-sdk' : flavor
222-
}`
223-
break
224-
default:
225-
throw new Error(`Unknown flavor: '${flavor}`)
226-
}
227-
228-
const baseURL = 'https://dev.azure.com/git-for-windows/git/_apis/build/builds'
229-
const data = await fetchJSONFromURL<{
230-
count: number
231-
value: [{id: string; downloadURL: string}]
232-
}>(
233-
`${baseURL}?definitions=${definitionId}&statusFilter=completed&resultFilter=succeeded&$top=1`
234-
)
235-
if (data.count !== 1) {
236-
throw new Error(`Unexpected number of builds: ${data.count}`)
237-
}
238-
const id = `${artifactName}-${data.value[0].id}`
239-
const download = async (
240-
outputDirectory: string,
241-
verbose: number | boolean = false
242-
): Promise<void> => {
243-
const data2 = await fetchJSONFromURL<{
244-
count: number
245-
value: [
246-
{
247-
name: string
248-
resource: {downloadUrl: string; properties: {artifactsize: number}}
249-
}
250-
]
251-
}>(`${baseURL}/${data.value[0].id}/artifacts`)
252-
const filtered = data2.value.filter(e => e.name === artifactName)
253-
if (filtered.length !== 1) {
254-
throw new Error(
255-
`Could not find ${artifactName} in ${JSON.stringify(data2, null, 4)}`
256-
)
257-
}
258-
const url = filtered[0].resource.downloadUrl
259-
const bytesToExtract = filtered[0].resource.properties.artifactsize
260-
let delayInSeconds = 1
261-
for (;;) {
262-
try {
263-
await unzip(
264-
url,
265-
bytesToExtract,
266-
`${artifactName}/`,
267-
outputDirectory,
268-
verbose,
269-
flavor === 'full' ? unpackTarXZInZipFromURL : undefined
270-
)
271-
break
272-
} catch (e) {
273-
delayInSeconds *= 2
274-
if (delayInSeconds >= 60) {
275-
throw e
276-
}
277-
process.stderr.write(
278-
`Encountered problem downloading/extracting ${url}: ${e}; Retrying in ${delayInSeconds} seconds...\n`
279-
)
280-
await new Promise((resolve, _reject) =>
281-
setTimeout(resolve, delayInSeconds * 1000)
282-
)
283-
}
284-
}
285-
}
286-
return {artifactName, download, id}
287-
}

0 commit comments

Comments
 (0)