Skip to content

Commit afc07fb

Browse files
jirfagjirfag
authored
Merge pull request #7 from golangci/feature/improve-cache
improve caching
2 parents ce34998 + b7926ca commit afc07fb

File tree

6 files changed

+189
-47
lines changed

6 files changed

+189
-47
lines changed

‎Dockerfile

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎README.md

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,14 @@ The restrictions of annotations are the following:
4949
1. Currently, they don't support markdown formatting (see the [feature request](https://github.community/t5/GitHub-API-Development-and/Checks-Ability-to-include-Markdown-in-line-annotations/m-p/56704))
5050
2. They aren't shown in list of comments like it was with [golangci.com](https://golangci.com). If you would like to have comments - please, up-vote [the issue](https://github.com/golangci/golangci-lint-action/issues/5).
5151
52+
## Performance
53+
54+
The action was implemented with performance in mind:
55+
56+
1. We cache data by [@actions/cache](https://github.com/actions/cache) between builds: Go build cache, Go modules cache, golangci-lint analysis cache.
57+
2. We don't use Docker because image pulling is slow.
58+
3. We do as much as we can in parallel, e.g. we download cache, go and golangci-lint binary in parallel.
59+
5260
## Internals
5361
5462
We use JavaScript-based action. We don't use Docker-based action because:
@@ -59,12 +67,21 @@ We use JavaScript-based action. We don't use Docker-based action because:
5967
Inside our action we perform 3 steps:
6068
6169
1. Setup environment running in parallel:
62-
* restore [cache](https://github.com/actions/cache) of previous analyzes into `$HOME/.cache/golangci-lint`
70+
* restore [cache](https://github.com/actions/cache) of previous analyzes
6371
* list [releases of golangci-lint](https://github.com/golangci/golangci-lint/releases) and find the latest patch version
6472
for needed version (users of this action can specify only minor version). After that install [golangci-lint](https://github.com/golangci/golangci-lint) using [@actions/tool-cache](https://github.com/actions/toolkit/tree/master/packages/tool-cache)
6573
* install the latest Go 1.x version using [@actions/setup-go](https://github.com/actions/setup-go)
6674
2. Run `golangci-lint` with specified by user `args`
67-
3. Save cache from `$HOME/.cache/golangci-lint` for later builds
75+
3. Save cache for later builds
76+
77+
### Caching internals
78+
79+
1. We save and restore the following directories: `~/.cache/golangci-lint`, `~/.cache/go-build`, `~/go/pkg`.
80+
2. The primary caching key looks like `golangci-lint.cache-{interval_number}-{go.mod_hash}`. Interval number ensures that we periodically invalidate
81+
our cache (every 7 days). `go.mod` hash ensures that we invalidate the cache early - as soon as dependencies have changed.
82+
3. We use [restore keys](https://help.github.com/en/actions/configuring-and-managing-workflows/caching-dependencies-to-speed-up-workflows#matching-a-cache-key): `golangci-lint.cache-{interval_number}-`, `golangci-lint.cache-`. GitHub matches keys by prefix if we have no exact match for the primary cache.
83+
84+
This scheme is basic and needs improvements. Pull requests and ideas are welcome.
6885

6986
## Development of this action
7087

‎dist/post_run/index.js

Lines changed: 54 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27870,29 +27870,73 @@ Object.defineProperty(exports, "__esModule", { value: true });
2787027870
const core = __importStar(__webpack_require__(470));
2787127871
const restore_1 = __importDefault(__webpack_require__(206));
2787227872
const save_1 = __importDefault(__webpack_require__(781));
27873-
// TODO: ensure dir exists, have access, etc
27874-
const getCacheDir = () => `${process.env.HOME}/.cache/golangci-lint`;
27875-
const setCacheInputs = () => {
27876-
process.env.INPUT_KEY = `golangci-lint.analysis-cache`;
27877-
process.env.INPUT_PATH = getCacheDir();
27873+
const crypto = __importStar(__webpack_require__(417));
27874+
const fs = __importStar(__webpack_require__(747));
27875+
function checksumFile(hashName, path) {
27876+
return new Promise((resolve, reject) => {
27877+
const hash = crypto.createHash(hashName);
27878+
const stream = fs.createReadStream(path);
27879+
stream.on("error", err => reject(err));
27880+
stream.on("data", chunk => hash.update(chunk));
27881+
stream.on("end", () => resolve(hash.digest("hex")));
27882+
});
27883+
}
27884+
const pathExists = (path) => __awaiter(void 0, void 0, void 0, function* () { return !!(yield fs.promises.stat(path).catch(e => false)); });
27885+
const getLintCacheDir = () => `${process.env.HOME}/.cache/golangci-lint`;
27886+
const getCacheDirs = () => {
27887+
// Not existing dirs are ok here: it works.
27888+
return [getLintCacheDir(), `${process.env.HOME}/.cache/go-build`, `${process.env.HOME}/go/pkg`];
27889+
};
27890+
const getIntervalKey = (invalidationIntervalDays) => {
27891+
const now = new Date();
27892+
const secondsSinceEpoch = now.getTime() / 1000;
27893+
const intervalNumber = Math.floor(secondsSinceEpoch / (invalidationIntervalDays * 86400));
27894+
return intervalNumber.toString();
2787827895
};
27896+
function buildCacheKeys() {
27897+
return __awaiter(this, void 0, void 0, function* () {
27898+
const keys = [];
27899+
let cacheKey = `golangci-lint.cache-`;
27900+
keys.push(cacheKey);
27901+
// Periodically invalidate a cache because a new code being added.
27902+
// TODO: configure it via inputs.
27903+
cacheKey += `${getIntervalKey(7)}-`;
27904+
keys.push(cacheKey);
27905+
if (yield pathExists(`go.mod`)) {
27906+
// Add checksum to key to invalidate a cache when dependencies change.
27907+
cacheKey += yield checksumFile(`cache-key`, `go.mod`);
27908+
}
27909+
else {
27910+
cacheKey += `nogomod`;
27911+
}
27912+
keys.push(cacheKey);
27913+
return keys;
27914+
});
27915+
}
2787927916
function restoreCache() {
2788027917
return __awaiter(this, void 0, void 0, function* () {
2788127918
const startedAt = Date.now();
27882-
setCacheInputs();
27919+
const keys = yield buildCacheKeys();
27920+
const primaryKey = keys.pop();
27921+
const restoreKeys = keys.reverse();
27922+
core.info(`Primary analysis cache key is '${primaryKey}', restore keys are '${restoreKeys.join(` | `)}'`);
27923+
process.env[`INPUT_RESTORE-KEYS`] = restoreKeys.join(`\n`);
27924+
process.env.INPUT_KEY = primaryKey;
27925+
process.env.INPUT_PATH = getCacheDirs().join(`\n`);
2788327926
// Tell golangci-lint to use our cache directory.
27884-
process.env.GOLANGCI_LINT_CACHE = getCacheDir();
27927+
process.env.GOLANGCI_LINT_CACHE = getLintCacheDir();
2788527928
yield restore_1.default();
27886-
core.info(`Restored golangci-lint analysis cache in ${Date.now() - startedAt}ms`);
27929+
core.info(`Restored cache for golangci-lint from key '${primaryKey}' in ${Date.now() - startedAt}ms`);
2788727930
});
2788827931
}
2788927932
exports.restoreCache = restoreCache;
2789027933
function saveCache() {
2789127934
return __awaiter(this, void 0, void 0, function* () {
2789227935
const startedAt = Date.now();
27893-
setCacheInputs();
27936+
const cacheDirs = getCacheDirs();
27937+
process.env.INPUT_PATH = cacheDirs.join(`\n`);
2789427938
yield save_1.default();
27895-
core.info(`Saved golangci-lint analysis cache in ${Date.now() - startedAt}ms`);
27939+
core.info(`Saved cache for golangci-lint from paths '${cacheDirs.join(`, `)}' in ${Date.now() - startedAt}ms`);
2789627940
});
2789727941
}
2789827942
exports.saveCache = saveCache;

‎dist/run/index.js

Lines changed: 54 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27882,29 +27882,73 @@ Object.defineProperty(exports, "__esModule", { value: true });
2788227882
const core = __importStar(__webpack_require__(470));
2788327883
const restore_1 = __importDefault(__webpack_require__(206));
2788427884
const save_1 = __importDefault(__webpack_require__(781));
27885-
// TODO: ensure dir exists, have access, etc
27886-
const getCacheDir = () => `${process.env.HOME}/.cache/golangci-lint`;
27887-
const setCacheInputs = () => {
27888-
process.env.INPUT_KEY = `golangci-lint.analysis-cache`;
27889-
process.env.INPUT_PATH = getCacheDir();
27885+
const crypto = __importStar(__webpack_require__(417));
27886+
const fs = __importStar(__webpack_require__(747));
27887+
function checksumFile(hashName, path) {
27888+
return new Promise((resolve, reject) => {
27889+
const hash = crypto.createHash(hashName);
27890+
const stream = fs.createReadStream(path);
27891+
stream.on("error", err => reject(err));
27892+
stream.on("data", chunk => hash.update(chunk));
27893+
stream.on("end", () => resolve(hash.digest("hex")));
27894+
});
27895+
}
27896+
const pathExists = (path) => __awaiter(void 0, void 0, void 0, function* () { return !!(yield fs.promises.stat(path).catch(e => false)); });
27897+
const getLintCacheDir = () => `${process.env.HOME}/.cache/golangci-lint`;
27898+
const getCacheDirs = () => {
27899+
// Not existing dirs are ok here: it works.
27900+
return [getLintCacheDir(), `${process.env.HOME}/.cache/go-build`, `${process.env.HOME}/go/pkg`];
27901+
};
27902+
const getIntervalKey = (invalidationIntervalDays) => {
27903+
const now = new Date();
27904+
const secondsSinceEpoch = now.getTime() / 1000;
27905+
const intervalNumber = Math.floor(secondsSinceEpoch / (invalidationIntervalDays * 86400));
27906+
return intervalNumber.toString();
2789027907
};
27908+
function buildCacheKeys() {
27909+
return __awaiter(this, void 0, void 0, function* () {
27910+
const keys = [];
27911+
let cacheKey = `golangci-lint.cache-`;
27912+
keys.push(cacheKey);
27913+
// Periodically invalidate a cache because a new code being added.
27914+
// TODO: configure it via inputs.
27915+
cacheKey += `${getIntervalKey(7)}-`;
27916+
keys.push(cacheKey);
27917+
if (yield pathExists(`go.mod`)) {
27918+
// Add checksum to key to invalidate a cache when dependencies change.
27919+
cacheKey += yield checksumFile(`cache-key`, `go.mod`);
27920+
}
27921+
else {
27922+
cacheKey += `nogomod`;
27923+
}
27924+
keys.push(cacheKey);
27925+
return keys;
27926+
});
27927+
}
2789127928
function restoreCache() {
2789227929
return __awaiter(this, void 0, void 0, function* () {
2789327930
const startedAt = Date.now();
27894-
setCacheInputs();
27931+
const keys = yield buildCacheKeys();
27932+
const primaryKey = keys.pop();
27933+
const restoreKeys = keys.reverse();
27934+
core.info(`Primary analysis cache key is '${primaryKey}', restore keys are '${restoreKeys.join(` | `)}'`);
27935+
process.env[`INPUT_RESTORE-KEYS`] = restoreKeys.join(`\n`);
27936+
process.env.INPUT_KEY = primaryKey;
27937+
process.env.INPUT_PATH = getCacheDirs().join(`\n`);
2789527938
// Tell golangci-lint to use our cache directory.
27896-
process.env.GOLANGCI_LINT_CACHE = getCacheDir();
27939+
process.env.GOLANGCI_LINT_CACHE = getLintCacheDir();
2789727940
yield restore_1.default();
27898-
core.info(`Restored golangci-lint analysis cache in ${Date.now() - startedAt}ms`);
27941+
core.info(`Restored cache for golangci-lint from key '${primaryKey}' in ${Date.now() - startedAt}ms`);
2789927942
});
2790027943
}
2790127944
exports.restoreCache = restoreCache;
2790227945
function saveCache() {
2790327946
return __awaiter(this, void 0, void 0, function* () {
2790427947
const startedAt = Date.now();
27905-
setCacheInputs();
27948+
const cacheDirs = getCacheDirs();
27949+
process.env.INPUT_PATH = cacheDirs.join(`\n`);
2790627950
yield save_1.default();
27907-
core.info(`Saved golangci-lint analysis cache in ${Date.now() - startedAt}ms`);
27951+
core.info(`Saved cache for golangci-lint from paths '${cacheDirs.join(`, `)}' in ${Date.now() - startedAt}ms`);
2790827952
});
2790927953
}
2791027954
exports.saveCache = saveCache;

‎entrypoint.sh

Lines changed: 0 additions & 10 deletions
This file was deleted.

‎src/cache.ts

Lines changed: 62 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,81 @@
11
import * as core from "@actions/core"
22
import restore from "cache/lib/restore"
33
import save from "cache/lib/save"
4+
import * as crypto from "crypto"
5+
import * as fs from "fs"
46

5-
// TODO: ensure dir exists, have access, etc
6-
const getCacheDir = (): string => `${process.env.HOME}/.cache/golangci-lint`
7+
function checksumFile(hashName: string, path: string) {
8+
return new Promise((resolve, reject) => {
9+
const hash = crypto.createHash(hashName)
10+
const stream = fs.createReadStream(path)
11+
stream.on("error", err => reject(err))
12+
stream.on("data", chunk => hash.update(chunk))
13+
stream.on("end", () => resolve(hash.digest("hex")))
14+
})
15+
}
16+
17+
const pathExists = async (path: string) => !!(await fs.promises.stat(path).catch(e => false))
18+
19+
const getLintCacheDir = (): string => `${process.env.HOME}/.cache/golangci-lint`
20+
21+
const getCacheDirs = (): string[] => {
22+
// Not existing dirs are ok here: it works.
23+
return [getLintCacheDir(), `${process.env.HOME}/.cache/go-build`, `${process.env.HOME}/go/pkg`]
24+
}
25+
26+
const getIntervalKey = (invalidationIntervalDays: number): string => {
27+
const now = new Date()
28+
const secondsSinceEpoch = now.getTime() / 1000
29+
const intervalNumber = Math.floor(secondsSinceEpoch / (invalidationIntervalDays * 86400))
30+
return intervalNumber.toString()
31+
}
732

8-
const setCacheInputs = (): void => {
9-
process.env.INPUT_KEY = `golangci-lint.analysis-cache`
10-
process.env.INPUT_PATH = getCacheDir()
33+
async function buildCacheKeys(): Promise<string[]> {
34+
const keys = []
35+
let cacheKey = `golangci-lint.cache-`
36+
keys.push(cacheKey)
37+
38+
// Periodically invalidate a cache because a new code being added.
39+
// TODO: configure it via inputs.
40+
cacheKey += `${getIntervalKey(7)}-`
41+
keys.push(cacheKey)
42+
43+
if (await pathExists(`go.mod`)) {
44+
// Add checksum to key to invalidate a cache when dependencies change.
45+
cacheKey += await checksumFile(`cache-key`, `go.mod`)
46+
} else {
47+
cacheKey += `nogomod`
48+
}
49+
keys.push(cacheKey)
50+
51+
return keys
1152
}
1253

1354
export async function restoreCache(): Promise<void> {
1455
const startedAt = Date.now()
15-
setCacheInputs()
56+
57+
const keys = await buildCacheKeys()
58+
const primaryKey = keys.pop()
59+
const restoreKeys = keys.reverse()
60+
core.info(`Primary analysis cache key is '${primaryKey}', restore keys are '${restoreKeys.join(` | `)}'`)
61+
process.env[`INPUT_RESTORE-KEYS`] = restoreKeys.join(`\n`)
62+
process.env.INPUT_KEY = primaryKey
63+
64+
process.env.INPUT_PATH = getCacheDirs().join(`\n`)
1665

1766
// Tell golangci-lint to use our cache directory.
18-
process.env.GOLANGCI_LINT_CACHE = getCacheDir()
67+
process.env.GOLANGCI_LINT_CACHE = getLintCacheDir()
1968

2069
await restore()
21-
core.info(`Restored golangci-lint analysis cache in ${Date.now() - startedAt}ms`)
70+
core.info(`Restored cache for golangci-lint from key '${primaryKey}' in ${Date.now() - startedAt}ms`)
2271
}
2372

2473
export async function saveCache(): Promise<void> {
2574
const startedAt = Date.now()
26-
setCacheInputs()
75+
76+
const cacheDirs = getCacheDirs()
77+
process.env.INPUT_PATH = cacheDirs.join(`\n`)
78+
2779
await save()
28-
core.info(`Saved golangci-lint analysis cache in ${Date.now() - startedAt}ms`)
80+
core.info(`Saved cache for golangci-lint from paths '${cacheDirs.join(`, `)}' in ${Date.now() - startedAt}ms`)
2981
}

0 commit comments

Comments
 (0)