Skip to content

Commit ddaab78

Browse files
rstzcopybara-github
rstz
authored and
copybara-github
committed
[YDF] Better Caching with GH Actions
PiperOrigin-RevId: 825645804
1 parent 6cc0dab commit ddaab78

File tree

3 files changed

+173
-15
lines changed

3 files changed

+173
-15
lines changed

‎.github/workflows/build_test.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@ permissions:
55

66
on:
77
workflow_dispatch:
8+
inputs:
9+
refresh_cache:
10+
description: 'Force refresh the cache for all child workflows'
11+
type: boolean
12+
default: false
813
push:
914
branches:
1015
- main
@@ -25,6 +30,8 @@ jobs:
2530
with:
2631
runner: linux-x86-n2-32
2732
compiler_config: ${{ matrix.compiler_config }}
33+
refresh_cache: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.refresh_cache == 'true') }}
34+
pull_request_base_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || '' }}
2835
strategy:
2936
fail-fast: false
3037
matrix:
@@ -36,6 +43,8 @@ jobs:
3643
with:
3744
runner: linux-x86-n2-32
3845
python_version: ${{ matrix.python_version }}
46+
refresh_cache: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.refresh_cache == 'true') }}
47+
pull_request_base_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || '' }}
3948
strategy:
4049
fail-fast: false
4150
matrix:

‎.github/workflows/cc_build_test.yml‎

Lines changed: 65 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,15 @@ on:
1717
type: string
1818
default: "gcc"
1919
description: "Configuration for the C++ compiler (e.g., 'gcc', 'clang')."
20+
refresh_cache:
21+
required: false
22+
type: boolean
23+
default: false
24+
description: "Force refresh the cache"
25+
pull_request_base_sha:
26+
required: false
27+
type: string
28+
description: "Force refresh the cache"
2029

2130
jobs:
2231
build-test:
@@ -31,20 +40,58 @@ jobs:
3140
--features=-fully_static_link
3241
--build_tag_filters=-tf_dep,-cuda_dep
3342
--test_tag_filters=-tf_dep,-cuda_dep
43+
REFRESH_CACHE: ${{ inputs.refresh_cache }}
44+
permissions:
45+
actions: write # For gh cache delete
46+
contents: read
47+
3448
steps:
3549
- name: Checkout Repository
3650
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
3751
- name: Set safe directory # https://github.com/actions/checkout/issues/2031
3852
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
3953

40-
- name: Cache Bazel Dependencies (C++ - ${{ inputs.compiler_config }})
41-
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830
54+
- name : Set up cache keys
55+
id: cache-keys
56+
run: |
57+
CACHE_PREFIX="${{ runner.os }}-bazel-cc-${{ inputs.compiler_config }}"
58+
CACHE_RESTORE_KEY_2="$CACHE_PREFIX"
59+
CACHE_RESTORE_KEY_1="$CACHE_RESTORE_KEY_2-${{ hashFiles('**/WORKSPACE', '.bazelrc') }}"
60+
CACHE_RESTORE_KEY_0="$CACHE_RESTORE_KEY_1-${{ hashFiles('**/BUILD*') }}"
61+
CACHE_KEY="$CACHE_RESTORE_KEY_0-${{ github.sha }}"
62+
echo "CACHE_RESTORE_KEY_2=$CACHE_RESTORE_KEY_2" >> "$GITHUB_OUTPUT"
63+
echo "CACHE_RESTORE_KEY_1=$CACHE_RESTORE_KEY_1" >> "$GITHUB_OUTPUT"
64+
echo "CACHE_RESTORE_KEY_0=$CACHE_RESTORE_KEY_0" >> "$GITHUB_OUTPUT"
65+
echo "CACHE_KEY=$CACHE_KEY" >> "$GITHUB_OUTPUT"
66+
if [[ -n "${{ inputs.pull_request_base_sha }}" ]]; then
67+
CACHE_RESTORE_KEY_HEAD="$CACHE_RESTORE_KEY_0-${{ inputs.pull_request_base_sha }}"
68+
echo "CACHE_RESTORE_KEY_HEAD=$CACHE_RESTORE_KEY_HEAD" >> "$GITHUB_OUTPUT"
69+
fi
70+
71+
- name: Clean build outputs if cache is being refreshed
72+
if: env.REFRESH_CACHE == 'true'
73+
run: |
74+
bazel clean --expunge
75+
rm -rf .bazel-cache
76+
77+
- name: Restore bazel cache
78+
id: bazel-cache
79+
if: env.REFRESH_CACHE != 'true'
80+
uses: actions/cache/restore@v4
4281
with:
4382
path: .bazel-cache
44-
key: ${{ runner.os }}-bazel-cc-${{ inputs.compiler_config }}-${{ hashFiles('**/WORKSPACE', '.bazelrc') }}
83+
key: ${{ steps.cache-keys.outputs.CACHE_KEY }}
4584
restore-keys: |
46-
${{ runner.os }}-bazel-cc-${{ inputs.compiler_config }}-
47-
${{ runner.os }}-bazel-cc-
85+
${{ steps.cache-keys.outputs.CACHE_RESTORE_KEY_HEAD }}
86+
${{ steps.cache-keys.outputs.CACHE_RESTORE_KEY_0 }}-
87+
${{ steps.cache-keys.outputs.CACHE_RESTORE_KEY_1 }}-
88+
${{ steps.cache-keys.outputs.CACHE_RESTORE_KEY_2 }}-
89+
90+
- name: Check cache hit
91+
run: |
92+
echo "Cache Hit: ${{ steps.bazel-cache.outputs.cache-hit }}"
93+
echo "Cache Primary Key: ${{ steps.bazel-cache.outputs.cache-primary-key }}"
94+
echo "Cache Matched Key: ${{ steps.bazel-cache.outputs.cache-matched-key }}"
4895
4996
- name: Create Bazel Cache Directories
5097
run: mkdir -p .bazel-cache/repository .bazel-cache/disk
@@ -72,4 +119,16 @@ jobs:
72119
testlogs_xml_path: bazel-testlogs/**/*.xml
73120
artifact_name: cc-test-logs-${{ inputs.compiler_config }}
74121
testlogs_log_path: bazel-testlogs/**/*.log
75-
failure_message: "CC tests failed for compiler config '${{ inputs.compiler_config }}'. See annotations in the Checks tab for details, or 'cc-test-logs-${{ inputs.compiler_config }}' artifact for full logs."
122+
failure_message: "CC tests failed for compiler config \'${{ inputs.compiler_config }}\'. See annotations in the Checks tab for details, or \'cc-test-logs-${{ inputs.compiler_config }}\' artifact for full logs."
123+
124+
- name: Remove cache if cache is being refreshed
125+
if: env.REFRESH_CACHE == 'true'
126+
continue-on-error: true # Ignore errors when cache is not found.
127+
run: gh cache delete ${{ steps.cache-keys.outputs.CACHE_KEY }}
128+
129+
- name: Save bazel cache
130+
uses: actions/cache/save@v4
131+
if: env.REFRESH_CACHE == 'true' || steps.bazel-cache.outputs.cache-hit != 'true'
132+
with:
133+
path: .bazel-cache
134+
key: ${{ steps.cache-keys.outputs.CACHE_KEY }}

‎.github/workflows/python_build_test.yml‎

Lines changed: 99 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,16 @@ on:
1313
python_version:
1414
required: true
1515
type: string
16-
description: "The Python version to set up (e.g., '3.9', '3.10'). Currently unused."
16+
description: "The Python version to set up (e.g., '3.9', '3.10')."
17+
refresh_cache:
18+
required: false
19+
type: boolean
20+
default: false
21+
description: "Force refresh the cache"
22+
pull_request_base_sha:
23+
required: false
24+
type: string
25+
description: "Force refresh the cache"
1726

1827
jobs:
1928
build-test:
@@ -27,33 +36,89 @@ jobs:
2736
--config=linux_cpp17
2837
--config=linux_avx2
2938
--features=-fully_static_link
39+
REFRESH_CACHE: ${{ inputs.refresh_cache }}
40+
GH_TOKEN: ${{ github.token }} # Required for gh cache delete
41+
permissions:
42+
actions: write # For gh cache delete
43+
contents: read
44+
3045
steps:
3146
- name: Checkout Repository
3247
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
3348
- name: Set safe directory # https://github.com/actions/checkout/issues/2031
3449
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
3550

36-
- name: Cache Bazel Dependencies (Python ${{ inputs.python_version }})
37-
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830
51+
# --- Bazel Cache Setup ---
52+
- name : Set up Bazel cache keys
53+
id: bazel-cache-keys
54+
run: |
55+
CACHE_PREFIX="${{ runner.os }}-bazel-py-${{ inputs.python_version }}"
56+
CACHE_RESTORE_KEY_2="$CACHE_PREFIX"
57+
CACHE_RESTORE_KEY_1="$CACHE_RESTORE_KEY_2-${{ hashFiles('**/WORKSPACE', '.bazelrc') }}"
58+
CACHE_RESTORE_KEY_0="$CACHE_RESTORE_KEY_1-${{ hashFiles('**/BUILD*') }}"
59+
CACHE_KEY="$CACHE_RESTORE_KEY_0-${{ github.sha }}"
60+
echo "CACHE_RESTORE_KEY_2=$CACHE_RESTORE_KEY_2" >> "$GITHUB_OUTPUT"
61+
echo "CACHE_RESTORE_KEY_1=$CACHE_RESTORE_KEY_1" >> "$GITHUB_OUTPUT"
62+
echo "CACHE_RESTORE_KEY_0=$CACHE_RESTORE_KEY_0" >> "$GITHUB_OUTPUT"
63+
echo "CACHE_KEY=$CACHE_KEY" >> "$GITHUB_OUTPUT"
64+
if [[ -n "${{ inputs.pull_request_base_sha }}" ]]; then
65+
CACHE_RESTORE_KEY_HEAD="$CACHE_RESTORE_KEY_0-${{ inputs.pull_request_base_sha }}"
66+
echo "CACHE_RESTORE_KEY_HEAD=$CACHE_RESTORE_KEY_HEAD" >> "$GITHUB_OUTPUT"
67+
fi
68+
69+
- name: Clean Bazel build outputs if cache is being refreshed
70+
if: env.REFRESH_CACHE == 'true'
71+
run: |
72+
bazel clean --expunge
73+
rm -rf .bazel-cache
74+
75+
- name: Restore Bazel cache
76+
id: bazel-cache
77+
if: env.REFRESH_CACHE != 'true'
78+
uses: actions/cache/restore@v4
3879
with:
3980
path: .bazel-cache
40-
key: ${{ runner.os }}-bazel-py-${{ inputs.python_version }}-${{ hashFiles('**/WORKSPACE', '.bazelrc') }}
81+
key: ${{ steps.bazel-cache-keys.outputs.CACHE_KEY }}
4182
restore-keys: |
42-
${{ runner.os }}-bazel-py-${{ inputs.python_version }}-
43-
${{ runner.os }}-bazel-py-
83+
${{ steps.bazel-cache-keys.outputs.CACHE_RESTORE_KEY_HEAD }}
84+
${{ steps.bazel-cache-keys.outputs.CACHE_RESTORE_KEY_0 }}-
85+
${{ steps.bazel-cache-keys.outputs.CACHE_RESTORE_KEY_1 }}-
86+
${{ steps.bazel-cache-keys.outputs.CACHE_RESTORE_KEY_2 }}-
87+
88+
- name: Check Bazel cache hit
89+
run: |
90+
echo "Bazel Cache Hit: ${{ steps.bazel-cache.outputs.cache-hit }}"
91+
echo "Bazel Cache Primary Key: ${{ steps.bazel-cache.outputs.cache-primary-key }}"
92+
echo "Bazel Cache Matched Key: ${{ steps.bazel-cache.outputs.cache-matched-key }}"
4493
4594
- name: Create Bazel Cache Directories
4695
run: mkdir -p .bazel-cache/repository .bazel-cache/disk
4796

48-
- name: Cache pip dependencies (Python ${{ inputs.python_version }})
49-
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830
97+
# --- pip Cache Setup ---
98+
- name: Set up pip cache key
99+
id: pip-cache-key
100+
run: |
101+
echo "CACHE_KEY=${{ runner.os }}-pip-${{ inputs.python_version }}-${{ hashFiles('yggdrasil_decision_forests/port/python/requirements.txt', 'yggdrasil_decision_forests/port/python/dev_requirements.txt') }}" >> "$GITHUB_OUTPUT"
102+
103+
- name: Clean pip cache if refresh is enabled
104+
if: env.REFRESH_CACHE == 'true'
105+
run: rm -rf ~/.cache/pip
106+
107+
- name: Restore pip cache
108+
id: pip-cache
109+
if: env.REFRESH_CACHE != 'true'
110+
uses: actions/cache/restore@v4
50111
with:
51112
path: ~/.cache/pip
52-
key: ${{ runner.os }}-pip-${{ inputs.python_version }}-${{ hashFiles('yggdrasil_decision_forests/port/python/requirements.txt', 'yggdrasil_decision_forests/port/python/dev_requirements.txt') }}
113+
key: ${{ steps.pip-cache-key.outputs.CACHE_KEY }}
53114
restore-keys: |
54115
${{ runner.os }}-pip-${{ inputs.python_version }}-
55116
${{ runner.os }}-pip-
56117
118+
- name: Check pip cache hit
119+
run: echo "Pip Cache Hit: ${{ steps.pip-cache.outputs.cache-hit }}"
120+
121+
# --- Build and Test Steps ---
57122
- name: Install Python Dependencies
58123
working-directory: yggdrasil_decision_forests/port/python
59124
run: |
@@ -88,3 +153,28 @@ jobs:
88153
testlogs_log_path: bazel-testlogs/**/*.log
89154
failure_message: "Python tests failed for Python ${{ inputs.python_version }}. See annotations in the Checks tab for details, or 'py-test-logs-${{ inputs.python_version }}' artifact for full logs."
90155
working_directory: yggdrasil_decision_forests/port/python
156+
157+
# --- Cache Saving ---
158+
- name: Remove old Bazel cache if refreshing
159+
if: env.REFRESH_CACHE == 'true'
160+
continue-on-error: true # Ignore errors when cache is not found.
161+
run: gh cache delete ${{ steps.bazel-cache-keys.outputs.CACHE_KEY }} --confirm
162+
163+
- name: Save Bazel cache
164+
uses: actions/cache/save@v4
165+
if: env.REFRESH_CACHE == 'true' || steps.bazel-cache.outputs.cache-hit != 'true'
166+
with:
167+
path: .bazel-cache
168+
key: ${{ steps.bazel-cache-keys.outputs.CACHE_KEY }}
169+
170+
- name: Remove old pip cache if refreshing
171+
if: env.REFRESH_CACHE == 'true'
172+
continue-on-error: true # Ignore errors when cache is not found.
173+
run: gh cache delete ${{ steps.pip-cache-key.outputs.CACHE_KEY }} --confirm
174+
175+
- name: Save pip cache
176+
uses: actions/cache/save@v4
177+
if: env.REFRESH_CACHE == 'true' || steps.pip-cache.outputs.cache-hit != 'true'
178+
with:
179+
path: ~/.cache/pip
180+
key: ${{ steps.pip-cache-key.outputs.CACHE_KEY }}

0 commit comments

Comments
 (0)