Skip to content

Commit 723372f

Browse files
mwestphalMeakk
mwestphal
and
Meakk
authored
Add action (#3)
 Co-authored-by: Michael MIGLIORE <[email protected]>
1 parent 82ccd49 commit 723372f

File tree

5 files changed

+303
-1
lines changed

5 files changed

+303
-1
lines changed

‎.github/workflows/check_size.cmake‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Check that a LFS data file is big enough to be an actual file
2+
file(SIZE "${CMAKE_ARGV3}" lfs_file_size)
3+
if (lfs_file_size LESS_EQUAL 500)
4+
message(FATAL_ERROR "File is smaller than 500 bytes, lfs data not correctly recovered")
5+
endif()

‎.github/workflows/ci.yml‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
name: CI
2+
3+
on:
4+
pull_request:
5+
types: [opened, synchronize, reopened, ready_for_review]
6+
push:
7+
branches:
8+
- main
9+
10+
concurrency:
11+
group: '${{ github.workflow }}-${{ github.ref_name }}'
12+
cancel-in-progress: true
13+
14+
jobs:
15+
cache_lfs:
16+
runs-on: ubuntu-latest
17+
name: Update LFS data cache
18+
outputs:
19+
lfs_sha: ${{ steps.lfs_sha.outputs.lfs_sha }}
20+
steps:
21+
22+
- name: Checkout
23+
uses: actions/checkout@v4
24+
with:
25+
path: 'lfs-data-cache_action'
26+
fetch-depth: 0
27+
28+
- name: Use lfs-data-cache action
29+
id: lfs-data-cache
30+
uses: ./lfs-data-cache_action
31+
with:
32+
type: 'producer'
33+
repository: 'f3d-app/f3d'
34+
cache_postfix: 'ci-cache'
35+
36+
- name: Set output
37+
id: lfs_sha
38+
shell: bash
39+
run: echo "lfs_sha=$(steps.lfs-data-cache.outputs.lfs_sha)" >> $GITHUB_OUTPUT
40+
41+
ci:
42+
name: CI
43+
needs: cache_lfs
44+
45+
strategy:
46+
fail-fast: false
47+
matrix:
48+
os: [ubuntu-latest, windows-latest, macos-13]
49+
50+
runs-on: ${{matrix.os}}
51+
52+
steps:
53+
54+
- name: Output directory
55+
shell: bash
56+
run: mkdir output_dir
57+
58+
- name: Checkout
59+
uses: actions/checkout@v4
60+
with:
61+
path: 'lfs-data-cache_action'
62+
fetch-depth: 0
63+
64+
- name: Use lfs-data-cache action
65+
uses: ./lfs-data-cache_action
66+
with:
67+
type: 'consumer'
68+
repository: 'f3d-app/f3d'
69+
lfs_sha: ${{ needs.cache_lfs.outputs.lfs_sha}}
70+
cache_postfix: 'ci-cache'
71+
target_directory: 'output_dir'
72+
73+
- name: Check output has expected size
74+
shell: bash
75+
run: cmake -P ./lfs-data-cache_action/.github/workflows/check_size.cmake output_dir/testing/data/f3d.vtp

‎README.md‎

Lines changed: 90 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,91 @@
11
# lfs-data-cache-action
2-
A github action to cache and recover LFS data
2+
3+
A producer/consumer github action to cache and recover LFS data.
4+
5+
Its main use is to avoid cloning LFS data in CI to avoid
6+
having to pay for LFS bandwidth because of CI needs.
7+
8+
It expects cmake to be available on the host.
9+
10+
The action can be used as a consumer or a producer, and must
11+
provide the repository containing the LFS data to recover from.
12+
13+
It is possible to provide a specific SHA to produce.
14+
If not provided, the last commit modyfing the LFS file will be produced.
15+
16+
Is has the following inputs:
17+
18+
- `type`: should be `producer` or `consumer`, default to `producer`
19+
- `repository`: the git repository to produce LFS data from, default: ${{ github.repository }}
20+
- `lfs_sha`: The git sha to recover LFS data from, optional
21+
- `cache_postfix`: An postfix added to the cache name, to support multiple caches, default to `cache`
22+
- `target_directory`: A target directory to copy LFS data to
23+
24+
## Logic
25+
26+
Producer/Consumer first use the classic cache action to recover a cache named
27+
`lfs-data-${{lfs_sha}}-${{cache_index}}`.
28+
29+
If Producer does not found it, it will clone the `repository` at `lfs_sha` commit
30+
and upload the content as an artifact.
31+
32+
If Consumer does not found it, it will try to download a potential artifact
33+
produced earlier by the Producer.
34+
35+
If it fails Consumer will clone the `repository` at `lfs_sha` commit.
36+
37+
Finally, Producer/Consumer will copy the LFS data only using cmake to the `target_directory`
38+
39+
## Usage
40+
41+
In an first job, use the `producer` action, which output the LFS sha that will be produced
42+
In a second job, usually a matrix job, depending on the first,
43+
recover the LFS sha from first job and use the `consumer` action.
44+
45+
```
46+
jobs:
47+
48+
#----------------------------------------------------------------------------
49+
# Cache LFS: Checkout LFS data and update the cache to limit LFS bandwidth
50+
#----------------------------------------------------------------------------
51+
cache_lfs:
52+
runs-on: ubuntu-latest
53+
name: Update LFS data cache
54+
outputs:
55+
lfs_sha: ${{ steps.lfs_sha_recover.outputs.lfs_sha }}
56+
steps:
57+
58+
# Checkout your repository WITHOUT LFS
59+
- name: Checkout
60+
uses: actions/checkout@v3
61+
with:
62+
path: 'source'
63+
fetch-depth: 1
64+
lfs: false
65+
66+
# Use producer action to recover the LFS data and upload it as cache/artifacts
67+
- name: Cache LFS Data
68+
id: lfs_sha_recover
69+
uses: f3d-app/lfs-data-cache-action:v1
70+
71+
#----------------------------------------------------------------------------
72+
# Actual CI: Recover LFS data first
73+
#----------------------------------------------------------------------------
74+
75+
recover_lfs:
76+
needs: cache_lfs
77+
78+
# Checkout your repository WITHOUT LFS
79+
- name: Checkout
80+
uses: actions/checkout@v3
81+
with:
82+
path: 'source'
83+
fetch-depth: 0
84+
lfs: false
85+
86+
- name: Recover LFS Data
87+
uses: f3d-app/lfs-data-cache-action:v1
88+
with:
89+
workflow_label: 'consumer'
90+
lfs_sha: ${{ needs.cache_lfs.outputs.lfs_sha}}
91+
```

‎action.yml‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
name: 'Copy LFS data'
2+
description: 'Copy LFS data using cache when possible'
3+
inputs:
4+
type:
5+
description: 'Whether this action produce/consume LFS data'
6+
default: 'producer'
7+
repository:
8+
description: 'Repository to recover LFS data from'
9+
required: false
10+
default: ${{ github.repository }}
11+
lfs_sha:
12+
description: 'LFS sha to recover. If not set, target directory will be used to recover it.'
13+
required: false
14+
cache_postfix:
15+
description: 'A postfix to differentiate between caches'
16+
default: 'cache'
17+
target_directory:
18+
description: 'Existing directory to copy LFS data to'
19+
default: 'source'
20+
outputs:
21+
lfs_sha:
22+
description: 'The lfs_sha generated by a producer action'
23+
value: ${{ steps.lfs_sha_recover.outputs.lfs_sha }}
24+
25+
runs:
26+
using: "composite"
27+
steps:
28+
29+
- name: Check required inputs
30+
shell: bash
31+
run: |
32+
[[ "${{ inputs.repository }}" ]] || { echo "repository input is empty" ; exit 1; }
33+
34+
- name: Create a working directory
35+
working-directory: ${{github.workspace}}
36+
shell: bash
37+
run: mkdir lfs_data_cache
38+
39+
- name: Checkout repository without LFS data
40+
if: inputs.lfs_sha == ''
41+
uses: actions/checkout@v4
42+
with:
43+
repository: ${{ inputs.repository }}
44+
path: 'lfs_data_cache/lfs_source'
45+
fetch-depth: 0
46+
lfs: false
47+
48+
- name: Set LFS sha env var from repository
49+
working-directory: ${{github.workspace}}/lfs_data_cache/lfs_source
50+
if: inputs.lfs_sha == ''
51+
shell: bash
52+
run: echo "lfs_data_cache_sha=$(git log -n 1 --pretty=format:%H -- `git-lfs ls-files -n`)" >> $GITHUB_ENV
53+
54+
- name: Set LFS sha env var from inputs
55+
if: inputs.lfs_sha != ''
56+
shell: bash
57+
run: echo "lfs_data_cache_sha=${{inputs.lfs_sha}}" >> $GITHUB_ENV
58+
59+
- name: Cache LFS data
60+
id: cache-lfs
61+
uses: actions/cache@v4
62+
with:
63+
path: 'lfs_data_cache/lfs_data'
64+
key: lfs-data-${{env.lfs_data_cache_sha}}-${{inputs.cache_postfix}}
65+
66+
- name: Set LFS output sha
67+
id: lfs_sha_recover
68+
shell: bash
69+
run: echo "lfs_sha=${{env.lfs_data_cache_sha}}" >> $GITHUB_OUTPUT
70+
71+
- name: Checkout LFS data for artifact producer
72+
if: |
73+
steps.cache-lfs.outputs.cache-hit != 'true' &&
74+
inputs.type == 'producer'
75+
uses: actions/checkout@v4
76+
with:
77+
repository: ${{ inputs.repository }}
78+
ref: ${{env.lfs_data_cache_sha}}
79+
path: 'lfs_data_cache/lfs_data'
80+
fetch-depth: 0
81+
lfs: true
82+
83+
- name: Upload LFS artifact
84+
if: |
85+
steps.cache-lfs.outputs.cache-hit != 'true' &&
86+
inputs.type == 'producer'
87+
uses: actions/upload-artifact@v4
88+
with:
89+
name: lfs-data-${{inputs.cache_postfix}}
90+
path: 'lfs_data_cache/lfs_data'
91+
overwrite: true
92+
include-hidden-files: true
93+
94+
- name: Download LFS artifact
95+
id: download-artifact
96+
if: |
97+
steps.cache-lfs.outputs.cache-hit != 'true' &&
98+
inputs.type == 'consumer'
99+
uses: actions/download-artifact@v4
100+
continue-on-error: true
101+
with:
102+
name: lfs-data-${{inputs.cache_postfix}}
103+
path: 'lfs_data_cache/lfs_data'
104+
105+
- name: Checkout LFS data (last resort)
106+
if: |
107+
steps.cache-lfs.outputs.cache-hit != 'true' &&
108+
steps.download-artifact.outcome != 'success' &&
109+
inputs.type == 'consumer'
110+
uses: actions/checkout@v4
111+
with:
112+
repository: ${{ inputs.repository }}
113+
ref: ${{env.lfs_data_cache_sha}}
114+
path: 'lfs_data_cache/lfs_data'
115+
fetch-depth: 0
116+
lfs: true
117+
118+
- name: Setup LFS data
119+
working-directory: ${{github.workspace}}
120+
shell: bash
121+
run: cmake -P $GITHUB_ACTION_PATH/copy_lfs.cmake 'lfs_data_cache/lfs_data' ${{ inputs.target_directory }}

‎copy_lfs.cmake‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
find_package(Git QUIET)
2+
execute_process(
3+
COMMAND ${GIT_EXECUTABLE} lfs ls-files -n
4+
WORKING_DIRECTORY ${CMAKE_ARGV3}
5+
OUTPUT_VARIABLE LFS_FILES
6+
OUTPUT_STRIP_TRAILING_WHITESPACE)
7+
string(REPLACE "\n" ";" LFS_FILES ${LFS_FILES})
8+
9+
foreach(LFS_FILE ${LFS_FILES})
10+
cmake_path(GET LFS_FILE PARENT_PATH LFS_PATH)
11+
file(COPY "${CMAKE_ARGV3}/${LFS_FILE}" DESTINATION "${CMAKE_ARGV4}/${LFS_PATH}")
12+
endforeach()

0 commit comments

Comments
 (0)