Skip to content

Commit f046887

Browse files
juanfeSanahujajuanfeSanahuja
committed
cicd: Added checkout to pull the code of the ci-core repository to get the scripts #TASK-8067
1 parent 587b305 commit f046887

File tree

3 files changed

+168
-8
lines changed

3 files changed

+168
-8
lines changed

‎.github/actions/setup-java-maven/action.yml‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,18 @@ outputs:
4646
runs:
4747
using: "composite"
4848
steps:
49+
- uses: actions/checkout@v4
50+
if: ${{ inputs.dependency_repos != '' }}
51+
name: Checkout ci-core repository
52+
id: "checkout-ci-core"
53+
## This checkout pulls the code of the ci-core repository to get the scripts
54+
## Checkout to "ci-core" folder.
55+
## Filter by ".github" folder
56+
with:
57+
repository: opencb/ci-core
58+
ref: main
59+
path: ci-core
60+
fetch-depth: '1'
4961
- name: Set up JDK
5062
if: ${{ inputs.deploy_maven != 'true' }}
5163
uses: actions/setup-java@v4
@@ -77,14 +89,14 @@ runs:
7789
# - update DEPENDENCIES_SHA in-place
7890
# - finally write `dependencies_sha=<hash>` into $GITHUB_OUTPUT
7991
80-
if [ -f "java-common-libs/.github/workflows/scripts/get_same_branch.sh" ]; then
81-
chmod +x java-common-libs/.github/workflows/scripts/get_same_branch.sh
92+
if [ -f "ci-core/.github/workflows/scripts/get_same_branch.sh" ]; then
93+
chmod +x ci-core/.github/workflows/scripts/get_same_branch.sh
8294
8395
# Initialize DEPENDENCIES_SHA with the current repo SHA so that
8496
# main project changes also affect the final cache key.
8597
export DEPENDENCIES_SHA="${{ github.sha }}"
8698
87-
java-common-libs/.github/workflows/scripts/get_same_branch.sh \
99+
ci-core/.github/workflows/scripts/get_same_branch.sh \
88100
"${{ github.ref_name }}" \
89101
"${{ inputs.dependency_repos }}"
90102
else
@@ -118,18 +130,18 @@ runs:
118130
# - run `mvn clean install -DskipTests` on each dependency
119131
# - populate ~/.m2/repository with the latest built artifacts
120132
121-
if [ -f "java-common-libs/.github/workflows/scripts/compile_same_branch.sh" ]; then
122-
chmod +x java-common-libs/.github/workflows/scripts/compile_same_branch.sh
123-
java-common-libs/.github/workflows/scripts/compile_same_branch.sh "${{ inputs.dependency_repos }}"
133+
if [ -f "ci-core/.github/workflows/scripts/compile_same_branch.sh" ]; then
134+
chmod +x ci-core/.github/workflows/scripts/compile_same_branch.sh
135+
ci-core/.github/workflows/scripts/compile_same_branch.sh "${{ inputs.dependency_repos }}"
124136
else
125137
echo "compile_same_branch.sh script not found. Skipping dependency compilation."
126138
fi
127139
- name: Prepare Hadoop profile
128140
if: ${{ inputs.needs_hadoop_preparation == 'true' }}
129141
shell: bash
130142
run: |
131-
chmod +x ./.github/workflows/scripts/prepare_hadoop.sh
132-
./.github/workflows/scripts/prepare_hadoop.sh --hadoop-flavour "${{ inputs.storage_hadoop }}"
143+
chmod +x ci-core/.github/workflows/scripts/prepare_hadoop.sh
144+
ci-core/.github/workflows/scripts/prepare_hadoop.sh --hadoop-flavour "${{ inputs.storage_hadoop }}"
133145
env:
134146
THIRDPARTY_READ_TOKEN: ${{ env.THIRDPARTY_READ_TOKEN }}
135147
- name: Cache summary

‎.github/workflows/scripts/get_same_branch.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash
2+
3+
BRANCH_NAME=$1
4+
## Comma separated list of repos to checkout
5+
REPOS=$2
6+
DEPENDENCIES_SHA=${DEPENDENCIES_SHA:-""}
7+
WORKSPACE=${WORKSPACE:-/home/runner/work/}
8+
9+
if [[ -z "$BRANCH_NAME" ]]; then
10+
echo "The first parameter is mandatory and must be a valid branch name."
11+
exit 1
12+
fi
13+
14+
function checkout(){
15+
local REPO=$1
16+
echo "::group::Installing '$REPO' project from branch $BRANCH_NAME"
17+
cd "${WORKSPACE}" || exit 2
18+
git clone https://github.com/opencb/"$REPO".git -b "$BRANCH_NAME"
19+
if [ -d "./$REPO" ]; then
20+
cd "$REPO" || exit 2
21+
DEPENDENCIES_SHA=${DEPENDENCIES_SHA}:$(git rev-parse HEAD)
22+
echo "Branch name $BRANCH_NAME already exists."
23+
else
24+
echo "Branch name $BRANCH_NAME does not exist in $REPO repository. Skipping installation."
25+
fi
26+
echo "::endgroup::"
27+
}
28+
29+
IFS=',' read -ra REPO_ARRAY <<<"$REPOS"
30+
for REPO in "${REPO_ARRAY[@]}"; do
31+
checkout "$REPO"
32+
done
33+
34+
## Apply sha1 to DEPENDENCIES_SHA if contains `:`
35+
if [[ "$DEPENDENCIES_SHA" == *":"* ]]; then
36+
DEPENDENCIES_SHA=$(echo -n "$DEPENDENCIES_SHA" | sha1sum | awk '{print $1}')
37+
fi
38+
39+
## Export DEPENDENCIES_SHA as github output
40+
echo "dependencies_sha=$DEPENDENCIES_SHA" >> "$GITHUB_OUTPUT"

‎.github/workflows/scripts/prepare_hadoop.sh‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#!/bin/bash
2+
3+
set -e
4+
set -o pipefail
5+
6+
7+
function main() {
8+
HADOOP_FLAVOUR="hbase2.0"
9+
10+
while [[ $# -gt 0 ]]; do
11+
key="$1"
12+
value="$2"
13+
case $key in
14+
--hadoop-flavour)
15+
HADOOP_FLAVOUR="$value"
16+
shift # past argument
17+
shift # past value
18+
;;
19+
--hadoop-thirdparty-version)
20+
HADOOP_THIRDPARTY_VERSION="$value"
21+
shift # past argument
22+
shift # past value
23+
;;
24+
--verbose)
25+
set -x
26+
shift # past key
27+
;;
28+
*) # unknown option
29+
echo "Unknown option $key"
30+
return 1
31+
;;
32+
esac
33+
done
34+
35+
if [ -z "$HADOOP_THIRDPARTY_VERSION" ]; then
36+
HADOOP_THIRDPARTY_VERSION=$(mvn help:evaluate -Dexpression=opencga.hadoop.thirdparty.version -q -DforceStdout)
37+
fi
38+
39+
# Check if HADOOP_THIRDPARTY can be download
40+
ARTIFACT="org.opencb.opencga.hadoop.thirdparty:opencga-hadoop-shaded-${HADOOP_FLAVOUR}:${HADOOP_THIRDPARTY_VERSION}"
41+
echo "Looking for artifact:"
42+
echo " - $ARTIFACT"
43+
if mvn dependency:get "-Dartifact=${ARTIFACT}" &> /dev/null; then
44+
echo "Hadoop thirdparty jar found."
45+
return 0;
46+
fi
47+
48+
echo "Hadoop thirdparty jar not found in local maven repository. Building opencga-hadoop-thirdparty..."
49+
local GIT_REF=
50+
if [[ "$HADOOP_THIRDPARTY_VERSION" == *"-SNAPSHOT" ]]; then
51+
local VERSION=$(echo "$HADOOP_THIRDPARTY_VERSION" | cut -d "-" -f 1)
52+
local MAJOR=$(echo "$VERSION" | cut -d "." -f 1)
53+
local MINOR=$(echo "$VERSION" | cut -d "." -f 2)
54+
local PATCH=$(echo "$VERSION" | cut -d "." -f 3)
55+
if [ $PATCH -gt 0 ]; then ## It's a hotfix
56+
GIT_REF="release-$MAJOR.$MINOR.x"
57+
elif [ $MINOR -eq 0 ]; then ## It's a develop branch
58+
GIT_REF="develop"
59+
else ## It's a release branch
60+
GIT_REF="release-$MAJOR.x.x"
61+
fi
62+
else
63+
GIT_REF="v$HADOOP_THIRDPARTY_VERSION"
64+
fi
65+
install "$GIT_REF" "$HADOOP_FLAVOUR"
66+
if [ $? -ne 0 ]; then
67+
echo "Failed to build opencga-hadoop-thirdparty."
68+
return 1
69+
fi
70+
71+
return 0;
72+
}
73+
74+
75+
function install(){
76+
local GIT_REF=${1:?"Git reference (branch, tag) is required"}
77+
local HADOOP=${2:?"Hadoop flavour is required"}
78+
echo "Installing $HADOOP hadoop flavour from $GIT_REF"
79+
local REPO="opencga-hadoop-thirdparty"
80+
local TMP_DIR_HOME="dependency-checkouts/"
81+
mkdir -p "$TMP_DIR_HOME"
82+
rm -rf "${TMP_DIR_HOME:?}"/*
83+
TEMP_DIR="$(mktemp -d --tmpdir="$TMP_DIR_HOME" --suffix="$(date +%Y%m%d%H%M%S)-$REPO")"
84+
cd "$TEMP_DIR" || return 2
85+
echo "Cloning repository $REPO with ref $GIT_REF"
86+
87+
# Build HTTPS clone URL using optional token for private access
88+
local CLONE_URL
89+
if [[ -n "${THIRDPARTY_READ_TOKEN:-}" ]]; then
90+
CLONE_URL="https://x-access-token:${THIRDPARTY_READ_TOKEN}@github.com/opencb/${REPO}.git"
91+
else
92+
CLONE_URL="git@github.com:opencb/${REPO}.git"
93+
fi
94+
95+
# Shallow clone at the requested ref
96+
git clone --depth 1 -b "$GIT_REF" "$CLONE_URL"
97+
PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
98+
echo "Cloned repository $REPO with ref $GIT_REF and version $PROJECT_VERSION"
99+
cd "$REPO" || return 2
100+
./dev/build.sh "$HADOOP"
101+
cd - || return 2
102+
}
103+
104+
105+
106+
main "$@"
107+
exit $?
108+

0 commit comments

Comments
 (0)