Skip to content

Commit eab41b3

Browse files
PengznaPengzna
authored
fix: improve category_X licenses & introducing validate-in-local script (#426)
1 parent 230a263 commit eab41b3

File tree

2 files changed

+357
-1
lines changed

2 files changed

+357
-1
lines changed

‎dist/validate-release-in-local.sh‎

Lines changed: 356 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,356 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
# This script is used to validate the release package, including:
19+
# 1. Check the release package name & content
20+
# 2. Check the release package sha512 & gpg signature
21+
# 3. Compile the source package & run server & toolchain
22+
# 4. Run server & toolchain in binary package
23+
24+
# exit when any error occurs
25+
set -e
26+
27+
# release version (input by committer)
28+
RELEASE_VERSION=$1 # like 1.2.0
29+
JAVA_VERSION=$2 # like 11
30+
USER=$3
31+
LOCAL_DIST_PATH=$4 # local directory path containing release files
32+
33+
# this URL is only valid during the release process
34+
SVN_URL_PREFIX="https://dist.apache.org/repos/dist/dev/incubator/hugegraph"
35+
36+
# git release branch (check it carefully)
37+
#GIT_BRANCH="release-${RELEASE_VERSION}"
38+
39+
RELEASE_VERSION=${RELEASE_VERSION:?"Please input the release version, like 1.2.0"}
40+
USER=${USER:-"imbajin"}
41+
WORK_DIR=$(
42+
cd "$(dirname "$0")"
43+
pwd
44+
)
45+
46+
# Use local directory if provided, otherwise use default dist path
47+
if [[ -n "${LOCAL_DIST_PATH}" ]]; then
48+
DIST_DIR="${LOCAL_DIST_PATH}"
49+
echo "Using local directory: ${DIST_DIR}"
50+
else
51+
DIST_DIR="${WORK_DIR}/dist/${RELEASE_VERSION}"
52+
echo "Using default directory: ${DIST_DIR}"
53+
fi
54+
55+
# Validate local directory exists
56+
if [[ ! -d "${DIST_DIR}" ]]; then
57+
echo "Error: Directory ${DIST_DIR} does not exist"
58+
exit 1
59+
fi
60+
61+
cd "${WORK_DIR}"
62+
echo "Current work dir: $(pwd)"
63+
echo "Release files directory: ${DIST_DIR}"
64+
65+
################################
66+
# Step 1: Validate Local Directory #
67+
################################
68+
cd "${DIST_DIR}"
69+
echo "Contents of ${DIST_DIR}:"
70+
ls -lh
71+
72+
##################################################
73+
# Step 2: Check Environment & Import Public Keys #
74+
##################################################
75+
shasum --version 1>/dev/null
76+
gpg --version 1>/dev/null
77+
78+
wget https://downloads.apache.org/incubator/hugegraph/KEYS
79+
echo "Import KEYS:" && gpg --import KEYS
80+
# TODO: how to trust all public keys in gpg list, currently only trust the first one
81+
echo -e "5\ny\n" | gpg --batch --command-fd 0 --edit-key $USER trust
82+
83+
echo "trust all pk"
84+
for key in $(gpg --no-tty --list-keys --with-colons | awk -F: '/^pub/ {print $5}'); do
85+
echo -e "5\ny\n" | gpg --batch --command-fd 0 --edit-key "$key" trust
86+
done
87+
88+
########################################
89+
# Step 3: Check SHA512 & GPG Signature #
90+
########################################
91+
cd "${DIST_DIR}"
92+
93+
for i in *.tar.gz; do
94+
echo "$i"
95+
shasum -a 512 --check "$i".sha512
96+
eval gpg "${GPG_OPT}" --verify "$i".asc "$i"
97+
done
98+
99+
####################################
100+
# Step 4: Validate Source Packages #
101+
####################################
102+
cd "${DIST_DIR}"
103+
104+
CATEGORY_X="\bGPL|\bLGPL|Sleepycat License|BSD-4-Clause|\bBCL\b|JSR-275|Amazon Software License|\bRSAL\b|\bQPL\b|\bSSPL|\bCPOL|\bNPL1|Creative Commons Non-Commercial|JSON\.org"
105+
CATEGORY_B="\bCDDL1|\bCPL|\bEPL|\bIPL|\bMPL|\bSPL|OSL-3.0|UnRAR License|Erlang Public License|\bOFL\b|Ubuntu Font License Version 1.0|IPA Font License Agreement v1.0|EPL2.0|CC-BY"
106+
ls -lh ./*.tar.gz
107+
for i in *src.tar.gz; do
108+
echo "$i"
109+
110+
# 4.1: check the directory name include "incubating"
111+
if [[ ! "$i" =~ "incubating" ]]; then
112+
echo "The package name $i should include incubating" && exit 1
113+
fi
114+
115+
MODULE_DIR=$(basename "$i" .tar.gz)
116+
rm -rf ${MODULE_DIR}
117+
tar -xzvf "$i"
118+
pushd ${MODULE_DIR}
119+
echo "Start to check the package content: ${MODULE_DIR}"
120+
121+
# 4.2: check the directory include "NOTICE" and "LICENSE" file and "DISCLAIMER" file
122+
if [[ ! -f "LICENSE" ]]; then
123+
echo "The package $i should include LICENSE file" && exit 1
124+
fi
125+
if [[ ! -f "NOTICE" ]]; then
126+
echo "The package $i should include NOTICE file" && exit 1
127+
fi
128+
if [[ ! -f "DISCLAIMER" ]]; then
129+
echo "The package $i should include DISCLAIMER file" && exit 1
130+
fi
131+
132+
# 4.3: ensure doesn't contains ASF CATEGORY X License dependencies in LICENSE and NOTICE files
133+
COUNT=$(grep -E "$CATEGORY_X" LICENSE NOTICE | wc -l)
134+
if [[ $COUNT -ne 0 ]]; then
135+
grep -E "$CATEGORY_X" LICENSE NOTICE
136+
echo "The package $i shouldn't include invalid ASF category X dependencies, but get $COUNT" && exit 1
137+
fi
138+
139+
# 4.4: ensure doesn't contains ASF CATEGORY B License dependencies in LICENSE and NOTICE files
140+
COUNT=$(grep -E "$CATEGORY_B" LICENSE NOTICE | wc -l)
141+
if [[ $COUNT -ne 0 ]]; then
142+
grep -E "$CATEGORY_B" LICENSE NOTICE
143+
echo "The package $i shouldn't include invalid ASF category B dependencies, but get $COUNT" && exit 1
144+
fi
145+
146+
# 4.5: ensure doesn't contains empty directory or file
147+
find . -type d -empty | while read -r EMPTY_DIR; do
148+
find . -type d -empty
149+
echo "The package $i shouldn't include empty directory: $EMPTY_DIR is empty" && exit 1
150+
done
151+
find . -type f -empty | while read -r EMPTY_FILE; do
152+
find . -type f -empty
153+
echo "The package $i shouldn't include empty file: $EMPTY_FILE is empty" && exit 1
154+
done
155+
156+
# 4.6: ensure any file should less than 800kb
157+
find . -type f -size +800k | while read -r FILE; do
158+
find . -type f -size +800k
159+
echo "The package $i shouldn't include file larger than 800kb: $FILE is larger than 800kb" && exit 1
160+
done
161+
162+
# 4.7: ensure all binary files are documented in LICENSE
163+
find . -type f | perl -lne 'print if -B' | while read -r BINARY_FILE; do
164+
FILE_NAME=$(basename "$BINARY_FILE")
165+
if grep -q "$FILE_NAME" LICENSE; then
166+
echo "Binary file $BINARY_FILE is documented in LICENSE, please check manually"
167+
else
168+
echo "Error: Binary file $BINARY_FILE is not documented in LICENSE" && exit 1
169+
fi
170+
done
171+
172+
# 4.8: test compile the packages
173+
if [[ ($JAVA_VERSION == 8 && "$i" =~ "hugegraph-computer") ]]; then
174+
echo "Skip compile $i module in java8"
175+
elif [[ "$i" =~ 'hugegraph-ai' ]]; then
176+
echo "Skip compile $i module in all versions"
177+
elif [[ "$i" =~ "hugegraph-commons" ]]; then
178+
mvn install -DskipTests -Papache-release -ntp -e
179+
elif [[ "$i" =~ "hugegraph-computer" ]]; then
180+
cd computer
181+
mvn install -DskipTests -Papache-release -ntp -e
182+
else
183+
# TODO: consider using commands that are entirely consistent with building binary packages
184+
mvn package -DskipTests -Papache-release -ntp -e
185+
ls -lh
186+
fi
187+
popd
188+
done
189+
190+
###########################################
191+
# Step 5: Run Compiled Packages of Server #
192+
###########################################
193+
cd "${DIST_DIR}"
194+
195+
ls -lh
196+
pushd ./*hugegraph-incubating*src/hugegraph-server/*hugegraph*"${RELEASE_VERSION}"
197+
bin/init-store.sh
198+
sleep 3
199+
bin/start-hugegraph.sh
200+
popd
201+
202+
#######################################################################
203+
# Step 6: Run Compiled Packages of ToolChain (Loader & Tool & Hubble) #
204+
#######################################################################
205+
cd "${DIST_DIR}"
206+
207+
pushd ./*toolchain*src
208+
ls -lh
209+
pushd ./*toolchain*"${RELEASE_VERSION}"
210+
ls -lh
211+
212+
# 6.1: load some data first
213+
echo "test loader"
214+
pushd ./*loader*"${RELEASE_VERSION}"
215+
bin/hugegraph-loader.sh -f ./example/file/struct.json -s ./example/file/schema.groovy \
216+
-g hugegraph
217+
popd
218+
219+
# 6.2: try some gremlin query & api in tool
220+
echo "test tool"
221+
pushd ./*tool*"${RELEASE_VERSION}"
222+
bin/hugegraph gremlin-execute --script 'g.V().count()'
223+
bin/hugegraph task-list
224+
bin/hugegraph backup -t all --directory ./backup-test
225+
popd
226+
227+
# 6.3: start hubble and connect to server
228+
echo "test hubble"
229+
pushd ./*hubble*"${RELEASE_VERSION}"
230+
# TODO: add hubble doc & test it
231+
cat conf/hugegraph-hubble.properties
232+
bin/start-hubble.sh
233+
bin/stop-hubble.sh
234+
popd
235+
236+
popd
237+
popd
238+
# stop server
239+
pushd ./*hugegraph-incubating*src/hugegraph-server/*hugegraph*"${RELEASE_VERSION}"
240+
bin/stop-hugegraph.sh
241+
popd
242+
243+
# clear source packages
244+
#rm -rf ./*src*
245+
#ls -lh
246+
247+
####################################
248+
# Step 7: Validate Binary Packages #
249+
####################################
250+
cd "${DIST_DIR}"
251+
252+
for i in *.tar.gz; do
253+
if [[ "$i" == *-src.tar.gz ]]; then
254+
# skip source packages
255+
continue
256+
fi
257+
258+
echo "$i"
259+
260+
# 7.1: check the directory name include "incubating"
261+
if [[ ! "$i" =~ "incubating" ]]; then
262+
echo "The package name $i should include incubating" && exit 1
263+
fi
264+
265+
MODULE_DIR=$(basename "$i" .tar.gz)
266+
rm -rf ${MODULE_DIR}
267+
tar -xzvf "$i"
268+
pushd ${MODULE_DIR}
269+
ls -lh
270+
echo "Start to check the package content: ${MODULE_DIR}"
271+
272+
# 7.2: check root dir include "NOTICE"/"LICENSE"/"DISCLAIMER" files & "licenses" dir
273+
if [[ ! -f "LICENSE" ]]; then
274+
echo "The package $i should include LICENSE file" && exit 1
275+
fi
276+
if [[ ! -f "NOTICE" ]]; then
277+
echo "The package $i should include NOTICE file" && exit 1
278+
fi
279+
if [[ ! -f "DISCLAIMER" ]]; then
280+
echo "The package $i should include DISCLAIMER file" && exit 1
281+
fi
282+
if [[ ! -d "licenses" ]]; then
283+
echo "The package $i should include licenses dir" && exit 1
284+
fi
285+
286+
# 7.3: ensure doesn't contains ASF CATEGORY X License dependencies in LICENSE/NOTICE and licenses/* files
287+
COUNT=$(grep -r -E "$CATEGORY_X" LICENSE NOTICE licenses | wc -l)
288+
if [[ $COUNT -ne 0 ]]; then
289+
grep -r -E "$CATEGORY_X" LICENSE NOTICE licenses
290+
echo "The package $i shouldn't include invalid ASF category X dependencies, but get $COUNT" && exit 1
291+
fi
292+
293+
# 7.4: ensure doesn't contains empty directory or file
294+
find . -type d -empty | while read -r EMPTY_DIR; do
295+
find . -type d -empty
296+
echo "The package $i shouldn't include empty directory: $EMPTY_DIR is empty" && exit 1
297+
done
298+
find . -type f -empty | while read -r EMPTY_FILE; do
299+
find . -type f -empty
300+
echo "The package $i shouldn't include empty file: $EMPTY_FILE is empty" && exit 1
301+
done
302+
303+
popd
304+
done
305+
306+
# TODO: skip the following steps by comparing the artifacts built from source packages with binary packages
307+
#########################################
308+
# Step 8: Run Binary Packages of Server #
309+
#########################################
310+
cd "${DIST_DIR}"
311+
312+
# TODO: run pd & store
313+
pushd ./*hugegraph-incubating*"${RELEASE_VERSION}"/*hugegraph-server-incubating*"${RELEASE_VERSION}"
314+
bin/init-store.sh
315+
sleep 3
316+
bin/start-hugegraph.sh
317+
popd
318+
319+
#####################################################################
320+
# Step 9: Run Binary Packages of ToolChain (Loader & Tool & Hubble) #
321+
#####################################################################
322+
cd "${DIST_DIR}"
323+
324+
pushd ./*toolchain*"${RELEASE_VERSION}"
325+
ls -lh
326+
327+
# 9.1: load some data first
328+
echo "test loader"
329+
pushd ./*loader*"${RELEASE_VERSION}"
330+
bin/hugegraph-loader.sh -f ./example/file/struct.json -s ./example/file/schema.groovy -g hugegraph
331+
popd
332+
333+
# 9.2: try some gremlin query & api in tool
334+
echo "test tool"
335+
pushd ./*tool*"${RELEASE_VERSION}"
336+
bin/hugegraph gremlin-execute --script 'g.V().count()'
337+
bin/hugegraph task-list
338+
bin/hugegraph backup -t all --directory ./backup-test
339+
popd
340+
341+
# 9.3: start hubble and connect to server
342+
echo "test hubble"
343+
pushd ./*hubble*"${RELEASE_VERSION}"
344+
# TODO: add hubble doc & test it
345+
cat conf/hugegraph-hubble.properties
346+
bin/start-hubble.sh
347+
bin/stop-hubble.sh
348+
popd
349+
350+
popd
351+
# stop server
352+
pushd ./*hugegraph-incubating*"${RELEASE_VERSION}"/*hugegraph-server-incubating*"${RELEASE_VERSION}"
353+
bin/stop-hugegraph.sh
354+
popd
355+
356+
echo "Finish validate, please check all steps manually again!"

‎dist/validate-release.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ done
8585
####################################
8686
cd "${WORK_DIR}/dist/${RELEASE_VERSION}"
8787

88-
CATEGORY_X="\bGPL|\bLGPL|Sleepycat License|BSD-4-Clause|\bBCL\b|JSR-275|Amazon Software License|\bRSAL\b|\bQPL\b|\bSSPL|\bCPOL|\bNPL1|Creative Commons Non-Commercial|JSON"
88+
CATEGORY_X="\bGPL|\bLGPL|Sleepycat License|BSD-4-Clause|\bBCL\b|JSR-275|Amazon Software License|\bRSAL\b|\bQPL\b|\bSSPL|\bCPOL|\bNPL1|Creative Commons Non-Commercial|JSON\.org"
8989
CATEGORY_B="\bCDDL1|\bCPL|\bEPL|\bIPL|\bMPL|\bSPL|OSL-3.0|UnRAR License|Erlang Public License|\bOFL\b|Ubuntu Font License Version 1.0|IPA Font License Agreement v1.0|EPL2.0|CC-BY"
9090
ls -lh ./*.tar.gz
9191
for i in *src.tar.gz; do

0 commit comments

Comments
 (0)