Skip to content

Commit 0010046

Browse files
udesouudesou
authored
Adding CI support (#256)
1 parent 1649090 commit 0010046

File tree

13 files changed

+176
-95
lines changed

13 files changed

+176
-95
lines changed

‎.github/scripts/ci-build.sh‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ fi
1212
build_type=$1
1313
# plan to use
1414
plan=$2
15-
# moving vs non-moving
15+
# max-moving vs non-moving
1616
is_moving=$3
1717

1818
# helloworld.jl
@@ -27,10 +27,14 @@ fi
2727
plan_feature=${plan,,}
2828
moving_feature=${is_moving,,}
2929

30-
if [ "$is_moving" == "moving" ]; then
30+
if [ "$moving_feature" == "max_moving" ]; then
3131
MOVING=1
32+
ALWAYS_MOVING=1
33+
MAX_MOVING=1
3234
else
3335
MOVING=0
36+
ALWAYS_MOVING=0
37+
MAX_MOVING=0
3438
fi
3539

3640
cd $JULIA_PATH
@@ -39,6 +43,6 @@ make cleanall
3943
# This will build the binding in MMTK_JULIA_DIR (defined in common.sh), and link it
4044
# when building Julia, instead of using the set version defined in Julia itself
4145
cp $BINDING_PATH/.github/scripts/Make.user $JULIA_PATH/
42-
MMTK_MOVING=$MOVING MMTK_PLAN=$plan MMTK_BUILD=$build_type make
46+
MMTK_MOVING=$MOVING MMTK_ALWAYS_MOVING=$ALWAYS_MOVING MMTK_MAX_MOVING=$MAX_MOVING MMTK_PLAN=$plan MMTK_BUILD=$build_type make
4347
# Run hello world
4448
$JULIA_PATH/julia $HELLO_WORLD_JL

‎.github/scripts/ci-test-LinearAlgebra.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,9 @@ total_mem_restricted=$((total_mem- mem_threshold))
1717
num_workers=2
1818
export JULIA_TEST_MAXRSS_MB=$((total_mem_restricted/ num_workers))
1919

20+
# max-moving vs non-moving
21+
is_moving=$2
22+
moving_feature=${is_moving,,}
23+
2024
echo "-> Run single threaded"
21-
ci_run_jl_test "LinearAlgebra" 2
25+
ci_run_jl_test "LinearAlgebra" 2 $moving_feature

‎.github/scripts/ci-test-other.sh‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,27 @@ CHOOSE_TESTS_JL_CONTENT=`cat $CHOOSE_TESTS_JL_PATH`
1313

1414
REGEX_PATTERN='.*const TESTNAMES = \[([^\[]*)^\].*'
1515

16+
# max-moving vs non-moving
17+
is_moving=$2
18+
moving_feature=${is_moving,,}
19+
20+
declare -a tests_to_skip=(
21+
# see https://github.com/mmtk/mmtk-julia/issues/259
22+
"atomics"
23+
"abstractarray"
24+
"Artifacts"
25+
"cmdlineargs"
26+
"Downloads"
27+
"read"
28+
"threads"
29+
"LibCURL"
30+
"subarray"
31+
"rounding"
32+
"loading"
33+
"compileall"
34+
"misc"
35+
)
36+
1637
if [[ $CHOOSE_TESTS_JL_CONTENT =~ $REGEX_PATTERN ]]; then
1738
RAW_TEST_NAMES=${BASH_REMATCH[1]}
1839

@@ -22,7 +43,7 @@ if [[ $CHOOSE_TESTS_JL_CONTENT =~ $REGEX_PATTERN ]]; then
2243
for i in "${test_names[@]}"
2344
do
2445
# echo "Token: '$i'"
25-
test=`sed 's/\"\(.*\)\"/\1/' <<< $i`
46+
test=$(sed 's/\"\(.*\)\"/\1/' <<< "$i" | xargs)
2647
if [[ ! -z "$test" ]]; then
2748
echo $test
2849

@@ -33,14 +54,19 @@ if [[ $CHOOSE_TESTS_JL_CONTENT =~ $REGEX_PATTERN ]]; then
3354
continue
3455
fi
3556

57+
if [[ "${tests_to_skip[@]}" =~ "$test" ]]; then
58+
echo "-> Skip"
59+
continue
60+
fi
61+
3662
if [[ $test =~ "compiler_extras" ]]; then
3763
# Skipping compiler_extras for now
3864
echo "-> Skip compiler_extras"
3965
continue
4066
fi
4167

4268
echo "-> Run"
43-
ci_run_jl_test $test
69+
ci_run_jl_test $test 1 $moving_feature
4470
fi
4571
done
4672
else

‎.github/scripts/ci-test-stdlib.sh‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,18 @@ declare -a tests_to_skip=(
2929
"LinearAlgebra"
3030
# Skipping Distributed tests
3131
"Distributed"
32+
33+
# Skipping tests that fail for max moving Immix
34+
# see https://github.com/mmtk/mmtk-julia/issues/259
35+
"Artifacts"
36+
"Downloads"
37+
"REPL"
38+
"TOML"
39+
"Random"
40+
"LibCURL"
41+
"Mmap"
42+
"SharedArrays"
43+
"LazyArtifacts"
3244
)
3345
# These tests need multiple workers.
3446
declare -a tests_with_multi_workers=(
@@ -51,6 +63,10 @@ if [ $(find $stdlib_path -mindepth 1 -maxdepth 1 | wc -l) -ne 1 ]; then
5163
exit 1
5264
fi
5365

66+
# max-moving vs non-moving
67+
is_moving=$2
68+
moving_feature=${is_moving,,}
69+
5470
for dir in $(find $stdlib_version_path -depth -mindepth 1 -type d -o -type l)
5571
do
5672
# if there is a runtests.jl, we run it.
@@ -67,16 +83,16 @@ do
6783

6884
if [[ "${tests_with_multi_workers[@]}" =~ "$test" ]]; then
6985
echo "-> Run multi threaded"
70-
ci_run_jl_test $test 2
86+
ci_run_jl_test $test 2 $moving_feature
7187
continue
7288
fi
7389

7490
if [[ "${tests_with_single_worker[@]}" =~ "$test" ]]; then
7591
echo "-> Run single threaded"
76-
ci_run_jl_test $test 1
92+
ci_run_jl_test $test 1 $moving_feature
7793
continue
7894
fi
7995

80-
ci_run_jl_test $test
96+
ci_run_jl_test $test 1 $moving_feature
8197
fi
8298
done

‎.github/scripts/common.sh‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,18 @@ export JULIA_TEST_MAXRSS_MB=$total_mem
1717
ci_run_jl_test() {
1818
test=$1
1919
threads=$2
20+
moving_feature=$3
21+
22+
23+
if [ "$moving_feature" == "max_moving" ]; then
24+
MOVING=1
25+
ALWAYS_MOVING=1
26+
MAX_MOVING=1
27+
else
28+
MOVING=0
29+
ALWAYS_MOVING=0
30+
MAX_MOVING=0
31+
fi
2032

2133
# if no argument is given, use 2 as default
2234
if [ -z "$threads" ]; then
@@ -29,6 +41,13 @@ ci_run_jl_test() {
2941
# Directly run runtests.jl: There could be some issues with some test suites. We better just use their build script.
3042
# $JULIA_PATH/julia $JULIA_TEST_ARGS $JULIA_PATH/test/runtests.jl --exit-on-error $test
3143

32-
# Run with their build script
33-
make test-$test
44+
# if we run with a single thread, we could use Julia's own heap resizing to potentially avoid OOMs
45+
if [ "$threads" == "1" ]; then
46+
# Run with their build script
47+
MMTK_MIN_HSIZE_G=0 MMTK_MAX_HSIZE_G=0 MMTK_MOVING=$MOVING MMTK_ALWAYS_MOVING=$ALWAYS_MOVING MMTK_MAX_MOVING=$MAX_MOVING make test-$test
48+
else
49+
# Run with their build script
50+
MMTK_MOVING=$MOVING MMTK_ALWAYS_MOVING=$ALWAYS_MOVING MMTK_MAX_MOVING=$MAX_MOVING make test-$test
51+
fi
52+
3453
}

‎.github/workflows/binding-tests.yml‎

Lines changed: 48 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -39,51 +39,52 @@ jobs:
3939
run: |
4040
./.github/scripts/ci-style.sh
4141
42-
build-debug-regen-ffi:
43-
runs-on: ubuntu-22.04
44-
steps:
45-
- uses: actions/checkout@v2
46-
- name: Setup environments
47-
run: |
48-
./.github/scripts/ci-checkout.sh ${{ inputs.julia_repo }} ${{ inputs.julia_ref }}
49-
./.github/scripts/ci-setup.sh
50-
# removing these as they cause a conflict within bindgen
51-
sudo rm -rf /usr/lib/llvm-14
52-
sudo rm -rf /usr/lib/llvm-13
53-
- name: Check FFI bindings
54-
run: |
55-
./.github/scripts/ci-ffi.sh
56-
- name: Build Julia (Debug)
57-
run: |
58-
./.github/scripts/ci-build.sh debug ${{ inputs.gc_plan }} ${{ inputs.moving }}
59-
- name: Print Julia version info
60-
run: |
61-
JULIA_PATH=$(realpath $(dirname "$0"))/../mmtk-julia/mmtk-julia/vm/julia
62-
$JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()"
63-
- name: Style check
64-
run: |
65-
./.github/scripts/ci-style.sh
42+
# Commenting these out since they shouldn't be needed in the mmtk-support-moving-upstream branch
43+
# build-debug-regen-ffi:
44+
# runs-on: ubuntu-22.04
45+
# steps:
46+
# - uses: actions/checkout@v2
47+
# - name: Setup environments
48+
# run: |
49+
# ./.github/scripts/ci-checkout.sh ${{ inputs.julia_repo }} ${{ inputs.julia_ref }}
50+
# ./.github/scripts/ci-setup.sh
51+
# # removing these as they cause a conflict within bindgen
52+
# sudo rm -rf /usr/lib/llvm-14
53+
# sudo rm -rf /usr/lib/llvm-13
54+
# - name: Check FFI bindings
55+
# run: |
56+
# ./.github/scripts/ci-ffi.sh
57+
# - name: Build Julia (Debug)
58+
# run: |
59+
# ./.github/scripts/ci-build.sh debug ${{ inputs.gc_plan }} ${{ inputs.moving }}
60+
# - name: Print Julia version info
61+
# run: |
62+
# JULIA_PATH=$(realpath $(dirname "$0"))/../mmtk-julia/mmtk-julia/vm/julia
63+
# $JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()"
64+
# - name: Style check
65+
# run: |
66+
# ./.github/scripts/ci-style.sh
6667

67-
check-ffi:
68-
runs-on: ubuntu-22.04
69-
steps:
70-
- uses: actions/checkout@v2
71-
- name: Setup environments
72-
run: |
73-
./.github/scripts/ci-checkout.sh ${{ inputs.julia_repo }} ${{ inputs.julia_ref }}
74-
./.github/scripts/ci-setup.sh
75-
# removing these as they cause a conflict within bindgen
76-
sudo rm -rf /usr/lib/llvm-14
77-
sudo rm -rf /usr/lib/llvm-13
78-
- name: Regenerate FFI bindings
79-
run: |
80-
./.github/scripts/ci-ffi.sh
81-
- run: |
82-
MMTK_JULIA_DIR=$(realpath $(dirname "$0"))/../mmtk-julia/mmtk-julia
83-
if ! git diff --exit-code $MMTK_JULIA_DIR/mmtk/src/julia_types.rs; then
84-
echo "Rust FFI bindings in \`julia_types.rs\` are outdated. Run \`make regen-bindgen-ffi\` from the mmtk-julia directory and make sure to include the updated file in the pull request."
85-
exit 1
86-
fi
68+
# check-ffi:
69+
# runs-on: ubuntu-22.04
70+
# steps:
71+
# - uses: actions/checkout@v2
72+
# - name: Setup environments
73+
# run: |
74+
# ./.github/scripts/ci-checkout.sh ${{ inputs.julia_repo }} ${{ inputs.julia_ref }}
75+
# ./.github/scripts/ci-setup.sh
76+
# # removing these as they cause a conflict within bindgen
77+
# sudo rm -rf /usr/lib/llvm-14
78+
# sudo rm -rf /usr/lib/llvm-13
79+
# - name: Regenerate FFI bindings
80+
# run: |
81+
# ./.github/scripts/ci-ffi.sh
82+
# - run: |
83+
# MMTK_JULIA_DIR=$(realpath $(dirname "$0"))/../mmtk-julia/mmtk-julia
84+
# if ! git diff --exit-code $MMTK_JULIA_DIR/mmtk/src/julia_types.rs; then
85+
# echo "Rust FFI bindings in \`julia_types.rs\` are outdated. Run \`make regen-bindgen-ffi\` from the mmtk-julia directory and make sure to include the updated file in the pull request."
86+
# exit 1
87+
# fi
8788

8889
build-test-other:
8990
runs-on: ubuntu-22.04
@@ -109,7 +110,7 @@ jobs:
109110
$JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()"
110111
- name: Test Julia
111112
run: |
112-
./.github/scripts/ci-test-other.sh ${{ inputs.gc_plan }}
113+
./.github/scripts/ci-test-other.sh ${{ inputs.gc_plan }} ${{ inputs.moving }}
113114
114115
build-test-stdlib:
115116
runs-on: ubuntu-22.04
@@ -135,7 +136,7 @@ jobs:
135136
$JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()"
136137
- name: Test Julia
137138
run: |
138-
./.github/scripts/ci-test-stdlib.sh ${{ inputs.gc_plan }}
139+
./.github/scripts/ci-test-stdlib.sh ${{ inputs.gc_plan }} ${{ inputs.moving }}
139140
140141
build-test-LinearAlgebra:
141142
runs-on: ubuntu-22.04
@@ -158,4 +159,4 @@ jobs:
158159
$JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()"
159160
- name: Test Julia
160161
run: |
161-
./.github/scripts/ci-test-LinearAlgebra.sh ${{ inputs.gc_plan }}
162+
./.github/scripts/ci-test-LinearAlgebra.sh ${{ inputs.gc_plan }} ${{ inputs.moving }}

‎.github/workflows/ci.yml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ on:
77
- v1.8.2\+RAI
88
- v1.9.2\+RAI
99
- dev
10+
- mmtk-support-moving-upstream
1011

1112
concurrency:
1213
# Cancels pending runs when a PR gets updated.
@@ -17,8 +18,8 @@ jobs:
1718
julia-version:
1819
runs-on: ubuntu-latest
1920
env:
20-
JULIA_REPO_DEFAULT: JuliaLang/julia
21-
JULIA_REF_DEFAULT: master
21+
JULIA_REPO_DEFAULT: mmtk/julia
22+
JULIA_REF_DEFAULT: mmtk-support-moving-upstream
2223
outputs:
2324
julia_repo: ${{ steps.print.outputs.julia_repo }}
2425
julia_ref: ${{ steps.print.outputs.julia_ref }}
@@ -40,7 +41,7 @@ jobs:
4041
fail-fast: false
4142
matrix:
4243
gc_plan: [Immix]
43-
moving: [Non_Moving, Moving]
44+
moving: [Non_Moving, Max_Moving]
4445
uses: ./.github/workflows/binding-tests.yml
4546
with:
4647
gc_plan: ${{ matrix.gc_plan }}

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ release:
6161
@cd $(MMTK_JULIA_DIR)mmtk && $(PROJECT_DIRS) cargo build --features $(CARGO_FEATURES) --release
6262

6363
debug:
64-
@echo "Building the Rust project in $(MMTK_JULIA_DIR) using a debug build";
64+
@echo "Building the Rust project in $(MMTK_JULIA_DIR)mmtk using a debug build with MMTK_VARS: $(MMTK_VARS)";
6565
@cd $(MMTK_JULIA_DIR)mmtk && $(PROJECT_DIRS) cargo build --features $(CARGO_FEATURES)
6666

6767
# Build the Julia project (which will build the binding as part of their deps build)

‎mmtk/src/conservative.rs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ lazy_static! {
99
pub static ref CONSERVATIVE_ROOTS: Mutex<HashSet<ObjectReference>> = Mutex::new(HashSet::new());
1010
}
1111
pub fn pin_conservative_roots() {
12-
crate::early_return_for_non_moving_build!(());
12+
crate::early_return_for_non_moving_build!();
1313
crate::early_return_for_current_gc!();
1414

1515
let mut roots = CONSERVATIVE_ROOTS.lock().unwrap();
@@ -25,7 +25,7 @@ pub fn pin_conservative_roots() {
2525
log::debug!("Conservative roots: {}, pinned: {}", n_roots, n_pinned);
2626
}
2727
pub fn unpin_conservative_roots() {
28-
crate::early_return_for_non_moving_build!(());
28+
crate::early_return_for_non_moving_build!();
2929
crate::early_return_for_current_gc!();
3030

3131
let mut roots = CONSERVATIVE_ROOTS.lock().unwrap();
@@ -43,8 +43,8 @@ pub fn unpin_conservative_roots() {
4343
n_live
4444
);
4545
}
46-
pub fn mmtk_conservative_scan_task_stack(ta: *const jl_task_t) {
47-
crate::early_return_for_non_moving_build!(());
46+
pub unsafe fn mmtk_conservative_scan_task_stack(ta: *const jl_task_t) {
47+
crate::early_return_for_non_moving_build!();
4848
crate::early_return_for_current_gc!();
4949

5050
log::debug!("mmtk_conservative_scan_native_stack begin ta = {:?}", ta);
@@ -83,18 +83,18 @@ pub fn mmtk_conservative_scan_task_stack(ta: *const jl_task_t) {
8383
log::warn!("Skip stack for {:?}", ta);
8484
}
8585
}
86-
pub fn mmtk_conservative_scan_task_registers(ta: *const jl_task_t) {
87-
crate::early_return_for_non_moving_build!(());
86+
pub unsafe fn mmtk_conservative_scan_task_registers(ta: *const jl_task_t) {
87+
crate::early_return_for_non_moving_build!();
8888
crate::early_return_for_current_gc!();
8989

9090
let (lo, hi) = get_range(&unsafe { &*ta }.ctx);
9191
conservative_scan_range(lo, hi);
9292
}
9393
pub fn mmtk_conservative_scan_ptls_registers(ptls: &mut _jl_tls_states_t) {
94-
crate::early_return_for_non_moving_build!(());
94+
crate::early_return_for_non_moving_build!();
9595
crate::early_return_for_current_gc!();
9696

97-
let (lo, hi) = get_range(&((*ptls).gc_tls.ctx_at_the_time_gc_started));
97+
let (lo, hi) = get_range(&(ptls).gc_tls.ctx_at_the_time_gc_started);
9898
conservative_scan_range(lo, hi);
9999
}
100100
// TODO: This scans the entire context type, which is slower.

0 commit comments

Comments
 (0)