Skip to content

Commit 982d31f

Browse files
koukou
authored
GH-47505: [CI][C#][Integration] Use apache/arrow-dotnet (#47508)
### Rationale for this change We're moving the C# implementation to apache/arrow-dotnet. ### What changes are included in this PR? Use apache/arrow-dotnet instead of `csharp/` in apache/arrow. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * GitHub Issue: #47505 Authored-by: Sutou Kouhei <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
1 parent b64a6f9 commit 982d31f

File tree

7 files changed

+50
-46
lines changed

7 files changed

+50
-46
lines changed

‎.github/workflows/integration.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,11 @@ jobs:
9595
with:
9696
repository: apache/arrow-js
9797
path: js
98+
- name: Checkout Arrow .NET
99+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
100+
with:
101+
repository: apache/arrow-dotnet
102+
path: dotnet
98103
- name: Free up disk space
99104
run: |
100105
ci/scripts/util_free_space.sh
@@ -118,6 +123,7 @@ jobs:
118123
source ci/scripts/util_enable_core_dumps.sh
119124
archery docker run \
120125
-e ARCHERY_DEFAULT_BRANCH=${{ github.event.repository.default_branch }} \
126+
-e ARCHERY_INTEGRATION_WITH_DOTNET=1 \
121127
-e ARCHERY_INTEGRATION_WITH_GO=1 \
122128
-e ARCHERY_INTEGRATION_WITH_JAVA=1 \
123129
-e ARCHERY_INTEGRATION_WITH_JS=1 \

‎ci/scripts/integration_arrow.sh‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,8 @@ build_dir=${2}
2525
gold_dir=$arrow_dir/testing/data/arrow-ipc-stream/integration
2626

2727
: "${ARROW_INTEGRATION_CPP:=ON}"
28-
: "${ARROW_INTEGRATION_CSHARP:=ON}"
2928

30-
: "${ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS:=cpp,csharp}"
29+
: "${ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS:=cpp}"
3130
export ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS
3231

3332
. "${arrow_dir}/ci/scripts/util_log.sh"
@@ -38,7 +37,7 @@ github_actions_group_end
3837

3938
github_actions_group_begin "Integration: Prepare: Dependencies"
4039
# For C Data Interface testing
41-
if [ "${ARROW_INTEGRATION_CSHARP}" == "ON" ]; then
40+
if [ "${ARCHERY_INTEGRATION_WITH_DOTNET}" -gt "0" ]; then
4241
pip install pythonnet
4342
fi
4443
if [ "${ARCHERY_INTEGRATION_WITH_JAVA}" -gt "0" ]; then
@@ -59,15 +58,13 @@ export GOMEMLIMIT=200MiB
5958
export GODEBUG=gctrace=1,clobberfree=1
6059

6160
ARCHERY_WITH_CPP=$([ "$ARROW_INTEGRATION_CPP" == "ON" ] && echo "1" || echo "0")
62-
ARCHERY_WITH_CSHARP=$([ "$ARROW_INTEGRATION_CSHARP" == "ON" ] && echo "1" || echo "0")
6361

6462
# Rust can be enabled by exporting ARCHERY_INTEGRATION_WITH_RUST=1
6563
time archery integration \
6664
--run-c-data \
6765
--run-ipc \
6866
--run-flight \
6967
--with-cpp="${ARCHERY_WITH_CPP}" \
70-
--with-csharp="${ARCHERY_WITH_CSHARP}"\
7168
--gold-dirs="$gold_dir/0.14.1" \
7269
--gold-dirs="$gold_dir/0.17.1" \
7370
--gold-dirs="$gold_dir/1.0.0-bigendian" \

‎ci/scripts/integration_arrow_build.sh‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ arrow_dir=${1}
2323
build_dir=${2}
2424

2525
: "${ARROW_INTEGRATION_CPP:=ON}"
26-
: "${ARROW_INTEGRATION_CSHARP:=ON}"
2726

2827
. "${arrow_dir}/ci/scripts/util_log.sh"
2928

@@ -47,9 +46,10 @@ if [ "${ARROW_INTEGRATION_CPP}" == "ON" ]; then
4746
fi
4847
github_actions_group_end
4948

50-
github_actions_group_begin "Integration: Build: C#"
51-
if [ "${ARROW_INTEGRATION_CSHARP}" == "ON" ]; then
52-
"${arrow_dir}/ci/scripts/csharp_build.sh" "${arrow_dir}" "${build_dir}"
49+
github_actions_group_begin "Integration: Build: .NET"
50+
if [ "${ARCHERY_INTEGRATION_WITH_DOTNET}" -gt "0" ]; then
51+
"${arrow_dir}/dotnet/ci/scripts/build.sh" "${arrow_dir}/dotnet"
52+
cp -a "${arrow_dir}/dotnet" "${build_dir}/dotnet"
5353
fi
5454
github_actions_group_end
5555

‎dev/archery/archery/cli.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -670,8 +670,9 @@ def _set_default(opt, default):
670670
help="Seed for PRNG when generating test data")
671671
@click.option('--with-cpp', type=bool, default=False,
672672
help='Include C++ in integration tests')
673-
@click.option('--with-csharp', type=bool, default=False,
674-
help='Include C# in integration tests')
673+
@click.option('--with-dotnet', type=bool, default=False,
674+
help='Include .NET in integration tests',
675+
envvar="ARCHERY_INTEGRATION_WITH_DOTNET")
675676
@click.option('--with-java', type=bool, default=False,
676677
help='Include Java in integration tests',
677678
envvar="ARCHERY_INTEGRATION_WITH_JAVA")
@@ -783,7 +784,7 @@ def integration(with_all=False, random_seed=12345, **args):
783784

784785
gen_path = args['write_generated_json']
785786

786-
implementations = ['cpp', 'csharp', 'java', 'js', 'go', 'nanoarrow', 'rust']
787+
implementations = ['cpp', 'dotnet', 'java', 'js', 'go', 'nanoarrow', 'rust']
787788
formats = ['ipc', 'flight', 'c_data']
788789

789790
enabled_implementations = 0

‎dev/archery/archery/integration/datagen.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1429,7 +1429,7 @@ def write(self, path):
14291429
self.path = path
14301430

14311431
def skip_tester(self, tester):
1432-
"""Skip this test for the given tester (such as 'C#').
1432+
"""Skip this test for the given tester (such as '.NET').
14331433
"""
14341434
self.skipped_testers.add(tester)
14351435
return self
@@ -1951,23 +1951,23 @@ def _temp_path():
19511951
# TODO(https://github.com/apache/arrow-nanoarrow/issues/622)
19521952
.skip_tester('nanoarrow')
19531953
# TODO(https://github.com/apache/arrow/issues/38045)
1954-
.skip_format(SKIP_FLIGHT, 'C#'),
1954+
.skip_format(SKIP_FLIGHT, '.NET'),
19551955

19561956
generate_dictionary_unsigned_case()
19571957
.skip_tester('nanoarrow')
19581958
.skip_tester('Java') # TODO(ARROW-9377)
19591959
# TODO(https://github.com/apache/arrow/issues/38045)
1960-
.skip_format(SKIP_FLIGHT, 'C#'),
1960+
.skip_format(SKIP_FLIGHT, '.NET'),
19611961

19621962
generate_nested_dictionary_case()
19631963
# TODO(https://github.com/apache/arrow-nanoarrow/issues/622)
19641964
.skip_tester('nanoarrow')
19651965
.skip_tester('Java') # TODO(ARROW-7779)
19661966
# TODO(https://github.com/apache/arrow/issues/38045)
1967-
.skip_format(SKIP_FLIGHT, 'C#'),
1967+
.skip_format(SKIP_FLIGHT, '.NET'),
19681968

19691969
generate_run_end_encoded_case()
1970-
.skip_tester('C#')
1970+
.skip_tester('.NET')
19711971
.skip_tester('JS')
19721972
# TODO(https://github.com/apache/arrow-nanoarrow/issues/618)
19731973
.skip_tester('nanoarrow')
@@ -1980,7 +1980,7 @@ def _temp_path():
19801980
.skip_tester('Rust'),
19811981

19821982
generate_list_view_case()
1983-
.skip_tester('C#') # Doesn't support large list views
1983+
.skip_tester('.NET') # Doesn't support large list views
19841984
.skip_tester('JS')
19851985
# TODO(https://github.com/apache/arrow-nanoarrow/issues/618)
19861986
.skip_tester('nanoarrow')
@@ -1992,7 +1992,7 @@ def _temp_path():
19921992
.skip_format(SKIP_C_SCHEMA, 'C++')
19931993
.skip_format(SKIP_C_ARRAY, 'C++')
19941994
# TODO(https://github.com/apache/arrow/issues/38045)
1995-
.skip_format(SKIP_FLIGHT, 'C#'),
1995+
.skip_format(SKIP_FLIGHT, '.NET'),
19961996
]
19971997

19981998
generated_paths = []

‎dev/archery/archery/integration/runner.py‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ def _gold_tests(self, gold_dir):
193193
skip_testers.add("Java")
194194
skip_testers.add("JS")
195195
if prefix == '1.0.0-bigendian' or prefix == '1.0.0-littleendian':
196-
skip_testers.add("C#")
196+
skip_testers.add(".NET")
197197
skip_testers.add("Java")
198198
skip_testers.add("JS")
199199
skip_testers.add("Rust")
@@ -207,7 +207,7 @@ def _gold_tests(self, gold_dir):
207207
# disable specific compression type tests.
208208

209209
if prefix == '4.0.0-shareddict':
210-
skip_testers.add("C#")
210+
skip_testers.add(".NET")
211211
# https://github.com/apache/arrow-nanoarrow/issues/622
212212
skip_testers.add("nanoarrow")
213213

@@ -590,7 +590,7 @@ def get_static_json_files():
590590

591591

592592
def run_all_tests(with_cpp=True, with_java=True, with_js=True,
593-
with_csharp=True, with_go=True, with_rust=False,
593+
with_dotnet=True, with_go=True, with_rust=False,
594594
with_nanoarrow=False, run_ipc=False, run_flight=False,
595595
run_c_data=False, tempdir=None, target_implementations="",
596596
**kwargs):
@@ -619,9 +619,9 @@ def append_tester(implementation, tester):
619619
from .tester_js import JSTester
620620
append_tester("js", JSTester(**kwargs))
621621

622-
if with_csharp:
623-
from .tester_csharp import CSharpTester
624-
append_tester("csharp", CSharpTester(**kwargs))
622+
if with_dotnet:
623+
from .tester_dotnet import DotNetTester
624+
append_tester("dotnet", DotNetTester(**kwargs))
625625

626626
if with_go:
627627
from .tester_go import GoTester
@@ -644,42 +644,42 @@ def append_tester(implementation, tester):
644644
Scenario(
645645
"auth:basic_proto",
646646
description="Authenticate using the BasicAuth protobuf.",
647-
skip_testers={"C#"},
647+
skip_testers={".NET"},
648648
),
649649
Scenario(
650650
"middleware",
651651
description="Ensure headers are propagated via middleware.",
652-
skip_testers={"C#"},
652+
skip_testers={".NET"},
653653
),
654654
Scenario(
655655
"ordered",
656656
description="Ensure FlightInfo.ordered is supported.",
657-
skip_testers={"JS", "C#", "Rust"},
657+
skip_testers={"JS", ".NET", "Rust"},
658658
),
659659
Scenario(
660660
"expiration_time:do_get",
661661
description=("Ensure FlightEndpoint.expiration_time with "
662662
"DoGet is working as expected."),
663-
skip_testers={"JS", "C#", "Rust"},
663+
skip_testers={"JS", ".NET", "Rust"},
664664
),
665665
Scenario(
666666
"expiration_time:list_actions",
667667
description=("Ensure FlightEndpoint.expiration_time related "
668668
"pre-defined actions is working with ListActions "
669669
"as expected."),
670-
skip_testers={"JS", "C#", "Rust"},
670+
skip_testers={"JS", ".NET", "Rust"},
671671
),
672672
Scenario(
673673
"expiration_time:cancel_flight_info",
674674
description=("Ensure FlightEndpoint.expiration_time and "
675675
"CancelFlightInfo are working as expected."),
676-
skip_testers={"JS", "C#", "Rust"},
676+
skip_testers={"JS", ".NET", "Rust"},
677677
),
678678
Scenario(
679679
"expiration_time:renew_flight_endpoint",
680680
description=("Ensure FlightEndpoint.expiration_time and "
681681
"RenewFlightEndpoint are working as expected."),
682-
skip_testers={"JS", "C#", "Rust"},
682+
skip_testers={"JS", ".NET", "Rust"},
683683
),
684684
Scenario(
685685
"do_exchange:echo",
@@ -690,37 +690,37 @@ def append_tester(implementation, tester):
690690
Scenario(
691691
"location:reuse_connection",
692692
description="Ensure arrow-flight-reuse-connection is accepted.",
693-
skip_testers={"JS", "C#", "Rust"},
693+
skip_testers={"JS", ".NET", "Rust"},
694694
),
695695
Scenario(
696696
"session_options",
697697
description="Ensure Flight SQL Sessions work as expected.",
698-
skip_testers={"JS", "C#", "Rust"}
698+
skip_testers={"JS", ".NET", "Rust"}
699699
),
700700
Scenario(
701701
"poll_flight_info",
702702
description="Ensure PollFlightInfo is supported.",
703-
skip_testers={"JS", "C#", "Rust"}
703+
skip_testers={"JS", ".NET", "Rust"}
704704
),
705705
Scenario(
706706
"app_metadata_flight_info_endpoint",
707707
description="Ensure support FlightInfo and Endpoint app_metadata",
708-
skip_testers={"JS", "C#", "Rust"}
708+
skip_testers={"JS", ".NET", "Rust"}
709709
),
710710
Scenario(
711711
"flight_sql",
712712
description="Ensure Flight SQL protocol is working as expected.",
713-
skip_testers={"Rust", "C#"}
713+
skip_testers={"Rust", ".NET"}
714714
),
715715
Scenario(
716716
"flight_sql:extension",
717717
description="Ensure Flight SQL extensions work as expected.",
718-
skip_testers={"Rust", "C#"}
718+
skip_testers={"Rust", ".NET"}
719719
),
720720
Scenario(
721721
"flight_sql:ingestion",
722722
description="Ensure Flight SQL ingestion works as expected.",
723-
skip_testers={"JS", "C#", "Rust"}
723+
skip_testers={"JS", ".NET", "Rust"}
724724
),
725725
]
726726

dev/archery/archery/integration/tester_csharp.py renamed to dev/archery/archery/integration/tester_dotnet.py

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
from ..utils.source import ARROW_ROOT_DEFAULT
2626

2727

28-
_ARTIFACTS_PATH = os.path.join(ARROW_ROOT_DEFAULT, "csharp/artifacts")
28+
_ARTIFACTS_PATH = os.path.join(ARROW_ROOT_DEFAULT, "dotnet/artifacts")
2929
_BUILD_SUBDIR = "Debug/net8.0"
3030

3131
_EXE_PATH = os.path.join(_ARTIFACTS_PATH,
@@ -94,7 +94,7 @@ def _run_gc(self):
9494
CDataInterface.RunGC()
9595

9696

97-
class CSharpCDataExporter(CDataExporter, _CDataBase):
97+
class DotNetCDataExporter(CDataExporter, _CDataBase):
9898

9999
def export_schema_from_json(self, json_path, c_schema_ptr):
100100
from Apache.Arrow.IntegrationTest import CDataInterface
@@ -120,7 +120,7 @@ def run_gc(self):
120120
self._run_gc()
121121

122122

123-
class CSharpCDataImporter(CDataImporter, _CDataBase):
123+
class DotNetCDataImporter(CDataImporter, _CDataBase):
124124

125125
def import_schema_and_compare_to_json(self, json_path, c_schema_ptr):
126126
from Apache.Arrow.IntegrationTest import CDataInterface
@@ -152,7 +152,7 @@ def run_gc(self):
152152
self._run_gc()
153153

154154

155-
class CSharpTester(Tester):
155+
class DotNetTester(Tester):
156156
PRODUCER = True
157157
CONSUMER = True
158158
FLIGHT_SERVER = True
@@ -162,7 +162,7 @@ class CSharpTester(Tester):
162162
C_DATA_ARRAY_EXPORTER = True
163163
C_DATA_ARRAY_IMPORTER = True
164164

165-
name = 'C#'
165+
name = '.NET'
166166

167167
def _run(self, json_path=None, arrow_path=None, command='validate'):
168168
cmd = [_EXE_PATH]
@@ -199,10 +199,10 @@ def file_to_stream(self, file_path, stream_path):
199199
self.run_shell_command(cmd)
200200

201201
def make_c_data_exporter(self):
202-
return CSharpCDataExporter(self.debug, self.args)
202+
return DotNetCDataExporter(self.debug, self.args)
203203

204204
def make_c_data_importer(self):
205-
return CSharpCDataImporter(self.debug, self.args)
205+
return DotNetCDataImporter(self.debug, self.args)
206206

207207
def flight_request(self, port, json_path=None, scenario_name=None):
208208
cmd = [_FLIGHT_EXE_PATH, 'client', '--port', f'{port}']

0 commit comments

Comments
 (0)