GH-47650: [Archery][Integration] Add option to generate gold files (#47651)

### Rationale for this change

Generating "gold" IPC files for integration/regression testing between Arrow implementations is currently a tedious (and undocumented) manual process.

### What changes are included in this PR?

Add an option to `archery integration` to automate generation of gold files in the right format, using the Arrow implementation selected on the command line.

### Are these changes tested?

Only manually.

### Are there any user-facing changes?

No.

* GitHub Issue: #47650

Authored-by: Antoine Pitrou <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>

Author: pitrou

dev/archery/archery/cli.py

@@ -693,6 +693,8 @@ def _set_default(opt, default):
               envvar="ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS")
 @click.option('--write-generated-json', default="",
               help='Generate test JSON to indicated path')
+@click.option('--write-gold-files', default="",
+              help='Generate gold files to indicated path')
 @click.option('--run-ipc', is_flag=True, default=False,
               help='Run IPC integration tests')
 @click.option('--run-flight', is_flag=True, default=False,
@@ -715,7 +717,7 @@ def _set_default(opt, default):
               help=("Substring for test names to include in run, "
                     "e.g. -k primitive"))
 def integration(with_all=False, random_seed=12345, write_generated_json="",
-                **args):
+                write_gold_files="", **args):
     """If you don't specify the "--target-implementations" option nor
     the "ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS" environment
     variable, test patterns are product of all specified
@@ -774,8 +776,9 @@ def integration(with_all=False, random_seed=12345, write_generated_json="",
 
     """
 
-    from .integration.datagen import get_generated_json_files
-    from .integration.runner import run_all_tests
+    from .integration.datagen import (
+        get_generated_json_files, generate_gold_files)
+    from .integration.runner import run_all_tests, select_testers
     import numpy as np
 
     # FIXME(bkietz) Include help strings for individual testers.
@@ -784,33 +787,32 @@ def integration(with_all=False, random_seed=12345, write_generated_json="",
     # Make runs involving data generation deterministic
     np.random.seed(random_seed)
 
-    implementations = ['cpp', 'dotnet', 'java', 'js', 'go', 'nanoarrow', 'rust']
     formats = ['ipc', 'flight', 'c_data']
 
-    enabled_implementations = 0
-    for lang in implementations:
-        param = f'with_{lang}'
-        if with_all:
-            args[param] = with_all
-        enabled_implementations += args[param]
-
     enabled_formats = 0
     for fmt in formats:
         param = f'run_{fmt}'
         enabled_formats += args[param]
 
+    testers, other_testers = select_testers(**args)
+
     if write_generated_json:
         os.makedirs(write_generated_json, exist_ok=True)
         get_generated_json_files(tempdir=write_generated_json)
+    elif write_gold_files:
+        if len(testers) != 1 or len(other_testers) != 0:
+            raise click.UsageError(
+                "Need exactly one implementation to generate gold files; try --help")
+        generate_gold_files(testers[0], write_gold_files)
     else:
         if enabled_formats == 0:
             raise click.UsageError(
                 "Need to enable at least one format to test "
                 "(IPC, Flight, C Data Interface); try --help")
-        if enabled_implementations == 0:
+        if len(testers) == 0:
             raise click.UsageError(
                 "Need to enable at least one implementation to test; try --help")
-        run_all_tests(**args)
+        run_all_tests(testers, other_testers, **args)
 
 
 @archery.command()

dev/archery/archery/integration/datagen.py

@@ -17,10 +17,12 @@
 
 from collections import namedtuple, OrderedDict
 import binascii
+import gzip
 import json
 import os
 import random
 import tempfile
+import shutil
 
 import numpy as np
 
@@ -1885,9 +1887,6 @@ def generate_extension_case():
 def get_generated_json_files(tempdir=None):
     tempdir = tempdir or tempfile.mkdtemp(prefix='arrow-integration-')
 
-    def _temp_path():
-        return
-
     file_objs = [
         generate_primitive_case([], name='primitive_no_batches'),
         generate_primitive_case([17, 20], name='primitive'),
@@ -2003,3 +2002,25 @@ def _temp_path():
         generated_paths.append(file_obj)
 
     return generated_paths
+
+
+def generate_gold_files(tester, gold_dir):
+    os.makedirs(gold_dir, exist_ok=True)
+
+    # Generate JSON files
+    files = get_generated_json_files(gold_dir)
+    for f in files:
+        # For each JSON file, convert it to Arrow IPC file and stream
+        json_path = os.path.join(gold_dir, 'generated_' +
+                                 f.name + '.json')
+        arrow_file_path = os.path.join(gold_dir, 'generated_' +
+                                       f.name + '.arrow_file')
+        stream_path = os.path.join(gold_dir, 'generated_' +
+                                   f.name + '.stream')
+        tester.json_to_file(json_path, arrow_file_path)
+        tester.file_to_stream(arrow_file_path, stream_path)
+        # And GZip-compress the JSON file
+        with open(json_path, 'rb') as f_in:
+            with gzip.open(json_path + '.gz', 'wb') as f_out:
+                shutil.copyfileobj(f_in, f_out)
+        os.unlink(json_path)

dev/archery/archery/integration/runner.py

@@ -589,14 +589,12 @@ def get_static_json_files():
     ]
 
 
-def run_all_tests(with_cpp=True, with_java=True, with_js=True,
-                  with_dotnet=True, with_go=True, with_rust=False,
-                  with_nanoarrow=False, run_ipc=False, run_flight=False,
-                  run_c_data=False, tempdir=None, target_implementations="",
-                  **kwargs):
-    tempdir = tempdir or tempfile.mkdtemp(prefix='arrow-integration-')
-    target_implementations = \
-        target_implementations.split(",") if target_implementations else []
+def select_testers(with_cpp=True, with_java=True, with_js=True,
+                   with_dotnet=True, with_go=True, with_rust=False,
+                   with_nanoarrow=False, target_implementations="",
+                   **kwargs):
+    target_implementations = (target_implementations.split(",")
+                              if target_implementations else [])
 
     testers: List[Tester] = []
     other_testers: List[Tester] = []
@@ -635,6 +633,14 @@ def append_tester(implementation, tester):
         from .tester_rust import RustTester
         append_tester("rust", RustTester(**kwargs))
 
+    return testers, other_testers
+
+
+def run_all_tests(testers: List[Tester], other_testers: List[Tester],
+                  run_ipc=False, run_flight=False, run_c_data=False,
+                  tempdir=None, **kwargs):
+    tempdir = tempdir or tempfile.mkdtemp(prefix='arrow-integration-')
+
     static_json_files = get_static_json_files()
     generated_json_files = datagen.get_generated_json_files(tempdir=tempdir)
     json_files = static_json_files + generated_json_files

docs/source/format/Integration.rst

@@ -621,3 +621,20 @@ utility. Below are the test cases which are covered by them:
   - ZSTD
 
 * Batches with Shared Dictionaries
+
+Generating new Gold Files
+'''''''''''''''''''''''''
+
+From time to time, it is desirable to add new gold files, for example when the
+Columnar format or the IPC specification is update. Archery provides a dedicated
+option to do that.
+
+It is recommended to generate gold files using a well-known version of a Arrow
+implementation. For example, if a build of Arrow C++ exists in ``./build/release/``,
+one can generate new gold files in the ``/tmp/gold-files`` directory using the
+following command:
+
+.. code-block:: shell
+
+   export ARROW_CPP_EXE_PATH=./build/release/
+   archery integration --with-cpp 1 --write-gold-files=/tmp/gold-files