[jvm-packages] Update helper script for publishing Maven Central (dmlc#11393)

* Update helper script for publishing Maven Central

* Show progress for GPU variant

* Only upload xgboost4j-spark-gpu when gpu variant is selected

* Insert required metadata for xgboost4j-spark-gpu

* Restore xgboost4j-spark-gpu/pom.xml for every invocation

* Insert metadata first

* Work around issue with gtest from dmlc-core

* Put metadata in xgboost4j-spark-gpu/pom.xml

* Revert "Work around issue with gtest from dmlc-core"

This reverts commit bdf0a06.

* Support CMake 4.0

Author: hcho3

dev/prepare_jvm_release.py

@@ -1,3 +1,33 @@
+"""
+Helper script to prepare for releasing XGBoost JVM packages to
+Maven Central.
+
+## Prerequisite
+
+1. You must have the right to upload artifacts to the Maven Central repo.
+   If you do not, contact Hyunsu Cho ([email protected]) so that
+   he can contact Sonatype on your behalf in order to add you as a
+   "producer" user for the ml.dmlc namespace. See
+   https://central.sonatype.org/pages/support/#status to learn about
+   the process for adding or removing users who can publish to the project.
+
+2. Follow instructions in
+   https://central.sonatype.org/publish/publish-portal-maven/#credentials
+   to set up the authentication token in your machine.
+
+3. Set up GPG for signing artifacts:
+   https://central.sonatype.org/publish/requirements/gpg/
+
+## Making the release
+Run this script 4 times:
+
+python3 dev/prepare_jvm_release.py --scala-version 2.12 --variant cpu
+python3 dev/prepare_jvm_release.py --scala-version 2.12 --variant gpu
+python3 dev/prepare_jvm_release.py --scala-version 2.13 --variant cpu
+python3 dev/prepare_jvm_release.py --scala-version 2.13 --variant gpu
+
+"""
+
 import argparse
 import errno
 import glob
@@ -52,7 +82,7 @@ def cd(path):
 
 def run(command, **kwargs):
     print(command)
-    subprocess.check_call(command, shell=True, **kwargs)
+    subprocess.run(command, shell=True, check=True, **kwargs)
 
 
 def get_current_commit_hash():
@@ -74,141 +104,135 @@ def retrieve(url, filename=None):
 
 
 def main():
-    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawTextHelpFormatter
+    )
     parser.add_argument(
         "--release-version",
         type=str,
         required=True,
         help="Version of the release being prepared",
     )
+    parser.add_argument(
+        "--scala-version",
+        type=str,
+        required=True,
+        help="Version of Scala to use in the JVM packages",
+        choices=["2.12", "2.13"],
+    )
+    parser.add_argument(
+        "--variant",
+        type=str,
+        required=True,
+        choices=["cpu", "gpu"],
+        help="JVM package variant to package and publish",
+    )
+
     args = parser.parse_args()
     version = args.release_version
+    scala_version = args.scala_version
+    use_cuda = args.variant == "gpu"
 
     commit_hash = get_current_commit_hash()
     git_branch = get_current_git_branch()
-    print(
-        f"Using commit {commit_hash} of branch {git_branch}"
+    print(f"Using commit {commit_hash} of branch {git_branch}")
+    print(f"====Update pom.xml to use Scala {scala_version}====")
+    run(
+        f"{sys.executable} ops/script/change_scala_version.py "
+        f"--scala-version {scala_version} --purge-artifacts"
     )
 
     with cd("jvm-packages/"):
-        print("====copying pure-Python tracker====")
-        for use_cuda in [True, False]:
-            xgboost4j = "xgboost4j-gpu" if use_cuda else "xgboost4j"
-            cp(
-                "../python-package/xgboost/tracker.py",
-                f"{xgboost4j}/src/main/resources",
-            )
-
-        print("====copying resources for testing====")
+        print("====Copying resources for testing====")
         with cd("../demo/CLI/regression"):
             run(f"{sys.executable} mapfeat.py")
             run(f"{sys.executable} mknfold.py machine.txt 1")
-        for use_cuda in [True, False]:
-            xgboost4j = "xgboost4j-gpu" if use_cuda else "xgboost4j"
-            xgboost4j_spark = "xgboost4j-spark-gpu" if use_cuda else "xgboost4j-spark"
-            maybe_makedirs(f"{xgboost4j}/src/test/resources")
-            maybe_makedirs(f"{xgboost4j_spark}/src/test/resources")
-            for file in glob.glob("../demo/data/agaricus.*"):
-                cp(file, f"{xgboost4j}/src/test/resources")
-                cp(file, f"{xgboost4j_spark}/src/test/resources")
-            for file in glob.glob("../demo/CLI/regression/machine.txt.t*"):
-                cp(file, f"{xgboost4j_spark}/src/test/resources")
+        xgboost4j_spark = "xgboost4j-spark-gpu" if use_cuda else "xgboost4j-spark"
+        maybe_makedirs(f"xgboost4j/src/test/resources")
+        maybe_makedirs(f"{xgboost4j_spark}/src/test/resources")
+        for file in glob.glob("../demo/data/agaricus.*"):
+            cp(file, f"xgboost4j/src/test/resources")
+            cp(file, f"{xgboost4j_spark}/src/test/resources")
+        for file in glob.glob("../demo/CLI/regression/machine.txt.t*"):
+            cp(file, f"{xgboost4j_spark}/src/test/resources")
 
         print("====Creating directories to hold native binaries====")
-        for os_ident, arch in [
-            ("linux", "x86_64"),
-            ("linux", "aarch64"),
-            ("windows", "x86_64"),
-            ("macos", "x86_64"),
-            ("macos", "aarch64"),
-        ]:
+        if use_cuda:
+            # TODO(hcho3): Add GPU build for linux aarch64
+            matrix = [("linux", "x86_64")]
+        else:
+            matrix = [
+                ("linux", "x86_64"),
+                ("linux", "aarch64"),
+                ("windows", "x86_64"),
+                ("macos", "x86_64"),
+                ("macos", "aarch64"),
+            ]
+        for os_ident, arch in matrix:
             output_dir = f"xgboost4j/src/main/resources/lib/{os_ident}/{arch}"
             maybe_makedirs(output_dir)
-        for os_ident, arch in [("linux", "x86_64")]:
-            output_dir = f"xgboost4j-gpu/src/main/resources/lib/{os_ident}/{arch}"
-            maybe_makedirs(output_dir)
 
         print("====Downloading native binaries from CI====")
-        nightly_bucket_prefix = (
-            "https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds"
-        )
-        maven_repo_prefix = (
-            "https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/release/ml/dmlc"
-        )
-
-        retrieve(
-            url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/xgboost4j_{commit_hash}.dll",
-            filename="xgboost4j/src/main/resources/lib/windows/x86_64/xgboost4j.dll",
-        )
-        retrieve(
-            url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/libxgboost4j_linux_x86_64_{commit_hash}.so",
-            filename="xgboost4j/src/main/resources/lib/linux/x86_64/libxgboost4j.so",
-        )
-        retrieve(
-            url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/libxgboost4j_linux_arm64_{commit_hash}.so",
-            filename="xgboost4j/src/main/resources/lib/linux/aarch64/libxgboost4j.so",
-        )
-        retrieve(
-            url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/libxgboost4j_{commit_hash}.dylib",
-            filename="xgboost4j/src/main/resources/lib/macos/x86_64/libxgboost4j.dylib",
-        )
-        retrieve(
-            url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/libxgboost4j_m1_{commit_hash}.dylib",
-            filename="xgboost4j/src/main/resources/lib/macos/aarch64/libxgboost4j.dylib",
-        )
-
-        with tempfile.TemporaryDirectory() as tempdir:
-            # libxgboost4j.so for Linux x86_64, GPU support
-            zip_path = os.path.join(tempdir, "xgboost4j-gpu_2.12.jar")
-            extract_dir = os.path.join(tempdir, "xgboost4j-gpu")
-            retrieve(
-                url=f"{maven_repo_prefix}/xgboost4j-gpu_2.12/{version}/"
-                f"xgboost4j-gpu_2.12-{version}.jar",
-                filename=zip_path,
+        if use_cuda:
+            url_prefix = (
+                "https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/release/ml/dmlc"
+            )
+            with tempfile.TemporaryDirectory() as tempdir:
+                # libxgboost4j.so for Linux x86_64, GPU support
+                zip_path = os.path.join(tempdir, "xgboost4j-spark-gpu_2.12.jar")
+                extract_dir = os.path.join(tempdir, "xgboost4j-spark-gpu")
+                retrieve(
+                    url=f"{url_prefix}/xgboost4j-spark-gpu_2.12/{version}/"
+                    f"xgboost4j-spark-gpu_2.12-{version}.jar",
+                    filename=zip_path,
+                )
+                os.mkdir(extract_dir)
+                with zipfile.ZipFile(zip_path, "r") as t:
+                    t.extractall(extract_dir)
+                cp(
+                    os.path.join(
+                        extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"
+                    ),
+                    "xgboost4j/src/main/resources/lib/linux/x86_64/libxgboost4j.so",
+                )
+            run(
+                "mvn --no-transfer-progress install -Pgpu "
+                "-DskipTests -Dmaven.test.skip=true -Dskip.native.build=true"
             )
-            os.mkdir(extract_dir)
-            with zipfile.ZipFile(zip_path, "r") as t:
-                t.extractall(extract_dir)
-            cp(
-                os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"),
-                "xgboost4j-gpu/src/main/resources/lib/linux/x86_64/libxgboost4j.so",
+            run(
+                "mvn deploy -Pgpu,release -pl xgboost4j-spark-gpu "
+                "-DskipTests -Dmaven.test.skip=true -Dskip.native.build=true"
+            )
+        else:
+            url_prefix = "https://s3-us-west-2.amazonaws.com/xgboost-nightly-builds"
+            for os_ident, arch, src_libname, dest_libname in [
+                ("linux", "x86_64", "libxgboost4j_linux_x86_64.so", "libxgboost4j.so"),
+                (
+                    "linux",
+                    "aarch64",
+                    "libxgboost4j_linux_aarch64.so",
+                    "libxgboost4j.so",
+                ),
+                ("windows", "x86_64", "xgboost4j.dll", "xgboost4j.dll"),
+                ("macos", "x86_64", "libxgboost4j_intel.dylib", "libxgboost4j.dylib"),
+                ("macos", "aarch64", "libxgboost4j_m1.dylib", "libxgboost4j.dylib"),
+            ]:
+                retrieve(
+                    url=f"{url_prefix}/{git_branch}/{commit_hash}/{src_libname}",
+                    filename=(
+                        "xgboost4j/src/main/resources/lib/"
+                        f"{os_ident}/{arch}/{dest_libname}"
+                    ),
+                )
+            run(
+                "mvn --no-transfer-progress deploy -Pdefault,release "
+                "-DskipTests -Dmaven.test.skip=true -Dskip.native.build=true"
             )
 
     print("====Next Steps====")
-    print("1. Gain upload right to Maven Central repo.")
-    print("1-1. Sign up for a JIRA account at Sonatype: ")
-    print(
-        "1-2. File a JIRA ticket: "
-        "https://issues.sonatype.org/secure/CreateIssue.jspa?issuetype=21&pid=10134. Example: "
-        "https://issues.sonatype.org/browse/OSSRH-67724"
-    )
-    print(
-        "2. Store the Sonatype credentials in .m2/settings.xml. See insturctions in "
-        "https://central.sonatype.org/publish/publish-maven/"
-    )
-    print(
-        "3. Now on a Linux machine, run the following to build Scala 2.12 artifacts. "
-        "Make sure to use an Internet connection with fast upload speed:"
-    )
-    print(
-        "   # Skip native build, since we have all needed native binaries from CI\n"
-        "   GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests -Dskip.native.build=true"
-    )
-    print(
-        "4. Log into https://oss.sonatype.org/. On the left menu panel, click Staging "
-        "Repositories. Visit the URL https://oss.sonatype.org/content/repositories/mldmlc-xxxx "
-        "to inspect the staged JAR files. Finally, press Release button to publish the "
-        "artifacts to the Maven Central repository. The top-level metapackage should be "
-        "named xgboost-jvm_2.12."
-    )
-    print(
-        "5. Remove the Scala 2.12 artifacts and build Scala 2.13 artifacts:\n"
-        "   python ops/script/change_scala_version.py --scala-version 2.13 --purge-artifacts\n"
-        "   GPG_TTY=$(tty) mvn deploy -Prelease -DskipTests -Dskip.native.build=true"
-    )
     print(
-        "6. Go to https://oss.sonatype.org/ to release the Scala 2.13 artifacts. "
-        "The top-level metapackage should be named xgboost-jvm_2.13."
+        "Visit https://central.sonatype.com/publishing/deployments to verify the deployment. "
+        "You can either drop the deployment or publish it. Note: publishing is final."
     )
 
 

dmlc-core

@@ -197,14 +197,12 @@
                         </executions>
                     </plugin>
                     <plugin>
-                        <groupId>org.sonatype.plugins</groupId>
-                        <artifactId>nexus-staging-maven-plugin</artifactId>
-                        <version>1.7.0</version>
+                        <groupId>org.sonatype.central</groupId>
+                        <artifactId>central-publishing-maven-plugin</artifactId>
+                        <version>0.7.0</version>
                         <extensions>true</extensions>
                         <configuration>
-                            <serverId>ossrh</serverId>
-                            <nexusUrl>https://oss.sonatype.org/</nexusUrl>
-                            <autoReleaseAfterClose>false</autoReleaseAfterClose>
+                            <publishingServerId>central</publishingServerId>
                         </configuration>
                     </plugin>
                     <plugin>

jvm-packages/pom.xml

@@ -10,6 +10,37 @@
     </parent>
     <name>xgboost4j-spark-gpu</name>
     <artifactId>xgboost4j-spark-gpu_2.12</artifactId>
+    <description>JVM Package for XGBoost</description>
+    <url>https://github.com/dmlc/xgboost/tree/master/jvm-packages</url>
+    <licenses>
+        <license>
+            <name>The Apache License, Version 2.0</name>
+            <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+        </license>
+    </licenses>
+    <developers>
+        <developer>
+            <name>Bobby Wang</name>
+            <email>[email protected]</email>
+        </developer>
+        <developer>
+            <name>Jiaming Yuan</name>
+            <email>[email protected]</email>
+        </developer>
+        <developer>
+            <name>Hyunsu Cho</name>
+            <email>[email protected]</email>
+        </developer>
+        <developer>
+            <name>CodingCat</name>
+            <email>[email protected]</email>
+        </developer>
+    </developers>
+    <scm>
+        <connection>scm:git:git:/github.com/dmlc/xgboost.git</connection>
+        <developerConnection>scm:git:ssh://github.com/dmlc/xgboost.git</developerConnection>
+        <url>https://github.com/dmlc/xgboost</url>
+    </scm>
     <build>
         <plugins>
             <plugin>

jvm-packages/xgboost4j-spark-gpu/pom.xml

@@ -20,9 +20,10 @@ def main(args: argparse.Namespace) -> None:
             if target.is_dir():
                 print(f"Removing {target}...")
                 shutil.rmtree(target)
-        for target in pathlib.Path("jvm-packages/").glob("**/*.so"):
-            print(f"Removing {target}...")
-            target.unlink()
+        for ext in ["so", "dll", "dylib"]:
+            for target in pathlib.Path("jvm-packages/").glob(f"**/*.{ext}"):
+                print(f"Removing {target}...")
+                target.unlink()
 
     # Update pom.xml
     for pom in pathlib.Path("jvm-packages/").glob("**/pom.xml"):