Rebase changes.

Author: Shanthoosh Pazhanjur Venkataraman

.github/workflows/java-ci.yml

@@ -1,4 +1,3 @@
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -21,19 +20,18 @@ name: "Java CI"
 on:
   push:
     branches: ['openhouse-1.2.0']
-    tags-ignore: [v*] # release tags are autogenerated after a successful CI, no need to run CI against them
+    tags-ignore: [v*]
   pull_request:
     branches: ['**']
     paths-ignore:
-    - '.github/ISSUE_TEMPLATE/iceberg_bug_report.yml'
     - '.github/workflows/python-ci.yml'
     - '.github/workflows/spark-ci.yml'
     - '.github/workflows/flink-ci.yml'
     - '.github/workflows/hive-ci.yml'
     - '.gitignore'
-    - '.asf.yml'
     - 'dev/**'
     - 'python/**'
+    - 'python_legacy/**'
     - 'docs/**'
     - 'open-api/**'
     - 'format/**'
@@ -48,36 +46,38 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
-  build:
+  build-checks:
     runs-on: ubuntu-latest
-    if: "! contains(toJSON(github.event.commits.*.message), '[skip ci]')"
-
     steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: zulu
+        java-version: 8
+    - run: ./gradlew -DsparkVersions=3.1 build -x test -x javadoc -x integrationTest
 
-      - name: 1. Check out code
-        uses: actions/checkout@v2 # https://github.com/actions/checkout
+  release:
+    if: ${{ github.event_name == 'push' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
         with:
-          fetch-depth: '0' # https://github.com/shipkit/shipkit-changelog#fetch-depth-on-ci
-
-      - name: 2. Set up Java
-        uses: actions/setup-java@v1
+          fetch-depth: '0'
+      - uses: actions/setup-java@v4
         with:
-          java-version: 1.8
-
-      - name: 3. Perform build
-        run: ./gradlew build
+          distribution: zulu
+          java-version: 8
+      - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
 
-      - name: 4. Perform release
-        # Release job, only for pushes to the main development branch
-        if: github.event_name == 'push'
+      - name: Perform release
+        if: ${{ github.event_name == 'push'
           && github.ref == 'refs/heads/openhouse-1.2.0'
           && github.repository == 'linkedin/iceberg'
-          && !contains(toJSON(github.event.commits.*.message), '[skip release]')
-
-        run: ./gradlew githubRelease publishToSonatype closeAndReleaseStagingRepository
+          && !contains(toJSON(github.event.commits.*.message), '[skip release]') }}
+        run: ./gradlew -DsparkVersions=3.1 githubRelease publishToSonatype closeAndReleaseStagingRepository -x test -x integrationTest
         env:
           GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
-          SONATYPE_USER: ${{secrets.SONATYPE_USER}}
-          SONATYPE_PWD: ${{secrets.SONATYPE_PWD}}
+          SONATYPE_USER: ${{secrets.SONATYPE_USERNAME}}
+          SONATYPE_PWD: ${{secrets.SONATYPE_PASSWORD}}
           PGP_KEY: ${{secrets.PGP_KEY}}
-          PGP_PWD: ${{secrets.PGP_PWD}}
+          PGP_PWD: ${{secrets.PGP_PWD}}

.github/workflows/spark-ci.yml

@@ -1,4 +1,3 @@
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -21,18 +20,13 @@ name: "Spark CI"
 on:
   push:
     branches:
-    - 'master'
-    - '0.**'
-    tags:
-    - 'apache-iceberg-**'
+    - 'openhouse-1.2.0'
   pull_request:
     paths-ignore:
-    - '.github/ISSUE_TEMPLATE/iceberg_bug_report.yml'
     - '.github/workflows/python-ci.yml'
     - '.github/workflows/flink-ci.yml'
     - '.github/workflows/hive-ci.yml'
     - '.gitignore'
-    - '.asf.yml'
     - 'dev/**'
     - 'mr/**'
     - 'hive3/**'
@@ -41,6 +35,7 @@ on:
     - 'flink/**'
     - 'pig/**'
     - 'python/**'
+    - 'python_legacy/**'
     - 'docs/**'
     - 'open-api/**'
     - 'format/**'
@@ -55,88 +50,25 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
-  spark2-tests:
-    runs-on: ubuntu-22.04
-    env:
-      SPARK_LOCAL_IP: localhost
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-java@v4
-      with:
-        distribution: zulu
-        java-version: 8
-    - uses: actions/cache@v4
-      with:
-        path: |
-          ~/.gradle/caches
-          ~/.gradle/wrapper
-        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
-        restore-keys: ${{ runner.os }}-gradle-
-    - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    - run: ./gradlew -DsparkVersions=2.4 -DhiveVersions= -DflinkVersions= :iceberg-spark:check :iceberg-spark:iceberg-spark-2.4:check :iceberg-spark:iceberg-spark-runtime-2.4:check -Pquick=true -x javadoc
-    - uses: actions/upload-artifact@v4
-      if: failure()
-      with:
-        name: test logs
-        path: |
-          **/build/testlogs
-
   spark-3x-scala-2-12-tests:
     runs-on: ubuntu-22.04
+    if: "github.repository == 'linkedin/iceberg'"
     strategy:
       matrix:
-        jvm: [8, 11]
-        spark: ['3.1', '3.2', '3.3']
+        spark: ['3.1']
     env:
       SPARK_LOCAL_IP: localhost
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-java@v4
         with:
           distribution: zulu
-          java-version: ${{ matrix.jvm }}
+          java-version: 8
       - uses: actions/cache@v4
         with:
-          path: |
-            ~/.gradle/caches
-            ~/.gradle/wrapper
-          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
-          restore-keys: ${{ runner.os }}-gradle-
+          path: ~/.gradle/caches
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+          restore-keys: ${{ runner.os }}-gradle
       - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-      - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=2.12 -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_2.12:check -Pquick=true -x javadoc
-      - uses: actions/upload-artifact@v4
-        if: failure()
-        with:
-          name: test logs
-          path: |
-            **/build/testlogs
 
-  spark-3x-scala-2-13-tests:
-    runs-on: ubuntu-22.04
-    strategy:
-      matrix:
-        jvm: [8, 11]
-        spark: ['3.2','3.3']
-    env:
-      SPARK_LOCAL_IP: localhost
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-java@v4
-        with:
-          distribution: zulu
-          java-version: ${{ matrix.jvm }}
-      - uses: actions/cache@v4
-        with:
-          path: |
-            ~/.gradle/caches
-            ~/.gradle/wrapper
-          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
-          restore-keys: ${{ runner.os }}-gradle-
-      - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-      - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=2.13 -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.13:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_2.13:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_2.13:check -Pquick=true -x javadoc
-      - uses: actions/upload-artifact@v4
-        if: failure()
-        with:
-          name: test logs
-          path: |
-            **/build/testlogs
+      - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=2.12 -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_2.12:check -Pquick=true -x test -x integrationTest -x javadoc

build.gradle

@@ -28,7 +28,9 @@ buildscript {
   }
   dependencies {
     classpath 'gradle.plugin.com.github.jengelman.gradle.plugins:shadow:7.0.0'
-    classpath 'com.palantir.baseline:gradle-baseline-java:4.42.0'
+    classpath('com.palantir.baseline:gradle-baseline-java:4.42.0') {
+      exclude group: 'com.palantir.tritium'
+    }
     // com.palantir.baseline:gradle-baseline-java:4.42.0 (the last version supporting Java 8) pulls
     // in an old version of the errorprone, which doesn't work w/ Gradle 8, so bump errorpone as
     // well.
@@ -131,6 +133,15 @@ subprojects {
   apply plugin: 'nebula.dependency-recommender'
   apply plugin: 'java-library'
 
+  configurations.all {
+    resolutionStrategy {
+      // Force specific versions of tritium to resolve version conflicts
+      force 'com.palantir.tritium:tritium-registry:0.17.0'
+      force 'com.palantir.tritium:tritium-core:0.17.0'
+      force 'com.palantir.tritium:tritium-metrics:0.17.0'
+    }
+  }
+
   if (project.name in REVAPI_PROJECTS) {
     apply plugin: 'com.palantir.revapi'
     revapi {
@@ -867,19 +878,7 @@ String getVersionFromFile() {
 
 String getProjectVersion() {
   if (versionFileExists()) {
-    String version = getVersionFromFile()
-    // If version is in 900+ series, treat it as a pre-release
-    Pattern pattern = Pattern.compile("^([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)?\$")
-    Matcher matcher = pattern.matcher(version)
-    if (matcher.matches()) {
-      int lastNum = Integer.valueOf(matcher.group(4))
-      if (lastNum >= 900) {
-        // Convert 1.2.0.900+ to 1.2.0.0-rcX format
-        int rcNum = lastNum - 899
-        return matcher.group(1) + "." + matcher.group(2) + "." + matcher.group(3) + "." + rcNum
-      }
-    }
-    return version
+    return getVersionFromFile()
   }
 
   try {

deploy.gradle

@@ -95,7 +95,30 @@ subprojects {
             }
             issueManagement {
               system = 'GitHub'
-              url = 'https://github.com/apache/iceberg/issues'
+              url = 'https://github.com/linkedin/iceberg/issues'
+            }
+            developers {
+              [
+                'wmoustafa:Walaa Eldin Moustafa',
+                'sumedhsakdeo:Sumedh Sakdeo',
+                'maluchari:Malini Mahalakshmi Venkatachari',
+                'rdsr:Ratandeep Ratti'
+              ].each { devData ->
+                developer {
+                  def devInfo = devData.split(':')
+                  id = devInfo[0]
+                  name = devInfo[1]
+                  url = 'https://github.com/' + devInfo[0]
+                  roles = ["Core developer"]
+                }
+              }
+            }
+            scm {
+              url = 'https://github.com/linkedin/iceberg.git'
+            }
+            ciManagement {
+              url = 'https://github.com/linkedin/iceberg/actions'
+              system = 'GitHub Actions'
             }
           }
         }

shipkit.gradle

@@ -25,23 +25,48 @@ tasks.named("generateChangelog") {
   previousRevision = project.ext.'shipkit-auto-version.previous-tag'
   githubToken = System.getenv("GITHUB_TOKEN")
   repository = "linkedin/iceberg"
-  
-  // Make generateChangelog depend on all necessary subproject tasks
-  subprojects.each { subproject ->
-    // Compilation tasks
-    dependsOn subproject.tasks.withType(JavaCompile)
-    
-    // Documentation tasks
-    dependsOn subproject.tasks.withType(Javadoc)
-    
-    // Publication tasks
-    dependsOn subproject.tasks.matching { it.name == 'generatePomFileForJavaLibraryPublication' }
-    
-    // Resource processing tasks
-    dependsOn subproject.tasks.withType(ProcessResources)
+  // Write changelog into the root build directory to avoid broad root-dir overlaps
+  outputFile = file("${buildDir}/CHANGELOG.md")
+}
+
+// Ensure Gradle knows ordering relative to tasks that write inside the repository
+// to avoid implicit dependency validation errors in Gradle 8+
+gradle.projectsEvaluated {
+  def genTaskProvider = tasks.named("generateChangelog")
+  genTaskProvider.configure { t ->
+    // Ensure we run after root task that writes to root build output
+    t.mustRunAfter(rootProject.tasks.matching { it.name == 'generateGitProperties' })
+
+    rootProject.subprojects.each { prj ->
+      // Run after resource processing tasks
+      t.mustRunAfter(prj.tasks.matching { it.name == 'processResources' })
+      t.mustRunAfter(prj.tasks.matching { it.name == 'processTestResources' })
+
+      // Run after Java compilation tasks
+      t.mustRunAfter(prj.tasks.matching { it.name == 'compileJava' })
+      t.mustRunAfter(prj.tasks.matching { it.name == 'compileTestJava' })
+
+      // Run after documentation and archive tasks that may touch repo files
+      t.mustRunAfter(prj.tasks.withType(org.gradle.api.tasks.javadoc.Javadoc))
+      t.mustRunAfter(prj.tasks.withType(org.gradle.api.tasks.bundling.Jar))
+
+      // Run after any POM generation tasks created by maven-publish
+      t.mustRunAfter(prj.tasks.withType(org.gradle.api.publish.maven.tasks.GenerateMavenPom))
+
+      // Run after module metadata tasks, if present
+      t.mustRunAfter(prj.tasks.matching { it.name.startsWith('generateMetadataFileFor') })
+
+      // Run after signing tasks (from 'signing' plugin)
+      t.mustRunAfter(prj.tasks.matching { it.name.startsWith('sign') })
+
+      // Run after publish tasks to avoid implicit dependency warnings in release pipelines
+      t.mustRunAfter(prj.tasks.matching { it.name.startsWith('publish') })
+    }
   }
 }
 
+ 
+
 apply plugin: "org.shipkit.shipkit-github-release" //https://github.com/shipkit/shipkit-changelog
 tasks.named("githubRelease") {
   def genTask = tasks.named("generateChangelog").get()
@@ -59,12 +84,11 @@ nexusPublishing {
       sonatype {
         username = System.getenv("SONATYPE_USER")
         password = System.getenv("SONATYPE_PWD")
-        nexusUrl.set(uri("https://s01.oss.sonatype.org/service/local/"))
-        snapshotRepositoryUrl.set(uri("https://s01.oss.sonatype.org/content/repositories/snapshots/"))
+        nexusUrl.set(uri("https://ossrh-staging-api.central.sonatype.com/service/local/"))
+        // nexusUrl.set(uri("https://s01.oss.sonatype.org/service/local/"))
+        snapshotRepositoryUrl.set(uri("https://central.sonatype.com/repository/maven-snapshots/"))
+        // snapshotRepositoryUrl.set(uri("https://s01.oss.sonatype.org/content/repositories/snapshots/"))
       }
     }
   }
-  transitionCheckOptions {
-    maxRetries.set(100)
-  }
 }

version.properties

@@ -19,4 +19,4 @@
 # Version of the produced binaries.
 # The version is inferred by shipkit-auto-version Gradle plugin (https://github.com/shipkit/shipkit-auto-version)
 # Using 900+ series to indicate pre-release versions
-version=1.2.0.900
+version=1.2.0.901

version.txt

@@ -1 +1 @@
-1.2.0.0-rc1
+1.2.0.*

versions.props

@@ -1,4 +1,5 @@
 org.slf4j:* = 1.7.36
+com.palantir.tritium:* = 0.17.0
 org.apache.avro:avro = 1.11.1
 org.apache.calcite:* = 1.10.0
 org.apache.hadoop:* = 2.7.3