Allow xcframework generation to skip xcarchive recompilation.

arik-so · arik-so · commit 762bfbd2b7e2 · 2022-07-10T11:33:03.000-07:00
diff --git a/.github/actions/archive-framework/action.yml b/.github/actions/archive-framework/action.yml
@@ -1,30 +1,29 @@
 name: Archive Framework
 inputs:
-  scheme:
-    description: Scheme
-    required: true
-  destination:
-    description: Destination
-    required: true
-  platform:
-    description: Platform Name
-    required: true
-  project:
-    description: Project directory, relative to the xcode folder
-    required: true
+    destination:
+        description: Destination platform/architecture details. Typically looks like "generic/platform=iOS" or "platform=macOS,variant=Mac Catalyst,arch=x86_64"
+        required: true
+    human_readable_platform:
+        description: Human-readable platform name, used for directory naming
+        required: true
+description: Framework archivation description.
 runs:
-  using: composite
-  steps:
-  - shell: bash
-    working-directory: .
-    run: |
-      cd xcode
+    using: composite
+    steps:
+        -   shell: bash
+            working-directory: .
+            run: |
+                cd xcode
 
-      xcodebuild archive \
-        -scheme '${{ inputs.scheme }}' \
-        -project '${{ inputs.project }}' \
-        -destination '${{ inputs.destination }}' \
-        -archivePath "$BUILD_DIR/LDKFramework-${{ inputs.platform }}" \
-        ENABLE_BITCODE=NO \
-        CLANG_ADDRESS_SANITIZER=NO CLANG_ADDRESS_SANITIZER_ALLOW_ERROR_RECOVERY=NO CLANG_ADDRESS_SANITIZER_USE_AFTER_SCOPE=NO \
-        SKIP_INSTALL=NO BUILD_LIBRARY_FOR_DISTRIBUTION=YES
+                LDK_C_BINDINGS_BINARY_DIRECTORY="" \
+                xcodebuild archive \
+                  -scheme 'LDKFramework' \
+                  -project 'LDKFramework/LDKFramework.xcodeproj' \
+                  -destination '${{ inputs.destination }}' \
+                  -archivePath "/Users/runner/work/ldk-swift/ldk-swift/bindings/bin/release/${{ inputs.human_readable_platform }}/${{ inputs.human_readable_platform }}" \
+                  ENABLE_BITCODE=NO \
+                  CLANG_ADDRESS_SANITIZER=NO CLANG_ADDRESS_SANITIZER_ALLOW_ERROR_RECOVERY=NO CLANG_ADDRESS_SANITIZER_USE_AFTER_SCOPE=NO \
+                  SKIP_INSTALL=NO BUILD_LIBRARY_FOR_DISTRIBUTION=YES \
+                  LDK_C_BINDINGS_BASE="${LDK_C_BINDINGS_BASE}" \
+                  LDK_C_BINDINGS_BINARY_DIRECTORY="" \
+                  LDK_C_BINDINGS_BINARY_FORCED_REBUILD_OUTPUT_DIRECTORY="${LDK_C_BINDINGS_BINARY_DIRECTORY}"
diff --git a/.github/workflows/build-xcframework-parallel-archives.yml b/.github/workflows/build-xcframework-parallel-archives.yml
@@ -1,4 +1,4 @@
-name: Generate XCFramework from parallel XCArchive builds
+name: XCFramework from XCArchives
 permissions:
     contents: read
     statuses: read
@@ -14,29 +14,21 @@ on:
             - "*"
 
 jobs:
-    build:
-        name: Build XCArchive for ${{ matrix.configuration['platform'] }}
+    build-xcarchives:
+        name: Build xcarchive for ${{ matrix.configuration['human_readable_platform'] }}
         runs-on: macos-12
         strategy:
             fail-fast: false
             matrix:
                 configuration:
-                    -   scheme: LDKFramework
-                        destination: generic/platform=iOS
-                        platform: iOS
-                        project: LDKFramework/LDKFramework.xcodeproj
-                    -   scheme: LDKFramework
-                        destination: platform=iOS Simulator,OS=15.2,name=iPhone 13 Pro
-                        platform: iOS Simulator
-                        project: LDKFramework/LDKFramework.xcodeproj
-                    -   scheme: LDKFramework
-                        destination: generic/platform=OS X
-                        platform: OS X
-                        project: LDKFramework/LDKFramework.xcodeproj
-                    -   scheme: LDKFramework
-                        destination: platform=macOS,variant=Mac Catalyst,arch=x86_64
-                        platform: Mac Catalyst
-                        project: LDKFramework/LDKFramework.xcodeproj
+                    -   destination: generic/platform=iOS
+                        human_readable_platform: 'iphoneos'
+                    -   destination: platform=iOS Simulator,OS=15.2,name=iPhone 13 Pro
+                        human_readable_platform: 'iphonesimulator'
+                    -   destination: generic/platform=OS X
+                        human_readable_platform: 'macosx'
+                    -   destination: platform=macOS,variant=Mac Catalyst,arch=x86_64
+                        human_readable_platform: 'catalyst'
         env:
             DEVELOPER_DIR: /Applications/Xcode_13.2.1.app/Contents/Developer
             LDK_C_BINDINGS_BASE: /Users/runner/work/ldk-swift/ldk-swift/bindings/artifacts/ldk-c-bindings
@@ -52,34 +44,37 @@ jobs:
                 uses: ./.github/actions/install-dependencies
             -   name: Build
                 id: build
-                uses: ./.github/actions/build-for-testing
+                uses: ./.github/actions/archive-framework
                 with:
-                    project: ${{ matrix.configuration['project'] }}
-                    scheme: ${{ matrix.configuration['scheme'] }}
                     destination: ${{ matrix.configuration['destination'] }}
+                    human_readable_platform: ${{ matrix.configuration['human_readable_platform'] }}
+            -   name: Xcarchive hack
+                run: touch ./bindings_root.txt
+            -   name: Upload xcarchive
+                uses: actions/upload-artifact@v3
+                with:
+                    name: bindings
+                    path: ./bindings/**/bin/release/${{ matrix.configuration['human_readable_platform'] }}/${{ matrix.configuration['human_readable_platform'] }}.xcarchive
 
-#    generate:
-#        name: Generate XCFramework
-#        runs-on: macos-12
-#        env:
-#            DEVELOPER_DIR: /Applications/Xcode_13.2.1.app/Contents/Developer
-#            LDK_C_BINDINGS_BASE: /Users/runner/work/ldk-swift/ldk-swift/bindings/artifacts/ldk-c-bindings
-#        steps:
-#            -   name: Configure Xcode
-#                uses: maxim-lobanov/setup-xcode@v1
-#                with:
-#                    xcode-version: latest
-#            -   name: Checkout
-#                uses: actions/checkout@v2
-#            -   name: Install Dependencies
-#                uses: ./.github/actions/install-dependencies
-#            -   name: Generate XCFramework
-#                uses: ./.github/actions/generate-xcframework
-#            -   name: Zip Xcframework (remove before merging!)
-#                run: |
-#                    ditto -c -k --sequesterRsrc /Users/runner/work/ldk-swift/ldk-swift/bindings/bin/release/LDKFramework.xcframework LDKFramework.xcframework.zip
-#            -   name: Upload Framework Artifact (remove before merging!)
-#                uses: actions/upload-artifact@v3
-#                with:
-#                    name: LDKFramework.xcframework.zip
-#                    path: LDKFramework.xcframework.zip
+    generate:
+        name: Combine xcarchives into xcframework
+        runs-on: macos-12
+        needs: [ build-xcarchives ]
+        env:
+            DEVELOPER_DIR: /Applications/Xcode_13.2.1.app/Contents/Developer
+            LDK_C_BINDINGS_BASE: /Users/runner/work/ldk-swift/ldk-swift/bindings/artifacts/ldk-c-bindings
+        steps:
+            -   name: Configure Xcode
+                uses: maxim-lobanov/setup-xcode@v1
+                with:
+                    xcode-version: latest
+            -   name: Checkout
+                uses: actions/checkout@v2
+            -   name: Download xcarchives
+                uses: actions/download-artifact@v3
+            -   name: Generate xcframework
+                shell: bash
+                run:
+                    python3 ./src/scripts/generate_xcframework.py
+                env:
+                    PRESERVE_XCARCHIVES: true
diff --git a/.github/workflows/build-xcframework-parallel-libldk.yml b/.github/workflows/build-xcframework-parallel-libldk.yml
@@ -1,4 +1,4 @@
-name: Generate XCframework from parallel libldk.a builds
+name: XCframework from libldks
 permissions:
     contents: read
     statuses: read
@@ -15,9 +15,10 @@ on:
 
 jobs:
     build-libldks:
-        name: Build lipo libldk.a for ${{ matrix.configuration['human_readable_platform'] }}
+        name: Build & lipo libldk.a for ${{ matrix.configuration['human_readable_platform'] }}
         runs-on: macos-12
         env:
+            DEVELOPER_DIR: /Applications/Xcode_13.2.1.app/Contents/Developer
             LDK_C_BINDINGS_BASE: /Users/runner/work/ldk-swift/ldk-swift/bindings/artifacts/ldk-c-bindings
         strategy:
             fail-fast: true
@@ -60,11 +61,13 @@ jobs:
             -   name: Upload binary
                 uses: actions/upload-artifact@v3
                 with:
-                    name: libldk-${{ matrix.configuration['human_readable_platform'] }}.a
+                    name: bindings
                     path: |
-                        /Users/runner/work/ldk-swift/ldk-swift/bindings/bin/release/${{ matrix.configuration['human_readable_platform'] }}/libldk.a
+                        ./bindings/**/artifacts/ldk-c-bindings/lightning-c-bindings/include/*.h
+                        ./bindings/**/artifacts/ldk-c-bindings/ldk-net/ldk_net.*
+                        ./bindings/**/bin/release/${{ matrix.configuration['human_readable_platform'] }}/libldk.a
     generate-parallel:
-        name: Generate XCarchives and XCFramework based on parallelized libldk builds
+        name: Build xcarchives and generate xcframework
         needs: [ build-libldks ]
         runs-on: macos-12
         env:
@@ -77,7 +80,7 @@ jobs:
                     xcode-version: latest
             -   name: Checkout
                 uses: actions/checkout@v2
-            -   name: Download xcarchives
+            -   name: Download libldks
                 uses: actions/download-artifact@v3
             -   name: Generate Swift Bindings
                 shell: bash
diff --git a/.github/workflows/build-xcframework-sequential.yml b/.github/workflows/build-xcframework-sequential.yml
@@ -1,4 +1,4 @@
-name: Generate XCFramework from sequential bulk build script
+name: XCFramework from sequential Python script
 permissions:
     contents: read
     statuses: read
@@ -15,7 +15,7 @@ on:
 
 jobs:
     generate-sequential:
-        name: Generate XCFramework
+        name: Build libldks, xcarchives, and combine into xcframework
         runs-on: macos-12
         env:
             DEVELOPER_DIR: /Applications/Xcode_13.2.1.app/Contents/Developer
diff --git a/.github/workflows/test-artifacts.yml b/.github/workflows/test-artifacts.yml
@@ -0,0 +1,54 @@
+name: Test Artifacts
+permissions:
+    contents: read
+    statuses: read
+    pull-requests: read
+    actions: read
+    checks: write
+on:
+    workflow_dispatch:
+
+jobs:
+    create-artifact:
+        name: Add ${{ matrix.configuration['human_readable_platform'] }} to artifact
+        runs-on: macos-12
+        strategy:
+            fail-fast: false
+            matrix:
+                configuration:
+                    -   destination: generic/platform=iOS
+                        human_readable_platform: 'iphoneos'
+                    -   destination: platform=iOS Simulator,OS=15.2,name=iPhone 13 Pro
+                        human_readable_platform: 'iphonesimulator'
+                    -   destination: generic/platform=OS X
+                        human_readable_platform: 'macosx'
+                    -   destination: platform=macOS,variant=Mac Catalyst,arch=x86_64
+                        human_readable_platform: 'catalyst'
+        steps:
+            -   name: Checkout
+                uses: actions/checkout@v2
+            -   name: Xcarchive emulation
+                run: |
+                    mkdir -p ./bindings/bin/release/${{ matrix.configuration['human_readable_platform'] }}
+                    touch ./bindings/bin/release/${{ matrix.configuration['human_readable_platform'] }}/${{ matrix.configuration['human_readable_platform'] }}.xcarchive
+            -   name: Upload emulated xcarchive
+                uses: actions/upload-artifact@v3
+                with:
+                    name: uploaded-artifact-name
+                    path: ./bindings/**/bin/release/${{ matrix.configuration['human_readable_platform'] }}/${{ matrix.configuration['human_readable_platform'] }}.xcarchive
+
+    download-artifact:
+        name: Download artifact
+        runs-on: macos-12
+        needs: [ create-artifact ]
+        steps:
+            -   name: Checkout
+                uses: actions/checkout@v2
+            -   name: Download emulated xcarchives
+                uses: actions/download-artifact@v3
+                with:
+                    path: ./downloaded-artifact-name
+            -   name: Preview download result
+                run: |
+                    ls -ll -R
+
diff --git a/src/scripts/build_individual_libldk.py b/src/scripts/build_individual_libldk.py
@@ -11,7 +11,7 @@
 
 def run(config: ScriptConfig):
 	if len(config.LIBLDK_BUILD_CONFIGURATIONS) != 1:
-		print('Individual libldk build must have exactly 1 build configuration.')
+		print('Individual libldk build must have exactly 1 build configuration.', file=sys.stderr)
 		sys.exit(1)
 
 	ldkBuildConfig = config.LIBLDK_BUILD_CONFIGURATIONS[0]
diff --git a/src/scripts/generate_xcframework.py b/src/scripts/generate_xcframework.py
@@ -6,7 +6,7 @@
 
 
 def parse_config() -> ScriptConfig:
-	config = ScriptConfig.parse(allow_ldk_argument=True, parse_configuration=True)
+	config = ScriptConfig.parse(allow_ldk_argument=True, parse_configuration=True, parse_xcarchive_preservation=True)
 
 	individual_configurations: [ArchiveConfig] = [
 		ArchiveConfig('iOS Simulator', 'iphonesimulator'),
@@ -57,6 +57,16 @@ def run(config: ScriptConfig):
 			current_human_readable_platform  # the last component is the filename excluding .xcarchive
 		)
 
+		# XCFRAMEWORK_INPUT_FLAGS="${XCFRAMEWORK_INPUT_FLAGS}-framework ${CURRENT_ARCHIVE_PATH}.xcarchive/Products/Library/Frameworks/LDKFramework.framework "
+		framework_input_flags += [
+			'-framework',
+			f'{xcarchive_output_path}.xcarchive/Products/Library/Frameworks/LDKFramework.framework'
+		]
+
+		if config.PRESERVE_XCARCHIVES:
+			# we don't regenerate any new xcarchives, and simply reuse the existing ones
+			break
+
 		# create clean derived data directory
 		if os.path.exists(derived_data_directory):
 			shutil.rmtree(derived_data_directory, ignore_errors=True)
@@ -88,12 +98,6 @@ def run(config: ScriptConfig):
 		# clean up the derived data
 		shutil.rmtree(derived_data_directory, ignore_errors=True)
 
-		# XCFRAMEWORK_INPUT_FLAGS="${XCFRAMEWORK_INPUT_FLAGS}-framework ${CURRENT_ARCHIVE_PATH}.xcarchive/Products/Library/Frameworks/LDKFramework.framework "
-		framework_input_flags += [
-			'-framework',
-			f'{xcarchive_output_path}.xcarchive/Products/Library/Frameworks/LDKFramework.framework'
-		]
-
 	# xcodebuild -create-xcframework ${XCFRAMEWORK_INPUT_FLAGS} -output ${XCFRAMEWORK_OUTPUT_PATH}"
 	if os.path.exists(xcframework_output_path):
 		shutil.rmtree(xcframework_output_path, ignore_errors=True)
diff --git a/src/scripts/prepare_xcode_project.py b/src/scripts/prepare_xcode_project.py
@@ -22,11 +22,11 @@ def run(config: ScriptConfig):
 	copy_c_files.run(config)
 
 	if bindings_binary_directory and forced_rebuild_override_directory:
-		print('LDK_C_BINDINGS_BINARY_DIRECTORY and LDK_C_BINDINGS_BINARY_FORCED_REBUILD_OUTPUT_DIRECTORY may not both be set at the same time!')
+		print('LDK_C_BINDINGS_BINARY_DIRECTORY and LDK_C_BINDINGS_BINARY_FORCED_REBUILD_OUTPUT_DIRECTORY may not both be set at the same time!', file=sys.stderr)
 		sys.exit(1)
 
 	if not bindings_binary_directory and not forced_rebuild_override_directory:
-		print('One of LDK_C_BINDINGS_BINARY_DIRECTORY and LDK_C_BINDINGS_BINARY_FORCED_REBUILD_OUTPUT_DIRECTORY must be set!')
+		print('One of LDK_C_BINDINGS_BINARY_DIRECTORY and LDK_C_BINDINGS_BINARY_FORCED_REBUILD_OUTPUT_DIRECTORY must be set!', file=sys.stderr)
 		sys.exit(1)
 
 	if not forced_rebuild_override_directory:
diff --git a/src/scripts/script_config.py b/src/scripts/script_config.py
