diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 34f505c0..c0f519a4 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -58,7 +58,15 @@ jobs: install_system_dependencies: true - name: Install dependencies and build - run: conan build . -s build_type=Release --build=missing --update -g VirtualRunEnv ${{ inputs.conan_extra_args }} + run: conan build . -s build_type=Release --build=missing --update -g VirtualRunEnv ${{ inputs.conan_extra_args }} --lockfile-out=conan.lock + + - name: Commit lockfile + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') + uses: stefanzweifel/git-auto-commit-action@v5 + with: + file_pattern: conan.lock + commit_message: "Update Conan lockfile" + status_options: --untracked-files=no - name: Run benchmark id: run-test diff --git a/.github/workflows/conan-package.yml b/.github/workflows/conan-package.yml index 65376520..1fca4811 100644 --- a/.github/workflows/conan-package.yml +++ b/.github/workflows/conan-package.yml @@ -113,4 +113,12 @@ jobs: install_system_dependencies: ${{ inputs.install_system_dependencies }} - name: Create the Package (binaries) - run: conan create ${{ inputs.conan_recipe_root }} --version ${{ needs.conan-recipe-version.outputs.version_base }} --user ${{ needs.conan-recipe-version.outputs.user }} --channel ${{ needs.conan-recipe-version.outputs.channel }} ${{ inputs.conan_extra_args }} --build=missing ${{ matrix.conan_extra_args }} + run: conan create ${{ inputs.conan_recipe_root }} --version ${{ needs.conan-recipe-version.outputs.version_base }} --user ${{ needs.conan-recipe-version.outputs.user }} --channel ${{ needs.conan-recipe-version.outputs.channel }} ${{ inputs.conan_extra_args }} --build=missing ${{ matrix.conan_extra_args }} --lockfile-out=conan.lock + + - name: Commit lockfile + if: github.event_name == 'push' && (github.ref_type == 'tag' || (github.ref_type == 'branch' && github.ref_name != 'main' && github.ref_name != 'master')) + uses: stefanzweifel/git-auto-commit-action@v5 + with: + file_pattern: conan.lock + commit_message: "Update Conan lockfile" + status_options: --untracked-files=no diff --git a/.github/workflows/cura-installer-linux.yml b/.github/workflows/cura-installer-linux.yml index cfbc243c..97276d00 100644 --- a/.github/workflows/cura-installer-linux.yml +++ b/.github/workflows/cura-installer-linux.yml @@ -80,7 +80,15 @@ jobs: run: echo -n "${{ secrets.GPG_PRIVATE_KEY }}" | base64 --decode | gpg --import - name: Gather/build the packages - run: conan install --requires "${{ inputs.cura_conan_version != '' && inputs.cura_conan_version || format('cura/{0}@ultimaker/testing', steps.setup-environment.outputs.package_version) }}" ${{ inputs.conan_args }} --build=missing --update -of cura_inst --deployer-package="*" --profile ${{ steps.set-overrides.outputs.profile }} -c user.sentry:token="${{ secrets.CURAENGINE_SENTRY_TOKEN }}" ${{ inputs.enterprise && '-o "cura/*:enterprise=True"' || '' }} ${{ inputs.staging && '-o "cura/*:staging=True"' || '' }} ${{ inputs.private_data && '-o "cura/*:internal=True"' || '' }} + run: conan install --requires "${{ inputs.cura_conan_version != '' && inputs.cura_conan_version || format('cura/{0}@ultimaker/testing', steps.setup-environment.outputs.package_version) }}" ${{ inputs.conan_args }} --build=missing --update -of cura_inst --deployer-package="*" --profile ${{ steps.set-overrides.outputs.profile }} -c user.sentry:token="${{ secrets.CURAENGINE_SENTRY_TOKEN }}" ${{ inputs.enterprise && '-o "cura/*:enterprise=True"' || '' }} ${{ inputs.staging && '-o "cura/*:staging=True"' || '' }} ${{ inputs.private_data && '-o "cura/*:internal=True"' || '' }} --lockfile-out=conan.lock + + - name: Commit lockfile + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') + uses: stefanzweifel/git-auto-commit-action@v5 + with: + file_pattern: conan.lock + commit_message: "Update Conan lockfile" + status_options: --untracked-files=no - name: Create the Cura distribution with pyinstaller id: prepare-distribution diff --git a/.github/workflows/cura-installer-macos.yml b/.github/workflows/cura-installer-macos.yml index 90e25f3e..407deca9 100644 --- a/.github/workflows/cura-installer-macos.yml +++ b/.github/workflows/cura-installer-macos.yml @@ -110,7 +110,15 @@ jobs: run: security unlock -p "${{ steps.macos-keychain-developer-cert.outputs.keychain-password }}" signing_temp.keychain - name: Gather/build the packages - run: conan install --requires "${{ inputs.cura_conan_version != '' && inputs.cura_conan_version || format('cura/{0}@ultimaker/testing', steps.setup-environment.outputs.package_version) }}" ${{ inputs.conan_args }} --build=missing --update -of cura_inst --deployer-package="*" --profile ${{ steps.set-overrides.outputs.profile }} -c user.sentry:token="${{ secrets.CURAENGINE_SENTRY_TOKEN }}" ${{ inputs.enterprise && '-o "cura/*:enterprise=True"' || '' }} ${{ inputs.staging && '-o "cura/*:staging=True"' || '' }} ${{ inputs.private_data && '-o "cura/*:internal=True"' || '' }} + run: conan install --requires "${{ inputs.cura_conan_version != '' && inputs.cura_conan_version || format('cura/{0}@ultimaker/testing', steps.setup-environment.outputs.package_version) }}" ${{ inputs.conan_args }} --build=missing --update -of cura_inst --deployer-package="*" --profile ${{ steps.set-overrides.outputs.profile }} -c user.sentry:token="${{ secrets.CURAENGINE_SENTRY_TOKEN }}" ${{ inputs.enterprise && '-o "cura/*:enterprise=True"' || '' }} ${{ inputs.staging && '-o "cura/*:staging=True"' || '' }} ${{ inputs.private_data && '-o "cura/*:internal=True"' || '' }} --lockfile-out=conan.lock + + - name: Commit lockfile + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') + uses: stefanzweifel/git-auto-commit-action@v5 + with: + file_pattern: conan.lock + commit_message: "Update Conan lockfile" + status_options: --untracked-files=no - name: Create the Cura distribution with pyinstaller id: prepare-distribution diff --git a/.github/workflows/cura-installer-windows.yml b/.github/workflows/cura-installer-windows.yml index e40b94e3..60a511b5 100644 --- a/.github/workflows/cura-installer-windows.yml +++ b/.github/workflows/cura-installer-windows.yml @@ -87,7 +87,15 @@ jobs: run: | $pydir = type pydir.txt $env:PATH += ";$pydir;$pydir/Scripts" - conan install --requires "${{ inputs.cura_conan_version != '' && inputs.cura_conan_version || format('cura/{0}@ultimaker/testing', steps.setup-environment.outputs.package_version) }}" ${{ inputs.conan_args }} --build=missing --update -of cura_inst --deployer-package="*" --profile ${{ steps.set-overrides.outputs.profile }} -c user.sentry:token="${{ secrets.CURAENGINE_SENTRY_TOKEN }}" ${{ inputs.enterprise && '-o "cura/*:enterprise=True"' || '' }} ${{ inputs.staging && '-o "cura/*:staging=True"' || '' }} ${{ inputs.private_data && '-o "cura/*:internal=True"' || '' }} + conan install --requires "${{ inputs.cura_conan_version != '' && inputs.cura_conan_version || format('cura/{0}@ultimaker/testing', steps.setup-environment.outputs.package_version) }}" ${{ inputs.conan_args }} --build=missing --update -of cura_inst --deployer-package="*" --profile ${{ steps.set-overrides.outputs.profile }} -c user.sentry:token="${{ secrets.CURAENGINE_SENTRY_TOKEN }}" ${{ inputs.enterprise && '-o "cura/*:enterprise=True"' || '' }} ${{ inputs.staging && '-o "cura/*:staging=True"' || '' }} ${{ inputs.private_data && '-o "cura/*:internal=True"' || '' }} --lockfile-out=conan.lock + + - name: Commit lockfile + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') + uses: stefanzweifel/git-auto-commit-action@v5 + with: + file_pattern: conan.lock + commit_message: "Update Conan lockfile" + status_options: --untracked-files=no - name: Create the Cura distribution with pyinstaller id: prepare-distribution diff --git a/.github/workflows/lint-tidier.yml b/.github/workflows/lint-tidier.yml index 9f59dae0..a38d17bc 100644 --- a/.github/workflows/lint-tidier.yml +++ b/.github/workflows/lint-tidier.yml @@ -21,7 +21,15 @@ jobs: src/**/*.c* - name: Install dependencies - run: conan install . -c tools.build:skip_test=False -s *:build_type=Release --build=missing --update + run: conan install . -c tools.build:skip_test=False -s *:build_type=Release --build=missing --update --lockfile-out=conan.lock + + - name: Commit lockfile + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') + uses: stefanzweifel/git-auto-commit-action@v5 + with: + file_pattern: conan.lock + commit_message: "Update Conan lockfile" + status_options: --untracked-files=no - name: Build application and tests run: | diff --git a/.github/workflows/npm-package.yml b/.github/workflows/npm-package.yml index cdba75f2..24fcded2 100644 --- a/.github/workflows/npm-package.yml +++ b/.github/workflows/npm-package.yml @@ -35,7 +35,15 @@ jobs: repository_path: _sources - name: Gather/build the packages - run: conan install --requires "${{ inputs.package_version_full }}" -pr:h cura_wasm.jinja -g npm --build=missing --update ${{ inputs.conan_extra_args }} -of . + run: conan install --requires "${{ inputs.package_version_full }}" -pr:h cura_wasm.jinja -g npm --build=missing --update ${{ inputs.conan_extra_args }} -of . --lockfile-out=conan.lock + + - name: Commit lockfile + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') + uses: stefanzweifel/git-auto-commit-action@v5 + with: + file_pattern: conan.lock + commit_message: "Update Conan lockfile" + status_options: --untracked-files=no - name: Use Node.js uses: actions/setup-node@v4 diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index e30f8a1b..04e4cae8 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -39,7 +39,15 @@ jobs: install_system_dependencies: true - name: Install dependencies and build unit test - run: conan build . -s build_type=Release --build=missing --update -c tools.build:skip_test=False ${{ inputs.test_use_pytest && '-g VirtualPythonEnv -c user.generator.virtual_python_env:dev_tools=True' || '' }} ${{ inputs.conan_extra_args }} + run: conan build . -s build_type=Release --build=missing --update -c tools.build:skip_test=False ${{ inputs.test_use_pytest && '-g VirtualPythonEnv -c user.generator.virtual_python_env:dev_tools=True' || '' }} ${{ inputs.conan_extra_args }} --lockfile-out=conan.lock + + - name: Commit lockfile + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') + uses: stefanzweifel/git-auto-commit-action@v5 + with: + file_pattern: conan.lock + commit_message: "Update Conan lockfile" + status_options: --untracked-files=no - name: Run ctest-based unit test if: ${{ inputs.test_use_ctest }} diff --git a/.github/workflows/update-translation.yml b/.github/workflows/update-translation.yml index 03f4bbc5..f79caf51 100644 --- a/.github/workflows/update-translation.yml +++ b/.github/workflows/update-translation.yml @@ -21,10 +21,10 @@ jobs: branch: ${{ inputs.branch }} - name: Update translation files using Conan install - run: conan install . --build=missing --update -o "&:enable_i18n=True" + run: conan install . --build=missing --update -o "&:enable_i18n=True" --lockfile-out=conan.lock - - uses: stefanzweifel/git-auto-commit-action@v4 + - uses: stefanzweifel/git-auto-commit-action@v5 with: - file_pattern: resources/i18n/*.po resources/i18n/*.pot + file_pattern: resources/i18n/*.po resources/i18n/*.pot conan.lock status_options: --untracked-files=no commit_message: Update translations diff --git a/README.md b/README.md index f3e1a183..d6e6ab4e 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,37 @@ UltiMaker developers working on Cura need to install the conan-config from the ` conan config install https://github.com/Ultimaker/conan-config.git -a "-b dev" ``` +## Conan Lockfiles for Reproducible Builds + +All CI/CD workflows use Conan lockfiles to ensure reproducible builds across time and environments. Lockfiles pin exact package versions and revisions, preventing dependency drift between developer machines, CI servers, and release builds. + +### Key Benefits: +- **Reproducibility**: Same dependencies for every build, regardless of when or where it runs +- **Traceability**: Full audit trail of exact dependency versions used in each build +- **Safety**: Builds fail if dependencies can't be resolved from the lockfile (strict mode) +- **Consistency**: Developers, CI, and releases all use identical dependency graphs + +### How it Works: +- Workflows use `--lockfile-out=conan.lock` to generate/update the lockfile +- Conan 2 automatically loads `conan.lock` if it exists in the working directory +- Default strict mode ensures all dependencies must be in the lockfile +- Lockfiles are generated/updated automatically during CI builds +- **Auto-commit behavior varies by workflow:** + - **Most workflows**: Lockfiles committed on push to main/master branches + - **Package creation workflow**: Lockfiles committed on push to release branches/tags (not main/master) +- Each repository should maintain its own `conan.lock` file in version control + +### Maintenance: +- Lockfiles are automatically created on first CI run if they don't exist +- Lockfiles are automatically updated when new dependencies are resolved +- **Lockfiles are automatically committed based on workflow type:** + - Test/build workflows commit on push to main/master + - Package creation commits on release branches/tags +- **Note**: Calling workflows must have `contents: write` permission for auto-commit to work +- To manually add a new dependency: `conan lock add --requires=package/version --lockfile=conan.lock --lockfile-out=conan.lock` then run `conan install` to resolve it +- For development branches with frequently changing dependencies, use `--lockfile-partial` flag (add to `conan_extra_args` workflow input) +- If merge conflicts occur in lockfiles, regenerate by deleting the lockfile and running the CI workflow + ## Pipeline caching over workflows Ported a lot of workflows to this repo. All of the reusable workflows have pipeline chacing enabled for Conan downloads and Conan data folders. The caching key have a default fallback key. This ensures that the cache is updated with the latest changes on the Artifactory remote server but reuses the previous stored downloads and data, greatly reducing our download bandwidth and flexible costs.