diff --git a/.github/workflows/bootkit.yml b/.github/workflows/bootkit.yml new file mode 100644 index 0000000000..957733be34 --- /dev/null +++ b/.github/workflows/bootkit.yml @@ -0,0 +1,108 @@ +name: Test EMTBootkit image build +on: + workflow_dispatch: + # pull_request: + # branches: [3.0] + +permissions: + id-token: write + contents: read + +env: + RELEASE_NAME: Test EMTBootkit + CODE_REPO_URL: open-edge-platform/edge-microvisor-toolkit + CODE_REPO_NAME: edge-microvisor-toolkit + CODE_REPO_BRANCH: "3.0" + +jobs: + image_build: + name: EMTBootkit Image Build + runs-on: Microvisor + timeout-minutes: 1440 # extend timeout from default 6 hour to 24 hour + environment: production + outputs: + upload_dir: ${{ steps.extract-files.outputs.upload_dir }} + datetime_as_version: ${{ steps.set-version.outputs.datetime_as_version }} + + steps: + + - name: Checkout default repo + uses: actions/checkout@v4 + + - name: Checkout + uses: actions/checkout@v4 + with: + repository: ${{ env.CODE_REPO_URL }} + ref: ${{ env.CODE_REPO_BRANCH }} + path: ${{ env.CODE_REPO_NAME }} + + - name: Set DATETIME_AS_VERSION + id: set-version + run: | + DATETIME_AS_VERSION=$(date +'%Y%m%d.%H%M') + echo "datetime_as_version=$DATETIME_AS_VERSION" >> "$GITHUB_OUTPUT" + echo "DATETIME_AS_VERSION=$DATETIME_AS_VERSION" >> "$GITHUB_ENV" + + - name: Build Edge microvisor Bootkit Image + id: build-image + run: | + cd ${{ env.CODE_REPO_NAME }} + + pushd toolkit + sudo make image -j8 \ + REBUILD_TOOLS=y \ + REBUILD_PACKAGES=n \ + EMT_DISTRIB_SPECS=y \ + VALIDATE_TOOLCHAIN_GPG=n \ + DATETIME_AS_VERSION="${DATETIME_AS_VERSION}" \ + DAILY_BUILD_REPO=./resources/manifests/package/development.repo \ + CONFIG_FILE=./imageconfigs/edge-image-bootkit.json \ + REPO_LIST+=./resources/manifests/package/development.repo + # check for output image to determine the results. + shopt -s nullglob + tar_files=(../out/images/edge-image-bootkit/*.tar.gz) + if [ ${#tar_files[@]} -eq 0 ]; then + echo "No .tar files were found in the output directory." + exit 1 + else + echo "Found ..tar files in the output directory:" + printf '%s\n' "${tar_files[@]}" + fi + popd + + - name: Extract vmlinuz & initramfs + id: extract-files + run: | + cd ${{ env.CODE_REPO_NAME }} + UPLOAD_DIR=${GITHUB_WORKSPACE}/${{ env.CODE_REPO_NAME }}/$DATETIME_AS_VERSION + echo "upload_dir=$UPLOAD_DIR" >> "$GITHUB_OUTPUT" + echo "current pwd is" + pwd + mkdir -p ${UPLOAD_DIR} + echo "workspace is ${GITHUB_WORKSPACE}" + tarFile=$(find "${GITHUB_WORKSPACE}/edge-microvisor-toolkit/out/images/edge-image-bootkit" -type f -name "*.tar.gz" | head -n 1) + if [[ -z "$tarFile" ]]; then + echo "No .tar.gz file found " + exit 1 + fi + echo "Using tar file $tarFile" + echo "Extracting images vmlinuz.img & initramfs.emt3" + cd toolkit/imageconfigs/scripts + chmod +x generate-bootkit-initramfs.sh + sudo ./generate-bootkit-initramfs.sh -f "${tarFile}" -o $UPLOAD_DIR + echo "Validating extracted images in upload directory" + initramfsExists=$(find ${UPLOAD_DIR} -type f -name "initramfs-*.img" | wc -l) + vmlinuzExists=$(find ${UPLOAD_DIR} -type f -name "vmlinuz-*" | wc -l) + if [[ $initramfsExists -eq 0 || $vmlinuzExists -eq 0 ]]; then + echo "ERROR: Required files not found: " + [[ $initramfsExists -eq 0 ]] && echo "Missing initramfs-*.img " + [[ $vmlinuzExists -eq 0 ]] && echo "Missing vmlinuz-* " + exit 1 + fi + echo "Both initramfs-*.img and vmlinuz-* files exist in ${UPLOAD_DIR}" + cd $UPLOAD_DIR + tar -cvzf emb_uos_x86_64_${DATE_AS_VERSION}.tar.gz vmlinuz*.emt3 initramfs*.img + cp ../../../build/imagegen/edge-image-bootkit/image_deps.json ${UPLOAD_DIR}/ || echo 'image_deps.json not found' + cp ../../../build/imagegen/edge-image-bootkit/image_deps.json ${UPLOAD_DIR}/ || echo 'image_pkg_manifest.json not found' + + diff --git a/.github/workflows/check-entangled-specs.yml b/.github/workflows/check-entangled-specs.yml index 791f3ed0b3..c3005c6dc5 100644 --- a/.github/workflows/check-entangled-specs.yml +++ b/.github/workflows/check-entangled-specs.yml @@ -15,7 +15,7 @@ permissions: read-all jobs: check: name: Spec Entanglement Mismatch Check - runs-on: ubuntu-latest + runs-on: arc-runner-set steps: # Checkout the branch of our repo that triggered this action diff --git a/.github/workflows/check-files.yml b/.github/workflows/check-files.yml index bb6826a495..6e04738d43 100644 --- a/.github/workflows/check-files.yml +++ b/.github/workflows/check-files.yml @@ -13,7 +13,7 @@ jobs: build: name: Check Disallowed Files - runs-on: ubuntu-latest + runs-on: arc-runner-set steps: - name: Check out code @@ -137,4 +137,4 @@ jobs: exit 1 fi - echo "All files are appropriate for git storage." \ No newline at end of file + echo "All files are appropriate for git storage." diff --git a/.github/workflows/dv-test.yml b/.github/workflows/dv-test.yml new file mode 100644 index 0000000000..204791d2b3 --- /dev/null +++ b/.github/workflows/dv-test.yml @@ -0,0 +1,155 @@ +name: Test DV image build +on: + workflow_dispatch: + #pull_request: + # branches: [3.0] + +permissions: + id-token: write + contents: read + +env: + RELEASE_NAME: Test TiberOS DV + CODE_REPO_URL: open-edge-platform/edge-microvisor-toolkit + CODE_REPO_NAME: edge-microvisor-toolkit + CODE_REPO_BRANCH: "3.0" + +jobs: + image_build: + name: DV Image Build + runs-on: Microvisor + timeout-minutes: 1440 # extend timeout from default 6 hour to 24 hour + environment: production + outputs: + base_url: ${{ steps.compress-image.outputs.base_url }} + upload_dir: ${{ steps.compress-image.outputs.upload_dir }} + images: ${{ steps.compress-image.outputs.images }} + st_images: ${{ steps.compress-image.outputs.st_images }} + datetime_as_version: ${{ steps.set-version.outputs.datetime_as_version }} + release_name: ${{ steps.compress-image.outputs.release_name }} + artifactory_url: ${{ steps.compress-image.outputs.artifactory_url }} + artifactory_path: ${{ steps.compress-image.outputs.artifactory_path }} + kernel_version: ${{ steps.compress-image.outputs.kernel_version }} + + steps: + + - name: Checkout default repo + uses: actions/checkout@v4 + + - name: Checkout + uses: actions/checkout@v4 + with: + repository: ${{ env.CODE_REPO_URL }} + ref: ${{ env.CODE_REPO_BRANCH }} + path: ${{ env.CODE_REPO_NAME }} + + - name: Set DATETIME_AS_VERSION + id: set-version + run: | + DATETIME_AS_VERSION=$(date +'%Y%m%d.%H%M') + echo "datetime_as_version=$DATETIME_AS_VERSION" >> "$GITHUB_OUTPUT" + echo "DATETIME_AS_VERSION=$DATETIME_AS_VERSION" >> "$GITHUB_ENV" + + - name: Build packages + run: | + cd ${{ env.CODE_REPO_NAME }} + + pushd toolkit + sudo make -j16 toolchain \ + REBUILD_TOOLS=y \ + VALIDATE_TOOLCHAIN_GPG=n \ + DAILY_BUILD_REPO=./resources/manifests/package/development.repo + popd + + - name: Build default image + id: build-image + run: | + cd ${{ env.CODE_REPO_NAME }} + + pushd toolkit + sudo make image \ + REBUILD_PACKAGES=n \ + REBUILD_TOOLS=y \ + VALIDATE_TOOLCHAIN_GPG=n \ + DATETIME_AS_VERSION="${DATETIME_AS_VERSION}" \ + LICENSE_CHECK_MODE=warn \ + CONFIG_FILE=./imageconfigs/edge-image-desktop-virtualization.json \ + TIBER_DISTRIB_SPECS=y \ + REPO_LIST+=./resources/manifests/package/development.repo \ + DAILY_BUILD_REPO=./resources/manifests/package/development.repo + # check for output image to determine the results + shopt -s nullglob + raw_files=(../out/images/edge-image-desktop-virtualization/*.raw) + if [ ${#raw_files[@]} -eq 0 ]; then + echo "No .raw files were found in the output directory." + exit 1 + else + echo "Found .raw files in the output directory:" + printf '%s\n' "${raw_files[@]}" + fi + popd + + - name: Build dev image + id: build-dev-image + run: | + cd ${{ env.CODE_REPO_NAME }} + + pushd toolkit + sudo make image \ + REBUILD_PACKAGES=n \ + REBUILD_TOOLS=y \ + VALIDATE_TOOLCHAIN_GPG=n \ + DATETIME_AS_VERSION="${DATETIME_AS_VERSION}" \ + CONFIG_FILE=./imageconfigs/edge-image-desktop-virtualization-dev.json \ + DAILY_BUILD_REPO=./resources/manifests/package/development.repo \ + REPO_LIST+=./resources/manifests/package/development.repo + popd + + - name: compress image + id: compress-image + if: steps.build-image.outcome == 'success' + run: | + cd ${{ env.CODE_REPO_NAME }} + + UPLOAD_DIR=$DATETIME_AS_VERSION + echo "upload_dir=$UPLOAD_DIR" >> "$GITHUB_OUTPUT" + cp -r out/images/edge-image-desktop-virtualization/ "$UPLOAD_DIR" + cp -r out/images/edge-image-desktop-virtualization-dev/* "$UPLOAD_DIR" + + KERNEL_VERSION=$(jq '.[] | .[] | select(.Name=="kernel") | .Version' ./build/imagegen/edge-image-desktop-virtualization/image_pkg_manifest.json) + echo "kernel_version=$KERNEL_VERSION" >> "$GITHUB_OUTPUT" + echo "KERNEL_VERSION=$KERNEL_VERSION" + + # this is needed for the new code scanning wf + echo "base_url=${{ env.ARTIFACTORY_URL }}/${{ env.ARTIFACTORY_PATH }}" >> "$GITHUB_OUTPUT" + + find "$UPLOAD_DIR" -type f -exec gzip {} \; + for dir in build/imagegen/edge-image-desktop-virtualization build/imagegen/edge-image-desktop-virtualization-dev; + do + json_files=("$dir"/*.json) + if [ -e "${json_files[0]}" ]; then + for file in "${json_files[@]}"; + do + filename=$(basename "$file") + # Extract the directory name to use as a prefix + dirname=$(basename "$dir") + cp "$file" "$UPLOAD_DIR/$dirname-$filename" + echo "$file $UPLOAD_DIR/$dirname-$filename" + done + else + echo "No .json files found in $dir" + fi + done + + find "$UPLOAD_DIR" -type f -exec sh -c 'sha256sum "{}" > "{}".sha256sum' \; + # echo "build commit: $GITHUB_SHA" >> "$UPLOAD_DIR/build_ID" + commit_id=$(git rev-parse HEAD) + echo "build commit: $commit_id" >> "${UPLOAD_DIR}/build_ID" + echo "Github build ID: $GITHUB_RUN_ID" >> "$UPLOAD_DIR/build_ID" + IMAGES=$(find "$UPLOAD_DIR" -name *.raw.* | xargs -n 1 basename | tr '\n' ',') + STABLE_IMAGES=$(echo "$IMAGES" | tr ',' '\n' | grep -v 'edge-readonly-dv-dev-' | tr '\n' ',') + STABLE_IMAGES="${STABLE_IMAGES%,}" + echo "images=$STABLE_IMAGES" >> "$GITHUB_OUTPUT" + echo "release_name=${{ env.RELEASE_NAME }}" >> "$GITHUB_OUTPUT" + echo "artifactory_url=test-art-url" >> "$GITHUB_OUTPUT" + echo "artifactory_path=test-art-path" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/full-test.yml b/.github/workflows/full-test.yml new file mode 100644 index 0000000000..d1329d6868 --- /dev/null +++ b/.github/workflows/full-test.yml @@ -0,0 +1,116 @@ +name: Test Full image build +on: + workflow_dispatch: + pull_request: + branches: [3.0] + +permissions: + id-token: write + contents: read + +env: + RELEASE_NAME: Test TiberOS Full + CODE_REPO_URL: open-edge-platform/edge-microvisor-toolkit + CODE_REPO_NAME: edge-microvisor-toolkit + CODE_REPO_BRANCH: "3.0" + +jobs: + image_build: + name: full Image Build + runs-on: Microvisor + timeout-minutes: 1440 # extend timeout from default 6 hour to 24 hour + environment: production + outputs: + base_url: ${{ steps.compress-image.outputs.base_url }} + upload_dir: ${{ steps.compress-image.outputs.upload_dir }} + datetime_as_version: ${{ steps.set-version.outputs.datetime_as_version }} + + steps: + + - name: Checkout default repo + uses: actions/checkout@v4 + + - name: Checkout + uses: actions/checkout@v4 + with: + repository: ${{ env.CODE_REPO_URL }} + ref: ${{ env.CODE_REPO_BRANCH }} + path: ${{ env.CODE_REPO_NAME }} + + - name: Set DATETIME_AS_VERSION + id: set-version + run: | + DATETIME_AS_VERSION=$(date +'%Y%m%d.%H%M') + echo "datetime_as_version=$DATETIME_AS_VERSION" >> "$GITHUB_OUTPUT" + echo "DATETIME_AS_VERSION=$DATETIME_AS_VERSION" >> "$GITHUB_ENV" + + - name: Build packages + run: | + cd ${{ env.CODE_REPO_NAME }} + + pushd toolkit + sudo make toolchain \ + REBUILD_TOOLS=y \ + VALIDATE_TOOLCHAIN_GPG=n \ + DAILY_BUILD_REPO=./resources/manifests/package/development.repo + popd + + - name: Build Container Packages + run: | + cd ${{ env.CODE_REPO_NAME }} + + pushd toolkit + sudo make image \ + REBUILD_TOOLS=y \ + VALIDATE_TOOLCHAIN_GPG=n \ + DATETIME_AS_VERSION="${DATETIME_AS_VERSION}" \ + CONFIG_FILE=./imageconfigs/core-container.json \ + DAILY_BUILD_REPO=./resources/manifests/package/development.repo + popd + + - name: Build default image + id: build-image + run: | + cd ${{ env.CODE_REPO_NAME }} + + pushd toolkit + sudo make image -j8 \ + REBUILD_TOOLS=y \ + VALIDATE_TOOLCHAIN_GPG=n \ + CONCURRENT_PACKAGE_BUILDS=8 \ + DATETIME_AS_VERSION="${DATETIME_AS_VERSION}" \ + CONFIG_FILE=./imageconfigs/edge-image.json \ + DAILY_BUILD_REPO=./resources/manifests/package/development.repo || true + # check for output image to determine the results + shopt -s nullglob + raw_files=(../out/images/edge-image/*.raw) + if [ ${#raw_files[@]} -eq 0 ]; then + echo "No .raw files were found in the output directory." + exit 1 + else + echo "Found .raw files in the output directory:" + printf '%s\n' "${raw_files[@]}" + fi + popd + + - name: compress image + id: compress-image + if: steps.build-image.outcome == 'success' + run: | + cd ${{ env.CODE_REPO_NAME }} + + UPLOAD_DIR=$DATETIME_AS_VERSION + echo "upload_dir=$UPLOAD_DIR" >> "$GITHUB_OUTPUT" + cp -r out/images/edge-image/ "$UPLOAD_DIR" + find "$UPLOAD_DIR" -type f -exec gzip {} \; + cp -r out/images/core-container/ "$UPLOAD_DIR" + cp -r build/imagegen/edge-image/*.json "$UPLOAD_DIR" + cp -r build/spec_parser.json "$UPLOAD_DIR" + + echo "Generating SHA256 checksums for files in version directory" + find ${UPLOAD_DIR} -type f -exec sh -c 'sha256sum "{}" > "{}".sha256sum' \\; + commit_id=$(git rev-parse HEAD) + echo "build commit: $commit_id" >> "${UPLOAD_DIR}/build_ID" + echo "Github build ID: $GITHUB_RUN_ID" >> "$UPLOAD_DIR/build_ID" + IMAGES=$(find "$UPLOAD_DIR" -name *.raw.* | xargs -n 1 basename | tr '\n' ',') + echo "Images: $IMAGES" diff --git a/.github/workflows/iso-test.yml b/.github/workflows/iso-test.yml new file mode 100644 index 0000000000..fb82b9ae5d --- /dev/null +++ b/.github/workflows/iso-test.yml @@ -0,0 +1,129 @@ +name: Test ISO image build +on: + workflow_dispatch: + # pull_request: + # branches: [3.0] + +permissions: + id-token: write + contents: read + +env: + RELEASE_NAME: Test TiberOS ISO IMAGE + CODE_REPO_URL: open-edge-platform/edge-microvisor-toolkit + CODE_REPO_NAME: edge-microvisor-toolkit + CODE_REPO_BRANCH: "3.0" + CHROOT_DIR: $(GITHUB_WORKSPACE)/build/worker/chroot + +jobs: + iso_image_build: + name: ISO Image Build + runs-on: Microvisor + timeout-minutes: 1440 # extend timeout from default 6 hour to 24 hour + environment: production + outputs: + base_url: ${{ steps.compress-image.outputs.base_url }} + upload_dir: ${{ steps.compress-image.outputs.upload_dir }} + images: ${{ steps.compress-image.outputs.images }} + datetime_as_version: ${{ steps.set-version.outputs.datetime_as_version }} + release_name: ${{ steps.compress-image.outputs.release_name }} + artifactory_url: ${{ steps.compress-image.outputs.artifactory_url }} + artifactory_path: ${{ steps.compress-image.outputs.artifactory_path }} + + steps: + + - name: Checkout default repo + uses: actions/checkout@v4 + + - name: Checkout + uses: actions/checkout@v4 + with: + repository: ${{ env.CODE_REPO_URL }} + ref: ${{ env.CODE_REPO_BRANCH }} + path: ${{ env.CODE_REPO_NAME }} + + - name: Set DATETIME_AS_VERSION + id: set-version + run: | + DATETIME_AS_VERSION=$(date +'%Y%m%d.%H%M') + echo "datetime_as_version=$DATETIME_AS_VERSION" >> "$GITHUB_OUTPUT" + echo "DATETIME_AS_VERSION=$DATETIME_AS_VERSION" >> "$GITHUB_ENV" + + - name: Parse spec files + continue-on-error: true + run: | + cd ${{ env.CODE_REPO_NAME }} + + python3 -m pip install -r toolkit/scripts/requirements.txt + python3 toolkit/scripts/spec_parser.py + + - name: Build ISO image + id: build-iso-image + run: | + cd ${{ env.CODE_REPO_NAME }} + + pushd toolkit + sudo make iso \ + CONFIG_FILE=./imageconfigs/full.json \ + REBUILD_TOOLS=y \ + REBUILD_PACKAGES=n \ + EMT_DISTRIB_SPECS=y \ + VALIDATE_TOOLCHAIN_GPG=n \ + DATETIME_AS_VERSION="$DATETIME_AS_VERSION" \ + DAILY_BUILD_REPO=./resources/manifests/package/development.repo \ + REPO_LIST+=./resources/manifests/package/development.repo + + # check for output image to determine the results + shopt -s nullglob + iso_files=(../out/images/full/*.iso) + if [ ${#iso_files[@]} -eq 0 ]; then + echo "No .iso files were found in the output directory" + exit 1 + else + echo "Found .iso files in the output directory:" + printf '%s\n' "${iso_files[@]}" + fi + popd + + - name: compress image + id: compress-image + if: steps.build-iso-image.outcome == 'success' + run: | + cd ${{ env.CODE_REPO_NAME }} + + UPLOAD_DIR=$DATETIME_AS_VERSION + echo "upload_dir=$UPLOAD_DIR" >> "$GITHUB_OUTPUT" + cp -r out/images/full/ "$UPLOAD_DIR" + + # this is needed for the new code scanning wf + echo "base_url=${{ env.ARTIFACTORY_URL }}/${{ env.ARTIFACTORY_PATH }}" >> "$GITHUB_OUTPUT" + + #find "$UPLOAD_DIR" -type f -exec gzip {} \; + for dir in build/imagegen/full build/imagegen/iso_initrd; + do + json_files=("$dir"/*.json) + if [ -e "${json_files[0]}" ]; then + for file in "${json_files[@]}"; + do + filename=$(basename "$file") + # Extract the directory name to use as a prefix + dirname=$(basename "$dir") + cp "$file" "$UPLOAD_DIR/$dirname-$filename" + echo "$file $UPLOAD_DIR/$dirname-$filename" + done + else + echo "No .json files found in $dir" + fi + done + cp build/spec_parser.json "$UPLOAD_DIR/" | true + + find "$UPLOAD_DIR" -type f -exec sh -c 'sha256sum "{}" > "{}".sha256sum' \; + # echo "build commit: $GITHUB_SHA" >> "$UPLOAD_DIR/build_ID" + echo "Github build ID: $GITHUB_RUN_ID" >> "$UPLOAD_DIR/build_ID" + commit_id=$(git rev-parse HEAD) + echo "build commit: $commit_id" >> "${UPLOAD_DIR}/build_ID" + ISO_IMAGES=$(find "$UPLOAD_DIR" -name *.iso* | xargs -n 1 basename | tr '\n' ',') + echo "images=$ISO_IMAGES" >> "$GITHUB_OUTPUT" + echo "release_name=${{ env.RELEASE_NAME }}" >> "$GITHUB_OUTPUT" + echo "artifactory_url=test-art-url" >> "$GITHUB_OUTPUT" + echo "artifactory_path=test-art-path" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 860d605577..43545f145e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,20 +11,13 @@ name: Linter on: workflow_dispatch: - pull_request: - branches: [3.0, 3.0-dev] - paths-ignore: - - "**/CODEOWNERS" - - "**.spec" - - "**.patch" permissions: read-all jobs: lint: name: Lint Workflows and Code - runs-on: ubuntu-latest - + runs-on: arc-runner-set permissions: contents: read packages: read diff --git a/.github/workflows/minOS.yml b/.github/workflows/minOS.yml new file mode 100644 index 0000000000..faf1360e61 --- /dev/null +++ b/.github/workflows/minOS.yml @@ -0,0 +1,133 @@ +name: Test MinimalOS image build +on: + workflow_dispatch: + # pull_request: + # branches: [3.0] + +permissions: + id-token: write + contents: read + +env: + RELEASE_NAME: Test MinimalOS + CODE_REPO_URL: open-edge-platform/edge-microvisor-toolkit + CODE_REPO_NAME: edge-microvisor-toolkit + CODE_REPO_BRANCH: "3.0" + +jobs: + image_build: + name: MinimalOS Image Build + runs-on: Microvisor + timeout-minutes: 1440 # extend timeout from default 6 hour to 24 hour + environment: production + outputs: + base_url: ${{ steps.compress-image.outputs.base_url }} + upload_dir: ${{ steps.compress-image.outputs.upload_dir }} + images: ${{ steps.compress-image.outputs.images }} + datetime_as_version: ${{ steps.set-version.outputs.datetime_as_version }} + release_name: ${{ steps.compress-image.outputs.release_name }} + artifactory_url: ${{ steps.compress-image.outputs.artifactory_url }} + artifactory_path: ${{ steps.compress-image.outputs.artifactory_path }} + kernel_version: ${{ steps.compress-image.outputs.kernel_version }} + + steps: + + - name: Checkout default repo + uses: actions/checkout@v4 + + - name: Checkout + uses: actions/checkout@v4 + with: + repository: ${{ env.CODE_REPO_URL }} + ref: ${{ env.CODE_REPO_BRANCH }} + path: ${{ env.CODE_REPO_NAME }} + + - name: Set DATETIME_AS_VERSION + id: set-version + run: | + DATETIME_AS_VERSION=$(date +'%Y%m%d.%H%M') + echo "datetime_as_version=$DATETIME_AS_VERSION" >> "$GITHUB_OUTPUT" + echo "DATETIME_AS_VERSION=$DATETIME_AS_VERSION" >> "$GITHUB_ENV" + + - name: Build packages + run: | + cd ${{ env.CODE_REPO_NAME }} + + pushd toolkit + sudo make -j16 toolchain \ + REBUILD_TOOLS=y \ + VALIDATE_TOOLCHAIN_GPG=n \ + DAILY_BUILD_REPO=./resources/manifests/package/development.repo + popd + + - name: Build default image + id: build-image + run: | + cd ${{ env.CODE_REPO_NAME }} + + pushd toolkit + sudo make image \ + REBUILD_PACKAGES=n \ + REBUILD_TOOLS=y \ + DATETIME_AS_VERSION="${DATETIME_AS_VERSION}" \ + LICENSE_CHECK_MODE=warn \ + CONFIG_FILE=./imageconfigs/edge-image-minimal-os.json \ + EMT_DISTRIB_SPECS=y \ + VALIDATE_TOOLCHAIN_GPG=n || true + # check for output image to determine the results + shopt -s nullglob + raw_files=(../out/images/edge-image-minimal-os/*.raw) + if [ ${#raw_files[@]} -eq 0 ]; then + echo "No .raw files were found in the output directory. " + exit 1 + else + echo "Found .raw files in the output directory:" + printf '%s\n' "${raw_files[@]}" + fi + popd + + + - name: compress image + id: compress-image + if: steps.build-image.outcome == 'success' + run: | + cd ${{ env.CODE_REPO_NAME }} + + UPLOAD_DIR=$DATETIME_AS_VERSION + echo "upload_dir=$UPLOAD_DIR" >> "$GITHUB_OUTPUT" + cp -r out/images/edge-image-minimal-os/ "$UPLOAD_DIR" + + KERNEL_VERSION=$(jq '.[] | .[] | select(.Name=="kernel") | .Version' ./build/imagegen/edge-image-minimal-os/image_pkg_manifest.json) + echo "kernel_version=$KERNEL_VERSION" >> "$GITHUB_OUTPUT" + echo "KERNEL_VERSION=$KERNEL_VERSION" + + # this is needed for the new code scanning wf + echo "base_url=${{ env.ARTIFACTORY_URL }}/${{ env.ARTIFACTORY_PATH }}" >> "$GITHUB_OUTPUT" + + find "$UPLOAD_DIR" -type f -exec gzip {} \; + for dir in build/imagegen/edge-image-minimal-os; + do + json_files=("$dir"/*.json) + if [ -e "${json_files[0]}" ]; then + for file in "${json_files[@]}"; + do + filename=$(basename "$file") + # Extract the directory name to use as a prefix + dirname=$(basename "$dir") + cp "$file" "$UPLOAD_DIR/$dirname-$filename" + echo "$file $UPLOAD_DIR/$dirname-$filename" + done + else + echo "No .json files found in $dir" + fi + done + + find "$UPLOAD_DIR" -type f -exec sh -c 'sha256sum "{}" > "{}".sha256sum' \; + # echo "build commit: $GITHUB_SHA" >> "$UPLOAD_DIR/build_ID" + commit_id=$(git rev-parse HEAD) + echo "build commit: $commit_id" >> "${UPLOAD_DIR}/build_ID" + echo "Github build ID: $GITHUB_RUN_ID" >> "$UPLOAD_DIR/build_ID" + IMAGES=$(find "$UPLOAD_DIR" -name *.raw.* | xargs -n 1 basename | tr '\n' ',') + echo "release_name=${{ env.RELEASE_NAME }}" >> "$GITHUB_OUTPUT" + echo "artifactory_url=test-art-url" >> "$GITHUB_OUTPUT" + echo "artifactory_path=test-art-path" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/non-rt-test.yml b/.github/workflows/non-rt-test.yml new file mode 100644 index 0000000000..7857b394bf --- /dev/null +++ b/.github/workflows/non-rt-test.yml @@ -0,0 +1,189 @@ +name: Test Non RT image build +on: + workflow_dispatch: + # pull_request: + # branches: [3.0] + +permissions: + id-token: write + contents: read + +env: + RELEASE_NAME: Test TiberOS Non-RT + CODE_REPO_URL: open-edge-platform/edge-microvisor-toolkit + CODE_REPO_NAME: edge-microvisor-toolkit + CODE_REPO_BRANCH: "3.0" + +jobs: + image_build: + name: Image Build + runs-on: Microvisor + timeout-minutes: 1440 # extend timeout from default 6 hour to 24 hour + environment: production + outputs: + base_url: ${{ steps.compress-image.outputs.base_url }} + upload_dir: ${{ steps.compress-image.outputs.upload_dir }} + images: ${{ steps.compress-image.outputs.images }} + st_images: ${{ steps.compress-image.outputs.st_images }} + datetime_as_version: ${{ steps.set-version.outputs.datetime_as_version }} + release_name: ${{ steps.compress-image.outputs.release_name }} + artifactory_url: ${{ steps.compress-image.outputs.artifactory_url }} + artifactory_path: ${{ steps.compress-image.outputs.artifactory_path }} + kernel_version: ${{ steps.compress-image.outputs.kernel_version }} + + steps: + + - name: Checkout default repo + uses: actions/checkout@v4 + + - name: Checkout + uses: actions/checkout@v4 + with: + repository: ${{ env.CODE_REPO_URL }} + ref: ${{ env.CODE_REPO_BRANCH }} + path: ${{ env.CODE_REPO_NAME }} + + - name: Set DATETIME_AS_VERSION + id: set-version + run: | + DATETIME_AS_VERSION=$(date +'%Y%m%d.%H%M') + echo "datetime_as_version=$DATETIME_AS_VERSION" >> "$GITHUB_OUTPUT" + echo "DATETIME_AS_VERSION=$DATETIME_AS_VERSION" >> "$GITHUB_ENV" + + - name: Parse spec files + continue-on-error: true + run: | + ## + cd ${{ env.CODE_REPO_NAME }} + + python3 -m pip install -r toolkit/scripts/requirements.txt + python3 toolkit/scripts/spec_parser.py + + - name: Build packages + run: | + cd ${{ env.CODE_REPO_NAME }} + + pushd toolkit + sudo make -j16 toolchain \ + REBUILD_TOOLS=y \ + VALIDATE_TOOLCHAIN_GPG=n \ + DAILY_BUILD_REPO=./resources/manifests/package/development.repo + popd + + - name: Build container image + run: | + cd ${{ env.CODE_REPO_NAME }} + + pushd toolkit + sudo make image \ + REBUILD_PACKAGES=n \ + REBUILD_TOOLS=y \ + EMT_DISTRIB_SPECS=y \ + VALIDATE_TOOLCHAIN_GPG=n \ + DATETIME_AS_VERSION="${DATETIME_AS_VERSION}" \ + CONFIG_FILE=./imageconfigs/core-container.json \ + DAILY_BUILD_REPO=./resources/manifests/package/development.repo \ + REPO_LIST+=./resources/manifests/package/development.repo + popd + + - name: Build default image + id: build-image + run: | + cd ${{ env.CODE_REPO_NAME }} + + pushd toolkit + sudo make image \ + REBUILD_PACKAGES=n \ + REBUILD_TOOLS=y \ + VALIDATE_TOOLCHAIN_GPG=n \ + DATETIME_AS_VERSION="${DATETIME_AS_VERSION}" \ + LICENSE_CHECK_MODE=warn \ + CONFIG_FILE=./imageconfigs/edge-image.json \ + EMT_DISTRIB_SPECS=y \ + REPO_LIST+=./resources/manifests/package/development.repo \ + DAILY_BUILD_REPO=./resources/manifests/package/development.repo + + # check for output image to determine the results + shopt -s nullglob + raw_files=(../out/images/edge-image/*.raw) + if [ ${#raw_files[@]} -eq 0 ]; then + echo "No .raw files were found in the output directory." + exit 1 + else + echo "Found .raw files in the output directory:" + printf '%s\n' "${raw_files[@]}" + fi + popd + + - name: Build dev image + id: build-dev-image + run: | + cd ${{ env.CODE_REPO_NAME }} + + pushd toolkit + sudo make image \ + REBUILD_PACKAGES=n \ + REBUILD_TOOLS=y \ + EMT_DISTRIB_SPECS=y \ + VALIDATE_TOOLCHAIN_GPG=n \ + DATETIME_AS_VERSION="${DATETIME_AS_VERSION}" \ + CONFIG_FILE=./imageconfigs/edge-image-dev.json \ + DAILY_BUILD_REPO=./resources/manifests/package/development.repo \ + REPO_LIST+=./resources/manifests/package/development.repo + popd + + - name: compress image + id: compress-image + if: steps.build-image.outcome == 'success' + run: | + cd ${{ env.CODE_REPO_NAME }} + + UPLOAD_DIR=$DATETIME_AS_VERSION + echo "upload_dir=$UPLOAD_DIR" >> "$GITHUB_OUTPUT" + cp -r out/images/edge-image/ "$UPLOAD_DIR" + cp -r out/images/edge-image-dev/* "$UPLOAD_DIR" + + KERNEL_VERSION=$(jq '.[] | .[] | select(.Name=="kernel") | .Version' ./build/imagegen/edge-image/image_pkg_manifest.json) + echo "kernel_version=$KERNEL_VERSION" >> "$GITHUB_OUTPUT" + echo "KERNEL_VERSION=$KERNEL_VERSION" + + echo "base_url=${{ env.ARTIFACTORY_URL }}/${{ env.ARTIFACTORY_PATH }}" >> "$GITHUB_OUTPUT" + + # setup for image analysis + # sudo cp "out/images/edge-image/tiber-readonly-3.0.$DATETIME_AS_VERSION.raw" /data/tiber-images/tiber-analyze-${{ github.run_id }}.raw + # echo "tiber-readonly-3.0.$DATETIME_AS_VERSION.raw" | sudo tee /data/edge-images/tiber-rename-${{ github.run_id }} + + find "$UPLOAD_DIR" -type f -exec gzip {} \; + for dir in build/imagegen/edge-image build/imagegen/edge-image-dev; + do + json_files=("$dir"/*.json) + if [ -e "${json_files[0]}" ]; then + for file in "${json_files[@]}"; + do + filename=$(basename "$file") + # Extract the directory name to use as a prefix + dirname=$(basename "$dir") + cp "$file" "$UPLOAD_DIR/$dirname-$filename" + echo "$file $UPLOAD_DIR/$dirname-$filename" + done + else + echo "No .json files found in $dir" + fi + done + cp out/images/core-container/* "$UPLOAD_DIR/" + cp build/spec_parser.json "$UPLOAD_DIR/" | true + find "$UPLOAD_DIR" -type f -exec sh -c 'sha256sum "{}" > "{}".sha256sum' \; + # echo "build commit: $GITHUB_SHA" >> "$UPLOAD_DIR/build_ID" + commit_id=$(git rev-parse HEAD) + echo "build commit: $commit_id" >> "${UPLOAD_DIR}/build_ID" + echo "Github build ID: $GITHUB_RUN_ID" >> "$UPLOAD_DIR/build_ID" + IMAGES=$(find "$UPLOAD_DIR" -name *.raw.* | xargs -n 1 basename | tr '\n' ',') + STABLE_IMAGES=$(echo "$IMAGES" | tr ',' '\n' | grep -v 'edge-readonly-dev-' | tr '\n' ',') + STABLE_IMAGES="${STABLE_IMAGES%,}" + ST_IMAGES=$(echo "$IMAGES" | tr ',' '\n' | grep 'edge-readonly-dev-*' | tr '\n' ',') + ST_IMAGES="${ST_IMAGES%,}" + echo "images=$STABLE_IMAGES" >> "$GITHUB_OUTPUT" + echo "st_images=$ST_IMAGES" >> "$GITHUB_OUTPUT" + echo "release_name=${{ env.RELEASE_NAME }}" >> "$GITHUB_OUTPUT" + echo "artifactory_url=test-art-url" >> "$GITHUB_OUTPUT" + echo "artifactory_path=test-art-path" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/rt-test.yml b/.github/workflows/rt-test.yml new file mode 100644 index 0000000000..d1268ec3e5 --- /dev/null +++ b/.github/workflows/rt-test.yml @@ -0,0 +1,167 @@ +name: Test RT image Build +on: + workflow_dispatch: + # pull_request: + # branches: [3.0] + +permissions: + id-token: write + contents: read + +env: + RELEASE_NAME: Test TiberOS RT + CODE_REPO_URL: open-edge-platform/edge-microvisor-toolkit + CODE_REPO_NAME: edge-microvisor-toolkit + CODE_REPO_BRANCH: "3.0" + CHROOT_DIR: $(GITHUB_WORKSPACE)/build/worker/chroot + +jobs: + image_build: + name: Image Build + runs-on: Microvisor + timeout-minutes: 1440 # extend timeout from default 6 hour to 24 hour + environment: production + outputs: + base_url: ${{ steps.compress-image.outputs.base_url }} + upload_dir: ${{ steps.compress-image.outputs.upload_dir }} + images: ${{ steps.compress-image.outputs.images }} + st_images: ${{ steps.compress-image.outputs.st_images }} + raw_image_sizes: ${{ steps.build-rt-image.outputs.raw_image_sizes }} + datetime_as_version: ${{ steps.set-version.outputs.datetime_as_version }} + release_name: ${{ steps.compress-image.outputs.release_name }} + artifactory_url: ${{ steps.compress-image.outputs.artifactory_url }} + artifactory_path: ${{ steps.compress-image.outputs.artifactory_path }} + kernel_version: ${{ steps.compress-image.outputs.kernel_version }} + + steps: + - name: print workspace + run: echo "workspace is ${GITHUB_WORKSPACE} " + + - name: Checkout default repo + uses: actions/checkout@v4 + + - name: Checkout + uses: actions/checkout@v4 + with: + repository: ${{ env.CODE_REPO_URL }} + ref: ${{ env.CODE_REPO_BRANCH }} + path: ${{ env.CODE_REPO_NAME }} + + - name: Set DATETIME_AS_VERSION + id: set-version + run: | + DATETIME_AS_VERSION=$(date +'%Y%m%d.%H%M') + echo "datetime_as_version=$DATETIME_AS_VERSION" >> "$GITHUB_OUTPUT" + echo "DATETIME_AS_VERSION=$DATETIME_AS_VERSION" >> "$GITHUB_ENV" + + - name: Parse spec files + continue-on-error: true + run: | + cd ${{ env.CODE_REPO_NAME }} + + python3 -m pip install -r toolkit/scripts/requirements.txt + python3 toolkit/scripts/spec_parser.py + + - name: Build packages + run: | + cd ${{ env.CODE_REPO_NAME }} + pushd toolkit + sudo make -j16 toolchain \ + REBUILD_TOOLS=y \ + VALIDATE_TOOLCHAIN_GPG=n \ + DAILY_BUILD_REPO=./resources/manifests/package/development.repo \ + LOG_LEVEL=debug + popd + + - name: Build rt image (quick build) + id: build-rt-image + run: | + cd ${{ env.CODE_REPO_NAME }}/toolkit + sudo make image \ + REBUILD_PACKAGES=n \ + REBUILD_TOOLS=y \ + VALIDATE_TOOLCHAIN_GPG=n \ + DATETIME_AS_VERSION="${DATETIME_AS_VERSION}" \ + LICENSE_CHECK_MODE=warn \ + CONFIG_FILE=./imageconfigs/edge-image-rt.json \ + EMT_DISTRIB_SPECS=y \ + REPO_LIST+=./resources/manifests/package/development.repo \ + DAILY_BUILD_REPO=./resources/manifests/package/development.repo + + # check for output image to determine the results + shopt -s nullglob + raw_files=(../out/images/edge-image-rt/*.raw) + raw_image_sizes=$(du -sh ../out/images/edge-image-rt/*.raw | tr '\n' ' ') + echo "raw_image_sizes=$raw_image_sizes" >> "$GITHUB_OUTPUT" + if [ ${#raw_files[@]} -eq 0 ]; then + echo "No .raw files were found in the output directory." + exit 1 + else + echo "Found .raw files in the output directory:" + printf '%s\n' "${raw_files[@]}" + fi + + - name: Build rt dev image + id: build-rt-dev-image + run: | + cd ${{ env.CODE_REPO_NAME }}/toolkit + sudo make image \ + REBUILD_PACKAGES=n \ + REBUILD_TOOLS=y \ + EMT_DISTRIB_SPECS=y \ + VALIDATE_TOOLCHAIN_GPG=n \ + DATETIME_AS_VERSION="${DATETIME_AS_VERSION}" \ + CONFIG_FILE=./imageconfigs/edge-image-rt-dev.json \ + DAILY_BUILD_REPO=./resources/manifests/package/development.repo \ + REPO_LIST+=./resources/manifests/package/development.repo + + + - name: compress image + id: compress-image + if: steps.build-rt-image.outcome == 'success' + run: | + cd ${{ env.CODE_REPO_NAME }} + + UPLOAD_DIR=$DATETIME_AS_VERSION + echo "upload_dir=$UPLOAD_DIR" >> "$GITHUB_OUTPUT" + cp -r out/images/edge-image-rt/ "$UPLOAD_DIR" + cp -r out/images/edge-image-rt-dev/* "$UPLOAD_DIR" + + KERNEL_VERSION=$(jq '.[] | .[] | select(.Name=="kernel-rt") | .Version' ./build/imagegen/edge-image-rt/image_pkg_manifest.json) + echo "kernel_version=$KERNEL_VERSION" >> "$GITHUB_OUTPUT" + echo "KERNEL_VERSION=$KERNEL_VERSION" + + + find "$UPLOAD_DIR" -type f -exec gzip {} \; + for dir in build/imagegen/edge-image-rt build/imagegen/edge-image-rt-dev; + do + json_files=("$dir"/*.json) + if [ -e "${json_files[0]}" ]; then + for file in "${json_files[@]}"; + do + filename=$(basename "$file") + # Extract the directory name to use as a prefix + dirname=$(basename "$dir") + cp "$file" "$UPLOAD_DIR/$dirname-$filename" + echo "$file $UPLOAD_DIR/$dirname-$filename" + done + else + echo "No .json files found in $dir" + fi + done + cp build/spec_parser.json "$UPLOAD_DIR/" | true + find "$UPLOAD_DIR" -type f -exec sh -c 'sha256sum "{}" > "{}".sha256sum' \; + # echo "build commit: $GITHUB_SHA" >> "$UPLOAD_DIR/build_ID" + commit_id=$(git rev-parse HEAD) + echo "build commit: $commit_id" >> "${UPLOAD_DIR}/build_ID" + echo "Github build ID: $GITHUB_RUN_ID" >> "$UPLOAD_DIR/build_ID" + IMAGES=$(find "$UPLOAD_DIR" -name *.raw.* | xargs -n 1 basename | tr '\n' ',') + STABLE_IMAGES=$(echo "$IMAGES" | tr ',' '\n' | grep -v 'edge-readonly-rt-dev-' | tr '\n' ',') + STABLE_IMAGES="${STABLE_IMAGES%,}" + ST_IMAGES=$(echo "$IMAGES" | tr ',' '\n' | grep 'edge-readonly-rt-dev-*' | tr '\n' ',') + ST_IMAGES="${ST_IMAGES%,}" + echo "images=$STABLE_IMAGES" >> "$GITHUB_OUTPUT" + echo "st_images=$ST_IMAGES" >> "$GITHUB_OUTPUT" + echo "release_name=${{ env.RELEASE_NAME }}" >> "$GITHUB_OUTPUT" + echo "artifactory_url=test-art-url" >> "$GITHUB_OUTPUT" + echo "artifactory_path=test-art-path" >> "$GITHUB_OUTPUT"