Update main

# Conflicts:
#	tests/unittests/f1ap/common/test_helpers.cpp

Author: codebot

.gitlab-ci.yml

@@ -36,6 +36,9 @@ include:
   - project: softwareradiosystems/ci/tools
     ref: "8"
     file: .gitlab/ci-shared/tools/test_reporter.yml
+  - project: softwareradiosystems/ci/tools
+    ref: "8"
+    file: .gitlab/ci-shared/tools/tagger.yml
   - local: .gitlab/ci/build.yml
   - local: .gitlab/ci/trx.yml
   - local: .gitlab/ci/e2e.yml
@@ -49,6 +52,7 @@ stages:
   - documentation
   - private
   - public
+  - release
 
 variables:
   SLACK_CHANNEL_OK: "#ci_gnb"
@@ -195,15 +199,6 @@ coverity-dev:
     - export DESCRIPTION="srsRAN Project dev build"
     - export COV_TOKEN="${COVERITY_TOKEN_DEV}"
 
-coverity-agpl:
-  extends: .coverity_base
-  rules:
-    - if: $CI_DESCRIPTION =~ /AGPL-COVERITY/
-  before_script:
-    - export PROJECT_NAME="srsRAN_5G_agpl"
-    - export DESCRIPTION="srsRAN Project AGPL build"
-    - export COV_TOKEN="${COVERITY_TOKEN_AGPL}"
-
 e2e tests tox:
   stage: static
   rules:
@@ -237,10 +232,12 @@ unit coverage:
       variables:
         coverage_report: summary
       when: always # Even if previous stages/required jobs fail
+      allow_failure: true
     - if: $CI_DESCRIPTION =~ /Nightly/
       variables:
         coverage_report: full
       when: always # Even if previous stages/required jobs fail
+      allow_failure: true
   before_script:
     - PACKAGE_URL=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/coverage/${CI_COMMIT_BRANCH}${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}/coverage_history.tar.gz
     # Download coverage history from the registry
@@ -280,6 +277,11 @@ unit coverage:
       if [[ $coverage_report = "full" ]]; then
         publish_to_registry
       fi
+    - export PIP_EXTRA_INDEX_URL=""
+    - apk update; apk add python3 py3-pip
+    - pip3 install --verbose retina-reporter --index-url https://__token__:[email protected]/api/v4/projects/44296988/packages/pypi/simple
+    - retina-reporter --input build_time_metrics.json --bucket ci --db-config token=$INFLUXDB_TOKEN org=$INFLUXDB_ORG url=$INFLUXDB_URL
+
   coverage: /^\s*Line coverage:\s*\d+.\d+\%/
   artifacts:
     paths:
@@ -376,6 +378,32 @@ promote code:
     - git push
   dependencies: []
 
+update agpl main dryrun:
+  extends: .update agpl main
+  stage: .post
+  rules:
+    - if: $CI_DESCRIPTION =~ /Nightly/
+      when: always
+      allow_failure: true
+      variables:
+        MODE: "dryrun"
+  needs: []
+
+tag successful schedule:
+  extends: .create-tag
+  stage: .post
+  rules:
+    - if: $CI_DESCRIPTION =~ /Nightly/
+      when: manual
+    - if: $CI_DESCRIPTION =~ /Weekly/
+      when: manual
+  variables:
+    GIT_STRATEGY: none
+  interruptible: false
+  script:
+    - create_tag $(date +"%Y.%m.%d")
+  dependencies: []
+
 ################################################################################
 # Enable / Disable pipelines
 ################################################################################

.gitlab/ci/build.yml

@@ -122,6 +122,8 @@ variables:
     - !reference [.fetch_src_cache, before_script]
     - |
       build_srsgnb() {
+        start_time=$(date +%s)
+
         if [ -n "${UHD_VERSION}" ]; then
           BUILD_ARGS="${BUILD_ARGS} -u ${UHD_VERSION}"
         fi
@@ -186,6 +188,13 @@ variables:
           /usr/local/builder_tools/bin/python /usr/local/bin/changed_tests.py -b . -o "../$OUTPUT_FINGERPRINT"
           cd ..
         fi
+
+        end_time=$(date +%s)
+        execution_time=$((end_time - start_time))
+        echo "Build time is: $execution_time seconds"
+        export metric_prefix=""
+        if [[ $ON_MR == "true" ]]; then metric_prefix="_cached"; fi
+        echo '[{"name":"build","description":"Compilation","tests":[{"name":"'$OS'","description":"'$OS' build","metrics":[{"name":"build_time'$metric_prefix'","description":"Build time","value":['$execution_time'],"unit":"seconds","type":"time"}],"duts":[]}]}]' >> build_time_metrics.json
       }
     - |
       launch_tests() {
@@ -269,6 +278,7 @@ variables:
       - ${CI_JOB_ID}_coverage.xml
       - build/apps/gnb/gnb
       - build/Testing/Temporary/MemoryChecker.*.log
+      - build_time_metrics.json
     expire_in: 10 minutes
   cache:
     - !reference [.fetch_src_cache,  cache]

.gitlab/ci/release.yml

@@ -10,20 +10,22 @@ include:
   - project: softwareradiosystems/ci/tools
     ref: "8"
     file: .gitlab/ci-shared/setup/all.yml
+  - project: softwareradiosystems/ci/tools
+    ref: "8"
+    file: .gitlab/ci-shared/features/all.yml
 
 stages:
   - private
   - public
+  - release
 
-update agpl main:
+.update agpl main:
   stage: private
-  rules:
-    - if: $ON_TAG
-      allow_failure: true
   interruptible: false
   variables:
     GIT_STRATEGY: none
     BRANCH: agpl_main
+    MODE: ""
   image: ubuntu:22.04
   before_script:
     - DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y --no-install-recommends git git-lfs apt-transport-https ca-certificates
@@ -33,7 +35,14 @@ update agpl main:
     - git clone https://${CODEBOT_USERNAME}:${CODEBOT_TOKEN}@gitlab.com/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}.git /${CI_PROJECT_NAME}
     - cd /${CI_PROJECT_NAME}
     - git fetch -q origin ${CI_COMMIT_SHA} && git checkout ${CI_COMMIT_SHA}
-    - .gitlab/ci/release/auto_merge.sh ${CI_COMMIT_SHA} ${BRANCH}
+    - .gitlab/ci/release/auto_merge.sh ${CI_COMMIT_SHA} ${BRANCH} ${MODE}
+
+update agpl main:
+  extends: .update agpl main
+  rules:
+    - if: $ON_TAG
+      variables:
+        MODE: "push"
 
 generate testvector tar gz:
   stage: private
@@ -46,21 +55,31 @@ generate testvector tar gz:
   artifacts:
     paths:
       - phy_testvectors.tar.gz
-    expire_in: "7 days"
+    expire_in: "30 days"
 
-push:
-  stage: public
+coverity-agpl:
+  extends: .coverity_base
+  stage: private
   rules:
     - if: $ON_TAG
-      when: manual
-      allow_failure: false
+  before_script:
+    - export PROJECT_NAME="srsRAN_5G_agpl"
+    - export DESCRIPTION="srsRAN Project AGPL build"
+    - export COV_TOKEN="${COVERITY_TOKEN_AGPL}"
+  needs:
+    - job: update agpl main
+      optional: false
+      artifacts: false
+
+.publish:
+  stage: public
   interruptible: false
   image: alpine:3.16.0 # sh entrypoint
   variables:
     GIT_STRATEGY: none
     PRIVATE_BRANCH: agpl_main
     PUBLIC_REPO: srsran/srsRAN_Project
-    PUBLIC_BRANCH: main
+    PUBLIC_BRANCH: ""
   before_script:
     - apk add git
   script:
@@ -71,16 +90,57 @@ push:
     # Push code to github main
     - git push github_public ${PRIVATE_BRANCH}:${PUBLIC_BRANCH}
 
-release:
+publish branch test:
+  extends: .publish
+  rules:
+    - if: $ON_TAG
+      when: manual
+      allow_failure: false
+  variables:
+    PUBLIC_BRANCH: test
+  needs:
+    - job: update agpl main
+      optional: false
+      artifacts: false
+
+publish main:
+  extends: .publish
+  rules:
+    - if: $ON_TAG
+      when: manual
+      allow_failure: false
+  variables:
+    PUBLIC_BRANCH: main
+  needs:
+    - job: publish branch test
+      optional: false
+      artifacts: false
+
+notify main published:
+  extends: .notifier
   stage: public
+  rules:
+    - if: $ON_TAG
+  needs:
+    - job: publish main
+      optional: false
+      artifacts: false
+  variables:
+    CI_DESCRIPTION: "Public GitHub Updated"
+    MSG: "main branch has been updated."
+    SLACK_CHANNEL: $SLACK_CHANNEL_OK
+    LEVEL: "ok"
+
+release public:
+  stage: release
   rules:
     - if: $ON_TAG
       when: manual
       allow_failure: false
   interruptible: false
   image: alpine:3.16.0 # sh entrypoint
   needs:
-    - job: push
+    - job: publish main
       optional: false
       artifacts: false
     - job: generate testvector tar gz
@@ -131,3 +191,19 @@ release:
         -H "Content-Type: application/octet-stream" \
         https://uploads.github.com/repos/${PUBLIC_REPO}/releases/${RELEASE_ID}/assets?name=${ARTIFACT} \
         --data-binary "@${CI_PROJECT_DIR}/${ARTIFACT}"
+
+notify release published:
+  extends: .notifier
+  stage: release
+  rules:
+    - if: $ON_TAG
+  needs:
+    - job: release public
+      optional: false
+      artifacts: false
+  variables:
+    CI_DESCRIPTION: "Public Release created on GitHub"
+    SLACK_CHANNEL: $SLACK_CHANNEL_OK
+    LEVEL: "ok"
+  before_script:
+    - MSG="release_${CI_COMMIT_TAG//./_} created"

.gitlab/ci/release/auto_merge.sh

@@ -29,6 +29,10 @@ rm_deleted_by_them() {
 # Checks the git status and tries to `git rm` the files removed from
 # the target branch to the commit.
 rm_deleted_by_us() {
+    echo "=================="
+    echo "= Remove deleted ="
+    echo "=================="
+
     delete_name="deleted by us:"
     delete_size_next=${#delete_name}+3
     lines=$(git status)
@@ -45,6 +49,10 @@ rm_deleted_by_us() {
 
 # Fixes the conflicts using theirs and adds them to the commit.
 fix_conflicts_using_theirs() {
+    echo "================="
+    echo "= Fix conflicts ="
+    echo "================="
+
     conflict_name="both modified:"
     conflict_size_next=${#conflict_name}+3
     lines=$(git status)
@@ -54,6 +62,7 @@ fix_conflicts_using_theirs() {
         if [[ "$status" == "$conflict_name" ]]; then
             path_name=${line:$conflict_size_next:${#line}}
             echo "Resolving using theirs $path_name"
+            git diff ORIG_HEAD MERGE_HEAD "$path_name"
             git checkout --theirs "$path_name"
             git add "$path_name"
         fi
@@ -62,12 +71,20 @@ fix_conflicts_using_theirs() {
 }
 
 remove_lfs_files() {
+    echo "===================="
+    echo "= Remove lfs files ="
+    echo "===================="
+
     while read -r line; do
         git rm --cached "$line"
     done < <(git lfs ls-files | sed -r 's/^.{13}//')
 }
 
 update_headers() {
+    echo "=================="
+    echo "= Update headers ="
+    echo "=================="
+
     # for actual source and header files
     find . -type f \( -name "*.cpp" -o -name "*.h" -o -name "*.h.in" \) ! -path "*/external/*" ! -name "rfnoc_test.cc" -exec perl -0777 -pi -e "s{/\*.*?\*/}{/*
  *
@@ -92,7 +109,7 @@ update_headers() {
  */}s" {} \;
 
     # for CMake/YML files
-    find . -type f -\( -name "CMakeLists.txt" -o -name "*.cmake" -o -name "*.yml" -o -name "*.sh" \) ! -path "*/configs/*" ! -path "*/.gitlab/*" ! -name "FindBackward.cmake" ! -name "FindRapidJSON.cmake" ! -name "CheckCSourceRuns.cmake" ! -name "CheckFunctionExistsMath.cmake" -exec perl -0777 -pi -e "s/#[^!][\s\S]*?(?=\n.*?=|\n\n)/#
+    find . -type f -\( -name "CMakeLists.txt" -o -name "*.cmake" -o -name "*.yml" -o -name "*.sh" \) ! -path "*/configs/*" ! -path "*/.github/*" ! -path "*/.gitlab/*" ! -name "FindBackward.cmake" ! -name "FindRapidJSON.cmake" ! -name "CheckCSourceRuns.cmake" ! -name "CheckFunctionExistsMath.cmake" -exec perl -0777 -pi -e "s/#[^!][\s\S]*?(?=\n.*?=|\n\n)/#
 # Copyright 2021-$(date +%Y) Software Radio Systems Limited
 #
 # This file is part of srsRAN
@@ -115,14 +132,15 @@ update_headers() {
 
 main() {
     # Check number of args
-    if (($# != 2)); then
+    if [ $# != 2 ] && [ $# != 3 ]; then
         echo >&2 "Please call script with source branch and target branch as argument"
-        echo >&2 "E.g. ./auto_merge.sh <source_branch> <target_branch>"
+        echo >&2 "E.g. ./auto_merge.sh <source_branch> <target_branch> [dryrun]"
         exit 1
     fi
 
     local source_branch=$1
     local target_branch=$2
+    local mode="${3:-push}"
 
     # Checkout target branch
     git fetch -q origin "$target_branch"
@@ -132,26 +150,27 @@ main() {
     fi
 
     # Merge
+    echo "========="
+    echo "= Merge ="
+    echo "========="
     git fetch -q origin "$source_branch"
-    if ! git merge "$source_branch"; then
-        # Conflicts
+    # Git merge will always generate a merge commit because history is different in both branches
+    if ! git merge "$source_branch" -m "Update main"; then
+        # There are conflicts
         git status
         rm_deleted_by_us
         rm_deleted_by_them
         fix_conflicts_using_theirs
         git commit --no-edit
     fi
-
-    # Remove any remaining lfs files
     remove_lfs_files
-    git commit -a --amend --no-edit
-
-    # Update headers
     update_headers
-    git commit -a -m "Adding AGPL copyright to new files"
+    git commit -a --amend --no-edit
 
     # Push
-    git push origin "$target_branch"
+    if [ "$mode" = "push" ]; then
+        git push origin "$target_branch"
+    fi
 }
 
 main "$@"