Migrate nightly documentation build from BuildBot to Github actions #242 (#247)

avdivan · web-flow · commit 6dc2024d6734 · 2025-06-09T21:43:25.000+03:00
* testing use default python for Mac os ARM

* testing use default python for Mac os Intel

* Create docks_upload.yaml

* Update docks_upload.yaml

* Update docks_upload.yaml

* Create docks_upload2.yaml

* Update docks_upload2.yaml

* Update docks_upload.yaml

* Update docks_upload2.yaml

* Create build_docks.yaml

* Update docks_upload.yaml

* Update docks_upload2.yaml

* Update build_docks.yaml

* Create download_artifacts.sh

* Delete .github/workflows/docks_upload2.yaml

* Delete .github/workflows/docks_upload.yaml

* Create build_javadocs.yaml

* Update build_javadocs.yaml

* Update build_docks.yaml

* Delete .github/workflows/build_javadocs.yaml

* Update build_docks.yaml

* Update build_docks.yaml

* Update build_docks.yaml

* Update build_docks.yaml

* Update build_docks.yaml

* Update OCV-PR-4.x-macOS-ARM64.yaml

* Update OCV-PR-4.x-macOS-x86_64.yaml

* Update download_artifacts.sh

* Update and rename build_docks.yaml to build_docs_schedule.yaml

* Update build_docs_schedule.yaml

* Create pencv-docs-22.04-jdk-js

* Update OCV-PR-5.x-docs.yaml

* Update OCV-PR-4.x-docs.yaml

* Rename pencv-docs-22.04-jdk-js to Dockerfile-22-jdk-js
diff --git a/.github/workflows/OCV-PR-4.x-docs.yaml b/.github/workflows/OCV-PR-4.x-docs.yaml
@@ -21,6 +21,8 @@ env:
   OPENCV_DOCKER_WORKDIR: '/home/ci/opencv'
   OPENCV_EXTRA_DOCKER_WORKDIR: '/home/ci/opencv_extra'
   OPENCV_CONTRIB_DOCKER_WORKDIR: '/home/ci/opencv_contrib'
+  OPENCV_BUILD: /home/ci/opencv-build
+  JAVA_DOC: /home/ci/opencv-build/doc/doxygen/html/javadoc
 
 jobs:
   BuildAndTest:
@@ -29,7 +31,7 @@ jobs:
       run:
         shell: bash
     container:
-      image: quay.io/opencv-ci/opencv-docs-22.04:20250404
+      image: quay.io/opencv-ci/opencv-docs-jdk-js-22.04:20250526
       options: --user ci
     steps:
     - uses: actions/checkout@v4
@@ -130,6 +132,20 @@ jobs:
         cd ${{ env.OPENCV_CONTRIB_DOCKER_WORKDIR }}
         git bundle create test.bundle ${{ env.LATEST_COMMIT_OPENCV_CONTRIB }}..HEAD || true
         python3 $HOME/scripts/patch_size.py
+        
+    - name: Generate JavaDoc
+      run: |
+          mkdir -p ${{ env.OPENCV_BUILD }}
+          cd ${{ env.OPENCV_BUILD }}
+          cmake -DBUILD_SHARED_LIBS=OFF \
+                -DBUILD_opencv_java=ON \
+                -DCMAKE_BUILD_TYPE=Release \
+                -DOPENCV_EXTRA_MODULES_PATH=${{ env.OPENCV_CONTRIB_DOCKER_WORKDIR }}/modules \
+                -DANT_EXECUTABLE=/usr/bin/ant \
+                ${{ env.OPENCV_DOCKER_WORKDIR }}
+          make -j2
+          cd modules/java/jar/opencv
+          ant javadoc
 
     - name: Build js
       if: ${{ always() && steps.last-repo-step.outcome == 'success' }}
diff --git a/.github/workflows/OCV-PR-5.x-docs.yaml b/.github/workflows/OCV-PR-5.x-docs.yaml
@@ -21,6 +21,8 @@ env:
   OPENCV_DOCKER_WORKDIR: '/home/ci/opencv'
   OPENCV_EXTRA_DOCKER_WORKDIR: '/home/ci/opencv_extra'
   OPENCV_CONTRIB_DOCKER_WORKDIR: '/home/ci/opencv_contrib'
+  OPENCV_BUILD: /home/ci/opencv-build
+  JAVA_DOC: /home/ci/opencv-build/doc/doxygen/html/javadoc
 
 jobs:
   BuildAndTest:
@@ -29,7 +31,7 @@ jobs:
       run:
         shell: bash
     container:
-      image: quay.io/opencv-ci/opencv-docs-22.04:20250404
+      image: quay.io/opencv-ci/opencv-docs-jdk-js-22.04:20250526
       options: --user ci
     steps:
     - uses: actions/checkout@v4
@@ -130,6 +132,21 @@ jobs:
         cd ${{ env.OPENCV_CONTRIB_DOCKER_WORKDIR }}
         git bundle create test.bundle ${{ env.LATEST_COMMIT_OPENCV_CONTRIB }}..HEAD || true
         python3 $HOME/scripts/patch_size.py
+        
+    - name: Generate JavaDoc
+      run: |
+          mkdir -p ${{ env.OPENCV_BUILD }}
+          cd ${{ env.OPENCV_BUILD }}
+          cmake -DBUILD_SHARED_LIBS=OFF \
+                -DBUILD_opencv_java=ON \
+                -DCMAKE_BUILD_TYPE=Release \
+                -DOPENCV_EXTRA_MODULES_PATH=${{ env.OPENCV_CONTRIB_DOCKER_WORKDIR }}/modules \
+                -DANT_EXECUTABLE=/usr/bin/ant \
+                ${{ env.OPENCV_DOCKER_WORKDIR }}
+          make -j2
+          cd modules/java/jar/opencv
+          ant javadoc
+ 
     - name: Build js
       if: ${{ always() && steps.last-repo-step.outcome == 'success' }}
       timeout-minutes: 60
diff --git a/.github/workflows/build_docs_schedule.yaml b/.github/workflows/build_docs_schedule.yaml
@@ -0,0 +1,101 @@
+name: build docs schedule
+
+on:
+  schedule:
+    - cron: '0 3 * * *'
+  workflow_dispatch:
+
+env:
+  EXTRA_CMAKE_OPTIONS: '-DOPENCV_JS_LOCATION=/home/ci/build/js/bin/opencv.js -DOPENCV_ENABLE_NONFREE=ON -DBUILD_DOCS_JS_BINDINGS=OFF -DBUILD_SHARED_LIBS=ON -DBUILD_DOCS=ON -DOPENCV_DOCS_HAVE_DOT=YES -DBUILD_TESTS=ON -DWITH_OPENNI2=ON -DWITH_GDCM=ON -DPYTHON_DEFAULT_EXECUTABLE=/usr/bin/python3 -DOPENCV_EXTRA_MODULES_PATH=/home/ci/opencv_contrib/modules -DWITH_GDAL=ON -DBUILD_EXAMPLES=ON  -DBUILD_PERF_TESTS=ON -DWITH_IPP=OFF'
+  OPENCV_DOCKER_WORKDIR: '/home/ci/opencv'
+  OPENCV_EXTRA_DOCKER_WORKDIR: '/home/ci/opencv_extra'
+  OPENCV_CONTRIB_DOCKER_WORKDIR: '/home/ci/opencv_contrib'
+  OPENCV_SRC: /home/ci/opencv
+  OPENCV_BUILD: /home/ci/opencv-build
+  JAVA_DOC: /home/ci/opencv-build/doc/doxygen/html/javadoc
+
+jobs:
+
+  build-docs:
+    name: Build Docs ${{ matrix.branch }}
+    runs-on: ubuntu-22.04
+    container:
+      image: quay.io/opencv-ci/opencv-docs-jdk-js-22.04:20250526
+      options: --user ci
+    strategy:
+      matrix:
+        branch: [4.x, 5.x]
+    env:
+      SOURCE_BRANCH_NAME: ${{ github.head_ref || matrix.branch }}
+      TARGET_BRANCH_NAME: ${{ github.base_ref || matrix.branch }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: opencv/ci-gha-workflow
+          ref: ${{ github.repository == 'opencv/ci-gha-workflow' && github.ref || 'main' }}
+
+      - name: Define HOME
+        run: echo "HOME=/home/ci" >> $GITHUB_ENV
+
+      - name: Force TARGET_BRANCH_NAME (if in ci-gha-workflow repo)
+        if: ${{ github.event.repository.name == 'ci-gha-workflow' }}
+        run: echo "TARGET_BRANCH_NAME=${{ matrix.branch }}" >> $GITHUB_ENV
+
+      - name: Info
+        run: |
+          echo "SOURCE_BRANCH_NAME=${{ env.SOURCE_BRANCH_NAME }}"
+          echo "TARGET_BRANCH_NAME=${{ env.TARGET_BRANCH_NAME }}"
+
+      - name: Clean workspace
+        run: find ${{ env.OPENCV_DOCKER_WORKDIR }} -mindepth 1 -delete
+
+      - name: Clone Repositories
+        run: |
+          git clone --branch ${{ env.TARGET_BRANCH_NAME }} https://github.com/opencv/opencv.git ${{ env.OPENCV_DOCKER_WORKDIR }}
+          git clone --branch ${{ env.TARGET_BRANCH_NAME }} https://github.com/opencv/opencv_extra.git ${{ env.OPENCV_EXTRA_DOCKER_WORKDIR }}
+          git clone --branch ${{ env.TARGET_BRANCH_NAME }} https://github.com/opencv/opencv_contrib.git ${{ env.OPENCV_CONTRIB_DOCKER_WORKDIR }}
+
+      - name: Generate JavaDoc
+        run: |
+          mkdir -p ${{ env.OPENCV_BUILD }}
+          cd ${{ env.OPENCV_BUILD }}
+          cmake -DBUILD_SHARED_LIBS=OFF \
+                -DBUILD_opencv_java=ON \
+                -DCMAKE_BUILD_TYPE=Release \
+                -DOPENCV_EXTRA_MODULES_PATH=${{ env.OPENCV_CONTRIB_DOCKER_WORKDIR }}/modules \
+                -DANT_EXECUTABLE=/usr/bin/ant \
+                ${{ env.OPENCV_DOCKER_WORKDIR }}
+          make -j2
+          cd modules/java/jar/opencv
+          ant javadoc
+          
+      - name: Build js
+        timeout-minutes: 60
+        run: |
+          cd $HOME/build
+          emcmake python ../opencv/platforms/js/build_js.py js --build_test
+
+      - name: Run js tests
+        timeout-minutes: 60
+        run: |
+          cd $HOME/build/js/bin
+          npm install
+          node tests.js
+          
+      - name: Configure Doxygen
+        run: |
+          mkdir -p $HOME/build
+          cd $HOME/build
+          cmake ${{ env.EXTRA_CMAKE_OPTIONS }} ${{ env.OPENCV_DOCKER_WORKDIR }}
+  
+      - name: Build Doxygen
+        working-directory: ${{ env.HOME }}/build
+        run: cmake --build . --config release --target doxygen -- -j$(nproc) 2>&1 | tee log.txt
+
+      - name: Archive Doxygen Docs
+        uses: actions/upload-artifact@v4
+        with:
+          name: opencv-docs-${{ matrix.branch }}
+          path: |
+            ${{ env.HOME }}/build/doc/doxygen/html
+            ${{ env.HOME }}/build/js/bin/opencv.js
diff --git a/docker/docs/Dockerfile-22-jdk-js b/docker/docs/Dockerfile-22-jdk-js
@@ -0,0 +1,93 @@
+#Version: 20250526
+# Image name: quay.io/opencv-ci/opencv-docs-jdk-js-22.04
+
+FROM ubuntu:22.04
+
+RUN \
+  apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    software-properties-common \
+    gpg-agent curl git htop man zip unzip vim wget ncdu mc nano time \
+    file \
+    && \
+  add-apt-repository ppa:git-core/ppa && \
+  apt-get install -y --no-install-recommends git && \
+  rm -rf /var/lib/apt/lists/*
+
+RUN \
+  apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    cmake build-essential \
+    doxygen-latex graphviz \
+    p7zip-full \
+    pylint flake8 \
+    python3-bs4 \
+    openjdk-11-jdk \
+    default-jdk-headless \
+    ant \
+    && \
+  rm -rf /var/lib/apt/lists/*
+
+ENV EMSDK_VERSION=1.39.0-upstream
+ENV EMSDK_DIR=/opt/emsdk-portable
+ENV EMSDK_CACHE_DIR=/opt/build-worker/emsdk-${EMSDK_VERSION}
+
+RUN \
+  /bin/bash -c "mkdir -p ${EMSDK_DIR} && \
+    cd ${EMSDK_DIR} && \
+    git rev-parse HEAD || (rm -rf ./.git && git clone https://github.com/juj/emsdk.git .) && \
+    git reset --hard || true && \
+    git clean -f -d || true && \
+    git fetch origin && \
+    git checkout -B main origin/main && \
+    git rev-parse HEAD && \
+    ./emsdk list && \
+    ./emsdk install ${EMSDK_VERSION} && \
+    ./emsdk activate ${EMSDK_VERSION} && \
+    source ./emsdk_env.sh && \
+    mkdir -p ./.home && \
+    cp -rp ./.em* ./.home/ && \
+    # python script (opencv/platforms/js/build_js.py) has shebang and required hardly python
+    ln -s /usr/bin/python3 /usr/bin/python"
+
+
+# Install fresh Doxygen
+ARG doxygen_version=doxygen-1.12.0
+ARG doxygen_urlbase=https://github.com/doxygen/doxygen/releases/download/Release_1_12_0
+ARG doxygen_archive=${doxygen_version}.linux.bin.tar.gz
+ARG doxygen_sha1sum=c50fc71ba9b5e431f0f0f21c5dc8559f47f06007
+ADD ${doxygen_urlbase}/${doxygen_archive} /opt/
+RUN \
+  cd /opt \
+  && echo "${doxygen_sha1sum} ${doxygen_archive}" | sha1sum -c \
+  && tar -xvf ${doxygen_archive} \
+  && rm -rf ${doxygen_archive}
+ENV PATH=/opt/${doxygen_version}/bin:${PATH}
+
+
+# GitHub Actions user's UID is 1001
+RUN \
+  useradd ci -m -s /bin/bash -G users --uid=1001 && \
+  chown -R ci:ci ${EMSDK_DIR}
+
+USER ci
+
+RUN \
+  cd $HOME && \
+  mkdir opencv opencv_contrib build && \
+  git config --global --add safe.directory "*" && \
+  git config --global user.email "opencv.ci" && \
+  git config --global user.name "opencv.ci" && \
+  git config --global pull.rebase false
+
+# Run docker build from the root directory of the repository
+COPY scripts/patch_size.py /home/ci/scripts/patch_size.py
+COPY scripts/pack_docs.py /home/ci/scripts/
+
+ENV PATH=${EMSDK_DIR}:${EMSDK_DIR}/upstream/emscripten:${EMSDK_DIR}/node/20.18.0_64bit/bin:${PATH}
+ENV EMSDK=${EMSDK_DIR}
+ENV EM_CONFIG=${EMSDK_DIR}/.emscripten
+ENV EM_CACHE=${EMSDK_DIR}/upstream/emscripten/cache
+ENV EMSDK_NODE=${EMSDK_DIR}/node/20.18.0_64bit/bin/node
+ENV EMSCRIPTEN=${EMSDK_DIR}/upstream/emscripten
+ENTRYPOINT ["bash"]
diff --git a/scripts/download_artifacts.sh b/scripts/download_artifacts.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+
+set -e
+
+REPO="opencv/ci-gha-workflow"
+DEST_DIR="/home/www/site/webapps/docs/test"
+TMP_DIR="/tmp/artifacts"
+TOKEN="_insert_your_key"
+
+mkdir -p "$TMP_DIR"
+
+echo "[*] Fetching artifacts list..."
+ARTIFACTS=$(curl -s -H "Authorization: token $TOKEN" "https://api.github.com/repos/$REPO/actions/artifacts")
+
+for BRANCH in 4.x 5.x; do
+    ARTIFACT_NAME="opencv-docs-$BRANCH"
+    echo "[*] Looking for latest artifact named $ARTIFACT_NAME..."
+
+    # Парсим id и дату создания вручную
+    ARTIFACT_ID=$(echo "$ARTIFACTS" | awk -v name="$ARTIFACT_NAME" '
+        BEGIN { RS="{"; FS=","; max_time=0; best_id="" }
+        /"name": *"[^"]+"/ {
+            this_id=""
+            this_name=""
+            this_expired="false"
+            this_created=""
+            for (i=1; i<=NF; i++) {
+                if ($i ~ /"id":/) {
+                    gsub(/[^0-9]/, "", $i)
+                    this_id=$i
+                }
+                if ($i ~ /"name":/) {
+                    gsub(/.*"name": *"/, "", $i)
+                    gsub(/".*/, "", $i)
+                    this_name=$i
+                }
+                if ($i ~ /"expired":/) {
+                    if ($i ~ /true/) this_expired="true"
+                }
+                if ($i ~ /"created_at":/) {
+                    gsub(/.*"created_at": *"/, "", $i)
+                    gsub(/".*/, "", $i)
+                    this_created=$i
+                }
+            }
+
+            if (this_name == name && this_expired == "false") {
+                cmd = "date -d \"" this_created "\" +%s"
+                cmd | getline epoch
+                close(cmd)
+                if (epoch > max_time) {
+                    max_time = epoch
+                    best_id = this_id
+                }
+            }
+        }
+        END { print best_id }
+    ')
+
+    if [[ -z "$ARTIFACT_ID" ]]; then
+        echo "[!] No valid artifact found for $ARTIFACT_NAME"
+        continue
+    fi
+
+    ZIP_PATH="$TMP_DIR/$ARTIFACT_NAME.zip"
+    DEST_PATH="$DEST_DIR/$BRANCH"
+
+    echo "[*] Downloading $ARTIFACT_NAME (ID: $ARTIFACT_ID) -> $ZIP_PATH"
+    curl -sL -H "Authorization: token $TOKEN" \
+        -o "$ZIP_PATH" \
+        "https://api.github.com/repos/$REPO/actions/artifacts/$ARTIFACT_ID/zip"
+
+    echo "[*] Extracting to $DEST_PATH"
+    mkdir -p "$DEST_PATH"
+    echo "[*] Cleaning old contents in $DEST_PATH"
+    rm -rf "$DEST_PATH"/*
+    TMP_EXTRACT="$TMP_DIR/extract_$BRANCH"
+    rm -rf "$TMP_EXTRACT"
+    mkdir -p "$TMP_EXTRACT"
+    unzip -q -o "$ZIP_PATH" -d "$TMP_EXTRACT"
+    echo "[*] Moving relevant files to $DEST_PATH"
+    if [ -d "$TMP_EXTRACT/doc/doxygen/html" ]; then
+        mv "$TMP_EXTRACT/doc/doxygen/html"/* "$DEST_PATH/"
+    else
+        echo "[!] Warning: doc/doxygen/html not found in artifact"
+    fi
+
+    if [ -d "$TMP_EXTRACT/js/bin" ]; then
+        mv "$TMP_EXTRACT/js/bin"/* "$DEST_PATH/"
+    else
+        echo "[!] Warning: js/bin not found in artifact"
+    fi
+
+done
+
+#echo "[*] Reloading nginx..."
+#nginx -s reload
+
+echo "[*] Cleaning old temp zip files..."
+find "$TMP_DIR" -type f -name "*.zip" -mtime +1 -delete
+
+echo "[✔] Done."
