Merge pull request #12922 from ethereum/crowdin-cd-v2

Crowdin CI v2

Author: wackerow

.github/workflows/build-crowdin.yml

@@ -0,0 +1,68 @@
+name: Crowdin CI
+
+on:
+  schedule:
+    - cron: "20 4 1 * *" # Runs at 4:20 AM on the first day of every month
+  workflow_dispatch: # Can be dispatched manually
+
+jobs:
+  create_approved_language_bucket_prs:
+    runs-on: ubuntu-latest
+    env:
+      CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }}
+      CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    steps:
+      # Set up environment
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+
+      - name: Install dependencies
+        run: yarn install
+
+      - name: Install ts-node
+        run: yarn global add ts-node
+
+      - name: Set up git
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "GitHub Action"
+
+      - name: Fetch latest dev
+        run: git fetch origin dev
+
+      # Build translations
+      - name: Build Crowdin project
+        id: build-crowdin
+        run: |
+          npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/triggerBuild.ts;
+          grep BUILD_ID output.env >> $GITHUB_ENV;
+
+      - name: Await latest build to finish
+        run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/awaitLatestBuild.ts
+
+      - name: Check build success
+        run: |
+          if [ $(grep BUILD_SUCCESS output.env | cut -d'=' -f2) = false ]; then
+            echo "Build timed out, exiting"
+            exit 1
+          fi
+        shell: bash
+
+      # Prepare bucket ids
+      - name: Get latest translation bucket directory ids
+        run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/getBucketDirectoryIds.ts
+
+      # Import approved translations
+      - name: Get translations
+        run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/getTranslations.ts
+
+      # Post updates as language-specific PRs
+      - name: Process commits and post PRs by language
+        run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/postLangPRs.ts

.github/workflows/crowdin-ci.yml

@@ -1,4 +1,4 @@
-import { copyFileSync, existsSync, mkdirSync, readdirSync } from "fs"
+import { copyFileSync, existsSync, mkdirSync, readdirSync, statSync } from "fs"
 import { join } from "path"
 
 import i18Config from "../../../../i18n.config.json"
@@ -45,9 +45,7 @@ export const scrapeDirectory = (
       copyFileSync(source, jsonDestinationPath)
       // Update .json tracker
       trackers.langs[repoLangCode].jsonCopyCount++
-    } else if (
-      item.endsWith(".md")
-    ) {
+    } else if (item.endsWith(".md")) {
       const mdDestDirPath: string = join(
         TRANSLATIONS_DIR,
         repoLangCode,
@@ -60,6 +58,7 @@ export const scrapeDirectory = (
       // Update .md tracker
       trackers.langs[repoLangCode].mdCopyCount++
     } else {
+      if (!statSync(source).isDirectory()) return
       // If another directory, recursively call `scrapeDirectory`
       scrapeDirectory(
         `${path}/${item}`,

.github/workflows/get-translations.yml

@@ -0,0 +1,57 @@
+import { writeFileSync } from "fs"
+import { join } from "path"
+
+import crowdin from "../api-client/crowdinClient"
+
+const FINISHED = "finished"
+const TIMEOUT = 2 * 60 * 60 * 1000 // Timeout after 2 hours
+const INTERVAL = 10 * 1000 // 10 seconds between checks
+
+const OUTPUT_PATH = join(process.env["GITHUB_WORKSPACE"] || "", "output.env")
+
+async function awaitLatestBuild() {
+  const projectId = Number(process.env.CROWDIN_PROJECT_ID) || 363359
+
+  // BUILD_ID is provided by the triggerBuild script run in the same workflow prior to this script
+  const buildId = process.env.BUILD_ID
+
+  console.log("Build ID provided:", buildId)
+  const initialResponse = await crowdin.translationsApi.checkBuildStatus(
+    projectId,
+    Number(buildId)
+  )
+  let data = initialResponse.data
+
+  let isFinished = data.status === FINISHED
+
+  const timeoutTime = Date.now() + TIMEOUT
+  let tryAgainTime = Date.now() - 1
+  while (!isFinished && Date.now() < timeoutTime) {
+    if (Date.now() < tryAgainTime) continue
+    tryAgainTime = Date.now() + INTERVAL
+
+    const repeatCheck = await crowdin.translationsApi.checkBuildStatus(
+      projectId,
+      Number(buildId)
+    )
+    data = repeatCheck.data
+    isFinished = data.status === FINISHED
+    console.log(
+      `id: ${buildId}, status: ${data.status}, progress ${data.progress}`
+    )
+  }
+
+  if (data.status !== FINISHED) {
+    writeFileSync(OUTPUT_PATH, `BUILD_SUCCESS=false\n`, { flag: "a" })
+    throw new Error(
+      `Timeout: Build did not finish in ${TIMEOUT / 1000 / 60} minutes`
+    )
+  }
+
+  console.log("Latest build data:", data)
+  writeFileSync(OUTPUT_PATH, `BUILD_SUCCESS=true\n`, { flag: "a" })
+}
+
+awaitLatestBuild()
+
+export default awaitLatestBuild

src/scripts/crowdin/import/utils.ts

@@ -1,15 +1,15 @@
-import { resolve } from "path"
+import { join } from "path"
 
 export const DOT_CROWDIN = ".crowdin"
 
 export const CROWDIN_DATA_DIR = "src/data/crowdin"
 export const SAVE_FILE = "download.zip"
-export const FILE_PATH = resolve(CROWDIN_DATA_DIR, SAVE_FILE)
+export const FILE_PATH = join(CROWDIN_DATA_DIR, SAVE_FILE)
 
 export const SUMMARY_SAVE_FILE = "import-summary.json"
-export const SUMMARY_PATH = resolve(CROWDIN_DATA_DIR, SUMMARY_SAVE_FILE)
+export const SUMMARY_PATH = join(CROWDIN_DATA_DIR, SUMMARY_SAVE_FILE)
 
 export const BUCKETS_IMPORTED_FILE = "buckets-imported.json"
-export const BUCKETS_PATH = resolve(CROWDIN_DATA_DIR, BUCKETS_IMPORTED_FILE)
+export const BUCKETS_PATH = join(CROWDIN_DATA_DIR, BUCKETS_IMPORTED_FILE)
 
 export const APPROVAL_THRESHOLD = 100

src/scripts/crowdin/translations/awaitLatestBuild.ts

@@ -7,11 +7,11 @@ import type { BucketsList } from "../import/types"
 import { APPROVAL_THRESHOLD } from "./constants"
 
 async function getApprovedBuckets(): Promise<BucketsList> {
+  console.log("⏳ Getting approved buckets...")
   const projectId = Number(process.env.CROWDIN_PROJECT_ID) || 363359
 
   const bucketsList: BucketsList = {}
 
-  // TODO: Consider regenerating bucketDirs list on each run for fidelity
   for (const bucketDir of bucketDirs) {
     const directoryProgress =
       await crowdin.translationStatusApi.getDirectoryProgress(

src/scripts/crowdin/translations/constants.ts

@@ -0,0 +1,9 @@
+import getAndSaveDirectories from "../source-files/fetchAndSaveDirectories"
+
+async function main() {
+  await getAndSaveDirectories()
+}
+
+main()
+
+export default main

src/scripts/crowdin/translations/getApprovedBuckets.ts

@@ -1,7 +1,7 @@
 import fs from "fs"
 
 import { LOCALES_CODES } from "../../../lib/constants"
-import { BucketsList } from "../import/types"
+import type { BucketsList } from "../import/types"
 
 import { BUCKETS_PATH } from "./constants"
 import { createLocaleTranslationPR } from "./utils"

src/scripts/crowdin/translations/getBucketDirectoryIds.ts

@@ -1,14 +1,22 @@
+import { writeFileSync } from "fs"
+import { join } from "path"
+
 import crowdin from "../api-client/crowdinClient"
 
-import "dotenv/config"
+const OUTPUT_PATH = join(process.env["GITHUB_WORKSPACE"] || "", "output.env")
 
 async function triggerBuild() {
   const projectId = Number(process.env.CROWDIN_PROJECT_ID) || 363359
 
   try {
-    await crowdin.translationsApi.buildProject(projectId, {
-      exportApprovedOnly: true,
-    })
+    const response = await crowdin.translationsApi.buildProject(projectId)
+    const { id, status } = response.data
+    const isAlreadyFinished = status === "finished"
+    console.log(
+      `Build ${isAlreadyFinished ? "already finished" : "triggered"} id:`,
+      id
+    )
+    writeFileSync(OUTPUT_PATH, `BUILD_ID=${id}\n`, { flag: "a" })
   } catch (error: unknown) {
     console.error((error as Error).message)
   }