Merge pull request #12698 from ethereum/crowdin-cd

feat: Crowdin import automation

Author: wackerow

.github/workflows/get-translations.yml

@@ -0,0 +1,47 @@
+name: Import Crowdin translations
+
+on:
+  schedule:
+    - cron: "20 16 1 * *" # Runs at 4:20 PM on the first day of every month
+  workflow_dispatch:
+
+jobs:
+  create_pr:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+
+      - name: Install dependencies
+        run: yarn install
+
+      - name: Install ts-node
+        run: yarn global add ts-node
+
+      - name: Set up git
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "GitHub Action"
+
+      - name: Fetch latest dev
+        run: git fetch origin dev
+
+      - name: Get translations
+        run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/getTranslations.ts
+        env:
+          CROWDIN_API_KEY: ${{ secrets.CROWDIN_API_KEY }}
+          CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }}
+
+      - name: Authenticate GitHub CLI
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | gh auth login --with-token
+
+      - name: Process commits and post PRs by language
+        run: npx ts-node -O '{"module":"commonjs"}' ./src/scripts/crowdin/translations/postLangPRs.ts
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

package.json

@@ -70,11 +70,13 @@
     "@storybook/react": "7.6.6",
     "@storybook/testing-library": "0.2.2",
     "@svgr/webpack": "^8.1.0",
+    "@types/decompress": "^4.2.7",
     "@types/hast": "^3.0.0",
     "@types/node": "^20.4.2",
     "@types/react": "^18.2.15",
     "@types/react-dom": "^18.2.7",
     "chromatic": "^10.5.0",
+    "decompress": "^4.2.1",
     "eslint": "^8.45.0",
     "eslint-config-next": "^13.0.0",
     "eslint-config-prettier": "^9.0.0",

src/scripts/crowdin/getTranslationProgress.ts

@@ -1,5 +1,6 @@
 import fs from "fs"
 
+import { CROWDIN_API_MAX_LIMIT } from "../../lib/constants"
 import type { ProjectProgressData } from "../../lib/types"
 
 import crowdin from "./api-client/crowdinClient"
@@ -13,7 +14,7 @@ async function main() {
     const response = await crowdin.translationStatusApi.getProjectProgress(
       projectId,
       {
-        limit: 200,
+        limit: CROWDIN_API_MAX_LIMIT,
       }
     )
 

src/scripts/crowdin/import/main.ts

@@ -0,0 +1,29 @@
+import { existsSync, mkdirSync } from "fs"
+
+import { DOT_CROWDIN } from "../translations/constants"
+
+import type { BucketsList, SelectionItem, TrackerObject } from "./types"
+import { getImportSelection, handleSummary, processLanguage } from "./utils"
+
+const main = (bucketList: BucketsList) => {
+  console.log("Bucket list:", bucketList)
+
+  // If first time, create directory for user
+  if (!existsSync(DOT_CROWDIN)) mkdirSync(DOT_CROWDIN)
+
+  // Initialize trackers object for summary
+  const trackers: TrackerObject = { emptyBuckets: 0, langs: {} }
+
+  // Filter out empty requests and map selection to usable array
+  const importSelection: SelectionItem[] = getImportSelection(
+    bucketList,
+    trackers
+  )
+
+  // Iterate through each selected language
+  importSelection.forEach((item) => processLanguage(item, trackers))
+
+  handleSummary(importSelection, trackers)
+}
+
+export default main

src/scripts/crowdin/import/types.ts

@@ -0,0 +1,27 @@
+export type BucketsList = Record<string, number[]>
+
+export type LangTrackerEntry = {
+  buckets: number[]
+  jsonCopyCount: number
+  mdCopyCount: number
+  error: string
+}
+
+export type TrackerObject = {
+  emptyBuckets: number
+  langs: Record<string, LangTrackerEntry>
+}
+
+export type SelectionItem = {
+  repoLangCode: string
+  crowdinLangCode: string
+  buckets: number[]
+}
+
+export type SummaryItem = {
+  repoLangCode: string
+  buckets: string[] | number[]
+  jsonCopyCount: number
+  mdCopyCount: number
+  error?: string
+}

src/scripts/crowdin/import/utils.ts

@@ -0,0 +1,170 @@
+import { copyFileSync, existsSync, mkdirSync, readdirSync } from "fs"
+import { join } from "path"
+
+import i18Config from "../../../../i18n.config.json"
+import { INTL_JSON_DIR, TRANSLATIONS_DIR } from "../../../lib/constants"
+import { DOT_CROWDIN } from "../translations/constants"
+
+import { BucketsList, SelectionItem, SummaryItem, TrackerObject } from "./types"
+
+/**
+ * Some language codes used in the repo differ from those used by Crowdin.
+ * This is used to convert any codes that may differ when performing folder lookup.
+ */
+export const getCrowdinCode = (code: string): string =>
+  i18Config.filter((lang) => lang.code === code)?.[0]?.crowdinCode || code
+
+/**
+ * Reads `ls` file contents of `path`, moving .md and .json files
+ * to their corresponding destinations in the repo. Function is called
+ * again recursively for subdirectories.
+ *
+ * @param path An absolute path to the directory being scraped.
+ * @param contentSubpath The subpath deep to the lang-code directory,
+ * used to construct destination for markdown content files
+ * @param repoLangCode Language code used within the repo
+ * @returns void
+ */
+export const scrapeDirectory = (
+  path: string,
+  contentSubpath: string,
+  repoLangCode: string,
+  trackers: TrackerObject
+): void => {
+  if (!existsSync(path)) return
+  const ls: string[] = readdirSync(path).filter(
+    (dir: string) => !dir.startsWith(".")
+  )
+  ls.forEach((item: string) => {
+    const source: string = join(path, item)
+    if (item.endsWith(".json")) {
+      const jsonDestDirPath: string = join(INTL_JSON_DIR, repoLangCode)
+      if (!existsSync(jsonDestDirPath))
+        mkdirSync(jsonDestDirPath, { recursive: true })
+      const jsonDestinationPath: string = join(jsonDestDirPath, item)
+      copyFileSync(source, jsonDestinationPath)
+      // Update .json tracker
+      trackers.langs[repoLangCode].jsonCopyCount++
+    } else if (
+      item.endsWith(".md") ||
+      item.endsWith(".svg") ||
+      item.endsWith(".xlsx")
+    ) {
+      const mdDestDirPath: string = join(
+        TRANSLATIONS_DIR,
+        repoLangCode,
+        contentSubpath
+      )
+      if (!existsSync(mdDestDirPath))
+        mkdirSync(mdDestDirPath, { recursive: true })
+      const mdDestinationPath: string = join(mdDestDirPath, item)
+      copyFileSync(source, mdDestinationPath)
+      // Update .md tracker
+      trackers.langs[repoLangCode].mdCopyCount++
+    } else {
+      // If another directory, recursively call `scrapeDirectory`
+      scrapeDirectory(
+        `${path}/${item}`,
+        `${contentSubpath}/${item}`,
+        repoLangCode,
+        trackers
+      )
+    }
+  })
+}
+
+export const getImportSelection = (
+  buckets: BucketsList,
+  trackers: TrackerObject
+): SelectionItem[] =>
+  Object.keys(buckets)
+    .filter((repoLangCode: string): boolean => {
+      if (!buckets[repoLangCode].length) trackers.emptyBuckets++
+      return !!buckets[repoLangCode].length
+    })
+    .map(
+      (repoLangCode: string): SelectionItem => ({
+        repoLangCode,
+        crowdinLangCode: getCrowdinCode(repoLangCode),
+        buckets: buckets[repoLangCode],
+      })
+    )
+
+/**
+ * ./postLangPRs.ts
+ */
+
+export const processBucket = (
+  bucket: number,
+  crowdinLangCode: string,
+  repoLangCode: string,
+  langLs: string[],
+  trackers: TrackerObject
+): void => {
+  const paddedBucket: string = bucket.toString().padStart(2, "0")
+  let bucketDirName = ""
+  langLs.forEach((bucketName: string) => {
+    bucketDirName += bucketName.startsWith(paddedBucket) ? bucketName : ""
+  })
+  const bucketDirectoryPath: string = `${DOT_CROWDIN}/${crowdinLangCode}/${bucketDirName}`
+  // Initial scrapeDirectory function call
+  scrapeDirectory(bucketDirectoryPath, ".", repoLangCode, trackers)
+  // Update tracker
+  trackers.langs[repoLangCode].buckets.push(bucket)
+}
+
+export const processLanguage = (
+  { repoLangCode, crowdinLangCode, buckets }: SelectionItem,
+  trackers: TrackerObject
+): void => {
+  // Initialize tracker for language
+  trackers.langs[repoLangCode] = {
+    buckets: [],
+    jsonCopyCount: 0,
+    mdCopyCount: 0,
+    error: "",
+  }
+  // Initialize working directory and check for existence
+  const path: string = join(DOT_CROWDIN, crowdinLangCode)
+  if (!existsSync(path)) {
+    trackers.langs[
+      repoLangCode
+    ].error = `Path doesn't exist for lang ${crowdinLangCode}`
+    return
+  }
+  const langLs: string[] = readdirSync(path)
+  // Iterate over each selected bucket, scraping contents with `scrapeDirectory`
+  buckets.forEach((bucket) =>
+    processBucket(bucket, crowdinLangCode, repoLangCode, langLs, trackers)
+  )
+}
+
+export const handleSummary = (
+  selection: SelectionItem[],
+  trackers: TrackerObject
+) => {
+  // Construct console summary
+  const summary: SummaryItem[] = selection.map(
+    (item: SelectionItem): SummaryItem => {
+      const { buckets, repoLangCode } = item
+      const { jsonCopyCount, mdCopyCount, error } = trackers.langs[repoLangCode]
+      return {
+        repoLangCode,
+        buckets,
+        jsonCopyCount,
+        mdCopyCount,
+        error,
+      }
+    }
+  )
+
+  // Print summary logs
+  if (!summary.length) {
+    console.warn(
+      "Nothing imported, see instruction at top of crowdin-imports.ts"
+    )
+    return
+  }
+  console.table(summary)
+  console.log("🎉 Crowdin import complete.")
+}

src/scripts/crowdin/translations/constants.ts

@@ -0,0 +1,15 @@
+import { resolve } from "path"
+
+export const DOT_CROWDIN = ".crowdin"
+
+export const CROWDIN_DATA_DIR = "src/data/crowdin"
+export const SAVE_FILE = "download.zip"
+export const FILE_PATH = resolve(CROWDIN_DATA_DIR, SAVE_FILE)
+
+export const SUMMARY_SAVE_FILE = "import-summary.json"
+export const SUMMARY_PATH = resolve(CROWDIN_DATA_DIR, SUMMARY_SAVE_FILE)
+
+export const BUCKETS_IMPORTED_FILE = "buckets-imported.json"
+export const BUCKETS_PATH = resolve(CROWDIN_DATA_DIR, BUCKETS_IMPORTED_FILE)
+
+export const APPROVAL_THRESHOLD = 100

src/scripts/crowdin/translations/getApprovedBuckets.ts

@@ -0,0 +1,43 @@
+import i18n from "../../../../i18n.config.json"
+import bucketDirs from "../../../data/crowdin/translation-buckets-dirs.json"
+import { CROWDIN_API_MAX_LIMIT } from "../../../lib/constants"
+import crowdin from "../api-client/crowdinClient"
+import type { BucketsList } from "../import/types"
+
+import { APPROVAL_THRESHOLD } from "./constants"
+
+async function getApprovedBuckets(): Promise<BucketsList> {
+  const projectId = Number(process.env.CROWDIN_PROJECT_ID) || 363359
+
+  const bucketsList: BucketsList = {}
+
+  // TODO: Consider regenerating bucketDirs list on each run for fidelity
+  for (const bucketDir of bucketDirs) {
+    const directoryProgress =
+      await crowdin.translationStatusApi.getDirectoryProgress(
+        projectId,
+        bucketDir.id,
+        { limit: CROWDIN_API_MAX_LIMIT }
+      )
+
+    const onlyApproved = directoryProgress.data.filter(
+      ({ data: { approvalProgress } }) => approvalProgress >= APPROVAL_THRESHOLD
+    )
+
+    for (const { code, crowdinCode } of i18n) {
+      const match = onlyApproved.find(
+        ({ data: { languageId } }) => languageId === crowdinCode
+      )
+      if (!match) continue
+
+      if (!bucketsList[code]) bucketsList[code] = []
+
+      const n = parseInt(bucketDir.name.substring(0, 2))
+      bucketsList[code].push(n)
+    }
+  }
+
+  return bucketsList
+}
+
+export default getApprovedBuckets