A GitHub Actions workflow to update the lists of templates and cloud provider actions weekly or manually (DOC-146) (#282)

Author: gorohoroh

.github/workflows/update-templates-and-cloud-provider-actions.yml

@@ -0,0 +1,78 @@
+name: Update Templates And Cloud Provider Actions
+
+on:
+  schedule:
+    - cron: '45 15 * * 3'
+  workflow_dispatch:
+
+jobs:
+  update-templates-and-actions:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/[email protected]
+        with:
+          ref: ${{ github.ref_name }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Generate templates list
+        run: node scripts/generate-list-of-templates.js
+        env:
+          OPENOPS_USERNAME: ${{ secrets.OPENOPS_USERNAME }}
+          OPENOPS_PASSWORD: ${{ secrets.OPENOPS_PASSWORD }}
+
+      - name: Generate cloud provider actions list
+        run: node scripts/generate-lists-of-cloud-provider-actions.js
+
+      - name: Check for changes
+        id: changes
+        run: |
+          if git status --porcelain | grep .; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+            echo "No changes detected. Exiting successfully."
+          fi
+
+      - name: Stop if no changes
+        if: steps.changes.outputs.changed == 'false'
+        run: exit 0
+
+      - name: Create update branch
+        id: branch
+        run: |
+          TS=$(date -u +"%Y-%m-%d-%H-%M")
+          BRANCH="updated-templates-and-actions-$TS"
+          echo "branch=$BRANCH" >> "$GITHUB_OUTPUT"
+          git checkout -b "$BRANCH"
+
+      - name: Commit and push changes
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          git add .
+          git commit -m "Templates and cloud provider actions updated as of $(date -u +'%Y-%m-%d')"
+          git push origin HEAD
+
+      - name: Create pull request
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh pr create \
+            --head "${{ steps.branch.outputs.branch }}" \
+            --base "${{ github.ref_name }}" \
+            --title "Update templates and cloud provider actions" \
+            --body "Automated update of templates and cloud provider actions." \
+            --reviewer gorohoroh

scripts/generate-list-of-templates.js

@@ -1,8 +1,8 @@
-const credentials = require('./credentials.json');
 const fs = require('fs/promises');
 const path = require('path');
 
-const API_URL = 'https://app.openops.com/api/v1/cloud-templates?version=9999';
+const AUTH_URL = 'https://app.openops.com/api/v1/authentication/sign-in';
+const TEMPLATES_URL = 'https://app.openops.com/api/v1/flow-templates?version=9999';
 const OUT_RELATIVE = '../snippets/templates-generated.mdx';
 
 const caseInsensitive = () => (a, b) => a.localeCompare(b, undefined, {sensitivity: 'base'});
@@ -53,27 +53,56 @@ const writeMarkdownToFile = async (lines, stats) => {
     console.log(`Wrote ${stats.categories} categories, ${stats.names} template names to ${outPath}`);
 };
 
-const main = async () => {
+const getAuthToken = async () => {
+    let credentialsFile = {};
     try {
-    const res = await fetch(API_URL, {
-      headers: {
-        Accept: 'application/json',
-        Cookie: credentials.cookie
-      }
+        credentialsFile = require('./credentials.json');
+    } catch {
+        console.log('No credentials.json found. Using environment variables instead.');
+    }
+
+    const username = process.env.OPENOPS_USERNAME || credentialsFile.username;
+    const password = process.env.OPENOPS_PASSWORD || credentialsFile.password;
+
+    const auth = await fetch(AUTH_URL, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json"
+        },
+        body: JSON.stringify({
+                "email": username,
+                "password": password
+            }
+        )
     });
 
-    if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
+    if (!auth.ok) throw new Error(`Unable to authenticate with app.openops.com: error ${auth.status}, ${auth.statusText}`);
 
-    const templates = await res.json();
-    if (!Array.isArray(templates)) throw new Error('Unexpected API response: expected array');
+    return (await auth.json()).token;
+};
 
-    const categoryMap = groupTemplatesByCategory(templates);
-    const {lines, stats} = prepareMarkdownLines(categoryMap, caseInsensitive);
-    await writeMarkdownToFile(lines, stats);
-  } catch (err) {
-    console.error('Error:', err?.message ?? err);
-    process.exitCode = 1;
-  }
+const main = async () => {
+    try {
+        const token = await getAuthToken()
+        const res = await fetch(TEMPLATES_URL, {
+            headers: {
+                Accept: 'application/json',
+                Authorization: `Bearer ${token}`
+            }
+        });
+
+        if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
+
+        const templates = await res.json();
+        if (!Array.isArray(templates)) throw new Error('Unexpected API response: expected array');
+
+        const categoryMap = groupTemplatesByCategory(templates);
+        const {lines, stats} = prepareMarkdownLines(categoryMap, caseInsensitive);
+        await writeMarkdownToFile(lines, stats);
+    } catch (err) {
+        console.error('Error:', err?.message ?? err);
+        process.exitCode = 1;
+    }
 };
 
 main();

scripts/generate-lists-of-cloud-provider-actions.js

@@ -1,4 +1,3 @@
-const credentials = require('./credentials.json');
 const fs = require('fs/promises');
 const path = require('path');
 
@@ -28,13 +27,7 @@ const writeMarkdownToFile = async (lines, cloudProvider, number) => {
 const main = async () => {
     try {
         for (const cloudProvider of CLOUD_PROVIDERS) {
-            const res = await fetch(API_BASE_URL + cloudProvider.url, {
-                headers: {
-                    Accept: 'application/json',
-                    Cookie: credentials.cookie
-                }
-            });
-
+            const res = await fetch(API_BASE_URL + cloudProvider.url);
             if (!res.ok) throw new Error(`Fetch failed: ${res.status}`);
 
             const block = await res.json();