Merge pull request #285 from segmentio/repo-sync

repo sync

Author: bot-docsteam

.github/Vocab/Docs/accept.txt

@@ -13,6 +13,7 @@
 (?:U|u)rls?\b
 Adwords
 allowlist
+Amberflo
 Appboy
 blocklist
 boolean

.yarn/cache/ansi-colors-npm-4.1.1-97ad42f223-138d04a510.zip

@@ -154,6 +154,10 @@ vendor/bundle:
 update: 
 	@node scripts/update.js
 
+.PHONY: add-id
+add-id:
+	@node scripts/add_id.js
+	
 .PHONY: lint
 lint: node_modules
 	@echo "Checking yml files..."

.yarn/cache/enquirer-npm-2.3.6-7899175762-1c0911e14a.zip

@@ -26,7 +26,7 @@
     "babel-loader": "^8.2.2",
     "concurrently": "^6.2.1",
     "front-matter": "^4.0.2",
-    "glob": "^7.1.7",
+    "glob": "^7.2.0",
     "js-yaml": "^4.1.0",
     "remark": "^14.0.1",
     "remark-cli": "^10.0.0",
@@ -50,6 +50,7 @@
     "check-links": "^1.1.8",
     "clipboard": "^2.0.8",
     "dotenv": "^10.0.0",
+    "enquirer": "^2.3.6",
     "fast-csv": "^4.3.6",
     "glightbox": "^3.1.0",
     "globby": "11.0.4",

.yarn/install-state.gz

@@ -0,0 +1,179 @@
+// Purpose: Add id values to integrations that don't have them
+// Why it's important: We look up integration metadata by ID, rather than slug
+// Instructions: run `make add-id`, select the integration type, enter the slug
+// The script:
+// 1. Get's the list of public integrations from the API
+// 2. Checks the slug you entered for an id
+// 3. If there is no ID, it adds the ID retrieved from the API to the file.
+
+
+
+const axios = require('axios');
+const path = require('path');
+const fs = require('fs');
+const fm = require('front-matter');
+const yaml = require('js-yaml');
+const {
+  Select
+} = require('enquirer');
+const {
+  AutoComplete
+} = require('enquirer');
+
+const {
+  type
+} = require('os');
+const {
+  autocomplete
+} = require('@algolia/autocomplete-js');
+
+require('dotenv').config();
+
+// Here, global variables are set
+const PAPI_URL = "https://api.segmentapis.com"
+const slugOverrides = yaml.load(fs.readFileSync(path.resolve(__dirname, `../src/_data/catalog/slugs.yml`)))
+
+const slugify = (displayName) => {
+  let slug = displayName
+    .toLowerCase()
+    .replace(/\s+/g, '-')
+    .replace('-&-', '-')
+    .replace('/', '-')
+    .replace(/[\(\)]/g, '')
+    .replace('.', '-')
+
+  for (key in slugOverrides) {
+    let original = slugOverrides[key].original
+    let override = slugOverrides[key].override
+
+    if (slug == original) {
+      slug = override
+    }
+  }
+
+  return slug
+}
+
+const getCatalog = async (url, page_token = "MA==") => {
+  try {
+    const res = await axios.get(url, {
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${process.env.PAPI_TOKEN}`
+      },
+      data: {
+        "pagination": {
+          "count": 200,
+          "cursor": page_token
+        }
+      }
+    });
+
+    return res.data
+  } catch (error) {
+    console.log(error)
+  }
+}
+
+
+const updateId = async () => {
+  const type_select = new Select({
+    name: 'int_type',
+    message: 'What type of integration?',
+    choices: ['sources', 'destinations']
+  })
+
+  let int_type = await type_select.run()
+
+
+
+  let nextPageToken = "MA=="
+  let integrations = []
+  let paredIntegrations = []
+
+  while (nextPageToken !== undefined) {
+    const res = await getCatalog(`${PAPI_URL}/catalog/${int_type}/`, nextPageToken)
+    if (int_type == "sources") {
+      integrations = integrations.concat(res.data.sourcesCatalog)
+    } else {
+      integrations = integrations.concat(res.data.destinationsCatalog)
+    }
+    nextPageToken = res.data.pagination.next
+  }
+
+
+
+  integrations.forEach(integration => {
+
+    const urlParse = (slug, type) => {
+      let url = ""
+      if (type == "destinations") {
+        url = `connections/destinations/catalog/${slug}`
+      } else {
+        const libraryCategories = [
+          'server',
+          'mobile',
+          'ott',
+          'roku',
+          'website'
+        ]
+
+        let mainCategory = integration.categories[0] ? integration.categories[0].toLowerCase() : ''
+
+        if (libraryCategories.includes(mainCategory)) {
+          url = `connections/sources/catalog/libraries/${mainCategory}/${slug}`
+        } else {
+          url = `connections/sources/catalog/cloud-apps/${slug}`
+          mainCategory = 'cloud-app'
+        }
+      }
+
+      return url
+    }
+    let slug = slugify(integration.name)
+    let url = urlParse(slug, int_type)
+
+    let updatedIntegration = {
+      id: integration.id,
+      slug,
+      url
+    }
+    paredIntegrations.push(updatedIntegration)
+  })
+
+  let integrationSlugs = paredIntegrations.map(a => a.slug)
+
+  const slug_select = new AutoComplete({
+    name: "slug",
+    message: "Enter the integration slug",
+    limit: 10,
+    initial: 2,
+    choices: integrationSlugs
+
+  })
+
+  let slug_value = await slug_select.run()
+
+  let final = paredIntegrations.find(x => x.slug == slug_value)
+  let itemURL = final.url
+
+  const catalogPath = path.resolve('src/', itemURL, 'index.md')
+  if (fs.existsSync(catalogPath)) {
+    const f = fm(fs.readFileSync(catalogPath, 'utf8'));
+    const fmatter = f.frontmatter
+    const re_id = new RegExp("(id: )\S*")
+    if (!re_id.test(fmatter)) {
+      const attr = `---\n${f.frontmatter}\nid: ${final.id}\n---\n`
+      const body = f.body
+      const content = attr + body
+      fs.writeFileSync(catalogPath, content)
+      console.log(`${final.slug} updated`)
+    } else {
+      console.log("integration already has an ID")
+    }
+  } else {
+    console.log("can't find that integration")
+  }
+}
+
+updateId()

Makefile

@@ -51,14 +51,14 @@ const slugify = (displayName) => {
     .replace(/[\(\)]/g, '')
     .replace('.', '-')
 
-// This is how we handle manual slug overrides right now.
-// If a slug appears in the slugOverrides file, we want to use the 'override' value instead.
+  // This is how we handle manual slug overrides right now.
+  // If a slug appears in the slugOverrides file, we want to use the 'override' value instead.
   for (key in slugOverrides) {
     let original = slugOverrides[key].original
     let override = slugOverrides[key].override
 
     if (slug == original) {
-      console.log(original + " -> " + override)
+      // console.log(original + " -> " + override)
       slug = override
     }
   }
@@ -76,11 +76,17 @@ const addIdToExisting = (integration) => {
     const catalogPath = path.resolve('src', itemURL, 'index.md')
     if (fs.existsSync(catalogPath)) {
       const f = fm(fs.readFileSync(catalogPath, 'utf8'));
-      const attr = `---\n${f.frontmatter}\nid: ${integration.id}\n---\n`
-      const body = f.body
-      const content = attr + body
-      console.log(attr)
-      fs.writeFileSync(catalogPath, content)
+
+      const fmatter = f.frontmatter
+      const re_id = new RegExp("(id: )\S*")
+      if (!re_id.test(fmatter)) {
+        const attr = `---\n${f.frontmatter}\nid: ${integration.id}\n---\n`
+        const body = f.body
+        const content = attr + body
+        console.log(attr)
+        fs.writeFileSync(catalogPath, content)
+      }
+
     }
   } catch (e) {
     console.log(error)
@@ -96,7 +102,7 @@ const updateSources = async () => {
   let sources = []
   let nextPageToken = "MA=="
 
-  while (nextPageToken !== null) {
+  while (nextPageToken !== undefined) {
     const res = await getCatalog(`${PAPI_URL}/catalog/sources/`, nextPageToken)
     sources = sources.concat(res.data.sourcesCatalog)
     nextPageToken = res.data.pagination.next
@@ -119,7 +125,7 @@ const updateSources = async () => {
       url = `connections/sources/catalog/cloud-apps/${slug}`
       mainCategory = 'cloud-app'
     }
-// So, we retrieve and store only the id and the URL, which is defined in the if statement on line 116.
+    // So, we retrieve and store only the id and the URL, which is defined in the if statement on line 116.
     let updatedSource = {
       id: source.id,
       url,
@@ -133,7 +139,7 @@ const updateDestinations = async () => {
   let destinations = []
   let nextPageToken = "MA=="
 
-  while (nextPageToken !== null) {
+  while (nextPageToken !== undefined) {
     const res = await getCatalog(`${PAPI_URL}/catalog/destinations/`, nextPageToken)
     destinations = destinations.concat(res.data.destinationsCatalog)
     nextPageToken = res.data.pagination.next

package.json

@@ -24,8 +24,10 @@ The following dependencies are installed in the function environment by default.
 - [`@sendgrid/mail v7.4.7`](https://www.npmjs.com/package/@sendgrid/mail) exposed as `sendgrid.mail`
 - [`stripe v8.115.0`](https://www.npmjs.com/package/stripe) exposed as `stripe`
 - [`twilio v3.68.0`](https://www.npmjs.com/package/twilio) exposed as `twilio`
+- [`uuidv5 v1.0.0`](https://www.npmjs.com/package/uuidv5) exposed as `uuidv5`
 - [`xml v1.0.1`](https://www.npmjs.com/package/xml) exposed as `xml`
 - [`xml2js v0.4.23`](https://www.npmjs.com/package/xml2js) exposed as `xml2js`
+- [`zlib v1.0.5`](https://www.npmjs.com/package/zlib) exposed as `zlib`
 
 Only the [`crypto` Node.js module](https://nodejs.org/dist/latest-v10.x/docs/api/crypto.html ) is included (exposed as `crypto`). [Other built-in Node.js modules](https://nodejs.org/api/modules.html) are not available.