segmentio
diff --git a/‎.github/workflows/check-links-external.yml
Lines changed: 49 additions & 0 deletions b/‎.github/workflows/check-links-external.yml
Lines changed: 49 additions & 0 deletions
diff --git a/‎.github/workflows/check-links.yml renamed to ‎.github/workflows/check-links-internal.yml
Lines changed: 19 additions & 15 deletions b/‎.github/workflows/check-links.yml renamed to ‎.github/workflows/check-links-internal.yml
Lines changed: 19 additions & 15 deletions
diff --git a/‎.vscode/yml.code-snippets
Lines changed: 5 additions & 1 deletion b/‎.vscode/yml.code-snippets
Lines changed: 5 additions & 1 deletion
diff --git a/‎Makefile
Lines changed: 23 additions & 7 deletions b/‎Makefile
Lines changed: 23 additions & 7 deletions
diff --git a/‎netlify.toml
Lines changed: 5 additions & 1 deletion b/‎netlify.toml
Lines changed: 5 additions & 1 deletion
diff --git a/‎package.json
Lines changed: 6 additions & 0 deletions b/‎package.json
Lines changed: 6 additions & 0 deletions
diff --git a/‎scripts/catalog_papi.js
Lines changed: 9 additions & 1 deletion b/‎scripts/catalog_papi.js
Lines changed: 9 additions & 1 deletion
diff --git a/‎scripts/checklinks-external.js
Lines changed: 75 additions & 0 deletions b/‎scripts/checklinks-external.js
Lines changed: 75 additions & 0 deletions
@@ -0,0 +1,49 @@
+name: CheckLinks
+
+# **What it does**: Runs a weekly check for broken external links.
+# **Why we have it**: We want to make sure that pages we link to work.
+# **Who does it impact**: Everyone
+
+
+
+on:
+  schedule:
+    - cron: "5 4 * * SUN"
+
+jobs:
+  checklinks-external:
+    if: github.repository == 'segmentio/segment-docs'
+    name: Linux
+    runs-on: ubuntu-latest
+    env:
+        GITHUB_TOKEN: ${{ secrets.DOCS_BOT_PAT_REPO_SCOPE }}
+        REPORT_AUTHOR: docsbot
+        REPORT_LABEL: report, automated issue, broken links
+        REPORT_REPOSITORY: segmentio/segment-docs
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+      - name: Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.6 # Not needed with a .ruby-version file
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - name: Yarn install
+        run: yarn install
+      - name: Build jekyll website with
+        run: bundle exec jekyll build
+      - name: Run Script
+        run: |
+          sudo make linkcheck-external > broken_links.md
+      - if: ${{ failure() }}
+        name: Create issue from file
+        id: broken-link-report
+        uses: peter-evans/create-issue-from-file@b4f9ee0a9d4abbfc6986601d9b1a4f8f8e74c77e
+        with:
+          token: ${{ env.GITHUB_TOKEN }}
+
+          title: Broken Links Report - External
+          content-filepath: ./broken_links.md
+          repository: ${{ env.REPORT_REPOSITORY }}
+          labels: ${{ env.REPORT_LABEL }}
@@ -11,10 +11,15 @@ on:
     - cron: "5 4 * * SAT"
 
 jobs:
-  checklinks:
+  checklinks-internal:
     if: github.repository == 'segmentio/segment-docs'
     name: Linux
     runs-on: ubuntu-latest
+    env:
+        GITHUB_TOKEN: ${{ secrets.DOCS_BOT_PAT_REPO_SCOPE }}
+        REPORT_AUTHOR: docsbot
+        REPORT_LABEL: report, automated issue, broken links
+        REPORT_REPOSITORY: segmentio/segment-docs
     strategy:
       fail-fast: false
     steps:
@@ -28,18 +33,17 @@ jobs:
         run: yarn install
       - name: Build jekyll website with
         run: bundle exec jekyll build
-      - name: Link Checker
-        uses: lycheeverse/[email protected]
+      - name: Run Script
+        run: |
+          sudo make linkcheck-internal > broken_links.md
+      - if: ${{ failure() }}
+        name: Create issue from file
+        id: broken-link-report
+        uses: peter-evans/create-issue-from-file@b4f9ee0a9d4abbfc6986601d9b1a4f8f8e74c77e
         with:
-          args: --no-progress _site/**/*.html -a 429 -c ./lychee.toml
-        env:
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
-      # - name: "exclude non-400 errors"
-      #   run: |
-      #     sudo grep -v "error sending request" ./lychee/out.md > ./lychee/out2.md
-      - name: Create Issue From File
-        uses: peter-evans/create-issue-from-file@v3
-        with:
-          title: Link Checker Report
-          content-filepath: ./lychee/out.md
-          labels: report, automated issue
+          token: ${{ env.GITHUB_TOKEN }}
+
+          title: Broken Links Report - Local
+          content-filepath: ./broken_links.md
+          repository: ${{ env.REPORT_REPOSITORY }}
+          labels: ${{ env.REPORT_LABEL }}
@@ -20,5 +20,9 @@
 		"prefix": ["- name"],
 		"body":["- name: ${1:name}\r\tdescription: ${2:description}\r\tdefault: ${3}"],
 		"description": "Yaml settings"
-	}
+	},
+	"ext_link":{
+		"prefix": ["{:}"],
+		"body":["{:target='${1:_blank}'}"]
+	},
 }
@@ -29,13 +29,19 @@ intialize-work-dir:
 
 .PHONY: build
 build: node_modules vendor/bundle
-	@echo "Jekyll env: ${JEKYLL_ENV}"
-	@chown -R jekyll /workdir
-	@chmod -R 777 /workdir
-	@echo "env: ${JEKYLL_ENV}"
-	@$(BIN)/webpack --mode=production
-	@JEKYLL_ENV=${JEKYLL_ENV} bundle exec jekyll build --trace
-	@if [ '${BUILDKITE_BRANCH}' == 'staging' ]; then echo "updating sitemap.xml..." && sed -i -r 's/segment.com/segment.build/g' ./_site/sitemap.xml; fi;
+	@$(BIN)/concurrently --raw --kill-others -n webpack,jekyll \
+		"$(BIN)/webpack --mode=development --watch" \
+		"bundle exec jekyll clean && bundle exec jekyll build -V"
+
+# .PHONY: build
+# build: node_modules vendor/bundle
+# 	@echo "Jekyll env: ${JEKYLL_ENV}"
+# 	@chown -R jekyll /workdir
+# 	@chmod -R 777 /workdir
+# 	@echo "env: ${JEKYLL_ENV}"
+# 	@$(BIN)/webpack --mode=production
+# 	@JEKYLL_ENV=${JEKYLL_ENV} bundle exec jekyll build --trace
+# 	@if [ '${BUILDKITE_BRANCH}' == 'staging' ]; then echo "updating sitemap.xml..." && sed -i -r 's/segment.com/segment.build/g' ./_site/sitemap.xml; fi;
 
 .PHONY: upload-docs
 upload-docs:
@@ -82,6 +88,16 @@ changelog: vendor/bundle
 sidenav: vendor/bundle
 	@node scripts/nav.js
 
+# check internal links
+.PHONY: linkcheck-internal
+linkcheck-internal:
+	@node scripts/checklinks-internal.js
+
+# check external links
+.PHONY: linkcheck-external
+linkcheck-external:
+	@node scripts/checklinks-external.js
+
 .PHONY: zip-artifacts
 zip-artifacts:
 	@tar czf build_package.tar.gz _site
 
@@ -18,7 +18,6 @@
 [context.develop]
   command = "yarn develop"
 
-
 [[redirects]]
   from = "/docs/*"
   to = "/:splat"
@@ -29,3 +28,8 @@
   to = "/assets/docs.bundle.js"
   status = 200
 
+[[plugins]]
+  package = "netlify-plugin-jekyll-cache"
+
+	[plugins.inputs]
+  	jekyllSource = "/src"
@@ -47,10 +47,16 @@
     "ajv": "^6.10.2",
     "algoliasearch": "^4.10.5",
     "ansi-regex": "^6.0.1",
+    "browser-sync": "^2.27.7",
+    "check-links": "^1.1.8",
     "clipboard": "^2.0.8",
     "dotenv": "^10.0.0",
     "glightbox": "^3.1.0",
+    "globby": "11.0.4",
     "handlebars": "^4.7.7",
+    "ora": "5.4.1",
+    "posthtml": "^0.16.5",
+    "posthtml-urls": "^1.0.0",
     "search-insights": "^2.0.3",
     "tap-spot": "^1.1.1",
     "tippy.js": "5.2.0",
 
@@ -118,7 +118,6 @@ const getConnectionModes = (destination) => {
     }
     fs.mkdirSync(docsPath)
     fs.writeFileSync(`${docsPath}/index.md`, content)
-    fs.appendFileSync('src/_data/catalog/incompleteDocs.txt', `${docsPath}\n`)
   }
 }
 
@@ -167,6 +166,7 @@ const updateSources = async () => {
   const hiddenSources = [
     'amp',
     'factual-engine',
+    'twilio-event-streams-beta'
   ]
 
   sources.forEach(source => {
@@ -197,6 +197,7 @@ const updateSources = async () => {
 
     // create the catalog metadata
     let updatedSource = {
+      id: source.id,
       display_name: source.name,
       slug,
       url,
@@ -272,7 +273,14 @@ const updateDestinations = async () => {
 
 
   destinations.forEach(destination => {
+    
+    // We need to be able to keep the system slug in some cases.
+    const slugOverrides = ['actions-google-enhanced-conversions', 'actions-google-analytics-4', 'actions-facebook-conversions-api']
     let slug = slugify(destination.name)
+    if (slugOverrides.includes(destination.slug)) {
+      slug = destination.slug
+    }
+    
 
     // Flip the slug of Actions destinations
     const actionsDests = [
 
@@ -0,0 +1,75 @@
+const globby = require('globby')
+const posthtml = require('posthtml')
+const fs = require('fs')
+const server = require('browser-sync').create()
+const checkLinks = require('check-links')
+const {
+  $dataMetaSchema
+} = require('ajv')
+const ora = require('ora')
+
+
+
+const checkForDeadExternalUrls = async () => {
+  try {
+    const files = await globby('_site/**/*.html')
+    const throbber = ora('Link Check Starting').start()
+    const urls = new Set()
+
+    const ph = posthtml([
+      require('posthtml-urls')({
+        eachURL: (url) => {
+          if (!url.startsWith('http://0') && !url.startsWith('/') && !url.startsWith('https://github.com/segmentio'))  {
+            urls.add(url)
+          }
+        },
+      }),
+    ])
+    throbber.succeed()
+    throbber.start('Processing files')
+    
+    files.forEach((file) => {
+      ph.process(fs.readFileSync(file))
+    })
+    throbber.succeed()
+    throbber.start('Starting server')
+    await new Promise((resolve) => {
+      server.init({
+          port: 3000,
+          server: {
+            baseDir: '_site',
+          },
+          open: false,
+          logLevel: 'silent',
+        },
+        resolve,
+      )
+      throbber.succeed()
+    })
+    
+    throbber.start('Checking the links')
+    const results = await checkLinks(
+      Array.from(urls).map((url) =>
+        url
+      ),
+    )
+    const deadUrls = Array.from(urls).filter(
+      (url) => results[url].status === 'dead',
+    )
+
+    if (deadUrls.length > 0) {
+      throbber.fail(`Dead URLS: ${deadUrls.length}\n\n`)
+      console.log(`Dead URLS: ${deadUrls.length}\n\n${deadUrls.join('\n')}`)
+      process.exit(1)
+    } else {
+      console.log('All links work!')
+      process.exit
+    }
+    throbber.stop()
+    server.exit()
+  } catch (e) {
+    console.error(e)
+    server.exit()
+  }
+}
+checkForDeadExternalUrls()