Merge pull request github#16168 from github/repo-sync

repo sync

Author: Octomerger

.github/workflows/test.yml

@@ -132,6 +132,11 @@ jobs:
       - name: Run build script
         run: npm run build
 
+      - name: Warm possible disk caching
+        env:
+          NODE_ENV: test
+        run: ./script/warm-before-tests.mjs
+
       - name: Run tests
         env:
           DIFF_FILE: get_diff_files.txt

lib/site-data.js

@@ -1,21 +1,48 @@
+import fs from 'fs'
+import os from 'os'
 import path from 'path'
 import flat from 'flat'
 import { get, set } from 'lodash-es'
 import languages from './languages.js'
 import dataDirectory from './data-directory.js'
 import encodeBracketedParentheses from './encode-bracketed-parentheses.js'
 
-const loadSiteDataFromDir = (dir) => ({
-  site: {
-    data: dataDirectory(path.join(dir, 'data'), {
-      preprocess: (dataString) => encodeBracketedParentheses(dataString.trimEnd()),
-      ignorePatterns: [/README\.md$/],
-    }),
-  },
+const TEMP_DIRECTORY = process.env.RUNNER_TEMP || os.tmpdir()
+
+function diskMemoize(prefix, fn) {
+  const useCache = process.env.NODE_ENV !== 'development'
+  return (dir) => {
+    const cacheFileName = `${prefix}.${dir.replace(/[^\w]+/g, '-').toLowerCase() || 'en'}.json`
+    if (useCache) {
+      try {
+        return JSON.parse(fs.readFileSync(cacheFileName, 'utf-8'))
+      } catch (err) {
+        if (!(err.code === 'ENOENT' || err instanceof SyntaxError)) throw err
+      }
+    }
+
+    const result = fn(dir)
+    if (useCache) {
+      fs.writeFileSync(cacheFileName, JSON.stringify(result), 'utf-8')
+      console.log(`Disk-cache miss on ${cacheFileName}`, new Date())
+    }
+    return result
+  }
+}
+
+const loadSiteDataFromDir = diskMemoize(path.join(TEMP_DIRECTORY, 'docs-site-data'), (dir) => {
+  return {
+    site: {
+      data: dataDirectory(path.join(dir, 'data'), {
+        preprocess: (dataString) => encodeBracketedParentheses(dataString.trimEnd()),
+        ignorePatterns: [/README\.md$/],
+      }),
+    },
+  }
 })
 
 export default function loadSiteData() {
-  // load english site data
+  // load English site data
   const siteData = {
     en: loadSiteDataFromDir(languages.en.dir),
   }

script/warm-before-tests.mjs

@@ -0,0 +1,22 @@
+#!/usr/bin/env node
+
+// [start-readme]
+//
+// It runs the warmServer() function because that function can do things
+// like writing to disk as a caching mechanism.
+// When jest runs tests, it starts multiple concurrent processes,
+// even if it runs it serially (`--runInBand`) so it's highly likely
+// that two concurrent processes both attempt to writing to
+// the same exact file. By running this script before anything
+// begins, we can be certain that files that should have been created
+// are created.
+//
+// [end-readme]
+
+import warmServer from '../lib/warm-server.js'
+
+main()
+
+async function main() {
+  await warmServer()
+}

tests/content/category-pages.js

@@ -17,13 +17,7 @@ const slugger = new GithubSlugger()
 const contentDir = path.join(__dirname, '../../content')
 
 describe('category pages', () => {
-  let siteData
-
-  beforeAll(async () => {
-    // Load the English site data
-    const allSiteData = await loadSiteData()
-    siteData = allSiteData.en.site
-  })
+  const siteData = loadSiteData().en.site
 
   const walkOptions = {
     globs: ['*/index.md', 'enterprise/*/index.md'],

tests/content/glossary.js

@@ -1,10 +1,7 @@
 import loadSiteData from '../../lib/site-data.js'
 
 describe('glossaries', () => {
-  let glossaries
-  beforeAll(async () => {
-    glossaries = (await loadSiteData()).en.site.data.glossaries
-  })
+  const glossaries = loadSiteData().en.site.data.glossaries
 
   test('are broken into external, internal, and candidates', async () => {
     const keys = Object.keys(glossaries)
@@ -41,7 +38,7 @@ describe('glossaries', () => {
   })
 
   test('non-English external glossary is in correct order', async () => {
-    const vals = (await loadSiteData()).ja.site.data.glossaries.external
+    const vals = loadSiteData().ja.site.data.glossaries.external
     vals.forEach((val, i) => {
       expect(val.term.localeCompare(vals[i + 1], 'ja')).toBeGreaterThan(0)
     })

tests/content/site-data-references.js

@@ -11,17 +11,12 @@ import { jest } from '@jest/globals'
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 
+const siteData = loadSiteData()
+const pages = (await loadPages()).filter((page) => page.languageCode === 'en')
+
 describe('data references', () => {
   jest.setTimeout(60 * 1000)
 
-  let data, pages
-
-  beforeAll(async () => {
-    data = await loadSiteData()
-    pages = await loadPages()
-    pages = pages.filter((page) => page.languageCode === 'en')
-  })
-
   test('every data reference found in English content files is defined and has a value', () => {
     let errors = []
     expect(pages.length).toBeGreaterThan(0)
@@ -30,7 +25,7 @@ describe('data references', () => {
       const file = path.join('content', page.relativePath)
       const pageRefs = getDataReferences(page.markdown)
       pageRefs.forEach((key) => {
-        const value = get(data.en, key)
+        const value = get(siteData.en, key)
         if (typeof value !== 'string') errors.push({ key, value, file })
       })
     })
@@ -50,7 +45,7 @@ describe('data references', () => {
         const { data: metadata } = frontmatter(fileContents, { filepath: page.fullPath })
         const metadataRefs = getDataReferences(JSON.stringify(metadata))
         metadataRefs.forEach((key) => {
-          const value = get(data.en, key)
+          const value = get(siteData.en, key)
           if (typeof value !== 'string') errors.push({ key, value, metadataFile })
         })
       })
@@ -62,7 +57,7 @@ describe('data references', () => {
 
   test('every data reference found in English reusable files is defined and has a value', async () => {
     let errors = []
-    const allReusables = data.en.site.data.reusables
+    const allReusables = siteData.en.site.data.reusables
     const reusables = Object.values(allReusables)
     expect(reusables.length).toBeGreaterThan(0)
 
@@ -78,7 +73,7 @@ describe('data references', () => {
         const reusableRefs = getDataReferences(JSON.stringify(reusablesPerFile))
 
         reusableRefs.forEach((key) => {
-          const value = get(data.en, key)
+          const value = get(siteData.en, key)
           if (typeof value !== 'string') errors.push({ key, value, reusableFile })
         })
       })
@@ -90,7 +85,7 @@ describe('data references', () => {
 
   test('every data reference found in English variable files is defined and has a value', async () => {
     let errors = []
-    const allVariables = data.en.site.data.variables
+    const allVariables = siteData.en.site.data.variables
     const variables = Object.values(allVariables)
     expect(variables.length).toBeGreaterThan(0)
 
@@ -106,7 +101,7 @@ describe('data references', () => {
         const variableRefs = getDataReferences(JSON.stringify(variablesPerFile))
 
         variableRefs.forEach((key) => {
-          const value = get(data.en, key)
+          const value = get(siteData.en, key)
           if (typeof value !== 'string') errors.push({ key, value, variableFile })
         })
       })

tests/content/site-data.js

@@ -1,21 +1,17 @@
 import { fileURLToPath } from 'url'
 import path from 'path'
-import fs from 'fs'
 import { get, isPlainObject, has } from 'lodash-es'
 import flat from 'flat'
+import walkSync from 'walk-sync'
 import { ParseError } from 'liquidjs'
 import loadSiteData from '../../lib/site-data.js'
 import patterns from '../../lib/patterns.js'
 import { liquid } from '../../lib/render-content/index.js'
-import walkSync from 'walk-sync'
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 
 describe('siteData module (English)', () => {
-  let data
-  beforeAll(async () => {
-    data = await loadSiteData()
-  })
+  const data = loadSiteData()
 
   test('makes an object', async () => {
     expect(isPlainObject(data)).toBe(true)
@@ -49,21 +45,6 @@ describe('siteData module (English)', () => {
     expect(reusable.includes('任意のページの左上で')).toBe(true)
   })
 
-  test('backfills missing translated site data with English values', async () => {
-    const newFile = path.join(__dirname, '../../data/newfile.yml')
-    fs.writeFileSync(newFile, 'newvalue: bar')
-    try {
-      const data = loadSiteData()
-      expect(get(data, 'en.site.data.newfile.newvalue')).toEqual('bar')
-      expect(get(data, 'ja.site.data.newfile.newvalue')).toEqual('bar')
-    } finally {
-      // If an error is thrown above, it will still "bubble up"
-      // to the jest reporter, but we still always need to clean up
-      // the temporary file.
-      fs.unlinkSync(newFile)
-    }
-  })
-
   test('all Liquid tags are valid', async () => {
     const dataMap = flat(data)
     for (const key in dataMap) {

tests/unit/liquid.js

@@ -191,11 +191,7 @@ describe('liquid template parser', () => {
     // Create a fake req so we can test the feature versions middleware
     const req = { language: 'en', query: {} }
 
-    let siteData
-    beforeAll(async () => {
-      const allSiteData = await loadSiteData()
-      siteData = allSiteData.en.site
-    })
+    const siteData = loadSiteData().en.site
 
     test('does not render in FPT because feature is not available in FPT', async () => {
       req.context = {