refactor(ui-scripts): refactor scss generator code and add debug logging

The debug statements help to investigate flaky CI builds that fail with "JSON5: invalid end of input
at 1:1"
Also remove 'fse', since the erro seems to be that 'styleDictionarySource' is not created  in some
cases

Author: matyasf

package-lock.json

@@ -23,6 +23,7 @@
  */
 
 import path from 'path'
+import fs from 'fs'
 import { error } from '@instructure/command-utils'
 import { handleMapJSTokensToSource } from '../utils/handle-map-js-tokens-to-source.js'
 import { handleGenerateTokens } from '../utils/handle-generate-tokens.js'
@@ -90,7 +91,21 @@ export default {
         ? path.join(themePath, outputDir, themeKey)
         : path.join(themePath, outputDir)
       const sourcePath = path.join(outputPath, 'source.json')
-
+      // this is just a debug code to find the JSON error here that occurs only in CI
+      // eslint-disable-next-line no-console
+      console.log(
+        '!!!!! starting',
+        themeKey,
+        sourceTokens,
+        outputPath,
+        sourcePath,
+        fs.existsSync(sourcePath)
+      )
+      // eslint-disable-next-line no-console
+      console.log(
+        '!!!! styleDictionarySource',
+        JSON.stringify(styleDictionarySource)?.substring(0, 400)
+      )
       generators.push(
         handleGenerateTokens({
           themeKey,

packages/ui-scripts/lib/build/generate-all-tokens.js

@@ -22,53 +22,61 @@
  * SOFTWARE.
  */
 
-import fse from 'fs-extra'
 import path from 'path'
 import StyleDictionary from 'style-dictionary'
+import fs from 'fs'
 
 export async function handleGenerateTokens({
   themeKey,
   sourcePath,
   styleDictionarySource,
   outputPath
 }) {
-  return fse
-    .outputFile(sourcePath, JSON.stringify(styleDictionarySource))
-    .then(() => {
-      const dictionary = new StyleDictionary({
-        source: [sourcePath],
-        platforms: {
-          scss: {
-            transformGroup: 'scss',
-            prefix: 'instui-' + themeKey,
-            buildPath: path.join(outputPath, 'scss/'),
-            files: [
-              {
-                destination: '_variables.scss',
-                format: 'scss/variables'
-              }
-            ]
+  const dir = path.dirname(sourcePath)
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true })
+  }
+  fs.writeFileSync(sourcePath, JSON.stringify(styleDictionarySource))
+  // this is just a debug code to find the JSON error here that occurs only in CI
+  // eslint-disable-next-line no-console
+  console.log(
+    '!!!! handleGenerateTokens',
+    sourcePath,
+    fs.existsSync(sourcePath),
+    JSON.stringify(styleDictionarySource)?.substring(0, 400)
+  )
+  const dictionary = new StyleDictionary({
+    source: [sourcePath],
+    platforms: {
+      scss: {
+        transformGroup: 'scss',
+        prefix: 'instui-' + themeKey,
+        buildPath: path.join(outputPath, 'scss/'),
+        files: [
+          {
+            destination: '_variables.scss',
+            format: 'scss/variables'
           }
-          // TODO: Revisit the best way to distribute tokens for vanilla JS
-          // js: {
-          //   transformGroup: 'js',
-          //   buildPath: path.join(outputPath, 'js/'),
-          //   files: [
-          //     {
-          //       destination: 'umd.js',
-          //       format: 'javascript/umd'
-          //     },
-          //     {
-          //       destination: 'es6.js',
-          //       format: 'javascript/es6'
-          //     }
-          //   ]
-          // }
-        }
-      })
-      return dictionary.buildAllPlatforms()
-    })
-    .then((builds) => {
-      return builds
-    })
+        ]
+      }
+      // TODO: Revisit the best way to distribute tokens for vanilla JS
+      // js: {
+      //   transformGroup: 'js',
+      //   buildPath: path.join(outputPath, 'js/'),
+      //   files: [
+      //     {
+      //       destination: 'umd.js',
+      //       format: 'javascript/umd'
+      //     },
+      //     {
+      //       destination: 'es6.js',
+      //       format: 'javascript/es6'
+      //     }
+      //   ]
+      // }
+    }
+  })
+  dictionary.buildAllPlatforms().then((builds) => {
+    return builds
+  })
 }

packages/ui-scripts/lib/utils/handle-generate-tokens.js

@@ -26,15 +26,14 @@
     "@lerna/project": "^6.4.1",
     "dotenv": "^16.4.7",
     "find-up": "^7.0.0",
-    "fs-extra": "^11.2.0",
     "http-server": "^14.1.1",
     "jscodeshift": "^0.16.1",
     "karma-cli": "^2.0.0",
     "lerna": "^8.1.9",
     "lodash.isplainobject": "^4.0.6",
     "mocha": "^10.7.3",
     "semver": "^7.6.3",
-    "style-dictionary": "4.1.2",
+    "style-dictionary": "4.4.0",
     "webpack-cli": "^5.1.4",
     "webpack-dev-server": "^5.1.0",
     "yargs": "^17.7.2"