Merge pull request #56 from NHSDigital/feature/CCM-8881-exclude-tool-versions-from-repo-sync

CCM-8881: Extend sync script to merge some config files rather than replacing them

Author: m-houston

.github/workflows/cicd-1-pull-request.yaml

@@ -40,9 +40,9 @@ jobs:
           echo "build_datetime=$datetime" >> $GITHUB_OUTPUT
           echo "build_timestamp=$(date --date=$datetime -u +'%Y%m%d%H%M%S')" >> $GITHUB_OUTPUT
           echo "build_epoch=$(date --date=$datetime -u +'%s')" >> $GITHUB_OUTPUT
-          echo "nodejs_version=$(grep "^nodejs " .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
-          echo "python_version=$(grep "^python " .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
-          echo "terraform_version=$(grep "^terraform " .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
+          echo "nodejs_version=$(grep "^nodejs\s" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
+          echo "python_version=$(grep "^python\s" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
+          echo "terraform_version=$(grep "^terraform\s" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
           echo "version=$(echo $version)" >> $GITHUB_OUTPUT
           echo "is_version_prerelease=$(if [[ $version == *-* ]]; then echo "true"; else echo "false"; fi)" >> $GITHUB_OUTPUT
 

.github/workflows/cicd-3-deploy.yaml

@@ -45,9 +45,9 @@ jobs:
           echo "build_datetime=$datetime" >> $GITHUB_OUTPUT
           echo "build_timestamp=$(date --date=$datetime -u +'%Y%m%d%H%M%S')" >> $GITHUB_OUTPUT
           echo "build_epoch=$(date --date=$datetime -u +'%s')" >> $GITHUB_OUTPUT
-          echo "nodejs_version=$(grep "^nodejs " .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
-          echo "python_version=$(grep "^python " .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
-          echo "terraform_version=$(grep "^terraform " .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
+          echo "nodejs_version=$(grep "^nodejs\s" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
+          echo "python_version=$(grep "^python\s" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
+          echo "terraform_version=$(grep "^terraform\s" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
           # TODO: Get the version, but it may not be the .version file as this should come from the CI/CD Pull Request Workflow
           echo "version=$(head -n 1 .version 2> /dev/null || echo unknown)" >> $GITHUB_OUTPUT
           # echo "tag=${{ github.event.inputs.tag }}" >> $GITHUB_OUTPUT

scripts/config/.repository-template-sync-ignore

@@ -1,5 +1,4 @@
 # Files and folders to ignore when syncing nhs-notify-repository-template back in to this repository
-scripts/config/.repository-template-sync-ignore
 .github/workflows/
 nhs-notify-repository-template/
 

scripts/config/.repository-template-sync-merge

@@ -0,0 +1,7 @@
+# Files and folders to merge when syncing nhs-notify-repository-template back in to this repository
+scripts/config/.repository-template-sync-ignore
+scripts/config/.repository-template-sync-merge
+.tool-versions
+.gitignore
+scripts/config/vale/styles/config/vocabularies/words/accept.txt
+scripts/config/vale/styles/config/vocabularies/words/reject.txt

scripts/config/sonar-scanner.properties

@@ -7,3 +7,4 @@ sonar.sources=.
 
 #sonar.python.coverage.reportPaths=.coverage/coverage.xml
 #sonar.[javascript|typescript].lcov.reportPaths=.coverage/lcov.info
+sonar.coverage.exclusions=scripts/**/*

scripts/githooks/sync-template-repo.sh

@@ -5,21 +5,24 @@ set -euo pipefail
 # Script to synchronise the nhs-notify-template-repository with this repository
 #
 # Usage:
-#   $ [options] ./check-terraform-format.sh
+#   $ [options] ./sync-template-repo.sh
 #
 # Options:
 #   new_only=true      # Only identify new files from the template-repository
 #   changes_only=true  # Only identify files which have drifted from the template-repository
 
 # ==============================================================================
 
-# Command line prameters
+scriptdir=$(realpath "$(dirname "$0")")
+
+# Command line parameters
 new_only=${new_only:-false}
 changes_only=${changes_only:-false}
 
 # Set variables
 TEMPLATE_REPO_DIR="nhs-notify-repository-template"
 IGNORE_FILE="scripts/config/.repository-template-sync-ignore"
+MERGE_FILE="scripts/config/.repository-template-sync-merge"
 
 # Check if the template directory exists
 if [ ! -d "${TEMPLATE_REPO_DIR}" ]; then
@@ -34,11 +37,21 @@ if [ ! -f "${IGNORE_FILE}" ]; then
   echo "# Files and Folders in the template repository to disregard" >> ${IGNORE_FILE}
 fi
 
+# Check if the .template-merge file exists, create an empty one if not
+if [ ! -f "${MERGE_FILE}" ]; then
+  echo "# Files and folders to merge when syncing ${TEMPLATE_REPO_DIR} back in to this repository" > ${MERGE_FILE}
+fi
+
 # Read the .template-ignore file into an array
 while IFS= read -r line || [ -n "$line" ]; do
   IGNORED_PATHS+=("$line")
 done < "$IGNORE_FILE"
 
+# Read the .template-merge file into an array
+while IFS= read -r line || [ -n "$line" ]; do
+  MERGED_PATHS+=("$line")
+done < "$MERGE_FILE"
+
 # Check if a file is ignored.
 is_ignored() {
   local file=${1}
@@ -56,13 +69,24 @@ is_ignored() {
   return 1
 }
 
+is_merge() {
+  local file=${1}
+
+  for merged in "${MERGED_PATHS[@]}"; do
+    if [[ -n "$merged" && "$file" =~ $merged ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
 # Navigate to the template directory
 cd "${TEMPLATE_REPO_DIR}" || exit
 FILES_ADDED=()
 FILES_WITH_CHANGES=()
 
 # Loop through all files in the template directory
-while IFS= read -r -d '' file; do
+while IFS= read -r -d '' file || [[ -n $file ]]; do
   relative_path="${file#./}"  # Remove leading './'
 
   # Check if the file is ignored
@@ -84,27 +108,38 @@ while IFS= read -r -d '' file; do
     # If the file exists, check if it's different
     if [ "$new_only" == false ]; then
       if ! diff -q "$file" "$target_path" > /dev/null 2>&1; then
-        echo "Merging changes from $relative_path"
-        FILES_WITH_CHANGES+=("${relative_path}")
-        cp "$file" "$target_path"
+        if is_merge "$relative_path"; then
+          echo "Merging changes from $relative_path"
+          cp "$target_path" "${target_path}.bak"
+          node "${scriptdir}/../maintenance/merge.js" "$target_path" "$file" > "${target_path}.merged"
+          if ! cmp -s "${target_path}.merged" "${target_path}.bak"; then
+            FILES_WITH_CHANGES+=("${relative_path}")
+            mv "${target_path}.merged" "$target_path"
+          fi
+          rm -f "${target_path}.merged" "${target_path}.bak"
+        else
+          echo "Copying changes from $relative_path"
+          cp "$file" "$target_path"
+          FILES_WITH_CHANGES+=("${relative_path}")
+        fi
       fi
     fi
   fi
 done < <(find . -type f -print0)
 
-echo "${#FILES_ADDED[@]}" files added, "${#FILES_WITH_CHANGES[@]}" files with changes detected.
-
 echo ------------------------------------------
+echo "${#FILES_ADDED[@]} files added, ${#FILES_WITH_CHANGES[@]} files with changes detected."
 
-if [ "$changes_only" == false ]; then
+if [[ "$changes_only" == false && ${#FILES_ADDED[@]} -gt 0 ]]; then
   echo ------------------------------------------
   echo "New files added:"
   printf ' - %s\n' "${FILES_ADDED[@]}"
 fi
 
-
-if [ "$new_only" == false ]; then
+if [[ "$new_only" == false && ${#FILES_WITH_CHANGES[@]} -gt 0 ]]; then
   echo ------------------------------------------
   echo "Changed files:"
   printf ' - %s\n' "${FILES_WITH_CHANGES[@]}"
 fi
+
+echo ------------------------------------------

scripts/maintenance/merge.js

@@ -0,0 +1,81 @@
+/*******************************************************************************
+ * Script to merge all added lines from source to target.
+ * Modified lines will be left intact.
+ *
+ * This is intended for updating config files like .tool-versions and .gitignore
+ * where the first token on the line remains the same
+ *
+ * Usage:
+ *   $ node merge.js <target> <source>
+ *
+ * Output:
+ *   Outputs the merged file to stdout
+ *
+ *******************************************************************************/
+
+const fs = require("fs");
+
+// Read files
+const [file1, file2] = process.argv.slice(2);
+const lines1 = fs.readFileSync(file1).toString().split('\n');
+const lines2 = fs.readFileSync(file2).toString().split('\n');
+
+// Tokenize lines in file1 for later comparison
+const tokenize = line => {
+  if (line === '') {
+    return [];
+  }
+  return line.split(/\s+/).filter(x => x !== '#').slice(0, 1);
+};
+const lines1Tokens = lines1.flatMap(tokenize);
+
+// Step through the files
+let pos1 = 0, pos2 = 0;
+while (pos1 < lines1.length || pos2 < lines2.length) {
+
+  const l1 = pos1 < lines1.length && lines1[pos1] || '';
+  const l2 = pos2 < lines2.length && lines2[pos2] || '';
+
+  // If the lines match, print l1 and skip l2
+  if (l1 !== '' && l1 === l2) {
+    process.stdout.write(`${l1}\n`);
+    pos1++;
+    pos2++;
+    continue;
+  }
+
+  if (pos2 < lines2.length) {
+    // If l2 is empty, skip l2
+    if (l2 === '') {
+      pos2++;
+      continue;
+    }
+
+    const [l2token] = tokenize(l2);
+
+    // If l2 token matches l1 token, print l1
+    if (pos1 < lines1.length && lines1Tokens[pos1] === l2token) {
+      process.stdout.write(`${l1}\n`);
+      pos1++;
+      pos2++;
+      continue;
+    }
+
+    // If l2 doesn't match any lines in file1, print l2
+    if (l2token && !lines1Tokens.includes(l2token)) {
+      process.stdout.write(`${l2}\n`);
+    }
+    pos2++
+    continue;
+  }
+
+  // If we're not at the end of file1, print l1
+  if (pos1 < lines1.length) {
+    if (pos1 === lines1.length - 1 && l1 === '') {
+      // Don't print tailing newline
+    } else {
+      process.stdout.write(`${l1}\n`);
+    }
+    pos1++;
+  }
+}