fix script updating endpoint_large_scale test data

Author: jhelie

javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/README.md

@@ -0,0 +1,9 @@
+# autogenerated folder
+
+This folder contains test data for the ATM endpoint CodeQL tests that has been autogenerated from the standard JS CodeQL libraries.
+
+It is helpful, but not required, to periodically update this test data to incorporate new test data introduced in the standard JS CodeQL libraries.
+
+To update this test data, run `python /path/to/codeql-lib/ql/javascript/test/update_endpoint_test_files.py`.
+
+For more information view the source code of [`update_endpoint_test_files.py`](../../update_endpoint_test_files.py).

javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py

@@ -2,10 +2,17 @@
 
 # This script updates the JavaScript test data used by the endpoint CodeQL tests.
 
-import glob
+import git
 import logging
-import os
 import shutil
+from pathlib import Path
+
+# Get relevant paths
+script_path = Path(__file__).absolute()
+git_repo = git.Repo(__file__, search_parent_directories=True)
+git_root = Path(git_repo.git.rev_parse('--show-toplevel'))
+autogenerated_dest_path = script_path.parent.joinpath('endpoint_large_scale',
+                                                      'autogenerated')
 
 # File extensions that should be copied to the endpoint tests. This should include source code files
 # e.g. .js, but not the tests themselves e.g. .expected, .ql, .qlref, etc.
@@ -15,35 +22,31 @@
 # path of that test relative to a checkout of github/codeql.
 test_root_relative_paths = {
     'NosqlAndSqlInjection': 'javascript/ql/test/query-tests/Security/CWE-089',
-    'TaintedPath': 'javascript/ql/test/query-tests/Security/CWE-022/TaintedPath',
+    'TaintedPath':
+    'javascript/ql/test/query-tests/Security/CWE-022/TaintedPath',
     'Xss': 'javascript/ql/test/query-tests/Security/CWE-079',
+    'XssThroughDom': 'javascript/ql/test/query-tests/Security/CWE-116'
 }
-# The path of the endpoint tests, relative to a checkout of github/codeql
-test_path = 'javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale'
 
 logging.basicConfig(level=logging.INFO)
 
-codeql_lib_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
-
-autogenerated_dest_path = os.path.join(codeql_lib_path, test_path, 'autogenerated')
-if os.path.exists(autogenerated_dest_path):
+if autogenerated_dest_path.exists():
     logging.info(f'Deleting existing autogenerated test files...')
     shutil.rmtree(autogenerated_dest_path)
 
 for key, rel_path in test_root_relative_paths.items():
-    test_files_path = os.path.join(codeql_lib_path, rel_path)
+    test_files_path = git_root.joinpath(rel_path)
     logging.info(f'Copying test files for {key}...')
-
-    for file in glob.glob(test_files_path + '/**', recursive=True):
-        # Ignore .testproj directories
-        if '.testproj' in file:
+    counter = 0
+    for file in Path(test_files_path).glob('**/*'):
+        if file.is_dir() or '.test_proj' in str(file):
             continue
-
-        file_extension = os.path.splitext(file)[1]
-        if file_extension in file_extensions_to_copy:
-            dest_path = os.path.normpath(
-                os.path.join(autogenerated_dest_path, key, os.path.relpath(file, test_files_path))
-            )
+        if file.suffix in file_extensions_to_copy:
+            autogenerated_dest_path.joinpath(key, )
+            dest_path = autogenerated_dest_path.joinpath(
+                key, file.relative_to(test_files_path))
             logging.debug(f'Copying {file} to {dest_path}')
-            os.makedirs(os.path.dirname(dest_path), exist_ok=True)
+            dest_path.parent.mkdir(parents=True, exist_ok=True)
             shutil.copyfile(file, dest_path)
+            counter += 1
+    logging.info(f'copied {counter} files')