Add d2d step to map javascript using string literals

Signed-off-by: Keshav Priyadarshi <[email protected]>

Author: keshav-space

scanpipe/filters.py

@@ -483,6 +483,7 @@ def filter(self, qs, value):
     ("about_file", "about file"),
     ("java_to_class", "java to class"),
     ("jar_to_source", "jar to source"),
+    ("javascript_strings", "js strings"),
     ("javascript_symbols", "js symbols"),
     ("js_compiled", "js compiled"),
     ("js_colocation", "js colocation"),

scanpipe/pipelines/deploy_to_develop.py

@@ -72,6 +72,7 @@ def steps(cls):
             cls.map_jar_to_source,
             cls.map_javascript,
             cls.map_javascript_symbols,
+            cls.map_javascript_strings,
             cls.map_elf,
             cls.map_macho,
             cls.map_winpe,
@@ -204,6 +205,11 @@ def map_javascript_symbols(self):
         """Map deployed JavaScript, TypeScript to its sources using symbols."""
         d2d.map_javascript_symbols(project=self.project, logger=self.log)
 
+    @optional_step("JavaScript")
+    def map_javascript_strings(self):
+        """Map deployed JavaScript, TypeScript to its sources using string literals."""
+        d2d.map_javascript_strings(project=self.project, logger=self.log)
+
     @optional_step("Elf")
     def map_elf(self):
         """Map ELF binaries to their sources using dwarf paths and symbols."""

scanpipe/pipes/d2d.py

@@ -62,6 +62,7 @@
 from scanpipe.pipes import purldb
 from scanpipe.pipes import resolve
 from scanpipe.pipes import scancode
+from scanpipe.pipes import strings
 from scanpipe.pipes import symbolmap
 from scanpipe.pipes import symbols
 
@@ -2146,3 +2147,84 @@ def _map_javascript_symbols(to_resource, javascript_from_resources, logger):
         to_resource.update(status=flag.MAPPED)
         return 1
     return 0
+
+
+def map_javascript_strings(project, logger=None):
+    """Map deployed JavaScript, TypeScript to its sources using string literals."""
+    project_files = project.codebaseresources.files()
+
+    javascript_to_resources = (
+        project_files.to_codebase()
+        .has_no_relation()
+        .filter(extension__in=[".ts", ".js"])
+        .exclude(extra_data={})
+    )
+
+    javascript_from_resources = (
+        project_files.from_codebase()
+        .exclude(path__contains="/test/")
+        .filter(extension__in=[".ts", ".js"])
+        .exclude(extra_data={})
+    )
+
+    if not (javascript_from_resources.exists() and javascript_to_resources.exists()):
+        return
+
+    javascript_from_resources_count = javascript_from_resources.count()
+    javascript_to_resources_count = javascript_to_resources.count()
+    if logger:
+        logger(
+            f"Mapping {javascript_to_resources_count:,d} JavaScript resources"
+            f" using string literals against {javascript_from_resources_count:,d}"
+            " from/ resources."
+        )
+
+    resource_iterator = javascript_to_resources.iterator(chunk_size=2000)
+    progress = LoopProgress(javascript_to_resources_count, logger)
+
+    resource_mapped = 0
+    for to_resource in progress.iter(resource_iterator):
+        resource_mapped += _map_javascript_strings(
+            to_resource, javascript_from_resources, logger
+        )
+    if logger:
+        logger(f"{resource_mapped:,d} resource mapped using strings")
+
+
+def _map_javascript_strings(to_resource, javascript_from_resources, logger):
+    """
+    Map a deployed JavaScript resource to its source using string literals and
+    return 1 if match is found otherwise return 0.
+    """
+    ignoreable_string_threshold = 5
+    to_strings = to_resource.extra_data.get("source_strings")
+
+    if not to_strings and len(to_strings) > ignoreable_string_threshold:
+        return 0
+
+    best_matching_score = 0
+    best_match = None
+    for source_js in javascript_from_resources:
+        from_strings = source_js.extra_data.get("source_strings")
+        if not from_strings and len(from_strings) > ignoreable_string_threshold:
+            continue
+
+        is_match, similarity = strings.match_source_strings_to_deployed(
+            source_strings=from_strings,
+            deployed_strings=to_strings,
+        )
+
+        if is_match and similarity > best_matching_score:
+            best_matching_score = similarity
+            best_match = source_js
+
+    if best_match:
+        pipes.make_relation(
+            from_resource=best_match,
+            to_resource=to_resource,
+            map_type="javascript_strings",
+            extra_data={"js_string_map_score": similarity},
+        )
+        to_resource.update(status=flag.MAPPED)
+        return 1
+    return 0

scanpipe/pipes/strings.py

@@ -20,8 +20,14 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
+from collections import Counter
+
 from aboutcode.pipeline import LoopProgress
 
+STRING_MATCHING_RATIO_JAVASCRIPT = 0.7
+SMALL_FILE_STRING_THRESHOLD_JAVASCRIPT = 10
+STRING_MATCHING_RATIO_JAVASCRIPT_SMALL_FILE = 0.5
+
 
 class XgettextNotFound(Exception):
     pass
@@ -66,3 +72,37 @@ def _collect_and_store_resource_strings(resource):
     result = strings_xgettext.collect_strings(resource.location)
     strings = [item["string"] for item in result if "string" in item]
     resource.update_extra_data({"source_strings": strings})
+
+
+def match_source_strings_to_deployed(source_strings, deployed_strings):
+    common_strings_ratio = 0
+    is_match = False
+    deployed_strings_set = set(deployed_strings)
+    source_strings_set = set(source_strings)
+    source_strings_count = len(source_strings)
+    deployed_strings_count = len(deployed_strings)
+    total_strings_count = source_strings_count + deployed_strings_count
+    source_strings_counter = Counter(source_strings)
+    deployed_strings_counter = Counter(deployed_strings)
+
+    common_strings = source_strings_set.intersection(deployed_strings_set)
+    total_common_strings_count = sum(
+        [
+            source_strings_counter.get(string, 0)
+            + deployed_strings_counter.get(string, 0)
+            for string in common_strings
+        ]
+    )
+
+    if total_common_strings_count:
+        common_strings_ratio = total_common_strings_count / total_strings_count
+
+    if common_strings_ratio > STRING_MATCHING_RATIO_JAVASCRIPT:
+        is_match = True
+    elif (
+        source_strings_count > SMALL_FILE_STRING_THRESHOLD_JAVASCRIPT
+        and common_strings_ratio > STRING_MATCHING_RATIO_JAVASCRIPT_SMALL_FILE
+    ):
+        is_match = True
+
+    return is_match, common_strings_ratio