cli: add --token-level-targets \w dpu_utils.codeutils.identifiersplitting

Signed-off-by: Alexander Bezzubov <[email protected]>

Author: bzz

notebooks/codesearchnet-opennmt.py

@@ -86,6 +86,69 @@ def __len__(self) -> int:
         return len(self.pd)
 
 
+# id splitting from
+# https://github.com/microsoft/dpu-utils/blob/dfc44e354b57a4e2617828bdf4d76c1c4d81c021/python/dpu_utils/codeutils/identifiersplitting.py
+from functools import lru_cache
+from typing import List
+
+def split_camelcase(camel_case_identifier: str) -> List[str]:
+    """
+    Split camelCase identifiers.
+    """
+    if not len(camel_case_identifier):
+        return []
+
+    # split into words based on adjacent cases being the same
+    result = []
+    current = str(camel_case_identifier[0])
+    prev_upper = camel_case_identifier[0].isupper()
+    prev_digit = camel_case_identifier[0].isdigit()
+    prev_special = not camel_case_identifier[0].isalnum()
+    for c in camel_case_identifier[1:]:
+        upper = c.isupper()
+        digit = c.isdigit()
+        special = not c.isalnum()
+        new_upper_word = upper and not prev_upper
+        new_digit_word = digit and not prev_digit
+        new_special_word = special and not prev_special
+        if new_digit_word or new_upper_word or new_special_word:
+            result.append(current)
+            current = c
+        elif not upper and prev_upper and len(current) > 1:
+            result.append(current[:-1])
+            current = current[-1] + c
+        elif not digit and prev_digit:
+            result.append(current)
+            current = c
+        elif not special and prev_special:
+            result.append(current)
+            current = c
+        else:
+            current += c
+        prev_digit = digit
+        prev_upper = upper
+        prev_special = special
+    result.append(current)
+    return result
+
+
+@lru_cache(maxsize=5000)
+def split_identifier_into_parts(identifier: str) -> List[str]:
+    """
+    Split a single identifier into parts on snake_case and camelCase
+    """
+    snake_case = identifier.split("_")
+
+    identifier_parts = []  # type: List[str]
+    for i in range(len(snake_case)):
+        part = snake_case[i]
+        if len(part) > 0:
+            identifier_parts.extend(s.lower() for s in split_camelcase(part))
+    if len(identifier_parts) == 0:
+        return [identifier]
+    return identifier_parts
+
+
 def main(args: Namespace) -> None:
     dataset = CodeSearchNetRAM(Path(args.data_dir), args.newline)
     split_name = Path(args.data_dir).name
@@ -96,7 +159,13 @@ def main(args: Namespace) -> None:
             if not fn_name or not fn_body:
                 continue
             src = " ".join(fn_body_tokens) if args.token_level_sources else fn_body
-            tgt = fn_name if args.word_level_targets else " ".join(fn_name)
+
+            if args.word_level_targets:
+                tgt = fn_name
+            elif args.token_level_targets:
+                tgt = " ".join(split_identifier_into_parts(fn_name))
+            else:
+                tgt = " ".join(fn_name)
             if args.print:
                 print(f"'{tgt[:40]:40}' - '{src[:40]:40}'")
             else:
@@ -121,6 +190,12 @@ def main(args: Namespace) -> None:
         help="Use language-specific token sources instead of word level ones",
     )
 
+    parser.add_argument(
+        "--token-level-targets",
+        action="store_true",
+        help="Use camlCase and snake_case split token sources instead of word or char level ones",
+    )
+
     parser.add_argument(
         "--word-level-targets",
         action="store_true",