|
| 1 | +# Copyright 2024 Google LLC |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# https://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | +import difflib |
| 15 | +import json |
| 16 | +import os |
| 17 | +import sys |
| 18 | +import unittest |
| 19 | +import xml.etree.ElementTree as tree |
| 20 | +from collections import Counter |
| 21 | +from filecmp import dircmp |
| 22 | + |
| 23 | +script_dir = os.path.dirname(os.path.realpath(__file__)) |
| 24 | + |
| 25 | + |
| 26 | +class IntegrationTest(unittest.TestCase): |
| 27 | + |
| 28 | + def test_monorepo_generation(self): |
| 29 | + repo_dest = "/workspace/google-cloud-java" |
| 30 | + golden_dir = "/workspace/golden" |
| 31 | + library_names = [ |
| 32 | + "java-apigee-connect", |
| 33 | + "java-alloydb", |
| 34 | + "java-alloydb-connectors", |
| 35 | + "java-cloudcontrolspartner", |
| 36 | + ] |
| 37 | + for library_name in library_names: |
| 38 | + actual_library = f"{repo_dest}/{library_name}" |
| 39 | + print("*" * 50) |
| 40 | + print(f"Checking for differences in '{library_name}'.") |
| 41 | + print(f" The expected library is in {golden_dir}/{library_name}.") |
| 42 | + print(f" The actual library is in {actual_library}.") |
| 43 | + compare_result = dircmp( |
| 44 | + f"{golden_dir}/{library_name}", |
| 45 | + actual_library, |
| 46 | + ignore=[".repo-metadata.json"], |
| 47 | + ) |
| 48 | + diff_files = [] |
| 49 | + golden_only = [] |
| 50 | + generated_only = [] |
| 51 | + # compare source code |
| 52 | + self.__recursive_diff_files( |
| 53 | + compare_result, diff_files, golden_only, generated_only |
| 54 | + ) |
| 55 | + |
| 56 | + # print all found differences for inspection |
| 57 | + def print_file(f: str) -> None: |
| 58 | + return print(f" - {f}") |
| 59 | + |
| 60 | + if len(diff_files) > 0: |
| 61 | + print(" Some files (found in both folders) are differing:") |
| 62 | + for diff_file in diff_files: |
| 63 | + print(f"Difference in {diff_file}:") |
| 64 | + with open( |
| 65 | + f"{golden_dir}/{library_name}/{diff_file}" |
| 66 | + ) as expected_file: |
| 67 | + with open(f"{actual_library}/{diff_file}") as actual_file: |
| 68 | + [ |
| 69 | + print(line) |
| 70 | + for line in difflib.unified_diff( |
| 71 | + expected_file.readlines(), |
| 72 | + actual_file.readlines(), |
| 73 | + ) |
| 74 | + ] |
| 75 | + if len(golden_only) > 0: |
| 76 | + print(" There were files found only in the golden dir:") |
| 77 | + [print_file(f) for f in golden_only] |
| 78 | + if len(generated_only) > 0: |
| 79 | + print(" There were files found only in the generated dir:") |
| 80 | + [print_file(f) for f in generated_only] |
| 81 | + |
| 82 | + self.assertTrue(len(golden_only) == 0) |
| 83 | + self.assertTrue(len(generated_only) == 0) |
| 84 | + self.assertTrue(len(diff_files) == 0) |
| 85 | + |
| 86 | + print(f" No differences found in {library_name}") |
| 87 | + # compare .repo-metadata.json |
| 88 | + self.assertTrue( |
| 89 | + self.__compare_json_files( |
| 90 | + f"{golden_dir}/{library_name}/.repo-metadata.json", |
| 91 | + f"{actual_library}/.repo-metadata.json", |
| 92 | + ), |
| 93 | + msg=f" The generated {library_name}/.repo-metadata.json is different from golden.", |
| 94 | + ) |
| 95 | + print(" .repo-metadata.json comparison succeed.") |
| 96 | + # compare gapic-libraries-bom/pom.xml and pom.xml |
| 97 | + self.assertFalse( |
| 98 | + self.compare_xml( |
| 99 | + f"{golden_dir}/gapic-libraries-bom/pom.xml", |
| 100 | + f"{repo_dest}/gapic-libraries-bom/pom.xml", |
| 101 | + False, |
| 102 | + ) |
| 103 | + ) |
| 104 | + print(" gapic-libraries-bom/pom.xml comparison succeed.") |
| 105 | + self.assertFalse( |
| 106 | + self.compare_xml( |
| 107 | + f"{golden_dir}/pom.xml", |
| 108 | + f"{repo_dest}/pom.xml", |
| 109 | + False, |
| 110 | + ) |
| 111 | + ) |
| 112 | + print(" pom.xml comparison succeed.") |
| 113 | + |
| 114 | + @classmethod |
| 115 | + def __compare_json_files(cls, expected: str, actual: str) -> bool: |
| 116 | + return cls.__load_json_to_sorted_list( |
| 117 | + expected |
| 118 | + ) == cls.__load_json_to_sorted_list(actual) |
| 119 | + |
| 120 | + @classmethod |
| 121 | + def __load_json_to_sorted_list(cls, path: str) -> list[tuple]: |
| 122 | + with open(path) as f: |
| 123 | + data = json.load(f) |
| 124 | + res = [(key, value) for key, value in data.items()] |
| 125 | + |
| 126 | + return sorted(res, key=lambda x: x[0]) |
| 127 | + |
| 128 | + @classmethod |
| 129 | + def __recursive_diff_files( |
| 130 | + cls, |
| 131 | + dcmp: dircmp, |
| 132 | + diff_files: list[str], |
| 133 | + left_only: list[str], |
| 134 | + right_only: list[str], |
| 135 | + dirname: str = "", |
| 136 | + ): |
| 137 | + """ |
| 138 | + Recursively compares two subdirectories. The found differences are |
| 139 | + passed to three expected list references. |
| 140 | + """ |
| 141 | + |
| 142 | + def append_dirname(d: str) -> str: |
| 143 | + return dirname + d |
| 144 | + |
| 145 | + diff_files.extend(map(append_dirname, dcmp.diff_files)) |
| 146 | + left_only.extend(map(append_dirname, dcmp.left_only)) |
| 147 | + right_only.extend(map(append_dirname, dcmp.right_only)) |
| 148 | + for sub_dirname, sub_dcmp in dcmp.subdirs.items(): |
| 149 | + cls.__recursive_diff_files( |
| 150 | + sub_dcmp, diff_files, left_only, right_only, dirname + sub_dirname + "/" |
| 151 | + ) |
| 152 | + |
| 153 | + @classmethod |
| 154 | + def compare_xml(cls, expected, actual, print_trees): |
| 155 | + """ |
| 156 | + compares two XMLs for content differences |
| 157 | + the argument print_whole_trees determines if both trees should be printed |
| 158 | + """ |
| 159 | + try: |
| 160 | + expected_tree = tree.parse(expected) |
| 161 | + actual_tree = tree.parse(actual) |
| 162 | + except tree.ParseError as e: |
| 163 | + cls.eprint(f"Error parsing XML") |
| 164 | + raise e |
| 165 | + except FileNotFoundError as e: |
| 166 | + cls.eprint(f"Error reading file") |
| 167 | + raise e |
| 168 | + |
| 169 | + expected_elements = [] |
| 170 | + actual_elements = [] |
| 171 | + |
| 172 | + cls.append_to_element_list(expected_tree.getroot(), "/", expected_elements) |
| 173 | + cls.append_to_element_list(actual_tree.getroot(), "/", actual_elements) |
| 174 | + |
| 175 | + expected_counter = Counter(expected_elements) |
| 176 | + actual_counter = Counter(actual_elements) |
| 177 | + intersection = expected_counter & actual_counter |
| 178 | + only_in_expected = expected_counter - intersection |
| 179 | + only_in_actual = actual_counter - intersection |
| 180 | + if print_trees: |
| 181 | + cls.eprint("expected") |
| 182 | + cls.print_counter(actual_counter) |
| 183 | + cls.eprint("actual") |
| 184 | + cls.print_counter(expected_counter) |
| 185 | + if len(only_in_expected) > 0 or len(only_in_actual) > 0: |
| 186 | + cls.eprint("only in " + expected) |
| 187 | + cls.print_counter(only_in_expected) |
| 188 | + cls.eprint("only in " + actual) |
| 189 | + cls.print_counter(only_in_actual) |
| 190 | + return True |
| 191 | + return False |
| 192 | + |
| 193 | + @classmethod |
| 194 | + def append_to_element_list(cls, node, path, elements): |
| 195 | + """ |
| 196 | + Recursively traverses a node tree and appends element text to a given |
| 197 | + `elements` array. If the element tag is `dependency` |
| 198 | + then the maven coordinates for its children will be computed as well |
| 199 | + """ |
| 200 | + namespace_start, namespace_end, tag_name = node.tag.rpartition("}") |
| 201 | + namespace = namespace_start + namespace_end |
| 202 | + if tag_name == "dependency": |
| 203 | + group_id = cls.get_text_from_element(node, "groupId", namespace) |
| 204 | + artifact_id = cls.get_text_from_element(node, "artifactId", namespace) |
| 205 | + artifact_str = "" |
| 206 | + artifact_str += group_id |
| 207 | + artifact_str += ":" + artifact_id |
| 208 | + elements.append(path + "/" + tag_name + "=" + artifact_str) |
| 209 | + if node.text and len(node.text.strip()) > 0: |
| 210 | + elements.append(path + "/" + tag_name + "=" + node.text) |
| 211 | + |
| 212 | + if tag_name == "version": |
| 213 | + # versions may be yet to be processed, we disregard them |
| 214 | + return elements |
| 215 | + |
| 216 | + for child in node: |
| 217 | + child_path = path + "/" + tag_name |
| 218 | + cls.append_to_element_list(child, child_path, elements) |
| 219 | + |
| 220 | + return elements |
| 221 | + |
| 222 | + @classmethod |
| 223 | + def get_text_from_element(cls, node, element_name, namespace): |
| 224 | + """ |
| 225 | + Convenience method to access a node's child elements via path and get |
| 226 | + its text. |
| 227 | + """ |
| 228 | + child = node.find(namespace + element_name) |
| 229 | + return child.text if child is not None else "" |
| 230 | + |
| 231 | + @classmethod |
| 232 | + def eprint(cls, *args, **kwargs): |
| 233 | + """ |
| 234 | + prints to stderr |
| 235 | + """ |
| 236 | + print(*args, file=sys.stderr, **kwargs) |
| 237 | + |
| 238 | + @classmethod |
| 239 | + def print_counter(cls, counter): |
| 240 | + """ |
| 241 | + Convenience method to pretty print the contents of a Counter (or dict) |
| 242 | + """ |
| 243 | + for key, value in counter.items(): |
| 244 | + cls.eprint(f"{key}: {value}") |
0 commit comments