Licenses_cpp: Parses NOTICES and DEPS (flutter#172044)

changes:
1) Now NOTICES files can be parsed
1) Now DEP files are parsed to make sure we are scanning cipd
dependencies. Previously we just checked git repos.
1) Added a `secondary` directory to `data` that allows us to inject our
own licenses. In a perfect world we wouldn't have this. When the dust
settles it may not be necessary anymore.

After this we are about 60 copyrights shy of having everything.

## Pre-launch Checklist

- [x] I read the [Contributor Guide] and followed the process outlined
there for submitting PRs.
- [x] I read the [Tree Hygiene] wiki page, which explains my
responsibilities.
- [x] I read and followed the [Flutter Style Guide], including [Features
we expect every widget to implement].
- [x] I signed the [CLA].
- [x] I listed at least one issue that this PR fixes in the description
above.
- [x] I updated/added relevant documentation (doc comments with `///`).
- [x] I added new tests to check the change I am making, or this PR is
[test-exempt].
- [x] I followed the [breaking change policy] and added [Data Driven
Fixes] where supported.
- [x] All existing and new tests are passing.

If you need help, consider asking for advice on the #hackers-new channel
on [Discord].

<!-- Links -->
[Contributor Guide]:
https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#overview
[Tree Hygiene]:
https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md
[test-exempt]:
https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#tests
[Flutter Style Guide]:
https://github.com/flutter/flutter/blob/main/docs/contributing/Style-guide-for-Flutter-repo.md
[Features we expect every widget to implement]:
https://github.com/flutter/flutter/blob/main/docs/contributing/Style-guide-for-Flutter-repo.md#features-we-expect-every-widget-to-implement
[CLA]: https://cla.developers.google.com/
[flutter/tests]: https://github.com/flutter/tests
[breaking change policy]:
https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#handling-breaking-changes
[Discord]:
https://github.com/flutter/flutter/blob/main/docs/contributing/Chat.md
[Data Driven Fixes]:
https://github.com/flutter/flutter/blob/main/docs/contributing/Data-driven-Fixes.md

Author: gaaclarke

engine/src/flutter/ci/licenses_cpp.sh

@@ -83,5 +83,10 @@ fi
   --licenses_path "$LICENSES_OUTPUT_PATH" \
   --v $VERBOSITY
 
-git diff --no-index  --ignore-cr-at-eol "$LICENSES_PATH" "$LICENSES_OUTPUT_PATH"
+git diff \
+  --no-index \
+  --ignore-cr-at-eol \
+  --ignore-matching-lines="^You may obtain a copy of this library's Source Code Form from:.*" \
+  "$LICENSES_PATH" \
+  "$LICENSES_OUTPUT_PATH"
 rm "$LICENSES_OUTPUT_PATH"

engine/src/flutter/sky/packages/sky_engine/LICENSE_CPP

@@ -8,6 +8,8 @@ source_set("licenses") {
     "src/comments_util.h",
     "src/data.cc",
     "src/data.h",
+    "src/deps_parser.cc",
+    "src/deps_parser.h",
     "src/filter.cc",
     "src/filter.h",
     "src/license_checker.cc",
@@ -52,6 +54,7 @@ executable("licenses_cpp_testrunner") {
   sources = [
     "src/catalog_unittests.cc",
     "src/comments_unittests.cc",
+    "src/deps_parser_unittests.cc",
     "src/filter_unittests.cc",
     "src/license_checker_unittests.cc",
   ]

engine/src/flutter/tools/licenses_cpp/BUILD.gn

@@ -6,6 +6,7 @@ validate the engine repository.  There are 3 main divisions of this data:
 - include.txt -- A list of all the files that will be checked.
 - exclude.txt -- A list of all the files that will be excluded.
 - data/ -- A catalog of all the accepted and known licenses.
+- secondary/ -- Secondary licenses to be included.
 
 All regex are in the [re2 format](https://github.com/google/re2/wiki/syntax).
 
@@ -34,3 +35,8 @@ show up in source code or in its own LICENSE file.  The format is the following:
    regexes.
 3) Remaining lines - Matcher regex that will be used to extract the full text
    of the accepted license.
+
+## secondary/
+
+This directory structure needs to match the one found in the working directory.
+License files here will be added verbatim to the output.

engine/src/flutter/tools/licenses_cpp/data/README.md

@@ -8,10 +8,13 @@
 .*/tests/.*
 .*third_party/googletest/.*
 ^build/.*
+^buildtools/.*
 ^impeller/fixtures/.*
+^prebuilts/.*
 ^shell/platform/fuchsia/dart-pkg/zircon_ffi/lib/zircon_ffi.dart
 ^sky/packages/sky_engine/LICENSE
 ^testing/.*
+^third_party/android_tools/.*
 ^third_party/angle/third_party/.*
 ^third_party/angle/tools/.*
 ^third_party/angle/util/.*
@@ -20,10 +23,14 @@
 ^third_party/dart/third_party/binary_size/.*
 ^third_party/dart/third_party/binaryen/.*
 ^third_party/dart/third_party/d3/.*
+^third_party/dart/third_party/devtools/web/.*\.frag$
+^third_party/dart/third_party/devtools/web/.*\.js$
 ^third_party/dart/third_party/pkg/.*
 ^third_party/dart/third_party/requirejs/.*
 ^third_party/depot_tools/.*
 ^third_party/freetype2/docs/.*
+^third_party/google_fonts_for_unit_tests/.*
+^third_party/gradle/.*
 ^third_party/harfbuzz/.*\.py
 ^third_party/harfbuzz/perf/.*
 ^third_party/harfbuzz/src/harfbuzz.cc
@@ -32,6 +39,7 @@
 ^third_party/imgui/.*
 ^third_party/inja/third_party/amalgamate/.*
 ^third_party/inja/third_party/include/doctest/.*
+^third_party/java/.*
 ^third_party/libpng/contrib/.*
 ^third_party/libwebp/.*\.py
 ^third_party/libwebp/examples/.*

engine/src/flutter/tools/licenses_cpp/data/exclude.txt

@@ -25,3 +25,5 @@
 .*\.txt$
 .*\.ucm$
 .*\.vert$
+^third_party/dart/tools/sdks/dart-sdk/bin/resources/devtools/assets/NOTICES$
+^third_party/dart/third_party/devtools/web/assets/NOTICES$

engine/src/flutter/tools/licenses_cpp/data/include.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2009 Florian Loitsch.
+
+Permission is hereby  granted, free of charge, to any  person obtaining a copy
+of this software and associated  documentation files (the "Software"), to deal
+in the Software  without restriction, including without  limitation the rights
+to  use, copy,  modify, merge,  publish, distribute,  sublicense, and/or  sell
+copies  of  the Software,  and  to  permit persons  to  whom  the Software  is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE  IS PROVIDED "AS  IS", WITHOUT WARRANTY  OF ANY KIND,  EXPRESS OR
+IMPLIED,  INCLUDING BUT  NOT  LIMITED TO  THE  WARRANTIES OF  MERCHANTABILITY,
+FITNESS FOR  A PARTICULAR PURPOSE AND  NONINFRINGEMENT. IN NO EVENT  SHALL THE
+AUTHORS  OR COPYRIGHT  HOLDERS  BE  LIABLE FOR  ANY  CLAIM,  DAMAGES OR  OTHER
+LIABILITY, WHETHER IN AN ACTION OF  CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE  OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

engine/src/flutter/tools/licenses_cpp/data/secondary/third_party/inja/third_party/include/nlohmann/loitsch.txt

@@ -12,7 +12,8 @@ const char* kExcludeFilename = "exclude.txt";
 namespace fs = std::filesystem;
 
 absl::StatusOr<Data> Data::Open(std::string_view data_dir) {
-  fs::path include_path = fs::path(data_dir) / kIncludeFilename;
+  fs::path data_path = fs::path(data_dir);
+  fs::path include_path = data_path / kIncludeFilename;
   absl::StatusOr<Filter> include_filter = Filter::Open(include_path.string());
   if (!include_filter.ok()) {
     return absl::InvalidArgumentError("Can't open include.txt at " +
@@ -33,7 +34,11 @@ absl::StatusOr<Data> Data::Open(std::string_view data_dir) {
                                       catalog.status().ToString());
   }
 
-  return Data{.include_filter = std::move(*include_filter),
-              .exclude_filter = std::move(*exclude_filter),
-              .catalog = std::move(*catalog)};
+  fs::path secondary_dir = data_path / "secondary";
+
+  return Data{
+      .include_filter = std::move(*include_filter),
+      .exclude_filter = std::move(*exclude_filter),
+      .catalog = std::move(*catalog),
+      .secondary_dir = fs::exists(secondary_dir) ? secondary_dir : fs::path()};
 }

engine/src/flutter/tools/licenses_cpp/src/data.cc

@@ -5,6 +5,8 @@
 #ifndef FLUTTER_TOOLS_LICENSES_CPP_SRC_DATA_H_
 #define FLUTTER_TOOLS_LICENSES_CPP_SRC_DATA_H_
 
+#include <filesystem>
+
 #include "flutter/third_party/abseil-cpp/absl/status/statusor.h"
 #include "flutter/tools/licenses_cpp/src/catalog.h"
 #include "flutter/tools/licenses_cpp/src/filter.h"
@@ -17,6 +19,7 @@ struct Data {
   Filter include_filter;
   Filter exclude_filter;
   Catalog catalog;
+  std::filesystem::path secondary_dir;
 };
 
 #endif  // FLUTTER_TOOLS_LICENSES_CPP_SRC_DATA_H_

engine/src/flutter/tools/licenses_cpp/src/data.h

@@ -0,0 +1,147 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "flutter/tools/licenses_cpp/src/deps_parser.h"
+
+#include <re2/re2.h>
+
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "flutter/third_party/re2/re2/re2.h"
+
+DepsParser::DepsParser() = default;
+
+DepsParser::~DepsParser() = default;
+
+// TODO(gaaclarke): Convert this to flex/bison, this is getting a bit
+// complicated. It technically doesn't handle commas in trailing comments.
+std::vector<std::string> DepsParser::Parse(std::string_view input) {
+  std::vector<std::string> result;
+  re2::StringPiece mutable_input(input);
+
+  // 1. Find the start of the deps block
+  if (!re2::RE2::FindAndConsume(&mutable_input, R"((?:^|\n)deps\s*=\s*\{)")) {
+    return result;
+  }
+
+  // 2. Find the matching closing brace for the entire deps block
+  size_t brace_close_pos = std::string_view::npos;
+  int brace_count = 1;
+  for (size_t i = 0; i < mutable_input.length(); ++i) {
+    if (mutable_input[i] == '{') {
+      brace_count++;
+    } else if (mutable_input[i] == '}') {
+      brace_count--;
+      if (brace_count == 0) {
+        brace_close_pos = i;
+        break;
+      }
+    }
+  }
+
+  if (brace_close_pos == std::string_view::npos) {
+    return result;
+  }
+
+  // 3. Extract the content of the deps block
+  std::string_view deps_content = mutable_input.substr(0, brace_close_pos);
+
+  // 4. Filter out comments and build a new string
+  std::string cleaned_content;
+  size_t current_pos_clean = 0;
+  while (current_pos_clean < deps_content.length()) {
+    size_t next_newline = deps_content.find('\n', current_pos_clean);
+    if (next_newline == std::string_view::npos) {
+      next_newline = deps_content.length();
+    }
+
+    std::string_view line = deps_content.substr(
+        current_pos_clean, next_newline - current_pos_clean);
+
+    size_t first_char = line.find_first_not_of(" \t");
+    if (first_char != std::string_view::npos) {
+      line = line.substr(first_char);
+    } else {
+      line = "";
+    }
+
+    if (line.empty() || line[0] != '#') {
+      cleaned_content.append(line);
+      cleaned_content.append("\n");
+    }
+
+    current_pos_clean = next_newline + 1;
+  }
+
+  // 5. Parse the cleaned content
+  size_t current_pos = 0;
+  while (current_pos < cleaned_content.length()) {
+    // Find the start of the key
+    size_t key_start = cleaned_content.find('\'', current_pos);
+    if (key_start == std::string_view::npos) {
+      break;
+    }
+    key_start++;
+
+    // Find the end of the key
+    size_t key_end = cleaned_content.find('\'', key_start);
+    if (key_end == std::string_view::npos) {
+      break;
+    }
+
+    std::string_view key =
+        std::string_view(&cleaned_content[key_start], key_end - key_start);
+
+    // Find the colon after the key
+    size_t colon_pos = cleaned_content.find(':', key_end);
+    if (colon_pos == std::string_view::npos) {
+      break;
+    }
+
+    // Find the start of the value
+    size_t value_start =
+        cleaned_content.find_first_not_of(" \t\n", colon_pos + 1);
+    if (value_start == std::string_view::npos) {
+      break;
+    }
+
+    // Find the end of the value
+    size_t value_end = std::string_view::npos;
+    int brace_level = 0;
+    int bracket_level = 0;
+    for (size_t i = value_start; i < cleaned_content.length(); ++i) {
+      char c = cleaned_content[i];
+      if (c == '{') {
+        brace_level++;
+      } else if (c == '}') {
+        brace_level--;
+      } else if (c == '[') {
+        bracket_level++;
+      } else if (c == ']') {
+        bracket_level--;
+      } else if (c == ',' && brace_level == 0 && bracket_level == 0) {
+        value_end = i;
+        break;
+      }
+    }
+
+    if (value_end == std::string_view::npos) {
+      value_end = cleaned_content.length();
+    }
+
+    std::string_view value = std::string_view(&cleaned_content[value_start],
+                                              value_end - value_start);
+
+    // Check for 'dep_type': 'cipd'
+    if (value.find("'dep_type': 'cipd'") != std::string_view::npos) {
+      result.emplace_back(key);
+    }
+
+    current_pos = value_end + 1;
+  }
+
+  return result;
+}