From a0a89a9d6c9f66c8da866ea05aedb92e8f67e0ac Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Tue, 15 Apr 2025 14:35:05 -0400 Subject: [PATCH 01/41] dataconnect.yml: improve send-notifications message and allow specifying issue via trksmnkncd_notification_issue=6863 in the PR body --- .github/workflows/dataconnect.yml | 50 +++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index f5a58ef6896..a9240d84d77 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -267,19 +267,57 @@ jobs: issues: write runs-on: ubuntu-latest steps: - - name: Post Comment on Issue #6857 - if: github.event_name == 'schedule' + - id: issue-id + name: Determine GitHub Issue For Commenting + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + set -xv + + # If this job was triggered by a pull request, then check to see if + # the description of the PR specifies a GitHub Issue to which to post + # a comment. The GitHub Issue number is specified by including a line + # of the form "trksmnkncd_notification_issue=6863" in the description. + # Specifying an issue number in this manner is primarily intended for + # debugging/development purposes of the comment-posting logic itself. + # For example, issue number 6863 was specifically created for testing. + # (see https://github.com/firebase/firebase-android-sdk/issues/6863) + readonly pr_number + pr_number=$(echo '${{ github.ref }}' | sed -En 's#^refs/pull/([0-9]+)/merge$#\1#p') + if [[ -n $pr_number ]] ; then + readonly issue_from_pr_body + issue_from_pr_body=$(gh issue view "$pr_number" --json body --jq '.body' | sed -En 's#(\s|^)trksmnkncd_notification_issue=([0-9]+)(\s|$)#\2#p') + fi + + if [[ -v $issue_from_pr_body ]] && [[ -n $issue_from_pr_body ]] ; then + readonly issue="$issue_from_pr_body" + elif [[ '${{ github.event_name }}' == 'schedule' ]] ; then + # See https://github.com/firebase/firebase-android-sdk/issues/6857 + readonly issue=6857 + else + readonly issue= + fi + + echo "issue=$issue" >> "$GITHUB_OUTPUT" + + - name: Post Comment on GitHub Issue + if: steps.issue-id.outputs.issue != '' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail cat >message.txt < Date: Tue, 15 Apr 2025 15:30:34 -0400 Subject: [PATCH 02/41] ci skeleton added --- .github/workflows/dataconnect.yml | 59 ++++++++++++++++++++++++ firebase-dataconnect/ci/notify_issue.py | 0 firebase-dataconnect/ci/pyproject.toml | 25 ++++++++++ firebase-dataconnect/ci/requirements.txt | 8 ++++ 4 files changed, 92 insertions(+) create mode 100644 firebase-dataconnect/ci/notify_issue.py create mode 100644 firebase-dataconnect/ci/pyproject.toml create mode 100644 firebase-dataconnect/ci/requirements.txt diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index a9240d84d77..5c0755a95d4 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -9,6 +9,7 @@ on: firebaseToolsVersion: gradleInfoLog: type: boolean + pythonVersion: pull_request: paths: - .github/workflows/dataconnect.yml @@ -27,6 +28,7 @@ env: FDC_FIREBASE_TOOLS_VERSION: ${{ inputs.firebaseToolsVersion || '13.29.1' }} FDC_FIREBASE_TOOLS_DIR: /tmp/firebase-tools FDC_FIREBASE_COMMAND: /tmp/firebase-tools/node_modules/.bin/firebase + FDC_PYTHON_VERSION: ${{ inputs.pythonVersion || '3.13' }} concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -254,12 +256,69 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + sparse-checkout: '.github/' with: show-progress: false - uses: docker://rhysd/actionlint:1.7.7 with: args: -color /github/workspace/.github/workflows/dataconnect.yml + python-ci-unit-tests: + continue-on-error: false + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + sparse-checkout: 'firebase-dataconnect/ci/' + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + with: + python-version: ${{ env.FDC_PYTHON_VERSION }} + - run: pip install -r firebase-dataconnect/ci/requirements.txt + - name: pytest + working-directory: firebase-dataconnect/ci + run: pytest --verbose --full-trace --color=no --strict-config + + pytnon-ci-lint: + continue-on-error: false + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + sparse-checkout: 'firebase-dataconnect/ci/' + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + with: + python-version: ${{ env.FDC_PYTHON_VERSION }} + - run: pip install -r firebase-dataconnect/ci/requirements.txt + - name: ruff check + working-directory: firebase-dataconnect/ci + run: ruff check --diff --verbose --no-cache --output-format=github --exit-non-zero-on-fix + + pytnon-ci-format: + continue-on-error: false + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + sparse-checkout: 'firebase-dataconnect/ci/' + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + with: + python-version: ${{ env.FDC_PYTHON_VERSION }} + - run: pip install -r firebase-dataconnect/ci/requirements.txt + - name: ruff format + working-directory: firebase-dataconnect/ci + run: ruff format --diff --verbose --no-cache + + python-ci-type-check: + continue-on-error: false + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + sparse-checkout: 'firebase-dataconnect/ci/' + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + with: + python-version: ${{ env.FDC_PYTHON_VERSION }} + - run: pip install -r firebase-dataconnect/ci/requirements.txt + - name: pyright + working-directory: firebase-dataconnect/ci + run: pyright --warnings --stats + send-notifications: needs: [integration-test, actionlint-dataconnect-yml] if: always() diff --git a/firebase-dataconnect/ci/notify_issue.py b/firebase-dataconnect/ci/notify_issue.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/firebase-dataconnect/ci/pyproject.toml b/firebase-dataconnect/ci/pyproject.toml new file mode 100644 index 00000000000..d23ab765e33 --- /dev/null +++ b/firebase-dataconnect/ci/pyproject.toml @@ -0,0 +1,25 @@ +[project] +name = "Firebase Data Connect Android SDK Continuous Integration Tools" +requires-python = ">= 3.13" + +[tool.pyright] +include = ["**/*.py"] +typeCheckingMode = "strict" + +[tool.ruff] +line-length = 100 +indent-width = 2 + +[tool.ruff.lint] +select = ["ALL"] +ignore = [ + "D100", # Missing docstring in public module + "D211", # no-blank-line-before-class + "D212", # multi-line-summary-second-line +] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +docstring-code-format = true diff --git a/firebase-dataconnect/ci/requirements.txt b/firebase-dataconnect/ci/requirements.txt new file mode 100644 index 00000000000..9825cdd3d53 --- /dev/null +++ b/firebase-dataconnect/ci/requirements.txt @@ -0,0 +1,8 @@ +iniconfig==2.1.0 +nodeenv==1.9.1 +packaging==24.2 +pluggy==1.5.0 +pyright==1.1.399 +pytest==8.3.5 +ruff==0.11.5 +typing_extensions==4.13.2 From 4c9ba074fcfc782737dd658fcf434b1dd45af0d7 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Tue, 15 Apr 2025 15:35:50 -0400 Subject: [PATCH 03/41] add copyright --- firebase-dataconnect/ci/notify_issue.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/firebase-dataconnect/ci/notify_issue.py b/firebase-dataconnect/ci/notify_issue.py index e69de29bb2d..114b92c3a04 100644 --- a/firebase-dataconnect/ci/notify_issue.py +++ b/firebase-dataconnect/ci/notify_issue.py @@ -0,0 +1,14 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + From 14c4a8172833c32e58622f9d4a25e9bf91488237 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Tue, 15 Apr 2025 15:46:12 -0400 Subject: [PATCH 04/41] dataconnect.yml: fix typos --- .github/workflows/dataconnect.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index 5c0755a95d4..b11a3b9a55d 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -256,9 +256,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - sparse-checkout: '.github/' with: show-progress: false + sparse-checkout: '.github/' - uses: docker://rhysd/actionlint:1.7.7 with: args: -color /github/workspace/.github/workflows/dataconnect.yml @@ -268,7 +268,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - sparse-checkout: 'firebase-dataconnect/ci/' + with: + show-progress: false + sparse-checkout: 'firebase-dataconnect/ci/' - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: ${{ env.FDC_PYTHON_VERSION }} @@ -282,7 +284,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - sparse-checkout: 'firebase-dataconnect/ci/' + with: + show-progress: false + sparse-checkout: 'firebase-dataconnect/ci/' - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: ${{ env.FDC_PYTHON_VERSION }} @@ -296,7 +300,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - sparse-checkout: 'firebase-dataconnect/ci/' + with: + show-progress: false + sparse-checkout: 'firebase-dataconnect/ci/' - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: ${{ env.FDC_PYTHON_VERSION }} @@ -310,7 +316,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - sparse-checkout: 'firebase-dataconnect/ci/' + with: + show-progress: false + sparse-checkout: 'firebase-dataconnect/ci/' - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: ${{ env.FDC_PYTHON_VERSION }} From 7b7cb4cbe7956294ba177d8fc871bc679ee39686 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Tue, 15 Apr 2025 15:48:27 -0400 Subject: [PATCH 05/41] fixes --- firebase-dataconnect/ci/notify_issue.py | 17 +++++++++++++++++ firebase-dataconnect/ci/pyproject.toml | 1 + 2 files changed, 18 insertions(+) diff --git a/firebase-dataconnect/ci/notify_issue.py b/firebase-dataconnect/ci/notify_issue.py index 114b92c3a04..45ff8b029c8 100644 --- a/firebase-dataconnect/ci/notify_issue.py +++ b/firebase-dataconnect/ci/notify_issue.py @@ -12,3 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys +import typing +from collections.abc import Sequence + +type ExitCode = int + +def main(args: Sequence[str], stdout: typing.TextIO, stderr: typing.TextIO) -> ExitCode: + return 0 + +if __name__ == "__main__": + try: + exit_code = main(sys.argv, sys.stdout, sys.stderr) + except KeyboardInterrupt: + print("ERROR: application terminated by keyboard interrupt", file=sys.stderr) + exit_code = 1 + + sys.exit(exit_code) diff --git a/firebase-dataconnect/ci/pyproject.toml b/firebase-dataconnect/ci/pyproject.toml index d23ab765e33..bbea6c41d59 100644 --- a/firebase-dataconnect/ci/pyproject.toml +++ b/firebase-dataconnect/ci/pyproject.toml @@ -16,6 +16,7 @@ ignore = [ "D100", # Missing docstring in public module "D211", # no-blank-line-before-class "D212", # multi-line-summary-second-line + "T201", # print` found ] [tool.ruff.format] From ee1e41d8c2abfd845f018e8f25346eb1ec3c4bf1 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Tue, 15 Apr 2025 16:06:26 -0400 Subject: [PATCH 06/41] work --- firebase-dataconnect/ci/notify_issue.py | 56 +++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/firebase-dataconnect/ci/notify_issue.py b/firebase-dataconnect/ci/notify_issue.py index 45ff8b029c8..18352f47d2d 100644 --- a/firebase-dataconnect/ci/notify_issue.py +++ b/firebase-dataconnect/ci/notify_issue.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse +import dataclasses +import logging import sys import typing from collections.abc import Sequence @@ -19,8 +22,61 @@ type ExitCode = int def main(args: Sequence[str], stdout: typing.TextIO, stderr: typing.TextIO) -> ExitCode: + try: + parsed_args = parse_args(args[0], args[1:], stdout) + except MyArgumentParser.Error as e: + if e.exit_code != 0: + print(f"ERROR: invalid command-line arguments: {e}", file=stderr) + print(f"Run with --help for help", file=stderr) + return e.exit_code + return 0 +@dataclasses.dataclass(frozen=True) +class GetIssueNumberCommand: + github_ref: str + github_event_name: str + default_github_issue: int + +@dataclasses.dataclass(frozen=True) +class ParsedArgs: + log_level: int + command: GetIssueNumberCommand + +class MyArgumentParser(argparse.ArgumentParser): + + def __init__(self, prog: str, stdout: typing.TextIO) -> None: + super().__init__(prog=prog, usage="%(prog)s [options]") + self.stdout = stdout + + @typing.override + def exit(self, status: int = 0, message: str | None = None) -> typing.Never: + raise self.Error(exit_code=status, message=message) + + @typing.override + def error(self, message: str) -> typing.Never: + self.exit(2, message) + + @typing.override + def print_usage(self, file: typing.TextIO | None = None) -> None: + file = file if file is not None else self.stdout + super().print_usage(file) + + @typing.override + def print_help(self, file: typing.TextIO | None =None) -> None: + file = file if file is not None else self.stdout + super().print_help(file) + + class Error(Exception): + def __init__(self, exit_code: ExitCode, message: str | None) -> None: + super().__init__(message) + self.exit_code = exit_code + + +def parse_args(prog: str, args: Sequence[str], stdout: typing.TextIO) -> ExitCode: + arg_parser = MyArgumentParser(prog, stdout) + parsed_args = arg_parser.parse_args(args) + if __name__ == "__main__": try: exit_code = main(sys.argv, sys.stdout, sys.stderr) From 25e7a8f08466ac2362eeed45fe1041ed6da0e362 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Tue, 15 Apr 2025 16:08:04 -0400 Subject: [PATCH 07/41] fix typo --- .github/workflows/dataconnect.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index b11a3b9a55d..ef62c47ef2d 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -279,7 +279,7 @@ jobs: working-directory: firebase-dataconnect/ci run: pytest --verbose --full-trace --color=no --strict-config - pytnon-ci-lint: + python-ci-lint: continue-on-error: false runs-on: ubuntu-latest steps: @@ -295,7 +295,7 @@ jobs: working-directory: firebase-dataconnect/ci run: ruff check --diff --verbose --no-cache --output-format=github --exit-non-zero-on-fix - pytnon-ci-format: + python-ci-format: continue-on-error: false runs-on: ubuntu-latest steps: From 44ea96fae032344d6affed2f1e7f90ae5e108f86 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Tue, 15 Apr 2025 16:23:53 -0400 Subject: [PATCH 08/41] fixes --- firebase-dataconnect/ci/notify_issue.py | 50 ++++++++++++++++++------- firebase-dataconnect/ci/pyproject.toml | 4 ++ 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/firebase-dataconnect/ci/notify_issue.py b/firebase-dataconnect/ci/notify_issue.py index 18352f47d2d..05df4e926de 100644 --- a/firebase-dataconnect/ci/notify_issue.py +++ b/firebase-dataconnect/ci/notify_issue.py @@ -12,58 +12,70 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + import argparse import dataclasses import logging import sys import typing -from collections.abc import Sequence +from typing import override + +if typing.TYPE_CHECKING: + from collections.abc import Sequence + from typing import Never, TextIO + + from _typeshed import SupportsWrite type ExitCode = int -def main(args: Sequence[str], stdout: typing.TextIO, stderr: typing.TextIO) -> ExitCode: + +def main(args: Sequence[str], stdout: TextIO, stderr: TextIO) -> ExitCode: try: parsed_args = parse_args(args[0], args[1:], stdout) except MyArgumentParser.Error as e: if e.exit_code != 0: print(f"ERROR: invalid command-line arguments: {e}", file=stderr) - print(f"Run with --help for help", file=stderr) + print("Run with --help for help", file=stderr) return e.exit_code + print(f"Successfully parsed arguments: {parsed_args!r}") return 0 + @dataclasses.dataclass(frozen=True) class GetIssueNumberCommand: github_ref: str github_event_name: str default_github_issue: int + @dataclasses.dataclass(frozen=True) class ParsedArgs: log_level: int command: GetIssueNumberCommand -class MyArgumentParser(argparse.ArgumentParser): - def __init__(self, prog: str, stdout: typing.TextIO) -> None: +class MyArgumentParser(argparse.ArgumentParser): + def __init__(self, prog: str, stdout: SupportsWrite[str]) -> None: super().__init__(prog=prog, usage="%(prog)s [options]") self.stdout = stdout - @typing.override - def exit(self, status: int = 0, message: str | None = None) -> typing.Never: + @override + def exit(self, status: int = 0, message: str | None = None) -> Never: raise self.Error(exit_code=status, message=message) - @typing.override - def error(self, message: str) -> typing.Never: + @override + def error(self, message: str) -> Never: self.exit(2, message) - @typing.override - def print_usage(self, file: typing.TextIO | None = None) -> None: + @override + def print_usage(self, file: SupportsWrite[str] | None = None) -> None: file = file if file is not None else self.stdout super().print_usage(file) - @typing.override - def print_help(self, file: typing.TextIO | None =None) -> None: + @override + def print_help(self, file: SupportsWrite[str] | None = None) -> None: file = file if file is not None else self.stdout super().print_help(file) @@ -73,9 +85,19 @@ def __init__(self, exit_code: ExitCode, message: str | None) -> None: self.exit_code = exit_code -def parse_args(prog: str, args: Sequence[str], stdout: typing.TextIO) -> ExitCode: +def parse_args(prog: str, args: Sequence[str], stdout: TextIO) -> ParsedArgs: arg_parser = MyArgumentParser(prog, stdout) parsed_args = arg_parser.parse_args(args) + print(f"parsed_args: {parsed_args!r}") + return ParsedArgs( + log_level=logging.INFO, + command=GetIssueNumberCommand( + github_ref="sample_github_ref", + github_event_name="sample_github_event_name", + default_github_issue=123456, + ), + ) + if __name__ == "__main__": try: diff --git a/firebase-dataconnect/ci/pyproject.toml b/firebase-dataconnect/ci/pyproject.toml index bbea6c41d59..8700f43d3e5 100644 --- a/firebase-dataconnect/ci/pyproject.toml +++ b/firebase-dataconnect/ci/pyproject.toml @@ -14,6 +14,10 @@ indent-width = 2 select = ["ALL"] ignore = [ "D100", # Missing docstring in public module + "D101", # Missing docstring in public class + "D103", # Missing docstring in public function + "D106", # Missing docstring in public nested class + "D107", # Missing docstring in `__init__` "D211", # no-blank-line-before-class "D212", # multi-line-summary-second-line "T201", # print` found From 9db34707efd215d9cedbb4bcaa4cbde6bb45c8d8 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Tue, 15 Apr 2025 16:31:25 -0400 Subject: [PATCH 09/41] notify_issue_test.py added --- firebase-dataconnect/ci/notify_issue_test.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 firebase-dataconnect/ci/notify_issue_test.py diff --git a/firebase-dataconnect/ci/notify_issue_test.py b/firebase-dataconnect/ci/notify_issue_test.py new file mode 100644 index 00000000000..e1865d714d4 --- /dev/null +++ b/firebase-dataconnect/ci/notify_issue_test.py @@ -0,0 +1,19 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + + +def test() -> None: + pass From 407983a6d7f3f62340870bea8417649789756fa4 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Tue, 15 Apr 2025 16:45:33 -0400 Subject: [PATCH 10/41] dataconnect.yml: remove `readonly` modifier to fix error: `pr_number: readonly variable` --- .github/workflows/dataconnect.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index a9240d84d77..b321e85662a 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -283,20 +283,18 @@ jobs: # debugging/development purposes of the comment-posting logic itself. # For example, issue number 6863 was specifically created for testing. # (see https://github.com/firebase/firebase-android-sdk/issues/6863) - readonly pr_number pr_number=$(echo '${{ github.ref }}' | sed -En 's#^refs/pull/([0-9]+)/merge$#\1#p') if [[ -n $pr_number ]] ; then - readonly issue_from_pr_body issue_from_pr_body=$(gh issue view "$pr_number" --json body --jq '.body' | sed -En 's#(\s|^)trksmnkncd_notification_issue=([0-9]+)(\s|$)#\2#p') fi if [[ -v $issue_from_pr_body ]] && [[ -n $issue_from_pr_body ]] ; then - readonly issue="$issue_from_pr_body" + issue="$issue_from_pr_body" elif [[ '${{ github.event_name }}' == 'schedule' ]] ; then # See https://github.com/firebase/firebase-android-sdk/issues/6857 - readonly issue=6857 + issue=6857 else - readonly issue= + issue= fi echo "issue=$issue" >> "$GITHUB_OUTPUT" From 97229fb2dc943a7b3100de5d7a113525d6f8150f Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Tue, 15 Apr 2025 17:01:45 -0400 Subject: [PATCH 11/41] dataconnect.yml: add `-R ${{ github.repository }}` to `gh` command line --- .github/workflows/dataconnect.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index b321e85662a..c79338992a6 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -256,6 +256,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: show-progress: false + sparse-checkout: '.github/' - uses: docker://rhysd/actionlint:1.7.7 with: args: -color /github/workspace/.github/workflows/dataconnect.yml @@ -285,7 +286,7 @@ jobs: # (see https://github.com/firebase/firebase-android-sdk/issues/6863) pr_number=$(echo '${{ github.ref }}' | sed -En 's#^refs/pull/([0-9]+)/merge$#\1#p') if [[ -n $pr_number ]] ; then - issue_from_pr_body=$(gh issue view "$pr_number" --json body --jq '.body' | sed -En 's#(\s|^)trksmnkncd_notification_issue=([0-9]+)(\s|$)#\2#p') + issue_from_pr_body=$(gh issue view "$pr_number" --json body --jq '.body' -R ${{ github.repository }} | sed -En 's#(\s|^)trksmnkncd_notification_issue=([0-9]+)(\s|$)#\2#p') fi if [[ -v $issue_from_pr_body ]] && [[ -n $issue_from_pr_body ]] ; then From 4cef939cdb270b3f1404c26f492db354336bb67c Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 11:43:47 -0400 Subject: [PATCH 12/41] dataconnect.yml: fix `[[ -v $issue_from_pr_body ]]` to `[[ -v issue_from_pr_body ]]` (remove the `$`) --- .github/workflows/dataconnect.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index c79338992a6..b7648526a79 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -289,7 +289,7 @@ jobs: issue_from_pr_body=$(gh issue view "$pr_number" --json body --jq '.body' -R ${{ github.repository }} | sed -En 's#(\s|^)trksmnkncd_notification_issue=([0-9]+)(\s|$)#\2#p') fi - if [[ -v $issue_from_pr_body ]] && [[ -n $issue_from_pr_body ]] ; then + if [[ -v issue_from_pr_body ]] && [[ -n $issue_from_pr_body ]] ; then issue="$issue_from_pr_body" elif [[ '${{ github.event_name }}' == 'schedule' ]] ; then # See https://github.com/firebase/firebase-android-sdk/issues/6857 From e679e1d6d6519ec7e059e8a70e0050c422a1c2e8 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 12:27:51 -0400 Subject: [PATCH 13/41] calculate_github_issue_for_commenting.py added (incomplete) --- .../calculate_github_issue_for_commenting.py | 109 +++++++++++++++ firebase-dataconnect/ci/notify_issue.py | 128 ++++++++---------- firebase-dataconnect/ci/pyproject.toml | 3 + 3 files changed, 167 insertions(+), 73 deletions(-) create mode 100644 firebase-dataconnect/ci/calculate_github_issue_for_commenting.py diff --git a/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py b/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py new file mode 100644 index 00000000000..05df4e926de --- /dev/null +++ b/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py @@ -0,0 +1,109 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import argparse +import dataclasses +import logging +import sys +import typing +from typing import override + +if typing.TYPE_CHECKING: + from collections.abc import Sequence + from typing import Never, TextIO + + from _typeshed import SupportsWrite + +type ExitCode = int + + +def main(args: Sequence[str], stdout: TextIO, stderr: TextIO) -> ExitCode: + try: + parsed_args = parse_args(args[0], args[1:], stdout) + except MyArgumentParser.Error as e: + if e.exit_code != 0: + print(f"ERROR: invalid command-line arguments: {e}", file=stderr) + print("Run with --help for help", file=stderr) + return e.exit_code + + print(f"Successfully parsed arguments: {parsed_args!r}") + return 0 + + +@dataclasses.dataclass(frozen=True) +class GetIssueNumberCommand: + github_ref: str + github_event_name: str + default_github_issue: int + + +@dataclasses.dataclass(frozen=True) +class ParsedArgs: + log_level: int + command: GetIssueNumberCommand + + +class MyArgumentParser(argparse.ArgumentParser): + def __init__(self, prog: str, stdout: SupportsWrite[str]) -> None: + super().__init__(prog=prog, usage="%(prog)s [options]") + self.stdout = stdout + + @override + def exit(self, status: int = 0, message: str | None = None) -> Never: + raise self.Error(exit_code=status, message=message) + + @override + def error(self, message: str) -> Never: + self.exit(2, message) + + @override + def print_usage(self, file: SupportsWrite[str] | None = None) -> None: + file = file if file is not None else self.stdout + super().print_usage(file) + + @override + def print_help(self, file: SupportsWrite[str] | None = None) -> None: + file = file if file is not None else self.stdout + super().print_help(file) + + class Error(Exception): + def __init__(self, exit_code: ExitCode, message: str | None) -> None: + super().__init__(message) + self.exit_code = exit_code + + +def parse_args(prog: str, args: Sequence[str], stdout: TextIO) -> ParsedArgs: + arg_parser = MyArgumentParser(prog, stdout) + parsed_args = arg_parser.parse_args(args) + print(f"parsed_args: {parsed_args!r}") + return ParsedArgs( + log_level=logging.INFO, + command=GetIssueNumberCommand( + github_ref="sample_github_ref", + github_event_name="sample_github_event_name", + default_github_issue=123456, + ), + ) + + +if __name__ == "__main__": + try: + exit_code = main(sys.argv, sys.stdout, sys.stderr) + except KeyboardInterrupt: + print("ERROR: application terminated by keyboard interrupt", file=sys.stderr) + exit_code = 1 + + sys.exit(exit_code) diff --git a/firebase-dataconnect/ci/notify_issue.py b/firebase-dataconnect/ci/notify_issue.py index 05df4e926de..d7383e4ae9a 100644 --- a/firebase-dataconnect/ci/notify_issue.py +++ b/firebase-dataconnect/ci/notify_issue.py @@ -15,95 +15,77 @@ from __future__ import annotations import argparse -import dataclasses import logging +import re import sys import typing -from typing import override if typing.TYPE_CHECKING: - from collections.abc import Sequence - from typing import Never, TextIO - from _typeshed import SupportsWrite type ExitCode = int -def main(args: Sequence[str], stdout: TextIO, stderr: TextIO) -> ExitCode: - try: - parsed_args = parse_args(args[0], args[1:], stdout) - except MyArgumentParser.Error as e: - if e.exit_code != 0: - print(f"ERROR: invalid command-line arguments: {e}", file=stderr) - print("Run with --help for help", file=stderr) - return e.exit_code +def main() -> None: + args = parse_args() + logging.basicConfig(format="%(message)s", level=logging.INFO) + run(args, sys.stdout) + + +def run(args: ParsedArgs, stdout: SupportsWrite[str]) -> None: + + # Determine the number of the pull request triggering this job, if any. - print(f"Successfully parsed arguments: {parsed_args!r}") - return 0 +def parse_github_ref(github_ref: str) -> int | None: + logging.info("Checking the value of github_ref for a PR number: %s", args.github_ref) + match = re.fullmatch("refs/pull/([0-9]+)/merge", args.github_ref) + if not match: + return None -@dataclasses.dataclass(frozen=True) -class GetIssueNumberCommand: + + +class ParsedArgs(typing.Protocol): github_ref: str + github_repository: str github_event_name: str - default_github_issue: int - - -@dataclasses.dataclass(frozen=True) -class ParsedArgs: - log_level: int - command: GetIssueNumberCommand - - -class MyArgumentParser(argparse.ArgumentParser): - def __init__(self, prog: str, stdout: SupportsWrite[str]) -> None: - super().__init__(prog=prog, usage="%(prog)s [options]") - self.stdout = stdout - - @override - def exit(self, status: int = 0, message: str | None = None) -> Never: - raise self.Error(exit_code=status, message=message) - - @override - def error(self, message: str) -> Never: - self.exit(2, message) - - @override - def print_usage(self, file: SupportsWrite[str] | None = None) -> None: - file = file if file is not None else self.stdout - super().print_usage(file) - - @override - def print_help(self, file: SupportsWrite[str] | None = None) -> None: - file = file if file is not None else self.stdout - super().print_help(file) - - class Error(Exception): - def __init__(self, exit_code: ExitCode, message: str | None) -> None: - super().__init__(message) - self.exit_code = exit_code - - -def parse_args(prog: str, args: Sequence[str], stdout: TextIO) -> ParsedArgs: - arg_parser = MyArgumentParser(prog, stdout) - parsed_args = arg_parser.parse_args(args) - print(f"parsed_args: {parsed_args!r}") - return ParsedArgs( - log_level=logging.INFO, - command=GetIssueNumberCommand( - github_ref="sample_github_ref", - github_event_name="sample_github_event_name", - default_github_issue=123456, - ), + pr_body_github_issue_key: str + github_issue_for_scheduled_run: str + + +def parse_args() -> ParsedArgs: + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + "--github-ref", + required=True, + help="The value of ${{ github.ref }} in the workflow", + ) + arg_parser.add_argument( + "--github-repository", + required=True, + help="The value of ${{ github.repository }} in the workflow", + ) + arg_parser.add_argument( + "--github-event-name", + required=True, + help="The value of ${{ github.event_name }} in the workflow", + ) + arg_parser.add_argument( + "--pr-body-github-issue-key", + required=True, + help="The string to search for in a Pull Request body to determine the GitHub Issue number " + "for commenting. For example, if the value is 'foobar' then this script searched a PR " + "body for a line of the form 'foobar=NNNN' where 'NNNN' is the GitHub issue number", + ) + arg_parser.add_argument( + "--github-issue-for-scheduled-run", + required=True, + help="The GitHub Issue number to use for commenting when --github-event-name is 'schedule'", ) + parse_result = arg_parser.parse_args() + return typing.cast("ParsedArgs", parse_result) -if __name__ == "__main__": - try: - exit_code = main(sys.argv, sys.stdout, sys.stderr) - except KeyboardInterrupt: - print("ERROR: application terminated by keyboard interrupt", file=sys.stderr) - exit_code = 1 - sys.exit(exit_code) +if __name__ == "__main__": + main() diff --git a/firebase-dataconnect/ci/pyproject.toml b/firebase-dataconnect/ci/pyproject.toml index 8700f43d3e5..3527f47a7fc 100644 --- a/firebase-dataconnect/ci/pyproject.toml +++ b/firebase-dataconnect/ci/pyproject.toml @@ -13,11 +13,14 @@ indent-width = 2 [tool.ruff.lint] select = ["ALL"] ignore = [ + "COM812", # missing-trailing-comma "D100", # Missing docstring in public module "D101", # Missing docstring in public class + "D102", # Missing docstring in public method "D103", # Missing docstring in public function "D106", # Missing docstring in public nested class "D107", # Missing docstring in `__init__` + "D203", # incorrect-blank-line-before-class "D211", # no-blank-line-before-class "D212", # multi-line-summary-second-line "T201", # print` found From dcf440e21291476295cf0eb89211dced39c88527 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 14:14:34 -0400 Subject: [PATCH 14/41] work --- .../calculate_github_issue_for_commenting.py | 121 +++++++----------- ...culate_github_issue_for_commenting_test.py | 82 ++++++++++++ firebase-dataconnect/ci/notify_issue.py | 91 ------------- firebase-dataconnect/ci/notify_issue_test.py | 19 --- firebase-dataconnect/ci/pyproject.toml | 11 ++ 5 files changed, 140 insertions(+), 184 deletions(-) create mode 100644 firebase-dataconnect/ci/calculate_github_issue_for_commenting_test.py delete mode 100644 firebase-dataconnect/ci/notify_issue.py delete mode 100644 firebase-dataconnect/ci/notify_issue_test.py diff --git a/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py b/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py index 05df4e926de..d9d3fd8b250 100644 --- a/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py +++ b/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py @@ -15,95 +15,68 @@ from __future__ import annotations import argparse -import dataclasses import logging -import sys +import re import typing -from typing import override -if typing.TYPE_CHECKING: - from collections.abc import Sequence - from typing import Never, TextIO +type ExitCode = int - from _typeshed import SupportsWrite -type ExitCode = int +def main() -> None: + args = parse_args() + logging.basicConfig(format="%(message)s", level=logging.INFO) + logging.info("Extracting PR number from ${{ github.ref }}: %s", args.github_ref) + pr_number: int | None = pr_number_from_github_ref(args.github_ref) + logging.info("Extracted PR number: %s", pr_number) -def main(args: Sequence[str], stdout: TextIO, stderr: TextIO) -> ExitCode: - try: - parsed_args = parse_args(args[0], args[1:], stdout) - except MyArgumentParser.Error as e: - if e.exit_code != 0: - print(f"ERROR: invalid command-line arguments: {e}", file=stderr) - print("Run with --help for help", file=stderr) - return e.exit_code - print(f"Successfully parsed arguments: {parsed_args!r}") - return 0 +def pr_number_from_github_ref(github_ref: str) -> int | None: + match = re.fullmatch("refs/pull/([0-9]+)/merge", github_ref) + return int(match.group(1)) if match else None -@dataclasses.dataclass(frozen=True) -class GetIssueNumberCommand: +class ParsedArgs(typing.Protocol): github_ref: str + github_repository: str github_event_name: str - default_github_issue: int - - -@dataclasses.dataclass(frozen=True) -class ParsedArgs: - log_level: int - command: GetIssueNumberCommand - - -class MyArgumentParser(argparse.ArgumentParser): - def __init__(self, prog: str, stdout: SupportsWrite[str]) -> None: - super().__init__(prog=prog, usage="%(prog)s [options]") - self.stdout = stdout - - @override - def exit(self, status: int = 0, message: str | None = None) -> Never: - raise self.Error(exit_code=status, message=message) - - @override - def error(self, message: str) -> Never: - self.exit(2, message) - - @override - def print_usage(self, file: SupportsWrite[str] | None = None) -> None: - file = file if file is not None else self.stdout - super().print_usage(file) + pr_body_github_issue_key: str + github_issue_for_scheduled_run: str - @override - def print_help(self, file: SupportsWrite[str] | None = None) -> None: - file = file if file is not None else self.stdout - super().print_help(file) - class Error(Exception): - def __init__(self, exit_code: ExitCode, message: str | None) -> None: - super().__init__(message) - self.exit_code = exit_code - - -def parse_args(prog: str, args: Sequence[str], stdout: TextIO) -> ParsedArgs: - arg_parser = MyArgumentParser(prog, stdout) - parsed_args = arg_parser.parse_args(args) - print(f"parsed_args: {parsed_args!r}") - return ParsedArgs( - log_level=logging.INFO, - command=GetIssueNumberCommand( - github_ref="sample_github_ref", - github_event_name="sample_github_event_name", - default_github_issue=123456, - ), +def parse_args() -> ParsedArgs: + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + "--github-ref", + required=True, + help="The value of ${{ github.ref }} in the workflow", + ) + arg_parser.add_argument( + "--github-repository", + required=True, + help="The value of ${{ github.repository }} in the workflow", + ) + arg_parser.add_argument( + "--github-event-name", + required=True, + help="The value of ${{ github.event_name }} in the workflow", + ) + arg_parser.add_argument( + "--pr-body-github-issue-key", + required=True, + help="The string to search for in a Pull Request body to determine the GitHub Issue number " + "for commenting. For example, if the value is 'foobar' then this script searched a PR " + "body for a line of the form 'foobar=NNNN' where 'NNNN' is the GitHub issue number", + ) + arg_parser.add_argument( + "--github-issue-for-scheduled-run", + required=True, + help="The GitHub Issue number to use for commenting when --github-event-name is 'schedule'", ) + parse_result = arg_parser.parse_args() + return typing.cast("ParsedArgs", parse_result) -if __name__ == "__main__": - try: - exit_code = main(sys.argv, sys.stdout, sys.stderr) - except KeyboardInterrupt: - print("ERROR: application terminated by keyboard interrupt", file=sys.stderr) - exit_code = 1 - sys.exit(exit_code) +if __name__ == "__main__": + main() diff --git a/firebase-dataconnect/ci/calculate_github_issue_for_commenting_test.py b/firebase-dataconnect/ci/calculate_github_issue_for_commenting_test.py new file mode 100644 index 00000000000..decb16d994d --- /dev/null +++ b/firebase-dataconnect/ci/calculate_github_issue_for_commenting_test.py @@ -0,0 +1,82 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import hypothesis +import hypothesis.strategies as st +import pytest + +import calculate_github_issue_for_commenting as sut + + +class Test_pr_number_from_github_ref: + @hypothesis.given(number=st.integers(min_value=0, max_value=10000)) + def test_returns_number_from_valid_github_ref(self, number: int) -> None: + github_ref = f"refs/pull/{number}/merge" + assert sut.pr_number_from_github_ref(github_ref) == number + + @hypothesis.given(invalid_github_ref=st.text()) + def test_returns_none_on_random_input(self, invalid_github_ref: str) -> None: + assert sut.pr_number_from_github_ref(invalid_github_ref) is None + + @pytest.mark.parametrize( + "invalid_number", + [ + "", + "123a", + "a123", + "12a34", + "1.2", + pytest.param( + "1234", + marks=pytest.mark.xfail( + reason="make sure that the test would otherwise pass on valid int values", + strict=True, + ), + ), + ], + ) + def test_returns_none_on_invalid_number(self, invalid_number: str) -> None: + invalid_github_ref = f"refs/pull/{invalid_number}/merge" + assert sut.pr_number_from_github_ref(invalid_github_ref) is None + + @pytest.mark.parametrize( + "malformed_ref", + [ + "", + "refs", + "refs/", + "refs/pull", + "refs/pull/", + "refs/pull/1234", + "refs/pull/1234/", + "Refs/pull/1234/merge", + "refs/Pull/1234/merge", + "refs/pull/1234/Merge", + "Arefs/pull/1234/merge", + "refs/pull/1234/mergeZ", + " refs/pull/1234/merge", + "refs/pull/1234/merge ", + pytest.param( + "refs/pull/1234/merge", + marks=pytest.mark.xfail( + reason="make sure that the test would otherwise pass on valid ref", + strict=True, + ), + ), + ], + ) + def test_returns_none_on_malformed_ref(self, malformed_ref: str) -> None: + assert sut.pr_number_from_github_ref(malformed_ref) is None diff --git a/firebase-dataconnect/ci/notify_issue.py b/firebase-dataconnect/ci/notify_issue.py deleted file mode 100644 index d7383e4ae9a..00000000000 --- a/firebase-dataconnect/ci/notify_issue.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import argparse -import logging -import re -import sys -import typing - -if typing.TYPE_CHECKING: - from _typeshed import SupportsWrite - -type ExitCode = int - - -def main() -> None: - args = parse_args() - logging.basicConfig(format="%(message)s", level=logging.INFO) - run(args, sys.stdout) - - -def run(args: ParsedArgs, stdout: SupportsWrite[str]) -> None: - - # Determine the number of the pull request triggering this job, if any. - - -def parse_github_ref(github_ref: str) -> int | None: - logging.info("Checking the value of github_ref for a PR number: %s", args.github_ref) - match = re.fullmatch("refs/pull/([0-9]+)/merge", args.github_ref) - if not match: - return None - - - -class ParsedArgs(typing.Protocol): - github_ref: str - github_repository: str - github_event_name: str - pr_body_github_issue_key: str - github_issue_for_scheduled_run: str - - -def parse_args() -> ParsedArgs: - arg_parser = argparse.ArgumentParser() - arg_parser.add_argument( - "--github-ref", - required=True, - help="The value of ${{ github.ref }} in the workflow", - ) - arg_parser.add_argument( - "--github-repository", - required=True, - help="The value of ${{ github.repository }} in the workflow", - ) - arg_parser.add_argument( - "--github-event-name", - required=True, - help="The value of ${{ github.event_name }} in the workflow", - ) - arg_parser.add_argument( - "--pr-body-github-issue-key", - required=True, - help="The string to search for in a Pull Request body to determine the GitHub Issue number " - "for commenting. For example, if the value is 'foobar' then this script searched a PR " - "body for a line of the form 'foobar=NNNN' where 'NNNN' is the GitHub issue number", - ) - arg_parser.add_argument( - "--github-issue-for-scheduled-run", - required=True, - help="The GitHub Issue number to use for commenting when --github-event-name is 'schedule'", - ) - - parse_result = arg_parser.parse_args() - return typing.cast("ParsedArgs", parse_result) - - -if __name__ == "__main__": - main() diff --git a/firebase-dataconnect/ci/notify_issue_test.py b/firebase-dataconnect/ci/notify_issue_test.py deleted file mode 100644 index e1865d714d4..00000000000 --- a/firebase-dataconnect/ci/notify_issue_test.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - - -def test() -> None: - pass diff --git a/firebase-dataconnect/ci/pyproject.toml b/firebase-dataconnect/ci/pyproject.toml index 3527f47a7fc..88a88416bed 100644 --- a/firebase-dataconnect/ci/pyproject.toml +++ b/firebase-dataconnect/ci/pyproject.toml @@ -2,6 +2,10 @@ name = "Firebase Data Connect Android SDK Continuous Integration Tools" requires-python = ">= 3.13" +[tool.pytest.ini_options] +addopts = "--strict-markers" +markers = [ "paramaterize" ] + [tool.pyright] include = ["**/*.py"] typeCheckingMode = "strict" @@ -23,9 +27,16 @@ ignore = [ "D203", # incorrect-blank-line-before-class "D211", # no-blank-line-before-class "D212", # multi-line-summary-second-line + "LOG015", # root-logger-call "T201", # print` found ] +[tool.ruff.lint.per-file-ignores] +"*_test.py" = [ + "N801", # invalid-class-name + "S101", # Use of `assert` detected +] + [tool.ruff.format] quote-style = "double" indent-style = "space" From 7dbe558f38fd4ecffb9633c8528cab320fdfc356 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 14:18:16 -0400 Subject: [PATCH 15/41] README.md added --- firebase-dataconnect/ci/README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 firebase-dataconnect/ci/README.md diff --git a/firebase-dataconnect/ci/README.md b/firebase-dataconnect/ci/README.md new file mode 100644 index 00000000000..251427e85f9 --- /dev/null +++ b/firebase-dataconnect/ci/README.md @@ -0,0 +1,21 @@ +# Firebase Data Connect Android SDK Continuous Integration Scripts + +These scripts are used by GitHub Actions. + +There are GitHub Actions workflows that verify code formatting, lint checks, type annotations, +and running unit tests. + +The minimum required Python version (at the time of writing, April 2025) is 3.13. +See `pyproject.toml` for the most up-to-date requirement. + +To set up Python, install the required dependencies by running: + +``` +pip install -r requirements.txt +``` + +Then, run all of these presubmit checks by running the following command: + +``` +ruff check --fix && ruff format && pyright && pytest && echo 'SUCCESS!!!!!!!!!!!!!!!' +``` From 9d434f2257063170846c5a335bebab0edfa59f2d Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 14:20:41 -0400 Subject: [PATCH 16/41] work --- .../ci/calculate_github_issue_for_commenting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py b/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py index d9d3fd8b250..8b53b1cc98f 100644 --- a/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py +++ b/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py @@ -26,7 +26,7 @@ def main() -> None: args = parse_args() logging.basicConfig(format="%(message)s", level=logging.INFO) - logging.info("Extracting PR number from ${{ github.ref }}: %s", args.github_ref) + logging.info("Extracting PR number from string: %s", args.github_ref) pr_number: int | None = pr_number_from_github_ref(args.github_ref) logging.info("Extracted PR number: %s", pr_number) From dcdaef7411565def8c4bd02b5186803c45ad8f4f Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 15:53:47 -0400 Subject: [PATCH 17/41] work --- .../calculate_github_issue_for_commenting.py | 91 ++++++++++++++++++- ...culate_github_issue_for_commenting_test.py | 80 ++++++++++++++++ firebase-dataconnect/ci/pyproject.toml | 2 +- 3 files changed, 167 insertions(+), 6 deletions(-) diff --git a/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py b/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py index 8b53b1cc98f..d859f3abec2 100644 --- a/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py +++ b/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py @@ -17,18 +17,69 @@ import argparse import logging import re +import subprocess import typing -type ExitCode = int +if typing.TYPE_CHECKING: + from collections.abc import Iterable def main() -> None: args = parse_args() logging.basicConfig(format="%(message)s", level=logging.INFO) - logging.info("Extracting PR number from string: %s", args.github_ref) - pr_number: int | None = pr_number_from_github_ref(args.github_ref) - logging.info("Extracted PR number: %s", pr_number) + github_issue = calculate_github_issue( + github_event_name=args.github_event_name, + github_issue_for_scheduled_run=args.github_issue_for_scheduled_run, + github_ref=args.github_ref, + github_repository=args.github_repository, + pr_body_github_issue_key=args.pr_body_github_issue_key, + ) + + logging.info(github_issue) + + +def calculate_github_issue( + github_event_name: str, + github_issue_for_scheduled_run: int, + github_ref: str, + github_repository: str, + pr_body_github_issue_key: str, +) -> int | None: + if github_event_name == "schedule": + logging.info( + "GitHub Event name is: %s; using GitHub Issue: %s", + github_event_name, + github_issue_for_scheduled_run, + ) + return github_issue_for_scheduled_run + + logging.info("Extracting PR number from string: %s", github_ref) + pr_number: int | None = pr_number_from_github_ref(github_ref) + + if pr_number is None: + logging.info("No PR number extracted") + return None + + logging.info("PR number extracted: %s", pr_number) + logging.info("Loading body text of PR: %s", pr_number) + pr_body_text = load_pr_body( + pr_number=pr_number, + github_repository=github_repository, + ) + + logging.info("Looking for GitHub Issue key in PR body text: %s=NNNN", pr_body_github_issue_key) + github_issue = github_issue_from_pr_body( + pr_body=pr_body_text, + issue_key=pr_body_github_issue_key, + ) + + if github_issue is None: + logging.info("No GitHub Issue key found in PR body") + return None + + logging.info("Found GitHub Issue key in PR body: %s", github_issue) + return github_issue def pr_number_from_github_ref(github_ref: str) -> int | None: @@ -36,12 +87,41 @@ def pr_number_from_github_ref(github_ref: str) -> int | None: return int(match.group(1)) if match else None +def load_pr_body(pr_number: int, github_repository: str) -> str: + gh_args = log_pr_body_gh_args(pr_number=pr_number, github_repository=github_repository) + gh_args = tuple(gh_args) + logging.info("Running command: %s", subprocess.list2cmdline(gh_args)) + return subprocess.check_output(gh_args, encoding="utf8", errors="replace") # noqa: S603 + + +def log_pr_body_gh_args(pr_number: int, github_repository: str) -> Iterable[str]: + yield "gh" + yield "issue" + yield "view" + yield str(pr_number) + yield "--json" + yield "body" + yield "--jq" + yield ".body" + yield "-R" + yield github_repository + + +def github_issue_from_pr_body(pr_body: str, issue_key: str) -> int | None: + expr = re.compile(r"\s*" + re.escape(issue_key) + r"\s*=\s*(\d+)\s*") + for line in pr_body.splitlines(): + match = expr.fullmatch(line.strip()) + if match: + return int(match.group(1)) + return None + + class ParsedArgs(typing.Protocol): github_ref: str github_repository: str github_event_name: str pr_body_github_issue_key: str - github_issue_for_scheduled_run: str + github_issue_for_scheduled_run: int def parse_args() -> ParsedArgs: @@ -70,6 +150,7 @@ def parse_args() -> ParsedArgs: ) arg_parser.add_argument( "--github-issue-for-scheduled-run", + type=int, required=True, help="The GitHub Issue number to use for commenting when --github-event-name is 'schedule'", ) diff --git a/firebase-dataconnect/ci/calculate_github_issue_for_commenting_test.py b/firebase-dataconnect/ci/calculate_github_issue_for_commenting_test.py index decb16d994d..a7e1df7da03 100644 --- a/firebase-dataconnect/ci/calculate_github_issue_for_commenting_test.py +++ b/firebase-dataconnect/ci/calculate_github_issue_for_commenting_test.py @@ -27,6 +27,11 @@ def test_returns_number_from_valid_github_ref(self, number: int) -> None: github_ref = f"refs/pull/{number}/merge" assert sut.pr_number_from_github_ref(github_ref) == number + @hypothesis.given(number=st.integers(min_value=0, max_value=10000)) + def test_ignores_leading_zeroes(self, number: int) -> None: + github_ref = f"refs/pull/0{number}/merge" + assert sut.pr_number_from_github_ref(github_ref) == number + @hypothesis.given(invalid_github_ref=st.text()) def test_returns_none_on_random_input(self, invalid_github_ref: str) -> None: assert sut.pr_number_from_github_ref(invalid_github_ref) is None @@ -35,6 +40,7 @@ def test_returns_none_on_random_input(self, invalid_github_ref: str) -> None: "invalid_number", [ "", + "-1", "123a", "a123", "12a34", @@ -80,3 +86,77 @@ def test_returns_none_on_invalid_number(self, invalid_number: str) -> None: ) def test_returns_none_on_malformed_ref(self, malformed_ref: str) -> None: assert sut.pr_number_from_github_ref(malformed_ref) is None + + +class Test_github_issue_from_pr_body: + @hypothesis.given(number=st.integers(min_value=0, max_value=10000)) + def test_returns_number(self, number: int) -> None: + text = f"zzyzx={number}" + assert sut.github_issue_from_pr_body(text, "zzyzx") == number + + @hypothesis.given(number=st.integers(min_value=0, max_value=10000)) + def test_ignores_leading_zeroes(self, number: int) -> None: + text = f"zzyzx=0{number}" + assert sut.github_issue_from_pr_body(text, "zzyzx") == number + + @hypothesis.given(number=st.integers(min_value=0, max_value=10000)) + def test_ignores_whitespace(self, number: int) -> None: + text = f" zzyzx = {number} " + assert sut.github_issue_from_pr_body(text, "zzyzx") == number + + @hypothesis.given( + number1=st.integers(min_value=0, max_value=10000), + number2=st.integers(min_value=0, max_value=10000), + ) + def test_does_not_ignore_whitespace_in_key(self, number1: int, number2: int) -> None: + text = f"zzyzx={number1}\n z z y z x = {number2} " + assert sut.github_issue_from_pr_body(text, "z z y z x") == number2 + + @hypothesis.given( + number1=st.integers(min_value=0, max_value=10000), + number2=st.integers(min_value=0, max_value=10000), + ) + def test_returns_first_number_ignoring_second(self, number1: int, number2: int) -> None: + text = f"zzyzx={number1}\nzzyzx={number2}" + assert sut.github_issue_from_pr_body(text, "zzyzx") == number1 + + @hypothesis.given(number=st.integers(min_value=0, max_value=10000)) + def test_returns_first_valid_number_ignoring_invalid(self, number: int) -> None: + text = f"zzyzx=12X34\nzzyzx={number}" + assert sut.github_issue_from_pr_body(text, "zzyzx") == number + + @hypothesis.given(number=st.integers(min_value=0, max_value=10000)) + def test_returns_number_amidst_other_lines(self, number: int) -> None: + text = f"line 1\nline 2\nzzyzx={number}\nline 3" + assert sut.github_issue_from_pr_body(text, "zzyzx") == number + + @hypothesis.given(number=st.integers(min_value=0, max_value=10000)) + def test_returns_escapes_regex_special_chars_in_key(self, number: int) -> None: + text = f"*+={number}" + assert sut.github_issue_from_pr_body(text, "*+") == number + + @pytest.mark.parametrize( + "text", + [ + "", + "asdf", + "zzyzx=", + "=zzyzx", + "zzyzx=a", + "zzyzx=-1", + "zzyzx=a123", + "zzyzx=123a", + "zzyzx=1.2", + "a zzyzx=1234", + "zzyzx=1234 a", + pytest.param( + "zzyzx=1234", + marks=pytest.mark.xfail( + reason="make sure that the test would otherwise pass on valid text", + strict=True, + ), + ), + ], + ) + def test_returns_none_when_key_not_found_or_cannot_parse_int(self, text: str) -> None: + assert sut.github_issue_from_pr_body(text, "zzyzx") is None diff --git a/firebase-dataconnect/ci/pyproject.toml b/firebase-dataconnect/ci/pyproject.toml index 88a88416bed..f22b30f525b 100644 --- a/firebase-dataconnect/ci/pyproject.toml +++ b/firebase-dataconnect/ci/pyproject.toml @@ -27,8 +27,8 @@ ignore = [ "D203", # incorrect-blank-line-before-class "D211", # no-blank-line-before-class "D212", # multi-line-summary-second-line + "E501", # Line too long (will be fixed by the formatter) "LOG015", # root-logger-call - "T201", # print` found ] [tool.ruff.lint.per-file-ignores] From 34f82ac02aa9859dbe8ec96f6a74aa01e3216639 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 16:46:05 -0400 Subject: [PATCH 18/41] post_comment_for_job_results.py added --- .../ci/post_comment_for_job_results.py | 188 ++++++++++++++++++ firebase-dataconnect/ci/pyproject.toml | 3 + 2 files changed, 191 insertions(+) create mode 100644 firebase-dataconnect/ci/post_comment_for_job_results.py diff --git a/firebase-dataconnect/ci/post_comment_for_job_results.py b/firebase-dataconnect/ci/post_comment_for_job_results.py new file mode 100644 index 00000000000..11ff5fb3efb --- /dev/null +++ b/firebase-dataconnect/ci/post_comment_for_job_results.py @@ -0,0 +1,188 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import argparse +import dataclasses +import logging +import pathlib +import subprocess +import tempfile +import typing + +if typing.TYPE_CHECKING: + from collections.abc import Iterable, Sequence + + +def main() -> None: + args = parse_args() + logging.basicConfig(format="%(message)s", level=logging.INFO) + + message_lines = tuple(generate_message_lines(args)) + + issue_url = f"{args.github_repository_html_url}/issues/{args.github_issue}" + logging.info("Posting the following comment to GitHub Issue %s", issue_url) + for line in message_lines: + logging.info(line) + + message_bytes = "\n".join(message_lines).encode("utf8", errors="replace") + with tempfile.TemporaryDirectory() as tempdir_path: + message_file = pathlib.Path(tempdir_path) / "message_text.txt" + message_file.write_bytes(message_bytes) + post_github_issue_comment( + issue_number=args.github_issue, + body_file=message_file, + github_repository=args.github_repository, + ) + + +def generate_message_lines(data: ParsedArgs) -> Iterable[str]: + yield f"Result of workflows at {data.github_sha}:" + + for job_result in data.job_results: + result_symbol = "✅" if job_result.result == "success" else "❌" + yield f" - {job_result.job_id}: {result_symbol} {job_result.result}" + + yield "" + yield f"{data.github_repository_html_url}/actions/runs/{data.github_run_id}" + + yield "" + yield ( + f"run_id={data.github_run_id} " + f"run_number={data.github_run_number} " + f"run_attempt={data.github_run_attempt}" + ) + + +def post_github_issue_comment( + issue_number: int, body_file: pathlib.Path, github_repository: str +) -> None: + gh_args = post_issue_comment_gh_args( + issue_number=issue_number, body_file=body_file, github_repository=github_repository + ) + gh_args = tuple(gh_args) + logging.info("Running command: %s", subprocess.list2cmdline(gh_args)) + subprocess.check_call(gh_args) # noqa: S603 + + +def post_issue_comment_gh_args( + issue_number: int, body_file: pathlib.Path, github_repository: str +) -> Iterable[str]: + yield "gh" + yield "issue" + yield "comment" + yield str(issue_number) + yield "--body-file" + yield str(body_file) + yield "-R" + yield github_repository + + +@dataclasses.dataclass(frozen=True) +class JobResult: + job_id: str + result: str + + @classmethod + def parse(cls, s: str) -> JobResult: + colon_index = s.find(":") + if colon_index < 0: + raise ParseError( + "no colon (:) character found in job result specfication, " + "which is required to delimit the job ID from the job result" + ) + job_id = s[:colon_index] + job_result = s[colon_index + 1 :] + return cls(job_id=job_id, result=job_result) + + +class ParsedArgs(typing.Protocol): + job_results: Sequence[JobResult] + github_issue: int + github_repository: str + github_sha: str + github_repository_html_url: str + github_run_id: str + github_run_number: str + github_run_attempt: str + + +class ParseError(Exception): + pass + + +def parse_args() -> ParsedArgs: + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + "job_results", + nargs="+", + help="The results of the jobs in question, of the form " + "'job-id:${{ needs.job-id.result }}' where 'job-id' is the id of the corresponding job " + "in the 'needs' section of the job.", + ) + arg_parser.add_argument( + "--github-issue", + required=True, + help="The GitHub Issue number to which to post a comment", + ) + arg_parser.add_argument( + "--github-repository", + required=True, + help="The value of ${{ github.repository }} in the workflow", + ) + arg_parser.add_argument( + "--github-sha", + required=True, + help="The value of ${{ github.sha }} in the workflow", + ) + arg_parser.add_argument( + "--github-repository-html-url", + required=True, + help="The value of ${{ github.event.repository.html_url }} in the workflow", + ) + arg_parser.add_argument( + "--github-run-id", + required=True, + help="The value of ${{ github.run_id }} in the workflow", + ) + arg_parser.add_argument( + "--github-run-number", + required=True, + help="The value of ${{ github.run_number }} in the workflow", + ) + arg_parser.add_argument( + "--github-run-attempt", + required=True, + help="The value of ${{ github.run_attempt }} in the workflow", + ) + + parse_result = arg_parser.parse_args() + + job_results: list[JobResult] = [] + for job_result_str in parse_result.job_results: + try: + job_result = JobResult.parse(job_result_str) + except ParseError as e: + arg_parser.error(f"invalid job result specification: {job_result_str} ({e})") + typing.assert_never("the line above should have raised an exception") + else: + job_results.append(job_result) + parse_result.job_results = tuple(job_results) + + return typing.cast("ParsedArgs", parse_result) + + +if __name__ == "__main__": + main() diff --git a/firebase-dataconnect/ci/pyproject.toml b/firebase-dataconnect/ci/pyproject.toml index f22b30f525b..85fcfb0f2db 100644 --- a/firebase-dataconnect/ci/pyproject.toml +++ b/firebase-dataconnect/ci/pyproject.toml @@ -28,7 +28,9 @@ ignore = [ "D211", # no-blank-line-before-class "D212", # multi-line-summary-second-line "E501", # Line too long (will be fixed by the formatter) + "EM101", # Exception must not use a string literal, assign to variable first "LOG015", # root-logger-call + "TRY003", # Avoid specifying long messages outside the exception class ] [tool.ruff.lint.per-file-ignores] @@ -42,3 +44,4 @@ quote-style = "double" indent-style = "space" skip-magic-trailing-comma = false docstring-code-format = true +SUCCESS!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! From 63fdb92c2374bdc66135ca7e7af36c4d29aaf2bf Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 16:55:56 -0400 Subject: [PATCH 19/41] dataconnect.yaml: use calculate_github_issue_for_commenting.py --- .github/workflows/dataconnect.yml | 42 +++++++++---------- .../calculate_github_issue_for_commenting.py | 13 +++++- firebase-dataconnect/ci/pyproject.toml | 1 - 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index 47154258b95..91ce3c41275 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -334,36 +334,36 @@ jobs: issues: write runs-on: ubuntu-latest steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + show-progress: false + sparse-checkout: 'firebase-dataconnect/ci/' + + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + with: + python-version: ${{ env.FDC_PYTHON_VERSION }} + + - run: pip install -r firebase-dataconnect/ci/requirements.txt + - id: issue-id name: Determine GitHub Issue For Commenting env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + working-directory: firebase-dataconnect/ci run: | set -euo pipefail set -xv - # If this job was triggered by a pull request, then check to see if - # the description of the PR specifies a GitHub Issue to which to post - # a comment. The GitHub Issue number is specified by including a line - # of the form "trksmnkncd_notification_issue=6863" in the description. - # Specifying an issue number in this manner is primarily intended for - # debugging/development purposes of the comment-posting logic itself. - # For example, issue number 6863 was specifically created for testing. - # (see https://github.com/firebase/firebase-android-sdk/issues/6863) - pr_number=$(echo '${{ github.ref }}' | sed -En 's#^refs/pull/([0-9]+)/merge$#\1#p') - if [[ -n $pr_number ]] ; then - issue_from_pr_body=$(gh issue view "$pr_number" --json body --jq '.body' -R ${{ github.repository }} | sed -En 's#(\s|^)trksmnkncd_notification_issue=([0-9]+)(\s|$)#\2#p') - fi - - if [[ -v issue_from_pr_body ]] && [[ -n $issue_from_pr_body ]] ; then - issue="$issue_from_pr_body" - elif [[ '${{ github.event_name }}' == 'schedule' ]] ; then - # See https://github.com/firebase/firebase-android-sdk/issues/6857 - issue=6857 - else - issue= - fi + python \ + calculate_github_issue_for_commenting.py \ + --output-file=github_issue_number.txt \ + --github-ref='${{ github.ref }}' \ + --github-repository='${{ github.repository }}' \ + --github-event-name='${{ github.event_name }}' \ + --pr-body-github-issue-key=trksmnkncd_notification_issue \ + --github-issue-for-scheduled-run=6857 # https://github.com/firebase/firebase-android-sdk/issues/6857 + issue="$(cat github_issue_number.txt)" echo "issue=$issue" >> "$GITHUB_OUTPUT" - name: Post Comment on GitHub Issue diff --git a/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py b/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py index d859f3abec2..4c2c224d3b6 100644 --- a/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py +++ b/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py @@ -16,6 +16,7 @@ import argparse import logging +import pathlib import re import subprocess import typing @@ -36,7 +37,9 @@ def main() -> None: pr_body_github_issue_key=args.pr_body_github_issue_key, ) - logging.info(github_issue) + file_text = "" if github_issue is None else str(github_issue) + logging.info("Writing '%s' to %s", file_text, args.output_file) + args.output_file.write_text(file_text, encoding="utf8", errors="replace") def calculate_github_issue( @@ -117,6 +120,7 @@ def github_issue_from_pr_body(pr_body: str, issue_key: str) -> int | None: class ParsedArgs(typing.Protocol): + output_file: pathlib.Path github_ref: str github_repository: str github_event_name: str @@ -126,6 +130,12 @@ class ParsedArgs(typing.Protocol): def parse_args() -> ParsedArgs: arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + "--output-file", + required=True, + help="The file to which to write the calculated issue number" + "if no issue number was found, then an empty file will be written", + ) arg_parser.add_argument( "--github-ref", required=True, @@ -156,6 +166,7 @@ def parse_args() -> ParsedArgs: ) parse_result = arg_parser.parse_args() + parse_result.output_file = pathlib.Path(parse_result.output_file) return typing.cast("ParsedArgs", parse_result) diff --git a/firebase-dataconnect/ci/pyproject.toml b/firebase-dataconnect/ci/pyproject.toml index 85fcfb0f2db..a380c3a1ad2 100644 --- a/firebase-dataconnect/ci/pyproject.toml +++ b/firebase-dataconnect/ci/pyproject.toml @@ -44,4 +44,3 @@ quote-style = "double" indent-style = "space" skip-magic-trailing-comma = false docstring-code-format = true -SUCCESS!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! From efd400c6901ef961097ddf5901b40946cb510506 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 17:03:03 -0400 Subject: [PATCH 20/41] dataconnect.yml: use post_comment_for_job_results.py --- .github/workflows/dataconnect.yml | 39 +++++++++++++++++++------------ 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index 91ce3c41275..bdb05cd0f0d 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -328,7 +328,13 @@ jobs: run: pyright --warnings --stats send-notifications: - needs: [integration-test, actionlint-dataconnect-yml] + needs: + - 'integration-test' + - 'actionlint-dataconnect-yml' + - 'python-ci-unit-tests' + - 'python-ci-lint' + - 'python-ci-format' + - 'python-ci-type-check' if: always() permissions: issues: write @@ -357,8 +363,8 @@ jobs: python \ calculate_github_issue_for_commenting.py \ --output-file=github_issue_number.txt \ - --github-ref='${{ github.ref }}' \ --github-repository='${{ github.repository }}' \ + --github-ref='${{ github.ref }}' \ --github-event-name='${{ github.event_name }}' \ --pr-body-github-issue-key=trksmnkncd_notification_issue \ --github-issue-for-scheduled-run=6857 # https://github.com/firebase/firebase-android-sdk/issues/6857 @@ -372,17 +378,20 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail + set -xv - cat >message.txt < Date: Wed, 16 Apr 2025 17:09:36 -0400 Subject: [PATCH 21/41] fix working directory --- .github/workflows/dataconnect.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index bdb05cd0f0d..6ef2ac85a71 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -355,13 +355,12 @@ jobs: name: Determine GitHub Issue For Commenting env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - working-directory: firebase-dataconnect/ci run: | set -euo pipefail set -xv python \ - calculate_github_issue_for_commenting.py \ + firebase-dataconnect/ci/calculate_github_issue_for_commenting.py \ --output-file=github_issue_number.txt \ --github-repository='${{ github.repository }}' \ --github-ref='${{ github.ref }}' \ @@ -381,7 +380,7 @@ jobs: set -xv exec python \ - post_comment_for_job_results.py \ + firebase-dataconnect/ci/post_comment_for_job_results.py \ --github-issue='${{ steps.issue-id.outputs.issue }}' \ --github-repository='${{ github.repository }}' \ --github-sha='${{ github.sha }}' \ From 2ef276340d0ee14e3e494a660a9f56eb384c79d6 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 17:16:56 -0400 Subject: [PATCH 22/41] requirements.txt updated --- firebase-dataconnect/ci/requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/firebase-dataconnect/ci/requirements.txt b/firebase-dataconnect/ci/requirements.txt index 9825cdd3d53..3440d3b19f8 100644 --- a/firebase-dataconnect/ci/requirements.txt +++ b/firebase-dataconnect/ci/requirements.txt @@ -1,3 +1,5 @@ +attrs==25.3.0 +hypothesis==6.131.0 iniconfig==2.1.0 nodeenv==1.9.1 packaging==24.2 @@ -5,4 +7,5 @@ pluggy==1.5.0 pyright==1.1.399 pytest==8.3.5 ruff==0.11.5 +sortedcontainers==2.4.0 typing_extensions==4.13.2 From 5713afb380f351a8095b56f5ff66552ddeb2dcfd Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 17:50:36 -0400 Subject: [PATCH 23/41] dataconnect_demo_app.yml: post notifications like dataconnect.yml does --- .github/workflows/dataconnect_demo_app.yml | 68 ++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/.github/workflows/dataconnect_demo_app.yml b/.github/workflows/dataconnect_demo_app.yml index 35a5079c96e..d20bee3fe99 100644 --- a/.github/workflows/dataconnect_demo_app.yml +++ b/.github/workflows/dataconnect_demo_app.yml @@ -8,6 +8,7 @@ on: javaVersion: gradleInfoLog: type: boolean + pythonVersion: pull_request: paths: - firebase-dataconnect/demo/** @@ -21,6 +22,7 @@ env: FDC_JAVA_VERSION: ${{ inputs.javaVersion || '17' }} FDC_FIREBASE_TOOLS_DIR: ${{ github.workspace }}/firebase-tools FDC_FIREBASE_COMMAND: ${{ github.workspace }}/firebase-tools/node_modules/.bin/firebase + FDC_PYTHON_VERSION: ${{ inputs.pythonVersion || '3.13' }} defaults: run: @@ -167,3 +169,69 @@ jobs: --no-daemon \ ${{ (inputs.gradleInfoLog && '--info') || '' }} \ spotlessCheck + + send-notifications: + needs: + - 'test' + - 'spotlessCheck' + if: always() + permissions: + issues: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + show-progress: false + sparse-checkout: 'firebase-dataconnect/ci/' + + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + with: + python-version: ${{ env.FDC_PYTHON_VERSION }} + + - run: pip install -r requirements.txt + working-directory: firebase-dataconnect/ci + + - id: issue-id + name: Determine GitHub Issue For Commenting + working-directory: firebase-dataconnect/ci + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + args=( + python + calculate_github_issue_for_commenting.py + --output-file=github_issue_number.txt + --github-repository='${{ github.repository }}' + --github-ref='${{ github.ref }}' + --github-event-name='${{ github.event_name }}' + --pr-body-github-issue-key=trksmnkncd_notification_issue + --github-issue-for-scheduled-run=6891 # https://github.com/firebase/firebase-android-sdk/issues/6891 + ) + echo "${args[*]}" + "${args[@]}" + + set -xv + issue="$(cat github_issue_number.txt)" + echo "issue=$issue" >> "$GITHUB_OUTPUT" + + - name: Post Comment on GitHub Issue + if: steps.issue-id.outputs.issue != '' + working-directory: firebase-dataconnect/ci + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + args=( + python + post_comment_for_job_results.py + --github-issue='${{ steps.issue-id.outputs.issue }}' + --github-repository='${{ github.repository }}' + --github-sha='${{ github.sha }}' + --github-repository-html-url='${{ github.event.repository.html_url }}' + --github-run-id='${{ github.run_id }}' + --github-run-number='${{ github.run_number }}' + --github-run-attempt='${{ github.run_attempt }}' + 'test:${{ needs.test.result }}' + 'spotlessCheck:${{ needs.spotlessCheck.result }}' + ) + echo "${args[*]}" + exec "${args[@]}" From d242e6784109693d47601c8df161e6492421c186 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 17:57:00 -0400 Subject: [PATCH 24/41] dataconnect.yml: code cleanup to match dataconnect_demo_app.yml --- .github/workflows/dataconnect.yml | 66 +++++++++++++++++-------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index 6ef2ac85a71..af04a8c6747 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -328,7 +328,7 @@ jobs: run: pyright --warnings --stats send-notifications: - needs: + needs: - 'integration-test' - 'actionlint-dataconnect-yml' - 'python-ci-unit-tests' @@ -349,48 +349,54 @@ jobs: with: python-version: ${{ env.FDC_PYTHON_VERSION }} - - run: pip install -r firebase-dataconnect/ci/requirements.txt + - run: pip install -r requirements.txt + working-directory: firebase-dataconnect/ci - id: issue-id name: Determine GitHub Issue For Commenting + working-directory: firebase-dataconnect/ci env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - set -euo pipefail - set -xv - - python \ - firebase-dataconnect/ci/calculate_github_issue_for_commenting.py \ - --output-file=github_issue_number.txt \ - --github-repository='${{ github.repository }}' \ - --github-ref='${{ github.ref }}' \ - --github-event-name='${{ github.event_name }}' \ - --pr-body-github-issue-key=trksmnkncd_notification_issue \ + args=( + python + calculate_github_issue_for_commenting.py + --output-file=github_issue_number.txt + --github-repository='${{ github.repository }}' + --github-ref='${{ github.ref }}' + --github-event-name='${{ github.event_name }}' + --pr-body-github-issue-key=trksmnkncd_notification_issue --github-issue-for-scheduled-run=6857 # https://github.com/firebase/firebase-android-sdk/issues/6857 + ) + echo "${args[*]}" + "${args[@]}" + set -xv issue="$(cat github_issue_number.txt)" echo "issue=$issue" >> "$GITHUB_OUTPUT" - name: Post Comment on GitHub Issue if: steps.issue-id.outputs.issue != '' + working-directory: firebase-dataconnect/ci env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - set -euo pipefail - set -xv - - exec python \ - firebase-dataconnect/ci/post_comment_for_job_results.py \ - --github-issue='${{ steps.issue-id.outputs.issue }}' \ - --github-repository='${{ github.repository }}' \ - --github-sha='${{ github.sha }}' \ - --github-repository-html-url='${{ github.event.repository.html_url }}' \ - --github-run-id='${{ github.run_id }}' \ - --github-run-number='${{ github.run_number }}' \ - --github-run-attempt='${{ github.run_attempt }}' \ - 'integration-test:${{ needs.integration-test.result }}' \ - 'actionlint-dataconnect-yml:${{ needs.actionlint-dataconnect-yml.result }}' \ - 'python-ci-unit-tests:${{ needs.python-ci-unit-tests.result }}' \ - 'python-ci-lint:${{ needs.python-ci-lint.result }}' \ - 'python-ci-format:${{ needs.python-ci-format.result }}' \ - 'python-ci-type-check:${{ needs.python-ci-type-check.result }}' \ + args=( + python + post_comment_for_job_results.py + --github-issue='${{ steps.issue-id.outputs.issue }}' + --github-repository='${{ github.repository }}' + --github-sha='${{ github.sha }}' + --github-repository-html-url='${{ github.event.repository.html_url }}' + --github-run-id='${{ github.run_id }}' + --github-run-number='${{ github.run_number }}' + --github-run-attempt='${{ github.run_attempt }}' + 'integration-test:${{ needs.integration-test.result }}' + 'actionlint-dataconnect-yml:${{ needs.actionlint-dataconnect-yml.result }}' + 'python-ci-unit-tests:${{ needs.python-ci-unit-tests.result }}' + 'python-ci-lint:${{ needs.python-ci-lint.result }}' + 'python-ci-format:${{ needs.python-ci-format.result }}' + 'python-ci-type-check:${{ needs.python-ci-type-check.result }}' + ) + echo "${args[*]}" + exec "${args[@]}" From 4d5f2159423ab57352a859147fbf6d79758ffc4b Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 18:11:29 -0400 Subject: [PATCH 25/41] work --- .../calculate_github_issue_for_commenting.py | 18 ++++++++---- .../ci/post_comment_for_job_results.py | 29 ++++++++++++++++++- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py b/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py index 4c2c224d3b6..7fb20138cd2 100644 --- a/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py +++ b/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py @@ -38,8 +38,8 @@ def main() -> None: ) file_text = "" if github_issue is None else str(github_issue) - logging.info("Writing '%s' to %s", file_text, args.output_file) - args.output_file.write_text(file_text, encoding="utf8", errors="replace") + logging.info("Writing '%s' to %s", file_text, args.issue_output_file) + args.issue_output_file.write_text(file_text, encoding="utf8", errors="replace") def calculate_github_issue( @@ -120,7 +120,8 @@ def github_issue_from_pr_body(pr_body: str, issue_key: str) -> int | None: class ParsedArgs(typing.Protocol): - output_file: pathlib.Path + issue_output_file: pathlib.Path + pr_output_file: pathlib.Path github_ref: str github_repository: str github_event_name: str @@ -131,11 +132,17 @@ class ParsedArgs(typing.Protocol): def parse_args() -> ParsedArgs: arg_parser = argparse.ArgumentParser() arg_parser.add_argument( - "--output-file", + "--issue-output-file", required=True, help="The file to which to write the calculated issue number" "if no issue number was found, then an empty file will be written", ) + arg_parser.add_argument( + "--pr-output-file", + required=True, + help="The file to which to write the calculated triggering PR number" + "if no PR was found, then an empty file will be written", + ) arg_parser.add_argument( "--github-ref", required=True, @@ -166,7 +173,8 @@ def parse_args() -> ParsedArgs: ) parse_result = arg_parser.parse_args() - parse_result.output_file = pathlib.Path(parse_result.output_file) + parse_result.issue_output_file = pathlib.Path(parse_result.issue_output_file) + parse_result.pr_output_file = pathlib.Path(parse_result.pr_output_file) return typing.cast("ParsedArgs", parse_result) diff --git a/firebase-dataconnect/ci/post_comment_for_job_results.py b/firebase-dataconnect/ci/post_comment_for_job_results.py index 11ff5fb3efb..667cd302c28 100644 --- a/firebase-dataconnect/ci/post_comment_for_job_results.py +++ b/firebase-dataconnect/ci/post_comment_for_job_results.py @@ -49,7 +49,20 @@ def main() -> None: def generate_message_lines(data: ParsedArgs) -> Iterable[str]: - yield f"Result of workflows at {data.github_sha}:" + pr_str = data.triggering_pr.strip() + pr: int | None + if len(pr) == 0: + pr = None + else: + try: + pr = int(pr) + except ValueError: + logging.warning("WARNING: unable to parse PR number as an int: %s", pr) + pr = None + + yield f"Posting from Pull Request {pr}: {data.github_repository_html_url}/pull/{pr}" + + yield f"Result of workflow '{data.github_workflow}' at {data.github_sha}:" for job_result in data.job_results: result_symbol = "✅" if job_result.result == "success" else "❌" @@ -82,6 +95,7 @@ def post_issue_comment_gh_args( ) -> Iterable[str]: yield "gh" yield "issue" + yield "comment" yield str(issue_number) yield "--body-file" @@ -112,11 +126,13 @@ class ParsedArgs(typing.Protocol): job_results: Sequence[JobResult] github_issue: int github_repository: str + github_workflow: str github_sha: str github_repository_html_url: str github_run_id: str github_run_number: str github_run_attempt: str + triggering_pr: str class ParseError(Exception): @@ -134,14 +150,25 @@ def parse_args() -> ParsedArgs: ) arg_parser.add_argument( "--github-issue", + type=int, required=True, help="The GitHub Issue number to which to post a comment", ) + arg_parser.add_argument( + "--triggering-pr", + required=True, + help="The GitHub Pull Request number that triggered the workflow, or empty if not applicable.", + ) arg_parser.add_argument( "--github-repository", required=True, help="The value of ${{ github.repository }} in the workflow", ) + arg_parser.add_argument( + "--github-workflow", + required=True, + help="The value of ${{ github.workflow }} in the workflow", + ) arg_parser.add_argument( "--github-sha", required=True, From 5211fdfaac8226147f83ead3a45acc00652f2600 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 19:46:38 -0400 Subject: [PATCH 26/41] add pr info --- .../calculate_github_issue_for_commenting.py | 40 ++++++++++++++----- .../ci/post_comment_for_job_results.py | 7 ++-- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py b/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py index 7fb20138cd2..7ea125cd2b8 100644 --- a/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py +++ b/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py @@ -15,6 +15,7 @@ from __future__ import annotations import argparse +import dataclasses import logging import pathlib import re @@ -29,7 +30,7 @@ def main() -> None: args = parse_args() logging.basicConfig(format="%(message)s", level=logging.INFO) - github_issue = calculate_github_issue( + calculated_result = calculate_github_issue( github_event_name=args.github_event_name, github_issue_for_scheduled_run=args.github_issue_for_scheduled_run, github_ref=args.github_ref, @@ -37,9 +38,22 @@ def main() -> None: pr_body_github_issue_key=args.pr_body_github_issue_key, ) - file_text = "" if github_issue is None else str(github_issue) - logging.info("Writing '%s' to %s", file_text, args.issue_output_file) - args.issue_output_file.write_text(file_text, encoding="utf8", errors="replace") + github_issue = calculated_result.issue + github_pr = calculated_result.pr + + issue_file_text = "" if github_issue is None else str(github_issue) + logging.info("Writing '%s' to %s", issue_file_text, args.issue_output_file) + args.issue_output_file.write_text(issue_file_text, encoding="utf8", errors="replace") + + pr_file_text = "" if github_pr is None else str(github_pr) + logging.info("Writing '%s' to %s", pr_file_text, args.pr_output_file) + args.pr_output_file.write_text(pr_file_text, encoding="utf8", errors="replace") + + +@dataclasses.dataclass(frozen=True) +class CalculatedGitHubIssue: + issue: int | None + pr: int | None def calculate_github_issue( @@ -48,21 +62,24 @@ def calculate_github_issue( github_ref: str, github_repository: str, pr_body_github_issue_key: str, -) -> int | None: +) -> CalculatedGitHubIssue: if github_event_name == "schedule": logging.info( "GitHub Event name is: %s; using GitHub Issue: %s", github_event_name, github_issue_for_scheduled_run, ) - return github_issue_for_scheduled_run + return CalculatedGitHubIssue( + issue=github_issue_for_scheduled_run, + pr=None, # scheduled runs are, by definition, not associated with a PR. + ) logging.info("Extracting PR number from string: %s", github_ref) - pr_number: int | None = pr_number_from_github_ref(github_ref) - + pr_number = pr_number_from_github_ref(github_ref) if pr_number is None: logging.info("No PR number extracted") - return None + return CalculatedGitHubIssue(None, None) + typing.assert_type(pr_number, int) logging.info("PR number extracted: %s", pr_number) logging.info("Loading body text of PR: %s", pr_number) @@ -79,10 +96,11 @@ def calculate_github_issue( if github_issue is None: logging.info("No GitHub Issue key found in PR body") - return None + return CalculatedGitHubIssue(issue=None, pr=pr_number) + typing.assert_type(github_issue, int) logging.info("Found GitHub Issue key in PR body: %s", github_issue) - return github_issue + return CalculatedGitHubIssue(issue=github_issue, pr=pr_number) def pr_number_from_github_ref(github_ref: str) -> int | None: diff --git a/firebase-dataconnect/ci/post_comment_for_job_results.py b/firebase-dataconnect/ci/post_comment_for_job_results.py index 667cd302c28..83a33210139 100644 --- a/firebase-dataconnect/ci/post_comment_for_job_results.py +++ b/firebase-dataconnect/ci/post_comment_for_job_results.py @@ -49,15 +49,14 @@ def main() -> None: def generate_message_lines(data: ParsedArgs) -> Iterable[str]: - pr_str = data.triggering_pr.strip() pr: int | None - if len(pr) == 0: + if len(data.triggering_pr) == 0: pr = None else: try: - pr = int(pr) + pr = int(data.triggering_pr) except ValueError: - logging.warning("WARNING: unable to parse PR number as an int: %s", pr) + logging.warning("WARNING: unable to parse PR number as an int: %s", data.triggering_pr) pr = None yield f"Posting from Pull Request {pr}: {data.github_repository_html_url}/pull/{pr}" From 41c1500a80171da0cdd992c155ffefb0d109fdaa Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 19:51:28 -0400 Subject: [PATCH 27/41] update workflow files for new command-line args --- .github/workflows/dataconnect.yml | 7 ++++++- .github/workflows/dataconnect_demo_app.yml | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index af04a8c6747..b6cb05b394d 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -361,7 +361,8 @@ jobs: args=( python calculate_github_issue_for_commenting.py - --output-file=github_issue_number.txt + --issue-output-file=github_issue_number.txt + --pr-output-file=github_pr_number.txt --github-repository='${{ github.repository }}' --github-ref='${{ github.ref }}' --github-event-name='${{ github.event_name }}' @@ -374,6 +375,8 @@ jobs: set -xv issue="$(cat github_issue_number.txt)" echo "issue=$issue" >> "$GITHUB_OUTPUT" + pr="$(cat github_pr_number.txt)" + echo "pr=$pr" >> "$GITHUB_OUTPUT" - name: Post Comment on GitHub Issue if: steps.issue-id.outputs.issue != '' @@ -385,12 +388,14 @@ jobs: python post_comment_for_job_results.py --github-issue='${{ steps.issue-id.outputs.issue }}' + --github-workflow='${{ github.workflow }}' --github-repository='${{ github.repository }}' --github-sha='${{ github.sha }}' --github-repository-html-url='${{ github.event.repository.html_url }}' --github-run-id='${{ github.run_id }}' --github-run-number='${{ github.run_number }}' --github-run-attempt='${{ github.run_attempt }}' + --triggering-pr='${{ steps.issue-id.outputs.pr }}' 'integration-test:${{ needs.integration-test.result }}' 'actionlint-dataconnect-yml:${{ needs.actionlint-dataconnect-yml.result }}' 'python-ci-unit-tests:${{ needs.python-ci-unit-tests.result }}' diff --git a/.github/workflows/dataconnect_demo_app.yml b/.github/workflows/dataconnect_demo_app.yml index d20bee3fe99..edc8c13f43f 100644 --- a/.github/workflows/dataconnect_demo_app.yml +++ b/.github/workflows/dataconnect_demo_app.yml @@ -200,7 +200,8 @@ jobs: args=( python calculate_github_issue_for_commenting.py - --output-file=github_issue_number.txt + --issue-output-file=github_issue_number.txt + --pr-output-file=github_pr_number.txt --github-repository='${{ github.repository }}' --github-ref='${{ github.ref }}' --github-event-name='${{ github.event_name }}' @@ -213,6 +214,8 @@ jobs: set -xv issue="$(cat github_issue_number.txt)" echo "issue=$issue" >> "$GITHUB_OUTPUT" + pr="$(cat github_pr_number.txt)" + echo "pr=$pr" >> "$GITHUB_OUTPUT" - name: Post Comment on GitHub Issue if: steps.issue-id.outputs.issue != '' @@ -224,12 +227,14 @@ jobs: python post_comment_for_job_results.py --github-issue='${{ steps.issue-id.outputs.issue }}' + --github-workflow='${{ github.workflow }}' --github-repository='${{ github.repository }}' --github-sha='${{ github.sha }}' --github-repository-html-url='${{ github.event.repository.html_url }}' --github-run-id='${{ github.run_id }}' --github-run-number='${{ github.run_number }}' --github-run-attempt='${{ github.run_attempt }}' + --triggering-pr='${{ steps.issue-id.outputs.pr }}' 'test:${{ needs.test.result }}' 'spotlessCheck:${{ needs.spotlessCheck.result }}' ) From 8ff4c862cf8b5d39324bdda81c0ac5db0207322c Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 19:54:21 -0400 Subject: [PATCH 28/41] post_comment_for_job_results.py: avoid unnecessarily repeating the pr number --- firebase-dataconnect/ci/post_comment_for_job_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firebase-dataconnect/ci/post_comment_for_job_results.py b/firebase-dataconnect/ci/post_comment_for_job_results.py index 83a33210139..62c656bdb4e 100644 --- a/firebase-dataconnect/ci/post_comment_for_job_results.py +++ b/firebase-dataconnect/ci/post_comment_for_job_results.py @@ -59,7 +59,7 @@ def generate_message_lines(data: ParsedArgs) -> Iterable[str]: logging.warning("WARNING: unable to parse PR number as an int: %s", data.triggering_pr) pr = None - yield f"Posting from Pull Request {pr}: {data.github_repository_html_url}/pull/{pr}" + yield f"Posting from Pull Request {data.github_repository_html_url}/pull/{pr}" yield f"Result of workflow '{data.github_workflow}' at {data.github_sha}:" From e443015e3759150808c1249cdf840e5cb699f9e6 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 19:57:16 -0400 Subject: [PATCH 29/41] post_comment_for_job_results.py: add backticks around numbers so that they are not hyperlinked as phone numbers --- firebase-dataconnect/ci/post_comment_for_job_results.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/firebase-dataconnect/ci/post_comment_for_job_results.py b/firebase-dataconnect/ci/post_comment_for_job_results.py index 62c656bdb4e..f98b40cf31a 100644 --- a/firebase-dataconnect/ci/post_comment_for_job_results.py +++ b/firebase-dataconnect/ci/post_comment_for_job_results.py @@ -72,9 +72,9 @@ def generate_message_lines(data: ParsedArgs) -> Iterable[str]: yield "" yield ( - f"run_id={data.github_run_id} " - f"run_number={data.github_run_number} " - f"run_attempt={data.github_run_attempt}" + f"run_id=`{data.github_run_id}` " + f"run_number=`{data.github_run_number}` " + f"run_attempt=`{data.github_run_attempt}`" ) From 8177f762b586fc3d82ed1769d54a970c271e6316 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 22:06:52 -0400 Subject: [PATCH 30/41] README.md updated --- firebase-dataconnect/ci/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/firebase-dataconnect/ci/README.md b/firebase-dataconnect/ci/README.md index 251427e85f9..59a36dc1aa0 100644 --- a/firebase-dataconnect/ci/README.md +++ b/firebase-dataconnect/ci/README.md @@ -3,12 +3,13 @@ These scripts are used by GitHub Actions. There are GitHub Actions workflows that verify code formatting, lint checks, type annotations, -and running unit tests. +and running unit tests of code in this directory. Although they are not "required" checks, it +is requested to wait for these checks to pass. See `dataconnect.yaml`. The minimum required Python version (at the time of writing, April 2025) is 3.13. See `pyproject.toml` for the most up-to-date requirement. -To set up Python, install the required dependencies by running: +Before running the scripts, install the required dependencies by running: ``` pip install -r requirements.txt @@ -17,5 +18,5 @@ pip install -r requirements.txt Then, run all of these presubmit checks by running the following command: ``` -ruff check --fix && ruff format && pyright && pytest && echo 'SUCCESS!!!!!!!!!!!!!!!' +ruff check && ruff format && pyright && pytest && echo 'SUCCESS!!!!!!!!!!!!!!!' ``` From f746ee9255f1c03e0d8a9f879bba0036e6e50256 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 23:37:14 -0400 Subject: [PATCH 31/41] dataconnect_demo_app.yml: factor out notification sending into .github/actions/dataconnect-send-notifications/action.yml --- .../dataconnect-send-notifications/action.yml | 79 +++++++++++++++++++ .github/workflows/dataconnect_demo_app.yml | 71 +++-------------- 2 files changed, 92 insertions(+), 58 deletions(-) create mode 100644 .github/actions/dataconnect-send-notifications/action.yml diff --git a/.github/actions/dataconnect-send-notifications/action.yml b/.github/actions/dataconnect-send-notifications/action.yml new file mode 100644 index 00000000000..ab65bcdbcd5 --- /dev/null +++ b/.github/actions/dataconnect-send-notifications/action.yml @@ -0,0 +1,79 @@ +name: Data Connect Workflow Notifications +description: Notify a GitHub Issue with the results of a workflow. + +inputs: + python-version: + required: true + default: "3.13" + github-issue-for-scheduled-runs: + required: true + job-results-file: + required: true + +runs: + using: "composite" + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + show-progress: false + sparse-checkout: 'firebase-dataconnect/ci/' + + - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + with: + python-version: ${{ inputs.python-version }} + + - run: pip install -r requirements.txt + working-directory: firebase-dataconnect/ci + + - id: issue-id + name: Determine GitHub Issue For Commenting + working-directory: firebase-dataconnect/ci + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + args=( + python + calculate_github_issue_for_commenting.py + --issue-output-file=github_issue_number.txt + --pr-output-file=github_pr_number.txt + --github-repository='${{ github.repository }}' + --github-ref='${{ github.ref }}' + --github-event-name='${{ github.event_name }}' + --pr-body-github-issue-key=trksmnkncd_notification_issue + --github-issue-for-scheduled-run='${{ inputs.github-issue-for-scheduled-runs }}' + ) + echo "${args[*]}" + "${args[@]}" + + set -xv + issue="$(cat github_issue_number.txt)" + echo "issue=$issue" >> "$GITHUB_OUTPUT" + pr="$(cat github_pr_number.txt)" + echo "pr=$pr" >> "$GITHUB_OUTPUT" + + - name: Post Comment on GitHub Issue + if: steps.issue-id.outputs.issue != '' + working-directory: firebase-dataconnect/ci + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + args=( + python + post_comment_for_job_results.py + --github-issue='${{ steps.issue-id.outputs.issue }}' + --github-workflow='${{ github.workflow }}' + --github-repository='${{ github.repository }}' + --github-sha='${{ github.sha }}' + --github-repository-html-url='${{ github.event.repository.html_url }}' + --github-run-id='${{ github.run_id }}' + --github-run-number='${{ github.run_number }}' + --github-run-attempt='${{ github.run_attempt }}' + --triggering-pr='${{ steps.issue-id.outputs.pr }}' + ) + + while read -r line; do + args=("${args[@]}" "$line") + done <'${{ inputs.job-results-file }}' + + echo "${args[*]}" + exec "${args[@]}" diff --git a/.github/workflows/dataconnect_demo_app.yml b/.github/workflows/dataconnect_demo_app.yml index edc8c13f43f..0663e39407c 100644 --- a/.github/workflows/dataconnect_demo_app.yml +++ b/.github/workflows/dataconnect_demo_app.yml @@ -179,64 +179,19 @@ jobs: issues: write runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - show-progress: false - sparse-checkout: 'firebase-dataconnect/ci/' + - name: Create Job Results File + id: create-job-results-file + run: | + set -xveuo pipefail + job_results_file="$(mktemp)" - - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 - with: - python-version: ${{ env.FDC_PYTHON_VERSION }} + echo 'test:${{ needs.test.result }}' >>"$job_results_file" + echo 'spotlessCheck:${{ needs.spotlessCheck.result }}' >>"$job_results_file" - - run: pip install -r requirements.txt - working-directory: firebase-dataconnect/ci + echo "job_results_file=$job_results_file" >>"$GITHUB_OUTPUT" - - id: issue-id - name: Determine GitHub Issue For Commenting - working-directory: firebase-dataconnect/ci - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - args=( - python - calculate_github_issue_for_commenting.py - --issue-output-file=github_issue_number.txt - --pr-output-file=github_pr_number.txt - --github-repository='${{ github.repository }}' - --github-ref='${{ github.ref }}' - --github-event-name='${{ github.event_name }}' - --pr-body-github-issue-key=trksmnkncd_notification_issue - --github-issue-for-scheduled-run=6891 # https://github.com/firebase/firebase-android-sdk/issues/6891 - ) - echo "${args[*]}" - "${args[@]}" - - set -xv - issue="$(cat github_issue_number.txt)" - echo "issue=$issue" >> "$GITHUB_OUTPUT" - pr="$(cat github_pr_number.txt)" - echo "pr=$pr" >> "$GITHUB_OUTPUT" - - - name: Post Comment on GitHub Issue - if: steps.issue-id.outputs.issue != '' - working-directory: firebase-dataconnect/ci - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - args=( - python - post_comment_for_job_results.py - --github-issue='${{ steps.issue-id.outputs.issue }}' - --github-workflow='${{ github.workflow }}' - --github-repository='${{ github.repository }}' - --github-sha='${{ github.sha }}' - --github-repository-html-url='${{ github.event.repository.html_url }}' - --github-run-id='${{ github.run_id }}' - --github-run-number='${{ github.run_number }}' - --github-run-attempt='${{ github.run_attempt }}' - --triggering-pr='${{ steps.issue-id.outputs.pr }}' - 'test:${{ needs.test.result }}' - 'spotlessCheck:${{ needs.spotlessCheck.result }}' - ) - echo "${args[*]}" - exec "${args[@]}" + - uses: ./.github/actions/dataconnect-send-notifications + with: + python-version: ${{ env.FDC_PYTHON_VERSION }} + github-issue-for-scheduled-runs: "6891" + job-results-file: ${{ steps.create-job-results-file.outputs.job_results_file }} From 0f6d1488db169328a1de50d79c034eb9977a7266 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 23:43:50 -0400 Subject: [PATCH 32/41] dataconnect_demo_app.yml: remove default dir because it's causing complications for little benefit --- .github/workflows/dataconnect_demo_app.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/dataconnect_demo_app.yml b/.github/workflows/dataconnect_demo_app.yml index 0663e39407c..aaa6f82ce2c 100644 --- a/.github/workflows/dataconnect_demo_app.yml +++ b/.github/workflows/dataconnect_demo_app.yml @@ -24,11 +24,6 @@ env: FDC_FIREBASE_COMMAND: ${{ github.workspace }}/firebase-tools/node_modules/.bin/firebase FDC_PYTHON_VERSION: ${{ inputs.pythonVersion || '3.13' }} -defaults: - run: - shell: bash - working-directory: firebase-dataconnect/demo - concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -52,8 +47,8 @@ jobs: node-version: ${{ env.FDC_NODE_VERSION }} cache: 'npm' cache-dependency-path: | - firebase-dataconnect/demo/github_actions_demo_test_cache_key.txt - firebase-dataconnect/demo/github_actions_demo_assemble_firebase_tools_version.txt + github_actions_demo_test_cache_key.txt + github_actions_demo_assemble_firebase_tools_version.txt - name: cache package-lock.json id: package_json_lock @@ -86,7 +81,7 @@ jobs: firebase-dataconnect/demo/build.gradle.kts firebase-dataconnect/demo/gradle.properties firebase-dataconnect/demo/gradle/wrapper/gradle-wrapper.properties - firebase-dataconnect/demo/github_actions_demo_test_cache_key.txt + github_actions_demo_test_cache_key.txt - name: tool versions continue-on-error: true @@ -110,6 +105,7 @@ jobs: run: | set -x ./gradlew \ + --project-dir firebase-dataconnect/demo \ --no-daemon \ ${{ (inputs.gradleInfoLog && '--info') || '' }} \ --profile \ @@ -150,7 +146,7 @@ jobs: firebase-dataconnect/demo/build.gradle.kts firebase-dataconnect/demo/gradle.properties firebase-dataconnect/demo/gradle/wrapper/gradle-wrapper.properties - firebase-dataconnect/demo/github_actions_demo_spotless_cache_key.txt + github_actions_demo_spotless_cache_key.txt - name: tool versions continue-on-error: true @@ -166,6 +162,7 @@ jobs: run: | set -x ./gradlew \ + --project-dir firebase-dataconnect/demo \ --no-daemon \ ${{ (inputs.gradleInfoLog && '--info') || '' }} \ spotlessCheck From 65e4571f2a8d96bb3efb514fcb18c30610c1ded7 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 23:46:25 -0400 Subject: [PATCH 33/41] dataconnect_demo_app.yml: fix path to gradlew --- .github/workflows/dataconnect_demo_app.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dataconnect_demo_app.yml b/.github/workflows/dataconnect_demo_app.yml index aaa6f82ce2c..dc926ee1159 100644 --- a/.github/workflows/dataconnect_demo_app.yml +++ b/.github/workflows/dataconnect_demo_app.yml @@ -99,12 +99,12 @@ jobs: run_cmd which node run_cmd node --version run_cmd ${{ env.FDC_FIREBASE_COMMAND }} --version - run_cmd ./gradlew --version + run_cmd firebase-dataconnect/demo/gradlew --version - - name: ./gradlew assemble test + - name: gradle assemble test run: | set -x - ./gradlew \ + firebase-dataconnect/demo/gradlew \ --project-dir firebase-dataconnect/demo \ --no-daemon \ ${{ (inputs.gradleInfoLog && '--info') || '' }} \ @@ -156,12 +156,12 @@ jobs: java -version which javac javac -version - ./gradlew --version + firebase-dataconnect/demo/gradlew --version - - name: ./gradlew spotlessCheck + - name: gradle spotlessCheck run: | set -x - ./gradlew \ + firebase-dataconnect/demo/gradlew \ --project-dir firebase-dataconnect/demo \ --no-daemon \ ${{ (inputs.gradleInfoLog && '--info') || '' }} \ From 6e85e2f9047193c6b16766c018ca524db35387c2 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 16 Apr 2025 23:53:50 -0400 Subject: [PATCH 34/41] dataconnect_demo_app.yml: use runner.temp and checkout repo before calling the action --- .../dataconnect-send-notifications/action.yml | 5 ----- .github/workflows/dataconnect_demo_app.yml | 19 ++++++++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/actions/dataconnect-send-notifications/action.yml b/.github/actions/dataconnect-send-notifications/action.yml index ab65bcdbcd5..a38dce9e4ab 100644 --- a/.github/actions/dataconnect-send-notifications/action.yml +++ b/.github/actions/dataconnect-send-notifications/action.yml @@ -13,11 +13,6 @@ inputs: runs: using: "composite" steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - show-progress: false - sparse-checkout: 'firebase-dataconnect/ci/' - - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: ${{ inputs.python-version }} diff --git a/.github/workflows/dataconnect_demo_app.yml b/.github/workflows/dataconnect_demo_app.yml index dc926ee1159..1b498a71590 100644 --- a/.github/workflows/dataconnect_demo_app.yml +++ b/.github/workflows/dataconnect_demo_app.yml @@ -176,19 +176,24 @@ jobs: issues: write runs-on: ubuntu-latest steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + show-progress: false + sparse-checkout: | + firebase-dataconnect/ci/ + .github/actions/dataconnect-send-notifications/ + - name: Create Job Results File id: create-job-results-file run: | set -xveuo pipefail - job_results_file="$(mktemp)" - - echo 'test:${{ needs.test.result }}' >>"$job_results_file" - echo 'spotlessCheck:${{ needs.spotlessCheck.result }}' >>"$job_results_file" - - echo "job_results_file=$job_results_file" >>"$GITHUB_OUTPUT" + cat >'${{ runner.temp }}/job_results.txt' < Date: Thu, 17 Apr 2025 00:00:03 -0400 Subject: [PATCH 35/41] dataconnect_demo_app.yml/dataconnect-send-notifications/action.yml: move secrets into main workflow and explicitly specify shell:bash --- .github/actions/dataconnect-send-notifications/action.yml | 7 +++---- .github/workflows/dataconnect_demo_app.yml | 3 +++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/actions/dataconnect-send-notifications/action.yml b/.github/actions/dataconnect-send-notifications/action.yml index a38dce9e4ab..ff1718a57c9 100644 --- a/.github/actions/dataconnect-send-notifications/action.yml +++ b/.github/actions/dataconnect-send-notifications/action.yml @@ -18,13 +18,13 @@ runs: python-version: ${{ inputs.python-version }} - run: pip install -r requirements.txt + shell: bash working-directory: firebase-dataconnect/ci - id: issue-id name: Determine GitHub Issue For Commenting working-directory: firebase-dataconnect/ci - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash run: | args=( python @@ -49,8 +49,7 @@ runs: - name: Post Comment on GitHub Issue if: steps.issue-id.outputs.issue != '' working-directory: firebase-dataconnect/ci - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash run: | args=( python diff --git a/.github/workflows/dataconnect_demo_app.yml b/.github/workflows/dataconnect_demo_app.yml index 1b498a71590..64bee8a8312 100644 --- a/.github/workflows/dataconnect_demo_app.yml +++ b/.github/workflows/dataconnect_demo_app.yml @@ -183,6 +183,9 @@ jobs: firebase-dataconnect/ci/ .github/actions/dataconnect-send-notifications/ + - name: gh auth login + run: echo '${{ secrets.GITHUB_TOKEN }}' | gh auth login --with-token + - name: Create Job Results File id: create-job-results-file run: | From a18af72fe0df0caf9bbb36e71dc7a11c8e262824 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 17 Apr 2025 00:13:17 -0400 Subject: [PATCH 36/41] dataconnect.yml: use the composite action --- .github/workflows/dataconnect.yml | 90 ++++++++-------------- .github/workflows/dataconnect_demo_app.yml | 11 ++- 2 files changed, 41 insertions(+), 60 deletions(-) diff --git a/.github/workflows/dataconnect.yml b/.github/workflows/dataconnect.yml index b6cb05b394d..038145c897e 100644 --- a/.github/workflows/dataconnect.yml +++ b/.github/workflows/dataconnect.yml @@ -327,6 +327,15 @@ jobs: working-directory: firebase-dataconnect/ci run: pyright --warnings --stats + # The "send-notifications" job adds a comment to GitHub Issue + # https://github.com/firebase/firebase-android-sdk/issues/6857 with the results of the scheduled + # nightly runs. Interested parties can then subscribe to that issue to be aprised of the outcome + # of the nightly runs. + # + # When testing the comment-adding logic itself, you can add the line + # trksmnkncd_notification_issue=6863 + # into the PR's description to instead post a comment to issue #6863, an issue specifically + # created for testing, avoiding spamming the main issue to which others are subscribed. send-notifications: needs: - 'integration-test' @@ -343,65 +352,28 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: show-progress: false - sparse-checkout: 'firebase-dataconnect/ci/' - - - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 - with: - python-version: ${{ env.FDC_PYTHON_VERSION }} - - - run: pip install -r requirements.txt - working-directory: firebase-dataconnect/ci - - - id: issue-id - name: Determine GitHub Issue For Commenting - working-directory: firebase-dataconnect/ci - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - args=( - python - calculate_github_issue_for_commenting.py - --issue-output-file=github_issue_number.txt - --pr-output-file=github_pr_number.txt - --github-repository='${{ github.repository }}' - --github-ref='${{ github.ref }}' - --github-event-name='${{ github.event_name }}' - --pr-body-github-issue-key=trksmnkncd_notification_issue - --github-issue-for-scheduled-run=6857 # https://github.com/firebase/firebase-android-sdk/issues/6857 - ) - echo "${args[*]}" - "${args[@]}" + sparse-checkout: | + firebase-dataconnect/ci/ + .github/ - set -xv - issue="$(cat github_issue_number.txt)" - echo "issue=$issue" >> "$GITHUB_OUTPUT" - pr="$(cat github_pr_number.txt)" - echo "pr=$pr" >> "$GITHUB_OUTPUT" + - name: gh auth login + run: echo '${{ secrets.GITHUB_TOKEN }}' | gh auth login --with-token - - name: Post Comment on GitHub Issue - if: steps.issue-id.outputs.issue != '' - working-directory: firebase-dataconnect/ci - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Create Job Results File + id: create-job-results-file run: | - args=( - python - post_comment_for_job_results.py - --github-issue='${{ steps.issue-id.outputs.issue }}' - --github-workflow='${{ github.workflow }}' - --github-repository='${{ github.repository }}' - --github-sha='${{ github.sha }}' - --github-repository-html-url='${{ github.event.repository.html_url }}' - --github-run-id='${{ github.run_id }}' - --github-run-number='${{ github.run_number }}' - --github-run-attempt='${{ github.run_attempt }}' - --triggering-pr='${{ steps.issue-id.outputs.pr }}' - 'integration-test:${{ needs.integration-test.result }}' - 'actionlint-dataconnect-yml:${{ needs.actionlint-dataconnect-yml.result }}' - 'python-ci-unit-tests:${{ needs.python-ci-unit-tests.result }}' - 'python-ci-lint:${{ needs.python-ci-lint.result }}' - 'python-ci-format:${{ needs.python-ci-format.result }}' - 'python-ci-type-check:${{ needs.python-ci-type-check.result }}' - ) - echo "${args[*]}" - exec "${args[@]}" + set -xveuo pipefail + cat >'${{ runner.temp }}/job_results.txt' < Date: Thu, 17 Apr 2025 00:33:01 -0400 Subject: [PATCH 37/41] include pr title in posted comment --- .../dataconnect-send-notifications/action.yml | 5 +- .../calculate_github_issue_for_commenting.py | 70 +++---------------- .../ci/post_comment_for_job_results.py | 36 ++++++---- firebase-dataconnect/ci/util.py | 60 ++++++++++++++++ 4 files changed, 92 insertions(+), 79 deletions(-) create mode 100644 firebase-dataconnect/ci/util.py diff --git a/.github/actions/dataconnect-send-notifications/action.yml b/.github/actions/dataconnect-send-notifications/action.yml index ff1718a57c9..89b86859c3c 100644 --- a/.github/actions/dataconnect-send-notifications/action.yml +++ b/.github/actions/dataconnect-send-notifications/action.yml @@ -30,7 +30,6 @@ runs: python calculate_github_issue_for_commenting.py --issue-output-file=github_issue_number.txt - --pr-output-file=github_pr_number.txt --github-repository='${{ github.repository }}' --github-ref='${{ github.ref }}' --github-event-name='${{ github.event_name }}' @@ -43,8 +42,6 @@ runs: set -xv issue="$(cat github_issue_number.txt)" echo "issue=$issue" >> "$GITHUB_OUTPUT" - pr="$(cat github_pr_number.txt)" - echo "pr=$pr" >> "$GITHUB_OUTPUT" - name: Post Comment on GitHub Issue if: steps.issue-id.outputs.issue != '' @@ -57,12 +54,12 @@ runs: --github-issue='${{ steps.issue-id.outputs.issue }}' --github-workflow='${{ github.workflow }}' --github-repository='${{ github.repository }}' + --github-ref='${{ github.ref }}' --github-sha='${{ github.sha }}' --github-repository-html-url='${{ github.event.repository.html_url }}' --github-run-id='${{ github.run_id }}' --github-run-number='${{ github.run_number }}' --github-run-attempt='${{ github.run_attempt }}' - --triggering-pr='${{ steps.issue-id.outputs.pr }}' ) while read -r line; do diff --git a/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py b/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py index 7ea125cd2b8..2f71eea86ea 100644 --- a/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py +++ b/firebase-dataconnect/ci/calculate_github_issue_for_commenting.py @@ -15,22 +15,19 @@ from __future__ import annotations import argparse -import dataclasses import logging import pathlib import re -import subprocess import typing -if typing.TYPE_CHECKING: - from collections.abc import Iterable +from util import fetch_pr_info, pr_number_from_github_ref def main() -> None: args = parse_args() logging.basicConfig(format="%(message)s", level=logging.INFO) - calculated_result = calculate_github_issue( + github_issue = calculate_github_issue( github_event_name=args.github_event_name, github_issue_for_scheduled_run=args.github_issue_for_scheduled_run, github_ref=args.github_ref, @@ -38,23 +35,10 @@ def main() -> None: pr_body_github_issue_key=args.pr_body_github_issue_key, ) - github_issue = calculated_result.issue - github_pr = calculated_result.pr - issue_file_text = "" if github_issue is None else str(github_issue) logging.info("Writing '%s' to %s", issue_file_text, args.issue_output_file) args.issue_output_file.write_text(issue_file_text, encoding="utf8", errors="replace") - pr_file_text = "" if github_pr is None else str(github_pr) - logging.info("Writing '%s' to %s", pr_file_text, args.pr_output_file) - args.pr_output_file.write_text(pr_file_text, encoding="utf8", errors="replace") - - -@dataclasses.dataclass(frozen=True) -class CalculatedGitHubIssue: - issue: int | None - pr: int | None - def calculate_github_issue( github_event_name: str, @@ -62,70 +46,42 @@ def calculate_github_issue( github_ref: str, github_repository: str, pr_body_github_issue_key: str, -) -> CalculatedGitHubIssue: +) -> int | None: if github_event_name == "schedule": logging.info( "GitHub Event name is: %s; using GitHub Issue: %s", github_event_name, github_issue_for_scheduled_run, ) - return CalculatedGitHubIssue( - issue=github_issue_for_scheduled_run, - pr=None, # scheduled runs are, by definition, not associated with a PR. - ) + return github_issue_for_scheduled_run logging.info("Extracting PR number from string: %s", github_ref) pr_number = pr_number_from_github_ref(github_ref) if pr_number is None: logging.info("No PR number extracted") - return CalculatedGitHubIssue(None, None) + return None typing.assert_type(pr_number, int) logging.info("PR number extracted: %s", pr_number) logging.info("Loading body text of PR: %s", pr_number) - pr_body_text = load_pr_body( + pr_info = fetch_pr_info( pr_number=pr_number, github_repository=github_repository, ) logging.info("Looking for GitHub Issue key in PR body text: %s=NNNN", pr_body_github_issue_key) github_issue = github_issue_from_pr_body( - pr_body=pr_body_text, + pr_body=pr_info.body, issue_key=pr_body_github_issue_key, ) if github_issue is None: logging.info("No GitHub Issue key found in PR body") - return CalculatedGitHubIssue(issue=None, pr=pr_number) + return None typing.assert_type(github_issue, int) logging.info("Found GitHub Issue key in PR body: %s", github_issue) - return CalculatedGitHubIssue(issue=github_issue, pr=pr_number) - - -def pr_number_from_github_ref(github_ref: str) -> int | None: - match = re.fullmatch("refs/pull/([0-9]+)/merge", github_ref) - return int(match.group(1)) if match else None - - -def load_pr_body(pr_number: int, github_repository: str) -> str: - gh_args = log_pr_body_gh_args(pr_number=pr_number, github_repository=github_repository) - gh_args = tuple(gh_args) - logging.info("Running command: %s", subprocess.list2cmdline(gh_args)) - return subprocess.check_output(gh_args, encoding="utf8", errors="replace") # noqa: S603 - - -def log_pr_body_gh_args(pr_number: int, github_repository: str) -> Iterable[str]: - yield "gh" - yield "issue" - yield "view" - yield str(pr_number) - yield "--json" - yield "body" - yield "--jq" - yield ".body" - yield "-R" - yield github_repository + return github_issue def github_issue_from_pr_body(pr_body: str, issue_key: str) -> int | None: @@ -139,7 +95,6 @@ def github_issue_from_pr_body(pr_body: str, issue_key: str) -> int | None: class ParsedArgs(typing.Protocol): issue_output_file: pathlib.Path - pr_output_file: pathlib.Path github_ref: str github_repository: str github_event_name: str @@ -155,12 +110,6 @@ def parse_args() -> ParsedArgs: help="The file to which to write the calculated issue number" "if no issue number was found, then an empty file will be written", ) - arg_parser.add_argument( - "--pr-output-file", - required=True, - help="The file to which to write the calculated triggering PR number" - "if no PR was found, then an empty file will be written", - ) arg_parser.add_argument( "--github-ref", required=True, @@ -192,7 +141,6 @@ def parse_args() -> ParsedArgs: parse_result = arg_parser.parse_args() parse_result.issue_output_file = pathlib.Path(parse_result.issue_output_file) - parse_result.pr_output_file = pathlib.Path(parse_result.pr_output_file) return typing.cast("ParsedArgs", parse_result) diff --git a/firebase-dataconnect/ci/post_comment_for_job_results.py b/firebase-dataconnect/ci/post_comment_for_job_results.py index f98b40cf31a..0dfe0452053 100644 --- a/firebase-dataconnect/ci/post_comment_for_job_results.py +++ b/firebase-dataconnect/ci/post_comment_for_job_results.py @@ -22,6 +22,8 @@ import tempfile import typing +from util import fetch_pr_info, pr_number_from_github_ref + if typing.TYPE_CHECKING: from collections.abc import Iterable, Sequence @@ -49,17 +51,23 @@ def main() -> None: def generate_message_lines(data: ParsedArgs) -> Iterable[str]: - pr: int | None - if len(data.triggering_pr) == 0: - pr = None + logging.info("Extracting PR number from string: %s", data.github_ref) + pr_number = pr_number_from_github_ref(data.github_ref) + pr_title: str | None + if pr_number is None: + logging.info("No PR number extracted") + pr_title = None else: - try: - pr = int(data.triggering_pr) - except ValueError: - logging.warning("WARNING: unable to parse PR number as an int: %s", data.triggering_pr) - pr = None + pr_info = fetch_pr_info( + pr_number=pr_number, + github_repository=data.github_repository, + ) + pr_title = pr_info.title - yield f"Posting from Pull Request {data.github_repository_html_url}/pull/{pr}" + if pr_number is not None: + yield ( + f"Posting from Pull Request {data.github_repository_html_url}/pull/{pr_number} ({pr_title})" + ) yield f"Result of workflow '{data.github_workflow}' at {data.github_sha}:" @@ -125,13 +133,13 @@ class ParsedArgs(typing.Protocol): job_results: Sequence[JobResult] github_issue: int github_repository: str + github_ref: str github_workflow: str github_sha: str github_repository_html_url: str github_run_id: str github_run_number: str github_run_attempt: str - triggering_pr: str class ParseError(Exception): @@ -154,14 +162,14 @@ def parse_args() -> ParsedArgs: help="The GitHub Issue number to which to post a comment", ) arg_parser.add_argument( - "--triggering-pr", + "--github-repository", required=True, - help="The GitHub Pull Request number that triggered the workflow, or empty if not applicable.", + help="The value of ${{ github.repository }} in the workflow", ) arg_parser.add_argument( - "--github-repository", + "--github-ref", required=True, - help="The value of ${{ github.repository }} in the workflow", + help="The value of ${{ github.ref }} in the workflow", ) arg_parser.add_argument( "--github-workflow", diff --git a/firebase-dataconnect/ci/util.py b/firebase-dataconnect/ci/util.py new file mode 100644 index 00000000000..8196bf0023c --- /dev/null +++ b/firebase-dataconnect/ci/util.py @@ -0,0 +1,60 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import dataclasses +import json +import logging +import re +import subprocess +import typing + +if typing.TYPE_CHECKING: + from collections.abc import Iterable + + +@dataclasses.dataclass(frozen=True) +class GitHubPrInfo: + title: str + body: str + + +def fetch_pr_info(pr_number: int, github_repository: str) -> GitHubPrInfo: + gh_args = _fetch_pr_gh_args(pr_number=pr_number, github_repository=github_repository) + gh_args = tuple(gh_args) + logging.info("Running command: %s", subprocess.list2cmdline(gh_args)) + output_str = subprocess.check_output(gh_args, encoding="utf8", errors="replace") # noqa: S603 + logging.info("%s", output_str) + output = json.loads(output_str) + return GitHubPrInfo( + title=output["title"], + body=output["body"], + ) + + +def _fetch_pr_gh_args(pr_number: int, github_repository: str) -> Iterable[str]: + yield "gh" + yield "issue" + yield "view" + yield str(pr_number) + yield "--json" + yield "title,body" + yield "-R" + yield github_repository + + +def pr_number_from_github_ref(github_ref: str) -> int | None: + match = re.fullmatch("refs/pull/([0-9]+)/merge", github_ref) + return int(match.group(1)) if match else None From da2594c6cb833b6c66da2c7cd333a975faf3d482 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 17 Apr 2025 00:38:52 -0400 Subject: [PATCH 38/41] include github.event_name in posted comment --- .github/actions/dataconnect-send-notifications/action.yml | 1 + firebase-dataconnect/ci/post_comment_for_job_results.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/.github/actions/dataconnect-send-notifications/action.yml b/.github/actions/dataconnect-send-notifications/action.yml index 89b86859c3c..27133b5031c 100644 --- a/.github/actions/dataconnect-send-notifications/action.yml +++ b/.github/actions/dataconnect-send-notifications/action.yml @@ -55,6 +55,7 @@ runs: --github-workflow='${{ github.workflow }}' --github-repository='${{ github.repository }}' --github-ref='${{ github.ref }}' + --github-event-name='${{ github.event_name }}' --github-sha='${{ github.sha }}' --github-repository-html-url='${{ github.event.repository.html_url }}' --github-run-id='${{ github.run_id }}' diff --git a/firebase-dataconnect/ci/post_comment_for_job_results.py b/firebase-dataconnect/ci/post_comment_for_job_results.py index 0dfe0452053..affb0b3be26 100644 --- a/firebase-dataconnect/ci/post_comment_for_job_results.py +++ b/firebase-dataconnect/ci/post_comment_for_job_results.py @@ -80,6 +80,7 @@ def generate_message_lines(data: ParsedArgs) -> Iterable[str]: yield "" yield ( + f"event_name=`{data.github_event_name}` " f"run_id=`{data.github_run_id}` " f"run_number=`{data.github_run_number}` " f"run_attempt=`{data.github_run_attempt}`" @@ -133,6 +134,7 @@ class ParsedArgs(typing.Protocol): job_results: Sequence[JobResult] github_issue: int github_repository: str + github_event_name: str github_ref: str github_workflow: str github_sha: str @@ -166,6 +168,11 @@ def parse_args() -> ParsedArgs: required=True, help="The value of ${{ github.repository }} in the workflow", ) + arg_parser.add_argument( + "--github-event-name", + required=True, + help="The value of ${{ github.event_name }} in the workflow", + ) arg_parser.add_argument( "--github-ref", required=True, From 717dfdcd17210ce22fa880a84af2e8729e81b35d Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 17 Apr 2025 00:41:45 -0400 Subject: [PATCH 39/41] util.py: strip leading/trailing whitespace from output before printing --- firebase-dataconnect/ci/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firebase-dataconnect/ci/util.py b/firebase-dataconnect/ci/util.py index 8196bf0023c..9f71bdcb226 100644 --- a/firebase-dataconnect/ci/util.py +++ b/firebase-dataconnect/ci/util.py @@ -36,7 +36,7 @@ def fetch_pr_info(pr_number: int, github_repository: str) -> GitHubPrInfo: gh_args = tuple(gh_args) logging.info("Running command: %s", subprocess.list2cmdline(gh_args)) output_str = subprocess.check_output(gh_args, encoding="utf8", errors="replace") # noqa: S603 - logging.info("%s", output_str) + logging.info("%s", output_str.strip()) output = json.loads(output_str) return GitHubPrInfo( title=output["title"], From 5a352ebc05918d9f83a8bb40c70b58c783fd5049 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 17 Apr 2025 00:45:26 -0400 Subject: [PATCH 40/41] pyproject.toml: remove broken and unneeded line: `markers = [ "paramaterize" ]` --- firebase-dataconnect/ci/pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/firebase-dataconnect/ci/pyproject.toml b/firebase-dataconnect/ci/pyproject.toml index a380c3a1ad2..3afddaa1c6c 100644 --- a/firebase-dataconnect/ci/pyproject.toml +++ b/firebase-dataconnect/ci/pyproject.toml @@ -4,7 +4,6 @@ requires-python = ">= 3.13" [tool.pytest.ini_options] addopts = "--strict-markers" -markers = [ "paramaterize" ] [tool.pyright] include = ["**/*.py"] From 802fa362082f1e6f42eb44c02dc9fea07eaa9790 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 17 Apr 2025 00:47:31 -0400 Subject: [PATCH 41/41] fix spelling error in message: specfication --- firebase-dataconnect/ci/post_comment_for_job_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firebase-dataconnect/ci/post_comment_for_job_results.py b/firebase-dataconnect/ci/post_comment_for_job_results.py index affb0b3be26..738ac2c9b9d 100644 --- a/firebase-dataconnect/ci/post_comment_for_job_results.py +++ b/firebase-dataconnect/ci/post_comment_for_job_results.py @@ -122,7 +122,7 @@ def parse(cls, s: str) -> JobResult: colon_index = s.find(":") if colon_index < 0: raise ParseError( - "no colon (:) character found in job result specfication, " + "no colon (:) character found in job result specification, " "which is required to delimit the job ID from the job result" ) job_id = s[:colon_index]