From 930127a6a716b74c8e5bb6be21a24b3faec578e4 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 25 Sep 2024 19:32:14 +0000 Subject: [PATCH 1/4] dataconnect: upgrade toolkit version to 1.3.9 (was 1.3.8) --- .../gradle/plugin/DataConnectExecutable.kt | 13 +++++++++++++ .../gradle/plugin/DataConnectProviders.kt | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutable.kt b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutable.kt index b8730c0f8e6..22dd692e1cf 100644 --- a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutable.kt +++ b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutable.kt @@ -66,6 +66,13 @@ sealed interface DataConnectExecutable { "aea3583ebe1a36938eec5164de79405951ddf05b70a857ddb4f346f1424666f1d96" + "989a5f81326c7e2aef4a195d31ff356fdf2331ed98fa1048c4bd469cbfd97" ) + "1.3.9" -> + VerificationInfo( + fileSizeInBytes = 24_977_560L, + sha512DigestHex = + "4558928c2a84b54113e0d6918907eb75bdeb9bd059dcc4b6f22cb4a7c9c7421a357" + + "7f3b0d2eeb246b1df739b38f1eb91e5a6166b0e559707746d79e6ccdf9ed4" + ) else -> throw DataConnectGradleException( "3svd27ch8y", @@ -86,10 +93,16 @@ sealed interface DataConnectExecutable { data class Version(val version: String, val verificationInfo: VerificationInfo?) : DataConnectExecutable { companion object { + + private const val DEFAULT_VERSION = "1.3.9" + fun forVersionWithDefaultVerificationInfo(version: String): Version { val verificationInfo = DataConnectExecutable.VerificationInfo.forVersion(version) return Version(version, verificationInfo) } + + fun forDefaultVersionWithDefaultVerificationInfo(): Version = + forVersionWithDefaultVerificationInfo(DEFAULT_VERSION) } } } diff --git a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectProviders.kt b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectProviders.kt index 0adf63688bc..698f5b28ae2 100644 --- a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectProviders.kt +++ b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectProviders.kt @@ -50,7 +50,7 @@ class DataConnectProviders( .orElse(versionValueFromGradleProperty) .orElse(valueFromVariant) .orElse(valueFromProject) - .orElse(DataConnectExecutable.Version.forVersionWithDefaultVerificationInfo("1.3.8")) + .orElse(DataConnectExecutable.Version.forDefaultVersionWithDefaultVerificationInfo()) } val postgresConnectionUrl: Provider = run { From 754608627412ff320d7a4a0c9d2f9a26906d64c1 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 26 Sep 2024 14:49:51 +0000 Subject: [PATCH 2/4] macrobenchmark/commands.py: try to fix IndexError: list index out of range on line ftl_dirs = list(...) --- .../fireciplugins/macrobenchmark/commands.py | 73 ++++++++++++------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/ci/fireci/fireciplugins/macrobenchmark/commands.py b/ci/fireci/fireciplugins/macrobenchmark/commands.py index 430668240b4..fbc8086c4a1 100644 --- a/ci/fireci/fireciplugins/macrobenchmark/commands.py +++ b/ci/fireci/fireciplugins/macrobenchmark/commands.py @@ -151,7 +151,6 @@ def ci(pull_request: bool, changed_modules_file: Path, repeat: int): """Run tests in CI and upload results to the metric service.""" output_path = Path("macrobenchmark-test-output.json") - exception = None try: if pull_request: @@ -166,32 +165,54 @@ def ci(pull_request: bool, changed_modules_file: Path, repeat: int): ) else: asyncio.run(runner.start(build_only=False, local=False, repeat=repeat, output=output_path)) - except Exception as e: - logger.error(f"Error: {e}") - exception = e - - with open(output_path) as output_file: - output = json.load(output_file) + run_succeeded = True + except: + run_succeeded = False + logger.exception("Macrobenchmark failed") + raise + finally: project_name = 'test-changed' if pull_request else 'test-all' - ftl_dirs = list(filter(lambda x: x['project'] == project_name, output))[0]['successful_runs'] - ftl_bucket_name = 'fireescape-benchmark-results' - - log = ci_utils.ci_log_link() - ftl_results = list(map(lambda x: {'bucket': ftl_bucket_name, 'dir': x}, ftl_dirs)) - startup_time_data = {'log': log, 'ftlResults': ftl_results} - - if ftl_results: - metric_service_url = 'https://metric-service-tv5rmd4a6q-uc.a.run.app' - access_token = ci_utils.gcloud_identity_token() - uploader.post_report( - test_report=startup_time_data, - metrics_service_url=metric_service_url, - access_token=access_token, - metric_type='startup-time', - asynchronous=True - ) + try: + with open(output_path) as output_file: + output = json.load(output_file) + except: + logger.exception("Unable to parse JSON from file: %s", output_path) + if run_succeeded: + raise + else: + projects_by_project_name = {} + for project in output: + cur_project_name = project['project'] + projects_by_project_name.setdefault(cur_project_name, []).append(project) + + matching_projects = projects_by_project_name.get(project_name) + if not matching_projects: + project_names = projects_by_project_name.keys() + project_names_str = ", ".join(sorted(project_names)) + message = f'Project "{project_name}" not found in output file: {output_path} ' \ + f'(projects are: {project_names_str})', + logging.error(message) + raise ProjectNotFoundError(message) + + ftl_dirs = matching_projects[0]['successful_runs'] + ftl_bucket_name = 'fireescape-benchmark-results' + + log = ci_utils.ci_log_link() + ftl_results = list(map(lambda x: {'bucket': ftl_bucket_name, 'dir': x}, ftl_dirs)) + startup_time_data = {'log': log, 'ftlResults': ftl_results} + + if ftl_results: + metric_service_url = 'https://metric-service-tv5rmd4a6q-uc.a.run.app' + access_token = ci_utils.gcloud_identity_token() + uploader.post_report( + test_report=startup_time_data, + metrics_service_url=metric_service_url, + access_token=access_token, + metric_type='startup-time', + asynchronous=True + ) - if exception: - raise exception +class ProjectNotFoundError(Exception): + pass # TODO(yifany): support of command chaining From 965c2a8a8f5b444278391e96de5d4a0289eb0384 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 26 Sep 2024 15:10:31 +0000 Subject: [PATCH 3/4] Revert "macrobenchmark/commands.py: try to fix IndexError: list index out of range on line ftl_dirs = list(...)" This reverts commit 754608627412ff320d7a4a0c9d2f9a26906d64c1. --- .../fireciplugins/macrobenchmark/commands.py | 73 +++++++------------ 1 file changed, 26 insertions(+), 47 deletions(-) diff --git a/ci/fireci/fireciplugins/macrobenchmark/commands.py b/ci/fireci/fireciplugins/macrobenchmark/commands.py index fbc8086c4a1..430668240b4 100644 --- a/ci/fireci/fireciplugins/macrobenchmark/commands.py +++ b/ci/fireci/fireciplugins/macrobenchmark/commands.py @@ -151,6 +151,7 @@ def ci(pull_request: bool, changed_modules_file: Path, repeat: int): """Run tests in CI and upload results to the metric service.""" output_path = Path("macrobenchmark-test-output.json") + exception = None try: if pull_request: @@ -165,54 +166,32 @@ def ci(pull_request: bool, changed_modules_file: Path, repeat: int): ) else: asyncio.run(runner.start(build_only=False, local=False, repeat=repeat, output=output_path)) - run_succeeded = True - except: - run_succeeded = False - logger.exception("Macrobenchmark failed") - raise - finally: + except Exception as e: + logger.error(f"Error: {e}") + exception = e + + with open(output_path) as output_file: + output = json.load(output_file) project_name = 'test-changed' if pull_request else 'test-all' - try: - with open(output_path) as output_file: - output = json.load(output_file) - except: - logger.exception("Unable to parse JSON from file: %s", output_path) - if run_succeeded: - raise - else: - projects_by_project_name = {} - for project in output: - cur_project_name = project['project'] - projects_by_project_name.setdefault(cur_project_name, []).append(project) - - matching_projects = projects_by_project_name.get(project_name) - if not matching_projects: - project_names = projects_by_project_name.keys() - project_names_str = ", ".join(sorted(project_names)) - message = f'Project "{project_name}" not found in output file: {output_path} ' \ - f'(projects are: {project_names_str})', - logging.error(message) - raise ProjectNotFoundError(message) - - ftl_dirs = matching_projects[0]['successful_runs'] - ftl_bucket_name = 'fireescape-benchmark-results' - - log = ci_utils.ci_log_link() - ftl_results = list(map(lambda x: {'bucket': ftl_bucket_name, 'dir': x}, ftl_dirs)) - startup_time_data = {'log': log, 'ftlResults': ftl_results} - - if ftl_results: - metric_service_url = 'https://metric-service-tv5rmd4a6q-uc.a.run.app' - access_token = ci_utils.gcloud_identity_token() - uploader.post_report( - test_report=startup_time_data, - metrics_service_url=metric_service_url, - access_token=access_token, - metric_type='startup-time', - asynchronous=True - ) + ftl_dirs = list(filter(lambda x: x['project'] == project_name, output))[0]['successful_runs'] + ftl_bucket_name = 'fireescape-benchmark-results' + + log = ci_utils.ci_log_link() + ftl_results = list(map(lambda x: {'bucket': ftl_bucket_name, 'dir': x}, ftl_dirs)) + startup_time_data = {'log': log, 'ftlResults': ftl_results} + + if ftl_results: + metric_service_url = 'https://metric-service-tv5rmd4a6q-uc.a.run.app' + access_token = ci_utils.gcloud_identity_token() + uploader.post_report( + test_report=startup_time_data, + metrics_service_url=metric_service_url, + access_token=access_token, + metric_type='startup-time', + asynchronous=True + ) -class ProjectNotFoundError(Exception): - pass + if exception: + raise exception # TODO(yifany): support of command chaining From 0ac5f7ad2dfab28af0594e3813924868c533ea48 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 26 Sep 2024 16:20:55 +0000 Subject: [PATCH 4/4] bump default version to 1.4.0 --- .../dataconnect/gradle/plugin/DataConnectExecutable.kt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutable.kt b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutable.kt index 22dd692e1cf..0fcf9cb8b71 100644 --- a/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutable.kt +++ b/firebase-dataconnect/gradleplugin/plugin/src/main/kotlin/com/google/firebase/dataconnect/gradle/plugin/DataConnectExecutable.kt @@ -73,6 +73,13 @@ sealed interface DataConnectExecutable { "4558928c2a84b54113e0d6918907eb75bdeb9bd059dcc4b6f22cb4a7c9c7421a357" + "7f3b0d2eeb246b1df739b38f1eb91e5a6166b0e559707746d79e6ccdf9ed4" ) + "1.4.0" -> + VerificationInfo( + fileSizeInBytes = 25_018_520L, + sha512DigestHex = + "c06ccade89cb46459452f71c6d49a01b4b30c9f96cc4cb770ed168e7420ef0cb368" + + "cd602ff596137e6586270046cf0ffd9f8d294e44b036e5c5b373a074b7e5a" + ) else -> throw DataConnectGradleException( "3svd27ch8y", @@ -94,7 +101,7 @@ sealed interface DataConnectExecutable { DataConnectExecutable { companion object { - private const val DEFAULT_VERSION = "1.3.9" + private const val DEFAULT_VERSION = "1.4.0" fun forVersionWithDefaultVerificationInfo(version: String): Version { val verificationInfo = DataConnectExecutable.VerificationInfo.forVersion(version)