feat(client-glue): AWS Glue Data Quality now provides aggregated metrics in evaluation results when publishAggregatedMetrics with row-level results are enabled. These metrics include summary statistics showing total counts of processed, passed, and failed rows and rules in a single view.

Author: awstools

clients/client-glue/src/commands/BatchGetDataQualityResultCommand.ts

@@ -80,6 +80,9 @@ export interface BatchGetDataQualityResultCommandOutput extends BatchGetDataQual
  * //             "<keys>": Number("double"),
  * //           },
  * //           EvaluatedRule: "STRING_VALUE",
+ * //           RuleMetrics: { // RuleMetricsMap
+ * //             "<keys>": Number("double"),
+ * //           },
  * //         },
  * //       ],
  * //       AnalyzerResults: [ // DataQualityAnalyzerResults
@@ -110,6 +113,14 @@ export interface BatchGetDataQualityResultCommandOutput extends BatchGetDataQual
  * //           },
  * //         },
  * //       ],
+ * //       AggregatedMetrics: { // DataQualityAggregatedMetrics
+ * //         TotalRowsProcessed: Number("double"),
+ * //         TotalRowsPassed: Number("double"),
+ * //         TotalRowsFailed: Number("double"),
+ * //         TotalRulesProcessed: Number("double"),
+ * //         TotalRulesPassed: Number("double"),
+ * //         TotalRulesFailed: Number("double"),
+ * //       },
  * //     },
  * //   ],
  * //   ResultsNotFound: [ // DataQualityResultIds

clients/client-glue/src/commands/GetDataQualityResultCommand.ts

@@ -76,6 +76,9 @@ export interface GetDataQualityResultCommandOutput extends GetDataQualityResultR
  * //         "<keys>": Number("double"),
  * //       },
  * //       EvaluatedRule: "STRING_VALUE",
+ * //       RuleMetrics: { // RuleMetricsMap
+ * //         "<keys>": Number("double"),
+ * //       },
  * //     },
  * //   ],
  * //   AnalyzerResults: [ // DataQualityAnalyzerResults
@@ -106,6 +109,14 @@ export interface GetDataQualityResultCommandOutput extends GetDataQualityResultR
  * //       },
  * //     },
  * //   ],
+ * //   AggregatedMetrics: { // DataQualityAggregatedMetrics
+ * //     TotalRowsProcessed: Number("double"),
+ * //     TotalRowsPassed: Number("double"),
+ * //     TotalRowsFailed: Number("double"),
+ * //     TotalRulesProcessed: Number("double"),
+ * //     TotalRulesPassed: Number("double"),
+ * //     TotalRulesFailed: Number("double"),
+ * //   },
  * // };
  *
  * ```

clients/client-glue/src/commands/ResetJobBookmarkCommand.ts

@@ -6,7 +6,8 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";
 
 import { commonParams } from "../endpoint/EndpointParameters";
 import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
-import { ResetJobBookmarkRequest, ResetJobBookmarkResponse } from "../models/models_2";
+import { ResetJobBookmarkRequest } from "../models/models_2";
+import { ResetJobBookmarkResponse } from "../models/models_3";
 import { de_ResetJobBookmarkCommand, se_ResetJobBookmarkCommand } from "../protocols/Aws_json1_1";
 
 /**

clients/client-glue/src/models/models_0.ts

@@ -2786,6 +2786,48 @@ export interface BatchGetDataQualityResultRequest {
   ResultIds: string[] | undefined;
 }
 
+/**
+ * <p>A summary of metrics showing the total counts of processed rows and rules, including their pass/fail statistics based on row-level results.</p>
+ * @public
+ */
+export interface DataQualityAggregatedMetrics {
+  /**
+   * <p>The total number of rows that were processed during the data quality evaluation.</p>
+   * @public
+   */
+  TotalRowsProcessed?: number | undefined;
+
+  /**
+   * <p>The total number of rows that passed all applicable data quality rules.</p>
+   * @public
+   */
+  TotalRowsPassed?: number | undefined;
+
+  /**
+   * <p>The total number of rows that failed one or more data quality rules.</p>
+   * @public
+   */
+  TotalRowsFailed?: number | undefined;
+
+  /**
+   * <p>The total number of data quality rules that were evaluated.</p>
+   * @public
+   */
+  TotalRulesProcessed?: number | undefined;
+
+  /**
+   * <p>The total number of data quality rules that passed their evaluation criteria.</p>
+   * @public
+   */
+  TotalRulesPassed?: number | undefined;
+
+  /**
+   * <p>The total number of data quality rules that failed their evaluation criteria.</p>
+   * @public
+   */
+  TotalRulesFailed?: number | undefined;
+}
+
 /**
  * <p>Describes the result of the evaluation of a data quality analyzer.</p>
  * @public
@@ -3008,6 +3050,12 @@ export interface DataQualityRuleResult {
    * @public
    */
   EvaluatedRule?: string | undefined;
+
+  /**
+   * <p>A map containing metrics associated with the evaluation of the rule based on row-level results. </p>
+   * @public
+   */
+  RuleMetrics?: Record<string, number> | undefined;
 }
 
 /**
@@ -3098,6 +3146,12 @@ export interface DataQualityResult {
    * @public
    */
   Observations?: DataQualityObservation[] | undefined;
+
+  /**
+   * <p> A summary of <code>DataQualityAggregatedMetrics</code> objects showing the total counts of processed rows and rules, including their pass/fail statistics based on row-level results. </p>
+   * @public
+   */
+  AggregatedMetrics?: DataQualityAggregatedMetrics | undefined;
 }
 
 /**
@@ -9473,71 +9527,6 @@ export interface WorkflowRun {
   StartingEventBatchCondition?: StartingEventBatchCondition | undefined;
 }
 
-/**
- * <p>A workflow is a collection of multiple dependent Glue
- *       jobs and crawlers that are run to complete a complex ETL task. A
- *       workflow manages the execution and monitoring of all its jobs and crawlers.</p>
- * @public
- */
-export interface Workflow {
-  /**
-   * <p>The name of the workflow.</p>
-   * @public
-   */
-  Name?: string | undefined;
-
-  /**
-   * <p>A description of the workflow.</p>
-   * @public
-   */
-  Description?: string | undefined;
-
-  /**
-   * <p>A collection of properties to be used as part of each execution of the workflow.
-   *     The run properties are made available to each job in the workflow. A job can modify
-   *     the properties for the next jobs in the flow.</p>
-   * @public
-   */
-  DefaultRunProperties?: Record<string, string> | undefined;
-
-  /**
-   * <p>The date and time when the workflow was created.</p>
-   * @public
-   */
-  CreatedOn?: Date | undefined;
-
-  /**
-   * <p>The date and time when the workflow was last modified.</p>
-   * @public
-   */
-  LastModifiedOn?: Date | undefined;
-
-  /**
-   * <p>The information about the last execution of the workflow.</p>
-   * @public
-   */
-  LastRun?: WorkflowRun | undefined;
-
-  /**
-   * <p>The graph representing all the Glue components that belong to the workflow as nodes and directed
-   *       connections between them as edges.</p>
-   * @public
-   */
-  Graph?: WorkflowGraph | undefined;
-
-  /**
-   * <p>You can use this parameter to prevent unwanted multiple updates to data, to control costs, or in some cases, to prevent exceeding the maximum number of concurrent runs of any of the component jobs. If you leave this parameter blank, there is no limit to the number of concurrent workflow runs.</p>
-   * @public
-   */
-  MaxConcurrentRuns?: number | undefined;
-
-  /**
-   * <p>This structure indicates the details of the blueprint that this particular workflow is created from.</p>
-   * @public
-   */
-  BlueprintDetails?: BlueprintDetails | undefined;
-}
-
 /**
  * @internal
  */
@@ -9617,6 +9606,7 @@ export const DataQualityRuleResultFilterSensitiveLog = (obj: DataQualityRuleResu
   ...(obj.EvaluationMessage && { EvaluationMessage: SENSITIVE_STRING }),
   ...(obj.EvaluatedMetrics && { EvaluatedMetrics: SENSITIVE_STRING }),
   ...(obj.EvaluatedRule && { EvaluatedRule: SENSITIVE_STRING }),
+  ...(obj.RuleMetrics && { RuleMetrics: SENSITIVE_STRING }),
 });
 
 /**

clients/client-glue/src/models/models_1.ts

@@ -12,6 +12,7 @@ import {
   AuthenticationConfigurationInputFilterSensitiveLog,
   AuthenticationType,
   Blueprint,
+  BlueprintDetails,
   Column,
   ConnectionsList,
   CrawlerTargets,
@@ -33,9 +34,75 @@ import {
   TableOptimizerType,
   TriggerType,
   WorkerType,
-  Workflow,
+  WorkflowGraph,
+  WorkflowRun,
 } from "./models_0";
 
+/**
+ * <p>A workflow is a collection of multiple dependent Glue
+ *       jobs and crawlers that are run to complete a complex ETL task. A
+ *       workflow manages the execution and monitoring of all its jobs and crawlers.</p>
+ * @public
+ */
+export interface Workflow {
+  /**
+   * <p>The name of the workflow.</p>
+   * @public
+   */
+  Name?: string | undefined;
+
+  /**
+   * <p>A description of the workflow.</p>
+   * @public
+   */
+  Description?: string | undefined;
+
+  /**
+   * <p>A collection of properties to be used as part of each execution of the workflow.
+   *     The run properties are made available to each job in the workflow. A job can modify
+   *     the properties for the next jobs in the flow.</p>
+   * @public
+   */
+  DefaultRunProperties?: Record<string, string> | undefined;
+
+  /**
+   * <p>The date and time when the workflow was created.</p>
+   * @public
+   */
+  CreatedOn?: Date | undefined;
+
+  /**
+   * <p>The date and time when the workflow was last modified.</p>
+   * @public
+   */
+  LastModifiedOn?: Date | undefined;
+
+  /**
+   * <p>The information about the last execution of the workflow.</p>
+   * @public
+   */
+  LastRun?: WorkflowRun | undefined;
+
+  /**
+   * <p>The graph representing all the Glue components that belong to the workflow as nodes and directed
+   *       connections between them as edges.</p>
+   * @public
+   */
+  Graph?: WorkflowGraph | undefined;
+
+  /**
+   * <p>You can use this parameter to prevent unwanted multiple updates to data, to control costs, or in some cases, to prevent exceeding the maximum number of concurrent runs of any of the component jobs. If you leave this parameter blank, there is no limit to the number of concurrent workflow runs.</p>
+   * @public
+   */
+  MaxConcurrentRuns?: number | undefined;
+
+  /**
+   * <p>This structure indicates the details of the blueprint that this particular workflow is created from.</p>
+   * @public
+   */
+  BlueprintDetails?: BlueprintDetails | undefined;
+}
+
 /**
  * @public
  */
@@ -1847,6 +1914,7 @@ export interface DataQualityTargetTable {
 }
 
 /**
+ * <p>A request to create a data quality ruleset.</p>
  * @public
  */
 export interface CreateDataQualityRulesetRequest {
@@ -7892,23 +7960,6 @@ export const ComputationType = {
  */
 export type ComputationType = (typeof ComputationType)[keyof typeof ComputationType];
 
-/**
- * @public
- * @enum
- */
-export const ColumnStatisticsState = {
-  FAILED: "FAILED",
-  RUNNING: "RUNNING",
-  STARTING: "STARTING",
-  STOPPED: "STOPPED",
-  SUCCEEDED: "SUCCEEDED",
-} as const;
-
-/**
- * @public
- */
-export type ColumnStatisticsState = (typeof ColumnStatisticsState)[keyof typeof ColumnStatisticsState];
-
 /**
  * @internal
  */