From 954ab445b3b8991148f81ac199bc79dd020c9a15 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 4 Sep 2024 21:33:54 +0400 Subject: [PATCH 1/2] Add Redact processor --- output/openapi/elasticsearch-openapi.json | 51 ++++ .../elasticsearch-serverless-openapi.json | 51 ++++ output/schema/schema-serverless.json | 229 ++++++++++++++---- output/schema/schema.json | 229 ++++++++++++++---- output/typescript/types.ts | 11 + specification/_doc_ids/table.csv | 1 + specification/ingest/_types/Processors.ts | 43 ++++ 7 files changed, 517 insertions(+), 98 deletions(-) diff --git a/output/openapi/elasticsearch-openapi.json b/output/openapi/elasticsearch-openapi.json index ea702e6e12..182c2933ed 100644 --- a/output/openapi/elasticsearch-openapi.json +++ b/output/openapi/elasticsearch-openapi.json @@ -86452,6 +86452,9 @@ "pipeline": { "$ref": "#/components/schemas/ingest._types:PipelineProcessor" }, + "redact": { + "$ref": "#/components/schemas/ingest._types:RedactProcessor" + }, "remove": { "$ref": "#/components/schemas/ingest._types:RemoveProcessor" }, @@ -87449,6 +87452,54 @@ } ] }, + "ingest._types:RedactProcessor": { + "allOf": [ + { + "$ref": "#/components/schemas/ingest._types:ProcessorBase" + }, + { + "type": "object", + "properties": { + "field": { + "$ref": "#/components/schemas/_types:Field" + }, + "patterns": { + "description": "A list of grok expressions to match and redact named captures with", + "type": "array", + "items": { + "type": "string" + } + }, + "pattern_definitions": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "prefix": { + "description": "Start a redacted section with this token", + "type": "string" + }, + "suffix": { + "description": "End a redacted section with this token", + "type": "string" + }, + "ignore_missing": { + "description": "If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document.", + "type": "boolean" + }, + "skip_if_unlicensed": { + "description": "If `true` and the current license does not support running redact processors, then the processor quietly exits without modifying the document", + "type": "boolean" + } + }, + "required": [ + "field", + "patterns" + ] + } + ] + }, "ingest._types:RemoveProcessor": { "allOf": [ { diff --git a/output/openapi/elasticsearch-serverless-openapi.json b/output/openapi/elasticsearch-serverless-openapi.json index 682a5d3080..a79fd26418 100644 --- a/output/openapi/elasticsearch-serverless-openapi.json +++ b/output/openapi/elasticsearch-serverless-openapi.json @@ -55235,6 +55235,9 @@ "pipeline": { "$ref": "#/components/schemas/ingest._types:PipelineProcessor" }, + "redact": { + "$ref": "#/components/schemas/ingest._types:RedactProcessor" + }, "remove": { "$ref": "#/components/schemas/ingest._types:RemoveProcessor" }, @@ -56232,6 +56235,54 @@ } ] }, + "ingest._types:RedactProcessor": { + "allOf": [ + { + "$ref": "#/components/schemas/ingest._types:ProcessorBase" + }, + { + "type": "object", + "properties": { + "field": { + "$ref": "#/components/schemas/_types:Field" + }, + "patterns": { + "description": "A list of grok expressions to match and redact named captures with", + "type": "array", + "items": { + "type": "string" + } + }, + "pattern_definitions": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "prefix": { + "description": "Start a redacted section with this token", + "type": "string" + }, + "suffix": { + "description": "End a redacted section with this token", + "type": "string" + }, + "ignore_missing": { + "description": "If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document.", + "type": "boolean" + }, + "skip_if_unlicensed": { + "description": "If `true` and the current license does not support running redact processors, then the processor quietly exits without modifying the document", + "type": "boolean" + } + }, + "required": [ + "field", + "patterns" + ] + } + ] + }, "ingest._types:RemoveProcessor": { "allOf": [ { diff --git a/output/schema/schema-serverless.json b/output/schema/schema-serverless.json index e1bb4d9c6f..fdfc7b4343 100644 --- a/output/schema/schema-serverless.json +++ b/output/schema/schema-serverless.json @@ -92728,7 +92728,7 @@ "name": "ConvertType", "namespace": "ingest._types" }, - "specLocation": "ingest/_types/Processors.ts#L507-L515" + "specLocation": "ingest/_types/Processors.ts#L514-L522" }, { "kind": "enum", @@ -92744,7 +92744,7 @@ "name": "GeoGridTargetFormat", "namespace": "ingest._types" }, - "specLocation": "ingest/_types/Processors.ts#L406-L409" + "specLocation": "ingest/_types/Processors.ts#L413-L416" }, { "kind": "enum", @@ -92763,7 +92763,7 @@ "name": "GeoGridTileType", "namespace": "ingest._types" }, - "specLocation": "ingest/_types/Processors.ts#L400-L404" + "specLocation": "ingest/_types/Processors.ts#L407-L411" }, { "kind": "enum", @@ -92781,7 +92781,7 @@ "name": "JsonProcessorConflictStrategy", "namespace": "ingest._types" }, - "specLocation": "ingest/_types/Processors.ts#L939-L944" + "specLocation": "ingest/_types/Processors.ts#L946-L951" }, { "kind": "enum", @@ -92797,7 +92797,7 @@ "name": "ShapeType", "namespace": "ingest._types" }, - "specLocation": "ingest/_types/Processors.ts#L1164-L1167" + "specLocation": "ingest/_types/Processors.ts#L1207-L1210" }, { "kind": "enum", @@ -92837,7 +92837,7 @@ "name": "UserAgentProperty", "namespace": "ingest._types" }, - "specLocation": "ingest/_types/Processors.ts#L284-L295" + "specLocation": "ingest/_types/Processors.ts#L291-L302" }, { "kind": "enum", @@ -119564,6 +119564,20 @@ } } }, + { + "description": "The Redact processor uses the Grok rules engine to obscure text in the input document matching the given Grok patterns.\nThe processor can be used to obscure Personal Identifying Information (PII) by configuring it to detect known patterns such as email or IP addresses.\nText that matches a Grok pattern is replaced with a configurable string such as `` where an email address is matched or simply replace all matches with the text `` if preferred.", + "docId": "redact-processor", + "docUrl": "https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/redact-processor.html", + "name": "redact", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "RedactProcessor", + "namespace": "ingest._types" + } + } + }, { "description": "Removes existing fields.\nIf one field doesn’t exist, an exception will be thrown.", "docId": "remove-processor", @@ -119746,7 +119760,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L27-L257", + "specLocation": "ingest/_types/Processors.ts#L27-L264", "variants": { "kind": "container", "nonExhaustive": true @@ -119802,7 +119816,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L297-L312" + "specLocation": "ingest/_types/Processors.ts#L304-L319" }, { "kind": "interface", @@ -119875,7 +119889,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L259-L282" + "specLocation": "ingest/_types/Processors.ts#L266-L289" }, { "inherits": { @@ -119995,7 +120009,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L314-L355" + "specLocation": "ingest/_types/Processors.ts#L321-L362" }, { "inherits": { @@ -120049,7 +120063,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L464-L480" + "specLocation": "ingest/_types/Processors.ts#L471-L487" }, { "inherits": { @@ -120126,7 +120140,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L482-L505" + "specLocation": "ingest/_types/Processors.ts#L489-L512" }, { "inherits": { @@ -120192,7 +120206,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L517-L537" + "specLocation": "ingest/_types/Processors.ts#L524-L544" }, { "inherits": { @@ -120290,7 +120304,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L539-L572" + "specLocation": "ingest/_types/Processors.ts#L546-L579" }, { "inherits": { @@ -120372,7 +120386,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L614-L641" + "specLocation": "ingest/_types/Processors.ts#L621-L648" }, { "inherits": { @@ -120478,7 +120492,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L574-L612" + "specLocation": "ingest/_types/Processors.ts#L581-L619" }, { "inherits": { @@ -120544,7 +120558,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L643-L662" + "specLocation": "ingest/_types/Processors.ts#L650-L669" }, { "inherits": { @@ -120584,7 +120598,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L664-L675" + "specLocation": "ingest/_types/Processors.ts#L671-L682" }, { "inherits": { @@ -120599,7 +120613,7 @@ "namespace": "ingest._types" }, "properties": [], - "specLocation": "ingest/_types/Processors.ts#L677-L677" + "specLocation": "ingest/_types/Processors.ts#L684-L684" }, { "inherits": { @@ -120703,7 +120717,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L679-L718" + "specLocation": "ingest/_types/Processors.ts#L686-L725" }, { "inherits": { @@ -120731,7 +120745,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L720-L726" + "specLocation": "ingest/_types/Processors.ts#L727-L733" }, { "inherits": { @@ -120784,7 +120798,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L728-L742" + "specLocation": "ingest/_types/Processors.ts#L735-L749" }, { "inherits": { @@ -120911,7 +120925,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L357-L398" + "specLocation": "ingest/_types/Processors.ts#L364-L405" }, { "inherits": { @@ -121006,7 +121020,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L411-L440" + "specLocation": "ingest/_types/Processors.ts#L418-L447" }, { "inherits": { @@ -121098,7 +121112,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L744-L769" + "specLocation": "ingest/_types/Processors.ts#L751-L776" }, { "inherits": { @@ -121176,7 +121190,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L771-L795" + "specLocation": "ingest/_types/Processors.ts#L778-L802" }, { "inherits": { @@ -121230,7 +121244,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L797-L813" + "specLocation": "ingest/_types/Processors.ts#L804-L820" }, { "inherits": { @@ -121302,7 +121316,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L815-L834" + "specLocation": "ingest/_types/Processors.ts#L822-L841" }, { "kind": "interface", @@ -121336,7 +121350,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L836-L848", + "specLocation": "ingest/_types/Processors.ts#L843-L855", "variants": { "kind": "container" } @@ -121375,7 +121389,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L850-L861" + "specLocation": "ingest/_types/Processors.ts#L857-L868" }, { "kind": "interface", @@ -121449,7 +121463,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L863-L889" + "specLocation": "ingest/_types/Processors.ts#L870-L896" }, { "inherits": { @@ -121502,7 +121516,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L891-L906" + "specLocation": "ingest/_types/Processors.ts#L898-L913" }, { "inherits": { @@ -121581,7 +121595,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L908-L937" + "specLocation": "ingest/_types/Processors.ts#L915-L944" }, { "inherits": { @@ -121738,7 +121752,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L946-L998" + "specLocation": "ingest/_types/Processors.ts#L953-L1005" }, { "inherits": { @@ -121792,7 +121806,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1000-L1016" + "specLocation": "ingest/_types/Processors.ts#L1007-L1023" }, { "inherits": { @@ -121833,7 +121847,124 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1018-L1029" + "specLocation": "ingest/_types/Processors.ts#L1025-L1036" + }, + { + "inherits": { + "type": { + "name": "ProcessorBase", + "namespace": "ingest._types" + } + }, + "kind": "interface", + "name": { + "name": "RedactProcessor", + "namespace": "ingest._types" + }, + "properties": [ + { + "description": "The field to be redacted", + "name": "field", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "Field", + "namespace": "_types" + } + } + }, + { + "description": "A list of grok expressions to match and redact named captures with", + "name": "patterns", + "required": true, + "type": { + "kind": "array_of", + "value": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + } + }, + { + "name": "pattern_definitions", + "required": false, + "type": { + "key": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + }, + "kind": "dictionary_of", + "singleKey": false, + "value": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + } + }, + { + "description": "Start a redacted section with this token", + "name": "prefix", + "required": false, + "serverDefault": "<", + "type": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + }, + { + "description": "End a redacted section with this token", + "name": "suffix", + "required": false, + "serverDefault": ">", + "type": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + }, + { + "description": "If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document.", + "name": "ignore_missing", + "required": false, + "serverDefault": false, + "type": { + "kind": "instance_of", + "type": { + "name": "boolean", + "namespace": "_builtins" + } + } + }, + { + "description": "If `true` and the current license does not support running redact processors, then the processor quietly exits without modifying the document", + "name": "skip_if_unlicensed", + "required": false, + "serverDefault": false, + "type": { + "kind": "instance_of", + "type": { + "name": "boolean", + "namespace": "_builtins" + } + } + } + ], + "specLocation": "ingest/_types/Processors.ts#L1038-L1072" }, { "inherits": { @@ -121886,7 +122017,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1031-L1045" + "specLocation": "ingest/_types/Processors.ts#L1074-L1088" }, { "inherits": { @@ -121939,7 +122070,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1047-L1063" + "specLocation": "ingest/_types/Processors.ts#L1090-L1106" }, { "inherits": { @@ -122021,7 +122152,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1065-L1093" + "specLocation": "ingest/_types/Processors.ts#L1108-L1136" }, { "inherits": { @@ -122093,7 +122224,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1095-L1115" + "specLocation": "ingest/_types/Processors.ts#L1138-L1158" }, { "inherits": { @@ -122179,7 +122310,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1117-L1151" + "specLocation": "ingest/_types/Processors.ts#L1160-L1194" }, { "inherits": { @@ -122222,7 +122353,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1153-L1162" + "specLocation": "ingest/_types/Processors.ts#L1196-L1205" }, { "inherits": { @@ -122275,7 +122406,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1169-L1185" + "specLocation": "ingest/_types/Processors.ts#L1212-L1228" }, { "inherits": { @@ -122354,7 +122485,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1187-L1212" + "specLocation": "ingest/_types/Processors.ts#L1230-L1255" }, { "inherits": { @@ -122408,7 +122539,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1214-L1230" + "specLocation": "ingest/_types/Processors.ts#L1257-L1273" }, { "inherits": { @@ -122462,7 +122593,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1232-L1248" + "specLocation": "ingest/_types/Processors.ts#L1275-L1291" }, { "inherits": { @@ -122516,7 +122647,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1250-L1266" + "specLocation": "ingest/_types/Processors.ts#L1293-L1309" }, { "inherits": { @@ -122596,7 +122727,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1268-L1294" + "specLocation": "ingest/_types/Processors.ts#L1311-L1337" }, { "inherits": { @@ -122676,7 +122807,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L442-L462" + "specLocation": "ingest/_types/Processors.ts#L449-L469" }, { "kind": "interface", diff --git a/output/schema/schema.json b/output/schema/schema.json index c162abe023..b510b13e31 100644 --- a/output/schema/schema.json +++ b/output/schema/schema.json @@ -138982,7 +138982,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L297-L312" + "specLocation": "ingest/_types/Processors.ts#L304-L319" }, { "kind": "interface", @@ -139102,7 +139102,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L314-L355" + "specLocation": "ingest/_types/Processors.ts#L321-L362" }, { "kind": "interface", @@ -139156,7 +139156,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L464-L480" + "specLocation": "ingest/_types/Processors.ts#L471-L487" }, { "kind": "interface", @@ -139233,7 +139233,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L482-L505" + "specLocation": "ingest/_types/Processors.ts#L489-L512" }, { "kind": "interface", @@ -139299,7 +139299,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L517-L537" + "specLocation": "ingest/_types/Processors.ts#L524-L544" }, { "kind": "enum", @@ -139330,7 +139330,7 @@ "name": "ConvertType", "namespace": "ingest._types" }, - "specLocation": "ingest/_types/Processors.ts#L507-L515" + "specLocation": "ingest/_types/Processors.ts#L514-L522" }, { "kind": "interface", @@ -139428,7 +139428,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L539-L572" + "specLocation": "ingest/_types/Processors.ts#L546-L579" }, { "kind": "interface", @@ -139568,7 +139568,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L574-L612" + "specLocation": "ingest/_types/Processors.ts#L581-L619" }, { "kind": "interface", @@ -139650,7 +139650,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L614-L641" + "specLocation": "ingest/_types/Processors.ts#L621-L648" }, { "kind": "interface", @@ -139716,7 +139716,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L643-L662" + "specLocation": "ingest/_types/Processors.ts#L650-L669" }, { "kind": "interface", @@ -139756,7 +139756,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L664-L675" + "specLocation": "ingest/_types/Processors.ts#L671-L682" }, { "kind": "interface", @@ -139771,7 +139771,7 @@ "namespace": "ingest._types" }, "properties": [], - "specLocation": "ingest/_types/Processors.ts#L677-L677" + "specLocation": "ingest/_types/Processors.ts#L684-L684" }, { "kind": "interface", @@ -139875,7 +139875,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L679-L718" + "specLocation": "ingest/_types/Processors.ts#L686-L725" }, { "kind": "interface", @@ -139903,7 +139903,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L720-L726" + "specLocation": "ingest/_types/Processors.ts#L727-L733" }, { "kind": "interface", @@ -139956,7 +139956,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L728-L742" + "specLocation": "ingest/_types/Processors.ts#L735-L749" }, { "kind": "interface", @@ -140083,7 +140083,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L357-L398" + "specLocation": "ingest/_types/Processors.ts#L364-L405" }, { "kind": "enum", @@ -140099,7 +140099,7 @@ "name": "GeoGridTargetFormat", "namespace": "ingest._types" }, - "specLocation": "ingest/_types/Processors.ts#L406-L409" + "specLocation": "ingest/_types/Processors.ts#L413-L416" }, { "kind": "enum", @@ -140118,7 +140118,7 @@ "name": "GeoGridTileType", "namespace": "ingest._types" }, - "specLocation": "ingest/_types/Processors.ts#L400-L404" + "specLocation": "ingest/_types/Processors.ts#L407-L411" }, { "kind": "interface", @@ -140213,7 +140213,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L411-L440" + "specLocation": "ingest/_types/Processors.ts#L418-L447" }, { "kind": "interface", @@ -140305,7 +140305,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L744-L769" + "specLocation": "ingest/_types/Processors.ts#L751-L776" }, { "kind": "interface", @@ -140383,7 +140383,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L771-L795" + "specLocation": "ingest/_types/Processors.ts#L778-L802" }, { "kind": "interface", @@ -140437,7 +140437,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L797-L813" + "specLocation": "ingest/_types/Processors.ts#L804-L820" }, { "kind": "interface", @@ -140471,7 +140471,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L836-L848", + "specLocation": "ingest/_types/Processors.ts#L843-L855", "variants": { "kind": "container" } @@ -140548,7 +140548,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L863-L889" + "specLocation": "ingest/_types/Processors.ts#L870-L896" }, { "kind": "interface", @@ -140584,7 +140584,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L850-L861" + "specLocation": "ingest/_types/Processors.ts#L857-L868" }, { "kind": "interface", @@ -140656,7 +140656,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L815-L834" + "specLocation": "ingest/_types/Processors.ts#L822-L841" }, { "kind": "interface", @@ -140709,7 +140709,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L891-L906" + "specLocation": "ingest/_types/Processors.ts#L898-L913" }, { "kind": "interface", @@ -140788,7 +140788,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L908-L937" + "specLocation": "ingest/_types/Processors.ts#L915-L944" }, { "kind": "enum", @@ -140806,7 +140806,7 @@ "name": "JsonProcessorConflictStrategy", "namespace": "ingest._types" }, - "specLocation": "ingest/_types/Processors.ts#L939-L944" + "specLocation": "ingest/_types/Processors.ts#L946-L951" }, { "kind": "interface", @@ -140963,7 +140963,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L946-L998" + "specLocation": "ingest/_types/Processors.ts#L953-L1005" }, { "kind": "interface", @@ -141017,7 +141017,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1000-L1016" + "specLocation": "ingest/_types/Processors.ts#L1007-L1023" }, { "kind": "interface", @@ -141204,7 +141204,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1018-L1029" + "specLocation": "ingest/_types/Processors.ts#L1025-L1036" }, { "kind": "interface", @@ -141277,7 +141277,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L259-L282" + "specLocation": "ingest/_types/Processors.ts#L266-L289" }, { "kind": "interface", @@ -141635,6 +141635,20 @@ } } }, + { + "description": "The Redact processor uses the Grok rules engine to obscure text in the input document matching the given Grok patterns.\nThe processor can be used to obscure Personal Identifying Information (PII) by configuring it to detect known patterns such as email or IP addresses.\nText that matches a Grok pattern is replaced with a configurable string such as `` where an email address is matched or simply replace all matches with the text `` if preferred.", + "docId": "redact-processor", + "docUrl": "https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/redact-processor.html", + "name": "redact", + "required": false, + "type": { + "kind": "instance_of", + "type": { + "name": "RedactProcessor", + "namespace": "ingest._types" + } + } + }, { "description": "Removes existing fields.\nIf one field doesn’t exist, an exception will be thrown.", "docId": "remove-processor", @@ -141817,12 +141831,129 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L27-L257", + "specLocation": "ingest/_types/Processors.ts#L27-L264", "variants": { "kind": "container", "nonExhaustive": true } }, + { + "kind": "interface", + "inherits": { + "type": { + "name": "ProcessorBase", + "namespace": "ingest._types" + } + }, + "name": { + "name": "RedactProcessor", + "namespace": "ingest._types" + }, + "properties": [ + { + "description": "The field to be redacted", + "name": "field", + "required": true, + "type": { + "kind": "instance_of", + "type": { + "name": "Field", + "namespace": "_types" + } + } + }, + { + "description": "A list of grok expressions to match and redact named captures with", + "name": "patterns", + "required": true, + "type": { + "kind": "array_of", + "value": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + } + }, + { + "name": "pattern_definitions", + "required": false, + "type": { + "kind": "dictionary_of", + "key": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + }, + "singleKey": false, + "value": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + } + }, + { + "description": "Start a redacted section with this token", + "name": "prefix", + "required": false, + "serverDefault": "<", + "type": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + }, + { + "description": "End a redacted section with this token", + "name": "suffix", + "required": false, + "serverDefault": ">", + "type": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + }, + { + "description": "If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document.", + "name": "ignore_missing", + "required": false, + "serverDefault": false, + "type": { + "kind": "instance_of", + "type": { + "name": "boolean", + "namespace": "_builtins" + } + } + }, + { + "description": "If `true` and the current license does not support running redact processors, then the processor quietly exits without modifying the document", + "name": "skip_if_unlicensed", + "required": false, + "serverDefault": false, + "type": { + "kind": "instance_of", + "type": { + "name": "boolean", + "namespace": "_builtins" + } + } + } + ], + "specLocation": "ingest/_types/Processors.ts#L1038-L1072" + }, { "kind": "interface", "inherits": { @@ -141874,7 +142005,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1031-L1045" + "specLocation": "ingest/_types/Processors.ts#L1074-L1088" }, { "kind": "interface", @@ -141927,7 +142058,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1047-L1063" + "specLocation": "ingest/_types/Processors.ts#L1090-L1106" }, { "kind": "interface", @@ -142009,7 +142140,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1065-L1093" + "specLocation": "ingest/_types/Processors.ts#L1108-L1136" }, { "kind": "interface", @@ -142081,7 +142212,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1095-L1115" + "specLocation": "ingest/_types/Processors.ts#L1138-L1158" }, { "kind": "interface", @@ -142167,7 +142298,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1117-L1151" + "specLocation": "ingest/_types/Processors.ts#L1160-L1194" }, { "kind": "interface", @@ -142210,7 +142341,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1153-L1162" + "specLocation": "ingest/_types/Processors.ts#L1196-L1205" }, { "kind": "enum", @@ -142226,7 +142357,7 @@ "name": "ShapeType", "namespace": "ingest._types" }, - "specLocation": "ingest/_types/Processors.ts#L1164-L1167" + "specLocation": "ingest/_types/Processors.ts#L1207-L1210" }, { "kind": "interface", @@ -142279,7 +142410,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1169-L1185" + "specLocation": "ingest/_types/Processors.ts#L1212-L1228" }, { "kind": "interface", @@ -142358,7 +142489,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1187-L1212" + "specLocation": "ingest/_types/Processors.ts#L1230-L1255" }, { "kind": "interface", @@ -142412,7 +142543,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1214-L1230" + "specLocation": "ingest/_types/Processors.ts#L1257-L1273" }, { "kind": "interface", @@ -142466,7 +142597,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1232-L1248" + "specLocation": "ingest/_types/Processors.ts#L1275-L1291" }, { "kind": "interface", @@ -142546,7 +142677,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1268-L1294" + "specLocation": "ingest/_types/Processors.ts#L1311-L1337" }, { "kind": "interface", @@ -142600,7 +142731,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L1250-L1266" + "specLocation": "ingest/_types/Processors.ts#L1293-L1309" }, { "kind": "interface", @@ -142680,7 +142811,7 @@ } } ], - "specLocation": "ingest/_types/Processors.ts#L442-L462" + "specLocation": "ingest/_types/Processors.ts#L449-L469" }, { "kind": "enum", @@ -142720,7 +142851,7 @@ "name": "UserAgentProperty", "namespace": "ingest._types" }, - "specLocation": "ingest/_types/Processors.ts#L284-L295" + "specLocation": "ingest/_types/Processors.ts#L291-L302" }, { "kind": "request", diff --git a/output/typescript/types.ts b/output/typescript/types.ts index 941e0b1afc..e64cd8d774 100644 --- a/output/typescript/types.ts +++ b/output/typescript/types.ts @@ -12892,6 +12892,7 @@ export interface IngestProcessorContainer { kv?: IngestKeyValueProcessor lowercase?: IngestLowercaseProcessor pipeline?: IngestPipelineProcessor + redact?: IngestRedactProcessor remove?: IngestRemoveProcessor rename?: IngestRenameProcessor reroute?: IngestRerouteProcessor @@ -12907,6 +12908,16 @@ export interface IngestProcessorContainer { user_agent?: IngestUserAgentProcessor } +export interface IngestRedactProcessor extends IngestProcessorBase { + field: Field + patterns: string[] + pattern_definitions?: Record + prefix?: string + suffix?: string + ignore_missing?: boolean + skip_if_unlicensed?: boolean +} + export interface IngestRemoveProcessor extends IngestProcessorBase { field: Fields keep?: Fields diff --git a/specification/_doc_ids/table.csv b/specification/_doc_ids/table.csv index c1106c8143..f87158f01e 100644 --- a/specification/_doc_ids/table.csv +++ b/specification/_doc_ids/table.csv @@ -405,6 +405,7 @@ query-dsl-wildcard-query,https://www.elastic.co/guide/en/elasticsearch/reference query-dsl-wrapper-query,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/query-dsl-wrapper-query.html query-dsl,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/query-dsl.html realtime,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/docs-get.html#realtime +redact-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/redact-processor.html regexp-syntax,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/regexp-syntax.html remove-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/remove-processor.html rename-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/rename-processor.html diff --git a/specification/ingest/_types/Processors.ts b/specification/ingest/_types/Processors.ts index 9f9891e95b..0da7d0ba2a 100644 --- a/specification/ingest/_types/Processors.ts +++ b/specification/ingest/_types/Processors.ts @@ -174,6 +174,13 @@ export class ProcessorContainer { * @doc_id pipeline-processor */ pipeline?: PipelineProcessor + /** + * The Redact processor uses the Grok rules engine to obscure text in the input document matching the given Grok patterns. + * The processor can be used to obscure Personal Identifying Information (PII) by configuring it to detect known patterns such as email or IP addresses. + * Text that matches a Grok pattern is replaced with a configurable string such as `` where an email address is matched or simply replace all matches with the text `` if preferred. + * @doc_id redact-processor + */ + redact?: RedactProcessor /** * Removes existing fields. * If one field doesn’t exist, an exception will be thrown. @@ -1028,6 +1035,42 @@ export class PipelineProcessor extends ProcessorBase { ignore_missing_pipeline?: boolean } +export class RedactProcessor extends ProcessorBase { + /** + * The field to be redacted + */ + field: Field + /** + * A list of grok expressions to match and redact named captures with + */ + patterns: string[] + /* + * A map of pattern-name and pattern tuples defining custom patterns to be used by the processor. + * Patterns matching existing names will override the pre-existing definition + */ + pattern_definitions?: Dictionary + /** + * Start a redacted section with this token + * @server_default < + */ + prefix?: string + /** + * End a redacted section with this token + * @server_default > + */ + suffix?: string + /** + * If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document. + * @server_default false + */ + ignore_missing?: boolean + /** + * If `true` and the current license does not support running redact processors, then the processor quietly exits without modifying the document + * @server_default false + */ + skip_if_unlicensed?: boolean +} + export class RemoveProcessor extends ProcessorBase { /** * Fields to be removed. Supports template snippets. From 0bdc5af7a2a192141b3b25af8ce7c17c458d6ea3 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Thu, 5 Sep 2024 16:08:41 +0400 Subject: [PATCH 2/2] Add GrokPattern alias --- output/openapi/elasticsearch-openapi.json | 17 +-- .../elasticsearch-serverless-openapi.json | 7 +- output/schema/schema-serverless.json | 111 ++++++++------- output/schema/schema.json | 129 ++++++++++-------- output/typescript/types.ts | 14 +- specification/_types/common.ts | 1 + specification/ingest/_types/Processors.ts | 6 +- specification/ml/_types/Category.ts | 4 +- .../find_structure/FindStructureRequest.ts | 4 +- .../find_structure/FindStructureResponse.ts | 4 +- .../TestGrokPatternRequest.ts | 3 +- 11 files changed, 169 insertions(+), 131 deletions(-) diff --git a/output/openapi/elasticsearch-openapi.json b/output/openapi/elasticsearch-openapi.json index 83b921750f..ed49347950 100644 --- a/output/openapi/elasticsearch-openapi.json +++ b/output/openapi/elasticsearch-openapi.json @@ -35015,7 +35015,7 @@ "description": "If you have set format to semi_structured_text, you can specify a Grok pattern that is used to extract fields from every message in the text. The name of the timestamp field in the Grok pattern must match what is specified in the timestamp_field parameter. If that parameter is not specified, the name of the timestamp field in the Grok pattern must match \"timestamp\". If grok_pattern is not specified, the structure finder creates a Grok pattern.", "deprecated": false, "schema": { - "type": "string" + "$ref": "#/components/schemas/_types:GrokPattern" }, "style": "form" }, @@ -35173,7 +35173,7 @@ } }, "grok_pattern": { - "type": "string" + "$ref": "#/components/schemas/_types:GrokPattern" }, "multiline_start_pattern": { "type": "string" @@ -53255,8 +53255,7 @@ "type": "object", "properties": { "grok_pattern": { - "description": "Grok pattern to run on the text.", - "type": "string" + "$ref": "#/components/schemas/_types:GrokPattern" }, "text": { "description": "Lines of text to run the Grok pattern on.", @@ -87180,7 +87179,7 @@ "description": "An ordered list of grok expression to match and extract named captures with.\nReturns on the first expression in the list that matches.", "type": "array", "items": { - "type": "string" + "$ref": "#/components/schemas/_types:GrokPattern" } }, "trace_match": { @@ -87195,6 +87194,9 @@ } ] }, + "_types:GrokPattern": { + "type": "string" + }, "ingest._types:GsubProcessor": { "allOf": [ { @@ -87526,7 +87528,7 @@ "description": "A list of grok expressions to match and redact named captures with", "type": "array", "items": { - "type": "string" + "$ref": "#/components/schemas/_types:GrokPattern" } }, "pattern_definitions": { @@ -89865,8 +89867,7 @@ } }, "grok_pattern": { - "description": "[experimental] A Grok pattern that could be used in Logstash or an ingest pipeline to extract fields from messages that match the category. This field is experimental and may be changed or removed in a future release. The Grok patterns that are found are not optimal, but are often a good starting point for manual tweaking.", - "type": "string" + "$ref": "#/components/schemas/_types:GrokPattern" }, "job_id": { "$ref": "#/components/schemas/_types:Id" diff --git a/output/openapi/elasticsearch-serverless-openapi.json b/output/openapi/elasticsearch-serverless-openapi.json index a25eb72b68..17928e8a47 100644 --- a/output/openapi/elasticsearch-serverless-openapi.json +++ b/output/openapi/elasticsearch-serverless-openapi.json @@ -55960,7 +55960,7 @@ "description": "An ordered list of grok expression to match and extract named captures with.\nReturns on the first expression in the list that matches.", "type": "array", "items": { - "type": "string" + "$ref": "#/components/schemas/_types:GrokPattern" } }, "trace_match": { @@ -55975,6 +55975,9 @@ } ] }, + "_types:GrokPattern": { + "type": "string" + }, "ingest._types:GsubProcessor": { "allOf": [ { @@ -56306,7 +56309,7 @@ "description": "A list of grok expressions to match and redact named captures with", "type": "array", "items": { - "type": "string" + "$ref": "#/components/schemas/_types:GrokPattern" } }, "pattern_definitions": { diff --git a/output/schema/schema-serverless.json b/output/schema/schema-serverless.json index b46ba30926..73012bf0db 100644 --- a/output/schema/schema-serverless.json +++ b/output/schema/schema-serverless.json @@ -44927,7 +44927,7 @@ "name": "SequenceNumber", "namespace": "_types" }, - "specLocation": "_types/common.ts#L128-L128", + "specLocation": "_types/common.ts#L129-L129", "type": { "kind": "instance_of", "type": { @@ -44942,7 +44942,7 @@ "name": "VersionNumber", "namespace": "_types" }, - "specLocation": "_types/common.ts#L102-L102", + "specLocation": "_types/common.ts#L103-L103", "type": { "kind": "instance_of", "type": { @@ -45296,7 +45296,7 @@ "name": "ExpandWildcards", "namespace": "_types" }, - "specLocation": "_types/common.ts#L217-L217", + "specLocation": "_types/common.ts#L218-L218", "type": { "items": [ { @@ -45348,7 +45348,7 @@ "name": "ExpandWildcard", "namespace": "_types" }, - "specLocation": "_types/common.ts#L201-L215" + "specLocation": "_types/common.ts#L202-L216" }, { "kind": "type_alias", @@ -45411,7 +45411,7 @@ "name": "SearchType", "namespace": "_types" }, - "specLocation": "_types/common.ts#L275-L280" + "specLocation": "_types/common.ts#L276-L281" }, { "kind": "interface", @@ -47024,7 +47024,7 @@ "name": "Metadata", "namespace": "_types" }, - "specLocation": "_types/common.ts#L99-L99", + "specLocation": "_types/common.ts#L100-L100", "type": { "key": { "kind": "instance_of", @@ -48379,7 +48379,7 @@ "name": "MinimumShouldMatch", "namespace": "_types" }, - "specLocation": "_types/common.ts#L163-L167", + "specLocation": "_types/common.ts#L164-L168", "type": { "items": [ { @@ -48489,7 +48489,7 @@ "name": "Field", "namespace": "_types" }, - "specLocation": "_types/common.ts#L138-L139", + "specLocation": "_types/common.ts#L139-L140", "type": { "kind": "instance_of", "type": { @@ -50091,7 +50091,7 @@ "name": "MultiTermQueryRewrite", "namespace": "_types" }, - "specLocation": "_types/common.ts#L135-L136", + "specLocation": "_types/common.ts#L136-L137", "type": { "kind": "instance_of", "type": { @@ -50108,7 +50108,7 @@ "name": "Fuzziness", "namespace": "_types" }, - "specLocation": "_types/common.ts#L133-L134", + "specLocation": "_types/common.ts#L134-L135", "type": { "items": [ { @@ -51921,7 +51921,7 @@ "name": "Fields", "namespace": "_types" }, - "specLocation": "_types/common.ts#L140-L140", + "specLocation": "_types/common.ts#L141-L141", "type": { "items": [ { @@ -52899,7 +52899,7 @@ "name": "RelationName", "namespace": "_types" }, - "specLocation": "_types/common.ts#L131-L131", + "specLocation": "_types/common.ts#L132-L132", "type": { "kind": "instance_of", "type": { @@ -54892,7 +54892,7 @@ "name": "VersionType", "namespace": "_types" }, - "specLocation": "_types/common.ts#L106-L122" + "specLocation": "_types/common.ts#L107-L123" }, { "description": "Language value, such as _arabic_ or _thai_. Defaults to _english_.\nEach language value corresponds to a predefined list of stop words in Lucene. See Stop words by language for supported language values and their stop words.\nAlso accepts an array of stop words.", @@ -62651,7 +62651,7 @@ "namespace": "_types" }, "properties": [], - "specLocation": "_types/common.ts#L160-L161" + "specLocation": "_types/common.ts#L161-L162" }, { "inherits": { @@ -67052,7 +67052,7 @@ "name": "SuggestMode", "namespace": "_types" }, - "specLocation": "_types/common.ts#L282-L295" + "specLocation": "_types/common.ts#L283-L296" }, { "kind": "interface", @@ -68527,7 +68527,7 @@ "name": "AggregateName", "namespace": "_types" }, - "specLocation": "_types/common.ts#L145-L149", + "specLocation": "_types/common.ts#L146-L150", "type": { "kind": "instance_of", "type": { @@ -74070,7 +74070,7 @@ "name": "SuggestionName", "namespace": "_types" }, - "specLocation": "_types/common.ts#L151-L155", + "specLocation": "_types/common.ts#L152-L156", "type": { "kind": "instance_of", "type": { @@ -74947,7 +74947,7 @@ "name": "ByteSize", "namespace": "_types" }, - "specLocation": "_types/common.ts#L96-L97", + "specLocation": "_types/common.ts#L97-L98", "type": { "items": [ { @@ -75000,7 +75000,7 @@ "name": "Bytes", "namespace": "_types" }, - "specLocation": "_types/common.ts#L169-L181" + "specLocation": "_types/common.ts#L170-L182" }, { "kind": "type_alias", @@ -75040,7 +75040,7 @@ "name": "ClusterInfoTarget", "namespace": "_types" }, - "specLocation": "_types/common.ts#L381-L387" + "specLocation": "_types/common.ts#L382-L388" }, { "kind": "type_alias", @@ -75048,7 +75048,7 @@ "name": "ClusterInfoTargets", "namespace": "_types" }, - "specLocation": "_types/common.ts#L389-L389", + "specLocation": "_types/common.ts#L390-L390", "type": { "items": [ { @@ -75088,7 +75088,7 @@ "name": "Conflicts", "namespace": "_types" }, - "specLocation": "_types/common.ts#L183-L192" + "specLocation": "_types/common.ts#L184-L193" }, { "kind": "enum", @@ -75167,7 +75167,7 @@ "name": "DataStreamName", "namespace": "_types" }, - "specLocation": "_types/common.ts#L91-L92", + "specLocation": "_types/common.ts#L92-L93", "type": { "kind": "instance_of", "type": { @@ -75182,7 +75182,7 @@ "name": "DataStreamNames", "namespace": "_types" }, - "specLocation": "_types/common.ts#L94-L94", + "specLocation": "_types/common.ts#L95-L95", "type": { "items": [ { @@ -75253,6 +75253,21 @@ } } }, + { + "kind": "type_alias", + "name": { + "name": "GrokPattern", + "namespace": "_types" + }, + "specLocation": "_types/common.ts#L87-L87", + "type": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + }, { "kind": "enum", "members": [ @@ -75282,7 +75297,7 @@ "name": "HealthStatus", "namespace": "_types" }, - "specLocation": "_types/common.ts#L219-L239" + "specLocation": "_types/common.ts#L220-L240" }, { "kind": "type_alias", @@ -75305,7 +75320,7 @@ "name": "HttpHeaders", "namespace": "_types" }, - "specLocation": "_types/common.ts#L158-L158", + "specLocation": "_types/common.ts#L159-L159", "type": { "key": { "kind": "instance_of", @@ -75452,7 +75467,7 @@ "name": "Level", "namespace": "_types" }, - "specLocation": "_types/common.ts#L249-L253" + "specLocation": "_types/common.ts#L250-L254" }, { "kind": "enum", @@ -75616,7 +75631,7 @@ "name": "NodeName", "namespace": "_types" }, - "specLocation": "_types/common.ts#L88-L89", + "specLocation": "_types/common.ts#L89-L90", "type": { "kind": "instance_of", "type": { @@ -75737,7 +75752,7 @@ "name": "OpType", "namespace": "_types" }, - "specLocation": "_types/common.ts#L255-L264" + "specLocation": "_types/common.ts#L256-L265" }, { "kind": "type_alias", @@ -75745,7 +75760,7 @@ "name": "Password", "namespace": "_types" }, - "specLocation": "_types/common.ts#L195-L195", + "specLocation": "_types/common.ts#L196-L196", "type": { "kind": "instance_of", "type": { @@ -75802,7 +75817,7 @@ "name": "PropertyName", "namespace": "_types" }, - "specLocation": "_types/common.ts#L130-L130", + "specLocation": "_types/common.ts#L131-L131", "type": { "kind": "instance_of", "type": { @@ -75829,7 +75844,7 @@ "name": "Refresh", "namespace": "_types" }, - "specLocation": "_types/common.ts#L266-L273" + "specLocation": "_types/common.ts#L267-L274" }, { "kind": "enum", @@ -75969,7 +75984,7 @@ "name": "Slices", "namespace": "_types" }, - "specLocation": "_types/common.ts#L364-L369", + "specLocation": "_types/common.ts#L365-L370", "type": { "items": [ { @@ -76002,7 +76017,7 @@ "name": "SlicesCalculation", "namespace": "_types" }, - "specLocation": "_types/common.ts#L371-L379" + "specLocation": "_types/common.ts#L372-L380" }, { "kind": "type_alias", @@ -76010,7 +76025,7 @@ "name": "TaskId", "namespace": "_types" }, - "specLocation": "_types/common.ts#L132-L132", + "specLocation": "_types/common.ts#L133-L133", "type": { "items": [ { @@ -76054,7 +76069,7 @@ "name": "ThreadType", "namespace": "_types" }, - "specLocation": "_types/common.ts#L297-L303" + "specLocation": "_types/common.ts#L298-L304" }, { "description": "Time of day, expressed as HH:MM:SS", @@ -76195,7 +76210,7 @@ "name": "Username", "namespace": "_types" }, - "specLocation": "_types/common.ts#L194-L194", + "specLocation": "_types/common.ts#L195-L195", "type": { "kind": "instance_of", "type": { @@ -76210,7 +76225,7 @@ "name": "Uuid", "namespace": "_types" }, - "specLocation": "_types/common.ts#L125-L125", + "specLocation": "_types/common.ts#L126-L126", "type": { "kind": "instance_of", "type": { @@ -76225,7 +76240,7 @@ "name": "VersionString", "namespace": "_types" }, - "specLocation": "_types/common.ts#L104-L104", + "specLocation": "_types/common.ts#L105-L105", "type": { "kind": "instance_of", "type": { @@ -76248,7 +76263,7 @@ "name": "WaitForActiveShardOptions", "namespace": "_types" }, - "specLocation": "_types/common.ts#L305-L309" + "specLocation": "_types/common.ts#L306-L310" }, { "codegenNames": [ @@ -76260,7 +76275,7 @@ "name": "WaitForActiveShards", "namespace": "_types" }, - "specLocation": "_types/common.ts#L142-L143", + "specLocation": "_types/common.ts#L143-L144", "type": { "items": [ { @@ -76307,7 +76322,7 @@ "name": "WaitForEvents", "namespace": "_types" }, - "specLocation": "_types/common.ts#L311-L318" + "specLocation": "_types/common.ts#L312-L319" }, { "kind": "type_alias", @@ -104890,7 +104905,7 @@ } } ], - "specLocation": "_types/common.ts#L320-L333" + "specLocation": "_types/common.ts#L321-L334" }, { "attachedBehaviors": [ @@ -121188,8 +121203,8 @@ "value": { "kind": "instance_of", "type": { - "name": "string", - "namespace": "_builtins" + "name": "GrokPattern", + "namespace": "_types" } } } @@ -121979,8 +121994,8 @@ "value": { "kind": "instance_of", "type": { - "name": "string", - "namespace": "_builtins" + "name": "GrokPattern", + "namespace": "_types" } } } @@ -128180,7 +128195,7 @@ } } ], - "specLocation": "_types/common.ts#L335-L362" + "specLocation": "_types/common.ts#L336-L363" }, { "kind": "interface", diff --git a/output/schema/schema.json b/output/schema/schema.json index 3d48b37e7c..26ff08d58c 100644 --- a/output/schema/schema.json +++ b/output/schema/schema.json @@ -41423,7 +41423,7 @@ "name": "AggregateName", "namespace": "_types" }, - "specLocation": "_types/common.ts#L145-L149", + "specLocation": "_types/common.ts#L146-L150", "type": { "kind": "instance_of", "type": { @@ -41632,7 +41632,7 @@ "name": "ByteSize", "namespace": "_types" }, - "specLocation": "_types/common.ts#L96-L97", + "specLocation": "_types/common.ts#L97-L98", "type": { "kind": "union_of", "items": [ @@ -41685,7 +41685,7 @@ "name": "Bytes", "namespace": "_types" }, - "specLocation": "_types/common.ts#L169-L181" + "specLocation": "_types/common.ts#L170-L182" }, { "kind": "type_alias", @@ -41828,7 +41828,7 @@ "name": "ClusterInfoTarget", "namespace": "_types" }, - "specLocation": "_types/common.ts#L381-L387" + "specLocation": "_types/common.ts#L382-L388" }, { "kind": "type_alias", @@ -41836,7 +41836,7 @@ "name": "ClusterInfoTargets", "namespace": "_types" }, - "specLocation": "_types/common.ts#L389-L389", + "specLocation": "_types/common.ts#L390-L390", "type": { "kind": "union_of", "items": [ @@ -42055,7 +42055,7 @@ "name": "Conflicts", "namespace": "_types" }, - "specLocation": "_types/common.ts#L183-L192" + "specLocation": "_types/common.ts#L184-L193" }, { "kind": "interface", @@ -42188,7 +42188,7 @@ "name": "DataStreamName", "namespace": "_types" }, - "specLocation": "_types/common.ts#L91-L92", + "specLocation": "_types/common.ts#L92-L93", "type": { "kind": "instance_of", "type": { @@ -42203,7 +42203,7 @@ "name": "DataStreamNames", "namespace": "_types" }, - "specLocation": "_types/common.ts#L94-L94", + "specLocation": "_types/common.ts#L95-L95", "type": { "kind": "union_of", "items": [ @@ -42631,7 +42631,7 @@ "namespace": "_types" }, "properties": [], - "specLocation": "_types/common.ts#L160-L161" + "specLocation": "_types/common.ts#L161-L162" }, { "kind": "type_alias", @@ -42843,7 +42843,7 @@ "name": "ExpandWildcard", "namespace": "_types" }, - "specLocation": "_types/common.ts#L201-L215" + "specLocation": "_types/common.ts#L202-L216" }, { "kind": "type_alias", @@ -42851,7 +42851,7 @@ "name": "ExpandWildcards", "namespace": "_types" }, - "specLocation": "_types/common.ts#L217-L217", + "specLocation": "_types/common.ts#L218-L218", "type": { "kind": "union_of", "items": [ @@ -42882,7 +42882,7 @@ "name": "Field", "namespace": "_types" }, - "specLocation": "_types/common.ts#L138-L139", + "specLocation": "_types/common.ts#L139-L140", "type": { "kind": "instance_of", "type": { @@ -43196,7 +43196,7 @@ "name": "Fields", "namespace": "_types" }, - "specLocation": "_types/common.ts#L140-L140", + "specLocation": "_types/common.ts#L141-L141", "type": { "kind": "union_of", "items": [ @@ -43291,7 +43291,7 @@ "name": "Fuzziness", "namespace": "_types" }, - "specLocation": "_types/common.ts#L133-L134", + "specLocation": "_types/common.ts#L134-L135", "type": { "kind": "union_of", "items": [ @@ -43892,6 +43892,21 @@ ], "specLocation": "_types/Stats.ts#L130-L141" }, + { + "kind": "type_alias", + "name": { + "name": "GrokPattern", + "namespace": "_types" + }, + "specLocation": "_types/common.ts#L87-L87", + "type": { + "kind": "instance_of", + "type": { + "name": "string", + "namespace": "_builtins" + } + } + }, { "kind": "enum", "members": [ @@ -43921,7 +43936,7 @@ "name": "HealthStatus", "namespace": "_types" }, - "specLocation": "_types/common.ts#L219-L239" + "specLocation": "_types/common.ts#L220-L240" }, { "kind": "type_alias", @@ -43944,7 +43959,7 @@ "name": "HttpHeaders", "namespace": "_types" }, - "specLocation": "_types/common.ts#L158-L158", + "specLocation": "_types/common.ts#L159-L159", "type": { "kind": "dictionary_of", "key": { @@ -44421,7 +44436,7 @@ } } ], - "specLocation": "_types/common.ts#L335-L362" + "specLocation": "_types/common.ts#L336-L363" }, { "kind": "interface", @@ -44564,7 +44579,7 @@ } } ], - "specLocation": "_types/common.ts#L320-L333" + "specLocation": "_types/common.ts#L321-L334" }, { "kind": "type_alias", @@ -44970,7 +44985,7 @@ "name": "Level", "namespace": "_types" }, - "specLocation": "_types/common.ts#L249-L253" + "specLocation": "_types/common.ts#L250-L254" }, { "kind": "enum", @@ -45225,7 +45240,7 @@ "name": "Metadata", "namespace": "_types" }, - "specLocation": "_types/common.ts#L99-L99", + "specLocation": "_types/common.ts#L100-L100", "type": { "kind": "dictionary_of", "key": { @@ -45280,7 +45295,7 @@ "name": "MinimumShouldMatch", "namespace": "_types" }, - "specLocation": "_types/common.ts#L163-L167", + "specLocation": "_types/common.ts#L164-L168", "type": { "kind": "union_of", "items": [ @@ -45309,7 +45324,7 @@ "name": "MultiTermQueryRewrite", "namespace": "_types" }, - "specLocation": "_types/common.ts#L135-L136", + "specLocation": "_types/common.ts#L136-L137", "type": { "kind": "instance_of", "type": { @@ -45566,7 +45581,7 @@ "name": "NodeName", "namespace": "_types" }, - "specLocation": "_types/common.ts#L88-L89", + "specLocation": "_types/common.ts#L89-L90", "type": { "kind": "instance_of", "type": { @@ -45902,7 +45917,7 @@ "name": "OpType", "namespace": "_types" }, - "specLocation": "_types/common.ts#L255-L264" + "specLocation": "_types/common.ts#L256-L265" }, { "kind": "type_alias", @@ -45910,7 +45925,7 @@ "name": "Password", "namespace": "_types" }, - "specLocation": "_types/common.ts#L195-L195", + "specLocation": "_types/common.ts#L196-L196", "type": { "kind": "instance_of", "type": { @@ -46079,7 +46094,7 @@ "name": "PropertyName", "namespace": "_types" }, - "specLocation": "_types/common.ts#L130-L130", + "specLocation": "_types/common.ts#L131-L131", "type": { "kind": "instance_of", "type": { @@ -46406,7 +46421,7 @@ "name": "Refresh", "namespace": "_types" }, - "specLocation": "_types/common.ts#L266-L273" + "specLocation": "_types/common.ts#L267-L274" }, { "kind": "interface", @@ -46508,7 +46523,7 @@ "name": "RelationName", "namespace": "_types" }, - "specLocation": "_types/common.ts#L131-L131", + "specLocation": "_types/common.ts#L132-L132", "type": { "kind": "instance_of", "type": { @@ -47541,7 +47556,7 @@ "name": "SearchType", "namespace": "_types" }, - "specLocation": "_types/common.ts#L275-L280" + "specLocation": "_types/common.ts#L276-L281" }, { "kind": "interface", @@ -47856,7 +47871,7 @@ "name": "SequenceNumber", "namespace": "_types" }, - "specLocation": "_types/common.ts#L128-L128", + "specLocation": "_types/common.ts#L129-L129", "type": { "kind": "instance_of", "type": { @@ -48090,7 +48105,7 @@ "name": "Slices", "namespace": "_types" }, - "specLocation": "_types/common.ts#L364-L369", + "specLocation": "_types/common.ts#L365-L370", "type": { "kind": "union_of", "items": [ @@ -48123,7 +48138,7 @@ "name": "SlicesCalculation", "namespace": "_types" }, - "specLocation": "_types/common.ts#L371-L379" + "specLocation": "_types/common.ts#L372-L380" }, { "kind": "type_alias", @@ -48583,7 +48598,7 @@ "name": "SuggestMode", "namespace": "_types" }, - "specLocation": "_types/common.ts#L282-L295" + "specLocation": "_types/common.ts#L283-L296" }, { "kind": "type_alias", @@ -48592,7 +48607,7 @@ "name": "SuggestionName", "namespace": "_types" }, - "specLocation": "_types/common.ts#L151-L155", + "specLocation": "_types/common.ts#L152-L156", "type": { "kind": "instance_of", "type": { @@ -48661,7 +48676,7 @@ "name": "TaskId", "namespace": "_types" }, - "specLocation": "_types/common.ts#L132-L132", + "specLocation": "_types/common.ts#L133-L133", "type": { "kind": "union_of", "items": [ @@ -48737,7 +48752,7 @@ "name": "ThreadType", "namespace": "_types" }, - "specLocation": "_types/common.ts#L297-L303" + "specLocation": "_types/common.ts#L298-L304" }, { "kind": "type_alias", @@ -49093,7 +49108,7 @@ "name": "Username", "namespace": "_types" }, - "specLocation": "_types/common.ts#L194-L194", + "specLocation": "_types/common.ts#L195-L195", "type": { "kind": "instance_of", "type": { @@ -49108,7 +49123,7 @@ "name": "Uuid", "namespace": "_types" }, - "specLocation": "_types/common.ts#L125-L125", + "specLocation": "_types/common.ts#L126-L126", "type": { "kind": "instance_of", "type": { @@ -49123,7 +49138,7 @@ "name": "VersionNumber", "namespace": "_types" }, - "specLocation": "_types/common.ts#L102-L102", + "specLocation": "_types/common.ts#L103-L103", "type": { "kind": "instance_of", "type": { @@ -49138,7 +49153,7 @@ "name": "VersionString", "namespace": "_types" }, - "specLocation": "_types/common.ts#L104-L104", + "specLocation": "_types/common.ts#L105-L105", "type": { "kind": "instance_of", "type": { @@ -49170,7 +49185,7 @@ "name": "VersionType", "namespace": "_types" }, - "specLocation": "_types/common.ts#L106-L122" + "specLocation": "_types/common.ts#L107-L123" }, { "kind": "enum", @@ -49186,7 +49201,7 @@ "name": "WaitForActiveShardOptions", "namespace": "_types" }, - "specLocation": "_types/common.ts#L305-L309" + "specLocation": "_types/common.ts#L306-L310" }, { "kind": "type_alias", @@ -49198,7 +49213,7 @@ "name": "WaitForActiveShards", "namespace": "_types" }, - "specLocation": "_types/common.ts#L142-L143", + "specLocation": "_types/common.ts#L143-L144", "type": { "kind": "union_of", "items": [ @@ -49245,7 +49260,7 @@ "name": "WaitForEvents", "namespace": "_types" }, - "specLocation": "_types/common.ts#L311-L318" + "specLocation": "_types/common.ts#L312-L319" }, { "kind": "interface", @@ -140387,8 +140402,8 @@ "value": { "kind": "instance_of", "type": { - "name": "string", - "namespace": "_builtins" + "name": "GrokPattern", + "namespace": "_types" } } } @@ -141986,8 +142001,8 @@ "value": { "kind": "instance_of", "type": { - "name": "string", - "namespace": "_builtins" + "name": "GrokPattern", + "namespace": "_types" } } } @@ -147854,8 +147869,8 @@ "type": { "kind": "instance_of", "type": { - "name": "string", - "namespace": "_builtins" + "name": "GrokPattern", + "namespace": "_types" } } }, @@ -201986,8 +202001,8 @@ "type": { "kind": "instance_of", "type": { - "name": "string", - "namespace": "_builtins" + "name": "GrokPattern", + "namespace": "_types" } } }, @@ -202275,8 +202290,8 @@ "type": { "kind": "instance_of", "type": { - "name": "string", - "namespace": "_builtins" + "name": "GrokPattern", + "namespace": "_types" } } }, @@ -202503,8 +202518,8 @@ "type": { "kind": "instance_of", "type": { - "name": "string", - "namespace": "_builtins" + "name": "GrokPattern", + "namespace": "_types" } } }, @@ -202551,7 +202566,7 @@ } } ], - "specLocation": "text_structure/test_grok_pattern/TestGrokPatternRequest.ts#L22-L43" + "specLocation": "text_structure/test_grok_pattern/TestGrokPatternRequest.ts#L23-L44" }, { "kind": "response", diff --git a/output/typescript/types.ts b/output/typescript/types.ts index 94577b3621..e88b675907 100644 --- a/output/typescript/types.ts +++ b/output/typescript/types.ts @@ -2326,6 +2326,8 @@ export interface GetStats { total: long } +export type GrokPattern = string + export type HealthStatus = 'green' | 'GREEN' | 'yellow' | 'YELLOW' | 'red' | 'RED' export type Host = string @@ -12767,7 +12769,7 @@ export interface IngestGrokProcessor extends IngestProcessorBase { field: Field ignore_missing?: boolean pattern_definitions?: Record - patterns: string[] + patterns: GrokPattern[] trace_match?: boolean } @@ -12922,7 +12924,7 @@ export interface IngestProcessorContainer { export interface IngestRedactProcessor extends IngestProcessorBase { field: Field - patterns: string[] + patterns: GrokPattern[] pattern_definitions?: Record prefix?: string suffix?: string @@ -13527,7 +13529,7 @@ export type MlCategorizationStatus = 'ok' | 'warn' export interface MlCategory { category_id: ulong examples: string[] - grok_pattern?: string + grok_pattern?: GrokPattern job_id: Id max_matching_length: ulong partition_field_name?: string @@ -19230,7 +19232,7 @@ export interface TextStructureFindStructureRequest { ecs_compatibility?: string explain?: boolean format?: string - grok_pattern?: string + grok_pattern?: GrokPattern has_header_row?: boolean line_merge_size_limit?: uint lines_to_sample?: uint @@ -19257,7 +19259,7 @@ export interface TextStructureFindStructureResponse { num_lines_analyzed: integer column_names?: string[] explanation?: string[] - grok_pattern?: string + grok_pattern?: GrokPattern multiline_start_pattern?: string exclude_lines_pattern?: string java_timestamp_formats?: string[] @@ -19286,7 +19288,7 @@ export interface TextStructureTestGrokPatternMatchedText { export interface TextStructureTestGrokPatternRequest extends RequestBase { ecs_compatibility?: string body?: { - grok_pattern: string + grok_pattern: GrokPattern text: string[] } } diff --git a/specification/_types/common.ts b/specification/_types/common.ts index 4a8ad73fe1..33edc9911c 100644 --- a/specification/_types/common.ts +++ b/specification/_types/common.ts @@ -84,6 +84,7 @@ export type Namespace = string export type Service = string export type PipelineName = string +export type GrokPattern = string /** @doc_id modules-node */ export type NodeName = string diff --git a/specification/ingest/_types/Processors.ts b/specification/ingest/_types/Processors.ts index 13d467641e..33212aeaf9 100644 --- a/specification/ingest/_types/Processors.ts +++ b/specification/ingest/_types/Processors.ts @@ -19,7 +19,7 @@ import { Dictionary } from '@spec_utils/Dictionary' import { UserDefinedValue } from '@spec_utils/UserDefinedValue' -import { Field, Fields, Id, Name } from '@_types/common' +import { Field, Fields, GrokPattern, Id, Name } from '@_types/common' import { GeoShapeRelation } from '@_types/Geo' import { double, integer, long } from '@_types/Numeric' import { SortOrder } from '@_types/sort' @@ -785,7 +785,7 @@ export class GrokProcessor extends ProcessorBase { * An ordered list of grok expression to match and extract named captures with. * Returns on the first expression in the list that matches. */ - patterns: string[] + patterns: GrokPattern[] /** * When `true`, `_ingest._grok_match_index` will be inserted into your matched document’s metadata with the index into the pattern found in `patterns` that matched. * @server_default false @@ -1061,7 +1061,7 @@ export class RedactProcessor extends ProcessorBase { /** * A list of grok expressions to match and redact named captures with */ - patterns: string[] + patterns: GrokPattern[] /* * A map of pattern-name and pattern tuples defining custom patterns to be used by the processor. * Patterns matching existing names will override the pre-existing definition diff --git a/specification/ml/_types/Category.ts b/specification/ml/_types/Category.ts index 231257eb6f..3c347bdcc2 100644 --- a/specification/ml/_types/Category.ts +++ b/specification/ml/_types/Category.ts @@ -17,7 +17,7 @@ * under the License. */ -import { Id } from '@_types/common' +import { GrokPattern, Id } from '@_types/common' import { long, ulong } from '@_types/Numeric' export class Category { @@ -26,7 +26,7 @@ export class Category { /** A list of examples of actual values that matched the category. */ examples: string[] /** [experimental] A Grok pattern that could be used in Logstash or an ingest pipeline to extract fields from messages that match the category. This field is experimental and may be changed or removed in a future release. The Grok patterns that are found are not optimal, but are often a good starting point for manual tweaking. */ - grok_pattern?: string + grok_pattern?: GrokPattern /** Identifier for the anomaly detection job. */ job_id: Id /** The maximum length of the fields that matched the category. The value is increased by 10% to enable matching for similar fields that have not been analyzed. */ diff --git a/specification/text_structure/find_structure/FindStructureRequest.ts b/specification/text_structure/find_structure/FindStructureRequest.ts index 1a00af2c62..ae48be2faf 100644 --- a/specification/text_structure/find_structure/FindStructureRequest.ts +++ b/specification/text_structure/find_structure/FindStructureRequest.ts @@ -17,7 +17,7 @@ * under the License. */ -import { Field } from '@_types/common' +import { Field, GrokPattern } from '@_types/common' import { uint } from '@_types/Numeric' import { Duration } from '@_types/Time' @@ -44,7 +44,7 @@ export interface Request { /** The high level structure of the text. Valid values are ndjson, xml, delimited, and semi_structured_text. By default, the API chooses the format. In this default scenario, all rows must have the same number of fields for a delimited format to be detected. If the format is set to delimited and the delimiter is not set, however, the API tolerates up to 5% of rows that have a different number of columns than the first row. */ format?: string /** If you have set format to semi_structured_text, you can specify a Grok pattern that is used to extract fields from every message in the text. The name of the timestamp field in the Grok pattern must match what is specified in the timestamp_field parameter. If that parameter is not specified, the name of the timestamp field in the Grok pattern must match "timestamp". If grok_pattern is not specified, the structure finder creates a Grok pattern. */ - grok_pattern?: string + grok_pattern?: GrokPattern /** If you have set format to delimited, you can use this parameter to indicate whether the column names are in the first row of the text. If this parameter is not specified, the structure finder guesses based on the similarity of the first row of the text to other rows. */ has_header_row?: boolean /** diff --git a/specification/text_structure/find_structure/FindStructureResponse.ts b/specification/text_structure/find_structure/FindStructureResponse.ts index 104935b00d..b6fb84d5a9 100644 --- a/specification/text_structure/find_structure/FindStructureResponse.ts +++ b/specification/text_structure/find_structure/FindStructureResponse.ts @@ -19,7 +19,7 @@ import { PipelineConfig } from '@ingest/_types/Pipeline' import { Dictionary } from '@spec_utils/Dictionary' -import { Field } from '@_types/common' +import { Field, GrokPattern } from '@_types/common' import { TypeMapping } from '@_types/mapping/TypeMapping' import { integer } from '@_types/Numeric' import { FieldStat } from './types' @@ -40,7 +40,7 @@ export class Response { num_lines_analyzed: integer column_names?: string[] explanation?: string[] - grok_pattern?: string + grok_pattern?: GrokPattern multiline_start_pattern?: string exclude_lines_pattern?: string java_timestamp_formats?: string[] diff --git a/specification/text_structure/test_grok_pattern/TestGrokPatternRequest.ts b/specification/text_structure/test_grok_pattern/TestGrokPatternRequest.ts index df403dd95f..0088b4301f 100644 --- a/specification/text_structure/test_grok_pattern/TestGrokPatternRequest.ts +++ b/specification/text_structure/test_grok_pattern/TestGrokPatternRequest.ts @@ -18,6 +18,7 @@ */ import { RequestBase } from '@_types/Base' +import { GrokPattern } from '@_types/common' /** * @rest_spec_name text_structure.test_grok_pattern @@ -33,7 +34,7 @@ export interface Request extends RequestBase { /** * Grok pattern to run on the text. */ - grok_pattern: string + grok_pattern: GrokPattern /** * Lines of text to run the Grok pattern on.