Add community_id, fingerprint, network_direction, registered_domain processors (#3011)

Add ingest processor definitions for `community_id`, `fingerprint`, `network_direction`, and `registered_domain` processors. All parameters and descriptions were scraped from the 8.16 docs.

Add `ip` to the list of types supported by the `convert` processor.

Add `ecs_compatibility` to grok processor.

Add `output_format` to date processor.

Related issues

- Fixes #2309
- Fixes #2553
- Fixes #2593
- Fixes #2617

Author: andrewkroh

specification/_doc_ids/table.csv

@@ -78,6 +78,7 @@ cluster-stats,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/c
 cluster-update-settings,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/cluster-update-settings.html
 cluster,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/cluster.html
 common-options,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/common-options.html
+community-id-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/community-id-processor.html
 connector-sync-job-cancel,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/cancel-connector-sync-job-api.html
 connector-sync-job-delete,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/delete-connector-sync-job-api.html
 connector-sync-job-get,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/get-connector-sync-job-api.html
@@ -157,6 +158,7 @@ fail-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/
 field-and-document-access-control,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/field-and-document-access-control.html
 field-usage-stats,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/field-usage-stats.html
 find-structure,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/find-structure.html
+fingerprint-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/fingerprint-processor.html
 foreach-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/foreach-processor.html
 fuzziness,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/common-options.html#fuzziness
 gap-policy,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/search-aggregations-pipeline.html#gap-policy
@@ -329,6 +331,7 @@ modules-scripting,https://www.elastic.co/guide/en/elasticsearch/reference/{branc
 modules-snapshots,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/modules-snapshots.html
 monitor-elasticsearch-cluster,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/monitor-elasticsearch-cluster.html
 multi-fields,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/multi-fields.html
+network-direction-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/network-direction-processor.html
 node-roles,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/modules-node.html#node-roles
 paginate-search-results,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/paginate-search-results.html
 painless-contexts,https://www.elastic.co/guide/en/elasticsearch/painless/{branch}/painless-contexts.html
@@ -407,6 +410,7 @@ query-dsl,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/query
 realtime,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/docs-get.html#realtime
 redact-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/redact-processor.html
 regexp-syntax,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/regexp-syntax.html
+registered-domain-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/registered-domain-processor.html
 remove-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/remove-processor.html
 rename-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/rename-processor.html
 reroute-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/reroute-processor.html

specification/ingest/_types/Processors.ts

@@ -55,6 +55,13 @@ export class ProcessorContainer {
    * @doc_id ingest-circle-processor
    */
   circle?: CircleProcessor
+  /**
+   * Computes the Community ID for network flow data as defined in the
+   * Community ID Specification. You can use a community ID to correlate network
+   * events related to a single flow.
+   * @doc_id community-id-processor
+   */
+  community_id?: CommunityIDProcessor
   /**
    * Converts a field in the currently ingested document to a different type, such as converting a string to an integer.
    * If the field value is an array, all members will be converted.
@@ -106,6 +113,12 @@ export class ProcessorContainer {
    * @doc_id fail-processor
    */
   fail?: FailProcessor
+  /**
+   * Computes a hash of the document’s content. You can use this hash for
+   * content fingerprinting.
+   * @doc_id fingerprint-processor
+   */
+  fingerprint?: FingerprintProcessor
   /**
    * Runs an ingest processor on each element of an array or object.
    * @doc_id foreach-processor
@@ -169,6 +182,12 @@ export class ProcessorContainer {
    * @doc_id lowercase-processor
    */
   lowercase?: LowercaseProcessor
+  /**
+   * Calculates the network direction given a source IP address, destination IP
+   * address, and a list of internal networks.
+   * @doc_id network-direction-processor
+   */
+  network_direction?: NetworkDirectionProcessor
   /**
    * Executes another pipeline.
    * @doc_id pipeline-processor
@@ -181,6 +200,14 @@ export class ProcessorContainer {
    * @doc_id redact-processor
    */
   redact?: RedactProcessor
+  /**
+   * Extracts the registered domain (also known as the effective top-level
+   * domain or eTLD), sub-domain, and top-level domain from a fully qualified
+   * domain name (FQDN). Uses the registered domains defined in the Mozilla
+   * Public Suffix List.
+   * @doc_id registered-domain-processor
+   */
+  registered_domain?: RegisteredDomainProcessor
   /**
    * Removes existing fields.
    * If one field doesn’t exist, an exception will be thrown.
@@ -528,13 +555,77 @@ export class CircleProcessor extends ProcessorBase {
   target_field?: Field
 }
 
+export class CommunityIDProcessor extends ProcessorBase {
+  /**
+   * Field containing the source IP address.
+   * @server_default source.ip
+   */
+  source_ip?: Field
+  /**
+   * Field containing the source port.
+   * @server_default source.port
+   */
+  source_port?: Field
+  /**
+   * Field containing the destination IP address.
+   * @server_default destination.ip
+   */
+  destination_ip?: Field
+  /**
+   * Field containing the destination port.
+   * @server_default destination.port
+   */
+  destination_port?: Field
+  /**
+   * Field containing the IANA number.
+   * @server_default network.iana_number
+   */
+  iana_number?: Field
+  /**
+   * Field containing the ICMP type.
+   * @server_default icmp.type
+   */
+  icmp_type?: Field
+  /**
+   * Field containing the ICMP code.
+   * @server_default icmp.code
+   */
+  icmp_code?: Field
+  /**
+   * Field containing the transport protocol name or number. Used only when the
+   * iana_number field is not present. The following protocol names are currently
+   * supported: eigrp, gre, icmp, icmpv6, igmp, ipv6-icmp, ospf, pim, sctp, tcp, udp
+   * @server_default network.transport
+   */
+  transport?: Field
+  /**
+   * Output field for the community ID.
+   * @server_default network.community_id
+   */
+  target_field?: Field
+  /**
+   * Seed for the community ID hash. Must be between 0 and 65535 (inclusive). The
+   * seed can prevent hash collisions between network domains, such as a staging
+   * and production network that use the same addressing scheme.
+   * @server_default 0
+   */
+  seed?: integer
+  /**
+   * If true and any required fields are missing, the processor quietly exits
+   * without modifying the document.
+   * @server_default true
+   */
+  ignore_missing?: boolean
+}
+
 export enum ConvertType {
   integer,
   long,
-  float,
   double,
-  string,
+  float,
   boolean,
+  ip,
+  string,
   auto
 }
 
@@ -662,6 +753,12 @@ export class DateProcessor extends ProcessorBase {
    * @server_default UTC
    */
   timezone?: string
+  /**
+   * The format to use when writing the date to target_field. Must be a valid
+   * java time pattern.
+   * @server_default yyyy-MM-dd'T'HH:mm:ss.SSSXXX
+   */
+  output_format?: string
 }
 
 export class DissectProcessor extends ProcessorBase {
@@ -756,6 +853,44 @@ export class FailProcessor extends ProcessorBase {
   message: string
 }
 
+export enum FingerprintDigest {
+  md5 = 'MD5',
+  sha1 = 'SHA-1',
+  sha256 = 'SHA-256',
+  sha512 = 'SHA-512',
+  murmurHash3 = 'MurmurHash3'
+}
+
+export class FingerprintProcessor extends ProcessorBase {
+  /**
+   * Array of fields to include in the fingerprint. For objects, the processor
+   * hashes both the field key and value. For other fields, the processor hashes
+   * only the field value.
+   */
+  fields: Fields
+  /**
+   * Output field for the fingerprint.
+   * @server_default fingerprint
+   */
+  target_field?: Field
+  /**
+   * Salt value for the hash function.
+   */
+  salt?: string
+  /**
+   * The hash method used to compute the fingerprint. Must be one of MD5, SHA-1,
+   * SHA-256, SHA-512, or MurmurHash3.
+   * @server_default SHA-1
+   */
+  method?: FingerprintDigest
+  /**
+   * If true, the processor ignores any missing fields. If all fields are
+   * missing, the processor silently exits without modifying the document.
+   * @server_default false
+   */
+  ignore_missing?: boolean
+}
+
 export class ForeachProcessor extends ProcessorBase {
   /**
    * Field containing array or object values.
@@ -773,6 +908,12 @@ export class ForeachProcessor extends ProcessorBase {
 }
 
 export class GrokProcessor extends ProcessorBase {
+  /**
+   * Must be disabled or v1. If v1, the processor uses patterns with Elastic
+   * Common Schema (ECS) field names.
+   * @server_default disabled
+   */
+  ecs_compatibility?: string
   /**
    * The field to use for grok expression parsing.
    */
@@ -1046,6 +1187,42 @@ export class LowercaseProcessor extends ProcessorBase {
   target_field?: Field
 }
 
+export class NetworkDirectionProcessor extends ProcessorBase {
+  /**
+   * Field containing the source IP address.
+   * @server_default source.ip
+   */
+  source_ip?: Field
+  /**
+   * Field containing the destination IP address.
+   * @server_default destination.ip
+   */
+  destination_ip?: Field
+  /**
+   * Output field for the network direction.
+   * @server_default network.direction
+   */
+  target_field?: Field
+  /**
+   * List of internal networks. Supports IPv4 and IPv6 addresses and ranges in
+   * CIDR notation. Also supports the named ranges listed below. These may be
+   * constructed with template snippets. Must specify only one of
+   * internal_networks or internal_networks_field.
+   */
+  internal_networks?: string[]
+  /**
+   * A field on the given document to read the internal_networks configuration
+   * from.
+   */
+  internal_networks_field?: Field
+  /**
+   * If true and any required fields are missing, the processor quietly exits
+   * without modifying the document.
+   * @server_default true
+   */
+  ignore_missing?: boolean
+}
+
 export class PipelineProcessor extends ProcessorBase {
   /**
    * The name of the pipeline to execute.
@@ -1102,6 +1279,24 @@ export class RedactProcessor extends ProcessorBase {
   trace_redact?: boolean
 }
 
+export class RegisteredDomainProcessor extends ProcessorBase {
+  /**
+   * Field containing the source FQDN.
+   */
+  field: Field
+  /**
+   * Object field containing extracted domain components. If an empty string,
+   * the processor adds components to the document’s root.
+   */
+  target_field?: Field
+  /**
+   * If true and any required fields are missing, the processor quietly exits
+   * without modifying the document.
+   * @server_default true
+   */
+  ignore_missing?: boolean
+}
+
 export class RemoveProcessor extends ProcessorBase {
   /**
    * Fields to be removed. Supports template snippets.