Merge pull request #206 from yaauie/ecs-region_iso_code

ECS: support composite region_iso_code

Author: yaauie

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 7.2.11
+ - Improved compatibility with the Elastic Common Schema [#206](https://github.com/logstash-plugins/logstash-filter-geoip/pull/206)
+   - Added support for ECS's composite `region_iso_code` (`US-WA`), which _replaces_ the non-ECS `region_code` (`WA`) as a default field with City databases. To get the stand-alone `region_code` in ECS mode, you must include it in the `fields` directive. 
+   - [DOC] Improve ECS-related documentation
+
 ## 7.2.10
   - [DOC] Air-gapped environment requires both ASN and City databases [#204](https://github.com/logstash-plugins/logstash-filter-geoip/pull/204)
 

docs/index.asciidoc

@@ -169,14 +169,57 @@ Example response:
 }
 --------------------------------------------------
 
+[id="plugins-{type}s-{plugin}-field-mapping"]
+==== Field mapping
+
+When this plugin is run with <<plugins-{type}s-{plugin}-ecs_compatibility>> disabled, the MaxMind DB's fields are added directly to the <<plugins-{type}s-{plugin}-target>>.
+When ECS compatibility is enabled, the fields are structured to fit into an ECS shape.
+
+[cols="3,5,3"]
+|===========================
+| Database Field Name | ECS Field | Example
+
+| `ip`               | `[ip]`                     | `12.34.56.78`
+
+| `city_name`        | `[geo][city_name]`         | `Seattle`
+| `country_name`     | `[geo][country_name]`      | `United States`
+| `continent_code`   | `[geo][continent_code]`    | `NA`
+| `continent_name`   | `[geo][continent_name]`    | `North America`
+| `country_code2`    | `[geo][country_iso_code]`  | `US`
+| `country_code3`    | _N/A_                      | `US`
+
+                                                    _maintained for legacy
+                                                    support, but populated
+                                                    with 2-character country
+                                                    code_
+
+| `postal_code`      | `[geo][postal_code]`       | `98106`
+| `region_name`      | `[geo][region_name]`       | `Washington`
+| `region_code`      | `[geo][region_code]`       | `WA`
+| `region_iso_code`* | `[geo][region_iso_code]`   | `US-WA`
+| `timezone`         | `[geo][timezone]`          | `America/Los_Angeles`
+| `location`*        | `[geo][location]`          | `{"lat": 47.6062, "lon": -122.3321}"`
+| `latitude`         | `[geo][location][lat]`     | `47.6062`
+| `longitude`        | `[geo][location][lon]`     | `-122.3321`
+
+| `domain`           | `[domain]`                 | `example.com`
+
+| `asn`              | `[as][number]`             | `98765`
+| `as_org`           | `[as][organization][name]` | `Elastic, NV`
+
+| `isp`              | `[mmdb][isp]`              | `InterLink Supra LLC`
+| `dma_code`         | `[mmdb][dma_code]`         | `819`
+| `organization`     | `[mmdb][organization]`     | `Elastic, NV`
+|===========================
+
+NOTE: `*` indicates a composite field, which is only populated if GeoIP lookup result contains all components.
+
 ==== Details
 
-A `[geoip][location]` field is created if
-the GeoIP lookup returns a latitude and longitude. The field is stored in
-http://geojson.org/geojson-spec.html[GeoJSON] format. Additionally,
-the default Elasticsearch template provided with the
-{logstash-ref}/plugins-outputs-elasticsearch.html[elasticsearch output] maps
-the `[geoip][location]` field to an {ref}/geo-point.html[Elasticsearch Geo_point datatype].
+When using a City database, the enrichment is aborted if no latitude/longitude pair is available.
+
+The `location` field combines the latitude and longitude into a structure called https://datatracker.ietf.org/doc/html/rfc7946[GeoJSON].
+When you are using a default <<plugins-{type}s-{plugin}-target>>, the templates provided by the {logstash-ref}/plugins-outputs-elasticsearch.html[elasticsearch output] map the field to an {ref}/geo-point.html[Elasticsearch Geo_point datatype].
 
 As this field is a `geo_point` _and_ it is still valid GeoJSON, you get
 the awesomeness of Elasticsearch's geospatial query, facet and filter functions
@@ -242,16 +285,16 @@ number of cache misses and waste memory.
 ===== `database`
 
   * Value type is <<path,path>>
-  * If not specified, the database defaults to the GeoLite2 City database that ships with Logstash.
+  * If not specified, the database defaults to the `GeoLite2 City` database that ships with Logstash.
 
-The path to MaxMind's database file that Logstash should use. The default database is GeoLite2-City.
-GeoLite2-City, GeoLite2-Country, GeoLite2-ASN are the free databases from MaxMind that are supported.
-GeoIP2-City, GeoIP2-ISP, GeoIP2-Country are the commercial databases from MaxMind that are supported.
+The path to MaxMind's database file that Logstash should use.
+The default database is `GeoLite2-City`.
+This plugin supports several free databases (`GeoLite2-City`, `GeoLite2-Country`, `GeoLite2-ASN`)
+and a selection of commercially-licensed databases (`GeoIP2-City`, `GeoIP2-ISP`, `GeoIP2-Country`).
 
-Database auto-update applies to default distribution. When `database` points to user's database path,
-auto-update will be disabled.
-See
-<<plugins-{type}s-{plugin}-database_license,Database License>> for more information.
+Database auto-update applies to the default distribution.
+When `database` points to user's database path, auto-update is disabled.
+See <<plugins-{type}s-{plugin}-database_license,Database License>> for more information.
 
 [id="plugins-{type}s-{plugin}-default_database_type"]
 ===== `default_database_type`
@@ -270,21 +313,18 @@ This plugin now includes both the GeoLite2-City and GeoLite2-ASN databases.  If
 
 An array of geoip fields to be included in the event.
 
-Possible fields depend on the database type. By default, all geoip fields
-are included in the event.
+Possible fields depend on the database type.
+By default, all geoip fields from the relevant database are included in the event.
 
-For the built-in GeoLite2 City database, the following are available:
-`city_name`, `continent_code`, `country_code2`, `country_code3`, `country_name`,
-`dma_code`, `ip`, `latitude`, `location`, `longitude`, `postal_code`, `region_code`,
-`region_name` and `timezone`.
+For a complete list of available fields and how they map to an event's structure, see <<plugins-{type}s-{plugin}-field-mapping,field mapping>>.
 
 [id="plugins-{type}s-{plugin}-ecs_compatibility"]
 ===== `ecs_compatibility`
 
 * Value type is <<string,string>>
 * Supported values are:
 ** `disabled`: unstructured geo data added at root level
-** `v1`, `v8`: uses fields that are compatible with Elastic Common Schema (for example, `[client][geo][country_name]`)
+** `v1`, `v8`: use fields that are compatible with Elastic Common Schema. Example: `[client][geo][country_name]`. See <<plugins-{type}s-{plugin}-field-mapping,field mapping>> for more info. 
 * Default value depends on which version of Logstash is running:
 ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
 ** Otherwise, the default value is `disabled`.

logstash-filter-geoip.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-geoip'
-  s.version         = '7.2.10'
+  s.version         = '7.2.11'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Adds geographical information about an IP address"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/filters/geoip_ecs_spec.rb

@@ -27,6 +27,10 @@
         end
 
         context "with city database" do
+          # example.com, has been static for 10+ years
+          # and has city-level details
+          let(:ip) { "93.184.216.34" }
+
           let(:options) { common_options }
 
           it "should return geo in target" do
@@ -36,15 +40,23 @@
             expect( event.get ecs_select[disabled: "[#{target}][country_code2]", v1: "[#{target}][geo][country_iso_code]"] ).to eq 'US'
             expect( event.get ecs_select[disabled: "[#{target}][country_name]", v1: "[#{target}][geo][country_name]"] ).to eq 'United States'
             expect( event.get ecs_select[disabled: "[#{target}][continent_code]", v1: "[#{target}][geo][continent_code]"] ).to eq 'NA'
-            expect( event.get ecs_select[disabled: "[#{target}][location][lat]", v1: "[#{target}][geo][location][lat]"] ).to eq 37.751
-            expect( event.get ecs_select[disabled: "[#{target}][location][lon]", v1: "[#{target}][geo][location][lon]"] ).to eq -97.822
+            expect( event.get ecs_select[disabled: "[#{target}][location][lat]", v1: "[#{target}][geo][location][lat]"] ).to eq 42.1596
+            expect( event.get ecs_select[disabled: "[#{target}][location][lon]", v1: "[#{target}][geo][location][lon]"] ).to eq -70.8217
+            expect( event.get ecs_select[disabled: "[#{target}][city_name]", v1: "[#{target}][geo][city_name]"] ).to eq 'Norwell'
+            expect( event.get ecs_select[disabled: "[#{target}][dma_code]", v1: "[#{target}][mmdb][dma_code]"] ).to eq 506
+            expect( event.get ecs_select[disabled: "[#{target}][region_name]", v1: "[#{target}][geo][region_name]"] ).to eq 'Massachusetts'
 
             if ecs_select.active_mode == :disabled
               expect( event.get "[#{target}][country_code3]" ).to eq 'US'
+              expect( event.get "[#{target}][region_code]" ).to eq 'MA'
+              expect( event.get "[#{target}][region_iso_code]" ).to be_nil
             else
               expect( event.get "[#{target}][geo][country_code3]" ).to be_nil
               expect( event.get "[#{target}][country_code3]" ).to be_nil
+              expect( event.get "[#{target}][geo][region_iso_code]" ).to eq 'US-MA'
+              expect( event.get "[#{target}][region_code]" ).to be_nil
             end
+            puts event.to_hash.inspect
           end
         end
 

src/main/java/org/logstash/filters/geoip/Fields.java

@@ -39,6 +39,7 @@ enum Fields {
   DMA_CODE("mmdb.dma_code", "dma_code"),
   REGION_NAME("geo.region_name", "region_name"),
   REGION_CODE("geo.region_code", "region_code"),
+  REGION_ISO_CODE("geo.region_iso_code", "region_iso_code"),
   TIMEZONE("geo.timezone", "timezone"),
   LOCATION("geo.location", "location"),
   LATITUDE("geo.location.lat", "latitude"),
@@ -96,6 +97,14 @@ public String getFieldReferenceECSv1() {
       Fields.COUNTRY_CODE3, Fields.IP, Fields.POSTAL_CODE, Fields.DMA_CODE, Fields.REGION_NAME,
       Fields.REGION_CODE, Fields.TIMEZONE, Fields.LOCATION, Fields.LATITUDE, Fields.LONGITUDE);
 
+  // When ECS is enabled, the composite REGION_ISO_CODE field is preferred to separate REGION_CODE
+  static final EnumSet<Fields> DEFAULT_ECS_CITY_FIELDS;
+  static {
+    DEFAULT_ECS_CITY_FIELDS = EnumSet.copyOf(DEFAULT_CITY_FIELDS);
+    DEFAULT_ECS_CITY_FIELDS.remove(REGION_CODE);
+    DEFAULT_ECS_CITY_FIELDS.add(REGION_ISO_CODE);
+  }
+
   static final EnumSet<Fields> DEFAULT_COUNTRY_FIELDS = EnumSet.of(Fields.IP, Fields.COUNTRY_CODE2,
       Fields.IP, Fields.COUNTRY_NAME, Fields.CONTINENT_NAME);
 

src/main/java/org/logstash/filters/geoip/GeoIPFilter.java

@@ -91,7 +91,7 @@ public GeoIPFilter(String sourceField, String targetField, List<String> fields,
     } catch (IOException e) {
       throw new IllegalArgumentException("The database provided was not found in the path", e);
     }
-    this.desiredFields = createDesiredFields(fields);
+    this.desiredFields = createDesiredFields(fields, !ecsCompatibility.equals("disabled"));
   }
 
   public static boolean isDatabaseValid(String databasePath) {
@@ -107,7 +107,7 @@ public static boolean isDatabaseValid(String databasePath) {
     return false;
   }
 
-  private Set<Fields> createDesiredFields(List<String> fields) {
+  private Set<Fields> createDesiredFields(List<String> fields, final boolean ecsCompatibilityEnabled) {
     Set<Fields> desiredFields = EnumSet.noneOf(Fields.class);
     if (fields == null || fields.isEmpty()) {
       switch (databaseReader.getMetadata().getDatabaseType()) {
@@ -118,7 +118,7 @@ private Set<Fields> createDesiredFields(List<String> fields) {
         case CITY_EUROPE_DB_TYPE:
         case CITY_NORTH_AMERICA_DB_TYPE:
         case CITY_SOUTH_AMERICA_DB_TYPE:
-          desiredFields = Fields.DEFAULT_CITY_FIELDS;
+          desiredFields = ecsCompatibilityEnabled ? Fields.DEFAULT_ECS_CITY_FIELDS : Fields.DEFAULT_CITY_FIELDS;
           break;
         case COUNTRY_LITE_DB_TYPE:
         case COUNTRY_DB_TYPE:
@@ -311,6 +311,13 @@ private Map<Fields,Object> retrieveCityGeoData(InetAddress ipAddress) throws Geo
             geoData.put(Fields.REGION_CODE, subdivisionCode);
           }
           break;
+        case REGION_ISO_CODE:
+          String countryCodeForRegion = country.getIsoCode();
+          String regionCode2 = subdivision.getIsoCode();
+          if (countryCodeForRegion != null && regionCode2 != null) {
+            geoData.put(Fields.REGION_ISO_CODE, String.format("%s-%s", countryCodeForRegion, regionCode2));
+          }
+          break;
         case TIMEZONE:
           String locationTimeZone = location.getTimeZone();
           if (locationTimeZone != null) {