diff --git a/.gitignore b/.gitignore index 822e58b..8815375 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /cover.*.gcov /cover.*.lcov -/dist/** \ No newline at end of file +/cmd/mmdbinspect/mmdbinspect +/dist/** diff --git a/CHANGELOG.md b/CHANGELOG.md index 874279f..aa4c702 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,27 @@ ## 2.0.0 -* Upgrade to `github.com/oschwald/maxminddb-golang/v2`. This is a breaking - API change, but should not affect the use of the program. +* The default output format is now YAML. This was done to improve the + readability when using the tool as a standalone utility for doing lookups + in an MMDB database. Use the `-jsonl` flag to output as JSONL instead. +* When outputting as JSON, we now use JSONL. There is one JSON object per + line. +* The output format has been flattened. Each record that is output now + contains the following keys: `database_path`, `requested_lookup`, + `network`, and `record`. This allows for efficient streaming of large + lookups, makes the key naming more consistent, and reduces the depth of + the data structure. +* You may now use a glob for the `-db` argument. If there are multiple + matches, it will be treated as if multiple `-db` arguments were provided. + Note that you must quote the parameter when using globs to prevent the + shell's globbing from interfering. See the [pattern syntax](https://pkg.go.dev/path#Match) +* The following flags were added: + * `-include-networks-without-data` - include networks without any data in + the database in the output. + * `-include-build-time` - include the build time from the database's + metadata in the output. +* This repo no longer provides a public Go API. It is only intended to be + used as a CLI program. ## 0.2.0 (2024-01-10) diff --git a/README.md b/README.md index 26f8b7f..0ae7560 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,19 @@ ## Usage ```bash -mmdbinspect -db - db Path to a MMDB file. Can be specified multiple times. - An IP address, or network in CIDR notation. Can be - specified multiple times. +mmdbinspect [-include-aliased-networks] [-include-build-time] [-include-networks-without-data] [-jsonl] -db path/to/db [IP|network] [IP|network]... + -db value Path to an mmdb file. You may pass this arg more than once. + This may also be a glob pattern matching one or more MMDB files. + -include-aliased-networks + Include aliased networks (e.g. 6to4, Teredo). This option may + cause IPv4 networks to be listed more than once via aliases. + -include-build-time Include the build time of the database in the output. + -include-networks-without-data + Include networks that have no data in the database. + The "record" will be null for these. + -jsonl Output as JSONL instead of YAML. + [IP|network] An IP address, or network in CIDR notation. Can be + specified multiple times. ``` ## Description @@ -36,7 +45,7 @@ Any IPs specified will be treated as their single-host network counterparts (e.g `mmdbinspect` will look up each IP/network in each database specified. For each IP/network looked up in a database, the program will select all records for networks which are contained within the looked up IP/network. If no records for contained networks are found in the datafile, the program will select the record that is contained by the looked up IP/network. If no such records are found, none are selected. -The program outputs the selected records as a JSON array, with each item in the array corresponding to a single IP/network being looked up in a single DB. The `Database` and `Lookup` keys are added to each item to help correlate which set of records resulted from looking up which IP/network in which database. +The program outputs the selected records in YAML format by default (use `-jsonl` for JSONL format). Each output item corresponds to a single IP/network being looked up in a single DB. Each record contains the following keys: `database_path`, `requested_lookup`, `network`, and `record`. This format allows for efficient streaming of large lookups and makes the key naming more consistent. ## Beta Release @@ -99,61 +108,46 @@ This installs `mmdbinspect` to `$GOPATH/bin/mmdbinspect`. ```bash $ mmdbinspect -db GeoIP2-Country.mmdb 152.216.7.110 -[ - { - "Database": "GeoIP2-Country.mmdb", - "Records": [ - { - "Network": "152.216.7.110/12", - "Record": { - "continent": { - "code": "NA", - "geoname_id": 6255149, - "names": { - "de": "Nordamerika", - "en": "North America", - "es": "Norteamérica", - "fr": "Amérique du Nord", - "ja": "北アメリカ", - "pt-BR": "América do Norte", - "ru": "Северная Америка", - "zh-CN": "北美洲" - } - }, - "country": { - "geoname_id": 6252001, - "iso_code": "US", - "names": { - "de": "USA", - "en": "United States", - "es": "Estados Unidos", - "fr": "États-Unis", - "ja": "アメリカ合衆国", - "pt-BR": "Estados Unidos", - "ru": "США", - "zh-CN": "美国" - } - }, - "registered_country": { - "geoname_id": 6252001, - "iso_code": "US", - "names": { - "de": "USA", - "en": "United States", - "es": "Estados Unidos", - "fr": "États-Unis", - "ja": "アメリカ合衆国", - "pt-BR": "Estados Unidos", - "ru": "США", - "zh-CN": "美国" - } - } - } - } - ], - "Lookup": "152.216.7.110" - } -] +database_path: GeoIP2-Country.mmdb +requested_lookup: 152.216.7.110 +network: 152.208.0.0/12 +record: + continent: + code: NA + geoname_id: 6255149 + names: + de: Nordamerika + en: North America + es: Norteamérica + fr: Amérique du Nord + ja: 北アメリカ + pt-BR: América do Norte + ru: Северная Америка + zh-CN: 北美洲 + country: + geoname_id: 6252001 + iso_code: US + names: + de: USA + en: United States + es: Estados Unidos + fr: États Unis + ja: アメリカ + pt-BR: EUA + ru: США + zh-CN: 美国 + registered_country: + geoname_id: 6252001 + iso_code: US + names: + de: USA + en: United States + es: Estados Unidos + fr: États Unis + ja: アメリカ + pt-BR: EUA + ru: США + zh-CN: 美国 ``` @@ -162,120 +156,87 @@ $ mmdbinspect -db GeoIP2-Country.mmdb 152.216.7.110 ```bash $ mmdbinspect -db GeoIP2-Country.mmdb -db GeoIP2-City.mmdb 152.216.7.110 -[ - { - "Database": "GeoIP2-Country.mmdb", - "Records": [ - { - "Network": "152.216.7.110/12", - "Record": { - "continent": { - "code": "NA", - "geoname_id": 6255149, - "names": { - "de": "Nordamerika", - "en": "North America", - "es": "Norteamérica", - "fr": "Amérique du Nord", - "ja": "北アメリカ", - "pt-BR": "América do Norte", - "ru": "Северная Америка", - "zh-CN": "北美洲" - } - }, - "country": { - "geoname_id": 6252001, - "iso_code": "US", - "names": { - "de": "USA", - "en": "United States", - "es": "Estados Unidos", - "fr": "États-Unis", - "ja": "アメリカ合衆国", - "pt-BR": "Estados Unidos", - "ru": "США", - "zh-CN": "美国" - } - }, - "registered_country": { - "geoname_id": 6252001, - "iso_code": "US", - "names": { - "de": "USA", - "en": "United States", - "es": "Estados Unidos", - "fr": "États-Unis", - "ja": "アメリカ合衆国", - "pt-BR": "Estados Unidos", - "ru": "США", - "zh-CN": "美国" - } - } - } - } - ], - "Lookup": "152.216.7.110" - }, - { - "Database": "GeoIP2-City.mmdb", - "Records": [ - { - "Network": "152.216.7.110/14", - "Record": { - "continent": { - "code": "NA", - "geoname_id": 6255149, - "names": { - "de": "Nordamerika", - "en": "North America", - "es": "Norteamérica", - "fr": "Amérique du Nord", - "ja": "北アメリカ", - "pt-BR": "América do Norte", - "ru": "Северная Америка", - "zh-CN": "北美洲" - } - }, - "country": { - "geoname_id": 6252001, - "iso_code": "US", - "names": { - "de": "USA", - "en": "United States", - "es": "Estados Unidos", - "fr": "États-Unis", - "ja": "アメリカ合衆国", - "pt-BR": "Estados Unidos", - "ru": "США", - "zh-CN": "美国" - } - }, - "location": { - "accuracy_radius": 1000, - "latitude": 37.751, - "longitude": -97.822, - "time_zone": "America/Chicago" - }, - "registered_country": { - "geoname_id": 6252001, - "iso_code": "US", - "names": { - "de": "USA", - "en": "United States", - "es": "Estados Unidos", - "fr": "États-Unis", - "ja": "アメリカ合衆国", - "pt-BR": "Estados Unidos", - "ru": "США", - "zh-CN": "美国" - } - } - } - } - ], - "Lookup": "152.216.7.110" - } -] +database_path: GeoIP2-Country.mmdb +requested_lookup: 152.216.7.110 +network: 152.208.0.0/12 +record: + continent: + code: NA + geoname_id: 6255149 + names: + de: Nordamerika + en: North America + es: Norteamérica + fr: Amérique du Nord + ja: 北アメリカ + pt-BR: América do Norte + ru: Северная Америка + zh-CN: 北美洲 + country: + geoname_id: 6252001 + iso_code: US + names: + de: USA + en: United States + es: Estados Unidos + fr: États Unis + ja: アメリカ + pt-BR: EUA + ru: США + zh-CN: 美国 + registered_country: + geoname_id: 6252001 + iso_code: US + names: + de: USA + en: United States + es: Estados Unidos + fr: États Unis + ja: アメリカ + pt-BR: EUA + ru: США + zh-CN: 美国 +--- +database_path: GeoIP2-City.mmdb +requested_lookup: 152.216.7.110 +network: 152.216.4.0/22 +record: + continent: + code: NA + geoname_id: 6255149 + names: + de: Nordamerika + en: North America + es: Norteamérica + fr: Amérique du Nord + ja: 北アメリカ + pt-BR: América do Norte + ru: Северная Америка + zh-CN: 北美洲 + country: + geoname_id: 6252001 + iso_code: US + names: + de: USA + en: United States + es: Estados Unidos + fr: États Unis + ja: アメリカ + pt-BR: EUA + ru: США + zh-CN: 美国 + registered_country: + geoname_id: 6252001 + iso_code: US + names: + de: USA + en: United States + es: Estados Unidos + fr: États Unis + ja: アメリカ + pt-BR: EUA + ru: США + zh-CN: 美国 ``` @@ -284,32 +245,17 @@ $ mmdbinspect -db GeoIP2-Country.mmdb -db GeoIP2-City.mmdb 152.216.7.110 ```bash $ mmdbinspect -db GeoIP2-Connection-Type.mmdb 152.216.7.110/20 2001:0:98d8::/64 -[ - { - "Database": "GeoIP2-Connection-Type.mmdb", - "Records": [ - { - "Network": "152.216.0.0/13", - "Record": { - "connection_type": "Corporate" - } - } - ], - "Lookup": "152.216.7.110/20" - }, - { - "Database": "GeoIP2-Connection-Type.mmdb", - "Records": [ - { - "Network": "2001:0:98d8::/45", - "Record": { - "connection_type": "Corporate" - } - } - ], - "Lookup": "2001:0:98d8::/64" - } -] +database_path: GeoIP2-Connection-Type.mmdb +requested_lookup: 152.216.7.110/20 +network: 152.216.0.0/19 +record: + connection_type: Corporate +--- +database_path: GeoIP2-Connection-Type.mmdb +requested_lookup: 2001:0:98d8::/64 +network: 2001:0:98d8::/51 +record: + connection_type: Corporate ``` @@ -317,62 +263,61 @@ $ mmdbinspect -db GeoIP2-Connection-Type.mmdb 152.216.7.110/20 2001:0:98d8::/64 Look up multiple IPs/networks in multiple databases ```bash -$ mmdbinspect -db GeoIP2-DensityIncome.mmdb -db GeoIP2-User-Count.mmdb 152.216.7.32/27 2610:30::/64 -[ - { - "Database": "GeoIP2-DensityIncome.mmdb", - "Records": [ - { - "Network": "152.216.7.32/21", - "Record": { - "average_income": 26483, - "population_density": 1265 - } - } - ], - "Lookup": "152.216.7.32/27" - }, - { - "Database": "GeoIP2-DensityIncome.mmdb", - "Records": [ - { - "Network": "2610:30::/38", - "Record": { - "average_income": 30369, - "population_density": 934 - } - } - ], - "Lookup": "2610:30::/64" - }, - { - "Database": "GeoIP2-User-Count.mmdb", - "Records": [ - { - "Network": "152.216.7.32/27", - "Record": { - "ipv4_24": 6, - "ipv4_32": 0 - } - } - ], - "Lookup": "152.216.7.32/27" - }, - { - "Database": "GeoIP2-User-Count.mmdb", - "Records": [ - { - "Network": "2610:30::/27", - "Record": { - "ipv6_32": 0, - "ipv6_48": 0, - "ipv6_64": 0 - } - } - ], - "Lookup": "2610:30::/64" - } -] +$ mmdbinspect -db GeoLite2-ASN.mmdb -db GeoIP2-Connection-Type.mmdb 152.216.7.110/20 2001:0:98d8::/64 +database_path: GeoIP/GeoLite2-ASN.mmdb +requested_lookup: 152.216.7.110/20 +network: 152.216.0.0/19 +record: + autonomous_system_number: 30313 + autonomous_system_organization: IRS +--- +database_path: GeoIP/GeoLite2-ASN.mmdb +requested_lookup: 2001:0:98d8::/64 +network: 2001:0:98d8::/51 +record: + autonomous_system_number: 30313 + autonomous_system_organization: IRS +--- +database_path: GeoIP2-Connection-Type.mmdb +requested_lookup: 152.216.7.110/20 +network: 152.216.0.0/19 +record: + connection_type: Cable/DSL +--- +database_path: GeoIP2-Connection-Type.mmdb +requested_lookup: 2001:0:98d8::/64 +network: 2001:0:98d8::/51 +record: + connection_type: Cable/DSL +``` + + +
+ Using glob patterns to match multiple database files + +```bash +$ mmdbinspect -db "GeoIP2-*.mmdb" 152.216.7.110 +database_path: GeoIP2-Country.mmdb +requested_lookup: 152.216.7.110 +network: 152.208.0.0/12 +record: + continent: + code: NA + geoname_id: 6255149 + names: + de: Nordamerika + en: North America + # ... more names + country: + geoname_id: 6252001 + iso_code: US + # ... more country data +--- +database_path: GeoIP2-City.mmdb +requested_lookup: 152.216.7.110 +network: 152.216.4.0/22 +record: + # ... city data ```
@@ -384,89 +329,113 @@ $ cat list.txt 152.216.7.32/27 2610:30::/64 $ cat list.txt | xargs mmdbinspect -db GeoIP2-ISP.mmdb -[ - { - "Database": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb", - "Records": [ - { - "Network": "152.216.7.32/20", - "Record": { - "autonomous_system_number": 30313, - "autonomous_system_organization": "IRS", - "isp": "Internal Revenue Service", - "organization": "Internal Revenue Service" - } - } - ], - "Lookup": "152.216.7.32/27" - }, - { - "Database": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb", - "Records": [ - { - "Network": "2610:30::/32", - "Record": { - "autonomous_system_number": 30313, - "autonomous_system_organization": "IRS", - "isp": "Internal Revenue Service", - "organization": "Internal Revenue Service" - } - } - ], - "Lookup": "2610:30::/64" - } -] +database_path: GeoIP2-ISP.mmdb +requested_lookup: 152.216.7.32/27 +network: 152.216.0.0/19 +record: + autonomous_system_number: 30313 + autonomous_system_organization: IRS + isp: Internal Revenue Service + organization: Internal Revenue Service +--- +database_path: GeoIP/GeoIP2-ISP.mmdb +requested_lookup: 2610:30::/64 +network: 2610:30::/32 +record: + autonomous_system_number: 30313 + autonomous_system_organization: IRS + isp: Internal Revenue Service + organization: Internal Revenue Service ```
-Tame the output with the jq utility +Processing the output with the -jsonl flag and the jq utility Print out the `isp` field from each result found: ```bash -$ mmdbinspect -db GeoIP2-ISP.mmdb 152.216.7.32/27 | jq -r '.[].Records[].Record.isp' +$ mmdbinspect -jsonl -db GeoIP2-ISP.mmdb 152.216.7.32/27 | jq -r '.record.isp' Internal Revenue Service ``` Print out the `isp` field from each result found in a specific format using string addition: ```bash -$ mmdbinspect -db GeoIP2-ISP.mmdb 152.216.7.32/27 | jq -r '.[].Records[].Record | "isp=" + .isp' +$ mmdbinspect -jsonl -db GeoIP2-ISP.mmdb 152.216.7.32/27 | jq -r '.record | "isp=" + .isp' isp=Internal Revenue Service ``` Print out the `city` and `country` names from each record using string addition: ```bash -$ mmdbinspect -db GeoIP2-City.mmdb 2610:30::/64 | jq -r '.[].Records[].Record | .city.names.en + ", " + .country.names.en' +$ mmdbinspect -jsonl -db GeoIP2-City.mmdb 2610:30::/64 | jq -r '.record | .city.names.en + ", " + .country.names.en' Martinsburg, United States ``` Print out the `city` and `country` names from each record using array construction and `join`: ```bash -$ mmdbinspect -db GeoIP2-City.mmdb 2610:30::/64 | jq -r '.[].Records[].Record | [.city.names.en, .country.names.en] | join(", ")' +$ mmdbinspect -jsonl -db GeoIP2-City.mmdb 2610:30::/64 | jq -r '.record | [.city.names.en, .country.names.en] | join(", ")' Martinsburg, United States ``` Get the AS number for an IP: ```bash -$ mmdbinspect -db GeoLite2-ASN.mmdb 152.216.7.49 | jq -r '.[].Records[].Record.autonomous_system_number' +$ mmdbinspect -jsonl -db GeoLite2-ASN.mmdb 152.216.7.49 | jq -r '.record.autonomous_system_number' 30313 ``` +Create a CSV file with network and country code for all networks with data: +```bash +$ echo "network,country" > networks.csv +$ mmdbinspect -jsonl -db GeoIP2-Country.mmdb ::/0 | jq -r '[.network, .record.country.iso_code] | join(",")' >> networks.csv +$ cat networks.csv +network,country +1.1.1.0/24,AU +... +``` + When asking `jq` to print a path it can't find, it'll print `null`: ```bash -$ mmdbinspect -db GeoIP2-City.mmdb 152.216.7.49 | jq -r '.[].invalid.path' +$ mmdbinspect -jsonl -db GeoIP2-City.mmdb 152.216.7.49 | jq -r '.invalid.path' null ``` When asking `jq` to concatenate or join a path it can't find, it'll leave it blank: ```bash -$ mmdbinspect -db GeoIP2-City.mmdb 152.216.7.49 | jq -r '.[].Records[].Record | .city.names.en + ", " + .country.names.en' +$ mmdbinspect -jsonl -db GeoIP2-City.mmdb 152.216.7.49 | jq -r '.record | .city.names.en + ", " + .country.names.en' , United States -$ mmdbinspect -db GeoIP2-City.mmdb 152.216.7.49 | jq -r '.[].Records[].Record | [.city.names.en, .country.names.en] | join(", ")' +$ mmdbinspect -jsonl -db GeoIP2-City.mmdb 152.216.7.49 | jq -r '.record | [.city.names.en, .country.names.en] | join(", ")' , United States ```
+
+Using the `-include-*` flags for additional information + +Include build time information: +```bash +$ mmdbinspect -db GeoIP2-City.mmdb -include-build-time 152.216.7.110 +database_path: GeoIP2-City.mmdb +build_time: 2023-01-15T12:34:56Z +requested_lookup: 152.216.7.110 +network: 152.216.4.0/22 +record: + # ... city data +``` + +Include networks without data: +```bash +$ mmdbinspect -db GeoIP2-City.mmdb -include-networks-without-data 192.0.2.1 +database_path: GeoIP2-City.mmdb +requested_lookup: 192.0.2.1 +network: 192.0.2.0/24 +``` + +Include aliased networks: +```bash +$ mmdbinspect -db GeoIP2-City.mmdb -include-aliased-networks ::/0 +# ... All IPs in the database, including all aliased networks. +``` +
+ ## Bug Reports Please report bugs by filing an issue with our GitHub issue tracker at [https://github.com/maxmind/mmdbinspect/issues](https://github.com/maxmind/mmdbinspect/issues). diff --git a/cmd/mmdbinspect/main.go b/cmd/mmdbinspect/main.go index 319b2b2..42a76d7 100644 --- a/cmd/mmdbinspect/main.go +++ b/cmd/mmdbinspect/main.go @@ -8,8 +8,6 @@ import ( "log" "os" "strings" - - "github.com/maxmind/mmdbinspect/v2/pkg/mmdbinspect" ) type arrayFlags []string @@ -23,35 +21,38 @@ func (i *arrayFlags) Set(value string) error { return nil } -func usage() { - fmt.Printf( - "Usage: %s [-include-aliased-networks] -db path/to/db -db path/to/other/db 130.113.64.30/24 0:0:0:0:0:ffff:8064:a678\n", //nolint: lll - os.Args[0], - ) - flag.PrintDefaults() - fmt.Print(` -Any additional arguments passed are assumed to be networks to look up. If an -address range is not supplied, /32 will be assumed for ipv4 addresses and /128 -will be assumed for ipv6 addresses. -`) -} - func main() { var mmdb arrayFlags flag.Var(&mmdb, "db", "Path to an mmdb file. You may pass this arg more than once.") + includeAliasedNetworks := flag.Bool( - "include-aliased-networks", false, + "include-aliased-networks", + false, "Include aliased networks (e.g. 6to4, Teredo). This option may cause IPv4 networks to be listed more than once via aliases.", //nolint: lll ) + includeBuildTime := flag.Bool( + "include-build-time", + false, + "Include the build time of the database in the output.", + ) + + includeNetworksWithoutData := flag.Bool( + "include-networks-without-data", + false, + `Include networks that have no data in the database. The "record" will be null for these.`, + ) + + useJSONL := flag.Bool("jsonl", false, "Output as JSONL instead of YAML.") + flag.Usage = usage flag.Parse() // Any remaining arguments (not passed via flags) should be networks - network := flag.Args() + networks := flag.Args() - if len(network) == 0 { + if len(networks) == 0 { fmt.Println("You must provide at least one network address") usage() os.Exit(1) @@ -63,15 +64,34 @@ func main() { os.Exit(1) } - records, err := mmdbinspect.AggregatedRecords(network, mmdb, *includeAliasedNetworks) - if err != nil { - log.Fatal(err) - } + w := os.Stdout - json, err := mmdbinspect.RecordToString(records) + err := process( + w, + *useJSONL, + networks, + mmdb, + *includeAliasedNetworks, + *includeBuildTime, + *includeNetworksWithoutData, + ) if err != nil { log.Fatal(err) } +} + +func usage() { + fmt.Printf( + "Usage: %s [-include-aliased-networks] -db path/to/db -db path/to/other/db 130.113.64.30/24 0:0:0:0:0:ffff:8064:a678\n", //nolint: lll + os.Args[0], + ) + flag.PrintDefaults() + fmt.Print(` +The -db parameter may be a path to an MMDB file or a glob matching one or more +MMDB files. - fmt.Printf("%v\n", json) +Any additional arguments passed are assumed to be networks to look up. If an +address range is not supplied, /32 will be assumed for ipv4 addresses and /128 +will be assumed for ipv6 addresses. +`) } diff --git a/cmd/mmdbinspect/process.go b/cmd/mmdbinspect/process.go new file mode 100644 index 0000000..73c44e1 --- /dev/null +++ b/cmd/mmdbinspect/process.go @@ -0,0 +1,50 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/goccy/go-yaml" +) + +func process( + w io.Writer, + useJSONL bool, + networks, databases []string, + includeAliasedNetworks, + includeBuildTime, + includeNetworksWithoutData bool, +) error { + var encoder interface { + Encode(any) error + } + + if useJSONL { + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) // don't escape ampersands and angle brackets + encoder = enc + } else { + encoder = yaml.NewEncoder(w) + } + + iterator := records( + networks, + databases, + includeAliasedNetworks, + includeBuildTime, + includeNetworksWithoutData, + ) + + for r, err := range iterator { + if err != nil { + return err + } + + if err := encoder.Encode(r); err != nil { + return fmt.Errorf("encoding record: %w", err) + } + } + + return nil +} diff --git a/cmd/mmdbinspect/process_test.go b/cmd/mmdbinspect/process_test.go new file mode 100644 index 0000000..d2a21e0 --- /dev/null +++ b/cmd/mmdbinspect/process_test.go @@ -0,0 +1,72 @@ +package main + +import ( + "bytes" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestProcess(t *testing.T) { + // We switch directories so that we don't have to deal with path separator + // differences in the output. + // TODO: after we switch to Go 1.24, this can just be t.Chdir + originalDir, err := os.Getwd() + require.NoError(t, err) + + defer func() { + require.NoError(t, os.Chdir(originalDir)) + }() + + require.NoError(t, os.Chdir(testDataDir)) + + tests := []struct { + name string + useJSONL bool + networks []string + databases []string + expected string + }{ + { + name: "single record YAML", + useJSONL: false, + networks: []string{"81.2.69.142"}, + databases: []string{"GeoIP2-Country-Test.mmdb"}, + expected: "database_path:.*\nrequested_lookup: 81.2.69.142\nnetwork: 81.2.69.142/31\nrecord:\n", + }, + { + name: "Single record JSONL", + useJSONL: true, + networks: []string{"81.2.69.142"}, + databases: []string{"GeoIP2-Country-Test.mmdb"}, + //nolint:lll + expected: `{"database_path":"GeoIP2-Country-Test.mmdb","requested_lookup":"81.2.69.142","network":"81.2.69.142/31","record":{"continent":{"code":"EU","geoname_id":6255148,"names":{"de":"Europa","en":"Europe","es":"Europa","fr":"Europe","ja":"ヨーロッパ","pt-BR":"Europa","ru":"Европа","zh-CN":"欧洲"}},"country":{"geoname_id":2635167,"is_in_european_union":true,"iso_code":"GB","names":{"de":"Vereinigtes Königreich","en":"United Kingdom","es":"Reino Unido","fr":"Royaume-Uni","ja":"イギリス","pt-BR":"Reino Unido","ru":"Великобритания","zh-CN":"英国"}},"registered_country":{"geoname_id":6252001,"iso_code":"US","names":{"de":"USA","en":"United States","es":"Estados Unidos","fr":"États-Unis","ja":"アメリカ合衆国","pt-BR":"Estados Unidos","ru":"США","zh-CN":"美国"}}}}`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var buf bytes.Buffer + + err := process( + &buf, + test.useJSONL, + test.networks, + test.databases, + false, + false, + false, + ) + + require.NoError(t, err) + + if test.useJSONL { + assert.JSONEq(t, test.expected, buf.String()) + } else { + assert.Regexp(t, test.expected, buf.String()) + } + }) + } +} diff --git a/cmd/mmdbinspect/records.go b/cmd/mmdbinspect/records.go new file mode 100644 index 0000000..d0c774b --- /dev/null +++ b/cmd/mmdbinspect/records.go @@ -0,0 +1,148 @@ +package main + +import ( + "errors" + "fmt" + "io/fs" + "iter" + "net/netip" + "os" + "path/filepath" + "strings" + "time" + + "github.com/oschwald/maxminddb-golang/v2" +) + +// record holds the records for a lookup in a database. Note that +// the order here affects the output order. +type record struct { + DatabasePath string `json:"database_path"` + BuildTime *time.Time `json:"build_time,omitempty"` + RequestedLookup string `json:"requested_lookup"` + Network netip.Prefix `json:"network"` + Record any `json:"record,omitempty"` +} + +// records returns an iterator over the records for the networks and +// databases provided. +func records( + networks, databases []string, + includeAliasedNetworks, + includeBuildTime, + includeNetworksWithoutData bool, +) iter.Seq2[*record, error] { + var opts []maxminddb.NetworksOption + if includeAliasedNetworks { + opts = append(opts, maxminddb.IncludeAliasedNetworks) + } + if includeNetworksWithoutData { + opts = append(opts, maxminddb.IncludeNetworksWithoutData) + } + + return func(yield func(*record, error) bool) { + for _, glob := range databases { + matches, err := filepath.Glob(glob) + if err != nil { + yield(nil, fmt.Errorf("invalid file path or glob %q: %w", glob, err)) + return + } + for _, path := range matches { + reader, err := openDB(path) + if err != nil { + yield(nil, fmt.Errorf("could not open database %q: %w", path, err)) + return + } + + var buildTime *time.Time + if includeBuildTime { + //nolint:gosec // not a security issue. + t := time.Unix(int64(reader.Metadata.BuildEpoch), 0).UTC() + buildTime = &t + } + + for _, thisNetwork := range networks { + baseRecord := record{ + DatabasePath: path, + BuildTime: buildTime, + RequestedLookup: thisNetwork, + } + ok := recordsForNetwork(reader, opts, baseRecord, yield) + if !ok { + return + } + } + _ = reader.Close() + } + } + } +} + +// recordsForNetwork fetches mmdb records inside a given network. If an IP +// address is provided without a prefix length, it will be treated as a +// network containing a single address (i.e., /32 for IPv4 and /128 for IPv6). +func recordsForNetwork( + reader *maxminddb.Reader, + opts []maxminddb.NetworksOption, + record record, + yield func(*record, error) bool, +) bool { + var err error + var network netip.Prefix + if strings.Contains(record.RequestedLookup, "/") { + network, err = netip.ParsePrefix(record.RequestedLookup) + if err != nil { + yield(nil, fmt.Errorf("%s is not a valid network", record.RequestedLookup)) + return false + } + } else { + addr, err := netip.ParseAddr(record.RequestedLookup) + if err != nil { + yield(nil, fmt.Errorf("%s is not a valid IP address", record.RequestedLookup)) + return false + } + + bits := 32 + if addr.Is6() { + bits = 128 + } + + network = netip.PrefixFrom(addr, bits) + } + + for res := range reader.NetworksWithin(network, opts...) { + record.Network = res.Prefix() + + err := res.Decode(&record.Record) + if err != nil { + yield(nil, fmt.Errorf("could not get next network: %w", err)) + return false + } + + ok := yield(&record, nil) + if !ok { + return false + } + } + + return true +} + +// openDB returns a maxminddb.Reader. +func openDB(path string) (*maxminddb.Reader, error) { + _, err := os.Stat(path) + + if errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("%v does not exist", path) + } + if err != nil { + return nil, fmt.Errorf("stating %s: %w", path, err) + } + + db, err := maxminddb.Open(path) + if err != nil { + return nil, fmt.Errorf("%v could not be opened: %w", path, err) + } + + return db, nil +} diff --git a/cmd/mmdbinspect/records_test.go b/cmd/mmdbinspect/records_test.go new file mode 100644 index 0000000..ce0e6ea --- /dev/null +++ b/cmd/mmdbinspect/records_test.go @@ -0,0 +1,274 @@ +package main + +import ( + "net/netip" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testDataDir = "../../test/data/test-data/" +) + +var ( + CityDBPath = filepath.Join(testDataDir, "GeoIP2-City-Test.mmdb") + CountryDBPath = filepath.Join(testDataDir, "GeoIP2-Country-Test.mmdb") + ISPDBPath = filepath.Join(testDataDir, "GeoIP2-ISP-Test.mmdb") +) + +var city81_2_69_142 = map[string]any{ + "city": map[string]any{ + "geoname_id": uint64(2643743), + "names": map[string]any{ + "de": "London", + "en": "London", + "es": "Londres", + "fr": "Londres", + "ja": "ロンドン", + "pt-BR": "Londres", + "ru": "Лондон", + }, + }, + "continent": map[string]any{ + "code": "EU", + "geoname_id": uint64(6255148), + "names": map[string]any{ + "de": "Europa", + "en": "Europe", + "es": "Europa", + "fr": "Europe", + "ja": "ヨーロッパ", + "pt-BR": "Europa", + "ru": "Европа", + "zh-CN": "欧洲", + }, + }, + "country": map[string]any{ + "geoname_id": uint64(2635167), + "is_in_european_union": true, + "iso_code": "GB", + "names": map[string]any{ + "de": "Vereinigtes Königreich", + "en": "United Kingdom", + "es": "Reino Unido", + "fr": "Royaume-Uni", + "ja": "イギリス", + "pt-BR": "Reino Unido", + "ru": "Великобритания", + "zh-CN": "英国", + }, + }, + "location": map[string]any{ + "accuracy_radius": uint64(10), + "latitude": 51.5142, + "longitude": -0.0931, + "time_zone": "Europe/London", + }, + "registered_country": map[string]any{ + "geoname_id": uint64(6252001), + "iso_code": "US", + "names": map[string]any{ + "de": "USA", + "en": "United States", + "es": "Estados Unidos", + "fr": "États-Unis", + "ja": "アメリカ合衆国", + "pt-BR": "Estados Unidos", + "ru": "США", + "zh-CN": "美国", + }, + }, + "subdivisions": []any{map[string]any{ + "geoname_id": uint64(6269131), + "iso_code": "ENG", + "names": map[string]any{ + "en": "England", + "es": "Inglaterra", + "fr": "Angleterre", + "pt-BR": "Inglaterra", + }, + }}, +} + +var country81_2_69_142 = map[string]any{ + "continent": map[string]any{ + "code": "EU", + "geoname_id": uint64(6255148), + "names": map[string]any{ + "de": "Europa", + "en": "Europe", + "es": "Europa", + "fr": "Europe", + "ja": "ヨーロッパ", + "pt-BR": "Europa", + "ru": "Европа", + "zh-CN": "欧洲", + }, + }, + "country": map[string]any{ + "geoname_id": uint64(2635167), + "is_in_european_union": true, + "iso_code": "GB", + "names": map[string]any{ + "de": "Vereinigtes Königreich", + "en": "United Kingdom", + "es": "Reino Unido", + "fr": "Royaume-Uni", + "ja": "イギリス", + "pt-BR": "Reino Unido", + "ru": "Великобритания", + "zh-CN": "英国", + }, + }, + "registered_country": map[string]any{ + "geoname_id": uint64(6252001), + "iso_code": "US", + "names": map[string]any{ + "de": "USA", + "en": "United States", + "es": "Estados Unidos", + "fr": "États-Unis", + "ja": "アメリカ合衆国", + "pt-BR": "Estados Unidos", + "ru": "США", + "zh-CN": "美国", + }, + }, +} + +func TestRecords(t *testing.T) { + countryBuildTime := time.Date(2019, 11, 4, 16, 30, 59, 0, time.UTC) + + tests := []struct { + name string + dbs []string + networks []string + includeBuildTime bool + includeNetworksWithoutData bool + expectRecords []record + expectErr string + }{ + { + name: "multiple non-glob paths and multiple IPs", + dbs: []string{CityDBPath, CountryDBPath}, + networks: []string{"81.2.69.142", "8.8.8.8"}, + expectRecords: []record{ + { + DatabasePath: CityDBPath, + RequestedLookup: "81.2.69.142", + Network: netip.MustParsePrefix("81.2.69.142/31"), + Record: city81_2_69_142, + }, + { + DatabasePath: CountryDBPath, + RequestedLookup: "81.2.69.142", + Network: netip.MustParsePrefix("81.2.69.142/31"), + Record: country81_2_69_142, + }, + }, + }, + { + name: "with build time", + dbs: []string{CountryDBPath}, + networks: []string{"81.2.69.142"}, + includeBuildTime: true, + expectRecords: []record{ + { + DatabasePath: CountryDBPath, + BuildTime: &countryBuildTime, + RequestedLookup: "81.2.69.142", + Network: netip.MustParsePrefix("81.2.69.142/31"), + Record: country81_2_69_142, + }, + }, + }, + { + name: "glob path", + dbs: []string{filepath.Join(testDataDir, "GeoIP2-C*y-Test.mmdb")}, + networks: []string{"81.2.69.142"}, + expectRecords: []record{ + { + DatabasePath: CityDBPath, + RequestedLookup: "81.2.69.142", + Network: netip.MustParsePrefix("81.2.69.142/31"), + Record: city81_2_69_142, + }, + { + DatabasePath: CountryDBPath, + RequestedLookup: "81.2.69.142", + Network: netip.MustParsePrefix("81.2.69.142/31"), + Record: country81_2_69_142, + }, + }, + }, + { + name: "network missing from DB without including empty networks", + dbs: []string{CityDBPath}, + networks: []string{"10.0.0.0"}, + }, + { + name: "network missing from DB with including empty networks", + dbs: []string{CityDBPath}, + networks: []string{"10.0.0.0"}, + includeNetworksWithoutData: true, + expectRecords: []record{ + { + DatabasePath: CityDBPath, + RequestedLookup: "10.0.0.0", + Network: netip.MustParsePrefix("10.0.0.0/8"), + }, + }, + }, + { + name: "file does not exist", + dbs: []string{"does/not/exist.mmdb"}, + networks: []string{"81.2.69.142"}, + expectErr: "does/not/exist.mmdb does not exist", + }, + { + name: "invalid lookup IP", + dbs: []string{CityDBPath}, + networks: []string{"81.2.69.342"}, + expectErr: "81.2.69.342 is not a valid IP address", + }, + { + name: "invalid lookup network", + dbs: []string{CityDBPath}, + networks: []string{"81.2.69.42/33"}, + expectErr: "81.2.69.42/33 is not a valid network", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var recs []record + iterator := records( + test.networks, + test.dbs, + false, + test.includeBuildTime, + test.includeNetworksWithoutData, + ) + for record, err := range iterator { + // For now, we don't test errors that happen half way through an + // iteration. If we want to in the future, we will need to rework + // this a bit. + if test.expectErr == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, test.expectErr) + } + + if err == nil { + recs = append(recs, *record) + } + } + + assert.Equal(t, test.expectRecords, recs) + }) + } +} diff --git a/go.mod b/go.mod index 4d01514..16fbe4f 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/maxmind/mmdbinspect/v2 go 1.23 require ( + github.com/goccy/go-yaml v1.15.23 github.com/oschwald/maxminddb-golang/v2 v2.0.0-beta.3 github.com/stretchr/testify v1.10.0 ) diff --git a/go.sum b/go.sum index eba9133..1939d7c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/goccy/go-yaml v1.15.23 h1:WS0GAX1uNPDLUvLkNU2vXq6oTnsmfVFocjQ/4qA48qo= +github.com/goccy/go-yaml v1.15.23/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/oschwald/maxminddb-golang/v2 v2.0.0-beta.3 h1:dIygR/m/ArKzJy5HFVlDpBOG0hju03T+567HxrRRAF4= github.com/oschwald/maxminddb-golang/v2 v2.0.0-beta.3/go.mod h1:JojXVel5ck1JzMiz32OVBfSk7lAtfSDUfzxIyVEuahM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/pkg/mmdbinspect/mmdbinspect.go b/pkg/mmdbinspect/mmdbinspect.go deleted file mode 100644 index a191332..0000000 --- a/pkg/mmdbinspect/mmdbinspect.go +++ /dev/null @@ -1,130 +0,0 @@ -// Package mmdbinspect peeks at the contents of .mmdb files -package mmdbinspect - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io/fs" - "net/netip" - "os" - "strings" - - "github.com/oschwald/maxminddb-golang/v2" -) - -// RecordForNetwork holds a network and the corresponding record. -type RecordForNetwork struct { - Network netip.Prefix - Record any -} - -// RecordSet holds the records for a lookup in a database. -type RecordSet struct { - Database string - Records any - Lookup string -} - -// OpenDB returns a maxminddb.Reader. -func OpenDB(path string) (*maxminddb.Reader, error) { - _, err := os.Stat(path) - - if errors.Is(err, fs.ErrNotExist) { - return nil, fmt.Errorf("%v does not exist", path) - } - if err != nil { - return nil, fmt.Errorf("stating %s: %w", path, err) - } - - db, err := maxminddb.Open(path) - if err != nil { - return nil, fmt.Errorf("%v could not be opened: %w", path, err) - } - - return db, nil -} - -// RecordsForNetwork fetches mmdb records inside a given network. If an IP -// address is provided without a prefix length, it will be treated as a -// network containing a single address (i.e., /32 for IPv4 and /128 for IPv6). -func RecordsForNetwork(reader maxminddb.Reader, includeAliasedNetworks bool, maybeNetwork string) (any, error) { - lookupNetwork := maybeNetwork - - if !strings.Contains(lookupNetwork, "/") { - if strings.Count(maybeNetwork, ":") >= 2 { - lookupNetwork = maybeNetwork + "/128" - } else { - lookupNetwork = maybeNetwork + "/32" - } - } - - network, err := netip.ParsePrefix(lookupNetwork) - if err != nil { - return nil, fmt.Errorf("%v is not a valid IP address", maybeNetwork) - } - - var opts []maxminddb.NetworksOption - if includeAliasedNetworks { - opts = append(opts, maxminddb.IncludeAliasedNetworks) - } - - var found []any - - for res := range reader.NetworksWithin(network, opts...) { - var record any - - err := res.Decode(&record) - if err != nil { - return nil, fmt.Errorf("could not get next network: %w", err) - } - - found = append(found, RecordForNetwork{res.Prefix(), record}) - } - - return found, nil -} - -// AggregatedRecords returns the aggregated records for the networks and -// databases provided. -func AggregatedRecords(networks, databases []string, includeAliasedNetworks bool) (any, error) { - var recordSets []RecordSet - - for _, path := range databases { - reader, err := OpenDB(path) - if err != nil { - return nil, fmt.Errorf("could not open database %v: %w", path, err) - } - - for _, thisNetwork := range networks { - var records any - records, err = RecordsForNetwork(*reader, includeAliasedNetworks, thisNetwork) - if err != nil { - _ = reader.Close() - return nil, fmt.Errorf("could not get records from db %v: %w", path, err) - } - - set := RecordSet{path, records, thisNetwork} - recordSets = append(recordSets, set) - } - _ = reader.Close() - } - - return recordSets, nil -} - -// RecordToString converts an mmdb record into a JSON-formatted string. -func RecordToString(record any) (string, error) { - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - enc.SetEscapeHTML(false) // don't escape ampersands and angle brackets - enc.SetIndent("", " ") - - err := enc.Encode(record) - if err != nil { - return "", errors.New("could not convert record to string") - } - - return buf.String(), nil -} diff --git a/pkg/mmdbinspect/mmdbinspect_test.go b/pkg/mmdbinspect/mmdbinspect_test.go deleted file mode 100644 index f5a1ee6..0000000 --- a/pkg/mmdbinspect/mmdbinspect_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package mmdbinspect - -import ( - "testing" - - "github.com/oschwald/maxminddb-golang/v2" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const ( - CityDBPath = "../../test/data/test-data/GeoIP2-City-Test.mmdb" - CountryDBPath = "../../test/data/test-data/GeoIP2-Country-Test.mmdb" - ISPDBPath = "../../test/data/test-data/GeoIP2-ISP-Test.mmdb" -) - -func TestOpenDB(t *testing.T) { - a := assert.New(t) - - a.FileExists(CityDBPath, "database exists") - - reader, err := OpenDB(CityDBPath) - require.NoError(t, err, "no open error") - a.IsType(maxminddb.Reader{}, *reader) - - reader, err = OpenDB("foo/bar/baz") - require.Error(t, err, "open error when file does not exist") - a.Nil(reader) - a.Equal( - "foo/bar/baz does not exist", - err.Error(), - ) - - reader, err = OpenDB("../../test/data/test-data/README.md") - require.Error(t, err) - a.Contains(err.Error(), "README.md could not be opened: error opening database: invalid MaxMind DB file") - a.Nil(reader) - - reader, err = OpenDB("../../test/data/test-data/GeoIP2-City-Test-Invalid-Node-Count.mmdb") - require.Error(t, err) - a.Contains(err.Error(), "invalid metadata") - a.Nil(reader) - - if reader != nil { - require.NoError(t, reader.Close()) - } -} - -func TestRecordsForNetwork(t *testing.T) { - a := assert.New(t) - reader, err := OpenDB(CityDBPath) // ipv6 database - require.NoError(t, err, "no open error") - - records, err := RecordsForNetwork(*reader, false, "81.2.69.142") - require.NoError(t, err, "no error on lookup of 81.2.69.142") - a.NotNil(records, "records returned") - - records, err = RecordsForNetwork(*reader, false, "81.2.69.0/24") - require.NoError(t, err, "no error on lookup of 81.2.69.0/24") - a.NotNil(records, "records returned") - - records, err = RecordsForNetwork(*reader, false, "10.255.255.255/29") - require.NoError(t, err, "got no error when IP not found") - a.Nil(records, "no records returned for 10.255.255.255/29") - - records, err = RecordsForNetwork(*reader, false, "X.X.Y.Z") - require.Error(t, err, "got an error") - a.Nil(records, "no records returned for X.X.Y.Z") - a.Equal("X.X.Y.Z is not a valid IP address", err.Error()) - - require.NoError(t, reader.Close()) -} - -func TestRecordToString(t *testing.T) { - a := assert.New(t) - - reader, err := OpenDB(CityDBPath) - require.NoError(t, err, "no open error") - records, err := RecordsForNetwork(*reader, false, "81.2.69.142") - require.NoError(t, err, "no RecordsForNetwork error") - prettyJSON, err := RecordToString(records) - - require.NoError(t, err, "no error on stringification") - a.NotNil(prettyJSON, "records stringified") - a.Contains(prettyJSON, "London") - a.Contains(prettyJSON, "2643743") - - require.NoError(t, reader.Close()) -} - -// TestRecordToStringEscaping tests that certain HTML-related characters are not -// escaped in the JSON output. -func TestRecordToStringEscaping(t *testing.T) { - a := assert.New(t) - - reader, err := OpenDB(ISPDBPath) - require.NoError(t, err, "no open error") - records, err := RecordsForNetwork(*reader, false, "206.16.137.0/24") - require.NoError(t, err, "no RecordsForNetwork error") - prettyJSON, err := RecordToString(records) - - require.NoError(t, err, "no error on stringification") - a.NotNil(prettyJSON, "records stringified") - a.Contains(prettyJSON, "AT&T Synaptic Cloud Hosting") - - require.NoError(t, reader.Close()) -} - -func TestAggregatedRecords(t *testing.T) { - a := assert.New(t) - - dbs := []string{CityDBPath, CountryDBPath} - networks := []string{"81.2.69.142", "8.8.8.8"} - results, err := AggregatedRecords(networks, dbs, false) - - require.NoError(t, err) - a.NotNil(results) -}