Add a flag to include networks without data

Author: oschwald

CHANGELOG.md

@@ -16,6 +16,8 @@
   matches, it will be treated as if multiple `-db` arguments were provided.
   Note that you must quote the parameter when using globs to prevent the
   shell's globbing from interfering. See the [pattern syntax](https://pkg.go.dev/path#Match)
+* You may now include networks with no data in the output by using the
+  `include-networks-without-data` flag.
 * This repo no longer provides a public Go API. It is only intended to be
   used as a CLI program.
 

cmd/mmdbinspect/main.go

@@ -28,12 +28,19 @@ func main() {
 	var mmdb arrayFlags
 
 	flag.Var(&mmdb, "db", "Path to an mmdb file. You may pass this arg more than once.")
+
 	includeAliasedNetworks := flag.Bool(
 		"include-aliased-networks",
 		false,
 		"Include aliased networks (e.g. 6to4, Teredo). This option may cause IPv4 networks to be listed more than once via aliases.", //nolint: lll
 	)
 
+	includeNetworksWithoutData := flag.Bool(
+		"include-networks-without-data",
+		false,
+		`Include networks that have no data in the database. The "record" will be null for these.`,
+	)
+
 	useJSONL := flag.Bool("jsonl", false, "Output as JSONL instead of YAML.")
 
 	flag.Usage = usage
@@ -65,7 +72,12 @@ func main() {
 		encoder = yaml.NewEncoder(w)
 	}
 
-	iterator := records(network, mmdb, *includeAliasedNetworks)
+	iterator := records(
+		network,
+		mmdb,
+		*includeAliasedNetworks,
+		*includeNetworksWithoutData,
+	)
 
 	for r, err := range iterator {
 		if err != nil {

cmd/mmdbinspect/mmdbinspect.go

@@ -18,15 +18,24 @@ type record struct {
 	DatabasePath    string       `json:"database_path"`
 	RequestedLookup string       `json:"requested_lookup"`
 	Network         netip.Prefix `json:"network"`
-	Record          any          `json:"record"`
+	Record          any          `json:"record,omitempty"`
 }
 
 // records returns an iterator over the records for the networks and
 // databases provided.
 func records(
 	networks, databases []string,
-	includeAliasedNetworks bool,
+	includeAliasedNetworks,
+	includeNetworksWithoutData bool,
 ) iter.Seq2[*record, error] {
+	var opts []maxminddb.NetworksOption
+	if includeAliasedNetworks {
+		opts = append(opts, maxminddb.IncludeAliasedNetworks)
+	}
+	if includeNetworksWithoutData {
+		opts = append(opts, maxminddb.IncludeNetworksWithoutData)
+	}
+
 	return func(yield func(*record, error) bool) {
 		for _, glob := range databases {
 			matches, err := filepath.Glob(glob)
@@ -46,7 +55,7 @@ func records(
 						DatabasePath:    path,
 						RequestedLookup: thisNetwork,
 					}
-					ok := recordsForNetwork(reader, includeAliasedNetworks, baseRecord, yield)
+					ok := recordsForNetwork(reader, opts, baseRecord, yield)
 					if !ok {
 						return
 					}
@@ -62,7 +71,7 @@ func records(
 // network containing a single address (i.e., /32 for IPv4 and /128 for IPv6).
 func recordsForNetwork(
 	reader *maxminddb.Reader,
-	includeAliasedNetworks bool,
+	opts []maxminddb.NetworksOption,
 	record record,
 	yield func(*record, error) bool,
 ) bool {
@@ -89,11 +98,6 @@ func recordsForNetwork(
 		network = netip.PrefixFrom(addr, bits)
 	}
 
-	var opts []maxminddb.NetworksOption
-	if includeAliasedNetworks {
-		opts = append(opts, maxminddb.IncludeAliasedNetworks)
-	}
-
 	for res := range reader.NetworksWithin(network, opts...) {
 		record.Network = res.Prefix()
 

cmd/mmdbinspect/mmdbinspect_test.go

@@ -141,11 +141,12 @@ var country81_2_69_142 = map[string]any{
 
 func TestRecords(t *testing.T) {
 	tests := []struct {
-		name          string
-		dbs           []string
-		networks      []string
-		expectRecords []record
-		expectErr     string
+		name                       string
+		dbs                        []string
+		networks                   []string
+		includeNetworksWithoutData bool
+		expectRecords              []record
+		expectErr                  string
 	}{
 		{
 			name:     "multiple non-glob paths and multiple IPs",
@@ -186,10 +187,23 @@ func TestRecords(t *testing.T) {
 			},
 		},
 		{
-			name:     "network missing from DB",
+			name:     "network missing from DB without including empty networks",
 			dbs:      []string{CityDBPath},
 			networks: []string{"10.0.0.0"},
 		},
+		{
+			name:                       "network missing from DB with including empty networks",
+			dbs:                        []string{CityDBPath},
+			networks:                   []string{"10.0.0.0"},
+			includeNetworksWithoutData: true,
+			expectRecords: []record{
+				{
+					DatabasePath:    CityDBPath,
+					RequestedLookup: "10.0.0.0",
+					Network:         netip.MustParsePrefix("10.0.0.0/8"),
+				},
+			},
+		},
 		{
 			name:      "file does not exist",
 			dbs:       []string{"does/not/exist.mmdb"},
@@ -213,7 +227,7 @@ func TestRecords(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			var recs []record
-			for record, err := range records(test.networks, test.dbs, false) {
+			for record, err := range records(test.networks, test.dbs, false, test.includeNetworksWithoutData) {
 				// For now, we don't test errors that happen half way through an
 				// iteration. If we want to in the future, we will need to rework
 				// this a bit.