Rename and comment

Author: dennis-tra

cmd/nebula/cmd_resolve.go

@@ -18,7 +18,7 @@ import (
 // ResolveCommand contains the monitor sub-command configuration.
 var ResolveCommand = &cli.Command{
 	Name:   "resolve",
-	Usage:  "Resolves all **unresolved** multi addresses to their IP addresses and geo location information",
+	Usage:  "Resolves all multi addresses to their IP addresses and geo location information",
 	Action: ResolveAction,
 	Flags: []cli.Flag{
 		&cli.IntFlag{
@@ -28,13 +28,6 @@ var ResolveCommand = &cli.Command{
 			DefaultText: "100",
 			Value:       100,
 		},
-		&cli.BoolFlag{
-			Name:        "full",
-			Usage:       "If set nebula starts resolving all multi addresses in the database",
-			EnvVars:     []string{"NEBULA_RESOLVE_FULL"},
-			DefaultText: "false",
-			Value:       false,
-		},
 	},
 }
 
@@ -79,11 +72,7 @@ func ResolveAction(c *cli.Context) error {
 
 		var err error
 		var dbmaddrs models.MultiAddressSlice
-		if c.Bool("full") {
-			dbmaddrs, err = dbc.FetchMultiAddresses(c.Context, offset, limit)
-		} else {
-			dbmaddrs, err = dbc.FetchUnresolvedMultiAddresses(c.Context, offset, limit)
-		}
+		dbmaddrs, err = dbc.FetchMultiAddresses(c.Context, offset, limit)
 		if err != nil {
 			return errors.Wrap(err, "fetching multi addresses")
 		}

pkg/crawl/db.go

@@ -11,7 +11,7 @@ import (
 	"github.com/dennis-tra/nebula-crawler/pkg/models"
 )
 
-// updateCrawl writes crawl statistics to the database. TODO: comment
+// updateCrawl writes crawl statistics to the database
 func (s *Scheduler) updateCrawl(ctx context.Context, success bool) error {
 	log.Infoln("Persisting crawl result...")
 
@@ -49,7 +49,7 @@ func (s *Scheduler) persistCrawlProperties(ctx context.Context) error {
 	}
 	pps := map[string]map[string]int{
 		"agent_version":      avFull,
-		"agent_version_core": avCore,
+		"agent_version_core": avCore, // TODO: Not used currently
 		"protocol":           s.protocols,
 		"error":              s.errors,
 	}

pkg/crawl/persister.go

@@ -19,8 +19,6 @@ import (
 var persisterID = atomic.NewInt32(0)
 
 // Persister handles the insert/upsert/update operations for a particular crawl result.
-// We're doing it asynchronously as each insert can take multiple tens of milliseconds.
-// This would take too long to do synchronously during a crawl.
 type Persister struct {
 	*service.Service
 

pkg/crawl/utils.go

@@ -1,8 +1,6 @@
 package crawl
 
 import (
-	"time"
-
 	"github.com/libp2p/go-libp2p-core/peer"
 	ma "github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr/net"
@@ -17,18 +15,7 @@ func (s *Scheduler) TotalErrors() int {
 	return sum
 }
 
-func addrsToMaddrs(addrs []string) ([]ma.Multiaddr, error) {
-	maddrs := make([]ma.Multiaddr, len(addrs))
-	for i, addr := range addrs {
-		maddr, err := ma.NewMultiaddr(addr)
-		if err != nil {
-			return nil, err
-		}
-		maddrs[i] = maddr
-	}
-	return maddrs, nil
-}
-
+// maddrsToAddrs maps a slice of multi addresses to their string representation.
 func maddrsToAddrs(maddrs []ma.Multiaddr) []string {
 	addrs := make([]string, len(maddrs))
 	for i, maddr := range maddrs {
@@ -37,11 +24,6 @@ func maddrsToAddrs(maddrs []ma.Multiaddr) []string {
 	return addrs
 }
 
-// millisSince returns the number of milliseconds between now and the given time.
-func millisSince(start time.Time) float64 {
-	return float64(time.Since(start)) / float64(time.Millisecond)
-}
-
 // filterPrivateMaddrs strips private multiaddrs from the given peer address information.
 func filterPrivateMaddrs(pi peer.AddrInfo) peer.AddrInfo {
 	filtered := peer.AddrInfo{

pkg/db/client.go

@@ -3,12 +3,12 @@ package db
 import (
 	"context"
 	"database/sql"
-	_ "embed"
 	"fmt"
 	"sync"
 	"time"
 
 	"contrib.go.opencensus.io/integrations/ocsql"
+	_ "github.com/lib/pq"
 	"github.com/libp2p/go-libp2p-core/peer"
 	ma "github.com/multiformats/go-multiaddr"
 	"github.com/pkg/errors"

pkg/db/db.go

@@ -32,18 +32,9 @@ func NewClient() (*Client, error) {
 	}, nil
 }
 
-func (c *Client) AddrCountry(addr string) (string, error) {
-	ip := net.ParseIP(addr)
-	if ip == nil {
-		return "", fmt.Errorf("invalid address %s", addr)
-	}
-	record, err := c.reader.City(ip)
-	if err != nil {
-		return "", err
-	}
-	return record.Country.IsoCode, nil
-}
-
+// MaddrCountry resolve the give multi address to its corresponding
+// IP addresses (it could be multiple due to protocols like dnsaddr)
+// and returns a map of the form IP-address -> country ISO code.
 func (c *Client) MaddrCountry(ctx context.Context, maddr ma.Multiaddr) (map[string]string, error) {
 	resolved := resolveAddrs(ctx, maddr)
 	if len(resolved) == 0 {
@@ -61,6 +52,19 @@ func (c *Client) MaddrCountry(ctx context.Context, maddr ma.Multiaddr) (map[stri
 	return countries, nil
 }
 
+// AddrCountry takes an IP address string and tries to derive the country ISO code.
+func (c *Client) AddrCountry(addr string) (string, error) {
+	ip := net.ParseIP(addr)
+	if ip == nil {
+		return "", fmt.Errorf("invalid address %s", addr)
+	}
+	record, err := c.reader.Country(ip)
+	if err != nil {
+		return "", err
+	}
+	return record.Country.IsoCode, nil
+}
+
 func (c *Client) Close() error {
 	return c.reader.Close()
 }

pkg/db/db_test.go

@@ -44,17 +44,18 @@ func TestClient_MaddrCountry(t *testing.T) {
 	require.NoError(t, err)
 
 	tests := []struct {
-		addr    string
-		want    string
-		wantErr bool
+		addr        string
+		wantAddr    string
+		wantCountry string
+		wantErr     bool
 	}{
-		{addr: "/ip4/46.17.96.99/tcp/6666/p2p/Qme8g49gm3q4Acp7xWBKg3nAa9fxZ1YmyDJdyGgoG6LsXh/p2p-circuit", want: "NL", wantErr: false},
-		{addr: "/p2p-circuit/p2p/QmPG5bax9kfpQUVDrzfahmh44Ab6egDeZ2QDWeTY279HLJ", want: "", wantErr: true},
-		{addr: "/dnsaddr/bootstrap.libp2p.io", want: "", wantErr: true},
+		{addr: "/ip4/46.17.96.99/tcp/6666/p2p/Qme8g49gm3q4Acp7xWBKg3nAa9fxZ1YmyDJdyGgoG6LsXh/p2p-circuit", wantAddr: "46.17.96.99", wantCountry: "NL", wantErr: false},
+		{addr: "/p2p-circuit/p2p/QmPG5bax9kfpQUVDrzfahmh44Ab6egDeZ2QDWeTY279HLJ", wantAddr: "", wantCountry: "", wantErr: true},
+		{addr: "/dnsaddr/bootstrap.libp2p.io", wantAddr: "147.75.109.29", wantCountry: "US", wantErr: false},
 		//{addr: "/dns4/k8s-dev-ipfsp2pt-c0b76d02d7-969229bd37f82282.elb.ca-central-1.amazonaws.com/tcp/4001", want: "", wantErr: true},
 	}
 	for _, tt := range tests {
-		t.Run(fmt.Sprintf("%s | iso: %s | err: %v", tt.addr, tt.want, tt.wantErr), func(t *testing.T) {
+		t.Run(fmt.Sprintf("%s | iso: %s | err: %v", tt.addr, tt.wantCountry, tt.wantErr), func(t *testing.T) {
 			maddr, err := ma.NewMultiaddr(tt.addr)
 			require.NoError(t, err)
 
@@ -63,8 +64,8 @@ func TestClient_MaddrCountry(t *testing.T) {
 				assert.Error(t, err)
 			} else {
 				assert.NoError(t, err)
+				assert.Equal(t, tt.wantCountry, got[tt.wantAddr])
 			}
-			assert.Equal(t, tt.want, got)
 		})
 	}
 }