Add HTTP-retrieval check to ipfs-check

Author: hsanjuan

daemon.go

@@ -8,6 +8,9 @@ import (
 	"time"
 
 	vole "github.com/ipfs-shipyard/vole/lib"
+	bsmsg "github.com/ipfs/boxo/bitswap/message"
+	"github.com/ipfs/boxo/bitswap/message/pb"
+	"github.com/ipfs/boxo/bitswap/network/httpnet"
 	"github.com/ipfs/boxo/ipns"
 	"github.com/ipfs/boxo/routing/http/client"
 	"github.com/ipfs/boxo/routing/http/contentrouter"
@@ -19,6 +22,7 @@ import (
 	record "github.com/libp2p/go-libp2p-record"
 	"github.com/libp2p/go-libp2p/core/host"
 	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/libp2p/go-libp2p/core/peerstore"
 	"github.com/libp2p/go-libp2p/core/routing"
 	"github.com/libp2p/go-libp2p/p2p/net/connmgr"
 	"github.com/multiformats/go-multiaddr"
@@ -140,13 +144,14 @@ type providerOutput struct {
 	Addrs                    []string
 	ConnectionMaddrs         []string
 	DataAvailableOverBitswap BitswapCheckOutput
+	DataAvailableOverHTTP    HTTPCheckOutput
 	Source                   string
 }
 
 // runCidCheck finds providers of a given CID, using the DHT and IPNI
 // concurrently. A check of connectivity and Bitswap availability is performed
 // for each provider found.
-func (d *daemon) runCidCheck(ctx context.Context, cidKey cid.Cid, ipniURL string) (cidCheckOutput, error) {
+func (d *daemon) runCidCheck(ctx context.Context, cidKey cid.Cid, ipniURL string, httpRetrieval bool) (cidCheckOutput, error) {
 	crClient, err := client.New(ipniURL,
 		client.WithStreamResultsRequired(),               // // https://specs.ipfs.tech/routing/http-routing-v1/#streaming
 		client.WithProtocolFilter(defaultProtocolFilter), // IPIP-484
@@ -236,6 +241,7 @@ func (d *daemon) runCidCheck(ctx context.Context, cidKey cid.Cid, ipniURL string
 				ID:                       provider.ID.String(),
 				Addrs:                    outputAddrs,
 				DataAvailableOverBitswap: BitswapCheckOutput{},
+				DataAvailableOverHTTP:    HTTPCheckOutput{},
 				Source:                   src,
 			}
 
@@ -261,6 +267,10 @@ func (d *daemon) runCidCheck(ctx context.Context, cidKey cid.Cid, ipniURL string
 				p2pAddr, _ := multiaddr.NewMultiaddr("/p2p/" + provider.ID.String())
 				provOutput.DataAvailableOverBitswap = checkBitswapCID(ctx, testHost, cidKey, p2pAddr)
 
+				if httpRetrieval {
+					provOutput.DataAvailableOverHTTP = checkHTTPRetrieval(ctx, testHost, cidKey, p2pAddr)
+				}
+
 				for _, c := range testHost.Network().ConnsToPeer(provider.ID) {
 					provOutput.ConnectionMaddrs = append(provOutput.ConnectionMaddrs, c.RemoteMultiaddr().String())
 				}
@@ -286,10 +296,11 @@ type peerCheckOutput struct {
 	ProviderRecordFromPeerInIPNI bool
 	ConnectionMaddrs             []string
 	DataAvailableOverBitswap     BitswapCheckOutput
+	DataAvailableOverHTTP        HTTPCheckOutput
 }
 
 // runPeerCheck checks the connectivity and Bitswap availability of a CID from a given peer (either with just peer ID or specific multiaddr)
-func (d *daemon) runPeerCheck(ctx context.Context, ma multiaddr.Multiaddr, ai *peer.AddrInfo, c cid.Cid, ipniURL string) (*peerCheckOutput, error) {
+func (d *daemon) runPeerCheck(ctx context.Context, ma multiaddr.Multiaddr, ai *peer.AddrInfo, c cid.Cid, ipniURL string, httpRetrieval bool) (*peerCheckOutput, error) {
 	addrMap, peerAddrDHTErr := peerAddrsInDHT(ctx, d.dht, d.dhtMessenger, ai.ID)
 
 	var inDHT, inIPNI bool
@@ -353,6 +364,10 @@ func (d *daemon) runPeerCheck(ctx context.Context, ma multiaddr.Multiaddr, ai *p
 	// If so is the data available over Bitswap?
 	out.DataAvailableOverBitswap = checkBitswapCID(ctx, testHost, c, ma)
 
+	if httpRetrieval {
+		out.DataAvailableOverHTTP = checkHTTPRetrieval(ctx, testHost, c, ma)
+	}
+
 	// Get all connection maddrs to the peer (in case we hole punched, there will usually be two: limited relay and direct)
 	for _, c := range testHost.Network().ConnsToPeer(ai.ID) {
 		out.ConnectionMaddrs = append(out.ConnectionMaddrs, c.RemoteMultiaddr().String())
@@ -389,6 +404,111 @@ func checkBitswapCID(ctx context.Context, host host.Host, c cid.Cid, ma multiadd
 	return out
 }
 
+type HTTPCheckOutput struct {
+	Duration  time.Duration
+	Connected bool
+	Requested bool
+	HasCID    bool
+	Error     string
+}
+
+type httpReceiver struct {
+	msgCh   chan bsmsg.BitSwapMessage
+	errorCh chan error
+}
+
+func (recv *httpReceiver) ReceiveMessage(ctx context.Context, sender peer.ID, incoming bsmsg.BitSwapMessage) {
+	recv.msgCh <- incoming
+}
+
+func (recv *httpReceiver) ReceiveError(err error) {
+	recv.errorCh <- err
+}
+
+func (recv *httpReceiver) PeerConnected(p peer.ID) { // nop
+}
+
+func (recv *httpReceiver) PeerDisconnected(p peer.ID) { // nop
+}
+
+// FIXME: could expose this directly in Boxo.
+func supportsHEAD(pstore peerstore.Peerstore, p peer.ID) bool {
+	v, err := pstore.Get(p, "http-retrieval-head-support")
+	if err != nil {
+		return false
+	}
+
+	b, ok := v.(bool)
+	return ok && b
+}
+
+func checkHTTPRetrieval(ctx context.Context, host host.Host, c cid.Cid, ma multiaddr.Multiaddr) HTTPCheckOutput {
+	infos, err := peer.AddrInfosFromP2pAddrs(ma)
+	if err != nil {
+		return HTTPCheckOutput{
+			Error: err.Error(),
+		}
+	}
+
+	htnet := httpnet.New(host,
+		httpnet.WithUserAgent(userAgent),
+		httpnet.WithResponseHeaderTimeout(5*time.Second), // default: 10
+		httpnet.WithHTTPWorkers(1),
+	)
+	defer htnet.Stop()
+
+	recv := httpReceiver{
+		msgCh:   make(chan bsmsg.BitSwapMessage),
+		errorCh: make(chan error),
+	}
+	htnet.Start(&recv)
+
+	pid := infos[0].ID
+	err = htnet.Connect(ctx, infos[0])
+	if err != nil {
+		return HTTPCheckOutput{
+			Error: err.Error(),
+		}
+	}
+	if !supportsHEAD(host.Peerstore(), pid) {
+		return HTTPCheckOutput{
+			Error: "HTTP endpoint does not support HEAD requests",
+		}
+	}
+
+	// Now we are in a position of sending a HEAD request.
+	msg := bsmsg.New(true)
+	msg.AddEntry(c, 0, pb.Message_Wantlist_Have, true)
+	start := time.Now()
+	err = htnet.SendMessage(ctx, pid, msg)
+	if err != nil {
+		return HTTPCheckOutput{
+			Connected: true,
+			Error:     err.Error(),
+		}
+	}
+
+	resp := HTTPCheckOutput{
+		Connected: true,
+		Requested: true,
+	}
+	waitCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+	select {
+	case <-waitCtx.Done():
+	case msg := <-recv.msgCh:
+		if len(msg.Haves()) > 0 {
+			resp.HasCID = true
+		}
+
+	case err = <-recv.errorCh:
+		resp.Error = err.Error()
+	}
+
+	resp.Duration = time.Since(start)
+	return resp
+}
+
 func peerAddrsInDHT(ctx context.Context, d kademlia, messenger *dhtpb.ProtocolMessenger, p peer.ID) (map[string]int, error) {
 	closestPeers, err := d.GetClosestPeers(ctx, string(p))
 	if err != nil {

dht.go

@@ -9,8 +9,6 @@ import (
 	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/libp2p/go-libp2p/core/protocol"
-
-	//lint:ignore SA1019 TODO migrate away from gogo pb
 	"github.com/libp2p/go-msgio/pbio"
 )
 

main.go

@@ -98,10 +98,12 @@ func startServer(ctx context.Context, d *daemon, tcpListener, metricsUsername, m
 	checkHandler := func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Add("Access-Control-Allow-Origin", "*")
 
-		maStr := r.URL.Query().Get("multiaddr")
-		cidStr := r.URL.Query().Get("cid")
-		timeoutStr := r.URL.Query().Get("timeoutSeconds")
-		ipniURL := r.URL.Query().Get("ipniIndexer")
+		q := r.URL.Query()
+		maStr := q.Get("multiaddr")
+		cidStr := q.Get("cid")
+		timeoutStr := q.Get("timeoutSeconds")
+		ipniURL := q.Get("ipniIndexer")
+		httpRetrieval := q.Get("httpRetrieval") != ""
 
 		if cidStr == "" {
 			http.Error(w, "missing 'cid' query parameter", http.StatusBadRequest)
@@ -139,14 +141,14 @@ func startServer(ctx context.Context, d *daemon, tcpListener, metricsUsername, m
 
 		var data interface{}
 		if maStr == "" {
-			data, err = d.runCidCheck(withTimeout, cidKey, ipniURL)
+			data, err = d.runCidCheck(withTimeout, cidKey, ipniURL, httpRetrieval)
 		} else {
 			ma, ai, err400 := parseMultiaddr(maStr)
 			if err400 != nil {
 				http.Error(w, err400.Error(), http.StatusBadRequest)
 				return
 			}
-			data, err = d.runPeerCheck(withTimeout, ma, ai, cidKey, ipniURL)
+			data, err = d.runPeerCheck(withTimeout, ma, ai, cidKey, ipniURL, httpRetrieval)
 		}
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)