feat: add multistream http download

transport/httptransport/http_transport.go

@@ -5,7 +5,9 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"strconv"
 	"sync"
@@ -119,6 +121,16 @@ func (h *httpTransport) Execute(ctx context.Context, transportInfo []byte, dealI
 	}
 	tInfo.URL = u.Url
 
+	// don't allow private network addresses as per https://en.wikipedia.org/wiki/Private_network
+	pu, err := url.Parse(u.Url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse request url: %w", err)
+	}
+	ip := net.ParseIP(pu.Hostname())
+	if ip != nil && ip.IsPrivate() {
+		return nil, fmt.Errorf("downloading from private ip addresses is not allowed")
+	}
+
 	// check that the outputFile exists
 	fi, err := os.Stat(dealInfo.OutputFile)
 	if err != nil {
@@ -188,7 +200,13 @@ func (h *httpTransport) Execute(ctx context.Context, transportInfo []byte, dealI
 			h.libp2pHost.ConnManager().Unprotect(u.PeerID, tag)
 		})
 	} else {
-		t.client = http.DefaultClient
+		// do not follow http redirects for security reasons
+		t.client = &http.Client{
+			CheckRedirect: func(req *http.Request, via []*http.Request) error {
+				return http.ErrUseLastResponse
+			},
+		}
+
 		h.dl.Infow(duuid, "http url", "url", tInfo.URL)
 	}
 

transport/httptransport/http_transport_test.go

@@ -316,6 +316,69 @@ func TestConcurrentTransfers(t *testing.T) {
 	}
 }
 
+func TestDontAllowDownloadingFromPrivateAddresses(t *testing.T) {
+	ctx := context.Background()
+	of := getTempFilePath(t)
+	// deal info is irrelevant in this test
+	dealInfo := &types.TransportDealInfo{
+		OutputFile: of,
+		DealSize:   1000,
+	}
+	ht := New(nil, newDealLogger(t, ctx), NChunksOpt(nChunks))
+	bz, err := json.Marshal(types.HttpRequest{URL: "http://192.168.0.1/blah"})
+	require.NoError(t, err)
+
+	_, err = ht.Execute(ctx, bz, dealInfo)
+	require.Error(t, err, "downloading from private addresses is not allowed")
+}
+
+func TestDontFollowHttpRedirects(t *testing.T) {
+	// we should not follow http redirects for security reasons. If the target URL tries to redirect, the client should return 303 response instead.
+	// This test sets up two servers, with one redirecting to the other. Without the redirect check the download would have been completed successfully.
+	rawSize := (100 * readBufferSize) + 30
+	ctx := context.Background()
+	st := newServerTest(t, rawSize)
+	carSize := len(st.carBytes)
+
+	var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
+		switch r.Method {
+		case http.MethodGet:
+			offset := r.Header.Get("Range")
+
+			startend := strings.Split(strings.TrimPrefix(offset, "bytes="), "-")
+			start, _ := strconv.ParseInt(startend[0], 10, 64)
+			end, _ := strconv.ParseInt(startend[1], 10, 64)
+
+			w.WriteHeader(200)
+			if end == 0 {
+				w.Write(st.carBytes[start:]) //nolint:errcheck
+			} else {
+				w.Write(st.carBytes[start:end]) //nolint:errcheck
+			}
+		case http.MethodHead:
+			addContentLengthHeader(w, len(st.carBytes))
+		}
+	}
+	svr := httptest.NewServer(handler)
+	defer svr.Close()
+
+	var redirectHandler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, svr.URL, http.StatusSeeOther)
+	}
+	redirectSvr := httptest.NewServer(redirectHandler)
+	defer redirectSvr.Close()
+
+	of := getTempFilePath(t)
+	th := executeTransfer(t, ctx, New(nil, newDealLogger(t, ctx), NChunksOpt(nChunks)), carSize, types.HttpRequest{URL: redirectSvr.URL}, of)
+	require.NotNil(t, th)
+
+	evts := waitForTransferComplete(th)
+	require.NotEmpty(t, evts)
+	require.Equal(t, 1, len(evts))
+	require.EqualValues(t, 0, evts[0].NBytesReceived)
+	require.Contains(t, evts[0].Error.Error(), "303 See Other")
+}
+
 func TestCompletionOnMultipleAttemptsWithSameFile(t *testing.T) {
 	ctx := context.Background()
 	of := getTempFilePath(t)