Skip to content

Commit 06a923e

Browse files
jmozahacud
authored andcommitted
api, chunk: progress bar support (ethersphere#1649)
* api, chunk: HTTP API support to retrieve Tags
1 parent 5e982b6 commit 06a923e

File tree

13 files changed

+313
-83
lines changed

13 files changed

+313
-83
lines changed

api/client/client.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ func (c *Client) UploadRaw(r io.Reader, size int64, toEncrypt bool, toPin bool)
7676
return "", err
7777
}
7878
req.ContentLength = size
79-
req.Header.Set(swarmhttp.SwarmTagHeaderName, fmt.Sprintf("raw_upload_%d", time.Now().Unix()))
79+
req.Header.Set(swarmhttp.TagHeaderName, fmt.Sprintf("raw_upload_%d", time.Now().Unix()))
8080

8181
// Set the pinning header if the file needs to be pinned
8282
if toPin {
@@ -544,7 +544,7 @@ func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, t
544544
}
545545
log.Trace("setting upload tag", "tag", tag)
546546

547-
req.Header.Set(swarmhttp.SwarmTagHeaderName, tag)
547+
req.Header.Set(swarmhttp.TagHeaderName, tag)
548548

549549
// Set the pinning header if the file is to be pinned
550550
if toPin {
@@ -616,7 +616,7 @@ func (c *Client) MultipartUpload(hash string, uploader Uploader, toPin bool) (st
616616

617617
mw := multipart.NewWriter(reqW)
618618
req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary()))
619-
req.Header.Set(swarmhttp.SwarmTagHeaderName, fmt.Sprintf("multipart_upload_%d", time.Now().Unix()))
619+
req.Header.Set(swarmhttp.TagHeaderName, fmt.Sprintf("multipart_upload_%d", time.Now().Unix()))
620620
if toPin {
621621
req.Header.Set(swarmhttp.PinHeaderName, "true")
622622
}

api/http/middleware.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ func InitLoggingResponseWriter(h http.Handler) http.Handler {
8888
}
8989

9090
// InitUploadTag creates a new tag for an upload to the local HTTP proxy
91-
// if a tag is not named using the SwarmTagHeaderName, a fallback name will be used
91+
// if a tag is not named using the TagHeaderName, a fallback name will be used
9292
// when the Content-Length header is set, an ETA on chunking will be available since the
9393
// number of chunks to be split is known in advance (not including enclosing manifest chunks)
9494
// the tag can later be accessed using the appropriate identifier in the request context
@@ -99,7 +99,7 @@ func InitUploadTag(h http.Handler, tags *chunk.Tags) http.Handler {
9999
err error
100100
estimatedTotal int64 = 0
101101
contentType = r.Header.Get("Content-Type")
102-
headerTag = r.Header.Get(SwarmTagHeaderName)
102+
headerTag = r.Header.Get(TagHeaderName)
103103
)
104104
if headerTag != "" {
105105
tagName = headerTag

api/http/server.go

Lines changed: 74 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@ var (
6262
getFileFail = metrics.NewRegisteredCounter("api.http.get.file.fail", nil)
6363
getListCount = metrics.NewRegisteredCounter("api.http.get.list.count", nil)
6464
getListFail = metrics.NewRegisteredCounter("api.http.get.list.fail", nil)
65+
getTagCount = metrics.NewRegisteredCounter("api.http.get.tag.count", nil)
66+
getTagNotFound = metrics.NewRegisteredCounter("api.http.get.tag.notfound", nil)
67+
getTagFail = metrics.NewRegisteredCounter("api.http.get.tag.fail", nil)
6568
getPinCount = metrics.NewRegisteredCounter("api.http.get.pin.count", nil)
6669
getPinFail = metrics.NewRegisteredCounter("api.http.get.pin.fail", nil)
6770
postPinCount = metrics.NewRegisteredCounter("api.http.post.pin.count", nil)
@@ -71,9 +74,8 @@ var (
7174
)
7275

7376
const (
74-
SwarmTagHeaderName = "x-swarm-tag" // Presence of this in header indicates the tag
75-
PinHeaderName = "x-swarm-pin" // Presence of this in header indicates pinning required
76-
77+
TagHeaderName = "x-swarm-tag" // Presence of this in header indicates the tag
78+
PinHeaderName = "x-swarm-pin" // Presence of this in header indicates pinning required
7779
encryptAddr = "encrypt"
7880
tarContentType = "application/x-tar"
7981
)
@@ -171,6 +173,12 @@ func NewServer(api *api.API, pinAPI *pin.API, corsString string) *Server {
171173
defaultMiddlewares...,
172174
),
173175
})
176+
mux.Handle("/bzz-tag:/", methodHandler{
177+
"GET": Adapt(
178+
http.HandlerFunc(server.HandleGetTag),
179+
defaultMiddlewares...,
180+
),
181+
})
174182
mux.Handle("/bzz-pin:/", methodHandler{
175183
"GET": Adapt(
176184
http.HandlerFunc(server.HandleGetPins),
@@ -326,6 +334,7 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) {
326334
}
327335

328336
w.Header().Set("Content-Type", "text/plain")
337+
w.Header().Set(TagHeaderName, fmt.Sprint(tagUid))
329338
w.WriteHeader(http.StatusOK)
330339
fmt.Fprint(w, addr)
331340
}
@@ -402,7 +411,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) {
402411
log.Error("got an error retrieving tag for DoneSplit", "tagUid", tagUid, "err", err)
403412
}
404413

405-
log.Debug("done splitting, setting tag total", "SPLIT", tag.Get(chunk.StateSplit), "TOTAL", tag.Total())
414+
log.Debug("done splitting, setting tag total", "SPLIT", tag.Get(chunk.StateSplit), "TOTAL", tag.TotalCounter())
406415
tag.DoneSplit(newAddr)
407416

408417
// Pin the file
@@ -418,6 +427,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) {
418427
log.Debug("stored content", "ruid", ruid, "key", newAddr)
419428

420429
w.Header().Set("Content-Type", "text/plain")
430+
w.Header().Set(TagHeaderName, fmt.Sprint(tagUid))
421431
w.WriteHeader(http.StatusOK)
422432
fmt.Fprint(w, newAddr)
423433
}
@@ -919,6 +929,66 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) {
919929
http.ServeContent(w, r, fileName, time.Now(), newBufferedReadSeeker(reader, getFileBufferSize))
920930
}
921931

932+
// HandleGetTag responds to the following request
933+
// - bzz-tag:/<manifest> and
934+
// - bzz-tag:/?tagId=<tagId>
935+
// Clients should use root hash or the tagID to get the tag counters
936+
func (s *Server) HandleGetTag(w http.ResponseWriter, r *http.Request) {
937+
getTagCount.Inc(1)
938+
uri := GetURI(r.Context())
939+
if uri == nil {
940+
getTagFail.Inc(1)
941+
respondError(w, r, "Error decoding uri", http.StatusBadRequest)
942+
return
943+
}
944+
fileAddr := uri.Address()
945+
946+
var tag *chunk.Tag
947+
if fileAddr == nil {
948+
tagString := r.URL.Query().Get("Id")
949+
if tagString == "" {
950+
getTagFail.Inc(1)
951+
respondError(w, r, "Missing one of the mandatory argument", http.StatusBadRequest)
952+
return
953+
}
954+
955+
u64, err := strconv.ParseUint(tagString, 10, 32)
956+
if err != nil {
957+
getTagFail.Inc(1)
958+
respondError(w, r, "Invalid Id argument", http.StatusBadRequest)
959+
return
960+
}
961+
tagId := uint32(u64)
962+
963+
tag, err = s.api.Tags.Get(tagId)
964+
if err != nil {
965+
getTagNotFound.Inc(1)
966+
respondError(w, r, "Tag not found", http.StatusNotFound)
967+
return
968+
}
969+
} else {
970+
971+
tagByFile, err := s.api.Tags.GetByAddress(fileAddr)
972+
if err != nil {
973+
getTagNotFound.Inc(1)
974+
respondError(w, r, "Tag not found", http.StatusNotFound)
975+
return
976+
}
977+
tag = tagByFile
978+
}
979+
980+
w.Header().Set("Content-Type", "application/json")
981+
w.Header().Set("Cache-Control", "no-cache, private, max-age=0")
982+
r.Header.Del("ETag")
983+
w.WriteHeader(http.StatusOK)
984+
err := json.NewEncoder(w).Encode(&tag)
985+
if err != nil {
986+
getTagFail.Inc(1)
987+
respondError(w, r, "marshalling error", http.StatusInternalServerError)
988+
return
989+
}
990+
}
991+
922992
// HandlePin takes a root hash as argument and pins a given file or collection in the local Swarm DB
923993
func (s *Server) HandlePin(w http.ResponseWriter, r *http.Request) {
924994
postPinCount.Inc(1)

api/http/server_test.go

Lines changed: 123 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import (
4343
"github.com/ethereum/go-ethereum/crypto"
4444
"github.com/ethereum/go-ethereum/log"
4545
"github.com/ethersphere/swarm/api"
46+
"github.com/ethersphere/swarm/chunk"
4647
"github.com/ethersphere/swarm/storage"
4748
"github.com/ethersphere/swarm/storage/feed"
4849
"github.com/ethersphere/swarm/storage/feed/lookup"
@@ -68,6 +69,125 @@ func newTestSigner() (*feed.GenericSigner, error) {
6869
return feed.NewGenericSigner(privKey), nil
6970
}
7071

72+
// TestGetTag uploads a file, retrieves the tag using http GET and check if it matches
73+
func TestGetTagUsingHash(t *testing.T) {
74+
srv := NewTestSwarmServer(t, serverFunc, nil, nil)
75+
defer srv.Close()
76+
77+
// upload a file
78+
data := testutil.RandomBytes(1, 10000)
79+
resp, err := http.Post(fmt.Sprintf("%s/bzz-raw:/", srv.URL), "text/plain", bytes.NewReader(data))
80+
if err != nil {
81+
t.Fatal(err)
82+
}
83+
defer resp.Body.Close()
84+
if resp.StatusCode != http.StatusOK {
85+
t.Fatalf("err %s", resp.Status)
86+
}
87+
rootHash, err := ioutil.ReadAll(resp.Body)
88+
if err != nil {
89+
t.Fatal(err)
90+
}
91+
92+
// get the tag for the above upload using root hash of the file
93+
getBzzURL := fmt.Sprintf("%s/bzz-tag:/%s", srv.URL, string(rootHash))
94+
getResp, err := http.Get(getBzzURL)
95+
if err != nil {
96+
t.Fatal(err)
97+
}
98+
defer getResp.Body.Close()
99+
if getResp.StatusCode != http.StatusOK {
100+
t.Fatalf("err %s", getResp.Status)
101+
}
102+
retrievedData, err := ioutil.ReadAll(getResp.Body)
103+
if err != nil {
104+
t.Fatal(err)
105+
}
106+
tag := &chunk.Tag{}
107+
err = json.Unmarshal(retrievedData, &tag)
108+
if err != nil {
109+
t.Fatal(err)
110+
}
111+
112+
// check if the tag has valid values
113+
rcvdAddress, err := hex.DecodeString(string(rootHash))
114+
if err != nil {
115+
t.Fatal(err)
116+
}
117+
if !bytes.Equal(tag.Address, rcvdAddress) {
118+
t.Fatalf("retrieved address mismatch, expected %x, got %x", string(rootHash), tag.Address)
119+
}
120+
121+
if tag.TotalCounter() != 4 {
122+
t.Fatalf("retrieved total tag count mismatch, expected %x, got %x", 4, tag.TotalCounter())
123+
}
124+
125+
if !strings.HasPrefix(tag.Name, "unnamed_tag_") {
126+
t.Fatalf("retrieved name prefix mismatch, expected %x, got %x", "unnamed_tag_", tag.Name)
127+
}
128+
129+
}
130+
131+
// TestGetTag uploads a file, retrieves the tag using http GET and check if it matches
132+
func TestGetTagUsingTagId(t *testing.T) {
133+
srv := NewTestSwarmServer(t, serverFunc, nil, nil)
134+
defer srv.Close()
135+
136+
// upload a file
137+
data := testutil.RandomBytes(1, 10000)
138+
resp, err := http.Post(fmt.Sprintf("%s/bzz-raw:/", srv.URL), "text/plain", bytes.NewReader(data))
139+
if err != nil {
140+
t.Fatal(err)
141+
}
142+
defer resp.Body.Close()
143+
if resp.StatusCode != http.StatusOK {
144+
t.Fatalf("err %s", resp.Status)
145+
}
146+
rootHash, err := ioutil.ReadAll(resp.Body)
147+
if err != nil {
148+
t.Fatal(err)
149+
}
150+
tidString := resp.Header.Get(TagHeaderName)
151+
152+
// get the tag of the above upload using the tagId
153+
getBzzURL := fmt.Sprintf("%s/bzz-tag:/?Id=%s", srv.URL, tidString)
154+
getResp, err := http.Get(getBzzURL)
155+
if err != nil {
156+
t.Fatal(err)
157+
}
158+
defer getResp.Body.Close()
159+
if getResp.StatusCode != http.StatusOK {
160+
t.Fatalf("err %s", getResp.Status)
161+
}
162+
retrievedData, err := ioutil.ReadAll(getResp.Body)
163+
if err != nil {
164+
t.Fatal(err)
165+
}
166+
tag := &chunk.Tag{}
167+
err = json.Unmarshal(retrievedData, &tag)
168+
if err != nil {
169+
t.Fatal(err)
170+
}
171+
172+
// check if the received tags has valid values
173+
rcvdAddress, err := hex.DecodeString(string(rootHash))
174+
if err != nil {
175+
t.Fatal(err)
176+
}
177+
if !bytes.Equal(tag.Address, rcvdAddress) {
178+
t.Fatalf("retrieved address mismatch, expected %x, got %x", string(rootHash), tag.Address)
179+
}
180+
181+
if tag.TotalCounter() != 4 {
182+
t.Fatalf("retrieved total tag count mismatch, expected %x, got %x", 4, tag.TotalCounter())
183+
}
184+
185+
if !strings.HasPrefix(tag.Name, "unnamed_tag_") {
186+
t.Fatalf("retrieved name prefix mismatch, expected %x, got %x", "unnamed_tag_", tag.Name)
187+
}
188+
189+
}
190+
71191
// TestPinUnpinAPI function tests the pinning and unpinning through HTTP API.
72192
// It does the following
73193
// 1) upload a file
@@ -125,6 +245,7 @@ func TestPinUnpinAPI(t *testing.T) {
125245
if len(listInfosUnpin) != 0 {
126246
t.Fatalf("roothash is in list of pinned files")
127247
}
248+
128249
}
129250

130251
// Test the transparent resolving of feed updates with bzz:// scheme
@@ -814,7 +935,7 @@ func testBzzTar(encrypted bool, t *testing.T) {
814935
t.Fatal(err)
815936
}
816937
req.Header.Add("Content-Type", "application/x-tar")
817-
req.Header.Add(SwarmTagHeaderName, "test-upload")
938+
req.Header.Add(TagHeaderName, "test-upload")
818939
client := &http.Client{}
819940
resp2, err := client.Do(req)
820941
if err != nil {
@@ -933,7 +1054,7 @@ func TestBzzCorrectTagEstimate(t *testing.T) {
9331054

9341055
req = req.WithContext(ctx)
9351056
req.ContentLength = 1000000
936-
req.Header.Add(SwarmTagHeaderName, "1000000")
1057+
req.Header.Add(TagHeaderName, "1000000")
9371058

9381059
go func() {
9391060
for {

api/uri.go

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ func Parse(rawuri string) (*URI, error) {
8686

8787
// check the scheme is valid
8888
switch uri.Scheme {
89-
case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzz-feed", "bzz-pin":
89+
case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzz-feed", "bzz-tag", "bzz-pin":
9090
default:
9191
return nil, fmt.Errorf("unknown scheme %q", u.Scheme)
9292
}
@@ -108,6 +108,12 @@ func Parse(rawuri string) (*URI, error) {
108108
}
109109
return uri, nil
110110
}
111+
112+
// Tag returns the string representation of the tag uri scheme
113+
func (u *URI) Tag() bool {
114+
return u.Scheme == "bzz-tag"
115+
}
116+
111117
func (u *URI) Feed() bool {
112118
return u.Scheme == "bzz-feed"
113119
}
@@ -128,6 +134,7 @@ func (u *URI) Hash() bool {
128134
return u.Scheme == "bzz-hash"
129135
}
130136

137+
// Pin returns the string representation of the pin uri scheme
131138
func (u *URI) Pin() bool {
132139
return u.Scheme == "bzz-pin"
133140
}

0 commit comments

Comments
 (0)