Skip to content

Commit a276067

Browse files
istaenugaonacha-bill
authored
feat: gsoc subscribe (#4901)
Co-authored-by: Viktor Levente Tóth <[email protected]> Co-authored-by: Acha Bill <[email protected]>
1 parent 9e388ed commit a276067

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+1344
-248
lines changed

.github/workflows/beekeeper.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,9 @@ jobs:
143143
- name: Test soc
144144
id: soc
145145
run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-soc
146+
- name: Test gsoc
147+
id: gsoc
148+
run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-gsoc
146149
- name: Test pushsync (chunks)
147150
id: pushsync-chunks-1
148151
run: timeout ${TIMEOUT} beekeeper check --cluster-name local-dns --checks=ci-pushsync-chunks

openapi/Swarm.yaml

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -818,6 +818,28 @@ paths:
818818
default:
819819
description: Default response
820820

821+
"/gsoc/subscribe/{address}":
822+
get:
823+
summary: Subscribe to GSOC payloads
824+
tags:
825+
- GSOC
826+
- Subscribe
827+
- Websocket
828+
parameters:
829+
- in: path
830+
name: reference
831+
schema:
832+
$ref: "SwarmCommon.yaml#/components/schemas/SwarmReference"
833+
required: true
834+
description: "Single Owner Chunk address (which may have multiple payloads)"
835+
responses:
836+
"200":
837+
description: Returns a WebSocket with a subscription for incoming message data on the requested SOC address.
838+
"500":
839+
$ref: "SwarmCommon.yaml#/components/responses/500"
840+
default:
841+
description: Default response
842+
821843
"/soc/{owner}/{id}":
822844
post:
823845
summary: Upload single owner chunk
@@ -847,8 +869,6 @@ paths:
847869
schema:
848870
$ref: "SwarmCommon.yaml#/components/parameters/SwarmPostageBatchId"
849871
required: true
850-
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmPinParameter"
851-
required: false
852872
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmPostageStamp"
853873
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmAct"
854874
- $ref: "SwarmCommon.yaml#/components/parameters/SwarmActHistoryAddress"

pkg/api/accesscontrol_test.go

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ func TestAccessLogicEachEndpointWithAct(t *testing.T) {
100100
resp struct {
101101
Reference swarm.Address `json:"reference"`
102102
}
103+
direct bool
103104
}{
104105
{
105106
name: "bzz",
@@ -159,6 +160,7 @@ func TestAccessLogicEachEndpointWithAct(t *testing.T) {
159160
data: bytes.NewReader(sch.WrappedChunk.Data()),
160161
expdata: sch.Chunk().Data(),
161162
contenttype: "binary/octet-stream",
163+
direct: true,
162164
},
163165
}
164166

@@ -183,13 +185,24 @@ func TestAccessLogicEachEndpointWithAct(t *testing.T) {
183185
upTestOpts = append(upTestOpts, jsonhttptest.WithRequestHeader(api.SwarmCollectionHeader, "True"))
184186
}
185187
t.Run(v.name, func(t *testing.T) {
186-
client, _, _, _ := newTestServer(t, testServerOptions{
188+
client, _, _, chanStore := newTestServer(t, testServerOptions{
187189
Storer: storerMock,
188190
Logger: logger,
189191
Post: mockpost.New(mockpost.WithAcceptAll()),
190192
PublicKey: pk.PublicKey,
191193
AccessControl: mockac.New(),
194+
DirectUpload: v.direct,
192195
})
196+
197+
if chanStore != nil {
198+
chanStore.Subscribe(func(chunk swarm.Chunk) {
199+
err := storerMock.Put(context.Background(), chunk)
200+
if err != nil {
201+
t.Fatal(err)
202+
}
203+
})
204+
}
205+
193206
header := jsonhttptest.Request(t, client, http.MethodPost, v.upurl, http.StatusCreated,
194207
upTestOpts...,
195208
)

pkg/api/api.go

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
"github.com/ethersphere/bee/v2/pkg/file/pipeline"
3434
"github.com/ethersphere/bee/v2/pkg/file/pipeline/builder"
3535
"github.com/ethersphere/bee/v2/pkg/file/redundancy"
36+
"github.com/ethersphere/bee/v2/pkg/gsoc"
3637
"github.com/ethersphere/bee/v2/pkg/jsonhttp"
3738
"github.com/ethersphere/bee/v2/pkg/log"
3839
"github.com/ethersphere/bee/v2/pkg/p2p"
@@ -151,6 +152,7 @@ type Service struct {
151152
storer Storer
152153
resolver resolver.Interface
153154
pss pss.Interface
155+
gsoc gsoc.Listener
154156
steward steward.Interface
155157
logger log.Logger
156158
loggerV1 log.Logger
@@ -254,6 +256,7 @@ type ExtraOptions struct {
254256
Storer Storer
255257
Resolver resolver.Interface
256258
Pss pss.Interface
259+
Gsoc gsoc.Listener
257260
FeedFactory feeds.Factory
258261
Post postage.Service
259262
AccessControl accesscontrol.Controller
@@ -337,6 +340,7 @@ func (s *Service) Configure(signer crypto.Signer, tracer *tracing.Tracer, o Opti
337340
s.storer = e.Storer
338341
s.resolver = e.Resolver
339342
s.pss = e.Pss
343+
s.gsoc = e.Gsoc
340344
s.feedFactory = e.FeedFactory
341345
s.post = e.Post
342346
s.accesscontrol = e.AccessControl
@@ -682,7 +686,12 @@ type putterSessionWrapper struct {
682686
}
683687

684688
func (p *putterSessionWrapper) Put(ctx context.Context, chunk swarm.Chunk) error {
685-
stamp, err := p.stamper.Stamp(chunk.Address())
689+
idAddress, err := storage.IdentityAddress(chunk)
690+
if err != nil {
691+
return err
692+
}
693+
694+
stamp, err := p.stamper.Stamp(chunk.Address(), idAddress)
686695
if err != nil {
687696
return err
688697
}

pkg/api/api_test.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"github.com/ethersphere/bee/v2/pkg/file/pipeline"
3333
"github.com/ethersphere/bee/v2/pkg/file/pipeline/builder"
3434
"github.com/ethersphere/bee/v2/pkg/file/redundancy"
35+
"github.com/ethersphere/bee/v2/pkg/gsoc"
3536
"github.com/ethersphere/bee/v2/pkg/jsonhttp/jsonhttptest"
3637
"github.com/ethersphere/bee/v2/pkg/log"
3738
p2pmock "github.com/ethersphere/bee/v2/pkg/p2p/mock"
@@ -93,6 +94,7 @@ type testServerOptions struct {
9394
StateStorer storage.StateStorer
9495
Resolver resolver.Interface
9596
Pss pss.Interface
97+
Gsoc gsoc.Listener
9698
WsPath string
9799
WsPingPeriod time.Duration
98100
Logger log.Logger
@@ -194,6 +196,7 @@ func newTestServer(t *testing.T, o testServerOptions) (*http.Client, *websocket.
194196
Storer: o.Storer,
195197
Resolver: o.Resolver,
196198
Pss: o.Pss,
199+
Gsoc: o.Gsoc,
197200
FeedFactory: o.Feeds,
198201
Post: o.Post,
199202
AccessControl: o.AccessControl,
@@ -630,6 +633,7 @@ type chanStorer struct {
630633
lock sync.Mutex
631634
chunks map[string]struct{}
632635
quit chan struct{}
636+
subs []func(chunk swarm.Chunk)
633637
}
634638

635639
func newChanStore(cc <-chan *pusher.Op) *chanStorer {
@@ -647,6 +651,9 @@ func (c *chanStorer) drain(cc <-chan *pusher.Op) {
647651
case op := <-cc:
648652
c.lock.Lock()
649653
c.chunks[op.Chunk.Address().ByteString()] = struct{}{}
654+
for _, h := range c.subs {
655+
h(op.Chunk)
656+
}
650657
c.lock.Unlock()
651658
op.Err <- nil
652659
case <-c.quit:
@@ -667,6 +674,12 @@ func (c *chanStorer) Has(addr swarm.Address) bool {
667674
return ok
668675
}
669676

677+
func (c *chanStorer) Subscribe(f func(chunk swarm.Chunk)) {
678+
c.lock.Lock()
679+
defer c.lock.Unlock()
680+
c.subs = append(c.subs, f)
681+
}
682+
670683
func createRedistributionAgentService(
671684
t *testing.T,
672685
addr swarm.Address,

pkg/api/envelope.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ func (s *Service) envelopePostHandler(w http.ResponseWriter, r *http.Request) {
5959
return
6060
}
6161

62-
stamp, err := stamper.Stamp(paths.Address)
62+
stamp, err := stamper.Stamp(paths.Address, paths.Address)
6363
if err != nil {
6464
logger.Debug("split write all failed", "error", err)
6565
logger.Error(nil, "split write all failed")

pkg/api/gsoc.go

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
// Copyright 2024 The Swarm Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package api
6+
7+
import (
8+
"net/http"
9+
"time"
10+
11+
"github.com/ethersphere/bee/v2/pkg/jsonhttp"
12+
"github.com/ethersphere/bee/v2/pkg/swarm"
13+
"github.com/gorilla/mux"
14+
"github.com/gorilla/websocket"
15+
)
16+
17+
func (s *Service) gsocWsHandler(w http.ResponseWriter, r *http.Request) {
18+
logger := s.logger.WithName("gsoc_subscribe").Build()
19+
20+
paths := struct {
21+
Address swarm.Address `map:"address,resolve" validate:"required"`
22+
}{}
23+
24+
if response := s.mapStructure(mux.Vars(r), &paths); response != nil {
25+
response("invalid path params", logger, w)
26+
return
27+
}
28+
29+
upgrader := websocket.Upgrader{
30+
ReadBufferSize: swarm.ChunkSize,
31+
WriteBufferSize: swarm.ChunkSize,
32+
CheckOrigin: s.checkOrigin,
33+
}
34+
35+
conn, err := upgrader.Upgrade(w, r, nil)
36+
if err != nil {
37+
logger.Debug("upgrade failed", "error", err)
38+
logger.Error(nil, "upgrade failed")
39+
jsonhttp.InternalServerError(w, "upgrade failed")
40+
return
41+
}
42+
43+
s.wsWg.Add(1)
44+
go s.gsocListeningWs(conn, paths.Address)
45+
}
46+
47+
func (s *Service) gsocListeningWs(conn *websocket.Conn, socAddress swarm.Address) {
48+
defer s.wsWg.Done()
49+
50+
var (
51+
dataC = make(chan []byte)
52+
gone = make(chan struct{})
53+
ticker = time.NewTicker(s.WsPingPeriod)
54+
err error
55+
)
56+
defer func() {
57+
ticker.Stop()
58+
_ = conn.Close()
59+
}()
60+
cleanup := s.gsoc.Subscribe(socAddress, func(m []byte) {
61+
select {
62+
case dataC <- m:
63+
case <-gone:
64+
return
65+
case <-s.quit:
66+
return
67+
}
68+
})
69+
70+
defer cleanup()
71+
72+
conn.SetCloseHandler(func(code int, text string) error {
73+
s.logger.Debug("gsoc ws: client gone", "code", code, "message", text)
74+
close(gone)
75+
return nil
76+
})
77+
78+
for {
79+
select {
80+
case b := <-dataC:
81+
err = conn.SetWriteDeadline(time.Now().Add(writeDeadline))
82+
if err != nil {
83+
s.logger.Debug("gsoc ws: set write deadline failed", "error", err)
84+
return
85+
}
86+
87+
err = conn.WriteMessage(websocket.BinaryMessage, b)
88+
if err != nil {
89+
s.logger.Debug("gsoc ws: write message failed", "error", err)
90+
return
91+
}
92+
93+
case <-s.quit:
94+
// shutdown
95+
err = conn.SetWriteDeadline(time.Now().Add(writeDeadline))
96+
if err != nil {
97+
s.logger.Debug("gsoc ws: set write deadline failed", "error", err)
98+
return
99+
}
100+
err = conn.WriteMessage(websocket.CloseMessage, []byte{})
101+
if err != nil {
102+
s.logger.Debug("gsoc ws: write close message failed", "error", err)
103+
}
104+
return
105+
case <-gone:
106+
// client gone
107+
return
108+
case <-ticker.C:
109+
err = conn.SetWriteDeadline(time.Now().Add(writeDeadline))
110+
if err != nil {
111+
s.logger.Debug("gsoc ws: set write deadline failed", "error", err)
112+
return
113+
}
114+
if err = conn.WriteMessage(websocket.PingMessage, nil); err != nil {
115+
// error encountered while pinging client. client probably gone
116+
return
117+
}
118+
}
119+
}
120+
}

0 commit comments

Comments
 (0)