Skip to content

Commit fda0eca

Browse files
sukunrtsukunrt
authored
webrtc: close connection when remote closes (#2914)
1 parent 8a11b7c commit fda0eca

File tree

9 files changed

+132
-45
lines changed

9 files changed

+132
-45
lines changed

‎go.mod

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,11 @@ require (
4646
github.com/multiformats/go-varint v0.0.7
4747
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
4848
github.com/pion/datachannel v1.5.8
49-
github.com/pion/ice/v2 v2.3.32
49+
github.com/pion/ice/v2 v2.3.34
5050
github.com/pion/logging v0.2.2
5151
github.com/pion/sctp v1.8.20
5252
github.com/pion/stun v0.6.1
53-
github.com/pion/webrtc/v3 v3.2.50
53+
github.com/pion/webrtc/v3 v3.3.0
5454
github.com/prometheus/client_golang v1.19.1
5555
github.com/prometheus/client_model v0.6.1
5656
github.com/quic-go/quic-go v0.45.2
@@ -111,7 +111,7 @@ require (
111111
github.com/pion/rtp v1.8.8 // indirect
112112
github.com/pion/sdp/v3 v3.0.9 // indirect
113113
github.com/pion/srtp/v2 v2.0.20 // indirect
114-
github.com/pion/transport/v2 v2.2.9 // indirect
114+
github.com/pion/transport/v2 v2.2.10 // indirect
115115
github.com/pion/turn/v2 v2.1.6 // indirect
116116
github.com/pkg/errors v0.9.1 // indirect
117117
github.com/pmezard/go-difflib v1.0.0 // indirect

‎go.sum

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -280,8 +280,8 @@ github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu
280280
github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
281281
github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk=
282282
github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE=
283-
github.com/pion/ice/v2 v2.3.32 h1:VwE/uEeqiMm0zUWpdt1DJtnqEkj3UjEbhX92/CurtWI=
284-
github.com/pion/ice/v2 v2.3.32/go.mod h1:8fac0+qftclGy1tYd/nfwfHC729BLaxtVqMdMVCAVPU=
283+
github.com/pion/ice/v2 v2.3.34 h1:Ic1ppYCj4tUOcPAp76U6F3fVrlSw8A9JtRXLqw6BbUM=
284+
github.com/pion/ice/v2 v2.3.34/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ=
285285
github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M=
286286
github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4=
287287
github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
@@ -307,17 +307,16 @@ github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/
307307
github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g=
308308
github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
309309
github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
310-
github.com/pion/transport/v2 v2.2.8/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
311-
github.com/pion/transport/v2 v2.2.9 h1:WEDygVovkJlV2CCunM9KS2kds+kcl7zdIefQA5y/nkE=
312-
github.com/pion/transport/v2 v2.2.9/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
310+
github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q=
311+
github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
313312
github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0=
314313
github.com/pion/transport/v3 v3.0.6 h1:k1mQU06bmmX143qSWgXFqSH1KUJceQvIUuVH/K5ELWw=
315314
github.com/pion/transport/v3 v3.0.6/go.mod h1:HvJr2N/JwNJAfipsRleqwFoR3t/pWyHeZUs89v3+t5s=
316315
github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
317316
github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc=
318317
github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
319-
github.com/pion/webrtc/v3 v3.2.50 h1:C/rwL2mBfCxHv6tlLzDAO3krJpQXfVx8A8WHnGJ2j34=
320-
github.com/pion/webrtc/v3 v3.2.50/go.mod h1:dytYYoSBy7ZUWhJMbndx9UckgYvzNAfL7xgVnrIKxqo=
318+
github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I=
319+
github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0=
321320
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
322321
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
323322
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

‎p2p/test/transport/transport_test.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -770,3 +770,31 @@ func TestConnDroppedWhenBlocked(t *testing.T) {
770770
})
771771
}
772772
}
773+
774+
// TestConnClosedWhenRemoteCloses tests that a connection is closed locally when it's closed by remote
775+
func TestConnClosedWhenRemoteCloses(t *testing.T) {
776+
for _, tc := range transportsToTest {
777+
t.Run(tc.Name, func(t *testing.T) {
778+
server := tc.HostGenerator(t, TransportTestCaseOpts{})
779+
client := tc.HostGenerator(t, TransportTestCaseOpts{NoListen: true})
780+
defer server.Close()
781+
defer client.Close()
782+
783+
client.Peerstore().AddAddrs(server.ID(), server.Addrs(), peerstore.PermanentAddrTTL)
784+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
785+
defer cancel()
786+
err := client.Connect(ctx, peer.AddrInfo{ID: server.ID(), Addrs: server.Addrs()})
787+
require.NoError(t, err)
788+
789+
require.Eventually(t, func() bool {
790+
return server.Network().Connectedness(client.ID()) != network.NotConnected
791+
}, 5*time.Second, 50*time.Millisecond)
792+
for _, c := range client.Network().ConnsToPeer(server.ID()) {
793+
c.Close()
794+
}
795+
require.Eventually(t, func() bool {
796+
return server.Network().Connectedness(client.ID()) == network.NotConnected
797+
}, 5*time.Second, 50*time.Millisecond)
798+
})
799+
}
800+
}

‎p2p/transport/webrtc/connection.go

Lines changed: 41 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616

1717
ma "github.com/multiformats/go-multiaddr"
1818
"github.com/pion/datachannel"
19+
"github.com/pion/sctp"
1920
"github.com/pion/webrtc/v3"
2021
)
2122

@@ -31,6 +32,8 @@ func (errConnectionTimeout) Error() string { return "connection timeout" }
3132
func (errConnectionTimeout) Timeout() bool { return true }
3233
func (errConnectionTimeout) Temporary() bool { return false }
3334

35+
var errConnClosed = errors.New("connection closed")
36+
3437
type dataChannel struct {
3538
stream datachannel.ReadWriteCloser
3639
channel *webrtc.DataChannel
@@ -74,6 +77,7 @@ func newConnection(
7477
remoteKey ic.PubKey,
7578
remoteMultiaddr ma.Multiaddr,
7679
incomingDataChannels chan dataChannel,
80+
peerConnectionClosedCh chan struct{},
7781
) (*connection, error) {
7882
ctx, cancel := context.WithCancel(context.Background())
7983
c := &connection{
@@ -102,6 +106,18 @@ func newConnection(
102106
}
103107

104108
pc.OnConnectionStateChange(c.onConnectionStateChange)
109+
pc.SCTP().OnClose(func(err error) {
110+
if err != nil {
111+
c.closeWithError(fmt.Errorf("%w: %w", errConnClosed, err))
112+
}
113+
c.closeWithError(errConnClosed)
114+
})
115+
select {
116+
case <-peerConnectionClosedCh:
117+
c.Close()
118+
return nil, errConnClosed
119+
default:
120+
}
105121
return c, nil
106122
}
107123

@@ -112,27 +128,29 @@ func (c *connection) ConnState() network.ConnectionState {
112128

113129
// Close closes the underlying peerconnection.
114130
func (c *connection) Close() error {
115-
c.closeOnce.Do(func() { c.closeWithError(errors.New("connection closed")) })
131+
c.closeWithError(errConnClosed)
116132
return nil
117133
}
118134

119135
// closeWithError is used to Close the connection when the underlying DTLS connection fails
120136
func (c *connection) closeWithError(err error) {
121-
c.closeErr = err
122-
// cancel must be called after closeErr is set. This ensures interested goroutines waiting on
123-
// ctx.Done can read closeErr without holding the conn lock.
124-
c.cancel()
125-
// closing peerconnection will close the datachannels associated with the streams
126-
c.pc.Close()
127-
128-
c.m.Lock()
129-
streams := c.streams
130-
c.streams = nil
131-
c.m.Unlock()
132-
for _, s := range streams {
133-
s.closeForShutdown(err)
134-
}
135-
c.scope.Done()
137+
c.closeOnce.Do(func() {
138+
c.closeErr = err
139+
// cancel must be called after closeErr is set. This ensures interested goroutines waiting on
140+
// ctx.Done can read closeErr without holding the conn lock.
141+
c.cancel()
142+
// closing peerconnection will close the datachannels associated with the streams
143+
c.pc.Close()
144+
145+
c.m.Lock()
146+
streams := c.streams
147+
c.streams = nil
148+
c.m.Unlock()
149+
for _, s := range streams {
150+
s.closeForShutdown(err)
151+
}
152+
c.scope.Done()
153+
})
136154
}
137155

138156
func (c *connection) IsClosed() bool {
@@ -155,6 +173,12 @@ func (c *connection) OpenStream(ctx context.Context) (network.MuxedStream, error
155173
}
156174
rwc, err := c.detachChannel(ctx, dc)
157175
if err != nil {
176+
// There's a race between webrtc.SCTP.OnClose callback and the underlying
177+
// association closing. It's nicer to close the connection here.
178+
if errors.Is(err, sctp.ErrStreamClosed) {
179+
c.closeWithError(errConnClosed)
180+
return nil, c.closeErr
181+
}
158182
dc.Close()
159183
return nil, fmt.Errorf("detach channel failed for stream(%d): %w", streamID, err)
160184
}
@@ -209,9 +233,7 @@ func (c *connection) removeStream(id uint16) {
209233

210234
func (c *connection) onConnectionStateChange(state webrtc.PeerConnectionState) {
211235
if state == webrtc.PeerConnectionStateFailed || state == webrtc.PeerConnectionStateClosed {
212-
c.closeOnce.Do(func() {
213-
c.closeWithError(errConnectionTimeout{})
214-
})
236+
c.closeWithError(errConnectionTimeout{})
215237
}
216238
}
217239

‎p2p/transport/webrtc/listener.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,7 @@ func (l *listener) setupConnection(
276276
remotePubKey,
277277
remoteMultiaddr,
278278
w.IncomingDataChannels,
279+
w.PeerConnectionClosedCh,
279280
)
280281
if err != nil {
281282
return nil, err

‎p2p/transport/webrtc/transport.go

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,7 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement
269269
}
270270
if tConn != nil {
271271
_ = tConn.Close()
272+
tConn = nil
272273
}
273274
}
274275
}()
@@ -399,6 +400,7 @@ func (t *WebRTCTransport) dial(ctx context.Context, scope network.ConnManagement
399400
remotePubKey,
400401
remoteMultiaddrWithoutCerthash,
401402
w.IncomingDataChannels,
403+
w.PeerConnectionClosedCh,
402404
)
403405
if err != nil {
404406
return nil, err
@@ -572,9 +574,10 @@ func detachHandshakeDataChannel(ctx context.Context, dc *webrtc.DataChannel) (da
572574
// a small window of time where datachannels created by the peer may not surface to us and cause a
573575
// memory leak.
574576
type webRTCConnection struct {
575-
PeerConnection *webrtc.PeerConnection
576-
HandshakeDataChannel *webrtc.DataChannel
577-
IncomingDataChannels chan dataChannel
577+
PeerConnection *webrtc.PeerConnection
578+
HandshakeDataChannel *webrtc.DataChannel
579+
IncomingDataChannels chan dataChannel
580+
PeerConnectionClosedCh chan struct{}
578581
}
579582

580583
func newWebRTCConnection(settings webrtc.SettingEngine, config webrtc.Configuration) (webRTCConnection, error) {
@@ -613,10 +616,20 @@ func newWebRTCConnection(settings webrtc.SettingEngine, config webrtc.Configurat
613616
}
614617
})
615618
})
619+
620+
connectionClosedCh := make(chan struct{}, 1)
621+
pc.SCTP().OnClose(func(err error) {
622+
// We only need one message. Closing a connection is a problem as pion might invoke the callback more than once.
623+
select {
624+
case connectionClosedCh <- struct{}{}:
625+
default:
626+
}
627+
})
616628
return webRTCConnection{
617-
PeerConnection: pc,
618-
HandshakeDataChannel: handshakeDataChannel,
619-
IncomingDataChannels: incomingDataChannels,
629+
PeerConnection: pc,
630+
HandshakeDataChannel: handshakeDataChannel,
631+
IncomingDataChannels: incomingDataChannels,
632+
PeerConnectionClosedCh: connectionClosedCh,
620633
}, nil
621634
}
622635

‎p2p/transport/webrtc/transport_test.go

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1009,3 +1009,28 @@ func TestManyConnections(t *testing.T) {
10091009
}
10101010
}
10111011
}
1012+
1013+
func TestConnectionClosedWhenRemoteCloses(t *testing.T) {
1014+
listenT, p := getTransport(t)
1015+
listener, err := listenT.Listen(ma.StringCast("/ip4/127.0.0.1/udp/0/webrtc-direct"))
1016+
require.NoError(t, err)
1017+
1018+
dialer, _ := getTransport(t)
1019+
var wg sync.WaitGroup
1020+
wg.Add(1)
1021+
go func() {
1022+
defer wg.Done()
1023+
c, err := listener.Accept()
1024+
if !assert.NoError(t, err) {
1025+
return
1026+
}
1027+
assert.Eventually(t, func() bool {
1028+
return c.IsClosed()
1029+
}, 5*time.Second, 50*time.Millisecond)
1030+
}()
1031+
1032+
c, err := dialer.Dial(context.Background(), listener.Multiaddr(), p)
1033+
require.NoError(t, err)
1034+
c.Close()
1035+
wg.Wait()
1036+
}

‎test-plans/go.mod

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ require (
6767
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
6868
github.com/pion/datachannel v1.5.8 // indirect
6969
github.com/pion/dtls/v2 v2.2.12 // indirect
70-
github.com/pion/ice/v2 v2.3.32 // indirect
70+
github.com/pion/ice/v2 v2.3.34 // indirect
7171
github.com/pion/interceptor v0.1.29 // indirect
7272
github.com/pion/logging v0.2.2 // indirect
7373
github.com/pion/mdns v0.0.12 // indirect
@@ -78,9 +78,9 @@ require (
7878
github.com/pion/sdp/v3 v3.0.9 // indirect
7979
github.com/pion/srtp/v2 v2.0.20 // indirect
8080
github.com/pion/stun v0.6.1 // indirect
81-
github.com/pion/transport/v2 v2.2.9 // indirect
81+
github.com/pion/transport/v2 v2.2.10 // indirect
8282
github.com/pion/turn/v2 v2.1.6 // indirect
83-
github.com/pion/webrtc/v3 v3.2.50 // indirect
83+
github.com/pion/webrtc/v3 v3.3.0 // indirect
8484
github.com/pkg/errors v0.9.1 // indirect
8585
github.com/pmezard/go-difflib v1.0.0 // indirect
8686
github.com/prometheus/client_golang v1.19.1 // indirect

‎test-plans/go.sum

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -226,8 +226,8 @@ github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu
226226
github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
227227
github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk=
228228
github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE=
229-
github.com/pion/ice/v2 v2.3.32 h1:VwE/uEeqiMm0zUWpdt1DJtnqEkj3UjEbhX92/CurtWI=
230-
github.com/pion/ice/v2 v2.3.32/go.mod h1:8fac0+qftclGy1tYd/nfwfHC729BLaxtVqMdMVCAVPU=
229+
github.com/pion/ice/v2 v2.3.34 h1:Ic1ppYCj4tUOcPAp76U6F3fVrlSw8A9JtRXLqw6BbUM=
230+
github.com/pion/ice/v2 v2.3.34/go.mod h1:mBF7lnigdqgtB+YHkaY/Y6s6tsyRyo4u4rPGRuOjUBQ=
231231
github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M=
232232
github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4=
233233
github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
@@ -253,17 +253,16 @@ github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/
253253
github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g=
254254
github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
255255
github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
256-
github.com/pion/transport/v2 v2.2.8/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
257-
github.com/pion/transport/v2 v2.2.9 h1:WEDygVovkJlV2CCunM9KS2kds+kcl7zdIefQA5y/nkE=
258-
github.com/pion/transport/v2 v2.2.9/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
256+
github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q=
257+
github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
259258
github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0=
260259
github.com/pion/transport/v3 v3.0.6 h1:k1mQU06bmmX143qSWgXFqSH1KUJceQvIUuVH/K5ELWw=
261260
github.com/pion/transport/v3 v3.0.6/go.mod h1:HvJr2N/JwNJAfipsRleqwFoR3t/pWyHeZUs89v3+t5s=
262261
github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
263262
github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc=
264263
github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
265-
github.com/pion/webrtc/v3 v3.2.50 h1:C/rwL2mBfCxHv6tlLzDAO3krJpQXfVx8A8WHnGJ2j34=
266-
github.com/pion/webrtc/v3 v3.2.50/go.mod h1:dytYYoSBy7ZUWhJMbndx9UckgYvzNAfL7xgVnrIKxqo=
264+
github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I=
265+
github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0=
267266
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
268267
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
269268
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

0 commit comments

Comments
 (0)