Skip to content

Commit 8d4b74f

Browse files
cyrilstcyrilst
committed
new dump_limit_window parameter: limit dump size within a time window
1 parent fe85235 commit 8d4b74f

File tree

23 files changed

+129
-45
lines changed

23 files changed

+129
-45
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
SSHPROXY_VERSION ?= 1.2.0
1+
SSHPROXY_VERSION ?= 1.3.0
22
SSHPROXY_GIT_URL ?= github.com/cea-hpc/sshproxy
33

44
prefix ?= /usr

‎README.asciidoc‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,10 +75,11 @@ license. See the included files +Licence_CeCILL-B_V1-en.txt+ (English version)
7575
and +Licence_CeCILL-B_V1-fr.txt+ (French version) or visit
7676
http://www.cecill.info for details.
7777
78-
Author
79-
------
78+
Authors
79+
-------
8080

8181
Arnaud Guignard <[email protected]>
82+
Cyril Servant <[email protected]>
8283

8384
Documentation
8485
-------------

‎cmd/sshproxy-dumpd/sshproxy-dumpd.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// Copyright 2015-2020 CEA/DAM/DIF
2-
// Contributor: Arnaud Guignard <[email protected]>
2+
// Author: Arnaud Guignard <[email protected]>
3+
// Contributor: Cyril Servant <[email protected]>
34
//
45
// This software is governed by the CeCILL-B license under French law and
56
// abiding by the rules of distribution of free software. You can use,

‎cmd/sshproxy-replay/sshproxy-replay.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// Copyright 2015-2020 CEA/DAM/DIF
2-
// Contributor: Arnaud Guignard <[email protected]>
2+
// Author: Arnaud Guignard <[email protected]>
3+
// Contributor: Cyril Servant <[email protected]>
34
//
45
// This software is governed by the CeCILL-B license under French law and
56
// abiding by the rules of distribution of free software. You can use,

‎cmd/sshproxy/commands.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// Copyright 2015-2020 CEA/DAM/DIF
2-
// Contributor: Arnaud Guignard <[email protected]>
2+
// Author: Arnaud Guignard <[email protected]>
3+
// Contributor: Cyril Servant <[email protected]>
34
//
45
// This software is governed by the CeCILL-B license under French law and
56
// abiding by the rules of distribution of free software. You can use,

‎cmd/sshproxy/recorder.go‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// Copyright 2015-2020 CEA/DAM/DIF
2-
// Contributor: Arnaud Guignard <[email protected]>
2+
// Author: Arnaud Guignard <[email protected]>
3+
// Contributor: Cyril Servant <[email protected]>
34
//
45
// This software is governed by the CeCILL-B license under French law and
56
// abiding by the rules of distribution of free software. You can use,
@@ -98,6 +99,7 @@ type Recorder struct {
9899
command string // initial user command
99100
dumpfile string // path to filename where the raw records are dumped
100101
dumpLimitSize uint64 // number of bytes beyond which records are no longer dumped
102+
dumpLimitWindow time.Duration // time window in which dump size is accounted
101103
writer *record.Writer // *record.Writer where the raw records are dumped
102104
}
103105

@@ -106,7 +108,7 @@ type Recorder struct {
106108
// If dumpfile is not empty, the intercepted raw data will be written in this
107109
// file. Logging of basic statistics will be done every logStatsInterval seconds. Bandwidth will be updated in etcd every etcdStatsInterval seconds.
108110
// It will stop recording when the context is cancelled.
109-
func NewRecorder(conninfo *ConnInfo, dumpfile, command string, etcdStatsInterval time.Duration, logStatsInterval time.Duration, dumpLimitSize uint64) *Recorder {
111+
func NewRecorder(conninfo *ConnInfo, dumpfile, command string, etcdStatsInterval time.Duration, logStatsInterval time.Duration, dumpLimitSize uint64, dumpLimitWindow time.Duration) *Recorder {
110112
ch := make(chan record.Record)
111113

112114
return &Recorder{
@@ -122,6 +124,7 @@ func NewRecorder(conninfo *ConnInfo, dumpfile, command string, etcdStatsInterval
122124
command: command,
123125
dumpfile: dumpfile,
124126
dumpLimitSize: dumpLimitSize,
127+
dumpLimitWindow: dumpLimitWindow,
125128
writer: nil,
126129
}
127130
}
@@ -231,6 +234,12 @@ func (r *Recorder) Run(ctx context.Context, cli *utils.Client, etcdPath string)
231234
}
232235
}()
233236
}
237+
bw := uint64(0)
238+
bwBuf := uint64(0)
239+
var bwTimeout <-chan time.Time
240+
if r.dumpLimitWindow != 0 {
241+
bwTimeout = time.After(r.dumpLimitWindow)
242+
}
234243
if r.etcdStatsInterval != 0 {
235244
go func() {
236245
for {
@@ -252,16 +261,22 @@ func (r *Recorder) Run(ctx context.Context, cli *utils.Client, etcdPath string)
252261
r.bandwidth[i] = buf[i] / uint64(r.etcdStatsInterval.Seconds())
253262
buf[i] = 0
254263
}
264+
case <-bwTimeout:
265+
bwTimeout = time.After(r.dumpLimitWindow)
266+
bw = bwBuf
267+
bwBuf = 0
255268
case rec := <-r.ch:
256269
buf[rec.Fd] += uint64(rec.Size)
257270
r.totals[rec.Fd] += uint64(rec.Size)
258271
if r.writer != nil {
259-
r.dump(rec)
260-
if r.dumpLimitSize != 0 && r.totals[rec.Fd] > r.dumpLimitSize {
272+
if r.dumpLimitSize == 0 || (bw < r.dumpLimitSize && bwBuf < r.dumpLimitSize) {
273+
r.dump(rec)
274+
} else if r.dumpLimitWindow == 0 {
261275
fd.Close()
262276
fd = nil
263277
r.writer = nil
264278
}
279+
bwBuf += uint64(rec.Size)
265280
}
266281
case <-ctx.Done():
267282
return
@@ -270,15 +285,21 @@ func (r *Recorder) Run(ctx context.Context, cli *utils.Client, etcdPath string)
270285
} else {
271286
for {
272287
select {
288+
case <-bwTimeout:
289+
bwTimeout = time.After(r.dumpLimitWindow)
290+
bw = bwBuf
291+
bwBuf = 0
273292
case rec := <-r.ch:
274293
r.totals[rec.Fd] += uint64(rec.Size)
275294
if r.writer != nil {
276-
r.dump(rec)
277-
if r.dumpLimitSize != 0 && r.totals[rec.Fd] > r.dumpLimitSize {
295+
if r.dumpLimitSize == 0 || (bw < r.dumpLimitSize && bwBuf < r.dumpLimitSize) {
296+
r.dump(rec)
297+
} else if r.dumpLimitWindow == 0 {
278298
fd.Close()
279299
fd = nil
280300
r.writer = nil
281301
}
302+
bwBuf += uint64(rec.Size)
282303
}
283304
case <-ctx.Done():
284305
return

‎cmd/sshproxy/sshproxy.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// Copyright 2015-2020 CEA/DAM/DIF
2-
// Contributor: Arnaud Guignard <[email protected]>
2+
// Author: Arnaud Guignard <[email protected]>
3+
// Contributor: Cyril Servant <[email protected]>
34
//
45
// This software is governed by the CeCILL-B license under French law and
56
// abiding by the rules of distribution of free software. You can use,
@@ -301,6 +302,7 @@ func mainExitCode() int {
301302
log.Debugf("config.check_interval = %s", config.CheckInterval.Duration())
302303
log.Debugf("config.dump = %s", config.Dump)
303304
log.Debugf("config.dump_limit_size = %d", config.DumpLimitSize)
305+
log.Debugf("config.dump_limit_window = %s", config.DumpLimitWindow.Duration())
304306
log.Debugf("config.etcd_stats_interval = %s", config.EtcdStatsInterval.Duration())
305307
log.Debugf("config.log_stats_interval = %s", config.LogStatsInterval.Duration())
306308
log.Debugf("config.etcd = %+v", config.Etcd)
@@ -423,6 +425,10 @@ func mainExitCode() int {
423425
"-oForwardAgent=no", "-oPermitLocalCommand=no",
424426
"-oClearAllForwardings=yes", "-oProtocol=2",
425427
"-s", "--", host, "sftp")
428+
if config.Dump != "" {
429+
// We don't want to dump sftp connections
430+
config.Dump = "etcd"
431+
}
426432
} else {
427433
if interactiveCommand {
428434
// Force TTY allocation because the user probably asked for it.
@@ -438,7 +444,7 @@ func mainExitCode() int {
438444

439445
var recorder *Recorder
440446
if config.Dump != "" {
441-
recorder = NewRecorder(conninfo, config.Dump, originalCmd, config.EtcdStatsInterval.Duration(), config.LogStatsInterval.Duration(), config.DumpLimitSize)
447+
recorder = NewRecorder(conninfo, config.Dump, originalCmd, config.EtcdStatsInterval.Duration(), config.LogStatsInterval.Duration(), config.DumpLimitSize, config.DumpLimitWindow.Duration())
442448

443449
wg.Add(1)
444450
go func() {

‎cmd/sshproxyctl/sshproxyctl.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// Copyright 2015-2020 CEA/DAM/DIF
2-
// Contributor: Arnaud Guignard <[email protected]>
2+
// Author: Arnaud Guignard <[email protected]>
3+
// Contributor: Cyril Servant <[email protected]>
34
//
45
// This software is governed by the CeCILL-B license under French law and
56
// abiding by the rules of distribution of free software. You can use,

‎config/sshproxy.yaml‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,19 +28,32 @@
2828
# 'TCP:host:port' (the TCP is case sensitive), e.g. 'TCP:collector:5555'.
2929
#dump: ""
3030

31-
# Maximum size of a dump (bytes). Beyond the limit, records are no more dumped.
32-
# Defaults to 0 (no limit).
31+
# Maximum amount of bytes of a dump. Setting the 'dump_limit_window' option
32+
# will limit the amount of bytes per window. This option is only useful if the
33+
# 'dump' option is set to a file or to a network address. Defaults to 0 (no
34+
# limit).
3335
#dump_limit_size: 0
3436

37+
# Duration of a dump measurement window. "0" by default, the string can contain
38+
# a unit suffix such as 'h', 'm' and 's' (e.g. "2m30s"). When set to "0", a
39+
# dump will stop once it's too big (and the dump's file descriptor will be
40+
# closed. When set to a duration, the dump will pause when there is too much
41+
# data transferred in the current window (or the previous), and will resume
42+
# when few enough data are transferred during the previous window and the
43+
# current one. This option is only useful when the 'dump_limit_size' option is
44+
# set.
45+
#dump_limit_window: "0"
46+
3547
# Interval at which basic statistics of transferred bytes are logged.
36-
# "0" by default, the string can contain a unit suffix such as 'h', 'm' and
37-
# 's' (e.g. "2m30s"). These statistics are only available when the 'dump'
38-
# option is set.
48+
# "0" by default (i.e. disabled), the string can contain a unit suffix such as
49+
# 'h', 'm' and 's' (e.g. "2m30s"). These statistics are only available when the
50+
# 'dump' option is set.
3951
#log_stats_interval: "0"
4052

41-
# Interval at which bandwidth is updated in etcd. 0 by default, the string can
42-
# contain a unit suffix such as 'h', 'm' and 's' (e.g. "2m30s"). These
43-
# statistics are only available when the 'dump' option is set.
53+
# Interval at which bandwidth is updated in etcd. "0" by default (i.e.
54+
# disabled), the string can contain a unit suffix such as 'h', 'm' and 's'
55+
# (e.g. "2m30s"). These statistics are only available when the 'dump' option is
56+
# set.
4457
#etcd_stats_interval: "0"
4558

4659
# A command can be launched in the background for the session duration.

‎doc/sshproxy-dumpd.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,8 @@ SEE ALSO
4444

4545
AUTHORS
4646
-------
47-
sshproxy-dumpd was written by Arnaud Guignard <arnaud[email protected]>.
47+
sshproxy-dumpd was written by Arnaud Guignard <arnaud[email protected]> and
48+
Cyril Servant <cyril[email protected]>.
4849

4950
COPYING
5051
-------

0 commit comments

Comments
 (0)