Skip to content

Commit 733251c

Browse files
jhengyjhengy
authored
Issue #47: Collect files opened by envoy pre-termination (lsof) (#68)
* create function to write iostats to file Signed-off-by: Hengyuan <[email protected]> * rename variables Signed-off-by: Hengyuan <[email protected]> * adjust test file to test for presence of iostats.json Signed-off-by: Hengyuan <[email protected]> * standardize output of iostats.json file to be an array of objects - same format as output of networkInterfaces: allows standardized testing Signed-off-by: Hengyuan <[email protected]> * ignore golint: rangeValCopy Signed-off-by: Hengyuan <[email protected]> * handle error Signed-off-by: Hengyuan <[email protected]> * change variable name to start with smaller letters Signed-off-by: Hengyuan <[email protected]> * fix lint error Signed-off-by: Hengyuan <[email protected]> * avoid copying of objects in loop Signed-off-by: Hengyuan <[email protected]> * create initial files Signed-off-by: Hengyuan <[email protected]> * cleanup Signed-off-by: Hengyuan <[email protected]> * enable OpenFilesDataCollection Signed-off-by: Hengyuan <[email protected]> * unit test for lsof.go Signed-off-by: Hengyuan <[email protected]> * resolve dirty commits Signed-off-by: Hengyuan <[email protected]> * resolve dirty commit Signed-off-by: Hengyuan <[email protected]> * rename variables Signed-off-by: Hengyuan <[email protected]> * remove govet nolint Signed-off-by: Hengyuan <[email protected]> * Create and expose the method to get pid of the child process Signed-off-by: Hengyuan <[email protected]> * get pid of envoy from Runtime instance method - no longer needs to search through all processes to find the envoy instance Signed-off-by: Hengyuan <[email protected]> * remove directive to ignore lint Signed-off-by: Hengyuan <[email protected]>
1 parent 490c537 commit 733251c

File tree

5 files changed

+188
-0
lines changed

5 files changed

+188
-0
lines changed

‎pkg/binary/envoy/debug/lsof.go‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
// Copyright 2019 Tetrate
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package debug
16+
17+
import (
18+
"encoding/json"
19+
"fmt"
20+
"os"
21+
"path/filepath"
22+
"syscall"
23+
24+
"github.com/shirou/gopsutil/process"
25+
26+
"github.com/tetratelabs/getenvoy/pkg/binary"
27+
"github.com/tetratelabs/getenvoy/pkg/binary/envoy"
28+
"github.com/tetratelabs/log"
29+
)
30+
31+
// OpenFileStat defines the structure of statistics about a single opened file
32+
type OpenFileStat struct {
33+
// string enclosed in `` are known as struct tags
34+
// this particular tag is used by json.Marshal() to encode Command field in a specific manner
35+
// process dependent information
36+
Command string `json:"command"`
37+
Pid string `json:"pid"`
38+
User string `json:"user"`
39+
Fd string `json:"fd"`
40+
// non process dependent information
41+
Type string `json:"type"` // type of node associated with the file / FIFO for FIFO special file
42+
Device string `json:"device"` // device numbers associated with the file, separated by commas
43+
Size string `json:"size"`
44+
Node string `json:"node"` // inode number of a local file/ Internet protocol type
45+
Name string `json:"name"` // name of the mount point and file system on which the file resides
46+
}
47+
48+
// EnableOpenFilesDataCollection is a preset option that registers collection of statistics of files opened by envoy instance(s)
49+
func EnableOpenFilesDataCollection(r *envoy.Runtime) {
50+
if err := os.Mkdir(filepath.Join(r.DebugStore(), "lsof"), os.ModePerm); err != nil {
51+
log.Errorf("error in creating a directory to write open file data of envoy to: %v", err)
52+
}
53+
r.RegisterPreTermination(retrieveOpenFilesData)
54+
}
55+
56+
// retrieveOpenFilesData writes statistics of open files associated with envoy instance(s) to a json file
57+
// if succeeded, return nil, else return an error instance
58+
func retrieveOpenFilesData(r binary.Runner) error {
59+
// get pid of envoy instance
60+
pid, err := r.GetPid()
61+
if err != nil {
62+
return fmt.Errorf("error in getting pid of envoy instance: %v", err)
63+
}
64+
envoyProcess, err := process.NewProcess(int32(pid))
65+
if err != nil {
66+
return fmt.Errorf("error in creating an envoy instance: %v", err)
67+
}
68+
69+
f, err := os.Create(filepath.Join(r.DebugStore(), "lsof/lsof.json"))
70+
if err != nil {
71+
return fmt.Errorf("error in creating a file to write open file statistics to: %v", err)
72+
}
73+
defer f.Close() //nolint
74+
75+
result := make([]OpenFileStat, 0)
76+
// print open file stats for all envoy instances
77+
// relevant fields of the process
78+
username, _ := envoyProcess.Username()
79+
name, _ := envoyProcess.Name()
80+
81+
openFiles, err := envoyProcess.OpenFiles()
82+
if err != nil {
83+
return fmt.Errorf("error in getting open file statistics: %v", err)
84+
}
85+
86+
for _, stat := range openFiles {
87+
ofStat := OpenFileStat{
88+
Command: name,
89+
Pid: fmt.Sprint(pid),
90+
User: username,
91+
Fd: fmt.Sprint(stat.Fd),
92+
Name: stat.Path,
93+
}
94+
95+
var fstat syscall.Stat_t
96+
if errSyscall := syscall.Stat(stat.Path, &fstat); errSyscall != nil {
97+
// continue if the path is invalid
98+
result = append(result, ofStat)
99+
continue
100+
}
101+
102+
ofStat.Node = fmt.Sprint(fstat.Ino)
103+
ofStat.Size = fmt.Sprint(fstat.Size)
104+
result = append(result, ofStat)
105+
}
106+
107+
out, err := json.Marshal(result)
108+
if err != nil {
109+
return fmt.Errorf("unable to convert to json representation: %v", err)
110+
}
111+
112+
fmt.Fprintln(f, string(out))
113+
114+
return nil
115+
}

‎pkg/binary/envoy/debug/lsof_test.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// Copyright 2019 Tetrate
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package debug
16+
17+
import (
18+
"encoding/json"
19+
"io/ioutil"
20+
"os"
21+
"path/filepath"
22+
"strings"
23+
"testing"
24+
"time"
25+
26+
"github.com/tetratelabs/getenvoy/pkg/binary/envoy"
27+
"github.com/tetratelabs/getenvoy/pkg/binary/envoytest"
28+
)
29+
30+
func TestGetOpenFileStats(t *testing.T) {
31+
t.Run("creates non-empty files", func(t *testing.T) {
32+
r, _ := envoy.NewRuntime(EnableOpenFilesDataCollection)
33+
defer os.RemoveAll(r.DebugStore() + ".tar.gz")
34+
defer os.RemoveAll(r.DebugStore())
35+
envoytest.RunKill(r, filepath.Join("testdata", "null.yaml"), time.Second*10)
36+
37+
files := [...]string{"lsof/lsof.json"}
38+
39+
for _, file := range files {
40+
path := filepath.Join(r.DebugStore(), file)
41+
f, err := os.Stat(path)
42+
if err != nil {
43+
t.Errorf("error stating %v: %v", path, err)
44+
}
45+
if f.Size() < 1 {
46+
t.Errorf("file %v was empty", path)
47+
}
48+
if strings.HasSuffix(file, ".json") {
49+
raw, err := ioutil.ReadFile(path)
50+
if err != nil {
51+
t.Errorf("error to read the file %v: %v", path, err)
52+
}
53+
var is []interface{}
54+
if err := json.Unmarshal(raw, &is); err != nil {
55+
t.Errorf("error to unmarshal json string, %v: \"%v\"", err, raw)
56+
}
57+
if len(is) < 1 {
58+
t.Errorf("unmarshaled content is empty, expected to be a non-empty array: \"%v\"", raw)
59+
}
60+
}
61+
}
62+
})
63+
}

‎pkg/binary/envoy/runtime.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,14 @@ func (r *Runtime) Status() int {
9191
}
9292
}
9393

94+
// GetPid returns the pid of the child process
95+
func (r *Runtime) GetPid() (int, error) {
96+
if r.cmd == nil || r.cmd.Process == nil {
97+
return 0, fmt.Errorf("envoy process not yet started")
98+
}
99+
return r.cmd.Process.Pid, nil
100+
}
101+
94102
func (r *Runtime) envoyReady() bool {
95103
// Once we have seen its ready once stop spamming the ready endpoint.
96104
// If we expand the interface to support ready <-> not ready then

‎pkg/binary/interface.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ type Runner interface {
3232
RegisterDone()
3333
SendSignal(signal os.Signal)
3434
Status() int
35+
GetPid() (int, error)
3536
AppendArgs([]string)
3637
Wait(int)
3738
WaitWithContext(context.Context, int)

‎pkg/cmd/run.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ getenvoy run standard:1.11.1 -- --help
8080
debug.EnableEnvoyAdminDataCollection,
8181
debug.EnableEnvoyLogCollection,
8282
debug.EnableNodeCollection,
83+
debug.EnableOpenFilesDataCollection,
8384
controlplaneFunc(),
8485
)
8586
if err != nil {

0 commit comments

Comments
 (0)