Skip to content

Commit 4f6c174

Browse files
olafurpgolafurpg
authored
Add scip snapshot subcommand (#14)
* Add stats subcommand to get quick statistics about an index * Add `scip snapshot` command to snapshot a SCIP index * Address review feedback
1 parent 74aaacf commit 4f6c174

File tree

10 files changed

+216
-35
lines changed

10 files changed

+216
-35
lines changed

‎bindings/go/scip/symbol_formatter.go

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,22 @@ type SymbolFormatter struct {
1313
IncludeDescriptor func(descriptor string) bool
1414
}
1515

16+
// VerboseSymbolFormatter formats all parts of the symbol.
17+
var VerboseSymbolFormatter = SymbolFormatter{
18+
IncludeScheme: func(_ string) bool { return true },
19+
IncludePackageManager: func(_ string) bool { return true },
20+
IncludePackageName: func(_ string) bool { return true },
21+
IncludePackageVersion: func(_ string) bool { return true },
22+
IncludeDescriptor: func(_ string) bool { return true },
23+
}
24+
1625
// DescriptorOnlyFormatter formats only the descriptor part of the symbol.
1726
var DescriptorOnlyFormatter = SymbolFormatter{
1827
IncludeScheme: func(scheme string) bool { return scheme == "local" },
19-
IncludePackageManager: func(_unused string) bool { return false },
20-
IncludePackageName: func(_unused string) bool { return false },
21-
IncludePackageVersion: func(_unused string) bool { return false },
22-
IncludeDescriptor: func(_unused string) bool { return true },
28+
IncludePackageManager: func(_ string) bool { return false },
29+
IncludePackageName: func(_ string) bool { return false },
30+
IncludePackageVersion: func(_ string) bool { return false },
31+
IncludeDescriptor: func(_ string) bool { return true },
2332
}
2433

2534
func (f *SymbolFormatter) Format(symbol string) (string, error) {

‎bindings/go/scip/symbol_role.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package scip
2+
3+
func (r SymbolRole) Matches(occ *Occurrence) bool {
4+
return occ.SymbolRoles&int32(r) > 0
5+
}

‎cmd/Readme.md

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,17 @@ instead of being present at the root because:
1616
```
1717
Usage:
1818
scip convert [--from=<path>] [--to=<path>]
19+
scip stats [--from=<path>]
20+
scip snapshot [--from=<path>] [--output=<path>]
1921
scip --version
2022
scip -h | --help
2123
2224
Options:
23-
--from=<path> Input file for conversion [default: index.scip].
24-
--to=<path> Output file for conversion [default: dump.lsif].
25-
--version Show version.
26-
-h --help Show help text.
25+
--from=<path> Input file for conversion [default: index.scip].
26+
--to=<path> Output file for conversion [default: dump.lsif].
27+
--output=<path> Output directory [default: scip-snapshot].
28+
--version Show version.
29+
-h --help Show help text.
2730
2831
A single dash path ('-') for --from (--to) is interpreted as stdin (stdout).
2932

‎cmd/convert.go

Lines changed: 4 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -5,28 +5,16 @@ import (
55
"os"
66
"strings"
77

8-
"google.golang.org/protobuf/proto"
9-
108
"github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol/reader"
119
"github.com/sourcegraph/sourcegraph/lib/errors"
1210

1311
"github.com/sourcegraph/scip/bindings/go/scip"
1412
)
1513

1614
func convertMain(parsedArgs map[string]interface{}) error {
17-
var scipReader io.Reader
18-
fromPath := parsedArgs["from"].(string)
19-
if fromPath == "-" {
20-
scipReader = os.Stdin
21-
} else if !strings.HasSuffix(fromPath, ".scip") && !strings.HasSuffix(fromPath, ".lsif-typed") {
22-
return errors.Newf("expected file with .scip extension but found %s", fromPath)
23-
} else {
24-
scipFile, err := os.Open(fromPath)
25-
defer scipFile.Close()
26-
if err != nil {
27-
return err
28-
}
29-
scipReader = scipFile
15+
scipIndex, err := readFromOption(parsedArgs["from"].(string))
16+
if err != nil {
17+
return err
3018
}
3119

3220
var lsifWriter io.Writer
@@ -44,18 +32,7 @@ func convertMain(parsedArgs map[string]interface{}) error {
4432
lsifWriter = lsifFile
4533
}
4634

47-
scipBytes, err := io.ReadAll(scipReader)
48-
if err != nil {
49-
return errors.Wrapf(err, "failed to read SCIP index at path %s", fromPath)
50-
}
51-
52-
scipIndex := scip.Index{}
53-
err = proto.Unmarshal(scipBytes, &scipIndex)
54-
if err != nil {
55-
return errors.Wrapf(err, "failed to parse SCIP index at path %s", fromPath)
56-
}
57-
58-
lsifIndex, err := scip.ConvertSCIPToLSIF(&scipIndex)
35+
lsifIndex, err := scip.ConvertSCIPToLSIF(scipIndex)
5936
if err != nil {
6037
return errors.Wrap(err, "failed to convert SCIP index to LSIF index")
6138
}

‎cmd/main.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,14 @@ func main() {
4444
bailIfError(convertMain(parsedArgs))
4545
os.Exit(0)
4646
}
47+
if parsedArgs["stats"].(bool) {
48+
bailIfError(statsMain(parsedArgs))
49+
os.Exit(0)
50+
}
51+
if parsedArgs["snapshot"].(bool) {
52+
bailIfError(snapshotMain(parsedArgs))
53+
os.Exit(0)
54+
}
4755
// Normally, this should be impossible as docopt should properly handle
4856
// incorrect arguments, but might as well exit nicely. 🤷🏽
4957
os.Stderr.WriteString(helpText())

‎cmd/option_from.go

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package main
2+
3+
import (
4+
"io"
5+
"os"
6+
"strings"
7+
8+
"github.com/sourcegraph/scip/bindings/go/scip"
9+
"github.com/sourcegraph/sourcegraph/lib/errors"
10+
"google.golang.org/protobuf/proto"
11+
)
12+
13+
// readFromOptions reads the fromPath parameter into a SCIP Index message.
14+
// If fromPath has the value "-" then the SCIP Index is read from os.Stdin.
15+
// Otherwise, fromPath is interpreted as a file path and the bytes are read from disk.
16+
func readFromOption(fromPath string) (*scip.Index, error) {
17+
var scipReader io.Reader
18+
if fromPath == "-" {
19+
scipReader = os.Stdin
20+
} else if !strings.HasSuffix(fromPath, ".scip") && !strings.HasSuffix(fromPath, ".lsif-typed") {
21+
return nil, errors.Newf("expected file with .scip extension but found %s", fromPath)
22+
} else {
23+
scipFile, err := os.Open(fromPath)
24+
defer scipFile.Close()
25+
if err != nil {
26+
return nil, err
27+
}
28+
scipReader = scipFile
29+
}
30+
31+
scipBytes, err := io.ReadAll(scipReader)
32+
if err != nil {
33+
return nil, errors.Wrapf(err, "failed to read SCIP index at path %s", fromPath)
34+
}
35+
36+
scipIndex := scip.Index{}
37+
err = proto.Unmarshal(scipBytes, &scipIndex)
38+
if err != nil {
39+
return nil, errors.Wrapf(err, "failed to parse SCIP index at path %s", fromPath)
40+
}
41+
return &scipIndex, nil
42+
}

‎cmd/snapshot.go

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"path/filepath"
7+
8+
"github.com/sourcegraph/scip/bindings/go/scip"
9+
"github.com/sourcegraph/scip/bindings/go/scip/testutil"
10+
)
11+
12+
func snapshotMain(parsedArgs map[string]interface{}) error {
13+
from := parsedArgs["--from"].(string)
14+
index, err := readFromOption(from)
15+
if err != nil {
16+
return err
17+
}
18+
output := parsedArgs["--output"].(string)
19+
err = os.RemoveAll(output)
20+
if err != nil {
21+
return err
22+
}
23+
snapshots, err := testutil.FormatSnapshots(index, "//", scip.VerboseSymbolFormatter)
24+
if err != nil {
25+
return err
26+
}
27+
for _, snapshot := range snapshots {
28+
outputPath := filepath.Join(output, snapshot.RelativePath)
29+
err = os.MkdirAll(filepath.Dir(outputPath), 0755)
30+
if err != nil {
31+
return err
32+
}
33+
err = os.WriteFile(outputPath, []byte(snapshot.Text), 0755)
34+
if err != nil {
35+
return err
36+
}
37+
}
38+
fmt.Println("done: " + output)
39+
return nil
40+
}

‎cmd/stats.go

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
package main
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"net/url"
7+
"os"
8+
"path/filepath"
9+
10+
"github.com/hhatto/gocloc"
11+
"github.com/sourcegraph/scip/bindings/go/scip"
12+
"github.com/sourcegraph/sourcegraph/lib/errors"
13+
)
14+
15+
func statsMain(parsedArgs map[string]interface{}) error {
16+
from := parsedArgs["--from"].(string)
17+
index, err := readFromOption(from)
18+
if err != nil {
19+
return err
20+
}
21+
if index.Metadata == nil {
22+
return errors.Errorf("Index.Metadata is nil (--from=%s)", from)
23+
}
24+
output := map[string]interface{}{}
25+
indexStats, err := countStatistics(index)
26+
if err != nil {
27+
return err
28+
}
29+
jsonBytes, err := json.MarshalIndent(indexStats, "", " ")
30+
if err != nil {
31+
return errors.Wrapf(err, "failed to marshall into JSON %s", output)
32+
}
33+
fmt.Println(string(jsonBytes))
34+
return nil
35+
}
36+
37+
type indexStatistics struct {
38+
Documents int32 `json:"documents"`
39+
LinesOfCode int32 `json:"linesOfCode"`
40+
Occurrences int32 `json:"occurrences"`
41+
Definitions int32 `json:"definitions"`
42+
}
43+
44+
func countStatistics(index *scip.Index) (*indexStatistics, error) {
45+
loc, err := countLinesOfCode(index)
46+
if err != nil {
47+
return nil, err
48+
}
49+
stats := &indexStatistics{
50+
Documents: int32(len(index.Documents)),
51+
LinesOfCode: loc.Total.Code,
52+
Occurrences: 0,
53+
Definitions: 0,
54+
}
55+
for _, document := range index.Documents {
56+
for _, occurrence := range document.Occurrences {
57+
stats.Occurrences += 1
58+
if scip.SymbolRole_Definition.Matches(occurrence) {
59+
stats.Definitions += 1
60+
}
61+
}
62+
}
63+
return stats, nil
64+
}
65+
66+
func countLinesOfCode(index *scip.Index) (*gocloc.Result, error) {
67+
root, err := url.Parse(index.Metadata.ProjectRoot)
68+
if err != nil {
69+
return nil, errors.Wrapf(err, "failed to parse Index.Metadata.ProjectRoot as a URI %s", index.Metadata.ProjectRoot)
70+
}
71+
stat, err := os.Stat(root.Path)
72+
if err != nil {
73+
return nil, err
74+
}
75+
if !stat.IsDir() {
76+
return nil, errors.Errorf("index.Metadata.ProjectRoot is not a directory: %s", root.Path)
77+
}
78+
processor := gocloc.NewProcessor(gocloc.NewDefinedLanguages(), gocloc.NewClocOptions())
79+
var paths []string
80+
for _, document := range index.Documents {
81+
paths = append(paths, filepath.Join(root.Path, document.RelativePath))
82+
}
83+
return processor.Analyze(paths)
84+
}

‎go.mod

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,20 @@ require (
2626
github.com/davecgh/go-spew v1.1.1 // indirect
2727
github.com/envoyproxy/protoc-gen-validate v0.3.0-java // indirect
2828
github.com/getsentry/sentry-go v0.12.0 // indirect
29+
github.com/go-enry/go-enry/v2 v2.7.2 // indirect
30+
github.com/go-enry/go-oniguruma v1.2.1 // indirect
2931
github.com/gofrs/flock v0.8.1 // indirect
3032
github.com/gofrs/uuid v4.2.0+incompatible // indirect
3133
github.com/gogo/protobuf v1.3.2 // indirect
3234
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
3335
github.com/golang/protobuf v1.5.2 // indirect
3436
github.com/google/uuid v1.3.0 // indirect
37+
github.com/hhatto/gocloc v0.4.2 // indirect
3538
github.com/huandu/xstrings v1.0.0 // indirect
3639
github.com/imdario/mergo v0.3.4 // indirect
3740
github.com/inconshreveable/mousetrap v1.0.0 // indirect
3841
github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a // indirect
42+
github.com/jessevdk/go-flags v1.4.0 // indirect
3943
github.com/jhump/protocompile v0.0.0-20220216033700-d705409f108f // indirect
4044
github.com/jhump/protoreflect v1.12.1-0.20220417024638-438db461d753 // indirect
4145
github.com/json-iterator/go v1.1.12 // indirect

‎go.sum

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,10 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
8484
github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
8585
github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
8686
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
87+
github.com/go-enry/go-enry/v2 v2.7.2 h1:IBtFo783PgL7oyd/TL1/8HQFMNzOAl4NaLPbzNOvbwM=
88+
github.com/go-enry/go-enry/v2 v2.7.2/go.mod h1:GVzIiAytiS5uT/QiuakK7TF1u4xDab87Y8V5EJRpsIQ=
89+
github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo=
90+
github.com/go-enry/go-oniguruma v1.2.1/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4=
8791
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
8892
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
8993
github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
@@ -145,6 +149,8 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
145149
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
146150
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
147151
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
152+
github.com/hhatto/gocloc v0.4.2 h1:deh3Xb1uqiySNgOccMNYb3HbKsUoQDzsZRpfQmbTIhs=
153+
github.com/hhatto/gocloc v0.4.2/go.mod h1:YsPWNgXpEHv+5NAEMGlVIX1hkVypLp+b0AEB8T1NRfA=
148154
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
149155
github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk=
150156
github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
@@ -163,6 +169,8 @@ github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0Gqw
163169
github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
164170
github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a h1:d4+I1YEKVmWZrgkt6jpXBnLgV2ZjO0YxEtLDdfIZfH4=
165171
github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a/go.mod h1:Zi/ZFkEqFHTm7qkjyNJjaWH4LQA9LQhGJyF0lTYGpxw=
172+
github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
173+
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
166174
github.com/jhump/gopoet v0.0.0-20190322174617-17282ff210b3/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI=
167175
github.com/jhump/gopoet v0.1.0/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI=
168176
github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7UvLHRQ=
@@ -289,6 +297,7 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9
289297
github.com/sourcegraph/sourcegraph/lib v0.0.0-20220511160847-5a43d3ea24eb h1:D8ciD0FEcPNnVm6BC8vg7gWJ9VxoJe6k/4j+Qxparj0=
290298
github.com/sourcegraph/sourcegraph/lib v0.0.0-20220511160847-5a43d3ea24eb/go.mod h1:iFiGPqnhOvQziDZkpWasU+Uo1BaEt/Au9kNK4VpqyOw=
291299
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
300+
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
292301
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
293302
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
294303
github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q=

0 commit comments

Comments
 (0)