Skip to content

Commit 6638a8b

Browse files
yevgenypatsyevgenypats
authored
fix: Tests and spec unmarshalling (#3)
1 parent 02878b4 commit 6638a8b

File tree

13 files changed

+115
-131
lines changed

13 files changed

+115
-131
lines changed

‎.github/workflows/lint_golang.yml‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# DONT EDIT. This file is synced from https://github.com/cloudquery/.github/.github
2-
name: lint_golang
2+
name: Lint
33
on:
44
push:
55
branches:
@@ -14,20 +14,10 @@ jobs:
1414
runs-on: ubuntu-latest
1515
steps:
1616
- uses: actions/checkout@v3
17-
- uses: dorny/paths-filter@v2
18-
id: changes
19-
with:
20-
filters: |
21-
src:
22-
- '!(terraform/**/*|scripts/**/*)'
23-
2417
- uses: actions/setup-go@v3
25-
if: steps.changes.outputs.src == 'true' || github.event_name != 'pull_request'
2618
with:
27-
go-version: 1.17
28-
19+
go-version: '1.18'
2920
- name: golangci-lint
30-
if: steps.changes.outputs.src == 'true' || github.event_name != 'pull_request'
3121
uses: golangci/golangci-lint-action@v3
3222
with:
3323
version: v1.46.2

‎.github/workflows/unittest.yml‎

Lines changed: 5 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: UnitTest
1+
name: "Unit tests"
22
on:
33
push:
44
branches:
@@ -8,32 +8,12 @@ on:
88

99
jobs:
1010
unitests:
11-
strategy:
12-
matrix:
13-
dbversion: ['postgres:10']
14-
go: ['1.17']
15-
platform: [ ubuntu-latest]
16-
runs-on: ${{ matrix.platform }}
17-
services:
18-
postgres:
19-
image: ${{ matrix.dbversion }}
20-
env:
21-
POSTGRES_PASSWORD: pass
22-
POSTGRES_USER: postgres
23-
POSTGRES_DB: postgres
24-
ports:
25-
- 5432:5432
26-
# Set health checks to wait until postgres has started
27-
options: >-
28-
--health-cmd pg_isready
29-
--health-interval 10s
30-
--health-timeout 5s
31-
--health-retries 5
11+
runs-on: ubuntu-latest
3212
steps:
3313
- name: Set up Go 1.x
3414
uses: actions/setup-go@v3
3515
with:
36-
go-version: ^${{ matrix.go }}
16+
go-version: 1.18
3717
- name: Check out code into the Go module directory
3818
uses: actions/checkout@v3
3919
- uses: actions/cache@v3
@@ -48,9 +28,9 @@ jobs:
4828
~/.cache/go-build
4929
~/Library/Caches/go-build
5030
~\AppData\Local\go-build
51-
key: ${{ runner.os }}-go-${{ matrix.go }}-${{ hashFiles('**/go.sum') }}
31+
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
5232
restore-keys: |
53-
${{ runner.os }}-go-${{ matrix.go }}-
33+
${{ runner.os }}-go-
5434
- run: go mod download
5535
- name: Test
5636
run: go test -v ./...

‎CODEOWNERS‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
* @cloudquery/cloudquery-opensource
1+
* @yevgenypats

‎clients/template.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,5 @@ func templateFuncMap() template.FuncMap {
1313

1414
func indent(spaces int, v string) string {
1515
pad := strings.Repeat(" ", spaces)
16-
return pad + strings.Replace(v, "\n", "\n"+pad, -1)
16+
return pad + strings.ReplaceAll(v, "\n", "\n"+pad)
1717
}

‎go.mod‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ require (
1111
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2
1212
github.com/iancoleman/strcase v0.2.0
1313
github.com/lorenzosaino/go-sysctl v0.3.1
14-
github.com/mitchellh/hashstructure/v2 v2.0.2
1514
github.com/modern-go/reflect2 v1.0.2
1615
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
1716
github.com/pkg/errors v0.9.1

‎go.sum‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -202,8 +202,6 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
202202
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
203203
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
204204
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
205-
github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
206-
github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
207205
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
208206
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
209207
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@@ -420,7 +418,6 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
420418
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
421419
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
422420
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
423-
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
424421
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
425422
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
426423
google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w=

‎schema/md_docs.go‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,17 @@ import (
99
"strings"
1010
)
1111

12+
const tableTmpl = `
13+
# Table: {{.Name}}
14+
{{ $.Description }}
15+
## Columns
16+
| Name | Typ | Description |
17+
| ------------- | ------------- | ----------- |
18+
{{- range $column := $.Columns }}
19+
|{{$column.Name}}|{{$column.Type}}|{{$column.Description|removeLineBreaks}}|
20+
{{- end }}
21+
`
22+
1223
// We are taking a similar approach to Cobra command generation for tables
1324
// https://github.com/spf13/cobra/blob/main/doc/md_docs.go
1425
// We also use this in our CLI cloudquery/cloudquery
@@ -46,14 +57,3 @@ func generateMarkdownTree(table *Table, dir string) error {
4657
}
4758
return ioutil.WriteFile(filepath.Join(dir, fmt.Sprintf("%s.md", table.Name)), buf.Bytes(), 0644)
4859
}
49-
50-
const tableTmpl = `
51-
# Table: {{.Name}}
52-
{{ $.Description }}
53-
## Columns
54-
| Name | Typ | Description |
55-
| ------------- | ------------- | ----------- |
56-
{{- range $column := $.Columns }}
57-
|{{$column.Name}}|{{$column.Type}}|{{$column.Description|removeLineBreaks}}|
58-
{{- end }}
59-
`

‎schema/meta.go‎

Lines changed: 37 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
package schema
22

33
import (
4-
"context"
5-
"encoding/json"
64
"time"
75

86
"github.com/rs/zerolog"
@@ -20,24 +18,24 @@ type Meta struct {
2018
const FetchIdMetaKey = "cq_fetch_id"
2119

2220
var (
23-
cqMeta = Column{
24-
Name: "cq_meta",
25-
Type: TypeJSON,
26-
Description: "Meta column holds fetch information",
27-
Resolver: func(ctx context.Context, meta ClientMeta, resource *Resource, c Column) error {
28-
mi := Meta{
29-
LastUpdate: time.Now().UTC(),
30-
}
31-
if val, ok := resource.GetMeta(FetchIdMetaKey); ok {
32-
if s, ok := val.(string); ok {
33-
mi.FetchId = s
34-
}
35-
}
36-
b, _ := json.Marshal(mi)
37-
return resource.Set(c.Name, b)
38-
},
39-
internal: true,
40-
}
21+
// cqMeta = Column{
22+
// Name: "cq_meta",
23+
// Type: TypeJSON,
24+
// Description: "Meta column holds fetch information",
25+
// Resolver: func(ctx context.Context, meta ClientMeta, resource *Resource, c Column) error {
26+
// mi := Meta{
27+
// LastUpdate: time.Now().UTC(),
28+
// }
29+
// if val, ok := resource.GetMeta(FetchIdMetaKey); ok {
30+
// if s, ok := val.(string); ok {
31+
// mi.FetchId = s
32+
// }
33+
// }
34+
// b, _ := json.Marshal(mi)
35+
// return resource.Set(c.Name, b)
36+
// },
37+
// internal: true,
38+
// }
4139
cqIdColumn = Column{
4240
Name: "cq_id",
4341
Type: TypeUUID,
@@ -58,23 +56,23 @@ var (
5856
},
5957
internal: true,
6058
}
61-
cqFetchDateColumn = Column{
62-
Name: "cq_fetch_date",
63-
Type: TypeTimestamp,
64-
Description: "Time of fetch for this resource",
65-
// Resolver: func(ctx context.Context, meta ClientMeta, resource *Resource, c Column) error {
66-
// val, ok := resource.GetMeta("cq_fetch_date")
67-
// if !ok && !resource.executionStart.IsZero() {
68-
// val = resource.executionStart
69-
// }
70-
// if val == nil {
71-
// return fmt.Errorf("zero cq_fetch date")
72-
// }
73-
// return resource.Set(c.Name, val)
74-
// },
75-
CreationOptions: ColumnCreationOptions{
76-
NotNull: true,
77-
},
78-
internal: true,
79-
}
59+
// cqFetchDateColumn = Column{
60+
// Name: "cq_fetch_date",
61+
// Type: TypeTimestamp,
62+
// Description: "Time of fetch for this resource",
63+
// // Resolver: func(ctx context.Context, meta ClientMeta, resource *Resource, c Column) error {
64+
// // val, ok := resource.GetMeta("cq_fetch_date")
65+
// // if !ok && !resource.executionStart.IsZero() {
66+
// // val = resource.executionStart
67+
// // }
68+
// // if val == nil {
69+
// // return fmt.Errorf("zero cq_fetch date")
70+
// // }
71+
// // return resource.Set(c.Name, val)
72+
// // },
73+
// CreationOptions: ColumnCreationOptions{
74+
// NotNull: true,
75+
// },
76+
// internal: true,
77+
// }
8078
)

‎schema/resource.go‎

Lines changed: 23 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,9 @@
11
package schema
22

33
import (
4-
"crypto"
54
"fmt"
6-
"strings"
75

86
"github.com/google/uuid"
9-
"github.com/mitchellh/hashstructure/v2"
107
"github.com/thoas/go-funk"
118
)
129

@@ -22,7 +19,7 @@ type Resource struct {
2219
// internal fields
2320
Table *Table `msgpack:"table"`
2421
Data map[string]interface{} `msgpack:"data"`
25-
cqId uuid.UUID `msgpack:"cq_id"`
22+
cqId uuid.UUID
2623
metadata map[string]interface{}
2724
CColumns []string `msgpack:"columns"`
2825
}
@@ -141,14 +138,14 @@ func (r Resource) GetMeta(key string) (interface{}, bool) {
141138
return v, ok
142139
}
143140

144-
func (r Resource) getColumnByName(column string) *Column {
145-
for _, c := range r.Table.Columns {
146-
if strings.Compare(column, c.Name) == 0 {
147-
return &c
148-
}
149-
}
150-
return nil
151-
}
141+
// func (r Resource) getColumnByName(column string) *Column {
142+
// for _, c := range r.Table.Columns {
143+
// if strings.Compare(column, c.Name) == 0 {
144+
// return &c
145+
// }
146+
// }
147+
// return nil
148+
// }
152149

153150
func (rr Resources) GetIds() []uuid.UUID {
154151
rids := make([]uuid.UUID, len(rr))
@@ -171,17 +168,17 @@ func (rr Resources) ColumnNames() []string {
171168
return rr[0].CColumns
172169
}
173170

174-
func hashUUID(objs interface{}) (uuid.UUID, error) {
175-
// Use SHA1 because it's fast and is reasonably enough protected against accidental collisions.
176-
// There is no scenario here where intentional created collisions could do harm.
177-
digester := crypto.SHA1.New()
178-
hash, err := hashstructure.Hash(objs, hashstructure.FormatV2, nil)
179-
if err != nil {
180-
return uuid.Nil, err
181-
}
182-
if _, err := fmt.Fprint(digester, hash); err != nil {
183-
return uuid.Nil, err
184-
}
185-
data := digester.Sum(nil)
186-
return uuid.NewSHA1(uuid.Nil, data), nil
187-
}
171+
// func hashUUID(objs interface{}) (uuid.UUID, error) {
172+
// // Use SHA1 because it's fast and is reasonably enough protected against accidental collisions.
173+
// // There is no scenario here where intentional created collisions could do harm.
174+
// digester := crypto.SHA1.New()
175+
// hash, err := hashstructure.Hash(objs, hashstructure.FormatV2, nil)
176+
// if err != nil {
177+
// return uuid.Nil, err
178+
// }
179+
// if _, err := fmt.Fprint(digester, hash); err != nil {
180+
// return uuid.Nil, err
181+
// }
182+
// data := digester.Sum(nil)
183+
// return uuid.NewSHA1(uuid.Nil, data), nil
184+
// }

‎schema/table.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package schema
33
import (
44
"context"
55
"runtime/debug"
6-
"strings"
76

87
"github.com/cloudquery/plugin-sdk/helpers"
98
"github.com/iancoleman/strcase"
@@ -73,9 +72,9 @@ func (t Table) Column(name string) *Column {
7372
return nil
7473
}
7574

76-
func (tco TableCreationOptions) signature() string {
77-
return strings.Join(tco.PrimaryKeys, ";")
78-
}
75+
// func (tco TableCreationOptions) signature() string {
76+
// return strings.Join(tco.PrimaryKeys, ";")
77+
// }
7978

8079
func (t Table) TableNames() []string {
8180
ret := []string{t.Name}

0 commit comments

Comments
 (0)