Skip to content
This repository was archived by the owner on Dec 15, 2025. It is now read-only.

upd to bbolt v1.4.0 #131

Merged
gertd merged 2 commits into from
Mar 25, 2025
Merged

upd to bbolt v1.4.0 #131

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
407e0b6
upd to bbolt v1.4.0
gertd Mar 25, 2025
b817d1d
go.mod
gertd Mar 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ on:
env:
VAULT_ADDR: https://vault.eng.aserto.com/
PRE_RELEASE: ${{ github.ref == 'refs/heads/main' && 'development' || '' }}
GO_VERSION: "1.23"
GO_VERSION: "1.24"
GO_RELEASER_VERSION: "v2.3.2"
GO_LANGCI_LINT_VERSION: "v1.64.5"
GO_TESTSUM_VERSION: "1.11.0"
Expand Down
33 changes: 17 additions & 16 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/aserto-dev/go-edge-ds

go 1.23.6
go 1.23.7

// replace github.com/aserto-dev/azm => ../azm
// replace github.com/aserto-dev/go-directory => ../go-directory
Expand All @@ -10,22 +10,22 @@ require (
github.com/aserto-dev/aserto-grpc v0.2.9
github.com/aserto-dev/azm v0.2.11
github.com/aserto-dev/errors v0.0.15
github.com/aserto-dev/go-directory v0.33.8
github.com/aserto-dev/go-directory v0.33.9
github.com/authzen/access.go v0.0.0-20250225192010-dfb0e30b2a39
github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a
github.com/gonvenience/ytbx v1.4.6
github.com/gonvenience/ytbx v1.4.7
github.com/google/uuid v1.6.0
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
github.com/homeport/dyff v1.9.4
github.com/homeport/dyff v1.10.1
github.com/panmari/cuckoofilter v1.0.6
github.com/pkg/errors v0.9.1
github.com/rs/zerolog v1.33.0
github.com/rs/zerolog v1.34.0
github.com/samber/lo v1.49.1
github.com/stretchr/testify v1.10.0
go.etcd.io/bbolt v1.3.11
golang.org/x/sync v0.11.0
go.etcd.io/bbolt v1.4.0
golang.org/x/sync v0.12.0
google.golang.org/grpc v1.71.0
google.golang.org/protobuf v1.36.5
google.golang.org/protobuf v1.36.6
)

require (
Expand All @@ -35,10 +35,11 @@ require (
github.com/deckarep/golang-set/v2 v2.7.0 // indirect
github.com/dgryski/go-metro v0.0.0-20250106013310-edb8663e5e33 // indirect
github.com/gonvenience/bunt v1.4.0 // indirect
github.com/gonvenience/idem v0.0.1 // indirect
github.com/gonvenience/neat v1.3.15 // indirect
github.com/gonvenience/term v1.0.3 // indirect
github.com/gonvenience/text v1.0.8 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.1 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
Expand All @@ -53,13 +54,13 @@ require (
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
github.com/texttheater/golang-levenshtein v1.0.1 // indirect
github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 // indirect
golang.org/x/exp v0.0.0-20250228200357-dead58393ab7 // indirect
golang.org/x/net v0.36.0 // indirect
golang.org/x/sys v0.30.0 // indirect
golang.org/x/term v0.29.0 // indirect
golang.org/x/text v0.22.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
golang.org/x/net v0.37.0 // indirect
golang.org/x/sys v0.31.0 // indirect
golang.org/x/term v0.30.0 // indirect
golang.org/x/text v0.23.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
84 changes: 43 additions & 41 deletions go.sum
View file
Open in desktop

Large diffs are not rendered by default.

20 changes: 5 additions & 15 deletions makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ GOARCH := $(shell go env GOARCH)
GOPRIVATE := "github.com/aserto-dev"
DOCKER_BUILDKIT := 1

EXT_DIR := ./.ext
EXT_DIR := ${PWD}/.ext
EXT_BIN_DIR := ${EXT_DIR}/bin
EXT_TMP_DIR := ${EXT_DIR}/tmp

GO_VER := 1.23
GO_VER := 1.24
VAULT_VER := 1.8.12
SVU_VER := 3.1.0
GOTESTSUM_VER := 1.11.0
Expand Down Expand Up @@ -101,23 +101,13 @@ install-vault: ${EXT_BIN_DIR} ${EXT_TMP_DIR}
@${EXT_BIN_DIR}/vault --version

.PHONY: install-svu
install-svu: install-svu-${GOOS}
install-svu: ${EXT_BIN_DIR} ${EXT_TMP_DIR}
@echo -e "$(ATTN_COLOR)==> $@ $(NO_COLOR)"
@gh release download v${SVU_VER} --repo https://github.com/caarlos0/svu --pattern "*${GOOS}_all.tar.gz" --output "${EXT_TMP_DIR}/svu.tar.gz" --clobber
@tar -xvf ${EXT_TMP_DIR}/svu.tar.gz --directory ${EXT_BIN_DIR} svu &> /dev/null
@chmod +x ${EXT_BIN_DIR}/svu
@${EXT_BIN_DIR}/svu --version

.PHONY: install-svu-darwin
install-svu-darwin: ${EXT_TMP_DIR} ${EXT_BIN_DIR}
@echo -e "$(ATTN_COLOR)==> $@ $(NO_COLOR)"
@gh release download v${SVU_VER} --repo https://github.com/caarlos0/svu --pattern "*darwin_all.tar.gz" --output "${EXT_TMP_DIR}/svu.tar.gz" --clobber
@tar -xvf ${EXT_TMP_DIR}/svu.tar.gz --directory ${EXT_BIN_DIR} svu &> /dev/null

.PHONY: install-svu-linux
install-svu-linux: ${EXT_TMP_DIR} ${EXT_BIN_DIR}
@echo -e "$(ATTN_COLOR)==> $@ $(NO_COLOR)"
@gh release download v${SVU_VER} --repo https://github.com/caarlos0/svu --pattern "*linux_${GOARCH}.tar.gz" --output "${EXT_TMP_DIR}/svu.tar.gz" --clobber
@tar -xvf ${EXT_TMP_DIR}/svu.tar.gz --directory ${EXT_BIN_DIR} svu &> /dev/null

.PHONY: install-gotestsum
install-gotestsum: ${EXT_TMP_DIR} ${EXT_BIN_DIR}
@echo -e "$(ATTN_COLOR)==> $@ $(NO_COLOR)"
Expand Down
9 changes: 5 additions & 4 deletions pkg/bdb/boltdb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/pkg/errors"
"github.com/rs/zerolog"
bolt "go.etcd.io/bbolt"
berr "go.etcd.io/bbolt/errors"
"google.golang.org/grpc/codes"
)

Expand All @@ -27,8 +28,8 @@ var (
type Config struct {
DBPath string
RequestTimeout time.Duration
MaxBatchSize int
MaxBatchDelay time.Duration
MaxBatchSize int `json:"-"` // obsolete bbolt configuration value.
MaxBatchDelay time.Duration `json:"-"` // obsolete bbolt configuration value.
}

// BoltDB based key-value store.
Expand Down Expand Up @@ -147,7 +148,7 @@ func DeleteBucket(tx *bolt.Tx, path Path) error {
if len(path) == 1 {
err := tx.DeleteBucket([]byte(path[0]))
switch {
case errors.Is(err, bolt.ErrBucketNotFound):
case errors.Is(err, berr.ErrBucketNotFound):
return nil
case err != nil:
return err
Expand All @@ -163,7 +164,7 @@ func DeleteBucket(tx *bolt.Tx, path Path) error {

err = b.DeleteBucket([]byte(path[len(path)-1]))
switch {
case errors.Is(err, bolt.ErrBucketNotFound):
case errors.Is(err, berr.ErrBucketNotFound):
return nil
case err != nil:
return err
Expand Down
5 changes: 3 additions & 2 deletions pkg/bdb/migrate/mig/mig.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (

"github.com/Masterminds/semver/v3"
bolt "go.etcd.io/bbolt"
berr "go.etcd.io/bbolt/errors"
)

const (
Expand Down Expand Up @@ -95,7 +96,7 @@ func DeleteBucket(path bdb.Path) func(*zerolog.Logger, *bolt.DB, *bolt.DB) error
if len(path) == 1 {
err := tx.DeleteBucket([]byte(path[0]))
switch {
case errors.Is(err, bolt.ErrBucketNotFound):
case errors.Is(err, berr.ErrBucketNotFound):
return nil
case err != nil:
return err
Expand All @@ -111,7 +112,7 @@ func DeleteBucket(path bdb.Path) func(*zerolog.Logger, *bolt.DB, *bolt.DB) error

err = b.DeleteBucket([]byte(path[len(path)-1]))
switch {
case errors.Is(err, bolt.ErrBucketNotFound):
case errors.Is(err, berr.ErrBucketNotFound):
return nil
case err != nil:
return err
Expand Down
5 changes: 3 additions & 2 deletions pkg/bdb/scanner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/aserto-dev/azm/graph"
dsc3 "github.com/aserto-dev/go-directory/aserto/directory/common/v3"
"github.com/aserto-dev/go-edge-ds/pkg/x"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
bolt "go.etcd.io/bbolt"
Expand All @@ -14,7 +15,7 @@ import (
type Iterator[T any, M Message[T]] interface {
Next() bool // move cursor to next element.
RawKey() []byte // return raw key value ([]byte).
RawValue() []byte // return ram value value ([]byte).
RawValue() []byte // return raw value value ([]byte).
Key() string // return key value (string).
Value() M // return typed value (M).
Delete() error // delete element underneath cursor.
Expand Down Expand Up @@ -57,7 +58,7 @@ func WithKeyFilter(filter []byte) ScanOption {
}

func NewScanIterator[T any, M Message[T]](ctx context.Context, tx *bolt.Tx, path Path, opts ...ScanOption) (Iterator[T, M], error) {
args := &ScanArgs{startToken: nil, keyFilter: nil, pageSize: 100}
args := &ScanArgs{startToken: nil, keyFilter: nil, pageSize: x.MaxPageSize}
for _, opt := range opts {
opt(args)
}
Expand Down
5 changes: 0 additions & 5 deletions pkg/directory/directory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (

"github.com/Masterminds/semver/v3"
"github.com/rs/zerolog"
bolt "go.etcd.io/bbolt"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
Expand Down Expand Up @@ -78,8 +77,6 @@ func newDirectory(_ context.Context, config *Config, logger *zerolog.Logger) (*D
cfg := bdb.Config{
DBPath: config.DBPath,
RequestTimeout: config.RequestTimeout,
MaxBatchSize: bolt.DefaultMaxBatchSize,
MaxBatchDelay: bolt.DefaultMaxBatchDelay,
}

if ok, err := migrate.CheckSchemaVersion(&cfg, logger, semver.MustParse(schemaVersion)); !ok {
Expand All @@ -102,8 +99,6 @@ func newDirectory(_ context.Context, config *Config, logger *zerolog.Logger) (*D
store, err := bdb.New(&bdb.Config{
DBPath: config.DBPath,
RequestTimeout: config.RequestTimeout,
MaxBatchSize: bolt.DefaultMaxBatchSize,
MaxBatchDelay: bolt.DefaultMaxBatchDelay,
},
&newLogger,
)
Expand Down
6 changes: 3 additions & 3 deletions pkg/directory/v3/reader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/aserto-dev/go-directory/pkg/validator"
"github.com/aserto-dev/go-edge-ds/pkg/bdb"
"github.com/aserto-dev/go-edge-ds/pkg/ds"
"github.com/aserto-dev/go-edge-ds/pkg/x"
"github.com/pkg/errors"

"github.com/go-http-utils/headers"
Expand Down Expand Up @@ -126,7 +127,7 @@ func (s *Reader) GetObjects(ctx context.Context, req *dsr3.GetObjectsRequest) (*
}

if req.Page == nil {
req.Page = &dsc3.PaginationRequest{Size: 100}
req.Page = &dsc3.PaginationRequest{Size: x.MaxPageSize}
}

opts := []bdb.ScanOption{
Expand Down Expand Up @@ -201,7 +202,6 @@ func (s *Reader) GetRelation(ctx context.Context, req *dsr3.GetRelationRequest)
resp.Result = dbRel

inMD, _ := grpcmd.FromIncomingContext(ctx)
// optimistic concurrency check
if lo.Contains(inMD.Get(headers.IfNoneMatch), dbRel.Etag) {
_ = grpc.SetHeader(ctx, grpcmd.Pairs("x-http-code", "304"))

Expand Down Expand Up @@ -246,7 +246,7 @@ func (s *Reader) GetRelations(ctx context.Context, req *dsr3.GetRelationsRequest
}

if req.Page == nil {
req.Page = &dsc3.PaginationRequest{Size: 100}
req.Page = &dsc3.PaginationRequest{Size: x.MaxPageSize}
}

getRelations := ds.GetRelations(req)
Expand Down
18 changes: 3 additions & 15 deletions pkg/ds/relation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,8 @@ func (i *relation) SubFilter(buf *bytes.Buffer) {
buf.WriteString(i.GetObjectId())
}

// nolint: gocritic
const relationFilterCount int = 6

func (i *relation) Filter(keyFilter *bytes.Buffer) (path bdb.Path, valueFilter func(*dsc3.RelationIdentifier) bool) {
// #1 determine if object identifier is complete (has type+id)
// set index path accordingly
Expand All @@ -195,44 +196,39 @@ func (i *relation) Filter(keyFilter *bytes.Buffer) (path bdb.Path, valueFilter f
}

// #2 build valueFilter function
filters := make([]func(item *dsc3.RelationIdentifier) bool, 0, 6)
filters := make([]func(item *dsc3.RelationIdentifier) bool, 0, relationFilterCount)

if fv := i.GetObjectType(); fv != "" {
filters = append(filters, func(item *dsc3.RelationIdentifier) bool {
equal := strings.Compare(item.GetObjectType(), fv)
// log.Trace().Str("fv", fv).Str("item", item.GetObjectType()).Bool("equal", equal == 0).Msg("object_type filter")
return equal == 0
})
}

if fv := i.GetObjectId(); fv != "" {
filters = append(filters, func(item *dsc3.RelationIdentifier) bool {
equal := strings.Compare(fv, item.GetObjectId())
// log.Trace().Str("fv", fv).Str("item", item.GetObjectId()).Bool("equal", equal == 0).Msg("object_id filter")
return equal == 0
})
}

if fv := i.GetRelation(); fv != "" {
filters = append(filters, func(item *dsc3.RelationIdentifier) bool {
equal := strings.Compare(item.Relation, fv)
// log.Trace().Str("fv", fv).Str("item", item.Relation).Bool("equal", equal == 0).Msg("relation filter")
return equal == 0
})
}

if fv := i.GetSubjectType(); fv != "" {
filters = append(filters, func(item *dsc3.RelationIdentifier) bool {
equal := strings.Compare(item.GetSubjectType(), fv)
// log.Trace().Str("fv", fv).Str("item", item.GetSubjectType()).Bool("equal", equal == 0).Msg("subject_type filter")
return equal == 0
})
}

if fv := i.GetSubjectId(); fv != "" {
filters = append(filters, func(item *dsc3.RelationIdentifier) bool {
equal := strings.Compare(fv, item.GetSubjectId())
// log.Trace().Str("fv", fv).Str("item", item.GetSubjectId()).Bool("equal", equal == 0).Msg("subject_id filter")
return equal == 0
})
}
Expand All @@ -241,7 +237,6 @@ func (i *relation) Filter(keyFilter *bytes.Buffer) (path bdb.Path, valueFilter f
fv := i.GetSubjectRelation()
filters = append(filters, func(item *dsc3.RelationIdentifier) bool {
equal := strings.Compare(item.SubjectRelation, fv)
// log.Trace().Str("fv", fv).Str("item", item.SubjectRelation).Bool("equal", equal == 0).Msg("subject_relation filter")
return equal == 0
})
}
Expand All @@ -258,7 +253,6 @@ func (i *relation) Filter(keyFilter *bytes.Buffer) (path bdb.Path, valueFilter f
return path, valueFilter
}

// nolint: gocritic // commentedOutCode
func (i *relation) RelationValueFilter(keyFilter *bytes.Buffer) (path bdb.Path, valueFilter func(*dsc3.Relation) bool) {
// #1 determine if object identifier is complete (has type+id)
// set index path accordingly
Expand All @@ -282,39 +276,34 @@ func (i *relation) RelationValueFilter(keyFilter *bytes.Buffer) (path bdb.Path,
if fv := i.GetObjectType(); fv != "" {
filters = append(filters, func(item *dsc3.Relation) bool {
equal := strings.Compare(item.GetObjectType(), fv)
// log.Trace().Str("fv", fv).Str("item", item.GetObjectType()).Bool("equal", equal == 0).Msg("object_type filter")
return equal == 0
})
}

if fv := i.GetObjectId(); fv != "" {
filters = append(filters, func(item *dsc3.Relation) bool {
equal := strings.Compare(fv, item.GetObjectId())
// log.Trace().Str("fv", fv).Str("item", item.GetObjectId()).Bool("equal", equal == 0).Msg("object_id filter")
return equal == 0
})
}

if fv := i.GetRelation(); fv != "" {
filters = append(filters, func(item *dsc3.Relation) bool {
equal := strings.Compare(item.Relation, fv)
// log.Trace().Str("fv", fv).Str("item", item.Relation).Bool("equal", equal == 0).Msg("relation filter")
return equal == 0
})
}

if fv := i.GetSubjectType(); fv != "" {
filters = append(filters, func(item *dsc3.Relation) bool {
equal := strings.Compare(item.GetSubjectType(), fv)
// log.Trace().Str("fv", fv).Str("item", item.GetSubjectType()).Bool("equal", equal == 0).Msg("subject_type filter")
return equal == 0
})
}

if fv := i.GetSubjectId(); fv != "" {
filters = append(filters, func(item *dsc3.Relation) bool {
equal := strings.Compare(fv, item.GetSubjectId())
// log.Trace().Str("fv", fv).Str("item", item.GetSubjectId()).Bool("equal", equal == 0).Msg("subject_id filter")
return equal == 0
})
}
Expand All @@ -323,7 +312,6 @@ func (i *relation) RelationValueFilter(keyFilter *bytes.Buffer) (path bdb.Path,
fv := i.GetSubjectRelation()
filters = append(filters, func(item *dsc3.Relation) bool {
equal := strings.Compare(item.SubjectRelation, fv)
// log.Trace().Str("fv", fv).Str("item", item.SubjectRelation).Bool("equal", equal == 0).Msg("subject_relation filter")
return equal == 0
})
}
Expand Down
Loading