confdb, overlord/confdbstate: add visibility-based pruning to view.Get and authorization checks to view.Set and view.Unset (#16400)

* confdb: add pre-get pruning

* confdb, o/confdbstate: add visibility check during set

* confdb, overlord/confdbstate: add secrets check to unset

* confdb: minor - return nil instead of empty slice

* confdb: add comment in set explaining approach

Author: maykathm

confdb/confdb.go

@@ -1105,7 +1105,9 @@ func validateSetValue(initial any) error {
 
 // Set sets the named view to a specified non-nil value. Matches that cannot
 // extract their data from the provided value will be considered as being unset.
-func (v *View) Set(databag Databag, request string, value any) error {
+// If the userID does not have sufficient permissions to set the data indicated
+// by the request, then an UnauthorizedAccessError is returned.
+func (v *View) Set(databag Databag, request string, value any, userID int) error {
 	if request == "" {
 		return badRequestErrorFrom(v, "set", request, "")
 	}
@@ -1177,10 +1179,28 @@ func (v *View) Set(databag Databag, request string, value any) error {
 	getAccs = func(i int) []Accessor { return expandedMatches[i].storagePath }
 	sort.Slice(expandedMatches, byAccessor(getAccs))
 
+	visibilities := getVisibilitiesToPrune(userID)
+
 	for _, match := range expandedMatches {
 		if err := databag.Set(match.storagePath, match.value); err != nil {
 			return err
 		}
+		data, err := databag.Data()
+		if err != nil {
+			return err
+		}
+		// we can safely check visibility after setting data in the databag since the
+		// data is only persisted if this Set method returns a non-error
+		pruned, err := v.schema.DatabagSchema.PruneByVisibility(match.storagePath, visibilities, data)
+		if err != nil {
+			if errors.Is(err, &UnauthorizedAccessError{}) {
+				return &UnauthorizedAccessError{viewID: v.ID(), operation: "set", request: request}
+			}
+			return err
+		}
+		if len(pruned) != len(data) {
+			return &UnauthorizedAccessError{viewID: v.ID(), operation: "set", request: request}
+		}
 	}
 
 	data, err := databag.Data()
@@ -1220,7 +1240,10 @@ func byAccessor(getAccs accGetter) func(x, y int) bool {
 	}
 }
 
-func (v *View) Unset(databag Databag, request string) error {
+// Unset unsets the value at request in the named view.
+// If the userID does not have sufficient permissions to unset the data
+// indicated by the request, then an UnauthorizedAccessError is returned
+func (v *View) Unset(databag Databag, request string, userID int) error {
 	opts := ParseOptions{AllowPlaceholders: false}
 	accessors, err := ParsePathIntoAccessors(request, opts)
 	if err != nil {
@@ -1236,7 +1259,25 @@ func (v *View) Unset(databag Databag, request string) error {
 		return NewNoMatchError(v, "unset", []string{request})
 	}
 
+	visibilities := getVisibilitiesToPrune(userID)
 	for _, match := range matches {
+		if len(visibilities) > 0 {
+			data, err := databag.Data()
+			if err != nil {
+				return err
+			}
+			pruned, err := v.schema.DatabagSchema.PruneByVisibility(match.storagePath, visibilities, data)
+			if err != nil {
+				if errors.Is(err, &UnauthorizedAccessError{}) {
+					return &UnauthorizedAccessError{viewID: v.ID(), operation: "unset", request: request}
+				}
+				return err
+			}
+			if len(pruned) != len(data) {
+				return &UnauthorizedAccessError{viewID: v.ID(), operation: "unset", request: request}
+			}
+		}
+
 		if err := databag.Unset(match.storagePath); err != nil {
 			return err
 		}
@@ -1828,11 +1869,21 @@ func (v *View) CheckAllConstraintsAreUsed(requests []string, constraints map[str
 	return NewUnmatchedConstraintsError(v, requests, unusedConstraints)
 }
 
+func getVisibilitiesToPrune(userID int) []Visibility {
+	if userID == 0 {
+		return nil
+	}
+	return []Visibility{SecretVisibility}
+}
+
 // Get returns the view value identified by the request after the constraints
-// have been applied to any matching filter in the storage path. If the request
-// cannot be matched against any rule, it returns a NoMatchError, and if no data
-// is stored for the request, a NoDataError is returned.
-func (v *View) Get(databag Databag, request string, constraints map[string]any) (any, error) {
+// have been applied to any matching filter in the storage path. The userID
+// determines whether data with restrictive visibility levels are returned.
+// If the request cannot be matched against any rule, it returns a NoMatchError,
+// and if no data is stored for the request, a NoDataError is returned. If data
+// would have been returned but was removed due to the visibility level, then a
+// UnauthorizedAccessError is returned.
+func (v *View) Get(databag Databag, request string, constraints map[string]any, userID int) (any, error) {
 	var accessors []Accessor
 	if request != "" {
 		var err error
@@ -1852,9 +1903,35 @@ func (v *View) Get(databag Databag, request string, constraints map[string]any)
 		return nil, err
 	}
 
+	visibilities := getVisibilitiesToPrune(userID)
+	allUnauthorized := len(visibilities) > 0
 	var merged any
 	for _, match := range matches {
-		val, err := databag.Get(match.storagePath, constraints)
+		bag := databag
+		if len(visibilities) > 0 {
+			data, err := databag.Data()
+			if err != nil {
+				return nil, err
+			}
+			data, err = v.schema.DatabagSchema.PruneByVisibility(match.storagePath, visibilities, data)
+			if err != nil {
+				if errors.Is(err, &UnauthorizedAccessError{}) || errors.Is(err, &NoDataError{}) {
+					continue
+				}
+				return nil, err
+			}
+			allUnauthorized = false
+			var decoded map[string]json.RawMessage
+			if err := jsonutil.DecodeWithNumber(bytes.NewReader(data), &decoded); err != nil {
+				_, ok := err.(*json.UnmarshalTypeError)
+				if !ok {
+					return nil, err
+				}
+			}
+			bag = JSONDatabag(decoded)
+		}
+
+		val, err := bag.Get(match.storagePath, constraints)
 		if err != nil {
 			if errors.Is(err, &NoDataError{}) {
 				continue
@@ -1880,6 +1957,9 @@ func (v *View) Get(databag Databag, request string, constraints map[string]any)
 		if request != "" {
 			requests = []string{request}
 		}
+		if allUnauthorized {
+			return nil, &UnauthorizedAccessError{request: request, operation: "get", viewID: v.ID()}
+		}
 		return nil, NewNoDataError(v, requests)
 	}