diff --git a/models/yang/annotations/openconfig-system-annot.yang b/models/yang/annotations/openconfig-system-annot.yang index 7b9accc14..ef5b9518e 100644 --- a/models/yang/annotations/openconfig-system-annot.yang +++ b/models/yang/annotations/openconfig-system-annot.yang @@ -8,6 +8,7 @@ module openconfig-system-annot { import openconfig-system { prefix oc-sys; } import sonic-extensions {prefix sonic-ext; } import openconfig-system-grpc { prefix oc-sys-grpc; } + import gnsi-pathz { prefix gnsi-pathz; } deviation /oc-sys:system/oc-sys:ssh-server/oc-sys:state { deviate add { @@ -22,5 +23,11 @@ module openconfig-system-annot { sonic-ext:subtree-transformer "grpc_server_xfmr"; } } -} + deviation /oc-sys:system/gnsi-pathz:gnmi-pathz-policies { + deviate add { + sonic-ext:key-transformer "pathz_policies_key_xfmr"; + sonic-ext:subtree-transformer "pathz_policies_xfmr"; + } + } +} diff --git a/translib/transformer/xfmr_system.go b/translib/transformer/xfmr_system.go index 0e0ff52ec..efcab7364 100644 --- a/translib/transformer/xfmr_system.go +++ b/translib/transformer/xfmr_system.go @@ -5,6 +5,8 @@ import ( "fmt" "strconv" "strings" + "sync" + "time" "github.com/Azure/sonic-mgmt-common/translib/db" "github.com/Azure/sonic-mgmt-common/translib/ocbinds" @@ -14,7 +16,11 @@ import ( ) const ( - GNXI_ID = "gnxi" + PATHZ_TBL = "PATHZ_TABLE" + READS_GET = "get" + READS_SUB = "subscribe" + WRITES = "set" + GNXI_ID = "gnxi" /** Credential Tables **/ CREDENTIALS_TBL = "CREDENTIALS" @@ -44,6 +50,22 @@ const ( AUTHZ_SUCCESS_TIMESTAMP = AUTHZ_STATE + "/last-access-accept" AUTHZ_FAILED = AUTHZ_STATE + "/access-rejects" AUTHZ_FAILED_TIMESTAMP = AUTHZ_STATE + "/last-access-reject" + PATHZ_POLICY_COUNTERS = GRPC_SERVER + "/gnsi-pathz:gnmi-pathz-policy-counters" + ALL_PATHZ = PATHZ_POLICY_COUNTERS + "/paths" + SINGLE_PATHZ = ALL_PATHZ + "/path" + + PATHZ_STATE = SINGLE_PATHZ + "/state" + PATHZ_READS = PATHZ_STATE + "/reads" + PATHZ_WRITES = PATHZ_STATE + "/writes" + + PATHZ_READ_SUCCESS = PATHZ_READS + "/access-accepts" + PATHZ_READ_SUCCESS_TIMESTAMP = PATHZ_READS + "/last-access-accept" + PATHZ_READ_FAILED = PATHZ_READS + "/access-rejects" + PATHZ_READ_FAILED_TIMESTAMP = PATHZ_READS + "/last-access-reject" + PATHZ_WRITE_SUCCESS = PATHZ_WRITES + "/access-accepts" + PATHZ_WRITE_SUCCESS_TIMESTAMP = PATHZ_WRITES + "/last-access-accept" + PATHZ_WRITE_FAILED = PATHZ_WRITES + "/access-rejects" + PATHZ_WRITE_FAILED_TIMESTAMP = PATHZ_WRITES + "/last-access-reject" ) type sshState struct { @@ -65,6 +87,23 @@ type certData struct { created uint64 } +// XfmrCache a sync.Map for storing path values that need to be cached +var XfmrCache sync.Map + +var pathzOpers = [][]string{ + []string{READS_GET, ACCEPTS}, + []string{READS_GET, REJECTS}, + []string{READS_SUB, ACCEPTS}, + []string{READS_SUB, REJECTS}, + []string{WRITES, ACCEPTS}, + []string{WRITES, REJECTS}} + +var pathzMap = &pathzCounters{ + mu: sync.Mutex{}, + updated: make(map[string]time.Time), + data: make(map[string]map[string]map[string]*uint64), +} + func init() { XlateFuncBind("DbToYang_grpc_server_xfmr", DbToYang_grpc_server_xfmr) XlateFuncBind("Subscribe_grpc_server_xfmr", Subscribe_grpc_server_xfmr) @@ -73,6 +112,9 @@ func init() { XlateFuncBind("Subscribe_ssh_server_state_xfmr", Subscribe_ssh_server_state_xfmr) XlateFuncBind("DbToYang_authz_policy_xfmr", DbToYang_authz_policy_xfmr) XlateFuncBind("Subscribe_authz_policy_xfmr", Subscribe_authz_policy_xfmr) + XlateFuncBind("DbToYang_pathz_policies_xfmr", DbToYang_pathz_policies_xfmr) + XlateFuncBind("Subscribe_pathz_policies_xfmr", Subscribe_pathz_policies_xfmr) + XlateFuncBind("DbToYang_pathz_policies_key_xfmr", DbToYang_pathz_policies_key_xfmr) } type grpcState struct { @@ -90,6 +132,23 @@ type grpcState struct { pathzCreated uint64 } +type pathzCounters struct { + mu sync.Mutex + updated map[string]time.Time + data map[string]map[string]map[string]*uint64 +} + +type policyState struct { + instance ocbinds.E_OpenconfigSystem_System_GnmiPathzPolicies_Policies_Policy_State_Instance + version string + created uint64 +} + +var dbToYangPathzInstanceMap = map[string]ocbinds.E_OpenconfigSystem_System_GnmiPathzPolicies_Policies_Policy_State_Instance{ + "ACTIVE": ocbinds.OpenconfigSystem_System_GnmiPathzPolicies_Policies_Policy_State_Instance_ACTIVE, + "SANDBOX": ocbinds.OpenconfigSystem_System_GnmiPathzPolicies_Policies_Policy_State_Instance_SANDBOX, +} + func getAppRootObject(inParams XfmrParams) *ocbinds.OpenconfigSystem_System { deviceObj := (*inParams.ygRoot).(*ocbinds.Device) return deviceObj.System @@ -218,10 +277,145 @@ var DbToYang_authz_policy_xfmr SubTreeXfmrDbToYang = func(inParams XfmrParams) e return nil } -var DbToYang_grpc_server_key_xfmr KeyXfmrDbToYang = func(inParams XfmrParams) (map[string]interface{}, error) { - log.V(3).Info("DbToYang_grpc_server_key_xfmr root, uri: ", inParams.ygRoot, inParams.uri) +func (m *pathzCounters) getCounters(pathzTables db.Table, xpath string) map[string]map[string]*uint64 { + result := make(map[string]map[string]*uint64) + m.mu.Lock() + defer m.mu.Unlock() + if m.updated == nil || m.data == nil { + m.updated = make(map[string]time.Time) + m.data = make(map[string]map[string]map[string]*uint64) + } - return map[string]interface{}{"name": NewPathInfo(inParams.uri).Var("name")}, nil + // Update the map if necessary + updateTime, ok := m.updated[xpath] + if !ok { + result = GetPathzPolicyCounter(pathzTables, xpath) + if len(m.data) < 50 { + m.data[xpath] = result + m.updated[xpath] = time.Now() + } + } else if time.Now().After(updateTime.Add(30 * time.Second)) { + m.data[xpath] = GetPathzPolicyCounter(pathzTables, xpath) + m.updated[xpath] = time.Now() + } + + // Fetch the result or return the previously calculated result + if data, ok := m.data[xpath]; ok { + result = data + } + return result +} + +func GetPathzPolicyCounter(pathzTables db.Table, path string) map[string]map[string]*uint64 { + cntMap := make(map[string]*uint64) + tsMap := make(map[string]*uint64) + + for _, tmp := range pathzOpers { + pattern := PatternGenerator(tmp, path) + if pattern == "" { + log.V(3).Infof("Invalid pathz counter key pattern.") + continue + } + key := db.NewKey(tmp[0], path, tmp[1]) + + // Sum the data collected + value, err := pathzTables.GetEntry(*key) + if err != nil { + log.V(tlerr.ErrorSeverity(err)).Infof("Cannot get value from %v table for %v, err: %v", PATHZ_TBL, key, err) + continue + } + + c := value.Get("count") + if c == "" { + continue + } + dbCnt, err := strconv.ParseUint(c, 10, 64) + if err != nil { + log.V(tlerr.ErrorSeverity(err)).Infof("Failed to convert counters from DB for pathz, err: %v", err) + continue + } + tsval := value.Get("timestamp") + if tsval == "" { + continue + } + dbTs, err := strconv.ParseUint(tsval, 10, 64) + if err != nil { + log.V(tlerr.ErrorSeverity(err)).Infof("Failed to convert timestamp for counters from DB for pathz, err: %v", err) + continue + } + + cnt, cntExists := cntMap[pattern] + if cntExists && cnt != nil { + cntUpdate, err := strconv.ParseUint(strconv.FormatUint((*cnt+dbCnt), 10), 10, 64) + if err != nil { + log.V(tlerr.ErrorSeverity(err)).Infof("Failed to convert counters for pathz, err: %v", err) + continue + } + cntMap[pattern] = &cntUpdate + } else { + cntMap[pattern] = &dbCnt + } + + ts, tsExists := tsMap[pattern] + if !tsExists || ts == nil || *ts < dbTs { + tsMap[pattern] = &dbTs + } + } + return map[string]map[string]*uint64{cntResult: cntMap, tsResult: tsMap} +} + +func getAllXpaths(pathzTables db.Table) ([]string, error) { + var res []string + check := make(map[string]bool) + pathzTableKeys, err := pathzTables.GetKeys() + if err != nil { + log.V(tlerr.ErrorSeverity(err)).Infof("Cannot get all keys from %v table, err: %v", PATHZ_TBL, err) + return []string{}, err + } + for _, pathzTableKey := range pathzTableKeys { + if len(pathzTableKey.Comp) != 3 { + log.V(3).Infof("invalid number of Comps for pathzTableKey %v.", pathzTableKey) + continue + } + if pathzTableKey.Comp[1] != "" { + key := pathzTableKey.Comp[1] + if val, ok := check[key]; !ok || !val { + res = append(res, key) + check[key] = true + } + } + } + + return res, nil +} + +var pathToPatternKeysMap = map[string][]string{ + PATHZ_READ_SUCCESS: []string{"reads", ACCEPTS}, + PATHZ_READ_SUCCESS_TIMESTAMP: []string{"reads", ACCEPTS}, + PATHZ_READ_FAILED: []string{"reads", REJECTS}, + PATHZ_READ_FAILED_TIMESTAMP: []string{"reads", REJECTS}, + PATHZ_WRITE_SUCCESS: []string{"writes", ACCEPTS}, + PATHZ_WRITE_SUCCESS_TIMESTAMP: []string{"writes", ACCEPTS}, + PATHZ_WRITE_FAILED: []string{"writes", REJECTS}, + PATHZ_WRITE_FAILED_TIMESTAMP: []string{"writes", REJECTS}, +} + +func PatternGenerator(params []string, xpath string) string { + if len(params) != 2 { + log.V(3).Infof("Invalid params for patternGenerator %#v", params) + return "" + } + + if params[0] == READS_GET || params[0] == READS_SUB || params[0] == "reads" { + return "*|reads|" + xpath + "|" + params[1] + } + + if params[0] == WRITES || params[0] == "writes" { + return "*|writes|" + xpath + "|" + params[1] + } + + log.V(3).Infof("Invalid operation %v", params[0]) + return "" } var Subscribe_grpc_server_xfmr SubTreeXfmrSubscribe = func(inParams XfmrSubscInParams) (XfmrSubscOutParams, error) { @@ -238,8 +432,8 @@ var Subscribe_grpc_server_xfmr SubTreeXfmrSubscribe = func(inParams XfmrSubscInP result.dbDataMap = RedisDbSubscribeMap{ db.StateDB: map[string]map[string]map[string]string{ CREDENTIALS_TBL: { - "CERT|gnxi": {}, - }, + "CERT|gnxi": {}, + "PATHZ_POLICY|ACTIVE": {}}, }, } } else { @@ -247,8 +441,8 @@ var Subscribe_grpc_server_xfmr SubTreeXfmrSubscribe = func(inParams XfmrSubscInP dbDataMap: RedisDbSubscribeMap{ db.StateDB: map[string]map[string]map[string]string{ CREDENTIALS_TBL: { - "CERT|gnxi": {}, - }, + "CERT|gnxi": {}, + "PATHZ_POLICY|ACTIVE": {}}, }}, } } @@ -265,6 +459,65 @@ var Subscribe_grpc_server_xfmr SubTreeXfmrSubscribe = func(inParams XfmrSubscInP return result, nil } +var DbToYang_pathz_policies_xfmr SubTreeXfmrDbToYang = func(inParams XfmrParams) error { + pathInfo := NewPathInfo(inParams.uri) + instances := []string{pathInfo.Var("instance")} + targetUriPath, _ := getYangPathFromUri(pathInfo.Path) + log.V(3).Infof("DbToYang_pathz_policies_xfmr: targetUriPath: %s instances: %v", targetUriPath, instances) + + stateDb := inParams.dbs[db.StateDB] + if len(instances) == 0 || len(instances[0]) == 0 { + var err error + if instances, err = getAllKeys(stateDb, CRED_PATHZ_TBL); err != nil { + return err + } + } + sysObj := getAppRootObject(inParams) + ygot.BuildEmptyTree(sysObj) + ygot.BuildEmptyTree(sysObj.GnmiPathzPolicies) + ygot.BuildEmptyTree(sysObj.GnmiPathzPolicies.Policies) + + for _, instance := range instances { + log.V(3).Infof("instance: %v", instance) + i, ok := dbToYangPathzInstanceMap[instance] + if !ok { + log.V(0).Infof("Pathz Policy Instance not found: %v", instance) + continue + } + policyObj, ok := sysObj.GnmiPathzPolicies.Policies.Policy[i] + if !ok { + var err error + policyObj, err = sysObj.GnmiPathzPolicies.Policies.NewPolicy(i) + if err != nil { + log.V(0).Infof("sysObj.GnmiPathzPolicies.Policies.NewPolicy failed: %v", err) + continue + } + } + table, err := stateDb.GetEntry(&db.TableSpec{Name: CRED_PATHZ_TBL}, db.Key{Comp: []string{instance}}) + if err != nil { + log.V(0).Infof("Failed to read from StateDB %v, id: %v, err: %v", inParams.table, instance, err) + return err + } + var state policyState + + state.instance = i + state.version = table.Get("pathz_version") + time := table.Get("pathz_created_on") + if state.created, err = strconv.ParseUint(time, 10, 64); err != nil && time != "" { + return err + } + ygot.BuildEmptyTree(policyObj) + policyObj.State.Instance = state.instance + policyObj.State.CreatedOn = &state.created + policyObj.State.Version = &state.version + } + return nil +} +var DbToYang_grpc_server_key_xfmr KeyXfmrDbToYang = func(inParams XfmrParams) (map[string]interface{}, error) { + log.V(3).Info("DbToYang_grpc_server_key_xfmr root, uri: ", inParams.ygRoot, inParams.uri) + + return map[string]interface{}{"name": NewPathInfo(inParams.uri).Var("name")}, nil +} var DbToYang_grpc_server_xfmr SubTreeXfmrDbToYang = func(inParams XfmrParams) error { pathInfo := NewPathInfo(inParams.uri) @@ -336,7 +589,6 @@ var DbToYang_grpc_server_xfmr SubTreeXfmrDbToYang = func(inParams XfmrParams) er } } } - serverObj, ok := sysObj.GrpcServers.GrpcServer[serverName] if !ok { serverObj, err = sysObj.GrpcServers.NewGrpcServer(serverName) @@ -430,9 +682,106 @@ var DbToYang_grpc_server_xfmr SubTreeXfmrDbToYang = func(inParams XfmrParams) er } } } + // Pathz counter is for GNXI_ID only + if serverName != GNXI_ID { + continue + } + + // Pathz counter + pathzTables, err := stateDb.GetTable(&db.TableSpec{Name: PATHZ_TBL}) + if err != nil { + log.V(tlerr.ErrorSeverity(err)).Infof("getPathzPolicyCounter failed to get PATHZ_TBL, err: %v", err) + } + + xpath := pathInfo.Var("xpath") + xpaths := []string{xpath} + + if xpath == "" || xpath == "*" { + xpaths = []string{} + xpaths, err = getAllXpaths(pathzTables) + if err != nil { + log.V(tlerr.ErrorSeverity(err)).Infof("Failed get all paths, err: %v", err) + } + } + + ygot.BuildEmptyTree(serverObj.GnmiPathzPolicyCounters) + for _, xpath := range xpaths { + // Processing these counters is hard on the CPU. We will only update these counters every 30 seconds. + pathzPolicyData := pathzMap.getCounters(pathzTables, xpath) + + pathObj, ok := serverObj.GnmiPathzPolicyCounters.Paths.Path[xpath] + if !ok { + pathObj, err = serverObj.GnmiPathzPolicyCounters.Paths.NewPath(xpath) + if err != nil { + log.V(0).Infof("serverObj.GnmiPathzPolicyCounters.NewPath(%v) failed: %v", xpath, err) + continue + } + } + ygot.BuildEmptyTree(pathObj) + + // If targetUriPath is a parent PATHZ_STATE, i.e.root path, all counters and timestamps should be returned + allPathzCounter := strings.HasPrefix(PATHZ_STATE, targetUriPath) || targetUriPath == GRPC_OC_SERVERS + + tmpCnt := make(map[string]*uint64) + tmpTs := make(map[string]*uint64) + if cnt, ok := pathzPolicyData[cntResult]; ok { + tmpCnt = cnt + } + if ts, ok := pathzPolicyData[tsResult]; ok { + tmpTs = ts + } + + // Handle root paths here. + if allPathzCounter || targetUriPath == PATHZ_READS || targetUriPath == PATHZ_WRITES { + ygot.BuildEmptyTree(pathObj.State) + if allPathzCounter || targetUriPath == PATHZ_READS { + pathObj.State.Reads.AccessAccepts = tmpCnt[PatternGenerator(pathToPatternKeysMap[PATHZ_READ_SUCCESS], xpath)] + pathObj.State.Reads.LastAccessAccept = tmpTs[PatternGenerator(pathToPatternKeysMap[PATHZ_READ_SUCCESS_TIMESTAMP], xpath)] + pathObj.State.Reads.AccessRejects = tmpCnt[PatternGenerator(pathToPatternKeysMap[PATHZ_READ_FAILED], xpath)] + pathObj.State.Reads.LastAccessReject = tmpTs[PatternGenerator(pathToPatternKeysMap[PATHZ_READ_FAILED_TIMESTAMP], xpath)] + } + if allPathzCounter || targetUriPath == PATHZ_WRITES { + pathObj.State.Writes.AccessAccepts = tmpCnt[PatternGenerator(pathToPatternKeysMap[PATHZ_WRITE_SUCCESS], xpath)] + pathObj.State.Writes.LastAccessAccept = tmpTs[PatternGenerator(pathToPatternKeysMap[PATHZ_WRITE_SUCCESS_TIMESTAMP], xpath)] + pathObj.State.Writes.AccessRejects = tmpCnt[PatternGenerator(pathToPatternKeysMap[PATHZ_WRITE_FAILED], xpath)] + pathObj.State.Writes.LastAccessReject = tmpTs[PatternGenerator(pathToPatternKeysMap[PATHZ_WRITE_FAILED_TIMESTAMP], xpath)] + } + } else { + // Handle leaf paths here. + patternKeys := pathToPatternKeysMap[targetUriPath] + if patternKeys == nil { + log.V(0).Infof("Invalid pathz table key: %#v", targetUriPath) + continue + } + pattern := PatternGenerator([]string{patternKeys[0], patternKeys[1]}, xpath) + + switch targetUriPath { + case PATHZ_READ_SUCCESS: + pathObj.State.Reads.AccessAccepts = tmpCnt[pattern] + case PATHZ_READ_SUCCESS_TIMESTAMP: + pathObj.State.Reads.LastAccessAccept = tmpTs[pattern] + case PATHZ_READ_FAILED: + pathObj.State.Reads.AccessRejects = tmpCnt[pattern] + case PATHZ_READ_FAILED_TIMESTAMP: + pathObj.State.Reads.LastAccessReject = tmpTs[pattern] + case PATHZ_WRITE_SUCCESS: + pathObj.State.Writes.AccessAccepts = tmpCnt[pattern] + case PATHZ_WRITE_SUCCESS_TIMESTAMP: + pathObj.State.Writes.LastAccessAccept = tmpTs[pattern] + case PATHZ_WRITE_FAILED: + pathObj.State.Writes.AccessRejects = tmpCnt[pattern] + case PATHZ_WRITE_FAILED_TIMESTAMP: + pathObj.State.Writes.LastAccessReject = tmpTs[pattern] + } + } + } } return nil } +var DbToYang_pathz_policies_key_xfmr KeyXfmrDbToYang = func(inParams XfmrParams) (map[string]interface{}, error) { + log.V(3).Info("DbToYang_pathz_policies_key_xfmr root, uri: ", inParams.ygRoot, inParams.uri) + return map[string]interface{}{"instance": NewPathInfo(inParams.uri).Var("instance")}, nil +} func getAuthzPolicyCounter(authzTables db.Table, server string, rpcString string) map[string]map[string]*uint64 { cntMap := make(map[string]*uint64) @@ -512,3 +861,21 @@ func getAllRpcs(authzTables db.Table, server string) ([]string, error) { return res, nil } + +var Subscribe_pathz_policies_xfmr SubTreeXfmrSubscribe = func(inParams XfmrSubscInParams) (XfmrSubscOutParams, error) { + pathInfo := NewPathInfo(inParams.uri) + instance := pathInfo.Var("instance") + if instance == "" { + instance = "*" + } + targetUriPath, _ := getYangPathFromUri(pathInfo.Path) + log.V(3).Infof("Subscribe_pathz_policies_xfmr: targetUriPath: %s instance: %s", targetUriPath, instance) + + key := strings.Join([]string{"PATHZ_POLICY", instance}, "|") + return XfmrSubscOutParams{ + dbDataMap: RedisDbSubscribeMap{ + db.StateDB: {CREDENTIALS_TBL: {key: {}}}}, + onChange: OnchangeEnable, + nOpts: ¬ificationOpts{mInterval: 0, pType: OnChange}, + }, nil +}