From d0c29f9ff9f0da959df7f26f8032c964834f595a Mon Sep 17 00:00:00 2001 From: bobvands <40576376+bobvands@users.noreply.github.com> Date: Thu, 17 Jul 2025 08:35:22 -0700 Subject: [PATCH 1/5] Get cache implemenation --- client/client_service.go | 528 +++++++++++++++++++++++++++++++++------ 1 file changed, 448 insertions(+), 80 deletions(-) diff --git a/client/client_service.go b/client/client_service.go index 8659348..763a2b6 100644 --- a/client/client_service.go +++ b/client/client_service.go @@ -1,162 +1,530 @@ package client import ( + "bytes" + "crypto/tls" "errors" "fmt" + "io/ioutil" "log" + "math" + "math/rand/v2" + "net/http" "net/url" + "strconv" + "strings" + "sync" + "time" "github.com/ciscoecosystem/mso-go-client/container" "github.com/ciscoecosystem/mso-go-client/models" + "github.com/hashicorp/go-version" ) -func (c *Client) GetViaURL(endpoint string) (*container.Container, error) { +const msoAuthPayload = `{ + "username": "%s", + "password": "%s" +}` + +const ndAuthPayload = `{ + "userName": "%s", + "userPasswd": "%s" +}` + +const DefaultBackoffMinDelay int = 4 +const DefaultBackoffMaxDelay int = 60 +const DefaultBackoffDelayFactor float64 = 3 + +// Client is the main entry point +type Client struct { + BaseURL *url.URL + httpClient *http.Client + AuthToken *Auth + Mutex sync.Mutex + username string + password string + insecure bool + reqTimeoutSet bool + reqTimeoutVal uint32 + proxyUrl string + domain string + platform string + version string + skipLoggingPayload bool + maxRetries int + backoffMinDelay int + backoffMaxDelay int + backoffDelayFactor float64 +} - req, err := c.MakeRestRequest("GET", endpoint, nil, true) +type CallbackRetryFunc func(*container.Container) bool - if err != nil { - return nil, err +// singleton implementation of a client +var clientImpl *Client + +type Option func(*Client) + +func Insecure(insecure bool) Option { + return func(client *Client) { + client.insecure = insecure } +} - obj, _, err := c.Do(req) - if err != nil { - return nil, err +func Password(password string) Option { + return func(client *Client) { + client.password = password } +} - if obj == nil { - return nil, errors.New("Empty response body") +func ProxyUrl(pUrl string) Option { + return func(client *Client) { + client.proxyUrl = pUrl } - return obj, CheckForErrors(obj, "GET") +} +func Domain(domain string) Option { + return func(client *Client) { + client.domain = domain + } } -func (c *Client) GetPlatform() string { - return c.platform +func Platform(platform string) Option { + return func(client *Client) { + client.platform = platform + } } -func (c *Client) Put(endpoint string, obj models.Model) (*container.Container, error) { - jsonPayload, err := c.PrepareModel(obj) +func Version(version string) Option { + return func(client *Client) { + client.version = version + } +} - if err != nil { - return nil, err +func SkipLoggingPayload(skipLoggingPayload bool) Option { + return func(client *Client) { + client.skipLoggingPayload = skipLoggingPayload } - req, err := c.MakeRestRequest("PUT", endpoint, jsonPayload, true) - if err != nil { - return nil, err +} + +func MaxRetries(maxRetries int) Option { + return func(client *Client) { + client.maxRetries = maxRetries + } +} + +func BackoffMinDelay(backoffMinDelay int) Option { + return func(client *Client) { + client.backoffMinDelay = backoffMinDelay } +} + +func BackoffMaxDelay(backoffMaxDelay int) Option { + return func(client *Client) { + client.backoffMaxDelay = backoffMaxDelay + } +} - c.Mutex.Lock() - cont, _, err := c.Do(req) - c.Mutex.Unlock() +func BackoffDelayFactor(backoffDelayFactor float64) Option { + return func(client *Client) { + client.backoffDelayFactor = backoffDelayFactor + } +} + +func initClient(clientUrl, username string, options ...Option) *Client { + var transport *http.Transport + bUrl, err := url.Parse(clientUrl) if err != nil { - return nil, err + // cannot move forward if url is undefined + log.Fatal(err) + } + client := &Client{ + BaseURL: bUrl, + username: username, + httpClient: http.DefaultClient, + } + + for _, option := range options { + option(client) + } + + transport = client.useInsecureHTTPClient(client.insecure) + if client.proxyUrl != "" { + transport = client.configProxy(transport) } - return cont, CheckForErrors(cont, "PUT") + client.httpClient = &http.Client{ + Transport: transport, + } + + return client +} + +// GetClient returns a singleton +func GetClient(clientUrl, username string, options ...Option) *Client { + if clientImpl == nil { + clientImpl = initClient(clientUrl, username, options...) + } + return clientImpl } -func (c *Client) Save(endpoint string, obj models.Model) (*container.Container, error) { +func (c *Client) configProxy(transport *http.Transport) *http.Transport { + pUrl, err := url.Parse(c.proxyUrl) + if err != nil { + log.Fatal(err) + } + transport.Proxy = http.ProxyURL(pUrl) + return transport +} - jsonPayload, err := c.PrepareModel(obj) +func (c *Client) useInsecureHTTPClient(insecure bool) *http.Transport { + transport := &http.Transport{ + TLSClientConfig: &tls.Config{ + CipherSuites: []uint16{ + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + }, + PreferServerCipherSuites: true, + InsecureSkipVerify: insecure, + MinVersion: tls.VersionTLS11, + MaxVersion: tls.VersionTLS13, + }, + } + return transport +} + +func (c *Client) MakeRestRequest(method string, path string, body *container.Container, authenticated bool) (*http.Request, error) { + origPath := path + if c.platform == "nd" && path != "/login" { + if strings.HasPrefix(path, "/") { + path = path[1:] + } + path = fmt.Sprintf("mso/%v", path) + } + url, err := url.Parse(path) if err != nil { return nil, err } - req, err := c.MakeRestRequest("POST", endpoint, jsonPayload, true) + if method == "PATCH" { + validateString := url.Query() + validateString.Set("validate", "false") + url.RawQuery = validateString.Encode() + } + fURL := c.BaseURL.ResolveReference(url) + var req *http.Request + if method == "GET" || method == "DELETE" { + req, err = http.NewRequest(method, fURL.String(), nil) + } else { + req, err = http.NewRequest(method, fURL.String(), bytes.NewBuffer((body.Bytes()))) + } if err != nil { return nil, err } + if method == "PATCH" || method == "PUT" || method == "DELETE" || method == "POST" { + c.updateCacheForWrite(origPath) + log.Printf("[DEBUG] updating cache for write methods, endpoint %v", origPath) + } + req.Header.Set("Content-Type", "application/json") + log.Printf("[DEBUG] HTTP request %s %s", method, path) - cont, _, err := c.Do(req) - if err != nil { - return nil, err + if authenticated { + + req, err = c.InjectAuthenticationHeader(req, path) + if err != nil { + return req, err + } } + log.Printf("[DEBUG] HTTP request after injection %s %s", method, path) - return cont, CheckForErrors(cont, "POST") + return req, nil } -// CheckForErrors parses the response and checks of there is an error attribute in the response -func CheckForErrors(cont *container.Container, method string) error { +// Authenticate is used to +func (c *Client) Authenticate() error { + method := "POST" + path := "/api/v1/auth/login" + var authPayload string - if cont.Exists("code") && cont.Exists("message") { - return errors.New(fmt.Sprintf("%s%s", cont.S("message"), cont.S("info"))) - } else if cont.Exists("error") { - return errors.New(fmt.Sprintf("%s %s", models.StripQuotes(cont.S("error").String()), models.StripQuotes(cont.S("error_code").String()))) + if c.platform == "nd" { + authPayload = ndAuthPayload + if c.domain == "" { + c.domain = "DefaultAuth" + } + path = "/login" } else { - return nil + authPayload = msoAuthPayload + } + body, err := container.ParseJSON([]byte(fmt.Sprintf(authPayload, c.username, c.password))) + if err != nil { + return err } - return nil -} -func (c *Client) DeletebyId(url string) error { + if c.domain != "" { + if c.platform == "nd" { + body.Set(c.domain, "domain") + } else { + domainId, err := c.GetDomainId(c.domain) + if err != nil { + return err + } + body.Set(domainId, "domainId") + } + } - req, err := c.MakeRestRequest("DELETE", url, nil, true) + c.skipLoggingPayload = true + + req, err := c.MakeRestRequest(method, path, body, false) if err != nil { return err } - _, resp, err1 := c.Do(req) - if err1 != nil { - return err1 + obj, _, err := c.Do(req) + c.skipLoggingPayload = false + if err != nil { + return err } - if resp != nil { - if resp.StatusCode == 204 || resp.StatusCode == 200 { - return nil - } else { - return fmt.Errorf("Unable to delete the object") - } + + if obj == nil { + return errors.New("Empty response") + } + req.Header.Set("Content-Type", "application/json") + + token := models.StripQuotes(obj.S("token").String()) + + if token == "" || token == "{}" { + return errors.New("Invalid Username or Password") } + if c.AuthToken == nil { + c.AuthToken = &Auth{} + } + c.AuthToken.Token = stripQuotes(token) + c.AuthToken.CalculateExpiry(1200) //refreshTime=1200 Sec + return nil } -func (c *Client) PatchbyID(endpoint string, objList ...models.Model) (*container.Container, error) { +func (c *Client) GetDomainId(domain string) (string, error) { + req, err := c.MakeRestRequest("GET", "/api/v1/auth/login-domains", nil, false) + if err != nil { + return "", err + } - contJs := container.New() - contJs.Array() - for _, obj := range objList { - jsonPayload, err := c.PrepareModel(obj) + obj, _, err := c.Do(req) + + if err != nil { + return "", err + } + err = CheckForErrors(obj, "GET") + if err != nil { + return "", err + } + count, err := obj.ArrayCount("domains") + if err != nil { + return "", err + } + + for i := 0; i < count; i++ { + domainCont, err := obj.ArrayElement(i, "domains") if err != nil { - return nil, err + return "", err + } + domainName := models.StripQuotes(domainCont.S("name").String()) + + if domainName == domain { + return models.StripQuotes(domainCont.S("id").String()), nil } - contJs.ArrayAppend(jsonPayload.Data()) + } + return "", fmt.Errorf("Unable to find domain id for domain %s", domain) +} +func (c *Client) GetVersion() (string, error) { + req, err := c.MakeRestRequest("GET", "/api/v1/platform/version", nil, true) + if err != nil { + return "unknown", err } - log.Printf("[DEBUG] Patch Request Container: %v\n", contJs) - // URL encoding - baseUrl, _ := url.Parse(endpoint) - qs := url.Values{} - qs.Add("validate", "false") - baseUrl.RawQuery = qs.Encode() - req, err := c.MakeRestRequest("PATCH", baseUrl.String(), contJs, true) + obj, _, err := c.Do(req) if err != nil { - return nil, err + return "unknown", err } - c.Mutex.Lock() - cont, _, err := c.Do(req) - c.Mutex.Unlock() + err = CheckForErrors(obj, "GET") if err != nil { - return nil, err + return "unknown", err } - return cont, CheckForErrors(cont, "PATCH") + version := stripQuotes(obj.Search("version").String()) + if version == "" { + return "unknown", fmt.Errorf("Unable to identify version") + } + c.version = version + return version, nil } -func (c *Client) PrepareModel(obj models.Model) (*container.Container, error) { - con, err := obj.ToMap() - if err != nil { - return nil, err +// Compares the version to the retrieved version. +// This returns -1, 0, or 1 if this version is smaller, equal, or larger than the retrieved version, respectively. +func (c *Client) CompareVersion(v string) (int, error) { + if c.version == "" { + c.GetVersion() + } + if c.version == "unknown" { + return 0, fmt.Errorf("Could not retrieve version") } - payload := &container.Container{} + v1, err := version.NewVersion(c.version) if err != nil { - return nil, err + return 0, fmt.Errorf("Could not parse retrieved version") } + v2, err := version.NewVersion(v) + if err != nil { + return 0, fmt.Errorf("Could not parse version") + } + + return v2.Compare(v1), nil +} + +func StrtoInt(s string, startIndex int, bitSize int) (int64, error) { + return strconv.ParseInt(s, startIndex, bitSize) +} + +func (c *Client) Do(req *http.Request) (*container.Container, *http.Response, error) { + return c.DoWithRetryFunc(req, nil) +} + +func (c *Client) DoWithRetryFunc(req *http.Request, retryFunc CallbackRetryFunc) (*container.Container, *http.Response, error) { + log.Printf("[DEBUG] Begining DO method %s", req.URL.String()) + + for attempts := 1; ; attempts++ { + log.Printf("[TRACE] HTTP Request Method and URL: %s %s", req.Method, req.URL.String()) + + if !c.skipLoggingPayload { + log.Printf("[TRACE] HTTP Request Body: %v", req.Body) + } + + resp, err := c.httpClient.Do(req) + + if err != nil { + if ok := c.backoff(attempts); !ok { + log.Printf("[ERROR] HTTP Connection error occured: %+v", err) + log.Printf("[DEBUG] Exit from Do method") + return nil, nil, err + } else { + log.Printf("[ERROR] HTTP Connection failed: %s, retries: %v", err, attempts) + continue + } + } + + if !c.skipLoggingPayload { + log.Printf("[TRACE] HTTP Response: %d %s %v", resp.StatusCode, resp.Status, resp) + } else { + log.Printf("[TRACE] HTTP Response: %d %s", resp.StatusCode, resp.Status) + } + + bodyBytes, err := ioutil.ReadAll(resp.Body) + bodyStr := string(bodyBytes) + resp.Body.Close() + if !c.skipLoggingPayload { + log.Printf("[DEBUG] HTTP response unique string %s %s %s", req.Method, req.URL.String(), bodyStr) + } + + retry := false + + // 204 No Content for any requests + if resp.StatusCode == 204 { + log.Printf("[DEBUG] Exit from Do method") + return nil, nil, nil + } + + var obj *container.Container + // Check 2xx status codes + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + obj, err = container.ParseJSON(bodyBytes) + if err != nil { + // Attempt retry if JSON parsing fails but status code is 2xx + // Assumption here is that packets were somehow corrupted/lost during transmission + log.Printf("[ERROR] Error occurred while JSON parsing (2xx status): %+v", err) + retry = true + } else { + // JSON parsing was successful for a 2xx response. + // Now, check the custom retry function. + if retryFunc != nil && retryFunc(obj) { + log.Printf("[DEBUG] Custom retry function indicated a retry is needed for 2xx response") + retry = true + } else { + // If JSON parsed successfully and retryFunc does not indicate a retry, + // then this is a successful operation. + log.Printf("[DEBUG] Exit from Do method") + return obj, resp, nil + } + } + } + + // Attempt retry for the following error codes: + // 429 Too Many Requests + // 503 Service Unavailable + if resp.StatusCode == 429 || resp.StatusCode == 503 { + retry = true + } + + if retry { + log.Printf("[ERROR] HTTP Request failed with status code %d, retrying...", resp.StatusCode) + if ok := c.backoff(attempts); !ok { + log.Printf("[ERROR] HTTP Request failed with status code %d, retries exhausted", resp.StatusCode) + log.Printf("[DEBUG] Exit from Do method") + return obj, resp, fmt.Errorf("[ERROR] HTTP Request failed with status code %d after %d attempts", resp.StatusCode, attempts) + } else { + log.Printf("[DEBUG] Retrying HTTP Request after backoff") + continue + } + } + + log.Printf("[DEBUG] Exit from Do method") + return nil, resp, err + } +} + +func (c *Client) backoff(attempts int) bool { + log.Printf("[DEBUG] Begining backoff method: attempts %v on %v", attempts, c.maxRetries) + if attempts > c.maxRetries { + log.Printf("[DEBUG] Exit from backoff method with return value false") + return false + } + + minDelay := time.Duration(DefaultBackoffMinDelay) * time.Second + if c.backoffMinDelay != 0 { + minDelay = time.Duration(c.backoffMinDelay) * time.Second + } + + maxDelay := time.Duration(DefaultBackoffMaxDelay) * time.Second + if c.backoffMaxDelay != 0 { + maxDelay = time.Duration(c.backoffMaxDelay) * time.Second + } + + factor := DefaultBackoffDelayFactor + if c.backoffDelayFactor != 0 { + factor = c.backoffDelayFactor + } + + min := float64(minDelay) + backoff := min * math.Pow(factor, float64(attempts)) + if backoff > float64(maxDelay) { + backoff = float64(maxDelay) + } + backoff = (rand.Float64()/2+0.5)*(backoff-min) + min + backoffDuration := time.Duration(backoff) + log.Printf("[TRACE] Start sleeping for %v seconds", backoffDuration.Round(time.Second)) + time.Sleep(backoffDuration) + log.Printf("[DEBUG] Exit from backoff method with return value true") + return true +} - for key, value := range con { - payload.Set(value, key) +func stripQuotes(word string) string { + if strings.HasPrefix(word, "\"") && strings.HasSuffix(word, "\"") { + return strings.TrimSuffix(strings.TrimPrefix(word, "\""), "\"") } - return payload, nil + return word } From 4a2bdfe49d69323c3b7f7a13a2cbb1fb2d6411e2 Mon Sep 17 00:00:00 2001 From: bobvands <40576376+bobvands@users.noreply.github.com> Date: Thu, 17 Jul 2025 08:37:05 -0700 Subject: [PATCH 2/5] Get Cache issue --- client/client_service.go | 584 +++++++++++---------------------------- 1 file changed, 161 insertions(+), 423 deletions(-) diff --git a/client/client_service.go b/client/client_service.go index 763a2b6..ffe3ef3 100644 --- a/client/client_service.go +++ b/client/client_service.go @@ -1,530 +1,268 @@ package client import ( - "bytes" - "crypto/tls" "errors" "fmt" - "io/ioutil" "log" - "math" - "math/rand/v2" - "net/http" "net/url" - "strconv" + "regexp" "strings" "sync" "time" "github.com/ciscoecosystem/mso-go-client/container" "github.com/ciscoecosystem/mso-go-client/models" - "github.com/hashicorp/go-version" ) -const msoAuthPayload = `{ - "username": "%s", - "password": "%s" -}` - -const ndAuthPayload = `{ - "userName": "%s", - "userPasswd": "%s" -}` - -const DefaultBackoffMinDelay int = 4 -const DefaultBackoffMaxDelay int = 60 -const DefaultBackoffDelayFactor float64 = 3 - -// Client is the main entry point -type Client struct { - BaseURL *url.URL - httpClient *http.Client - AuthToken *Auth - Mutex sync.Mutex - username string - password string - insecure bool - reqTimeoutSet bool - reqTimeoutVal uint32 - proxyUrl string - domain string - platform string - version string - skipLoggingPayload bool - maxRetries int - backoffMinDelay int - backoffMaxDelay int - backoffDelayFactor float64 -} - -type CallbackRetryFunc func(*container.Container) bool +const ( + CACHE_TIMEOUT = 60 // 60 seconds +) -// singleton implementation of a client -var clientImpl *Client +type msoApi struct { + readTs time.Time + resp *container.Container + writeTs time.Time +} -type Option func(*Client) +var msoApiCache map[string]msoApi +var muApiCache sync.RWMutex // mutex lock for upating the map -func Insecure(insecure bool) Option { - return func(client *Client) { - client.insecure = insecure - } +// init of the package +func init() { + msoApiCache = make(map[string]msoApi) } -func Password(password string) Option { - return func(client *Client) { - client.password = password +// getFromCache: check the API cache and return the stored resp +// if it is with the timeout +func (c *Client) getFromCache(endpoint string) *container.Container { + defer muApiCache.RUnlock() + muApiCache.RLock() + updEndpoint := strings.Replace(endpoint, "mso/", "", 1) + if api, ok := msoApiCache[updEndpoint]; ok { + curTs := time.Now() + rDiff := curTs.Sub(api.readTs) + wDiff := curTs.Sub(api.writeTs) + log.Printf("[DEBUG] getFromCache readTs %v writeTs: %v rDiff %v wDiff %v\n", api.readTs, api.writeTs, rDiff.Seconds(), wDiff.Seconds()) + if rDiff.Seconds() >= CACHE_TIMEOUT || wDiff.Seconds() <= CACHE_TIMEOUT { + return nil + } + log.Printf("[DEBUG] Found GET response in cache for schema endpoint: %v\n", updEndpoint) + return api.resp } + return nil } -func ProxyUrl(pUrl string) Option { - return func(client *Client) { - client.proxyUrl = pUrl - } -} +// storeInCache: store the given response in the API cache +func (c *Client) storeInCache(endpoint string, resp *container.Container) { + updEndpoint := strings.Replace(endpoint, "mso/", "", 1) + var re = regexp.MustCompile(`^api/v1/schemas/(.*)$`) + matches := re.FindStringSubmatch(updEndpoint) -func Domain(domain string) Option { - return func(client *Client) { - client.domain = domain + if len(matches) != 2 { + return } -} -func Platform(platform string) Option { - return func(client *Client) { - client.platform = platform - } -} + defer muApiCache.Unlock() -func Version(version string) Option { - return func(client *Client) { - client.version = version + muApiCache.Lock() + if api, ok := msoApiCache[updEndpoint]; ok { + curTs := time.Now() + wDiff := curTs.Sub(api.writeTs) + if wDiff.Seconds() <= CACHE_TIMEOUT { + log.Printf("[DEBUG] Skip storing endpoint %v due to recent writeTs: %v\n", updEndpoint, api.writeTs) + return + } } -} -func SkipLoggingPayload(skipLoggingPayload bool) Option { - return func(client *Client) { - client.skipLoggingPayload = skipLoggingPayload + api := msoApi{ + readTs: time.Now(), + resp: resp, + writeTs: time.Now().Add(-180 * time.Second), } -} -func MaxRetries(maxRetries int) Option { - return func(client *Client) { - client.maxRetries = maxRetries - } + log.Printf("[DEBUG] Caching GET endpoint:: %s readTs %v writeTs %v", updEndpoint, api.readTs, api.writeTs) + msoApiCache[updEndpoint] = api } -func BackoffMinDelay(backoffMinDelay int) Option { - return func(client *Client) { - client.backoffMinDelay = backoffMinDelay +// invalidateCache: invalidate the cache +func (c *Client) updateCacheForWrite(endpoint string) { + updEndpoint := strings.Replace(endpoint, "mso/", "", 1) + var re = regexp.MustCompile(`^api/v1/schemas/(.*)(\?)?`) + matches := re.FindStringSubmatch(updEndpoint) + if len(matches) != 2 && len(matches) != 3 { + return } -} -func BackoffMaxDelay(backoffMaxDelay int) Option { - return func(client *Client) { - client.backoffMaxDelay = backoffMaxDelay + defer muApiCache.Unlock() + schEndPoint := "api/v1/schemas/" + matches[1] + muApiCache.Lock() + if api, ok := msoApiCache[schEndPoint]; ok { + api.writeTs = time.Now() + api.resp = nil + msoApiCache[schEndPoint] = api + log.Printf("[DEBUG] Update writeTs %v in cache for schema endpoint: %v\n", api.writeTs, schEndPoint) } } -func BackoffDelayFactor(backoffDelayFactor float64) Option { - return func(client *Client) { - client.backoffDelayFactor = backoffDelayFactor +func (c *Client) GetViaURL(endpoint string) (*container.Container, error) { + cobj := c.getFromCache(endpoint) + + if cobj != nil { + c.storeInCache(endpoint, cobj) + return cobj, nil } -} + req, err := c.MakeRestRequest("GET", endpoint, nil, true) -func initClient(clientUrl, username string, options ...Option) *Client { - var transport *http.Transport - bUrl, err := url.Parse(clientUrl) if err != nil { - // cannot move forward if url is undefined - log.Fatal(err) - } - client := &Client{ - BaseURL: bUrl, - username: username, - httpClient: http.DefaultClient, + return nil, err } - for _, option := range options { - option(client) + obj, _, err := c.Do(req) + if err != nil { + return nil, err } - transport = client.useInsecureHTTPClient(client.insecure) - if client.proxyUrl != "" { - transport = client.configProxy(transport) + if obj == nil { + return nil, errors.New("Empty response body") } + err = CheckForErrors(obj, "GET") - client.httpClient = &http.Client{ - Transport: transport, + if err != nil { + return obj, err } - return client -} + c.storeInCache(endpoint, obj) -// GetClient returns a singleton -func GetClient(clientUrl, username string, options ...Option) *Client { - if clientImpl == nil { - clientImpl = initClient(clientUrl, username, options...) - } - return clientImpl + return obj, nil } -func (c *Client) configProxy(transport *http.Transport) *http.Transport { - pUrl, err := url.Parse(c.proxyUrl) - if err != nil { - log.Fatal(err) - } - transport.Proxy = http.ProxyURL(pUrl) - return transport +func (c *Client) GetPlatform() string { + return c.platform } -func (c *Client) useInsecureHTTPClient(insecure bool) *http.Transport { - transport := &http.Transport{ - TLSClientConfig: &tls.Config{ - CipherSuites: []uint16{ - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - }, - PreferServerCipherSuites: true, - InsecureSkipVerify: insecure, - MinVersion: tls.VersionTLS11, - MaxVersion: tls.VersionTLS13, - }, - } +func (c *Client) Put(endpoint string, obj models.Model) (*container.Container, error) { + jsonPayload, err := c.PrepareModel(obj) - return transport -} - -func (c *Client) MakeRestRequest(method string, path string, body *container.Container, authenticated bool) (*http.Request, error) { - origPath := path - if c.platform == "nd" && path != "/login" { - if strings.HasPrefix(path, "/") { - path = path[1:] - } - path = fmt.Sprintf("mso/%v", path) - } - url, err := url.Parse(path) if err != nil { return nil, err } - if method == "PATCH" { - validateString := url.Query() - validateString.Set("validate", "false") - url.RawQuery = validateString.Encode() - } - fURL := c.BaseURL.ResolveReference(url) - var req *http.Request - if method == "GET" || method == "DELETE" { - req, err = http.NewRequest(method, fURL.String(), nil) - } else { - req, err = http.NewRequest(method, fURL.String(), bytes.NewBuffer((body.Bytes()))) - } + req, err := c.MakeRestRequest("PUT", endpoint, jsonPayload, true) if err != nil { return nil, err } - if method == "PATCH" || method == "PUT" || method == "DELETE" || method == "POST" { - c.updateCacheForWrite(origPath) - log.Printf("[DEBUG] updating cache for write methods, endpoint %v", origPath) - } - req.Header.Set("Content-Type", "application/json") - log.Printf("[DEBUG] HTTP request %s %s", method, path) - - if authenticated { - req, err = c.InjectAuthenticationHeader(req, path) - if err != nil { - return req, err - } + c.Mutex.Lock() + cont, _, err := c.Do(req) + c.Mutex.Unlock() + if err != nil { + return nil, err } - log.Printf("[DEBUG] HTTP request after injection %s %s", method, path) - return req, nil + return cont, CheckForErrors(cont, "PUT") } -// Authenticate is used to -func (c *Client) Authenticate() error { - method := "POST" - path := "/api/v1/auth/login" - var authPayload string +func (c *Client) Save(endpoint string, obj models.Model) (*container.Container, error) { - if c.platform == "nd" { - authPayload = ndAuthPayload - if c.domain == "" { - c.domain = "DefaultAuth" - } - path = "/login" - } else { - authPayload = msoAuthPayload - } - body, err := container.ParseJSON([]byte(fmt.Sprintf(authPayload, c.username, c.password))) - if err != nil { - return err - } + jsonPayload, err := c.PrepareModel(obj) - if c.domain != "" { - if c.platform == "nd" { - body.Set(c.domain, "domain") - } else { - domainId, err := c.GetDomainId(c.domain) - if err != nil { - return err - } - body.Set(domainId, "domainId") - } - } - - c.skipLoggingPayload = true - - req, err := c.MakeRestRequest(method, path, body, false) if err != nil { - return err + return nil, err } - - obj, _, err := c.Do(req) - c.skipLoggingPayload = false + req, err := c.MakeRestRequest("POST", endpoint, jsonPayload, true) if err != nil { - return err - } - - if obj == nil { - return errors.New("Empty response") - } - req.Header.Set("Content-Type", "application/json") - - token := models.StripQuotes(obj.S("token").String()) - - if token == "" || token == "{}" { - return errors.New("Invalid Username or Password") - } - - if c.AuthToken == nil { - c.AuthToken = &Auth{} + return nil, err } - c.AuthToken.Token = stripQuotes(token) - c.AuthToken.CalculateExpiry(1200) //refreshTime=1200 Sec - - return nil -} -func (c *Client) GetDomainId(domain string) (string, error) { - req, err := c.MakeRestRequest("GET", "/api/v1/auth/login-domains", nil, false) + cont, _, err := c.Do(req) if err != nil { - return "", err + return nil, err } - obj, _, err := c.Do(req) - - if err != nil { - return "", err - } - err = CheckForErrors(obj, "GET") - if err != nil { - return "", err - } - count, err := obj.ArrayCount("domains") - if err != nil { - return "", err - } + return cont, CheckForErrors(cont, "POST") +} - for i := 0; i < count; i++ { - domainCont, err := obj.ArrayElement(i, "domains") - if err != nil { - return "", err - } - domainName := models.StripQuotes(domainCont.S("name").String()) +// CheckForErrors parses the response and checks of there is an error attribute in the response +func CheckForErrors(cont *container.Container, method string) error { - if domainName == domain { - return models.StripQuotes(domainCont.S("id").String()), nil - } + if cont.Exists("code") && cont.Exists("message") { + return errors.New(fmt.Sprintf("%s%s", cont.S("message"), cont.S("info"))) + } else if cont.Exists("error") { + return errors.New(fmt.Sprintf("%s %s", models.StripQuotes(cont.S("error").String()), models.StripQuotes(cont.S("error_code").String()))) + } else { + return nil } - return "", fmt.Errorf("Unable to find domain id for domain %s", domain) + return nil } -func (c *Client) GetVersion() (string, error) { - req, err := c.MakeRestRequest("GET", "/api/v1/platform/version", nil, true) - if err != nil { - return "unknown", err - } +func (c *Client) DeletebyId(url string) error { - obj, _, err := c.Do(req) + req, err := c.MakeRestRequest("DELETE", url, nil, true) if err != nil { - return "unknown", err - } - - err = CheckForErrors(obj, "GET") - if err != nil { - return "unknown", err - } - - version := stripQuotes(obj.Search("version").String()) - if version == "" { - return "unknown", fmt.Errorf("Unable to identify version") + return err } - c.version = version - return version, nil -} -// Compares the version to the retrieved version. -// This returns -1, 0, or 1 if this version is smaller, equal, or larger than the retrieved version, respectively. -func (c *Client) CompareVersion(v string) (int, error) { - if c.version == "" { - c.GetVersion() - } - if c.version == "unknown" { - return 0, fmt.Errorf("Could not retrieve version") + _, resp, err1 := c.Do(req) + if err1 != nil { + return err1 } - - v1, err := version.NewVersion(c.version) - if err != nil { - return 0, fmt.Errorf("Could not parse retrieved version") - } - v2, err := version.NewVersion(v) - if err != nil { - return 0, fmt.Errorf("Could not parse version") + if resp != nil { + if resp.StatusCode == 204 || resp.StatusCode == 200 { + return nil + } else { + return fmt.Errorf("Unable to delete the object") + } } - return v2.Compare(v1), nil -} - -func StrtoInt(s string, startIndex int, bitSize int) (int64, error) { - return strconv.ParseInt(s, startIndex, bitSize) -} - -func (c *Client) Do(req *http.Request) (*container.Container, *http.Response, error) { - return c.DoWithRetryFunc(req, nil) + return nil } -func (c *Client) DoWithRetryFunc(req *http.Request, retryFunc CallbackRetryFunc) (*container.Container, *http.Response, error) { - log.Printf("[DEBUG] Begining DO method %s", req.URL.String()) - - for attempts := 1; ; attempts++ { - log.Printf("[TRACE] HTTP Request Method and URL: %s %s", req.Method, req.URL.String()) - - if !c.skipLoggingPayload { - log.Printf("[TRACE] HTTP Request Body: %v", req.Body) - } - - resp, err := c.httpClient.Do(req) +func (c *Client) PatchbyID(endpoint string, objList ...models.Model) (*container.Container, error) { + contJs := container.New() + contJs.Array() + for _, obj := range objList { + jsonPayload, err := c.PrepareModel(obj) if err != nil { - if ok := c.backoff(attempts); !ok { - log.Printf("[ERROR] HTTP Connection error occured: %+v", err) - log.Printf("[DEBUG] Exit from Do method") - return nil, nil, err - } else { - log.Printf("[ERROR] HTTP Connection failed: %s, retries: %v", err, attempts) - continue - } + return nil, err } + contJs.ArrayAppend(jsonPayload.Data()) - if !c.skipLoggingPayload { - log.Printf("[TRACE] HTTP Response: %d %s %v", resp.StatusCode, resp.Status, resp) - } else { - log.Printf("[TRACE] HTTP Response: %d %s", resp.StatusCode, resp.Status) - } - - bodyBytes, err := ioutil.ReadAll(resp.Body) - bodyStr := string(bodyBytes) - resp.Body.Close() - if !c.skipLoggingPayload { - log.Printf("[DEBUG] HTTP response unique string %s %s %s", req.Method, req.URL.String(), bodyStr) - } - - retry := false - - // 204 No Content for any requests - if resp.StatusCode == 204 { - log.Printf("[DEBUG] Exit from Do method") - return nil, nil, nil - } - - var obj *container.Container - // Check 2xx status codes - if resp.StatusCode >= 200 && resp.StatusCode < 300 { - obj, err = container.ParseJSON(bodyBytes) - if err != nil { - // Attempt retry if JSON parsing fails but status code is 2xx - // Assumption here is that packets were somehow corrupted/lost during transmission - log.Printf("[ERROR] Error occurred while JSON parsing (2xx status): %+v", err) - retry = true - } else { - // JSON parsing was successful for a 2xx response. - // Now, check the custom retry function. - if retryFunc != nil && retryFunc(obj) { - log.Printf("[DEBUG] Custom retry function indicated a retry is needed for 2xx response") - retry = true - } else { - // If JSON parsed successfully and retryFunc does not indicate a retry, - // then this is a successful operation. - log.Printf("[DEBUG] Exit from Do method") - return obj, resp, nil - } - } - } - - // Attempt retry for the following error codes: - // 429 Too Many Requests - // 503 Service Unavailable - if resp.StatusCode == 429 || resp.StatusCode == 503 { - retry = true - } - - if retry { - log.Printf("[ERROR] HTTP Request failed with status code %d, retrying...", resp.StatusCode) - if ok := c.backoff(attempts); !ok { - log.Printf("[ERROR] HTTP Request failed with status code %d, retries exhausted", resp.StatusCode) - log.Printf("[DEBUG] Exit from Do method") - return obj, resp, fmt.Errorf("[ERROR] HTTP Request failed with status code %d after %d attempts", resp.StatusCode, attempts) - } else { - log.Printf("[DEBUG] Retrying HTTP Request after backoff") - continue - } - } - - log.Printf("[DEBUG] Exit from Do method") - return nil, resp, err } -} + log.Printf("[DEBUG] Patch Request Container: %v\n", contJs) + // URL encoding + baseUrl, _ := url.Parse(endpoint) + qs := url.Values{} + qs.Add("validate", "false") + baseUrl.RawQuery = qs.Encode() -func (c *Client) backoff(attempts int) bool { - log.Printf("[DEBUG] Begining backoff method: attempts %v on %v", attempts, c.maxRetries) - if attempts > c.maxRetries { - log.Printf("[DEBUG] Exit from backoff method with return value false") - return false + req, err := c.MakeRestRequest("PATCH", baseUrl.String(), contJs, true) + if err != nil { + return nil, err } - minDelay := time.Duration(DefaultBackoffMinDelay) * time.Second - if c.backoffMinDelay != 0 { - minDelay = time.Duration(c.backoffMinDelay) * time.Second + c.Mutex.Lock() + cont, _, err := c.Do(req) + c.Mutex.Unlock() + if err != nil { + return nil, err } - maxDelay := time.Duration(DefaultBackoffMaxDelay) * time.Second - if c.backoffMaxDelay != 0 { - maxDelay = time.Duration(c.backoffMaxDelay) * time.Second - } + return cont, CheckForErrors(cont, "PATCH") +} - factor := DefaultBackoffDelayFactor - if c.backoffDelayFactor != 0 { - factor = c.backoffDelayFactor +func (c *Client) PrepareModel(obj models.Model) (*container.Container, error) { + con, err := obj.ToMap() + if err != nil { + return nil, err } - min := float64(minDelay) - backoff := min * math.Pow(factor, float64(attempts)) - if backoff > float64(maxDelay) { - backoff = float64(maxDelay) + payload := &container.Container{} + if err != nil { + return nil, err } - backoff = (rand.Float64()/2+0.5)*(backoff-min) + min - backoffDuration := time.Duration(backoff) - log.Printf("[TRACE] Start sleeping for %v seconds", backoffDuration.Round(time.Second)) - time.Sleep(backoffDuration) - log.Printf("[DEBUG] Exit from backoff method with return value true") - return true -} -func stripQuotes(word string) string { - if strings.HasPrefix(word, "\"") && strings.HasSuffix(word, "\"") { - return strings.TrimSuffix(strings.TrimPrefix(word, "\""), "\"") + for key, value := range con { + payload.Set(value, key) } - return word + return payload, nil } From cae76c17444d787b15df754b95a391a642adbcbd Mon Sep 17 00:00:00 2001 From: bobvands <40576376+bobvands@users.noreply.github.com> Date: Thu, 17 Jul 2025 08:38:06 -0700 Subject: [PATCH 3/5] Get cache implemenation --- client/client.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/client.go b/client/client.go index 68d2252..763a2b6 100644 --- a/client/client.go +++ b/client/client.go @@ -197,6 +197,7 @@ func (c *Client) useInsecureHTTPClient(insecure bool) *http.Transport { } func (c *Client) MakeRestRequest(method string, path string, body *container.Container, authenticated bool) (*http.Request, error) { + origPath := path if c.platform == "nd" && path != "/login" { if strings.HasPrefix(path, "/") { path = path[1:] @@ -222,6 +223,10 @@ func (c *Client) MakeRestRequest(method string, path string, body *container.Con if err != nil { return nil, err } + if method == "PATCH" || method == "PUT" || method == "DELETE" || method == "POST" { + c.updateCacheForWrite(origPath) + log.Printf("[DEBUG] updating cache for write methods, endpoint %v", origPath) + } req.Header.Set("Content-Type", "application/json") log.Printf("[DEBUG] HTTP request %s %s", method, path) From b03555e1b424e6917d320438f17d8e496d984c23 Mon Sep 17 00:00:00 2001 From: bobvands <40576376+bobvands@users.noreply.github.com> Date: Thu, 17 Jul 2025 08:38:50 -0700 Subject: [PATCH 4/5] Update go.mod --- go.mod | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d16e33a..13f2f78 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,7 @@ module github.com/ciscoecosystem/mso-go-client go 1.12 -require github.com/hashicorp/go-version v1.6.0 // indirect +require ( + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/stretchr/testify v1.10.0 // indirect +) From fb4e13c7dc447e890b7ffd06d4ab70b126350f30 Mon Sep 17 00:00:00 2001 From: bobvands <40576376+bobvands@users.noreply.github.com> Date: Thu, 17 Jul 2025 08:39:52 -0700 Subject: [PATCH 5/5] Get cache implemenation --- tests/client_test.go | 197 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 196 insertions(+), 1 deletion(-) diff --git a/tests/client_test.go b/tests/client_test.go index b9cced6..100f371 100644 --- a/tests/client_test.go +++ b/tests/client_test.go @@ -1,9 +1,15 @@ package tests import ( + "encoding/json" + "fmt" + "sync" "testing" + "time" "github.com/ciscoecosystem/mso-go-client/client" + "github.com/ciscoecosystem/mso-go-client/container" + "github.com/stretchr/testify/assert" ) func TestClientAuthenticate(t *testing.T) { @@ -14,13 +20,202 @@ func TestClientAuthenticate(t *testing.T) { t.Error(err) } + fmt.Printf("err is %v", err) + if client.AuthToken.Token == "{}" { t.Error("Token is empty") } - t.Error("all wrong") + + fmt.Printf("Got Token %v", client.AuthToken.Token) } func GetTestClient() *client.Client { return client.GetClient("https://173.36.219.193", "admin", client.Password("ins3965!ins3965!"), client.Insecure(true)) +} + +func TestParallelGetSchemas(t *testing.T) { + cl := GetTestClient() + err := cl.Authenticate() + if err != nil { + t.Error(err) + } + schId := "6878807a072d2d88bec9b3b3" // Test_Schema + schUrl := "api/v1/schemas/" + schId + _, err = cl.GetViaURL(schUrl) + + assert := assert.New(t) + assert.Equal(err, nil) + + numRequests := 6 + resps := make(map[int]*container.Container) + errs := []error{} + + numObjs := 100 + numBatches := numObjs / numRequests + + fmt.Printf("Requesting %v objects in %v batches in %v requests per batch", numObjs, numBatches, numRequests) + + for b := 1; b <= numBatches; b++ { + wgReqs := sync.WaitGroup{} + // Create the workers + for w := 1; w <= numRequests; w++ { + wgReqs.Add(numRequests) + go func(reqN int) { + defer wgReqs.Done() + var err error + resps[reqN], err = cl.GetViaURL(schUrl) + fmt.Printf("Batch: %v Request: %v GetViaURL err = [%v]\n", b, reqN, err) + errs = append(errs, err) + }(w) + } + wgReqs.Wait() + // time.Sleep(2 * time.Second) + time.Sleep(200000000) // 2*10^8 nano seconds = 200 ms + } + assert.Equal(err, nil) + fmt.Printf("len(resps) = %v\n", len(resps)) +} + +func TestParallelGetSchemasMso(t *testing.T) { + cl := GetTestClient() + err := cl.Authenticate() + if err != nil { + t.Error(err) + } + schId := "6878807a072d2d88bec9b3b3" // for Test_Schema + schUrl := "mso/api/v1/schemas/" + schId + _, err = cl.GetViaURL(schUrl) + + assert := assert.New(t) + assert.Equal(err, nil) + + numRequests := 6 + resps := make(map[int]*container.Container) + errs := []error{} + + numObjs := 120 + numBatches := numObjs / numRequests + + fmt.Printf("Requesting %v objects in %v batches in %v requests per batch", numObjs, numBatches, numRequests) + + for b := 1; b <= numBatches; b++ { + wgReqs := sync.WaitGroup{} + // Create the workers + for w := 1; w <= numRequests; w++ { + wgReqs.Add(1) + go func(reqN int) { + defer wgReqs.Done() + var err error + resps[reqN], err = cl.GetViaURL(schUrl) + fmt.Printf("Batch: %v Request: %v GetViaURL err = [%v]\n", b, reqN, err) + errs = append(errs, err) + }(w) + } + wgReqs.Wait() + // time.Sleep(2 * time.Second) + time.Sleep(200000000) // 2*10^8 nano seconds = 200 ms + } + assert.Equal(err, nil) + fmt.Printf("len(resps) = %v\n", len(resps)) +} + +func TestParallelPatchSchemas(t *testing.T) { + cl := GetTestClient() + err := cl.Authenticate() + if err != nil { + t.Error(err) + } + + return + + schemaID := "6878807a072d2d88bec9b3b3" + schUrl := "api/v1/schemas/" + schemaID + + assert := assert.New(t) + + _, err = cl.GetViaURL(schUrl) + + numBatches := 3 + numRequests := 3 + for b := 0; b < numBatches; b++ { + wgReqs := sync.WaitGroup{} + // Create the workers + for w := 1; w <= numRequests; w++ { + wgReqs.Add(1) + bdNum := b*numRequests + w + go func(bdN int) { + defer wgReqs.Done() + var err error + bdName := fmt.Sprintf("BD%v", bdN) + desc := fmt.Sprintf("new descr %v", 300+bdN) + err = patchBDDescr(cl, schemaID, "Tmpl1", bdName, desc) + assert.Equal(err, nil) + _, err = cl.GetViaURL(schUrl) + fmt.Printf("Batch: %v Request: %v GetViaURL err = [%v]\n", b, w, err) + }(bdNum) + } + wgReqs.Wait() + // time.Sleep(2 * time.Second) + time.Sleep(200000000) // 2*10^8 nano seconds = 200 ms + } + assert.Equal(err, nil) +} + +func doPatchRequest(msoClient *client.Client, path string, payloadCon *container.Container) error { + req, err := msoClient.MakeRestRequest("PATCH", path, payloadCon, true) + if err != nil { + return err + } + + cont, _, err := msoClient.Do(req) + if err != nil { + return err + } + + err = client.CheckForErrors(cont, "PATCH") + if err != nil { + return err + } + + return nil +} + +func addPatchPayloadToContainer(payloadContainer *container.Container, op, path string, value interface{}) error { + + payloadMap := map[string]interface{}{"op": op, "path": path, "value": value} + + payload, err := json.Marshal(payloadMap) + if err != nil { + return err + } + + jsonContainer, err := container.ParseJSON([]byte(payload)) + if err != nil { + return err + } + + err = payloadContainer.ArrayAppend(jsonContainer.Data()) + if err != nil { + return err + } + + return nil +} + +func patchBDDescr(cl *client.Client, schemaID string, templateName string, bdName string, desc string) error { + basePath := fmt.Sprintf("/templates/%s/bds/%s", templateName, bdName) + payloadCon := container.New() + payloadCon.Array() + + err := addPatchPayloadToContainer(payloadCon, "replace", fmt.Sprintf("%s/description", basePath), desc) + if err != nil { + return err + } + + err = doPatchRequest(cl, fmt.Sprintf("api/v1/schemas/%s", schemaID), payloadCon) + if err != nil { + return err + } + return nil }