Merge branch 'add_client_cache_for_rest_api_requests' of https://github.com/menai34/terraform-provider-powerdns into menai34-add_client_cache_for_rest_api_requests

Author: mbag

.gitignore

@@ -22,6 +22,7 @@ website/node_modules
 *~
 .*.swp
 .idea
+.vscode
 *.iml
 *.test
 *.iml

go.mod

@@ -6,4 +6,5 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.1
 	github.com/hashicorp/terraform-plugin-sdk v1.1.0
 	github.com/stretchr/testify v1.3.0
+	github.com/coocood/freecache v1.1.1
 )

go.sum

@@ -9,6 +9,7 @@ cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbf
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
 github.com/agext/levenshtein v1.2.2 h1:0S/Yg6LYmFJ5stwQeRp6EeOcCbj7xiqQSdNelsXvaqE=
 github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
@@ -30,8 +31,12 @@ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ
 github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bsm/go-vlq v0.0.0-20150828105119-ec6e8d4f5f4e/go.mod h1:N+BjUcTjSxc2mtRGSCPsat1kze3CUtvJN3/jTXlp29k=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc=
+github.com/coocood/freecache v1.1.1/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -168,6 +173,7 @@ github.com/posener/complete v1.2.1 h1:LrvDIY//XNo65Lq84G/akBuMGlawHvGBABv8f/ZN6D
 github.com/posener/complete v1.2.1/go.mod h1:6gapUrK/U1TAN7ciCoNRIdVC5sbdBTUh1DKN0g6uH7E=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=

powerdns/client.go

@@ -6,29 +6,37 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
 
+	freecache "github.com/coocood/freecache"
 	cleanhttp "github.com/hashicorp/go-cleanhttp"
 )
 
 // DefaultSchema is the value used for the URL in case
 // no schema is explicitly defined
 var DefaultSchema = "https"
 
+// DefaultCacheSize is client default cache size
+var DefaultCacheSize int
+
 // Client is a PowerDNS client representation
 type Client struct {
 	ServerURL     string // Location of PowerDNS server to use
 	ServerVersion string
 	APIKey        string // REST API Static authentication key
 	APIVersion    int    // API version to use
 	HTTP          *http.Client
+	CacheEnable   bool // Enable/Disable chache for REST API requests
+	Cache         *freecache.Cache
+	CacheTTL      int
 }
 
 // NewClient returns a new PowerDNS client
-func NewClient(serverURL string, apiKey string, configTLS *tls.Config) (*Client, error) {
+func NewClient(serverURL string, apiKey string, configTLS *tls.Config, cacheEnable bool, cacheSizeMB string, cacheTTL int) (*Client, error) {
 
 	cleanURL, err := sanitizeURL(serverURL)
 
@@ -39,11 +47,22 @@ func NewClient(serverURL string, apiKey string, configTLS *tls.Config) (*Client,
 		return nil, fmt.Errorf("Error while creating client: %s", err)
 	}
 
+	if cacheEnable {
+		cacheSize, err := strconv.Atoi(cacheSizeMB)
+		if err != nil {
+			return nil, fmt.Errorf("Error while creating client: %s", err)
+		}
+		DefaultCacheSize = cacheSize * 1024 * 1024
+	}
+
 	client := Client{
-		ServerURL:  cleanURL,
-		APIKey:     apiKey,
-		HTTP:       httpClient,
-		APIVersion: -1,
+		ServerURL:   cleanURL,
+		APIKey:      apiKey,
+		HTTP:        httpClient,
+		APIVersion:  -1,
+		CacheEnable: cacheEnable,
+		Cache:       freecache.NewCache(DefaultCacheSize),
+		CacheTTL:    cacheTTL,
 	}
 
 	if err := client.setServerVersion(); err != nil {
@@ -420,23 +439,62 @@ func (client *Client) DeleteZone(name string) error {
 	return nil
 }
 
+// GetZoneInfoFromCache return ZoneInfo struct
+func (client *Client) GetZoneInfoFromCache(zone string) (*ZoneInfo, error) {
+	if client.CacheEnable {
+		cacheZoneInfo, err := client.Cache.Get([]byte(zone))
+		if err != nil {
+			return nil, err
+		}
+
+		zoneInfo := new(ZoneInfo)
+		err = json.Unmarshal(cacheZoneInfo, &zoneInfo)
+		if err != nil {
+			return nil, err
+		}
+
+		return zoneInfo, err
+	}
+	return nil, nil
+}
+
 // ListRecords returns all records in Zone
 func (client *Client) ListRecords(zone string) ([]Record, error) {
-	req, err := client.newRequest("GET", fmt.Sprintf("/servers/localhost/zones/%s", zone), nil)
+	zoneInfo, err := client.GetZoneInfoFromCache(zone)
 	if err != nil {
-		return nil, err
+		log.Printf("[WARN] module.freecache: %s: %s", zone, err)
 	}
 
-	resp, err := client.HTTP.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
+	if zoneInfo == nil {
+		req, err := client.newRequest("GET", fmt.Sprintf("/servers/localhost/zones/%s", zone), nil)
+		if err != nil {
+			return nil, err
+		}
 
-	zoneInfo := new(ZoneInfo)
-	err = json.NewDecoder(resp.Body).Decode(zoneInfo)
-	if err != nil {
-		return nil, err
+		resp, err := client.HTTP.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+
+		zoneInfo = new(ZoneInfo)
+		err = json.NewDecoder(resp.Body).Decode(zoneInfo)
+		if err != nil {
+			return nil, err
+		}
+
+		if client.CacheEnable {
+			cacheValue, err := json.Marshal(zoneInfo)
+			if err != nil {
+				return nil, err
+			}
+
+			err = client.Cache.Set([]byte(zone), cacheValue, client.CacheTTL)
+			if err != nil {
+				return nil, fmt.Errorf("The cache for REST API requests is enabled but the size isn't enough: cacheSize: %db \n %s",
+					DefaultCacheSize, err)
+			}
+		}
 	}
 
 	records := zoneInfo.Records

powerdns/config.go

@@ -11,10 +11,13 @@ import (
 
 // Config describes de configuration interface of this provider
 type Config struct {
-	ServerURL     string
-	APIKey        string
-	InsecureHTTPS bool
-	CACertificate string
+	ServerURL       string
+	APIKey          string
+	InsecureHTTPS   bool
+	CACertificate   string
+	CacheEnable     bool
+	CacheMemorySize string
+	CacheTTL        int
 }
 
 // Client returns a new client for accessing PowerDNS
@@ -36,7 +39,7 @@ func (c *Config) Client() (*Client, error) {
 
 	tlsConfig.InsecureSkipVerify = c.InsecureHTTPS
 
-	client, err := NewClient(c.ServerURL, c.APIKey, tlsConfig)
+	client, err := NewClient(c.ServerURL, c.APIKey, tlsConfig, c.CacheEnable, c.CacheMemorySize, c.CacheTTL)
 
 	if err != nil {
 		return nil, fmt.Errorf("Error setting up PowerDNS client: %s", err)

powerdns/provider.go

@@ -33,6 +33,24 @@ func Provider() terraform.ResourceProvider {
 				DefaultFunc: schema.EnvDefaultFunc("PDNS_CACERT", ""),
 				Description: "Content or path of a Root CA to be used to verify PowerDNS's SSL certificate",
 			},
+			"cache_requests": {
+				Type:        schema.TypeBool,
+				Optional:    true,
+				DefaultFunc: schema.EnvDefaultFunc("PDNS_CACHE_REQUESTS", false),
+				Description: "Enable cache REST API requests",
+			},
+			"cache_mem_size": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				DefaultFunc: schema.EnvDefaultFunc("PDNS_CACHE_MEM_SIZE", "100"),
+				Description: "Set cache memory size in MB",
+			},
+			"cache_ttl": {
+				Type:        schema.TypeInt,
+				Optional:    true,
+				DefaultFunc: schema.EnvDefaultFunc("PDNS_CACHE_TTL", 30),
+				Description: "Set cache TTL in seconds",
+			},
 		},
 
 		ResourcesMap: map[string]*schema.Resource{
@@ -46,10 +64,13 @@ func Provider() terraform.ResourceProvider {
 
 func providerConfigure(data *schema.ResourceData) (interface{}, error) {
 	config := Config{
-		APIKey:        data.Get("api_key").(string),
-		ServerURL:     data.Get("server_url").(string),
-		InsecureHTTPS: data.Get("insecure_https").(bool),
-		CACertificate: data.Get("ca_certificate").(string),
+		APIKey:          data.Get("api_key").(string),
+		ServerURL:       data.Get("server_url").(string),
+		InsecureHTTPS:   data.Get("insecure_https").(bool),
+		CACertificate:   data.Get("ca_certificate").(string),
+		CacheEnable:     data.Get("cache_requests").(bool),
+		CacheMemorySize: data.Get("cache_mem_size").(string),
+		CacheTTL:        data.Get("cache_ttl").(int),
 	}
 
 	return config.Client()

website/docs/index.html.markdown

@@ -40,3 +40,6 @@ The following arguments are supported:
 * `server_url` - (Required) The address of PowerDNS server. This can also be specified with `PDNS_SERVER_URL` environment variable. When no schema is provided, the default is `https`.
 * `ca_certificate` - (Optional) A valid path of a Root CA Certificate in PEM format _or_ the content of a Root CA certificate in PEM format. This can also be specified with `PDNS_CACERT` environment variable.
 * `insecure_https` - (Optional) Set this to `true` to disable verification of the PowerDNS server's TLS certificate. This can also be specified with the `PDNS_INSECURE_HTTPS` environment variable.
+* `cache_requests` - (Optional) Set this to `true` to enable cache of the PowerDNS REST API requests. This can also be specified with the `PDNS_CACHE_REQUESTS` environment variable. `WARNING! Enabling this option can lead to the use of stale records when you use other automation to populate the DNS zone records at the same time.`
+* `cache_mem_size` - (Optional) Memory size in MB for a cache of the PowerDNS REST API requests. This can also be specified with the `PDNS_CACHE_MEM_SIZE` environment variable.
+* `cache_ttl` - (Optional) TTL in seconds for a cache of the PowerDNS REST API requests. This can also be specified with the `PDNS_CACHE_TTL` environment variable.