add license reporting process (#6309)

Author: Jim Ryan

build/scripts/common.sh

@@ -7,6 +7,7 @@ if [ -z "${BUILD_OS##*plus*}" ]; then
     mkdir -p /etc/nginx/oidc/
     cp -a /code/internal/configs/oidc/* /etc/nginx/oidc/
     mkdir -p /etc/nginx/state_files/
+    mkdir -p /etc/nginx/reporting/
     PLUS=-plus
 fi
 

cmd/nginx-ingress/main.go

@@ -21,6 +21,7 @@ import (
 	"github.com/nginxinc/kubernetes-ingress/internal/healthcheck"
 	"github.com/nginxinc/kubernetes-ingress/internal/k8s"
 	"github.com/nginxinc/kubernetes-ingress/internal/k8s/secrets"
+	license_reporting "github.com/nginxinc/kubernetes-ingress/internal/license_reporting"
 	"github.com/nginxinc/kubernetes-ingress/internal/metrics"
 	"github.com/nginxinc/kubernetes-ingress/internal/metrics/collectors"
 	"github.com/nginxinc/kubernetes-ingress/internal/nginx"
@@ -41,6 +42,7 @@ import (
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 
 	kitlog "github.com/go-kit/log"
+
 	"github.com/go-kit/log/level"
 
 	nl "github.com/nginxinc/kubernetes-ingress/internal/logger"
@@ -96,7 +98,13 @@ func main() {
 
 	managerCollector, controllerCollector, registry := createManagerAndControllerCollectors(ctx, constLabels)
 
-	nginxManager, useFakeNginxManager := createNginxManager(ctx, managerCollector)
+	var licenseReporter *license_reporting.LicenseReporter
+
+	if *nginxPlus {
+		licenseReporter = license_reporting.NewLicenseReporter(kubeClient)
+	}
+
+	nginxManager, useFakeNginxManager := createNginxManager(ctx, managerCollector, licenseReporter)
 
 	nginxVersion := getNginxVersionInfo(ctx, nginxManager)
 
@@ -452,14 +460,14 @@ func createTemplateExecutors(ctx context.Context) (*version1.TemplateExecutor, *
 	return templateExecutor, templateExecutorV2
 }
 
-func createNginxManager(ctx context.Context, managerCollector collectors.ManagerCollector) (nginx.Manager, bool) {
+func createNginxManager(ctx context.Context, managerCollector collectors.ManagerCollector, licenseReporter *license_reporting.LicenseReporter) (nginx.Manager, bool) {
 	useFakeNginxManager := *proxyURL != ""
 	var nginxManager nginx.Manager
 	if useFakeNginxManager {
 		nginxManager = nginx.NewFakeManager("/etc/nginx")
 	} else {
 		timeout := time.Duration(*nginxReloadTimeout) * time.Millisecond
-		nginxManager = nginx.NewLocalManager(ctx, "/etc/nginx/", *nginxDebug, managerCollector, timeout)
+		nginxManager = nginx.NewLocalManager(ctx, "/etc/nginx/", *nginxDebug, managerCollector, licenseReporter, timeout, *nginxPlus)
 	}
 	return nginxManager, useFakeNginxManager
 }

internal/common_cluster_info/common_cluster_info.go

@@ -0,0 +1,66 @@
+package commonclusterinfo
+
+import (
+	"context"
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+)
+
+// This file contains functions for data used in both product telemetry and license reporting
+
+// GetNodeCount returns the number of nodes in the cluster
+func GetNodeCount(ctx context.Context, client kubernetes.Interface) (int, error) {
+	nodes, err := client.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return 0, err
+	}
+	return len(nodes.Items), nil
+}
+
+// GetClusterID returns the UID of the kube-system namespace representing cluster id.
+// It returns an error if the underlying k8s API client errors.
+func GetClusterID(ctx context.Context, client kubernetes.Interface) (string, error) {
+	cluster, err := client.CoreV1().Namespaces().Get(ctx, "kube-system", metav1.GetOptions{})
+	if err != nil {
+		return "", err
+	}
+	return string(cluster.UID), nil
+}
+
+// GetInstallationID returns the Installation ID of the cluster
+func GetInstallationID(ctx context.Context, client kubernetes.Interface, podNSName types.NamespacedName) (_ string, err error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("error generating InstallationID: %w", err)
+		}
+	}()
+
+	pod, err := client.CoreV1().Pods(podNSName.Namespace).Get(ctx, podNSName.Name, metav1.GetOptions{})
+	if err != nil {
+		return "", err
+	}
+	podOwner := pod.GetOwnerReferences()
+	if len(podOwner) != 1 {
+		return "", fmt.Errorf("expected pod owner reference to be 1, got %d", len(podOwner))
+	}
+
+	switch podOwner[0].Kind {
+	case "ReplicaSet":
+		rs, err := client.AppsV1().ReplicaSets(podNSName.Namespace).Get(ctx, podOwner[0].Name, metav1.GetOptions{})
+		if err != nil {
+			return "", err
+		}
+		rsOwner := rs.GetOwnerReferences() // rsOwner holds information about replica's owner - Deployment object
+		if len(rsOwner) != 1 {
+			return "", fmt.Errorf("expected replicaset owner reference to be 1, got %d", len(rsOwner))
+		}
+		return string(rsOwner[0].UID), nil
+	case "DaemonSet":
+		return string(podOwner[0].UID), nil
+	default:
+		return "", fmt.Errorf("expected pod owner reference to be ReplicaSet or DeamonSet, got %s", podOwner[0].Kind)
+	}
+}

internal/configs/version1/nginx-plus.tmpl

@@ -358,8 +358,15 @@ stream {
     include /etc/nginx/stream-conf.d/*.conf;
 }
 
-{{- if (.NginxVersion.PlusGreaterThanOrEqualTo "nginx-plus-r31") }}
+{{- if .NginxVersion.PlusGreaterThanOrEqualTo "nginx-plus-r33" }}
+mgmt {
+    usage_report endpoint=product.connect.nginx.com interval=1h;
+    uuid_file /var/lib/nginx/nginx.id;
+    license_token conf/license.jwt;
+    deployment_context /etc/nginx/reporting/tracking.info;
+}
+{{- else if .NginxVersion.PlusGreaterThanOrEqualTo "nginx-plus-r31" }}
 mgmt {
     usage_report interval=0s;
 }
-{{- end}}
+{{- end }}

internal/license_reporting/license_reporting.go

@@ -0,0 +1,96 @@
+package licensereporting
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"time"
+
+	nl "github.com/nginxinc/kubernetes-ingress/internal/logger"
+
+	clusterInfo "github.com/nginxinc/kubernetes-ingress/internal/common_cluster_info"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+)
+
+var (
+	reportingDir  = "/etc/nginx/reporting"
+	reportingFile = "tracking.info"
+)
+
+type licenseInfo struct {
+	Integration      string `json:"integration"`
+	ClusterID        string `json:"cluster_id"`
+	ClusterNodeCount int    `json:"cluster_node_count"`
+	InstallationID   string `json:"installation_id"`
+}
+
+func newLicenseInfo(clusterID, installationID string, clusterNodeCount int) *licenseInfo {
+	return &licenseInfo{
+		Integration:      "nic",
+		ClusterID:        clusterID,
+		InstallationID:   installationID,
+		ClusterNodeCount: clusterNodeCount,
+	}
+}
+
+func writeLicenseInfo(l *slog.Logger, info *licenseInfo) {
+	jsonData, err := json.Marshal(info)
+	if err != nil {
+		nl.Errorf(l, "failed to marshal LicenseInfo to JSON: %v", err)
+		return
+	}
+	filePath := filepath.Join(reportingDir, reportingFile)
+	if err := os.WriteFile(filePath, jsonData, 0o600); err != nil {
+		nl.Errorf(l, "failed to write license reporting info to file: %v", err)
+	}
+}
+
+// LicenseReporter can start the license reporting process
+type LicenseReporter struct {
+	config LicenseReporterConfig
+}
+
+// LicenseReporterConfig contains the information needed for license reporting
+type LicenseReporterConfig struct {
+	Period          time.Duration
+	K8sClientReader kubernetes.Interface
+	PodNSName       types.NamespacedName
+}
+
+// NewLicenseReporter creates a new LicenseReporter
+func NewLicenseReporter(client kubernetes.Interface) *LicenseReporter {
+	return &LicenseReporter{
+		config: LicenseReporterConfig{
+			Period:          24 * time.Hour,
+			K8sClientReader: client,
+			PodNSName:       types.NamespacedName{Namespace: os.Getenv("POD_NAMESPACE"), Name: os.Getenv("POD_NAME")},
+		},
+	}
+}
+
+// Start begins the license report writer process for NIC
+func (lr *LicenseReporter) Start(ctx context.Context) {
+	wait.JitterUntilWithContext(ctx, lr.collectAndWrite, lr.config.Period, 0.1, true)
+}
+
+func (lr *LicenseReporter) collectAndWrite(ctx context.Context) {
+	l := nl.LoggerFromContext(ctx)
+	clusterID, err := clusterInfo.GetClusterID(ctx, lr.config.K8sClientReader)
+	if err != nil {
+		nl.Errorf(l, "Error collecting ClusterIDS: %v", err)
+	}
+	nodeCount, err := clusterInfo.GetNodeCount(ctx, lr.config.K8sClientReader)
+	if err != nil {
+		nl.Errorf(l, "Error collecting ClusterNodeCount: %v", err)
+	}
+	installationID, err := clusterInfo.GetInstallationID(ctx, lr.config.K8sClientReader, lr.config.PodNSName)
+	if err != nil {
+		nl.Errorf(l, "Error collecting InstallationID: %v", err)
+	}
+	info := newLicenseInfo(clusterID, installationID, nodeCount)
+	writeLicenseInfo(l, info)
+}

internal/license_reporting/license_reporting_test.go

@@ -0,0 +1,71 @@
+package licensereporting
+
+import (
+	"encoding/json"
+	"io"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"testing"
+
+	nic_glog "github.com/nginxinc/kubernetes-ingress/internal/logger/glog"
+	"github.com/nginxinc/kubernetes-ingress/internal/logger/levels"
+
+	"k8s.io/client-go/kubernetes/fake"
+)
+
+func TestNewLicenseInfo(t *testing.T) {
+	info := newLicenseInfo("test-cluster", "test-installation", 5)
+
+	if info.Integration != "nic" {
+		t.Errorf("newLicenseInfo() Integration = %v, want %v", info.Integration, "nic")
+	}
+	if info.ClusterID != "test-cluster" {
+		t.Errorf("newLicenseInfo() ClusterID = %v, want %v", info.ClusterID, "test-cluster")
+	}
+	if info.InstallationID != "test-installation" {
+		t.Errorf("newLicenseInfo() InstallationID = %v, want %v", info.InstallationID, "test-installation")
+	}
+	if info.ClusterNodeCount != 5 {
+		t.Errorf("newLicenseInfo() ClusterNodeCount = %v, want %v", info.ClusterNodeCount, 5)
+	}
+}
+
+func TestWriteLicenseInfo(t *testing.T) {
+	tempDir := t.TempDir()
+	oldReportingDir := reportingDir
+	reportingDir = tempDir
+	defer func() { reportingDir = oldReportingDir }()
+
+	l := slog.New(nic_glog.New(io.Discard, &nic_glog.Options{Level: levels.LevelInfo}))
+	info := newLicenseInfo("test-cluster", "test-installation", 5)
+	writeLicenseInfo(l, info)
+
+	filePath := filepath.Join(tempDir, reportingFile)
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		t.Fatalf("Expected file %s to exist, but it doesn't", filePath)
+	}
+
+	/* #nosec G304 */
+	content, err := os.ReadFile(filePath)
+	if err != nil {
+		t.Fatalf("Failed to read file: %v", err)
+	}
+
+	var readInfo licenseInfo
+	err = json.Unmarshal(content, &readInfo)
+	if err != nil {
+		t.Fatalf("Failed to unmarshal JSON: %v", err)
+	}
+
+	if readInfo != *info {
+		t.Errorf("Written info does not match original. Got %+v, want %+v", readInfo, *info)
+	}
+}
+
+func TestNewLicenseReporter(t *testing.T) {
+	reporter := NewLicenseReporter(fake.NewSimpleClientset())
+	if reporter == nil {
+		t.Fatal("NewLicenseReporter() returned nil")
+	}
+}

internal/nginx/manager.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	license_reporting "github.com/nginxinc/kubernetes-ingress/internal/license_reporting"
 	nl "github.com/nginxinc/kubernetes-ingress/internal/logger"
 	"github.com/nginxinc/kubernetes-ingress/internal/metrics/collectors"
 
@@ -101,7 +102,7 @@ type Manager interface {
 	DeleteKeyValStateFiles(virtualServerName string)
 }
 
-// LocalManager updates NGINX configuration, starts, reloads and quits NGINX,
+// LocalManager updates NGINX configuration, starts, reloads and quits NGINX, updates License Reporting file
 // updates NGINX Plus upstream servers. It assumes that NGINX is running in the same container.
 type LocalManager struct {
 	confdPath                    string
@@ -119,15 +120,18 @@ type LocalManager struct {
 	plusClient                   *client.NginxClient
 	plusConfigVersionCheckClient *http.Client
 	metricsCollector             collectors.ManagerCollector
+	licenseReporter              *license_reporting.LicenseReporter
+	licenseReporterCancel        context.CancelFunc
 	OpenTracing                  bool
 	appProtectPluginPid          int
 	appProtectDosAgentPid        int
 	agentPid                     int
 	logger                       *slog.Logger
+	nginxPlus                    bool
 }
 
 // NewLocalManager creates a LocalManager.
-func NewLocalManager(ctx context.Context, confPath string, debug bool, mc collectors.ManagerCollector, timeout time.Duration) *LocalManager {
+func NewLocalManager(ctx context.Context, confPath string, debug bool, mc collectors.ManagerCollector, lr *license_reporting.LicenseReporter, timeout time.Duration, nginxPlus bool) *LocalManager {
 	l := nl.LoggerFromContext(ctx)
 	verifyConfigGenerator, err := newVerifyConfigGenerator()
 	if err != nil {
@@ -148,6 +152,8 @@ func NewLocalManager(ctx context.Context, confPath string, debug bool, mc collec
 		configVersion:               0,
 		verifyClient:                newVerifyClient(timeout),
 		metricsCollector:            mc,
+		licenseReporter:             lr,
+		nginxPlus:                   nginxPlus,
 		logger:                      l,
 	}
 
@@ -296,6 +302,19 @@ func (lm *LocalManager) ClearAppProtectFolder(name string) {
 
 // Start starts NGINX.
 func (lm *LocalManager) Start(done chan error) {
+	if lm.nginxPlus {
+		isR33OrGreater, versionErr := lm.Version().PlusGreaterThanOrEqualTo("nginx-plus-r33")
+		if versionErr != nil {
+			nl.Errorf(lm.logger, "Error determining whether nginx version is >= r33: %v", versionErr)
+		}
+		if isR33OrGreater {
+			ctx, cancel := context.WithCancel(context.Background())
+			nl.ContextWithLogger(ctx, lm.logger)
+			go lm.licenseReporter.Start(ctx)
+			lm.licenseReporterCancel = cancel
+		}
+	}
+
 	nl.Debug(lm.logger, "Starting nginx")
 
 	binaryFilename := getBinaryFileName(lm.debug)
@@ -347,6 +366,16 @@ func (lm *LocalManager) Reload(isEndpointsUpdate bool) error {
 func (lm *LocalManager) Quit() {
 	nl.Debugf(lm.logger, "Quitting nginx")
 
+	if lm.nginxPlus {
+		isR33OrGreater, err := lm.Version().PlusGreaterThanOrEqualTo("nginx-plus-r33")
+		if err != nil {
+			nl.Errorf(lm.logger, "Error determining whether nginx version is >= r33: %v", err)
+		}
+		if isR33OrGreater && lm.licenseReporterCancel != nil {
+			lm.licenseReporterCancel()
+		}
+	}
+
 	binaryFilename := getBinaryFileName(lm.debug)
 	if err := shellOut(lm.logger, fmt.Sprintf("%v -s %v", binaryFilename, "quit")); err != nil {
 		nl.Fatalf(lm.logger, "Failed to quit nginx: %v", err)