diff --git a/kube-state-metrics/common_config.libsonnet b/kube-state-metrics/common_config.libsonnet new file mode 100644 index 000000000..65c99bf7b --- /dev/null +++ b/kube-state-metrics/common_config.libsonnet @@ -0,0 +1,129 @@ +local kausal = import 'github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libsonnet'; + +{ + local this = self, + + image:: error 'image is required', + + server_info:: { + port: 8080, + telemetry_host: '0.0.0.0', + telemetry_port: 8081, + }, + + local container = kausal.core.v1.container, + local containerPort = kausal.core.v1.containerPort, + container:: container.new('kube-state-metrics', this.image) + + container.withPorts([ + containerPort.new('ksm', 8080), + containerPort.new('self-metrics', 8081), + ]) + + kausal.util.resourcesRequests('50m', '50Mi') + + kausal.util.resourcesLimits('250m', '150Mi'), + + local policyRule = kausal.rbac.v1.policyRule, + rbac: + kausal.util.rbac('kube-state-metrics', [ + policyRule.new() + + policyRule.withApiGroups(['']) + + policyRule.withResources([ + 'configmaps', + 'secrets', + 'nodes', + 'pods', + 'services', + 'resourcequotas', + 'replicationcontrollers', + 'limitranges', + 'persistentvolumeclaims', + 'persistentvolumes', + 'namespaces', + 'endpoints', + ]) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.new() + + policyRule.withApiGroups(['extensions']) + + policyRule.withResources([ + 'daemonsets', + 'deployments', + 'replicasets', + 'ingresses', + ]) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.new() + + policyRule.withApiGroups(['apps']) + + policyRule.withResources([ + 'daemonsets', + 'deployments', + 'replicasets', + 'statefulsets', + ]) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.new() + + policyRule.withApiGroups(['batch']) + + policyRule.withResources([ + 'cronjobs', + 'jobs', + ]) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.new() + + policyRule.withApiGroups(['autoscaling']) + + policyRule.withResources([ + 'horizontalpodautoscalers', + ]) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.new() + + policyRule.withApiGroups(['authorization.k8s.io']) + + policyRule.withResources(['subjectaccessreviews']) + + policyRule.withVerbs(['create']), + + policyRule.new() + + policyRule.withApiGroups(['ingresses']) + + policyRule.withResources(['ingress']) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.new() + + policyRule.withApiGroups(['policy']) + + policyRule.withResources(['poddisruptionbudgets']) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.new() + + policyRule.withApiGroups(['certificates.k8s.io']) + + policyRule.withResources(['certificatesigningrequests']) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.new() + + policyRule.withApiGroups(['storage.k8s.io']) + + policyRule.withResources([ + 'storageclasses', + 'volumeattachments', + ]) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.new() + + policyRule.withApiGroups(['admissionregistration.k8s.io']) + + policyRule.withResources([ + 'mutatingwebhookconfigurations', + 'validatingwebhookconfigurations', + ]) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.new() + + policyRule.withApiGroups(['networking.k8s.io']) + + policyRule.withResources([ + 'networkpolicies', + 'ingresses', + ]) + + policyRule.withVerbs(['list', 'watch']), + + policyRule.new() + + policyRule.withApiGroups(['coordination.k8s.io']) + + policyRule.withResources(['leases']) + + policyRule.withVerbs(['list', 'watch']), + ]), +} diff --git a/kube-state-metrics/main.libsonnet b/kube-state-metrics/main.libsonnet index 8819bccae..0f03911b5 100644 --- a/kube-state-metrics/main.libsonnet +++ b/kube-state-metrics/main.libsonnet @@ -3,142 +3,31 @@ local kausal = import 'github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libso { new( namespace, + name='kube-state-metrics', image='registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.1.0', + sharded=false, + replicas=3, ):: { local k = kausal { _config+: { namespace: namespace, }, }, - - local container = k.core.v1.container, - local containerPort = k.core.v1.containerPort, - container:: - container.new('kube-state-metrics', image) - + container.withArgs([ - '--port=8080', - '--telemetry-host=0.0.0.0', - '--telemetry-port=8081', - ]) - + container.withPorts([ - containerPort.new('ksm', 8080), - containerPort.new('self-metrics', 8081), - ]) - + k.util.resourcesRequests('50m', '50Mi') - + k.util.resourcesLimits('250m', '150Mi'), - - local deployment = k.apps.v1.deployment, - deployment: - deployment.new('kube-state-metrics', 1, [self.container]) - + deployment.mixin.spec.template.spec.withServiceAccountName(self.rbac.service_account.metadata.name) - + deployment.mixin.spec.template.spec.securityContext.withRunAsUser(65534) - + deployment.mixin.spec.template.spec.securityContext.withRunAsGroup(65534), - - local policyRule = k.rbac.v1.policyRule, - rbac: - k.util.rbac('kube-state-metrics', [ - policyRule.new() - + policyRule.withApiGroups(['']) - + policyRule.withResources([ - 'configmaps', - 'secrets', - 'nodes', - 'pods', - 'services', - 'resourcequotas', - 'replicationcontrollers', - 'limitranges', - 'persistentvolumeclaims', - 'persistentvolumes', - 'namespaces', - 'endpoints', - ]) - + policyRule.withVerbs(['list', 'watch']), - - policyRule.new() - + policyRule.withApiGroups(['extensions']) - + policyRule.withResources([ - 'daemonsets', - 'deployments', - 'replicasets', - 'ingresses', - ]) - + policyRule.withVerbs(['list', 'watch']), - - policyRule.new() - + policyRule.withApiGroups(['apps']) - + policyRule.withResources([ - 'daemonsets', - 'deployments', - 'replicasets', - 'statefulsets', - ]) - + policyRule.withVerbs(['list', 'watch']), - - policyRule.new() - + policyRule.withApiGroups(['batch']) - + policyRule.withResources([ - 'cronjobs', - 'jobs', - ]) - + policyRule.withVerbs(['list', 'watch']), - - policyRule.new() - + policyRule.withApiGroups(['autoscaling']) - + policyRule.withResources([ - 'horizontalpodautoscalers', - ]) - + policyRule.withVerbs(['list', 'watch']), - - policyRule.new() - + policyRule.withApiGroups(['authorization.k8s.io']) - + policyRule.withResources(['subjectaccessreviews']) - + policyRule.withVerbs(['create']), - - policyRule.new() - + policyRule.withApiGroups(['ingresses']) - + policyRule.withResources(['ingress']) - + policyRule.withVerbs(['list', 'watch']), - - policyRule.new() - + policyRule.withApiGroups(['policy']) - + policyRule.withResources(['poddisruptionbudgets']) - + policyRule.withVerbs(['list', 'watch']), - - policyRule.new() - + policyRule.withApiGroups(['certificates.k8s.io']) - + policyRule.withResources(['certificatesigningrequests']) - + policyRule.withVerbs(['list', 'watch']), - - policyRule.new() - + policyRule.withApiGroups(['storage.k8s.io']) - + policyRule.withResources([ - 'storageclasses', - 'volumeattachments', - ]) - + policyRule.withVerbs(['list', 'watch']), - - policyRule.new() - + policyRule.withApiGroups(['admissionregistration.k8s.io']) - + policyRule.withResources([ - 'mutatingwebhookconfigurations', - 'validatingwebhookconfigurations', - ]) - + policyRule.withVerbs(['list', 'watch']), - - policyRule.new() - + policyRule.withApiGroups(['networking.k8s.io']) - + policyRule.withResources([ - 'networkpolicies', - 'ingresses', - ]) - + policyRule.withVerbs(['list', 'watch']), - - policyRule.new() - + policyRule.withApiGroups(['coordination.k8s.io']) - + policyRule.withResources(['leases']) - + policyRule.withVerbs(['list', 'watch']), - ]), + local common_config = { + image:: image, + } + import './common_config.libsonnet', + local singleton_ksm = common_config + import './singleton.libsonnet', + local sharded_ksm = common_config { + replicas:: replicas, + name:: name, + namespace:: namespace, + } + import './sharded.libsonnet', + + deployment: if sharded then {} else singleton_ksm.deployment, + [if sharded then 'statefulset']: sharded_ksm.statefulset, + [if sharded then 'configmaps']: sharded_ksm.configmaps, + + rbac: common_config.rbac, }, scrape_config: (import './scrape_config.libsonnet'), diff --git a/kube-state-metrics/sharded.libsonnet b/kube-state-metrics/sharded.libsonnet new file mode 100644 index 000000000..8a583c5d4 --- /dev/null +++ b/kube-state-metrics/sharded.libsonnet @@ -0,0 +1,68 @@ +local shared_config = import './shared_config.libsonnet'; +local kausal = import 'github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libsonnet'; + +{ + local this = self, + + replicas:: error 'replicas are required', + name:: error 'name is required', + namespace:: error 'namespace is required', + + config_file:: '$(POD_NAME).yml', + config_path:: '/etc/kube-state-metrics', + + rbac:: shared_config.rbac, + ksm_config:: { + port: shared_config.server_info.port, + telemetry_host: shared_config.server_info.telemetry_host, + telemetry_port: shared_config.server_info.telemetry_port, + }, + + local configMap = kausal.core.v1.configMap, + config_map: configMap.new('%s-config' % this.name) + + configMap.withData({ + ['%s-%s.yml' % [this.name, i]]: + kausal.util.manifestYaml( + this.ksm_config + { + total_shards: this.replicas, + shard: i, + } + ) + for i in std.range(0, this.replicas - 1) + }), + + local container = kausal.core.v1.container, + local containerPort = kausal.core.v1.containerPort, + local envVar = kausal.core.v1.envVar, + container:: shared_config.container + + container.withArgs([ + '--config=%s' % std.join('/', [self.config_path, self.config_file]), + ]) + + container.withEnvMixin([ + envVar.fromFieldPath('POD_NAME', 'metadata.name'), + ]), + + local statefulSet = kausal.apps.v1.statefulSet, + statefulset: + statefulSet.new(this.name, this.replicas, [self.container]) + + statefulSet.spec.withServiceName(this.name) + + statefulSet.mixin.spec.template.spec.withServiceAccountName(self.rbac.service_account.metadata.name) + + statefulSet.mixin.spec.template.spec.securityContext.withRunAsUser(65534) + + statefulSet.mixin.spec.template.spec.securityContext.withRunAsGroup(65534) + + statefulSet.configMapVolumeMount(self.config_map, self.config_path) + + statefulSet.spec.template.spec.affinity.podAntiAffinity.withRequiredDuringSchedulingIgnoredDuringExecution( + kausal.core.v1.podAffinityTerm.withTopologyKey('kubernetes.io/hostname') + + kausal.core.v1.podAffinityTerm.labelSelector.withMatchLabels({ + name: this.name, + }) + ), + + local podDisruptionBudget = kausal.policy.v1.podDisruptionBudget, + ksm_pdb: + podDisruptionBudget.new(this.name) + + podDisruptionBudget.mixin.metadata.withNamespace(this.namespace) + + podDisruptionBudget.mixin.metadata.withLabels({ name: '%s-pdb' % this.name }) + + podDisruptionBudget.mixin.spec.selector.withMatchLabels({ name: this.name }) + + podDisruptionBudget.mixin.spec.withMaxUnavailable(1), +} diff --git a/kube-state-metrics/singleton.libsonnet b/kube-state-metrics/singleton.libsonnet new file mode 100644 index 000000000..1ff23ce4e --- /dev/null +++ b/kube-state-metrics/singleton.libsonnet @@ -0,0 +1,21 @@ +local shared_config = import './shared_config.libsonnet'; +local kausal = import 'github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libsonnet'; + +{ + local container = kausal.core.v1.container, + local containerPort = kausal.core.v1.containerPort, + container+:: container.withArgs([ + '--port=%s' % shared_config.server_info.port, + '--telemetry-host=%s' % shared_config.server_info.telemetry_host, + '--telemetry-port=%s' % shared_config.server_info.telemetry_port, + ]), + + local deployment = kausal.apps.v1.deployment, + deployment: + deployment.new('kube-state-metrics', 1, [self.container]) + + deployment.mixin.spec.template.spec.withServiceAccountName(self.rbac.service_account.metadata.name) + + deployment.mixin.spec.template.spec.securityContext.withRunAsUser(65534) + + deployment.mixin.spec.template.spec.securityContext.withRunAsGroup(65534), + + rbac:: shared_config.rbac, +}