diff --git a/kompose b/kompose new file mode 100755 index 000000000..7670ab6ef Binary files /dev/null and b/kompose differ diff --git a/projects/kafka-cron/alert_rules.yml b/projects/kafka-cron/alert_rules.yml new file mode 100644 index 000000000..e69de29bb diff --git a/projects/kafka-cron/alertmanager.conf b/projects/kafka-cron/alertmanager.conf new file mode 100644 index 000000000..8c0a79b2c --- /dev/null +++ b/projects/kafka-cron/alertmanager.conf @@ -0,0 +1,13 @@ +# Minimal alertmanager config that doesn't send alerts anywhere - +# Visible at http://localhost:9093/#/alerts +global: + resolve_timeout: 5m +route: + receiver: "null" + group_by: + - job + group_wait: 30s + group_interval: 5m + repeat_interval: 12h +receivers: +- name: "null" \ No newline at end of file diff --git a/projects/kafka-cron/cmds/consumer/consumer.go b/projects/kafka-cron/cmds/consumer/consumer.go new file mode 100644 index 000000000..9cef01ac9 --- /dev/null +++ b/projects/kafka-cron/cmds/consumer/consumer.go @@ -0,0 +1,151 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "kafka-cron/configs" + "kafka-cron/message" + "os" + "os/exec" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/confluentinc/confluent-kafka-go/kafka" +) + +func main() { + var bootstrapServers, clusterName string + var retry bool + flag.StringVar(&bootstrapServers, "bootstrap.servers", "127.0.0.1:9092", "bootstrap servers") + flag.StringVar(&clusterName, "cluster", "cluster-a", "cluster name") + flag.BoolVar(&retry, "retry", false, "retry worker") + flag.Parse() + + c, err := kafka.NewConsumer(&kafka.ConfigMap{ + "bootstrap.servers": bootstrapServers, + "group.id": "foo", + }) + if err != nil { + fmt.Printf("Failed to create consumer: %s", err) + os.Exit(1) + } + + topic := configs.GetTopicName(clusterName) + if retry { + topic = configs.GetRetryTopicName(clusterName) + } + + err = c.Subscribe(topic, nil) + if err != nil { + fmt.Printf("Failed to subscribe to topic: %s %v\n", topic, err) + os.Exit(1) + } else { + fmt.Printf("Subscribed to topic: %s\n", topic) + } + + // Create Producer instance for retrying + p, err := kafka.NewProducer(&kafka.ConfigMap{ + "bootstrap.servers": bootstrapServers}) + if err != nil { + fmt.Printf("Failed to create producer: %s", err) + os.Exit(1) + } + + // Set up a channel for handling Ctrl-C, etc + sigchan := make(chan os.Signal, 1) + signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + + // Process messages + run := true + for run { + select { + case sig := <-sigchan: + fmt.Printf("Caught signal %v: terminating\n", sig) + run = false + default: + msg, err := c.ReadMessage(100 * time.Millisecond) + if err != nil { + // Errors are informational and automatically handled by the consumer + continue + } + recordKey := string(msg.Key) + recordValue := msg.Value + data := message.CronjobMessage{} + err = json.Unmarshal(recordValue, &data) + if err != nil { + fmt.Printf("Failed to decode JSON at offset %d: %v", msg.TopicPartition.Offset, err) + continue + } + fmt.Printf("Consumed record with key %s and value %s\n", recordKey, recordValue) + + args := strings.Split(data.Command, " ") + cmd := exec.Command(args[0], args[1:]...) + err = cmd.Run() + + if err != nil { + fmt.Printf("Failed to run command %s - re-enqueuing on retry queue: %v\n", data.Command, err) + if data.Retries > 1 { + queueRetryJob(p, data, msg.Key, clusterName) + } else { + // todo dlq + fmt.Printf("To the dead letter queue\n") + queueDLQ(p, data, msg.Key) + } + } else { + fmt.Printf("Command ran OK %s\n", data.Command) + } + } + } + + fmt.Printf("Closing consumer\n") + c.Close() +} + +func queueRetryJob(p *kafka.Producer, msg message.CronjobMessage, key []byte, cluster string) { + msg.Retries = msg.Retries - 1 + + jmsg, err := json.Marshal(msg) + if err != nil { + fmt.Println(err) + return + } + + topic := configs.GetRetryTopicName(cluster) + fmt.Printf("queuing job: %s on topic %s\n", string(jmsg), topic) + err = p.Produce(&kafka.Message{ + TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny}, + Key: key, + Value: jmsg}, + nil, + ) + if err != nil { + fmt.Println(err) + return + } +} + +func queueDLQ(p *kafka.Producer, msg message.CronjobMessage, key []byte) { + msg.Retries = 0 + + jmsg, err := json.Marshal(msg) + if err != nil { + fmt.Println(err) + return + } + + topic := configs.GetDLQName() + fmt.Printf("queuing job: %s on topic %s\n", string(jmsg), topic) + err = p.Produce(&kafka.Message{ + TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny}, + Key: key, + Value: jmsg}, + nil, + ) + if err != nil { + fmt.Println(err) + return + } +} diff --git a/projects/kafka-cron/cmds/producer/producer.go b/projects/kafka-cron/cmds/producer/producer.go new file mode 100644 index 000000000..d5c89e3d6 --- /dev/null +++ b/projects/kafka-cron/cmds/producer/producer.go @@ -0,0 +1,178 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "flag" + "fmt" + "kafka-cron/configs" + "kafka-cron/message" + "log" + "os" + "os/signal" + "strconv" + "strings" + "time" + + "github.com/confluentinc/confluent-kafka-go/kafka" + "github.com/google/uuid" + "github.com/robfig/cron/v3" +) + +type cronjob struct { + crontab string + command string + name string + cluster string + retries int +} + +func main() { + var bootstrapServers, configPath string + var partitions int + + flag.StringVar(&bootstrapServers, "bootstrap.servers", "127.0.0.1:9092", "bootstrap servers") + flag.StringVar(&configPath, "config", "./data/cronjobs.txt", "path to cronjob spec file") + flag.IntVar(&partitions, "partitions", 1, "number of partitions per topic") + flag.Parse() + + // Create Producer instance + p, err := kafka.NewProducer(&kafka.ConfigMap{ + "bootstrap.servers": bootstrapServers}) + if err != nil { + fmt.Printf("Failed to create producer: %s", err) + os.Exit(1) + } + + // Create topics if needed + for _, cluster := range configs.GetClusters() { + CreateTopic(p, configs.GetTopicName(cluster), partitions) + CreateTopic(p, configs.GetRetryTopicName(cluster), partitions) + } + CreateTopic(p, configs.GetDLQName(), partitions) + + // Parse cronjobs and create schedule + cronjobs := parseCronjobs(configPath) + fmt.Printf("cronjobs: %v\n", cronjobs) + + var secondParser = cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.DowOptional | cron.Descriptor) + c := cron.New(cron.WithParser(secondParser), cron.WithChain()) + for _, j := range cronjobs { + j := j // for closure + c.AddFunc(j.crontab, func() { + queueJob(p, j.command, j.command, j.cluster, j.retries) + }) + fmt.Printf("cronjobs: started cron for %+v\n", j) + } + go c.Start() + + fmt.Printf("cronjobs: cron entries are %+v\n", c.Entries()) + + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, os.Interrupt, os.Kill) + <-signalChan +} + +func queueJob(p *kafka.Producer, command string, name string, cluster string, retries int) { + msg := message.CronjobMessage{ + Command: strings.TrimSpace(command), + Exectime: time.Now().Format(time.RFC3339), + Name: strings.TrimSpace(name), + Retries: retries, + } + + jmsg, err := json.Marshal(msg) + if err != nil { + fmt.Println(err) + return + } + + key, _ := uuid.New().MarshalText() + topic := configs.GetTopicName(cluster) + fmt.Printf("queuing job: %s on topic %s\n", string(jmsg), topic) + err = p.Produce(&kafka.Message{ + TopicPartition: kafka.TopicPartition{Topic: &topic, Partition: kafka.PartitionAny}, + Key: key, + Value: jmsg}, + nil, + ) + if err != nil { + fmt.Println(err) + return + } +} + +func parseCronjobs(path string) []cronjob { + result := make([]cronjob, 0) + + f, err := os.Open(path) + + if err != nil { + log.Fatal(err) + } + + defer f.Close() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + val := scanner.Text() + fields := strings.Split(val, ",") + + if len(fields) != 5 { + log.Fatal(fmt.Errorf("wrong format %s %d fields, expect 5", val, len(fields))) + } + + retries, err := strconv.Atoi(strings.TrimSpace(fields[4])) + if err != nil { + log.Fatal(fmt.Errorf("wrong format could not parse %s as int (num retries)", fields[4])) + } + + cron := cronjob{fields[0], fields[1], fields[2], strings.TrimSpace(fields[3]), retries} + result = append(result, cron) + } + return result +} + +// CreateTopic creates a topic using the Admin Client API +func CreateTopic(p *kafka.Producer, topic string, partitions int) { + + a, err := kafka.NewAdminClientFromProducer(p) + if err != nil { + fmt.Printf("Failed to create new admin client from producer: %s", err) + os.Exit(1) + } + // Contexts are used to abort or limit the amount of time + // the Admin call blocks waiting for a result. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // Create topics on cluster. + // Set Admin options to wait up to 60s for the operation to finish on the remote cluster + maxDur, err := time.ParseDuration("60s") + if err != nil { + fmt.Printf("ParseDuration(60s): %s", err) + os.Exit(1) + } + results, err := a.CreateTopics( + ctx, + // Multiple topics can be created simultaneously + // by providing more TopicSpecification structs here. + []kafka.TopicSpecification{{ + Topic: topic, + NumPartitions: partitions, + ReplicationFactor: 1}}, + // Admin options + kafka.SetAdminOperationTimeout(maxDur)) + if err != nil { + fmt.Printf("Admin Client request error: %v\n", err) + os.Exit(1) + } + for _, result := range results { + if result.Error.Code() != kafka.ErrNoError && result.Error.Code() != kafka.ErrTopicAlreadyExists { + fmt.Printf("Failed to create topic: %v\n", result.Error) + os.Exit(1) + } + fmt.Printf("%v\n", result) + } + a.Close() + +} diff --git a/projects/kafka-cron/configs/cluster.go b/projects/kafka-cron/configs/cluster.go new file mode 100644 index 000000000..b547167a6 --- /dev/null +++ b/projects/kafka-cron/configs/cluster.go @@ -0,0 +1,24 @@ +package configs + +// Hardcoded, for demo purposes. + +import ( + "fmt" +) + +func GetClusters() []string { + clusters := []string{"cluster-a", "cluster-b"} + return clusters +} + +func GetTopicName(cluster string) string { + return fmt.Sprintf("%s-cronjobs", cluster) +} + +func GetRetryTopicName(cluster string) string { + return fmt.Sprintf("%s-cronjobs-retry", cluster) +} + +func GetDLQName() string { + return "dead-letter-queue" +} diff --git a/projects/kafka-cron/data/cronjobs copy.txt b/projects/kafka-cron/data/cronjobs copy.txt new file mode 100644 index 000000000..9f84c1eca --- /dev/null +++ b/projects/kafka-cron/data/cronjobs copy.txt @@ -0,0 +1,3 @@ +* * * * * *, /sbin/ping -c 1 www.microsoft.com, ms-pinger, cluster-a, 3 +* * * * * *, /sbin/ping -c 1 www.google.com, google-pinger, cluster-a, 3 +* * * * * *, badcmd -c 1 www.google.com, badjob, cluster-a, 3 \ No newline at end of file diff --git a/projects/kafka-cron/data/cronjobs.txt b/projects/kafka-cron/data/cronjobs.txt new file mode 100644 index 000000000..83c790128 --- /dev/null +++ b/projects/kafka-cron/data/cronjobs.txt @@ -0,0 +1 @@ +* * * * *, badcmd -c 1 www.google.com, badjob, cluster-a, 3 \ No newline at end of file diff --git a/projects/kafka-cron/docker-compose.yml b/projects/kafka-cron/docker-compose.yml new file mode 100644 index 000000000..ecd7c14c4 --- /dev/null +++ b/projects/kafka-cron/docker-compose.yml @@ -0,0 +1,85 @@ +# TODO add services for consumer and producer +version: "2" + +services: + zoo1: + image: confluentinc/cp-zookeeper:7.3.0 + hostname: zoo1 + container_name: zoo1 + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_SERVER_ID: 1 + ZOOKEEPER_SERVERS: zoo1:2888:3888 + + kafka1: + image: confluentinc/cp-kafka:7.3.0 + hostname: kafka1 + container_name: kafka1 + ports: + - "9092:9092" + - "29092:29092" + - "9999:9999" + - "8999:8999" + volumes: + - ./kafka-jmxagent/config.yaml:/kafka-jmx-config.yaml + environment: + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" + KAFKA_BROKER_ID: 1 + KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_JMX_PORT: 9999 + KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1} + KAFKA_OPTS: -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.14.0.jar=8999:/kafka-jmx-config.yaml + KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" + depends_on: + - zoo1 + + alertmanager: + container_name: alertmanager + hostname: alertmanager + image: prom/alertmanager + volumes: + - ./alertmanager.conf:/etc/alertmanager/alertmanager.conf + command: + - '--config.file=/etc/alertmanager/alertmanager.conf' + ports: + - 9093:9093 + + prometheus: + container_name: prometheus + hostname: prometheus + image: prom/prometheus + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - ./alert_rules.yml:/etc/prometheus/alert_rules.yml + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + links: + - alertmanager:alertmanager + ports: + - 9090:9090 + + grafana: + container_name: grafana + hostname: grafana + image: grafana/grafana + volumes: + - ./grafana_datasources.yml:/etc/grafana/provisioning/datasources/all.yaml + - ./grafana_config.ini:/etc/grafana/config.ini + - grafana_data:/var/lib/grafana + ports: + - 3000:3000 + +# Names our volume +volumes: + prometheus_data: + grafana_data: \ No newline at end of file diff --git a/projects/kafka-cron/go.mod b/projects/kafka-cron/go.mod new file mode 100644 index 000000000..87da07a47 --- /dev/null +++ b/projects/kafka-cron/go.mod @@ -0,0 +1,15 @@ +module kafka-cron + +go 1.19 + +require ( + github.com/confluentinc/confluent-kafka-go v1.9.2 + github.com/google/uuid v1.3.0 + github.com/robfig/cron/v3 v3.0.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/go-redis/redis/v9 v9.0.0-rc.1 // indirect +) diff --git a/projects/kafka-cron/go.sum b/projects/kafka-cron/go.sum new file mode 100644 index 000000000..2fca8f6d4 --- /dev/null +++ b/projects/kafka-cron/go.sum @@ -0,0 +1,222 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/actgardner/gogen-avro/v10 v10.1.0/go.mod h1:o+ybmVjEa27AAr35FRqU98DJu1fXES56uXniYFv4yDA= +github.com/actgardner/gogen-avro/v10 v10.2.1/go.mod h1:QUhjeHPchheYmMDni/Nx7VB0RsT/ee8YIgGY/xpEQgQ= +github.com/actgardner/gogen-avro/v9 v9.1.0/go.mod h1:nyTj6wPqDJoxM3qdnjcLv+EnMDSDFqE0qDpva2QRmKc= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/confluentinc/confluent-kafka-go v1.9.2 h1:gV/GxhMBUb03tFWkN+7kdhg+zf+QUM+wVkI9zwh770Q= +github.com/confluentinc/confluent-kafka-go v1.9.2/go.mod h1:ptXNqsuDfYbAE/LBW6pnwWZElUoWxHoV8E43DCrliyo= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.2.2/go.mod h1:Qh/WofXFeiAFII1aEBu529AtJo6Zg2VHscnEsbBnJ20= +github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= +github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= +github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/go-redis/redis/v9 v9.0.0-rc.1 h1:/+bS+yeUnanqAbuD3QwlejzQZ+4eqgfUtFTG4b+QnXs= +github.com/go-redis/redis/v9 v9.0.0-rc.1/go.mod h1:8et+z03j0l8N+DvsVnclzjf3Dl/pFHgRk+2Ct1qw66A= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hamba/avro v1.5.6/go.mod h1:3vNT0RLXXpFm2Tb/5KC71ZRJlOroggq1Rcitb6k4Fr8= +github.com/heetch/avro v0.3.1/go.mod h1:4xn38Oz/+hiEUTpbVfGVLfvOg0yKLlRP7Q9+gJJILgA= +github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= +github.com/invopop/jsonschema v0.4.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0= +github.com/jhump/gopoet v0.0.0-20190322174617-17282ff210b3/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= +github.com/jhump/gopoet v0.1.0/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= +github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7UvLHRQ= +github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuTd3Z9nFXJf5E= +github.com/jhump/protoreflect v1.12.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/juju/qthttptest v0.1.1/go.mod h1:aTlAv8TYaflIiTDIQYzxnl1QdPjAg8Q8qJMErpKy6A4= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM= +github.com/linkedin/goavro/v2 v2.10.0/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= +github.com/linkedin/goavro/v2 v2.10.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= +github.com/linkedin/goavro/v2 v2.11.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/nrwiersma/avro-benchmarks v0.0.0-20210913175520-21aec48c8f76/go.mod h1:iKyFMidsk/sVYONJRE372sJuX/QTRPacU7imPqqsu7g= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= +github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a/go.mod h1:4r5QyqhjIWCcK8DO4KMclc5Iknq5qVBAlbYYzAbUScQ= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200505041828-1ed23360d12c/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183/go.mod h1:FvqrFXt+jCsyQibeRv4xxEJBL5iG2DDW5aeJwzDiq4A= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v1 v1.0.0/go.mod h1:CxwszS/Xz1C49Ucd2i6Zil5UToP1EmyrFhKaMVbg1mk= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/httprequest.v1 v1.2.1/go.mod h1:x2Otw96yda5+8+6ZeWwHIJTFkEHWP/qP8pJOzqEtWPM= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/retry.v1 v1.0.3/go.mod h1:FJkXmWiMaAo7xB+xhvDF59zhfjDWyzmyAxiT4dB688g= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/projects/kafka-cron/grafana_config.ini b/projects/kafka-cron/grafana_config.ini new file mode 100644 index 000000000..e69de29bb diff --git a/projects/kafka-cron/grafana_datasources.yml b/projects/kafka-cron/grafana_datasources.yml new file mode 100644 index 000000000..e69de29bb diff --git a/projects/kafka-cron/jmx-exporter/Dockerfile b/projects/kafka-cron/jmx-exporter/Dockerfile new file mode 100644 index 000000000..5541c760f --- /dev/null +++ b/projects/kafka-cron/jmx-exporter/Dockerfile @@ -0,0 +1,6 @@ +FROM openjdk:17 +RUN mkdir -p /jmx +COPY ./jmx_prometheus_httpserver-0.17.2.jar /jmx/ +COPY ./config.yaml /jmx/ +WORKDIR /jmx +ENTRYPOINT ["java","-jar","/jmx/jmx_prometheus_httpserver-0.17.2.jar","8999","/jmx/config.yaml"] \ No newline at end of file diff --git a/projects/kafka-cron/jmx-exporter/config.yaml b/projects/kafka-cron/jmx-exporter/config.yaml new file mode 100644 index 000000000..dd7eba355 --- /dev/null +++ b/projects/kafka-cron/jmx-exporter/config.yaml @@ -0,0 +1,6 @@ +jmxUrl: service:jmx:rmi:///jndi/rmi://kafka1:9999/jmxrmi +ssl: false +username: +password: +rules: +- pattern: ".*" \ No newline at end of file diff --git a/projects/kafka-cron/jmx-exporter/jmx_prometheus_httpserver-0.17.2.jar b/projects/kafka-cron/jmx-exporter/jmx_prometheus_httpserver-0.17.2.jar new file mode 100644 index 000000000..2e96ba64e Binary files /dev/null and b/projects/kafka-cron/jmx-exporter/jmx_prometheus_httpserver-0.17.2.jar differ diff --git a/projects/kafka-cron/kafka-jmxagent/Dockerfile b/projects/kafka-cron/kafka-jmxagent/Dockerfile new file mode 100644 index 000000000..85e30d673 --- /dev/null +++ b/projects/kafka-cron/kafka-jmxagent/Dockerfile @@ -0,0 +1,4 @@ +FROM confluentinc/cp-kafka:7.3.0 +USER root +RUN mkdir -p /prometheus/ +COPY ./config.yaml /prometheus/config.yaml diff --git a/projects/kafka-cron/kafka-jmxagent/build.sh b/projects/kafka-cron/kafka-jmxagent/build.sh new file mode 100755 index 000000000..0b6d3e428 --- /dev/null +++ b/projects/kafka-cron/kafka-jmxagent/build.sh @@ -0,0 +1 @@ +sudo docker build --tag=lnolan-cp-kafka . \ No newline at end of file diff --git a/projects/kafka-cron/kafka-jmxagent/config copy.yaml b/projects/kafka-cron/kafka-jmxagent/config copy.yaml new file mode 100644 index 000000000..197342a69 --- /dev/null +++ b/projects/kafka-cron/kafka-jmxagent/config copy.yaml @@ -0,0 +1,58 @@ +lowercaseOutputName: true +rules: +- pattern: '"kafka.consumer"<>(Count)' + name: kafka_consumer_$1_$3_$4 + labels: + thread: $2 +- pattern: '"kafka.consumer"<>(Count|Value)' + name: kafka_consumer_$1_$5_$6 + labels: + hostport: $3:$4 + thread: $2 +- name: kafka_consumer_$1_$7_$8 + pattern: '"kafka.consumer"<>(Count|Value)' + labels: + hostport: $3:$4 + partition: $6 + thread: $2 + topic: $5 +- pattern: '"kafka.server"<>Count' + name: kafka_server_$1_$5_total + labels: + hostport: $3:$4 + thread: $2 +- pattern: '"kafka.server"<>(Count|Value)' + name: kafka_server_$1_$7_$8 + labels: + hostport: $3:$4 + partition: $6 + thread: $2 + topic: $5 +- name: kafka_cluster_$1_$4 + pattern: '"kafka.cluster"<>Value' + labels: + partition: $3 + topic: $2 +- pattern: '"kafka.log"<>Value' + name: kafka_log_$3 + labels: + partition: $2 + topic: $1 +- pattern: '"kafka.server"<>Count' + name: kafka_server_$1_$3_total + labels: + topic: $2 + type: COUNTER +- pattern: '"kafka.server"<>(Count|Value)' + name: kafka_server_$1_$3_$4 + labels: + topic: $2 +- pattern: '"kafka.network"<>Value' + name: kafka_network_$1_$3 + labels: + processor: $2 +- pattern: '"kafka.(\w+)"<>Count' + name: kafka_$1_$2_$3_total + type: COUNTER +- pattern: '"kafka.(\w+)"<>(Count|Value)' + name: kafka_$1_$2_$3_$4 \ No newline at end of file diff --git a/projects/kafka-cron/kafka-jmxagent/config.yaml b/projects/kafka-cron/kafka-jmxagent/config.yaml new file mode 100644 index 000000000..e82d043f5 --- /dev/null +++ b/projects/kafka-cron/kafka-jmxagent/config.yaml @@ -0,0 +1,2 @@ +rules: +- pattern: ".*" \ No newline at end of file diff --git a/projects/kafka-cron/kafka-jmxagent/jmx_prometheus_javaagent-0.17.2.jar b/projects/kafka-cron/kafka-jmxagent/jmx_prometheus_javaagent-0.17.2.jar new file mode 100644 index 000000000..48b3e4d84 Binary files /dev/null and b/projects/kafka-cron/kafka-jmxagent/jmx_prometheus_javaagent-0.17.2.jar differ diff --git a/projects/kafka-cron/kafka-jmxagent/jmx_prometheus_javaagent_java6-0.17.2.jar b/projects/kafka-cron/kafka-jmxagent/jmx_prometheus_javaagent_java6-0.17.2.jar new file mode 100644 index 000000000..4ffba1dfd Binary files /dev/null and b/projects/kafka-cron/kafka-jmxagent/jmx_prometheus_javaagent_java6-0.17.2.jar differ diff --git a/projects/kafka-cron/kubeconfigs/alertmanager-configmap.yaml b/projects/kafka-cron/kubeconfigs/alertmanager-configmap.yaml new file mode 100644 index 000000000..5ac9b8380 --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/alertmanager-configmap.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: amconfig +data: + # file-like keys + alertmanager.conf: | + # Minimal alertmanager config that doesn't send alerts anywhere - + # Visible at http://localhost:9093/#/alerts + global: + resolve_timeout: 5m + route: + receiver: "null" + group_by: + - job + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receivers: + - name: "null" \ No newline at end of file diff --git a/projects/kafka-cron/kubeconfigs/alertmanager-deployment.yaml b/projects/kafka-cron/kubeconfigs/alertmanager-deployment.yaml new file mode 100644 index 000000000..136ad2d4d --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/alertmanager-deployment.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: ../../kompose convert + kompose.version: 1.26.0 (40646f47) + creationTimestamp: null + labels: + io.kompose.service: alertmanager + name: alertmanager +spec: + replicas: 1 + selector: + matchLabels: + io.kompose.service: alertmanager + strategy: + type: Recreate + template: + metadata: + annotations: + kompose.cmd: ../../kompose convert + kompose.version: 1.26.0 (40646f47) + creationTimestamp: null + labels: + io.kompose.service: alertmanager + spec: + containers: + - args: + - --config.file=/etc/alertmanager/alertmanager.conf + image: prom/alertmanager + name: alertmanager + ports: + - containerPort: 9093 + resources: {} + volumeMounts: + - mountPath: /etc/alertmanager/ + name: amconfig + hostname: alertmanager + restartPolicy: Always + volumes: + - name: amconfig + configMap: + name: amconfig +status: {} diff --git a/projects/kafka-cron/kubeconfigs/alertmanager-service.yaml b/projects/kafka-cron/kubeconfigs/alertmanager-service.yaml new file mode 100644 index 000000000..7f8ac9ada --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/alertmanager-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: ../../kompose convert + kompose.version: 1.26.0 (40646f47) + creationTimestamp: null + labels: + io.kompose.service: alertmanager + name: alertmanager +spec: + ports: + - name: "9093" + port: 9093 + targetPort: 9093 + selector: + io.kompose.service: alertmanager +status: + loadBalancer: {} diff --git a/projects/kafka-cron/kubeconfigs/grafana-claim0-persistentvolumeclaim.yaml b/projects/kafka-cron/kubeconfigs/grafana-claim0-persistentvolumeclaim.yaml new file mode 100644 index 000000000..d0865dd01 --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/grafana-claim0-persistentvolumeclaim.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + creationTimestamp: null + labels: + io.kompose.service: grafana-claim0 + name: grafana-claim0 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi +status: {} diff --git a/projects/kafka-cron/kubeconfigs/grafana-claim1-persistentvolumeclaim.yaml b/projects/kafka-cron/kubeconfigs/grafana-claim1-persistentvolumeclaim.yaml new file mode 100644 index 000000000..0f5850a99 --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/grafana-claim1-persistentvolumeclaim.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + creationTimestamp: null + labels: + io.kompose.service: grafana-claim1 + name: grafana-claim1 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi +status: {} diff --git a/projects/kafka-cron/kubeconfigs/grafana-data-persistentvolumeclaim.yaml b/projects/kafka-cron/kubeconfigs/grafana-data-persistentvolumeclaim.yaml new file mode 100644 index 000000000..a7faf70b8 --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/grafana-data-persistentvolumeclaim.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + creationTimestamp: null + labels: + io.kompose.service: grafana-data + name: grafana-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi +status: {} diff --git a/projects/kafka-cron/kubeconfigs/grafana-deployment.yaml b/projects/kafka-cron/kubeconfigs/grafana-deployment.yaml new file mode 100644 index 000000000..b0b0e329f --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/grafana-deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: ../../kompose convert + kompose.version: 1.26.0 (40646f47) + creationTimestamp: null + labels: + io.kompose.service: grafana + name: grafana +spec: + replicas: 1 + selector: + matchLabels: + io.kompose.service: grafana + strategy: + type: Recreate + template: + metadata: + annotations: + kompose.cmd: ../../kompose convert + kompose.version: 1.26.0 (40646f47) + creationTimestamp: null + labels: + io.kompose.service: grafana + spec: + containers: + - image: grafana/grafana + name: grafana + ports: + - containerPort: 3000 + resources: {} + volumeMounts: + - mountPath: /etc/grafana/provisioning/datasources/all.yaml + name: grafana-claim0 + - mountPath: /etc/grafana/config.ini + name: grafana-claim1 + - mountPath: /var/lib/grafana + name: grafana-data + hostname: grafana + restartPolicy: Always + volumes: + - name: grafana-claim0 + persistentVolumeClaim: + claimName: grafana-claim0 + - name: grafana-claim1 + persistentVolumeClaim: + claimName: grafana-claim1 + - name: grafana-data + persistentVolumeClaim: + claimName: grafana-data +status: {} diff --git a/projects/kafka-cron/kubeconfigs/grafana-service.yaml b/projects/kafka-cron/kubeconfigs/grafana-service.yaml new file mode 100644 index 000000000..da193f614 --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/grafana-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: ../../kompose convert + kompose.version: 1.26.0 (40646f47) + creationTimestamp: null + labels: + io.kompose.service: grafana + name: grafana +spec: + ports: + - name: "3000" + port: 3000 + targetPort: 3000 + selector: + io.kompose.service: grafana +status: + loadBalancer: {} diff --git a/projects/kafka-cron/kubeconfigs/kafka1-claim0-persistentvolumeclaim.yaml b/projects/kafka-cron/kubeconfigs/kafka1-claim0-persistentvolumeclaim.yaml new file mode 100644 index 000000000..37cda4b77 --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/kafka1-claim0-persistentvolumeclaim.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + creationTimestamp: null + labels: + io.kompose.service: kafka1-claim0 + name: kafka1-claim0 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi +status: {} diff --git a/projects/kafka-cron/kubeconfigs/kafka1-deployment.yaml b/projects/kafka-cron/kubeconfigs/kafka1-deployment.yaml new file mode 100644 index 000000000..750242e22 --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/kafka1-deployment.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: ../../kompose convert + kompose.version: 1.26.0 (40646f47) + creationTimestamp: null + labels: + io.kompose.service: kafka1 + name: kafka1 +spec: + replicas: 1 + selector: + matchLabels: + io.kompose.service: kafka1 + strategy: + type: Recreate + template: + metadata: + annotations: + kompose.cmd: ../../kompose convert + kompose.version: 1.26.0 (40646f47) + creationTimestamp: null + labels: + io.kompose.service: kafka1 + spec: + containers: + - env: + - name: KAFKA_ADVERTISED_LISTENERS + value: INTERNAL://kafka1:19092,EXTERNAL://127.0.0.1:9092,DOCKER://host.docker.internal:29092 + - name: KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND + value: "true" + - name: KAFKA_AUTHORIZER_CLASS_NAME + value: kafka.security.authorizer.AclAuthorizer + - name: KAFKA_BROKER_ID + value: "1" + - name: KAFKA_INTER_BROKER_LISTENER_NAME + value: INTERNAL + - name: KAFKA_JMX_HOSTNAME + value: 127.0.0.1 + - name: KAFKA_JMX_PORT + value: "9999" + - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP + value: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT + - name: KAFKA_LOG4J_LOGGERS + value: kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO + - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR + value: "1" + - name: KAFKA_OPTS + value: -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.14.0.jar=8999:/kafka-jmx-config.yaml + - name: KAFKA_TRANSACTION_STATE_LOG_MIN_ISR + value: "1" + - name: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR + value: "1" + - name: KAFKA_ZOOKEEPER_CONNECT + value: zoo1:2181 + image: confluentinc/cp-kafka:7.3.0 + name: kafka1 + ports: + - containerPort: 9092 + - containerPort: 29092 + - containerPort: 9999 + - containerPort: 8999 + resources: {} + volumeMounts: + - mountPath: /kafka-jmx-config.yaml + name: kafka1-claim0 + hostname: kafka1 + restartPolicy: Always + volumes: + - name: kafka1-claim0 + persistentVolumeClaim: + claimName: kafka1-claim0 +status: {} diff --git a/projects/kafka-cron/kubeconfigs/kafka1-service.yaml b/projects/kafka-cron/kubeconfigs/kafka1-service.yaml new file mode 100644 index 000000000..b60168c06 --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/kafka1-service.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: ../../kompose convert + kompose.version: 1.26.0 (40646f47) + creationTimestamp: null + labels: + io.kompose.service: kafka1 + name: kafka1 +spec: + ports: + - name: "9092" + port: 9092 + targetPort: 9092 + - name: "29092" + port: 29092 + targetPort: 29092 + - name: "9999" + port: 9999 + targetPort: 9999 + - name: "8999" + port: 8999 + targetPort: 8999 + selector: + io.kompose.service: kafka1 +status: + loadBalancer: {} diff --git a/projects/kafka-cron/kubeconfigs/prometheus-claim0-persistentvolumeclaim.yaml b/projects/kafka-cron/kubeconfigs/prometheus-claim0-persistentvolumeclaim.yaml new file mode 100644 index 000000000..282635cba --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/prometheus-claim0-persistentvolumeclaim.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + creationTimestamp: null + labels: + io.kompose.service: prometheus-claim0 + name: prometheus-claim0 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi +status: {} diff --git a/projects/kafka-cron/kubeconfigs/prometheus-claim1-persistentvolumeclaim.yaml b/projects/kafka-cron/kubeconfigs/prometheus-claim1-persistentvolumeclaim.yaml new file mode 100644 index 000000000..99ca0f131 --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/prometheus-claim1-persistentvolumeclaim.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + creationTimestamp: null + labels: + io.kompose.service: prometheus-claim1 + name: prometheus-claim1 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi +status: {} diff --git a/projects/kafka-cron/kubeconfigs/prometheus-data-persistentvolumeclaim.yaml b/projects/kafka-cron/kubeconfigs/prometheus-data-persistentvolumeclaim.yaml new file mode 100644 index 000000000..3f80692e2 --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/prometheus-data-persistentvolumeclaim.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + creationTimestamp: null + labels: + io.kompose.service: prometheus-data + name: prometheus-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi +status: {} diff --git a/projects/kafka-cron/kubeconfigs/prometheus-deployment.yaml b/projects/kafka-cron/kubeconfigs/prometheus-deployment.yaml new file mode 100644 index 000000000..7eadf8831 --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/prometheus-deployment.yaml @@ -0,0 +1,54 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: ../../kompose convert + kompose.version: 1.26.0 (40646f47) + creationTimestamp: null + labels: + io.kompose.service: prometheus + name: prometheus +spec: + replicas: 1 + selector: + matchLabels: + io.kompose.service: prometheus + strategy: + type: Recreate + template: + metadata: + annotations: + kompose.cmd: ../../kompose convert + kompose.version: 1.26.0 (40646f47) + creationTimestamp: null + labels: + io.kompose.service: prometheus + spec: + containers: + - args: + - --config.file=/etc/prometheus/prometheus.yml + image: prom/prometheus + name: prometheus + ports: + - containerPort: 9090 + resources: {} + volumeMounts: + - mountPath: /etc/prometheus/prometheus.yml + name: prometheus-claim0 + - mountPath: /etc/prometheus/alert_rules.yml + name: prometheus-claim1 + - mountPath: /prometheus + name: prometheus-data + hostname: prometheus + restartPolicy: Always + volumes: + - name: prometheus-claim0 + persistentVolumeClaim: + claimName: prometheus-claim0 + - name: prometheus-claim1 + persistentVolumeClaim: + claimName: prometheus-claim1 + - name: prometheus-data + persistentVolumeClaim: + claimName: prometheus-data +status: {} diff --git a/projects/kafka-cron/kubeconfigs/prometheus-service.yaml b/projects/kafka-cron/kubeconfigs/prometheus-service.yaml new file mode 100644 index 000000000..ac7fcf710 --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/prometheus-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: ../../kompose convert + kompose.version: 1.26.0 (40646f47) + creationTimestamp: null + labels: + io.kompose.service: prometheus + name: prometheus +spec: + ports: + - name: "9090" + port: 9090 + targetPort: 9090 + selector: + io.kompose.service: prometheus +status: + loadBalancer: {} diff --git a/projects/kafka-cron/kubeconfigs/zoo1-deployment.yaml b/projects/kafka-cron/kubeconfigs/zoo1-deployment.yaml new file mode 100644 index 000000000..827d5cf0c --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/zoo1-deployment.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + kompose.cmd: ../../kompose convert + kompose.version: 1.26.0 (40646f47) + creationTimestamp: null + labels: + io.kompose.service: zoo1 + name: zoo1 +spec: + replicas: 1 + selector: + matchLabels: + io.kompose.service: zoo1 + strategy: {} + template: + metadata: + annotations: + kompose.cmd: ../../kompose convert + kompose.version: 1.26.0 (40646f47) + creationTimestamp: null + labels: + io.kompose.service: zoo1 + spec: + containers: + - env: + - name: ZOOKEEPER_CLIENT_PORT + value: "2181" + - name: ZOOKEEPER_SERVERS + value: zoo1:2888:3888 + - name: ZOOKEEPER_SERVER_ID + value: "1" + image: confluentinc/cp-zookeeper:7.3.0 + name: zoo1 + ports: + - containerPort: 2181 + resources: {} + hostname: zoo1 + restartPolicy: Always +status: {} diff --git a/projects/kafka-cron/kubeconfigs/zoo1-service.yaml b/projects/kafka-cron/kubeconfigs/zoo1-service.yaml new file mode 100644 index 000000000..c88ed55f0 --- /dev/null +++ b/projects/kafka-cron/kubeconfigs/zoo1-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: ../../kompose convert + kompose.version: 1.26.0 (40646f47) + creationTimestamp: null + labels: + io.kompose.service: zoo1 + name: zoo1 +spec: + ports: + - name: "2181" + port: 2181 + targetPort: 2181 + selector: + io.kompose.service: zoo1 +status: + loadBalancer: {} diff --git a/projects/kafka-cron/message/cronjob.go b/projects/kafka-cron/message/cronjob.go new file mode 100644 index 000000000..682a5622a --- /dev/null +++ b/projects/kafka-cron/message/cronjob.go @@ -0,0 +1,8 @@ +package message + +type CronjobMessage struct { + Command string + Exectime string // ISO 8601 + Name string + Retries int +} \ No newline at end of file diff --git a/projects/kafka-cron/prometheus.yml b/projects/kafka-cron/prometheus.yml new file mode 100644 index 000000000..b568ab1df --- /dev/null +++ b/projects/kafka-cron/prometheus.yml @@ -0,0 +1,28 @@ +# my global config +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + # - alertmanager:9093 + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + # - "first_rules.yml" + # - "second_rules.yml" + +# A scrape configuration containing exactly one endpoint to scrape: +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: "kafka" + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + static_configs: + - targets: ["kafka1:8999"] \ No newline at end of file