diff --git a/Makefile b/Makefile index 7404082..dbba333 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,9 @@ else GOBIN=$(shell go env GOBIN) endif +# Set license header files. +LICENSE_HEADER_GO ?= hack/boilerplate.go.txt + # Setting SHELL to bash allows bash commands to be executed by recipes. # This is a requirement for 'setup-envtest.sh' in the test target. # Options are set to exit when a recipe line exits non-zero or a piped command fails. @@ -56,6 +59,12 @@ tidy: ## Run go mod tidy. test: ## Run tests with coverage. Outputs to combined.cov. go test -v -coverprofile=coverage.txt -covermode=count ./... +.PHONY: generate +generate: generate-deepcopy fmt ## Generate Go code. + +generate-deepcopy: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(CONTROLLER_GEN) object:headerFile="$(LICENSE_HEADER_GO)" paths="./..." + ##@ Building .PHONY: build @@ -72,9 +81,11 @@ $(LOCALBIN): ## Ensure that the directory exists ## Tool Binaries GOIMPORTS ?= $(LOCALBIN)/goimports GOLANGCI_LINT ?= $(LOCALBIN)/golangci-lint@$(GOLANGCILINT_VERSION) +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen@$(CONTROLLER_TOOLS_VERSION) ## Tool Versions GOLANGCILINT_VERSION ?= v1.61.0 +CONTROLLER_TOOLS_VERSION ?= v0.16.1 .PHONY: goimports goimports: $(GOIMPORTS) ## Download goimports locally if necessary. @@ -87,3 +98,9 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. $(GOLANGCI_LINT): @[ -f $(GOLANGCI_LINT) ] || curl -sSfL $(GOLANGCILINT_INSTALL_SCRIPT) | sh -s $(GOLANGCILINT_VERSION) mv $(LOCALBIN)/golangci-lint $(LOCALBIN)/golangci-lint@$(GOLANGCILINT_VERSION) + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. +$(CONTROLLER_GEN): + GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) + mv $(LOCALBIN)/controller-gen $(LOCALBIN)/controller-gen@$(CONTROLLER_TOOLS_VERSION) diff --git a/README.md b/README.md index 95ea2d5..b165edf 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ For more detailed instructions, refer to the [official site](https://www.healthy ### Running the Server -You can deploy and run the server using Docker or as a standalone binary, which is packaged and released on the . +You can deploy and run the server using Docker or as a standalone binary, which is packaged and released on the [GitHub Releases page](https://github.com/irvinlim/apple-health-ingester/releases). ### Linux Download @@ -65,48 +65,99 @@ $ docker run --rm irvinlim/apple-health-ingester --help ## Configuration -### Command-line Flags +### Specifying Configuration + +Configuration can be specified in the following places, in order of precedence from lowest to highest: + +- Config file (**preferred**) +- Environment variables +- Command-line flags + +See the following sections for available configuration keys for each of the available configuration methods. + +#### Configuration File + +The preferred method of configuration is via configuration files. An example configuration file to get started is as follows: + +```sh +httpServer: + listenAddr: ":8080" +backends: + influxdb: + enabled: true + serverURL: "" + orgName: "" + bucketNames: + metrics: "apple_health_metrics" + workouts: "apple_health_workouts" +``` + +Configuration files can be specified in both JSON and YAML format. To pass the configuration to the application, use the `--config` or `-c` command-line flag such as follows: + +```sh +./build/ingester -c config.yaml +``` + +If any of the following configuration paths exist, the config file will be automatically discovered and loaded, listed below in order of preference: + +- `./config.yaml` +- `./config.json` +- `/config.yaml` +- `/config.json` + +#### Environment Variables + +Environment variables take greater precedence over those specified via configuration files. These can be used to override behaviour at runtime if necessary. + +Not all configuration fields will be exposed via environment variables, and therefore it is recommended to only make use of environment variables sparingly. + +#### Command-line Flags + +Finally, command-line flags take highest precedence over all other configuration methods available. Similar to environment variables, not all configuration can be configured via command-line flags, and therefore it is recommended to define complex configuration in a config file instead. + +Since command-line flags were introduced since the beginning, support for existing command-line flags will _not_ be dropped to maintain backwards compatibility. + +The following command-line flags are currently available: ```sh $ ./build/ingester --help Usage of ./build/ingester: --backend.influxdb Enable the InfluxDB storage backend. --backend.localfile Enable the LocalFile storage backend. + -c, --config string Path to YAML or JSON configuration file. + However, config specified via environment variables or command-line arguments will still take greater precedence. + If not specified, attempts to discover configuration files at the following locations in order: + config.yaml, config.json, /config/config.yaml, /config/config.json --http.authToken string Optional authorization token that will be used to authenticate incoming requests. + Environment variable: $AUTH_TOKEN --http.certFile string Certificate file for TLS support. + Environment variable: $TLS_CERT_FILE --http.enableTLS Enable TLS/HTTPS. Requires setting certificate and key files. + Environment variable: $TLS_ENABLED --http.keyFile string Key file for TLS support. - --http.listenAddr string Address to listen on. (default ":8080") + Environment variable: $TLS_KEY_FILE + --http.listenAddr string Address to listen on. + Environment variable: $LISTEN_ADDR --influxdb.authToken string Auth token to connect to InfluxDB. + Environment variable: $INFLUXDB_AUTH_TOKEN --influxdb.insecureSkipVerify Skip TLS verification of the certificate chain and host name for the InfluxDB server. + Environment variable: $INFLUXDB_INSECURE_SKIP_VERIFY --influxdb.metricsBucketName string InfluxDB bucket name for metrics. + Environment variable: $INFLUXDB_METRICS_BUCKET --influxdb.orgName string InfluxDB organization name. + Environment variable: $INFLUXDB_ORG_NAME --influxdb.serverURL string Server URL for InfluxDB. + Environment variable: $INFLUXDB_SERVER_URL --influxdb.staticTags strings Additional tags to add to InfluxDB for every single request, in key=value format. --influxdb.workoutsBucketName string InfluxDB bucket name for workouts. + Environment variable: $INFLUXDB_WORKOUTS_BUCKET --localfile.metricsPath string Output path to write metrics, with one metric per file. All data will be aggregated by timestamp. Any existing data will be merged together. --log string Log level to use. (default "info") ``` -### Global Configuration - -#### `http.listenAddr` - -Address to listen on, in `IP:port` format. Defaults to `:8080`. - -#### `http.authToken` - -Optional authorization token that will be used to authenticate incoming requests. The header name should be `Authorization`, and the header value should be `Bearer `. - -#### TLS Configuration +### Configuration Fields -To enable TLS, the following flags must be provided: - -* `http.enableTLS`: Starts a TLS/HTTPS server instead of HTTP. -* `http.keyFile`: TLS private key file. -* `http.certFile`: TLS certificate file. - -#### `log` +#### Log Level Specify the log level. The following log levels are supported, and in order of verbosity from lowest to highest: @@ -118,6 +169,70 @@ Specify the log level. The following log levels are supported, and in order of v - `debug` - `trace` +Increase the level to `debug` for more verbose logging. + +- Default: `info` +- Command-line: `--log` + +#### Listen Address + +Address to listen on, in `IP:port` format. + +- Default: `:8080` +- Config file: `httpServer.listenAddr` +- Environment variable: `$LISTEN_ADDR` +- Command-line: `--http.listenAddr` + +#### HTTP Authorization Token + +Optional authorization token that will be used to authenticate incoming requests. The header name should be `Authorization`, and the header value should be `Bearer `. + +- Config file: `httpServer.auth.authorizationToken` +- Environment variable: `$AUTH_TOKEN` +- Command-line: `--http.authToken` + +#### HTTP TLS Configuration + +_Enable TLS_ + +The following configuration fields are available to enable TLS: + +- Config file: `httpServer.tls.enabled` +- Environment variable: `$TLS_ENABLED` +- Command-line: `--http.enableTLS` + +_Certificate File / Key File_ + +If TLS is enabled, the TLS certificate needs to be provided that will be served. + +- Certificate File + - Config file: `httpServer.tls.certFile` + - Environment variable: `$TLS_CERT_FILE` + - Command-line: `--http.certFile` +- Key File + - Config file: `httpServer.tls.keyFile` + - Environment variable: `$TLS_KEY_FILE` + - Command-line: `--http.keyFile` + +_Certificate Data / Key Data_ + +Alternatively, encode the actual TLS certificate in the configuration file, such as follows: + +```yaml +httpServer: + tls: + enabled: true + certData: | + -----BEGIN CERTIFICATE----- + data omitted... + -----END CERTIFICATE----- +``` + +- Certificate Data + - Config file: `httpServer.tls.certData` +- Key File + - Config file: `httpServer.tls.keyData` + ## Supported Backends Each backend must be enabled explicitly. By default, no backends are enabled by default. @@ -132,14 +247,18 @@ Writes the ingested payloads into the local filesystem as JSON. Mainly intended #### Configuration -This backend is disabled by default. You can enable it by specifying `--backend.localfile`. +This backend is disabled by default. You can enable it by specifying: + +- Config file: `backends.localfile.enabled` +- Command-line: `--backend.localfile` You must also configure additional fields for the backend to work. Example configuration: -```sh -$ ingester \ - --backend.localfile \ - --localfile.metricsPath=/data/health-export-metrics +```yaml +backends: + localfile: + enabled: true + metricsPath: /data/health-export-metrics ``` **NOTE**: Workout data is currently not yet supported for this storage backend. @@ -172,23 +291,49 @@ Writes the ingested metrics and workout data into a configured InfluxDB backend. #### Configuration -This backend is disabled by default. You can enable it by specifying `--backend.influxdb`. +This backend is disabled by default. You can enable it by specifying: + +- Config file: `backends.influxdb.enabled` +- Command-line: `--backend.influxdb` You must also configure additional fields for the backend to work. Example configuration: -```sh -$ ingester \ - --backend.influxdb \ - --influxdb.serverURL=http://localhost:8086 \ - --influxdb.authToken=INFLUX_API_TOKEN \ - --influxdb.orgName=my-org \ - --influxdb.metricsBucketName=apple_health_metrics \ - --influxdb.workoutsBucketName=apple_health_workouts +```yaml +backends: + influxdb: + # Must be set to true to enable this backend. + enabled: true + # The URL to the InfluxDB server. + # Environment variable: $INFLUXDB_SERVER_URL. + serverURL: "http://localhost:8086" + # If true, skips TLS verification of the certificate chain and host name for + # the InfluxDB server. + # Environment variable: $INFLUXDB_INSECURE_SKIP_VERIFY + insecureSkipVerify: false + # Auth token to connect to InfluxDB. + # Environment variable: $INFLUXDB_AUTH_TOKEN + authToken: YOUR_INFLUX_API_TOKEN + # InfluxDB organization name. + # Environment variable: $INFLUXDB_ORG_NAME + orgName: my-org + # Configuration for InfluxDB bucket names for various data points. + bucketNames: + # InfluxDB bucket name for metrics. + # Environment variable: $INFLUXDB_METRICS_BUCKET + metrics: apple_health_metrics + # InfluxDB bucket name for workouts. + # Environment variable: $INFLUXDB_WORKOUTS_BUCKET + workouts: apple_health_workouts + # Additional tags to add to InfluxDB for every single request. + # Uncomment the following lines to add static tags to all metrics. + #staticTags: + # key1: value1 + # key2: value2 ``` #### Metrics Data Format -All metrics will be stored in the bucket named by `--influxdb.metricsBucketName` using the following format: +All metrics will be stored in the _Metrics Bucket_ using the following format: - Measurement: - Metric name (e.g. `active_energy`) + Unit (e.g. `kJ`) @@ -203,11 +348,11 @@ All metrics will be stored in the bucket named by `--influxdb.metricsBucketName` - The rest of the fields can be found here: https://github.com/Lybron/health-auto-export/wiki/API-Export---JSON-Format - Tags: - `target_name`: Optional, set by `?target=TARGET_NAME` query string from HTTP request. - - Additional tags can be set by `--influxdb.staticTags`. + - Additional tags can be set by `--influxdb.staticTags` / `backends.influxdb.staticTags`. #### Workouts Data Format -Workout data will be stored in the bucket named by `--influxdb.workoutsBucketName`. Workout data is slightly more complicated than metrics. You can read more about the workout data format here: https://github.com/Lybron/health-auto-export/wiki/API-Export---JSON-Format#workouts +Workout data will be stored in the _Workouts Bucket_. Workout data is slightly more complicated than metrics. You can read more about the workout data format here: https://github.com/Lybron/health-auto-export/wiki/API-Export---JSON-Format#workouts There are two kinds of data: @@ -232,7 +377,7 @@ All workout data have the following tags: - `target_name`: Optional, set by `?target=TARGET_NAME` query string from HTTP request. - `workout_name`: Name of the workout. - Example `Walking` - - Additional tags can be set by `--influxdb.staticTags`. + - Additional tags can be set by `--influxdb.staticTags` / `backends.influxdb.staticTags`. #### Example Output diff --git a/apis/config/v1/appconfig_types.go b/apis/config/v1/appconfig_types.go new file mode 100644 index 0000000..0d5494b --- /dev/null +++ b/apis/config/v1/appconfig_types.go @@ -0,0 +1,12 @@ +package v1 + +// Config contains the application configuration. +// +// Configuration be loaded from configuration file, or possibly overridden from +// environment variables if the `env` struct tag is specified. +type Config struct { + // Configuration for the ingester's HTTP server. + HttpServer HttpServerConfig `json:"httpServer"` + // Configuration for individual ingester backends. + Backends BackendsConfig `json:"backends"` +} diff --git a/apis/config/v1/backendconfig_types.go b/apis/config/v1/backendconfig_types.go new file mode 100644 index 0000000..fc3d2dd --- /dev/null +++ b/apis/config/v1/backendconfig_types.go @@ -0,0 +1,44 @@ +package v1 + +type BackendsConfig struct { + // If specified, enables the LocalFile backend. + LocalFile LocalFileBackendConfig `json:"localFile,omitempty"` + // If specified, enables the InfluxDB backend. + InfluxDB InfluxdbBackendConfig `json:"influxdb,omitempty"` +} + +type LocalFileBackendConfig struct { + // Whether the LocalFile backend is enabled. + // Defaults to false. + Enabled bool `json:"enabled"` + // Output path to write metrics, with one metric per file. Required. + // All data will be aggregated by timestamp. + // Any existing data will be merged together. + MetricsPath string `json:"metricsPath"` +} + +type InfluxdbBackendConfig struct { + // Whether the InfluxDB backend is enabled. + // Defaults to false. + Enabled bool `json:"enabled"` + // The URL to the InfluxDB server. + ServerURL string `json:"serverURL" env:"INFLUXDB_SERVER_URL"` + // If true, skips TLS verification of the certificate chain and host name for + // the InfluxDB server. + InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty" env:"INFLUXDB_INSECURE_SKIP_VERIFY"` + // Auth token to connect to InfluxDB. + AuthToken string `json:"authToken,omitempty" env:"INFLUXDB_AUTH_TOKEN"` + // InfluxDB organization name. + OrgName string `json:"orgName" env:"INFLUXDB_ORG_NAME"` + // Configuration for InfluxDB bucket names for various data points. + BucketNames InfluxdbBucketNamesConfig `json:"bucketNames"` + // Additional tags to add to InfluxDB for every single request, in key=value format. + StaticTags map[string]string `json:"staticTags,omitempty"` +} + +type InfluxdbBucketNamesConfig struct { + // InfluxDB bucket name for metrics. + Metrics string `json:"metrics" env:"INFLUXDB_METRICS_BUCKET"` + // InfluxDB bucket name for workouts. + Workouts string `json:"workouts" env:"INFLUXDB_WORKOUTS_BUCKET"` +} diff --git a/apis/config/v1/doc.go b/apis/config/v1/doc.go new file mode 100644 index 0000000..93ba80d --- /dev/null +++ b/apis/config/v1/doc.go @@ -0,0 +1,3 @@ +// Package v1 contains API Schema definitions for the config v1 API group +// +kubebuilder:object:generate=true +package v1 diff --git a/apis/config/v1/httpconfig_types.go b/apis/config/v1/httpconfig_types.go new file mode 100644 index 0000000..dd50d1b --- /dev/null +++ b/apis/config/v1/httpconfig_types.go @@ -0,0 +1,37 @@ +package v1 + +type HttpServerConfig struct { + // The address to listen on. Required. + // Must be specified as one of the following: + // - ":": Binds to the IP address on the given interface only + // - ":": Binds to all interfaces + ListenAddr string `json:"listenAddr" env:"LISTEN_ADDR"` + // Specify TLS configuration for the HTTP server. + // Defaults to running without TLS. + TLS HttpTLSConfig `json:"tls"` + // Specify authentication for the HTTP server. + // Defaults to no authentication. + Auth HttpAuthConfig `json:"auth,omitempty"` +} + +type HttpTLSConfig struct { + // Whether TLS is enabled for the HTTP server. + // If enabled, certificate and key files must also be specified. + Enabled bool `json:"enabled" env:"TLS_ENABLED"` + // The path to the TLS certificate to serve. + // At most one of CertFile or CertData can be specified. + CertFile string `json:"certFile,omitempty" env:"TLS_CERT_FILE"` + // The TLS certificate data to serve. + // At most one of CertFile or CertData can be specified. + CertData string `json:"certData,omitempty"` + // The path to the TLS private key. + KeyFile string `json:"keyFile,omitempty" env:"TLS_KEY_FILE"` + // The TLS private key. + // At most one of KeyFile or KeyData can be specified. + KeyData string `json:"keyData,omitempty"` +} + +type HttpAuthConfig struct { + // Optionally specify a fixed Bearer token that can be used to authenticate incoming requests. + AuthorizationToken string `json:"authorizationToken,omitempty" env:"AUTH_TOKEN"` +} diff --git a/apis/config/v1/zz_generated.deepcopy.go b/apis/config/v1/zz_generated.deepcopy.go new file mode 100644 index 0000000..823d181 --- /dev/null +++ b/apis/config/v1/zz_generated.deepcopy.go @@ -0,0 +1,139 @@ +//go:build !ignore_autogenerated + +// Code generated by controller-gen. DO NOT EDIT. + +package v1 + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackendsConfig) DeepCopyInto(out *BackendsConfig) { + *out = *in + out.LocalFile = in.LocalFile + in.InfluxDB.DeepCopyInto(&out.InfluxDB) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackendsConfig. +func (in *BackendsConfig) DeepCopy() *BackendsConfig { + if in == nil { + return nil + } + out := new(BackendsConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Config) DeepCopyInto(out *Config) { + *out = *in + out.HttpServer = in.HttpServer + in.Backends.DeepCopyInto(&out.Backends) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Config. +func (in *Config) DeepCopy() *Config { + if in == nil { + return nil + } + out := new(Config) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HttpAuthConfig) DeepCopyInto(out *HttpAuthConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HttpAuthConfig. +func (in *HttpAuthConfig) DeepCopy() *HttpAuthConfig { + if in == nil { + return nil + } + out := new(HttpAuthConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HttpServerConfig) DeepCopyInto(out *HttpServerConfig) { + *out = *in + out.TLS = in.TLS + out.Auth = in.Auth +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HttpServerConfig. +func (in *HttpServerConfig) DeepCopy() *HttpServerConfig { + if in == nil { + return nil + } + out := new(HttpServerConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HttpTLSConfig) DeepCopyInto(out *HttpTLSConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HttpTLSConfig. +func (in *HttpTLSConfig) DeepCopy() *HttpTLSConfig { + if in == nil { + return nil + } + out := new(HttpTLSConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InfluxdbBackendConfig) DeepCopyInto(out *InfluxdbBackendConfig) { + *out = *in + out.BucketNames = in.BucketNames + if in.StaticTags != nil { + in, out := &in.StaticTags, &out.StaticTags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InfluxdbBackendConfig. +func (in *InfluxdbBackendConfig) DeepCopy() *InfluxdbBackendConfig { + if in == nil { + return nil + } + out := new(InfluxdbBackendConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InfluxdbBucketNamesConfig) DeepCopyInto(out *InfluxdbBucketNamesConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InfluxdbBucketNamesConfig. +func (in *InfluxdbBucketNamesConfig) DeepCopy() *InfluxdbBucketNamesConfig { + if in == nil { + return nil + } + out := new(InfluxdbBucketNamesConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LocalFileBackendConfig) DeepCopyInto(out *LocalFileBackendConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalFileBackendConfig. +func (in *LocalFileBackendConfig) DeepCopy() *LocalFileBackendConfig { + if in == nil { + return nil + } + out := new(LocalFileBackendConfig) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/ingester/backends.go b/cmd/ingester/backends.go index f571420..ee9985e 100644 --- a/cmd/ingester/backends.go +++ b/cmd/ingester/backends.go @@ -5,6 +5,7 @@ import ( "github.com/pkg/errors" + configv1 "github.com/irvinlim/apple-health-ingester/apis/config/v1" "github.com/irvinlim/apple-health-ingester/pkg/backends/influxdb" "github.com/irvinlim/apple-health-ingester/pkg/backends/localfile" "github.com/irvinlim/apple-health-ingester/pkg/ingester" @@ -15,11 +16,15 @@ const ( ) // RegisterDebugBackend registers the Debug backend. -func RegisterDebugBackend(ingester *ingester.Ingester, mux *http.ServeMux) error { - if !enableLocalFile { +func RegisterDebugBackend(cfg *configv1.Config, ingester *ingester.Ingester, mux *http.ServeMux) error { + if cfg == nil { + cfg = &configv1.Config{} + } + backendCfg := cfg.Backends.LocalFile.DeepCopy() + if !backendCfg.Enabled { return nil } - backend, err := localfile.NewBackend() + backend, err := localfile.NewBackend(backendCfg) if err != nil { return err } @@ -27,15 +32,19 @@ func RegisterDebugBackend(ingester *ingester.Ingester, mux *http.ServeMux) error } // RegisterInfluxDBBackend registers the InfluxDB backend. -func RegisterInfluxDBBackend(ingester *ingester.Ingester, mux *http.ServeMux) error { - if !enableInfluxDB { +func RegisterInfluxDBBackend(cfg *configv1.Config, ingester *ingester.Ingester, mux *http.ServeMux) error { + if cfg == nil { + cfg = &configv1.Config{} + } + backendCfg := cfg.Backends.InfluxDB.DeepCopy() + if !backendCfg.Enabled { return nil } - client, err := influxdb.NewClient() + client, err := influxdb.NewClient(backendCfg) if err != nil { return errors.Wrapf(err, "cannot initialize client") } - backend, err := influxdb.NewBackend(client) + backend, err := influxdb.NewBackend(backendCfg, client) if err != nil { return err } diff --git a/cmd/ingester/flags.go b/cmd/ingester/flags.go index 898586e..0ec6d0e 100644 --- a/cmd/ingester/flags.go +++ b/cmd/ingester/flags.go @@ -1,28 +1,274 @@ package main import ( + "fmt" + "os" + "strings" + + "github.com/pkg/errors" "github.com/spf13/pflag" -) -var ( - logLevel string - listenAddr string - authorizationToken string - enableInfluxDB bool - enableLocalFile bool - enableTLS bool - certFile string - keyFile string + configv1 "github.com/irvinlim/apple-health-ingester/apis/config/v1" + "github.com/irvinlim/apple-health-ingester/pkg/config" ) -func init() { - pflag.StringVar(&listenAddr, "http.listenAddr", ":8080", "Address to listen on.") - pflag.StringVar(&logLevel, "log", "info", "Log level to use.") - pflag.StringVar(&authorizationToken, "http.authToken", "", - "Optional authorization token that will be used to authenticate incoming requests.") - pflag.BoolVar(&enableInfluxDB, "backend.influxdb", false, "Enable the InfluxDB storage backend.") - pflag.BoolVar(&enableLocalFile, "backend.localfile", false, "Enable the LocalFile storage backend.") - pflag.BoolVar(&enableTLS, "http.enableTLS", false, "Enable TLS/HTTPS. Requires setting certificate and key files.") - pflag.StringVar(&certFile, "http.certFile", "", "Certificate file for TLS support.") - pflag.StringVar(&keyFile, "http.keyFile", "", "Key file for TLS support.") +// Flags is the root data structure for defining command-line flags for the application. +// +// Currently, we support quite a fair bit of business logic flags which are also available to be specified via config. +// Any new flags that control business logic should not be added here, but instead should be added to the configuration schema. +// Using configuration files should be the preferred way moving forward. +// Existing flags will be retained for backwards compatibility for now. +type Flags struct { + // The configured log level. + LogLevel string + // The configuration file to load. + ConfigFile string + // Flags for the HTTP server. + HttpServer HttpServerFlags + // Flags for ingester backends. + Backends BackendFlags +} + +// BindFlagSet registers the flags into the given FlagSet. +func (f *Flags) BindFlagSet(flagSet *pflag.FlagSet) { + flagSet.StringVar(&f.LogLevel, "log", "info", "Log level to use.") + flagSet.StringVarP(&f.ConfigFile, "config", "c", "", + `Path to YAML or JSON configuration file. +However, config specified via environment variables or command-line arguments will still take greater precedence. +If not specified, attempts to discover configuration files at the following locations in order: + `+strings.Join(defaultConfigFileLocations, ", ")) + + f.HttpServer.BindFlagSet(flagSet) + f.Backends.BindFlagSet(flagSet) +} + +// Merge down the flags to override the config. +func (f *Flags) Merge(cfg *configv1.Config) error { + if err := f.HttpServer.Merge(&cfg.HttpServer); err != nil { + return errors.Wrap(err, "failed to merge HttpServerFlags") + } + if err := f.Backends.Merge(&cfg.Backends); err != nil { + return errors.Wrapf(err, "failed to merge BackendFlags") + } + return nil +} + +type HttpServerFlags struct { + // The address to listen on. + ListenAddr string + // Optional Bearer token to authorize incoming requests. + AuthorizationToken string + // Optionally enable TLS. + EnableTLS bool + // The TLS certificate to serve. + CertFile string + // The TLS private key. + KeyFile string +} + +// BindFlagSet registers the flags into the given FlagSet. +func (f *HttpServerFlags) BindFlagSet(flagSet *pflag.FlagSet) { + flagSet.StringVar(&f.ListenAddr, "http.listenAddr", "", `Address to listen on. +Environment variable: $LISTEN_ADDR`) + flagSet.StringVar(&f.AuthorizationToken, "http.authToken", "", + `Optional authorization token that will be used to authenticate incoming requests. +Environment variable: $AUTH_TOKEN`) + flagSet.BoolVar(&f.EnableTLS, "http.enableTLS", false, `Enable TLS/HTTPS. Requires setting certificate and key files. +Environment variable: $TLS_ENABLED`) + flagSet.StringVar(&f.CertFile, "http.certFile", "", `Certificate file for TLS support. +Environment variable: $TLS_CERT_FILE`) + flagSet.StringVar(&f.KeyFile, "http.keyFile", "", `Key file for TLS support. +Environment variable: $TLS_KEY_FILE`) +} + +// Merge down the flags to override the config. +func (f *HttpServerFlags) Merge(cfg *configv1.HttpServerConfig) error { + if f.ListenAddr != "" { + cfg.ListenAddr = f.ListenAddr + } + if f.AuthorizationToken != "" { + cfg.Auth.AuthorizationToken = f.AuthorizationToken + } + if f.EnableTLS { + cfg.TLS.Enabled = f.EnableTLS + } + if f.CertFile != "" { + cfg.TLS.CertFile = f.CertFile + } + if f.KeyFile != "" { + cfg.TLS.KeyFile = f.KeyFile + } + return nil +} + +type BackendFlags struct { + // Flags for InfluxDB backend. + InfluxDB InfluxdbBackendFlags + // Flags for LocalFile backend. + LocalFile LocalFileBackendFlags +} + +// BindFlagSet registers the flags into the given FlagSet. +func (f *BackendFlags) BindFlagSet(flagSet *pflag.FlagSet) { + f.InfluxDB.BindFlagSet(flagSet) + f.LocalFile.BindFlagSet(flagSet) +} + +// Merge down the flags to override the config. +func (f *BackendFlags) Merge(cfg *configv1.BackendsConfig) error { + if err := f.InfluxDB.Merge(&cfg.InfluxDB); err != nil { + return errors.Wrap(err, "failed to merge InfluxdbBackendFlags") + } + if err := f.LocalFile.Merge(&cfg.LocalFile); err != nil { + return errors.Wrap(err, "failed to merge LocalFileBackendFlags") + } + return nil +} + +type InfluxdbBackendFlags struct { + // Whether the backend is enabled. + Enabled bool + // The InfluxDB server URL. + ServerURL string + // Whether to skip TLS verification of the certificate chain and host name for the InfluxDB server. + InsecureSkipVerify bool + // Auth token to connect to InfluxDB. + AuthToken string + // InfluxDB organization name. + OrgName string + // InfluxDB bucket name for metrics. + MetricsBucketName string + // InfluxDB bucket name for workouts. + WorkoutsBucketName string + // Additional tags to add to InfluxDB for every single request, in key=value format. + StaticTags []string +} + +// BindFlagSet registers the flags into the given FlagSet. +func (f *InfluxdbBackendFlags) BindFlagSet(flagSet *pflag.FlagSet) { + flagSet.BoolVar(&f.Enabled, "backend.influxdb", false, "Enable the InfluxDB storage backend.") + flagSet.StringVar(&f.ServerURL, "influxdb.serverURL", "", `Server URL for InfluxDB. +Environment variable: $INFLUXDB_SERVER_URL`) + flagSet.BoolVar(&f.InsecureSkipVerify, "influxdb.insecureSkipVerify", false, + `Skip TLS verification of the certificate chain and host name for the InfluxDB server. +Environment variable: $INFLUXDB_INSECURE_SKIP_VERIFY`) + flagSet.StringVar(&f.AuthToken, "influxdb.authToken", "", `Auth token to connect to InfluxDB. +Environment variable: $INFLUXDB_AUTH_TOKEN`) + flagSet.StringVar(&f.OrgName, "influxdb.orgName", "", `InfluxDB organization name. +Environment variable: $INFLUXDB_ORG_NAME`) + flagSet.StringVar(&f.MetricsBucketName, "influxdb.metricsBucketName", "", `InfluxDB bucket name for metrics. +Environment variable: $INFLUXDB_METRICS_BUCKET`) + flagSet.StringVar(&f.WorkoutsBucketName, "influxdb.workoutsBucketName", "", `InfluxDB bucket name for workouts. +Environment variable: $INFLUXDB_WORKOUTS_BUCKET`) + flagSet.StringSliceVar(&f.StaticTags, "influxdb.staticTags", nil, + "Additional tags to add to InfluxDB for every single request, in key=value format.") +} + +// Merge down the flags to override the config. +func (f *InfluxdbBackendFlags) Merge(cfg *configv1.InfluxdbBackendConfig) error { + if f.Enabled { + cfg.Enabled = f.Enabled + } + if f.ServerURL != "" { + cfg.ServerURL = f.ServerURL + } + if f.InsecureSkipVerify { + cfg.InsecureSkipVerify = true + } + if f.AuthToken != "" { + cfg.AuthToken = f.AuthToken + } + if f.OrgName != "" { + cfg.OrgName = f.OrgName + } + if f.MetricsBucketName != "" { + cfg.BucketNames.Metrics = f.MetricsBucketName + } + if f.WorkoutsBucketName != "" { + cfg.BucketNames.Workouts = f.WorkoutsBucketName + } + if len(f.StaticTags) > 0 { + // Throw an error if both command-line args and config file args are specified. + if len(cfg.StaticTags) > 0 { + return errors.New("cannot specify static tags via both --influxdb.staticTags and from config file") + } + if cfg.StaticTags == nil { + cfg.StaticTags = make(map[string]string, len(f.StaticTags)) + } + + for _, tag := range f.StaticTags { + tokens := strings.SplitN(tag, "=", 2) + if len(tokens) != 2 { + return fmt.Errorf("invalid static tag %v", tag) + } + cfg.StaticTags[tokens[0]] = tokens[1] + } + } + return nil +} + +type LocalFileBackendFlags struct { + // Whether the backend is enabled. + Enabled bool + // Output path to write metrics + MetricsPath string +} + +// BindFlagSet registers the flags into the given FlagSet. +func (f *LocalFileBackendFlags) BindFlagSet(flagSet *pflag.FlagSet) { + flagSet.BoolVar(&f.Enabled, "backend.localfile", false, "Enable the LocalFile storage backend.") + flagSet.StringVar(&f.MetricsPath, "localfile.metricsPath", "", + "Output path to write metrics, with one metric per file. All data will be aggregated by timestamp. "+ + "Any existing data will be merged together.") +} + +// Merge down the flags to override the config. +func (f *LocalFileBackendFlags) Merge(cfg *configv1.LocalFileBackendConfig) error { + if f.MetricsPath != "" { + cfg.MetricsPath = f.MetricsPath + } + return nil +} + +// ParseFlags will initialize and parse Flags. +func ParseFlags(flagSet *pflag.FlagSet) (*Flags, error) { + flags := &Flags{} + + // Recursively bind the flagSet. + flags.BindFlagSet(flagSet) + + // Parse the arguments. + if err := flagSet.Parse(os.Args[1:]); err != nil { + return nil, errors.Wrapf(err, "failed to parse command line") + } + + return flags, nil +} + +// LoadConfigAndMergeFlags is the main entrypoint to load configuration from all +// configuration sources, while also merging down any command-line flags that +// can override the +// +// Specifically, configuration will be loaded from the following sources, in +// order of precedence from lowest to highest: +// +// 1. Default config, specified by DefaultConfig (see defaults.go) +// 2. Configuration file +// 3. Environment variables +// 4. Command-line arguments +func LoadConfigAndMergeFlags(flags *Flags, args config.LoadConfigArgs) (*configv1.Config, error) { + // First load the configuration from file. + // We don't need to validate the config yet as we'll override with command-line args later. + cfg, err := config.Load(args) + if err != nil { + return nil, errors.Wrapf(err, "cannot load config") + } + + // Override the configuration with flags if specified. + cfg = cfg.DeepCopy() + if err := flags.Merge(cfg); err != nil { + return nil, errors.Wrapf(err, "cannot merge flags with command-line arguments") + } + + return cfg, nil } diff --git a/cmd/ingester/ingester.go b/cmd/ingester/ingester.go index 0a980f2..9d3e368 100644 --- a/cmd/ingester/ingester.go +++ b/cmd/ingester/ingester.go @@ -7,11 +7,12 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + configv1 "github.com/irvinlim/apple-health-ingester/apis/config/v1" "github.com/irvinlim/apple-health-ingester/pkg/backends" "github.com/irvinlim/apple-health-ingester/pkg/ingester" ) -type RegisterBackendFunc func(ingester *ingester.Ingester, mux *http.ServeMux) error +type RegisterBackendFunc func(cfg *configv1.Config, ingester *ingester.Ingester, mux *http.ServeMux) error func RegisterBackend(backend backends.Backend, ingester *ingester.Ingester, mux *http.ServeMux, pattern string) error { mux.Handle(pattern, handleIngest(ingester, backend.Name())) diff --git a/cmd/ingester/main.go b/cmd/ingester/main.go index 70cb627..69d8f0e 100644 --- a/cmd/ingester/main.go +++ b/cmd/ingester/main.go @@ -3,35 +3,89 @@ package main import ( "context" "crypto/tls" + "errors" "net/http" "os" "os/signal" + "strings" "time" log "github.com/sirupsen/logrus" "github.com/spf13/pflag" + "sigs.k8s.io/yaml" + "github.com/irvinlim/apple-health-ingester/pkg/config" "github.com/irvinlim/apple-health-ingester/pkg/ingester" + "github.com/irvinlim/apple-health-ingester/pkg/util/logutils" +) + +var ( + defaultConfigFileLocations = []string{ + "config.yaml", // Load from current working directory + "config.json", // Support both JSON and YAML (prefer to use YAML first) + "/config/config.yaml", // Also support /config in case of container-based deployments (e.g. Docker, K8s) + "/config/config.json", // Finally also support JSON within /config + } ) func main() { - pflag.Parse() + // Parse flags from command-line. + flags, err := ParseFlags(pflag.CommandLine) + if err != nil { + log.Fatalf("failed to initialize command-line flags: %v", err) + } mux := http.NewServeMux() // Set log level - if logLevel != "" { - level, err := log.ParseLevel(logLevel) + if flags.LogLevel != "" { + level, err := log.ParseLevel(flags.LogLevel) if err != nil { - log.Fatalf("cannot parse log level: %v", logLevel) + log.Fatalf("cannot parse log level: %v", flags.LogLevel) } log.WithField("log_level", level).Info("setting log level") log.SetLevel(level) } + // Load config + cfg, err := LoadConfigAndMergeFlags(flags, config.LoadConfigArgs{ + ConfigFilePath: flags.ConfigFile, + OptionalConfigFilePaths: defaultConfigFileLocations, + }) + if err != nil { + log.Fatalf("failed to load config: %v", err) + } + + if log.IsLevelEnabled(log.DebugLevel) { + cfg := cfg.DeepCopy() + + // Redact certain fields. + if cfg.HttpServer.Auth.AuthorizationToken != "" { + cfg.HttpServer.Auth.AuthorizationToken = "REDACTED" + } + if len(cfg.HttpServer.TLS.CertData) > 0 { + cfg.HttpServer.TLS.CertData = "DATA+OMITTED" + } + if len(cfg.HttpServer.TLS.KeyData) > 0 { + cfg.HttpServer.TLS.KeyData = "DATA+OMITTED" + } + if cfg.Backends.InfluxDB.AuthToken != "" { + cfg.Backends.InfluxDB.AuthToken = "REDACTED" + } + + if out, err := yaml.Marshal(cfg); err == nil { + logutils.QuotesDisabled().WithField("config", "\n"+string(out)).Info("successfully loaded validated config") + } + } + + // Validate the configuration at this stage. + if err := config.Validate(cfg); err != nil { + log.WithError(err).Fatal("invalid config, see --help") + } + // Add middlewares middlewares := []Middleware{ createLoggingHandler(log.StandardLogger()), - createAuthenticateHandler(), + createAuthenticateHandler(cfg), } var handler http.Handler = mux for _, middleware := range middlewares { @@ -39,7 +93,7 @@ func main() { } server := &http.Server{ - Addr: listenAddr, + Addr: cfg.HttpServer.ListenAddr, Handler: handler, ReadHeaderTimeout: 10 * time.Second, TLSConfig: &tls.Config{ @@ -51,13 +105,34 @@ func main() { }, } + // Set up TLS. + if tlsCfg := cfg.HttpServer.TLS; tlsCfg.Enabled { + certData := strings.TrimSpace(tlsCfg.CertData) + keyData := strings.TrimSpace(tlsCfg.KeyData) + if len(certData) > 0 && len(keyData) > 0 { + cert, err := tls.X509KeyPair([]byte(certData), []byte(keyData)) + if err != nil { + log.Fatalf("failed to load server certificate and key: %v", err) + } + server.TLSConfig.Certificates = []tls.Certificate{cert} + } else if tlsCfg.CertFile != "" && tlsCfg.KeyFile != "" { + cert, err := tls.LoadX509KeyPair(tlsCfg.CertFile, tlsCfg.KeyFile) + if err != nil { + log.Fatalf("failed to load server certificate and key: %v", err) + } + server.TLSConfig.Certificates = []tls.Certificate{cert} + } else { + log.Fatalf("TLS cert and key are required, see --help") + } + } + // Initialize and register backends for ingester ingest := ingester.NewIngester() for _, register := range []RegisterBackendFunc{ RegisterDebugBackend, RegisterInfluxDBBackend, } { - if err := register(ingest, mux); err != nil { + if err := register(cfg, ingest, mux); err != nil { log.WithError(err).Fatal("add backend error") } } @@ -73,14 +148,15 @@ func main() { // Start http server go func() { - log.WithField("listen_addr", listenAddr).Info("starting http server") + log.WithField("listen_addr", cfg.HttpServer.ListenAddr).Info("starting http server") var err error - if enableTLS { - err = server.ListenAndServeTLS(certFile, keyFile) + if cfg.HttpServer.TLS.Enabled { + // Use the cert and key from the TLSConfig. + err = server.ListenAndServeTLS("", "") } else { err = server.ListenAndServe() } - if err != nil && err != http.ErrServerClosed { + if err != nil && !errors.Is(err, http.ErrServerClosed) { log.WithError(err).Panicf("cannot start http server") } }() diff --git a/cmd/ingester/middlewares.go b/cmd/ingester/middlewares.go index 4275ca9..4a14780 100644 --- a/cmd/ingester/middlewares.go +++ b/cmd/ingester/middlewares.go @@ -8,6 +8,8 @@ import ( "time" log "github.com/sirupsen/logrus" + + configv1 "github.com/irvinlim/apple-health-ingester/apis/config/v1" ) const ( @@ -56,7 +58,7 @@ func createLoggingHandler(logger *log.Logger) func(http.Handler) http.Handler { // createAuthenticateHandler returns a middleware that will authenticate // incoming http requests. -func createAuthenticateHandler() func(http.Handler) http.Handler { +func createAuthenticateHandler(cfg *configv1.Config) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { unauthorized := func(w http.ResponseWriter) { w.WriteHeader(http.StatusUnauthorized) @@ -64,7 +66,7 @@ func createAuthenticateHandler() func(http.Handler) http.Handler { } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if authorizationToken != "" { + if cfg.HttpServer.Auth.AuthorizationToken != "" { header := r.Header.Get("Authorization") if !strings.HasPrefix(header, bearerPrefix) { @@ -73,7 +75,7 @@ func createAuthenticateHandler() func(http.Handler) http.Handler { } token := header[len(bearerPrefix):] - if token != authorizationToken { + if token != cfg.HttpServer.Auth.AuthorizationToken { unauthorized(w) return } diff --git a/go.mod b/go.mod index 2d77a20..6fb14c0 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,8 @@ module github.com/irvinlim/apple-health-ingester go 1.23 require ( - github.com/google/go-cmp v0.5.6 + github.com/caarlos0/env/v11 v11.3.1 + github.com/google/go-cmp v0.6.0 github.com/hashicorp/go-multierror v1.1.1 github.com/influxdata/influxdb-client-go/v2 v2.6.0 github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 @@ -11,26 +12,28 @@ require ( github.com/mitchellh/mapstructure v1.1.2 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.6.0 - github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.7.0 + github.com/spf13/afero v1.12.0 + github.com/spf13/pflag v1.0.6 + github.com/stretchr/testify v1.10.0 k8s.io/client-go v0.23.1 + sigs.k8s.io/yaml v1.2.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/deepmap/oapi-codegen v1.8.2 // indirect - github.com/go-logr/logr v1.2.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect - golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e // indirect - golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect + golang.org/x/time v0.8.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apimachinery v0.23.1 // indirect k8s.io/klog/v2 v2.30.0 // indirect k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect diff --git a/go.sum b/go.sum index 71c096a..47c33b0 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,8 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA= +github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -88,8 +90,9 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= @@ -140,8 +143,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -239,8 +242,11 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -248,8 +254,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= @@ -345,8 +352,9 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20211209124913-491a49abca63 h1:iocB37TsdFuN6IBRZ+ry36wrkoV51/tl5vOWqkcPGvY= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -421,8 +429,9 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e h1:XMgFehsDnnLGtjvjOfqWSUzt0alpTR1RSEuznObga2c= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -435,13 +444,16 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= +golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -495,7 +507,6 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -614,8 +625,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -643,4 +655,5 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt new file mode 100644 index 0000000..e69de29 diff --git a/pkg/backends/influxdb/backend.go b/pkg/backends/influxdb/backend.go index f172423..82c147a 100644 --- a/pkg/backends/influxdb/backend.go +++ b/pkg/backends/influxdb/backend.go @@ -2,8 +2,6 @@ package influxdb import ( "context" - "fmt" - "strings" "time" "github.com/influxdata/influxdb-client-go/v2/api/write" @@ -11,6 +9,7 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + configv1 "github.com/irvinlim/apple-health-ingester/apis/config/v1" "github.com/irvinlim/apple-health-ingester/pkg/backends" "github.com/irvinlim/apple-health-ingester/pkg/healthautoexport" utiltime "github.com/irvinlim/apple-health-ingester/pkg/util/time" @@ -32,23 +31,23 @@ type Backend struct { var _ backends.Backend = &Backend{} -func NewBackend(client Client) (backends.Backend, error) { +func NewBackend(cfg *configv1.InfluxdbBackendConfig, client Client) (backends.Backend, error) { + if cfg == nil { + cfg = &configv1.InfluxdbBackendConfig{} + } + backend := &Backend{ ctx: context.TODO(), client: client, - staticTags: make([]lp.Tag, len(staticTags)), + staticTags: make([]lp.Tag, 0, len(cfg.StaticTags)), } // Prepare static tags. - for i, tag := range staticTags { - tokens := strings.SplitN(tag, "=", 2) - if len(tokens) != 2 { - return nil, fmt.Errorf("invalid static tag %v", tag) - } - backend.staticTags[i] = lp.Tag{ - Key: tokens[0], - Value: tokens[1], - } + for key, val := range cfg.StaticTags { + backend.staticTags = append(backend.staticTags, lp.Tag{ + Key: key, + Value: val, + }) } return backend, nil diff --git a/pkg/backends/influxdb/backend_test.go b/pkg/backends/influxdb/backend_test.go index 8299145..db38d56 100644 --- a/pkg/backends/influxdb/backend_test.go +++ b/pkg/backends/influxdb/backend_test.go @@ -145,7 +145,8 @@ type BackendTest struct { func NewBackendTest(t *testing.T) *BackendTest { client := influxdb.NewMockClient() - backend, err := influxdb.NewBackend(client) + // Use a nil config for now, but support overriding the client in tests in the future. + backend, err := influxdb.NewBackend(nil, client) if err != nil { t.Fatalf("init backend failed: %v", err) } diff --git a/pkg/backends/influxdb/client.go b/pkg/backends/influxdb/client.go index a63bf39..2776d9d 100644 --- a/pkg/backends/influxdb/client.go +++ b/pkg/backends/influxdb/client.go @@ -7,6 +7,7 @@ import ( influxdb2 "github.com/influxdata/influxdb-client-go/v2" "github.com/influxdata/influxdb-client-go/v2/api/write" + configv1 "github.com/irvinlim/apple-health-ingester/apis/config/v1" apierrors "github.com/irvinlim/apple-health-ingester/pkg/errors" ) @@ -27,16 +28,20 @@ type clientImpl struct { var _ Client = (*clientImpl)(nil) // NewClient returns a real influxdb Client initialized from flags. -func NewClient() (Client, error) { - client, err := NewInfluxDBClient() +func NewClient(cfg *configv1.InfluxdbBackendConfig) (Client, error) { + if cfg == nil { + cfg = &configv1.InfluxdbBackendConfig{} + } + + client, err := NewInfluxDBClient(cfg) if err != nil { return nil, err } impl := &clientImpl{ client: client, - orgName: orgName, - metricsBucketName: metricsBucketName, - workoutsBucketName: workoutsBucketName, + orgName: cfg.OrgName, + metricsBucketName: cfg.BucketNames.Metrics, + workoutsBucketName: cfg.BucketNames.Workouts, } return impl, nil } diff --git a/pkg/backends/influxdb/influxdb.go b/pkg/backends/influxdb/influxdb.go index 23ea0a2..f8d139a 100644 --- a/pkg/backends/influxdb/influxdb.go +++ b/pkg/backends/influxdb/influxdb.go @@ -5,41 +5,20 @@ import ( "errors" influxdb2 "github.com/influxdata/influxdb-client-go/v2" - "github.com/spf13/pflag" -) -var ( - serverURL string - insecureSkipVerify bool - authToken string - orgName string - metricsBucketName string - workoutsBucketName string - staticTags []string + configv1 "github.com/irvinlim/apple-health-ingester/apis/config/v1" ) -func NewInfluxDBClient() (influxdb2.Client, error) { - if serverURL == "" { - return nil, errors.New("--influxdb.serverURL is not set") +func NewInfluxDBClient(cfg *configv1.InfluxdbBackendConfig) (influxdb2.Client, error) { + if cfg.ServerURL == "" { + return nil, errors.New("serverURL is not set") } options := influxdb2.DefaultOptions(). SetTLSConfig(&tls.Config{ - InsecureSkipVerify: insecureSkipVerify, // nolint:gosec + InsecureSkipVerify: cfg.InsecureSkipVerify, // nolint:gosec }) - client := influxdb2.NewClientWithOptions(serverURL, authToken, options) + client := influxdb2.NewClientWithOptions(cfg.ServerURL, cfg.AuthToken, options) return client, nil } - -func init() { - pflag.StringVar(&serverURL, "influxdb.serverURL", "", "Server URL for InfluxDB.") - pflag.BoolVar(&insecureSkipVerify, "influxdb.insecureSkipVerify", false, - "Skip TLS verification of the certificate chain and host name for the InfluxDB server.") - pflag.StringVar(&authToken, "influxdb.authToken", "", "Auth token to connect to InfluxDB.") - pflag.StringVar(&orgName, "influxdb.orgName", "", "InfluxDB organization name.") - pflag.StringVar(&metricsBucketName, "influxdb.metricsBucketName", "", "InfluxDB bucket name for metrics.") - pflag.StringVar(&workoutsBucketName, "influxdb.workoutsBucketName", "", "InfluxDB bucket name for workouts.") - pflag.StringSliceVar(&staticTags, "influxdb.staticTags", nil, - "Additional tags to add to InfluxDB for every single request, in key=value format.") -} diff --git a/pkg/backends/localfile/backend.go b/pkg/backends/localfile/backend.go index c245d4f..8480c21 100644 --- a/pkg/backends/localfile/backend.go +++ b/pkg/backends/localfile/backend.go @@ -9,16 +9,12 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "github.com/spf13/pflag" + configv1 "github.com/irvinlim/apple-health-ingester/apis/config/v1" "github.com/irvinlim/apple-health-ingester/pkg/backends" "github.com/irvinlim/apple-health-ingester/pkg/healthautoexport" ) -var ( - metricsPath string -) - // Backend LocalFile is used to store ingested metrics in the local filesystem // as JSON files. It is not very performant as it would process all data at once // to produce a sorted JSON output file. As such, it should only be used for @@ -26,22 +22,25 @@ var ( // // TODO(irvinlim): Handle workout data type Backend struct { + cfg *configv1.LocalFileBackendConfig metrics map[string]*MetricFile mtx sync.RWMutex } var _ backends.Backend = &Backend{} -func NewBackend() (*Backend, error) { - backend := &Backend{} +func NewBackend(cfg *configv1.LocalFileBackendConfig) (*Backend, error) { + backend := &Backend{ + cfg: cfg, + } // Load metrics - if metricsPath == "" { + if cfg.MetricsPath == "" { return nil, errors.New("--localfile.metricsPath is not set") } metrics, err := backend.loadMetrics() if err != nil { - return nil, errors.Wrapf(err, "cannot load metrics from %v", metricsPath) + return nil, errors.Wrapf(err, "cannot load metrics from %v", cfg.MetricsPath) } backend.metrics = metrics @@ -103,7 +102,7 @@ func (b *Backend) handleMetric(metric *healthautoexport.Metric, target string) e metricFile.Data = updatedData // Write back - metricFilePath := path.Join(metricsPath, fileName) + metricFilePath := path.Join(b.cfg.MetricsPath, fileName) if err := b.writeMetricFile(metricFilePath, &metricFile); err != nil { return errors.Wrapf(err, "cannot write metrics to %v", metricFilePath) } @@ -113,7 +112,7 @@ func (b *Backend) handleMetric(metric *healthautoexport.Metric, target string) e func (b *Backend) loadMetrics() (map[string]*MetricFile, error) { output := make(map[string]*MetricFile) - files, err := os.ReadDir(metricsPath) + files, err := os.ReadDir(b.cfg.MetricsPath) if err != nil { // Directory doesn't exist, simply return empty map. if os.IsNotExist(err) { @@ -124,7 +123,7 @@ func (b *Backend) loadMetrics() (map[string]*MetricFile, error) { } for _, file := range files { - metricFilePath := path.Join(metricsPath, file.Name()) + metricFilePath := path.Join(b.cfg.MetricsPath, file.Name()) metricFile, err := b.loadMetricFile(metricFilePath) if err != nil { log.WithError(err).Warnf("could not read %v as metric file", metricFilePath) @@ -175,9 +174,3 @@ func (b *Backend) writeMetricFile(name string, metricFile *MetricFile) error { enc.SetIndent("", " ") return enc.Encode(metricFile) } - -func init() { - pflag.StringVar(&metricsPath, "localfile.metricsPath", "", - "Output path to write metrics, with one metric per file. All data will be aggregated by timestamp. "+ - "Any existing data will be merged together.") -} diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..ceaf1db --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,137 @@ +package config + +import ( + "github.com/caarlos0/env/v11" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/spf13/afero" + "sigs.k8s.io/yaml" + + configv1 "github.com/irvinlim/apple-health-ingester/apis/config/v1" +) + +type LoadConfigArgs struct { + // Specify an explicit config file path to load from. + // If specified but the file does not exist, Load will throw an error. + ConfigFilePath string + // List of configuration file paths to load from on a best-effort basis. + // The first file that is found will be used. + // Takes lower loading precedence than ConfigFilePath. + // No error will be thrown if the file paths do not exist. + OptionalConfigFilePaths []string + // Optionally specify a custom environment to load from. + // Useful for tests. + Environment map[string]string + // Optionally override the filesystem to load from. + // Useful for tests. + Fs afero.Fs +} + +// LoadAndValidate is a shortcut to load and return a validated configuration. +// See Load() and Validate(). +func LoadAndValidate(args LoadConfigArgs) (*configv1.Config, error) { + cfg, err := Load(args) + if err != nil { + return nil, errors.Wrapf(err, "failed to load config") + } + if err := Validate(cfg); err != nil { + return nil, errors.Wrapf(err, "invalid config") + } + return cfg, nil +} + +// Load attempts to load the configuration from the following sources, +// in order of precedence from lowest to highest: +// +// 1. Default config, specified by DefaultConfig (see defaults.go) +// 2. Configuration file +// 3. Environment variables +// +// Any configuration source that is not available will be simply skipped. +// +// Note that not all configuration fields should be configurable via files or +// environment variables. Examples include log verbosity, which often must be +// specified right at the beginning of the program, or complex configuration +// types that make it cumbersome to be specified via environment variables. +func Load(args LoadConfigArgs) (*configv1.Config, error) { + return LoadWithDefaultConfig(DefaultConfig, args) +} + +// LoadWithDefaultConfig is like Load, but allows you to specify a +// different DefaultConfig. +func LoadWithDefaultConfig(defaultCfg *configv1.Config, args LoadConfigArgs) (*configv1.Config, error) { + cfg := defaultCfg.DeepCopy() + + fs := args.Fs + if fs == nil { + fs = afero.NewOsFs() + } + + // First load from configuration file. + // Use the config file path specified explicitly if provided, otherwise use auto-discovery. + if args.ConfigFilePath != "" { + err := loadFromConfigFilePaths(cfg, fs, []string{args.ConfigFilePath}, true) + if err != nil { + return nil, errors.Wrapf(err, "failed to load config from file") + } + } else if len(args.OptionalConfigFilePaths) > 0 { + err := loadFromConfigFilePaths(cfg, fs, args.OptionalConfigFilePaths, false) + if err != nil { + return nil, errors.Wrapf(err, "failed to load config from file") + } + } + + // Next, load from environment variables. + if err := env.ParseWithOptions(cfg, env.Options{ + Environment: args.Environment, + }); err != nil { + return nil, errors.Wrapf(err, "failed to load config from environment") + } + + return cfg, nil +} + +func loadFromConfigFilePaths(target *configv1.Config, fs afero.Fs, paths []string, failOnError bool) error { + // Sanity check. + if len(paths) == 0 { + if failOnError { + return errors.New("empty path") + } + return nil + } + + // First attempt to load the first config file that exists. + for _, path := range paths { + path = expandUser(path) + exists, err := fileExists(fs, path, failOnError) + if err != nil { + return err + } + if !exists { + continue + } + + log.WithField("path", path).Info("loading config from file") + if err := unmarshalConfigFile(target, fs, path); err != nil { + return errors.Wrapf(err, `failed to unmarshal config file at "%v"`, path) + } + + log.WithField("path", path).Info("successfully loaded config from file") + return nil + } + + return nil +} + +func unmarshalConfigFile(target *configv1.Config, fs afero.Fs, path string) error { + data, err := afero.ReadFile(fs, path) + if err != nil { + return errors.Wrap(err, "failed to open file") + } + // Supports unmarshaling as either JSON or YAML. + if err := yaml.Unmarshal(data, target); err != nil { + log.WithField("path", path).WithError(err).Error("failed to unmarshal config file") + return errors.Wrap(err, "failed to unmarshal config") + } + return nil +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go new file mode 100644 index 0000000..f1fe542 --- /dev/null +++ b/pkg/config/config_test.go @@ -0,0 +1,258 @@ +package config + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + configv1 "github.com/irvinlim/apple-health-ingester/apis/config/v1" + "github.com/irvinlim/apple-health-ingester/pkg/util/testutils" +) + +func TestLoad(t *testing.T) { + type test struct { + name string + defaultCfg *configv1.Config + files map[string]string + env map[string]string + args LoadConfigArgs + wantErr assert.ErrorAssertionFunc + want *configv1.Config + } + tests := []test{ + { + name: "error if path does not exist, use default config", + args: LoadConfigArgs{ + ConfigFilePath: "/config.yaml", + }, + wantErr: testutils.AssertErrorContains("file does not exist"), + }, + { + name: "no error if optional file does not exist, use default config", + args: LoadConfigArgs{ + OptionalConfigFilePaths: []string{"/config.yaml"}, + }, + want: &configv1.Config{ + HttpServer: configv1.HttpServerConfig{ + ListenAddr: ":8080", + }, + }, + }, + { + name: "cannot load from directory", + files: map[string]string{ + "/config/config.yaml": ` +httpServer: + listenAddr: ":5050" +`, + }, + args: LoadConfigArgs{ + ConfigFilePath: "/config", + }, + wantErr: testutils.AssertErrorContains("/config is a directory"), + }, + { + name: "skip optional file path if directory", + files: map[string]string{ + "/config/config.yaml": ` +httpServer: + listenAddr: ":5050" +`, + }, + args: LoadConfigArgs{ + OptionalConfigFilePaths: []string{"/config"}, + }, + want: &configv1.Config{ + HttpServer: configv1.HttpServerConfig{ + ListenAddr: ":8080", + }, + }, + }, + { + name: "fail on first optional config file that exists but can't be parsed", + files: map[string]string{ + "/config.yaml": ` +httpServer: + listenAddr: ":5050 +`, + "/config2.yaml": ` +httpServer: + listenAddr: ":5051" +`, + }, + args: LoadConfigArgs{ + OptionalConfigFilePaths: []string{"/config.yaml", "/config2.yaml"}, + }, + wantErr: testutils.AssertErrorContains("found unexpected end of stream"), + }, + { + name: "successfully load YAML from config file", + files: map[string]string{ + "/config.yaml": ` +httpServer: + listenAddr: ":5050" +`, + }, + args: LoadConfigArgs{ + ConfigFilePath: "/config.yaml", + }, + want: &configv1.Config{ + HttpServer: configv1.HttpServerConfig{ + ListenAddr: ":5050", + }, + }, + }, + { + name: "successfully load JSON from config file", + files: map[string]string{ + "/config.json": `{"httpServer":{"listenAddr":":5050"}}`, + }, + args: LoadConfigArgs{ + ConfigFilePath: "/config.json", + }, + want: &configv1.Config{ + HttpServer: configv1.HttpServerConfig{ + ListenAddr: ":5050", + }, + }, + }, + { + name: "load only from non-optional config file path", + files: map[string]string{ + "/config.yaml": ` +httpServer: + listenAddr: ":5050" +`, + "/config2.yaml": ` +httpServer: + listenAddr: ":5051" +`, + }, + args: LoadConfigArgs{ + ConfigFilePath: "/config.yaml", + OptionalConfigFilePaths: []string{"/config2.yaml"}, + }, + want: &configv1.Config{ + HttpServer: configv1.HttpServerConfig{ + ListenAddr: ":5050", + }, + }, + }, + { + name: "load only from first existent optional file", + files: map[string]string{ + "/config.yaml": ` +httpServer: + listenAddr: ":5050" +`, + "/config2.yaml": ` +httpServer: + listenAddr: ":5051" +`, + }, + args: LoadConfigArgs{ + OptionalConfigFilePaths: []string{"/config.yaml", "/config2.yaml"}, + }, + want: &configv1.Config{ + HttpServer: configv1.HttpServerConfig{ + ListenAddr: ":5050", + }, + }, + }, + { + name: "override with environment variable", + files: map[string]string{ + "/config.yaml": ` +httpServer: + listenAddr: ":5050" +`, + }, + env: map[string]string{ + "LISTEN_ADDR": ":5051", + }, + args: LoadConfigArgs{ + ConfigFilePath: "/config.yaml", + }, + want: &configv1.Config{ + HttpServer: configv1.HttpServerConfig{ + ListenAddr: ":5051", + }, + }, + }, + { + name: "do not override with empty environment variable", + files: map[string]string{ + "/config.yaml": ` +httpServer: + listenAddr: ":5050" +`, + }, + env: map[string]string{ + "LISTEN_ADDR": "", + }, + args: LoadConfigArgs{ + ConfigFilePath: "/config.yaml", + }, + want: &configv1.Config{ + HttpServer: configv1.HttpServerConfig{ + ListenAddr: ":5050", + }, + }, + }, + { + name: "can load env var in nested struct", + files: map[string]string{ + "/config.yaml": ` +httpServer: + listenAddr: ":5050" +`, + }, + env: map[string]string{ + "TLS_ENABLED": "true", + "TLS_CERT_FILE": "server.crt", + "TLS_KEY_FILE": "server.key", + }, + args: LoadConfigArgs{ + ConfigFilePath: "/config.yaml", + }, + want: &configv1.Config{ + HttpServer: configv1.HttpServerConfig{ + ListenAddr: ":5050", + TLS: configv1.HttpTLSConfig{ + Enabled: true, + CertFile: "server.crt", + KeyFile: "server.key", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fs := afero.NewMemMapFs() + defaultCfg := tt.defaultCfg + if defaultCfg == nil { + defaultCfg = DefaultConfig + } + + // Set up mock FS and environment. + for path, data := range tt.files { + require.NoError(t, afero.WriteFile(fs, path, []byte(data), 0644)) + } + args := tt.args + args.Fs = fs + args.Environment = tt.env + + cfg, err := LoadWithDefaultConfig(defaultCfg, args) + if testutils.WantError(t, tt.wantErr, err) { + return + } + if !cmp.Equal(tt.want, cfg) { + t.Errorf("not equal:\ndiff = %v", cmp.Diff(tt.want, cfg)) + } + }) + } +} diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go new file mode 100644 index 0000000..8e4a7da --- /dev/null +++ b/pkg/config/defaults.go @@ -0,0 +1,13 @@ +package config + +import ( + configv1 "github.com/irvinlim/apple-health-ingester/apis/config/v1" +) + +var ( + DefaultConfig = &configv1.Config{ + HttpServer: configv1.HttpServerConfig{ + ListenAddr: ":8080", + }, + } +) diff --git a/pkg/config/util.go b/pkg/config/util.go new file mode 100644 index 0000000..14a0ee1 --- /dev/null +++ b/pkg/config/util.go @@ -0,0 +1,51 @@ +package config + +import ( + "os" + "os/user" + "path/filepath" + "strings" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/spf13/afero" +) + +func fileExists(fs afero.Fs, path string, failOnError bool) (bool, error) { + stat, err := fs.Stat(path) + if err != nil { + // If failOnError is false we can ignore ENOENT. + if !failOnError && os.IsNotExist(err) { + return false, nil + } + return false, errors.Wrapf(err, "failed to stat %s", path) + } + if stat.IsDir() { + if failOnError { + return false, errors.Errorf("%s is a directory", path) + } + // Show a warning if it is a directory instead of a file. + log.WithField("path", path).Warning("warning: config file path is a directory, skipping") + return false, nil + } + return true, nil +} + +// expandUser expands the given path (such as "~/.ssh") to refer to the path +// located within the home directory specific to the platform that the program +// is currently running on. +// +// For example on macOS, this expands to "/Users/johndoe/.ssh". +func expandUser(path string) string { + usr, _ := user.Current() + dir := usr.HomeDir + // Exact match. + if path == "~" { + return dir + } + // Only match prefixes. + if strings.HasPrefix(path, "~/") { + path = filepath.Join(dir, path[2:]) + } + return path +} diff --git a/pkg/config/validation.go b/pkg/config/validation.go new file mode 100644 index 0000000..0c15d06 --- /dev/null +++ b/pkg/config/validation.go @@ -0,0 +1,142 @@ +package config + +import ( + "strconv" + + "k8s.io/apimachinery/pkg/util/validation/field" + + configv1 "github.com/irvinlim/apple-health-ingester/apis/config/v1" +) + +// Validate performs validation on the given config. +func Validate(cfg *configv1.Config) error { + validator := &Validator{} + errList := validator.Validate(cfg) + if len(errList) > 0 { + return errList.ToAggregate() + } + return nil +} + +// Validator is a single instance of a config validator. +type Validator struct{} + +// Validate is the entrypoint to validate a config. +func (v *Validator) Validate(cfg *configv1.Config) field.ErrorList { + fldPath := field.NewPath("config") + allErrs := field.ErrorList{} + allErrs = append(allErrs, v.validateHttpServerConfig(fldPath.Child("httpServer"), cfg.HttpServer)...) + allErrs = append(allErrs, v.validateBackendsConfig(fldPath.Child("backends"), cfg.Backends)...) + return allErrs +} + +func (v *Validator) validateHttpServerConfig(fldPath *field.Path, cfg configv1.HttpServerConfig) field.ErrorList { + allErrs := field.ErrorList{} + + if cfg.ListenAddr == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("listenAddr"), "listenAddr is required")) + } else if _, err := strconv.Atoi(cfg.ListenAddr); err == nil { + // Show an error if we managed to parse it as a number, which is incorrect. + allErrs = append(allErrs, field.Invalid(fldPath.Child("listenAddr"), cfg.ListenAddr, `should be specified as ":" or ":" format`)) + } + + allErrs = append(allErrs, v.validateHttpTLSConfig(fldPath.Child("tls"), cfg.TLS)...) + return allErrs +} + +func (v *Validator) validateHttpTLSConfig(fldPath *field.Path, cfg configv1.HttpTLSConfig) field.ErrorList { + allErrs := field.ErrorList{} + + // Skip validation if not enabled. + if !cfg.Enabled { + return allErrs + } + + // Exactly one of certFile / certData must be specified. + var certSpecified int + if cfg.CertFile != "" { + certSpecified++ + } + if len(cfg.CertData) > 0 { + certSpecified++ + } + if certSpecified == 0 { + allErrs = append(allErrs, field.Required(fldPath, "must specify either certFile or certData")) + } else if certSpecified > 1 { + allErrs = append(allErrs, field.Invalid(fldPath, cfg.CertFile, "exactly one or certFile or certData must be specified")) + } + + // Exactly one of keyFile / keyData must be specified. + var keySpecified int + if cfg.KeyFile != "" { + keySpecified++ + } + if len(cfg.KeyData) > 0 { + keySpecified++ + } + if keySpecified == 0 { + allErrs = append(allErrs, field.Required(fldPath, "must specify either keyFile or keyData")) + } else if keySpecified > 1 { + allErrs = append(allErrs, field.Invalid(fldPath, cfg.KeyFile, "exactly one or keyFile or keyData must be specified")) + } + + return allErrs +} + +func (v *Validator) validateBackendsConfig(fldPath *field.Path, cfg configv1.BackendsConfig) field.ErrorList { + allErrs := field.ErrorList{} + + var enabled int + if cfg.InfluxDB.Enabled { + enabled++ + allErrs = append(allErrs, v.validateInfluxdbBackendConfig(fldPath.Child("influxdb"), cfg.InfluxDB)...) + } + if cfg.LocalFile.Enabled { + enabled++ + allErrs = append(allErrs, v.validateLocalFileBackendConfig(fldPath.Child("localFile"), cfg.LocalFile)...) + } + if enabled == 0 { + allErrs = append(allErrs, field.Required(fldPath, "at least one backend must be specified")) + } + + return allErrs +} + +func (v *Validator) validateInfluxdbBackendConfig(fldPath *field.Path, cfg configv1.InfluxdbBackendConfig) field.ErrorList { + allErrs := field.ErrorList{} + + // Skip validation if not enabled. + if !cfg.Enabled { + return allErrs + } + + if cfg.ServerURL == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("serverURL"), "serverURL is required")) + } + if cfg.OrgName == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("orgName"), "organization name is required")) + } + if cfg.BucketNames.Metrics == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("bucketNames", "metrics"), "bucket name is required")) + } + if cfg.BucketNames.Workouts == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("bucketNames", "workouts"), "bucket name is required")) + } + + return allErrs +} + +func (v *Validator) validateLocalFileBackendConfig(fldPath *field.Path, cfg configv1.LocalFileBackendConfig) field.ErrorList { + allErrs := field.ErrorList{} + + // Skip validation if not enabled. + if !cfg.Enabled { + return allErrs + } + + if cfg.MetricsPath == "" { + allErrs = append(allErrs, field.Required(fldPath.Child("metricsPath"), "metrics path is required")) + } + + return allErrs +} diff --git a/pkg/util/logutils/logutils.go b/pkg/util/logutils/logutils.go new file mode 100644 index 0000000..843551b --- /dev/null +++ b/pkg/util/logutils/logutils.go @@ -0,0 +1,15 @@ +package logutils + +import ( + log "github.com/sirupsen/logrus" +) + +// QuotesDisabled returns a new Logger that disables quotes. +func QuotesDisabled() *log.Logger { + logger := log.New() + logger.SetLevel(log.GetLevel()) + logger.SetFormatter(&log.TextFormatter{ + DisableQuote: true, + }) + return logger +}