chore: support non-linux GOOS in e2e tests

Signed-off-by: Pranshu Srivastava <[email protected]>

Author: rexagod

.github/workflows/freebsd.yml

@@ -30,7 +30,7 @@ jobs:
           prepare: |
             pkg update -f
             pkg install -y bash curl git gmake go${GO_VERSION} python
-          run: | # The steps below are executed on the FreeBSD VM.
+          run: | # The steps below are executed inside the FreeBSD VM.
             set -eux
             mkdir golang-bin
             ln -s $(which go${GO_VERSION}) golang-bin/go

Makefile.common

@@ -201,7 +201,13 @@ common-unused:
 .PHONY: common-build
 common-build: promu
 	@echo ">> building binaries"
-	$(PROMU) build --prefix $(PREFIX) $(PROMU_BINARIES)
+	@$(PROMU) build --prefix $(PREFIX) $(PROMU_BINARIES)
+	@# Use the codesign binary directly instead of `promu codesign` as it depends on an image that doesn't support linux/arm64 (w.r.t. darwin's rosetta emulation) yet.
+	@if [ "$(GOHOSTOS)" == "darwin" ]; then \
+        if command -v codesign > /dev/null 2>&1; then \
+			codesign --sign - --force --preserve-metadata=entitlements,requirements,flags,runtime node_exporter ;\
+		fi; \
+	fi
 
 .PHONY: common-tarball
 common-tarball: promu

collector/fixtures/e2e-output-freebsd.txt

@@ -0,0 +1,181 @@
+# HELP go_gc_duration_seconds A summary of the wall-time pause (stop-the-world) duration in garbage collection cycles.
+# TYPE go_gc_duration_seconds summary
+# HELP go_gc_gogc_percent Heap size target percentage configured by the user, otherwise 100. This value is set by the GOGC environment variable, and the runtime/debug.SetGCPercent function. Sourced from /gc/gogc:percent
+# TYPE go_gc_gogc_percent gauge
+# HELP go_gc_gomemlimit_bytes Go runtime memory limit configured by the user, otherwise math.MaxInt64. This value is set by the GOMEMLIMIT environment variable, and the runtime/debug.SetMemoryLimit function. Sourced from /gc/gomemlimit:bytes
+# TYPE go_gc_gomemlimit_bytes gauge
+# HELP go_goroutines Number of goroutines that currently exist.
+# TYPE go_goroutines gauge
+# HELP go_info Information about the Go environment.
+# TYPE go_info gauge
+# HELP go_memstats_alloc_bytes Number of bytes allocated in heap and currently in use. Equals to /memory/classes/heap/objects:bytes.
+# TYPE go_memstats_alloc_bytes gauge
+# HELP go_memstats_alloc_bytes_total Total number of bytes allocated in heap until now, even if released already. Equals to /gc/heap/allocs:bytes.
+# TYPE go_memstats_alloc_bytes_total counter
+# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. Equals to /memory/classes/profiling/buckets:bytes.
+# TYPE go_memstats_buck_hash_sys_bytes gauge
+# HELP go_memstats_frees_total Total number of heap objects frees. Equals to /gc/heap/frees:objects + /gc/heap/tiny/allocs:objects.
+# TYPE go_memstats_frees_total counter
+# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. Equals to /memory/classes/metadata/other:bytes.
+# TYPE go_memstats_gc_sys_bytes gauge
+# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and currently in use, same as go_memstats_alloc_bytes. Equals to /memory/classes/heap/objects:bytes.
+# TYPE go_memstats_heap_alloc_bytes gauge
+# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. Equals to /memory/classes/heap/released:bytes + /memory/classes/heap/free:bytes.
+# TYPE go_memstats_heap_idle_bytes gauge
+# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. Equals to /memory/classes/heap/objects:bytes + /memory/classes/heap/unused:bytes
+# TYPE go_memstats_heap_inuse_bytes gauge
+# HELP go_memstats_heap_objects Number of currently allocated objects. Equals to /gc/heap/objects:objects.
+# TYPE go_memstats_heap_objects gauge
+# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. Equals to /memory/classes/heap/released:bytes.
+# TYPE go_memstats_heap_released_bytes gauge
+# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. Equals to /memory/classes/heap/objects:bytes + /memory/classes/heap/unused:bytes + /memory/classes/heap/released:bytes + /memory/classes/heap/free:bytes.
+# TYPE go_memstats_heap_sys_bytes gauge
+# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection.
+# TYPE go_memstats_last_gc_time_seconds gauge
+# HELP go_memstats_mallocs_total Total number of heap objects allocated, both live and gc-ed. Semantically a counter version for go_memstats_heap_objects gauge. Equals to /gc/heap/allocs:objects + /gc/heap/tiny/allocs:objects.
+# TYPE go_memstats_mallocs_total counter
+# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. Equals to /memory/classes/metadata/mcache/inuse:bytes.
+# TYPE go_memstats_mcache_inuse_bytes gauge
+# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. Equals to /memory/classes/metadata/mcache/inuse:bytes + /memory/classes/metadata/mcache/free:bytes.
+# TYPE go_memstats_mcache_sys_bytes gauge
+# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. Equals to /memory/classes/metadata/mspan/inuse:bytes.
+# TYPE go_memstats_mspan_inuse_bytes gauge
+# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. Equals to /memory/classes/metadata/mspan/inuse:bytes + /memory/classes/metadata/mspan/free:bytes.
+# TYPE go_memstats_mspan_sys_bytes gauge
+# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. Equals to /gc/heap/goal:bytes.
+# TYPE go_memstats_next_gc_bytes gauge
+# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. Equals to /memory/classes/other:bytes.
+# TYPE go_memstats_other_sys_bytes gauge
+# HELP go_memstats_stack_inuse_bytes Number of bytes obtained from system for stack allocator in non-CGO environments. Equals to /memory/classes/heap/stacks:bytes.
+# TYPE go_memstats_stack_inuse_bytes gauge
+# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. Equals to /memory/classes/heap/stacks:bytes + /memory/classes/os-stacks:bytes.
+# TYPE go_memstats_stack_sys_bytes gauge
+# HELP go_memstats_sys_bytes Number of bytes obtained from system. Equals to /memory/classes/total:byte.
+# TYPE go_memstats_sys_bytes gauge
+# HELP go_sched_gomaxprocs_threads The current runtime.GOMAXPROCS setting, or the number of operating system threads that can execute user-level Go code simultaneously. Sourced from /sched/gomaxprocs:threads
+# TYPE go_sched_gomaxprocs_threads gauge
+# HELP go_threads Number of OS threads created.
+# TYPE go_threads gauge
+# HELP node_boot_time_seconds Unix time of last boot, including microseconds.
+# TYPE node_boot_time_seconds gauge
+node_boot_time_seconds 1.732329528488754e+09
+# HELP node_exec_software_interrupts_total Software interrupts since system boot.  Resets at architecture unsigned integer.
+# TYPE node_exec_software_interrupts_total counter
+node_exec_software_interrupts_total 43
+# HELP node_exporter_build_info A metric with a constant '1' value labeled by version, revision, branch, goversion from which node_exporter was built, and the goos and goarch for the build.
+# TYPE node_exporter_build_info gauge
+# HELP node_memory_cache_bytes Almost free, backed by swap or files, available for re-allocation
+# TYPE node_memory_cache_bytes gauge
+node_memory_cache_bytes 0
+# HELP node_memory_size_bytes Total physical memory size
+# TYPE node_memory_size_bytes gauge
+node_memory_size_bytes 6.214455296e+09
+# HELP node_memory_swap_in_bytes_total Bytes paged in from swap devices
+# TYPE node_memory_swap_in_bytes_total counter
+node_memory_swap_in_bytes_total 0
+# HELP node_memory_swap_out_bytes_total Bytes paged out to swap devices
+# TYPE node_memory_swap_out_bytes_total counter
+node_memory_swap_out_bytes_total 0
+# HELP node_memory_swap_size_bytes Total swap memory size
+# TYPE node_memory_swap_size_bytes gauge
+node_memory_swap_size_bytes 1.073741824e+09
+# HELP node_memory_swap_used_bytes Currently allocated swap
+# TYPE node_memory_swap_used_bytes gauge
+node_memory_swap_used_bytes 0
+# HELP node_memory_user_wired_bytes Locked in memory by user, mlock, etc
+# TYPE node_memory_user_wired_bytes gauge
+node_memory_user_wired_bytes 0
+# HELP node_netisr_bindthreads netisr threads bound to CPUs
+# TYPE node_netisr_bindthreads gauge
+node_netisr_bindthreads 0
+# HELP node_netisr_defaultqlimit netisr default queue limit
+# TYPE node_netisr_defaultqlimit gauge
+node_netisr_defaultqlimit 256
+# HELP node_netisr_maxprot netisr maximum protocols
+# TYPE node_netisr_maxprot gauge
+node_netisr_maxprot 16
+# HELP node_netisr_maxqlimit netisr maximum queue limit
+# TYPE node_netisr_maxqlimit gauge
+node_netisr_maxqlimit 10240
+# HELP node_netisr_maxthreads netisr maximum thread count
+# TYPE node_netisr_maxthreads gauge
+node_netisr_maxthreads 1
+# HELP node_netisr_numthreads netisr current thread count
+# TYPE node_netisr_numthreads gauge
+node_netisr_numthreads 1
+# HELP node_network_receive_bytes_total Network device statistic receive_bytes.
+# TYPE node_network_receive_bytes_total counter
+# HELP node_network_receive_drop_total Network device statistic receive_drop.
+# TYPE node_network_receive_drop_total counter
+node_network_receive_drop_total{device="em0"} 0
+node_network_receive_drop_total{device="lo0"} 0
+# HELP node_network_receive_errs_total Network device statistic receive_errs.
+# TYPE node_network_receive_errs_total counter
+node_network_receive_errs_total{device="em0"} 0
+node_network_receive_errs_total{device="lo0"} 0
+# HELP node_network_receive_multicast_total Network device statistic receive_multicast.
+# TYPE node_network_receive_multicast_total counter
+node_network_receive_multicast_total{device="em0"} 3
+node_network_receive_multicast_total{device="lo0"} 0
+# HELP node_network_receive_packets_total Network device statistic receive_packets.
+# TYPE node_network_receive_packets_total counter
+# HELP node_network_transmit_bytes_total Network device statistic transmit_bytes.
+# TYPE node_network_transmit_bytes_total counter
+# HELP node_network_transmit_drop_total Network device statistic transmit_drop.
+# TYPE node_network_transmit_drop_total counter
+node_network_transmit_drop_total{device="em0"} 0
+node_network_transmit_drop_total{device="lo0"} 0
+# HELP node_network_transmit_errs_total Network device statistic transmit_errs.
+# TYPE node_network_transmit_errs_total counter
+node_network_transmit_errs_total{device="em0"} 0
+node_network_transmit_errs_total{device="lo0"} 0
+# HELP node_network_transmit_multicast_total Network device statistic transmit_multicast.
+# TYPE node_network_transmit_multicast_total counter
+node_network_transmit_multicast_total{device="em0"} 8
+node_network_transmit_multicast_total{device="lo0"} 0
+# HELP node_network_transmit_packets_total Network device statistic transmit_packets.
+# TYPE node_network_transmit_packets_total counter
+# HELP node_os_info A metric with a constant '1' value labeled by build_id, id, id_like, image_id, image_version, name, pretty_name, variant, variant_id, version, version_codename, version_id.
+# TYPE node_os_info gauge
+node_os_info{build_id="",id="ubuntu",id_like="debian",image_id="",image_version="",name="Ubuntu",pretty_name="Ubuntu 20.04.2 LTS",variant="",variant_id="",version="20.04.2 LTS (Focal Fossa)",version_codename="focal",version_id="20.04"} 1
+# HELP node_os_version Metric containing the major.minor part of the OS version.
+# TYPE node_os_version gauge
+node_os_version{id="ubuntu",id_like="debian",name="Ubuntu"} 20.04
+# HELP node_scrape_collector_duration_seconds node_exporter: Duration of a collector scrape.
+# TYPE node_scrape_collector_duration_seconds gauge
+# HELP node_scrape_collector_success node_exporter: Whether a collector succeeded.
+# TYPE node_scrape_collector_success gauge
+node_scrape_collector_success{collector="boottime"} 1
+node_scrape_collector_success{collector="cpu"} 1
+node_scrape_collector_success{collector="exec"} 1
+node_scrape_collector_success{collector="loadavg"} 1
+node_scrape_collector_success{collector="meminfo"} 1
+node_scrape_collector_success{collector="netdev"} 1
+node_scrape_collector_success{collector="netisr"} 1
+node_scrape_collector_success{collector="os"} 1
+node_scrape_collector_success{collector="textfile"} 1
+node_scrape_collector_success{collector="time"} 1
+node_scrape_collector_success{collector="uname"} 1
+node_scrape_collector_success{collector="zfs"} 1
+# HELP node_textfile_scrape_error 1 if there was an error opening or reading a file, 0 otherwise
+# TYPE node_textfile_scrape_error gauge
+node_textfile_scrape_error 0
+# HELP node_time_seconds System time in seconds since epoch (1970).
+# TYPE node_time_seconds gauge
+# HELP node_time_zone_offset_seconds System time zone offset in seconds.
+# TYPE node_time_zone_offset_seconds gauge
+# HELP node_uname_info Labeled system information as provided by the uname system call.
+# TYPE node_uname_info gauge
+node_uname_info{domainname="(none)",machine="amd64",nodename="freebsd",release="14.1-RELEASE",sysname="FreeBSD",version="FreeBSD 14.1-RELEASE releng/14.1-n267679-10e31f0946d8 GENERIC "} 1
+# HELP promhttp_metric_handler_errors_total Total number of internal errors encountered by the promhttp metric handler.
+# TYPE promhttp_metric_handler_errors_total counter
+promhttp_metric_handler_errors_total{cause="encoding"} 0
+promhttp_metric_handler_errors_total{cause="gathering"} 0
+# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served.
+# TYPE promhttp_metric_handler_requests_in_flight gauge
+promhttp_metric_handler_requests_in_flight 1
+# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code.
+# TYPE promhttp_metric_handler_requests_total counter
+promhttp_metric_handler_requests_total{code="200"} 0
+promhttp_metric_handler_requests_total{code="500"} 0
+promhttp_metric_handler_requests_total{code="503"} 0

end-to-end-test.sh

@@ -2,6 +2,8 @@
 
 set -euf -o pipefail
 
+GOHOSTOS=$(go env GOHOSTOS)
+
 enabled_collectors=$(cat << COLLECTORS
   arp
   bcache
@@ -57,13 +59,31 @@ enabled_collectors=$(cat << COLLECTORS
   zoneinfo
 COLLECTORS
 )
+
+supported_enabled_collectors=""
+for collector in ${enabled_collectors}; do
+  collector_paths=$(find ./collector -type f -name "${collector}"_"${GOHOSTOS}".go -exec basename {} _"${GOHOSTOS}".go \;)
+  if [ -n "$collector_paths" ]; then
+    supported_enabled_collectors="${supported_enabled_collectors} ${collector}"
+  fi
+done
+
 disabled_collectors=$(cat << COLLECTORS
   selinux
   filesystem
   timex
   uname
 COLLECTORS
 )
+
+supported_disabled_collectors=""
+for collector in ${disabled_collectors}; do
+  collector_paths=$(find ./collector -type f -name "${collector}"_"${GOHOSTOS}".go -exec basename {} _"${GOHOSTOS}".go \;)
+  if [ -n "$collector_paths" ]; then
+    supported_disabled_collectors="${supported_disabled_collectors} ${collector}"
+  fi
+done
+
 cd "$(dirname $0)"
 
 port="$((10000 + (RANDOM % 10000)))"
@@ -74,8 +94,8 @@ skip_re="^(go_|node_exporter_build_info|node_scrape_collector_duration_seconds|p
 arch="$(uname -m)"
 
 case "${arch}" in
-  aarch64|ppc64le) fixture='collector/fixtures/e2e-64k-page-output.txt' ;;
-  *) fixture='collector/fixtures/e2e-output.txt' ;;
+  aarch64|ppc64le) fixture_metrics='collector/fixtures/e2e-64k-page-output.txt' ;;
+  *) fixture_metrics='collector/fixtures/e2e-output.txt' ;;
 esac
 
 # Only test CPU info collection on x86_64.
@@ -109,7 +129,7 @@ do
     *)
       echo "Usage: $0 [-k] [-u] [-v]"
       echo "  -k: keep temporary files and leave node_exporter running"
-      echo "  -u: update fixture"
+      echo "  -u: update fixture_metrics"
       echo "  -v: verbose output"
       exit 1
       ;;
@@ -122,13 +142,7 @@ then
     exit 1
 fi
 
-./node_exporter \
-  --path.rootfs="collector/fixtures" \
-  --path.procfs="collector/fixtures/proc" \
-  --path.sysfs="collector/fixtures/sys" \
-  --path.udev.data="collector/fixtures/udev/data" \
-  $(for c in ${enabled_collectors}; do echo --collector.${c}  ; done) \
-  $(for c in ${disabled_collectors}; do echo --no-collector.${c}  ; done) \
+collector_flags=$(cat << FLAGS
   --collector.textfile.directory="collector/fixtures/textfile/two_metric_files/" \
   --collector.wifi.fixtures="collector/fixtures/wifi" \
   --collector.qdisc.fixtures="collector/fixtures/qdisc/" \
@@ -147,12 +161,60 @@ fi
   --collector.sysctl.include="kernel.threads-max" \
   --collector.sysctl.include="fs.file-nr" \
   --collector.sysctl.include="fs.file-nr:total,current,max" \
-  --collector.sysctl.include-info="kernel.seccomp.actions_avail" \
+  --collector.sysctl.include-info="kernel.seccomp.actions_avail"
+FLAGS
+)
+
+# The procedure below does three things, based on the gathered list of supported collectors (by the host) from above:
+# * Fetch the files that are associated with these collectors.
+# * Extract all flags defined within these files.
+# * Filter out flags (from $collector_flags) that are not supported by the supported collectors.
+filtered_collector_flags=""
+# Filter out flags that are not provided by the supported collectors.
+for collector in ${supported_enabled_collectors} ${supported_disabled_collectors}; do
+  # Currently, the following file formats (OS-ARCH combinations) exist in the repository. This is used to narrow down
+  # the search scope to the files supported by host's OS and/or ARCH.
+  maybe_flag_search_scope="${collector}_${GOHOSTOS}.go ${collector}_${GOHOSTOS}_${arch}.go"
+  # Check if these files exist.
+  flag_search_scope=""
+  for file in ${maybe_flag_search_scope}; do
+    if [ -f "collector/${file}" ]; then
+      flag_search_scope="${flag_search_scope} collector/${file}"
+    fi
+  done
+  # Collect all flags defined within these files.
+  supported_collector_flags=$(grep -h -E -o -- 'kingpin.Flag\("collector[.a-zA-Z0-9_\-]+' ${flag_search_scope} | sed 's/kingpin.Flag("//' | sort | uniq || true)
+  if [ -z "${supported_collector_flags}" ]; then
+    continue
+  fi
+  # Filter out flags that are not provided by the supported collectors.
+  for flag in ${collector_flags}; do
+    flag_name=$(echo "${flag}" | tr -d "\" " | cut -d"=" -f1 | cut -c 3-)
+    if echo "${supported_collector_flags}" | grep -q "${flag_name}"; then
+      filtered_collector_flags="${filtered_collector_flags} ${flag}"
+    fi
+  done
+done
+
+./node_exporter \
+  --path.rootfs="collector/fixtures" \
+  --path.procfs="collector/fixtures/proc" \
+  --path.sysfs="collector/fixtures/sys" \
+  --path.udev.data="collector/fixtures/udev/data" \
+  $(for c in ${supported_enabled_collectors}; do echo --collector.${c}  ; done) \
+  $(for c in ${supported_disabled_collectors}; do echo --no-collector.${c}  ; done) \
+  ${filtered_collector_flags} \
   --web.listen-address "127.0.0.1:${port}" \
   --log.level="debug" > "${tmpdir}/node_exporter.log" 2>&1 &
 
 echo $! > "${tmpdir}/node_exporter.pid"
 
+generated_metrics="${tmpdir}/e2e-output.txt"
+if [ "${GOHOSTOS}" != "linux" ]; then
+  generated_metrics="${tmpdir}/e2e-output-${GOHOSTOS}.txt"
+  fixture_metrics="${fixture_metrics::-4}-${GOHOSTOS}.txt"
+fi
+
 finish() {
   if [ $? -ne 0 -o ${verbose} -ne 0 ]
   then
@@ -165,7 +227,7 @@ EOF
 
   if [ ${update} -ne 0 ]
   then
-    cp "${tmpdir}/e2e-output.txt" "${fixture}"
+    cp "${generated_metrics}" "${fixture_metrics}"
   fi
 
   if [ ${keep} -eq 0 ]
@@ -195,8 +257,36 @@ get() {
 
 sleep 1
 
-get "127.0.0.1:${port}/metrics" | grep -E -v "${skip_re}" > "${tmpdir}/e2e-output.txt"
+get "127.0.0.1:${port}/metrics" | grep -E -v "${skip_re}" > "${generated_metrics}"
+
+non_deterministic_metrics=$(cat << METRICS
+  node_cpu_seconds_total
+  node_exec_context_switches_total
+  node_exec_device_interrupts_total
+  node_exec_forks_total
+  node_exec_system_calls_total
+  node_exec_traps_total
+  node_load1
+  node_load15
+  node_load5
+  node_memory_active_bytes
+  node_memory_buffer_bytes
+  node_memory_free_bytes
+  node_memory_inactive_bytes
+  node_memory_laundry_bytes
+  node_memory_wired_bytes
+METRICS
+)
+
+# Remove non-deterministic metrics from the generated metrics file.
+for os in freebsd; do
+  if [ "${GOHOSTOS}" = "${os}" ]; then
+    for metric in ${non_deterministic_metrics}; do
+      sed -i "/^${metric}/d" "${generated_metrics}"
+    done
+  fi
+done
 
 diff -u \
-  "${fixture}" \
-  "${tmpdir}/e2e-output.txt"
+  "${fixture_metrics}" \
+  "${generated_metrics}"