diff --git a/modules/default.nix b/modules/default.nix index ed5802d4..c9131a9f 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -5,5 +5,6 @@ ./pyroscope ./folder-size-metrics ./shard-split + ./random-alerts ]; } diff --git a/modules/random-alerts/default.nix b/modules/random-alerts/default.nix new file mode 100644 index 00000000..121d65c1 --- /dev/null +++ b/modules/random-alerts/default.nix @@ -0,0 +1,77 @@ +{ withSystem, ... }: +{ + flake.modules.nixos.random-alerts = + { + pkgs, + config, + lib, + ... + }: + let + cfg = config.services.random-alerts; + pkg = withSystem pkgs.stdenv.hostPlatform.system ({ config, ... }: config.packages.random-alerts); + in + { + options.services.random-alerts = with lib; { + enable = mkEnableOption (lib.mdDoc "Random Alerts"); + args = { + url = mkOption { + type = types.str; + example = "http://localhost:9093"; + description = ''Alertmanager URL''; + }; + + min-wait-time = mkOption { + type = types.int; + default = 3600; + example = 360; + description = ''Minimum wait time before alert in seconds''; + }; + + max-wait-time = mkOption { + type = types.int; + default = 14400; + example = 6000; + description = ''Maximum wait time before alert in seconds''; + }; + + alert-duration = mkOption { + type = types.int; + default = 3600; + example = 360; + description = ''Time after alerts ends in seconds''; + }; + + log-level = mkOption { + type = types.enum [ + "info" + "trace" + "error" + ]; + default = "info"; + }; + }; + }; + + config = + let + concatMapAttrsStringSep = + sep: f: attrs: + lib.concatStringsSep sep (lib.attrValues (lib.mapAttrs f attrs)); + + args = concatMapAttrsStringSep " " (n: v: "--${n}=${toString v}") cfg.args; + in + lib.mkIf cfg.enable { + systemd.services.random-alerts = { + description = "Random Alerts"; + + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + DynamicUser = lib.mkDefault true; + Restart = lib.mkDefault "on-failure"; + ExecStart = "${lib.getExe pkg} ${args}"; + }; + }; + }; + }; +} diff --git a/packages/default.nix b/packages/default.nix index 41543953..b9981703 100644 --- a/packages/default.nix +++ b/packages/default.nix @@ -57,6 +57,7 @@ { lido-withdrawals-automation = pkgs.callPackage ./lido-withdrawals-automation { }; pyroscope = pkgs.callPackage ./pyroscope { }; + random-alerts = pkgs.callPackage ./random-alerts { }; } // optionalAttrs (system == "x86_64-linux" || system == "aarch64-darwin") { grafana-agent = import ./grafana-agent { inherit inputs'; }; diff --git a/packages/random-alerts/.gitignore b/packages/random-alerts/.gitignore new file mode 100644 index 00000000..378eac25 --- /dev/null +++ b/packages/random-alerts/.gitignore @@ -0,0 +1 @@ +build diff --git a/packages/random-alerts/default.nix b/packages/random-alerts/default.nix new file mode 100644 index 00000000..c5e55abf --- /dev/null +++ b/packages/random-alerts/default.nix @@ -0,0 +1,14 @@ +{ buildDubPackage, ... }: +buildDubPackage rec { + pname = "random-alerts"; + version = "1.0.0"; + src = ./.; + dubLock = { + dependencies = { }; + }; + installPhase = '' + mkdir -p $out/bin + install -m755 ./build/${pname} $out/bin/${pname} + ''; + meta.mainProgram = pname; +} diff --git a/packages/random-alerts/dub.sdl b/packages/random-alerts/dub.sdl new file mode 100644 index 00000000..1d87f9ef --- /dev/null +++ b/packages/random-alerts/dub.sdl @@ -0,0 +1,3 @@ +name "random-alerts" + +targetPath "build" diff --git a/packages/random-alerts/dub.selections.json b/packages/random-alerts/dub.selections.json new file mode 100644 index 00000000..322586b1 --- /dev/null +++ b/packages/random-alerts/dub.selections.json @@ -0,0 +1,5 @@ +{ + "fileVersion": 1, + "versions": { + } +} diff --git a/packages/random-alerts/src/main.d b/packages/random-alerts/src/main.d new file mode 100755 index 00000000..3f41f241 --- /dev/null +++ b/packages/random-alerts/src/main.d @@ -0,0 +1,164 @@ +import core.thread : Thread; +import std.datetime : Duration, Clock, seconds, TimeOfDay; +import std.format : format; +import std.getopt : getopt, getOptConfig = config; +import std.json : JSONValue, parseJSON, JSONOptions; +import std.logger : infof, errorf, tracef, LogLevel; +import std.random : uniform; +import std.exception : enforce; + +import utils.json : toJSON; + +struct Params +{ + Duration minWaitTime; + Duration maxWaitTime; + Duration alertDuration; + string url; + TimeOfDay startTime; + TimeOfDay endTime; +} + +struct Alert +{ + string[string] labels; + Annotation annotations; + string startsAt; + string endsAt; + string generatorURL; + + struct Annotation + { + string alert_type; + string title; + string summary; + } +} + +int main(string[] args) +{ + LogLevel logLevel = LogLevel.info; + string url; + string startTime = "00:00:00"; + string endTime = "23:59:59"; + uint minWaitTimeInSeconds = 3600; // 1 hour + uint maxWaitTimeInSeconds = 14400; // 4 hours + uint alertDurationInSeconds = 3600; // 1 hour + + try + { + args.getopt( + getOptConfig.required, "url", &url, + "start-time", &startTime, + "end-time", &endTime, + "min-wait-time", &minWaitTimeInSeconds, + "max-wait-time", &maxWaitTimeInSeconds, + "alert-duration", &alertDurationInSeconds, + "log-level", &logLevel, + ); + + enforce(minWaitTimeInSeconds <= maxWaitTimeInSeconds, "Make sure that `max-wait-time` is greater than `min-wait-time`."); + + setLogLevel(logLevel); + + executeAtRandomIntervals( + Params( + url: url, + startTime: TimeOfDay.fromISOExtString(startTime), + endTime: TimeOfDay.fromISOExtString(endTime), + minWaitTime: minWaitTimeInSeconds.seconds(), + maxWaitTime: maxWaitTimeInSeconds.seconds(), + alertDuration: alertDurationInSeconds.seconds(), + ) + ); + } + catch (Exception e) + { + errorf("Exception: %s", e.message); + return 1; + } + return 0; +} + +auto getRandomDuration(Duration min, Duration max) => + uniform(min.total!"seconds", max.total!"seconds").seconds; + +void executeAtRandomIntervals(Params params) +{ + with(params) while (true) + { + auto currentTime = Clock.currTime(); + auto currentTimeTOD = cast(TimeOfDay)currentTime; + Duration randomDuration = getRandomDuration(minWaitTime, maxWaitTime); + auto randomTime = currentTime + randomDuration; + + tracef("The operating time is: [%s .. %s]", startTime, endTime); + tracef("The next alarm will be activated in %s", randomTime); + + Thread.sleep(randomDuration); // sleep till the request is ready to be posted. + + if (currentTimeTOD >= startTime && currentTimeTOD <= endTime) + { + infof("Posting alert... "); + postAlert(url, alertDuration); + infof("Alert posted successfully."); + } + else + { + infof("This service is outside working hours. The operating time is: [%s .. %s]", startTime, endTime); + } + + Duration remainingTime = maxWaitTime - randomDuration; + + tracef("The current interval's end will be at %s", (currentTime + remainingTime)); + Thread.sleep(remainingTime); // sleep till the cycle is over. + } +} + +void postAlert(string alertManagerEndpoint, Duration alertDuration) +{ + string url = alertManagerEndpoint ~ "/api/v2/alerts"; + + postJson(url, [ + Alert( + startsAt: Clock.currTime.toUTC.toISOExtString(0), + endsAt: (Clock.currTime + alertDuration).toUTC.toISOExtString(0), + generatorURL: "http://localhost:9090", + labels: [ + "alertname": "Random alert", + "severity": "critical", + "environment": "staging", + "job": "test-monitoring", + ], + annotations: Alert.Annotation( + alert_type: "critical", + title: "Write report", + summary: "The alert was triggered at '%s'".format(Clock.currTime.toUTC) + ) + ) + ]); +} + +JSONValue postJson(T)(string url, in T value) +{ + import std.net.curl : HTTP, post, HTTPStatusException; + + auto jsonRequest = value + .toJSON + .toPrettyString(JSONOptions.doNotEscapeSlashes); + + tracef("Sending request to '%s':\n%s", url, jsonRequest); + + auto http = HTTP(); + http.addRequestHeader("Content-Type", "application/json"); + + auto response = post(url, jsonRequest, http); + return response.parseJSON; +} + +void setLogLevel(LogLevel l) +{ + import std.logger : globalLogLevel, sharedLog; + globalLogLevel = l; + (cast()sharedLog()).logLevel = l; +} diff --git a/packages/random-alerts/src/utils/json.d b/packages/random-alerts/src/utils/json.d new file mode 100644 index 00000000..078499d9 --- /dev/null +++ b/packages/random-alerts/src/utils/json.d @@ -0,0 +1,66 @@ +module utils.json; + +import std.traits: isNumeric, isArray, isSomeChar, ForeachType, isBoolean; +import std.json: JSONValue; +import std.conv: to; +import std.string: strip; +import std.range: front; +import std.algorithm: map; +import std.array: array; +import std.datetime: SysTime; +import core.stdc.string: strlen; + +JSONValue toJSON(T)(in T value, bool simplify = false) +{ + static if (is(T == enum)) + { + return JSONValue(value.enumToString); + } + else static if (is(T == bool) || is(T == string) || isSomeChar!T || isNumeric!T) + return JSONValue(value); + else static if ((isArray!T && isSomeChar!(ForeachType!T)) ) { + return JSONValue(value.idup[0..(strlen(value.ptr)-1)]); + } + else static if (isArray!T) + { + if (simplify && value.length == 1) + return value.front.toJSON(simplify); + else if (simplify && isBoolean!(ForeachType!T) ) { + static if (isBoolean!(ForeachType!T)) { + return JSONValue((value.map!(a => a ? '1' : '0').array).to!string); + } + else {assert(0);} + } + else { + JSONValue[] result; + foreach (elem; value) + result ~= elem.toJSON(simplify); + return JSONValue(result); + } + } + else static if (is(T == SysTime)) { + return JSONValue(value.toISOExtString()); + } + else static if (is(T == struct)) + { + JSONValue[string] result; + auto name = ""; + static foreach (idx, field; T.tupleof) + { + name = __traits(identifier, field).strip("_"); + result[name] = value.tupleof[idx].toJSON(simplify); + } + return JSONValue(result); + } + else static if (is(T == K[V], K, V)) + { + JSONValue[string] result; + foreach (key, field; value) + { + result[key] = field.toJSON(simplify); + } + return JSONValue(result); + } + else + static assert(false, "Unsupported type: `" ~ __traits(identifier, T) ~ "`"); +}