Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions modules/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,6 @@
./pyroscope
./folder-size-metrics
./shard-split
./random-alerts
];
}
77 changes: 77 additions & 0 deletions modules/random-alerts/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
{ withSystem, ... }:
{
flake.modules.nixos.random-alerts =
{
pkgs,
config,
lib,
...
}:
let
cfg = config.services.random-alerts;
pkg = withSystem pkgs.stdenv.hostPlatform.system ({ config, ... }: config.packages.random-alerts);
in
{
options.services.random-alerts = with lib; {
enable = mkEnableOption (lib.mdDoc "Random Alerts");
args = {
url = mkOption {
type = types.str;
example = "http://localhost:9093";
description = ''Alertmanager URL'';
};

min-wait-time = mkOption {
type = types.int;
default = 3600;
example = 360;
description = ''Minimum wait time before alert in seconds'';
};

max-wait-time = mkOption {
type = types.int;
default = 14400;
example = 6000;
description = ''Maximum wait time before alert in seconds'';
};

alert-duration = mkOption {
type = types.int;
default = 3600;
example = 360;
description = ''Time after alerts ends in seconds'';
};

log-level = mkOption {
type = types.enum [
"info"
"trace"
"error"
];
default = "info";
};
};
};

config =
let
concatMapAttrsStringSep =
sep: f: attrs:
lib.concatStringsSep sep (lib.attrValues (lib.mapAttrs f attrs));

args = concatMapAttrsStringSep " " (n: v: "--${n}=${toString v}") cfg.args;
in
lib.mkIf cfg.enable {
systemd.services.random-alerts = {
description = "Random Alerts";

wantedBy = [ "multi-user.target" ];
serviceConfig = {
DynamicUser = lib.mkDefault true;
Restart = lib.mkDefault "on-failure";
ExecStart = "${lib.getExe pkg} ${args}";
};
};
};
};
}
1 change: 1 addition & 0 deletions packages/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
{
lido-withdrawals-automation = pkgs.callPackage ./lido-withdrawals-automation { };
pyroscope = pkgs.callPackage ./pyroscope { };
random-alerts = pkgs.callPackage ./random-alerts { };
}
// optionalAttrs (system == "x86_64-linux" || system == "aarch64-darwin") {
grafana-agent = import ./grafana-agent { inherit inputs'; };
Expand Down
1 change: 1 addition & 0 deletions packages/random-alerts/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
build
14 changes: 14 additions & 0 deletions packages/random-alerts/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{ buildDubPackage, ... }:
buildDubPackage rec {
pname = "random-alerts";
version = "1.0.0";
src = ./.;
dubLock = {
dependencies = { };
};
installPhase = ''
mkdir -p $out/bin
install -m755 ./build/${pname} $out/bin/${pname}
'';
meta.mainProgram = pname;
}
3 changes: 3 additions & 0 deletions packages/random-alerts/dub.sdl
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
name "random-alerts"

targetPath "build"
5 changes: 5 additions & 0 deletions packages/random-alerts/dub.selections.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"fileVersion": 1,
"versions": {
}
}
164 changes: 164 additions & 0 deletions packages/random-alerts/src/main.d
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
import core.thread : Thread;
import std.datetime : Duration, Clock, seconds, TimeOfDay;
import std.format : format;
import std.getopt : getopt, getOptConfig = config;
import std.json : JSONValue, parseJSON, JSONOptions;
import std.logger : infof, errorf, tracef, LogLevel;
import std.random : uniform;
import std.exception : enforce;

import utils.json : toJSON;

struct Params
{
Duration minWaitTime;
Duration maxWaitTime;
Duration alertDuration;
string url;
TimeOfDay startTime;
TimeOfDay endTime;
}

struct Alert
{
string[string] labels;
Annotation annotations;
string startsAt;
string endsAt;
string generatorURL;

struct Annotation
{
string alert_type;
string title;
string summary;
}
}

int main(string[] args)
{
LogLevel logLevel = LogLevel.info;
string url;
string startTime = "00:00:00";
string endTime = "23:59:59";
uint minWaitTimeInSeconds = 3600; // 1 hour
uint maxWaitTimeInSeconds = 14400; // 4 hours
uint alertDurationInSeconds = 3600; // 1 hour

try
{
args.getopt(
getOptConfig.required, "url", &url,
"start-time", &startTime,
"end-time", &endTime,
"min-wait-time", &minWaitTimeInSeconds,
"max-wait-time", &maxWaitTimeInSeconds,
"alert-duration", &alertDurationInSeconds,
"log-level", &logLevel,
);

enforce(minWaitTimeInSeconds <= maxWaitTimeInSeconds, "Make sure that `max-wait-time` is greater than `min-wait-time`.");

setLogLevel(logLevel);

executeAtRandomIntervals(
Params(
url: url,
startTime: TimeOfDay.fromISOExtString(startTime),
endTime: TimeOfDay.fromISOExtString(endTime),
minWaitTime: minWaitTimeInSeconds.seconds(),
maxWaitTime: maxWaitTimeInSeconds.seconds(),
alertDuration: alertDurationInSeconds.seconds(),
)
);
}
catch (Exception e)
{
errorf("Exception: %s", e.message);
return 1;
}
return 0;
}

auto getRandomDuration(Duration min, Duration max) =>
uniform(min.total!"seconds", max.total!"seconds").seconds;

void executeAtRandomIntervals(Params params)
{
with(params) while (true)
{
auto currentTime = Clock.currTime();
auto currentTimeTOD = cast(TimeOfDay)currentTime;
Duration randomDuration = getRandomDuration(minWaitTime, maxWaitTime);
auto randomTime = currentTime + randomDuration;

tracef("The operating time is: [%s .. %s]", startTime, endTime);
tracef("The next alarm will be activated in %s", randomTime);

Thread.sleep(randomDuration); // sleep till the request is ready to be posted.

if (currentTimeTOD >= startTime && currentTimeTOD <= endTime)
{
infof("Posting alert... ");
postAlert(url, alertDuration);
infof("Alert posted successfully.");
}
else
{
infof("This service is outside working hours. The operating time is: [%s .. %s]", startTime, endTime);
}

Duration remainingTime = maxWaitTime - randomDuration;

tracef("The current interval's end will be at %s", (currentTime + remainingTime));
Thread.sleep(remainingTime); // sleep till the cycle is over.
}
}

void postAlert(string alertManagerEndpoint, Duration alertDuration)
{
string url = alertManagerEndpoint ~ "/api/v2/alerts";

postJson(url, [
Alert(
startsAt: Clock.currTime.toUTC.toISOExtString(0),
endsAt: (Clock.currTime + alertDuration).toUTC.toISOExtString(0),
generatorURL: "http://localhost:9090",
labels: [
"alertname": "Random alert",
"severity": "critical",
"environment": "staging",
"job": "test-monitoring",
],
annotations: Alert.Annotation(
alert_type: "critical",
title: "Write report",
summary: "The alert was triggered at '%s'".format(Clock.currTime.toUTC)
)
)
]);
}

JSONValue postJson(T)(string url, in T value)
{
import std.net.curl : HTTP, post, HTTPStatusException;

auto jsonRequest = value
.toJSON
.toPrettyString(JSONOptions.doNotEscapeSlashes);

tracef("Sending request to '%s':\n%s", url, jsonRequest);

auto http = HTTP();
http.addRequestHeader("Content-Type", "application/json");

auto response = post(url, jsonRequest, http);
return response.parseJSON;
}

void setLogLevel(LogLevel l)
{
import std.logger : globalLogLevel, sharedLog;
globalLogLevel = l;
(cast()sharedLog()).logLevel = l;
}
66 changes: 66 additions & 0 deletions packages/random-alerts/src/utils/json.d
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
module utils.json;

import std.traits: isNumeric, isArray, isSomeChar, ForeachType, isBoolean;
import std.json: JSONValue;
import std.conv: to;
import std.string: strip;
import std.range: front;
import std.algorithm: map;
import std.array: array;
import std.datetime: SysTime;
import core.stdc.string: strlen;

JSONValue toJSON(T)(in T value, bool simplify = false)
{
static if (is(T == enum))
{
return JSONValue(value.enumToString);
}
else static if (is(T == bool) || is(T == string) || isSomeChar!T || isNumeric!T)
return JSONValue(value);
else static if ((isArray!T && isSomeChar!(ForeachType!T)) ) {
return JSONValue(value.idup[0..(strlen(value.ptr)-1)]);
}
else static if (isArray!T)
{
if (simplify && value.length == 1)
return value.front.toJSON(simplify);
else if (simplify && isBoolean!(ForeachType!T) ) {
static if (isBoolean!(ForeachType!T)) {
return JSONValue((value.map!(a => a ? '1' : '0').array).to!string);
}
else {assert(0);}
}
else {
JSONValue[] result;
foreach (elem; value)
result ~= elem.toJSON(simplify);
return JSONValue(result);
}
}
else static if (is(T == SysTime)) {
return JSONValue(value.toISOExtString());
}
else static if (is(T == struct))
{
JSONValue[string] result;
auto name = "";
static foreach (idx, field; T.tupleof)
{
name = __traits(identifier, field).strip("_");
result[name] = value.tupleof[idx].toJSON(simplify);
}
return JSONValue(result);
}
else static if (is(T == K[V], K, V))
{
JSONValue[string] result;
foreach (key, field; value)
{
result[key] = field.toJSON(simplify);
}
return JSONValue(result);
}
else
static assert(false, "Unsupported type: `" ~ __traits(identifier, T) ~ "`");
}