Cleaned up a lot of stuff, prepared metric aggregation.

Author: andrewlalis

README.md

@@ -5,15 +5,16 @@ Discord bot for recording and analyzing user engagement, retention, and more.
 
 Use this URL to add the bot to your server:
 
-https://discord.com/api/oauth2/authorize?client_id=817842720848347137&permissions=650304&redirect_uri=https%3A%2F%2Fjavadiscord.net%2F&response_type=code&scope=bot%20messages.read
+https://discord.com/api/oauth2/authorize?client_id=817842720848347137&permissions=650432&redirect_uri=https%3A%2F%2Fjavadiscord.net%2F&scope=bot
 
 ### Configuration
-Currently, the application is only configured for the `development` profile. Please therefore use only `development` as the active profile when running the application.
+The application is configured for both a `development` and `production` spring profile.
 
 For security reasons, the following properties must be declared as environment variables available to this program at runtime:
 
 | Environment Variable   | Description                                                  |
 | ---------------------- | ------------------------------------------------------------ |
 | `INSIGHTS_BOT_DB_URL`  | The JDBC URL used to access the data source, usually of the form `jdbc:<db_type>://<ip>:<port>/<db_name>`. |
 | `INSIGHTS_BOT_DB_USER` | The user with which the bot will access the data source.     |
-| `INSIGHTS_BOT_DB_PASS` | The password for the above user.                             |
+| `INSIGHTS_BOT_DB_PASS` | The password for the above user.                             |
+| `INSIGHTS_BOT_TOKEN`   | The Discord Bot token to use for the bot.                    |

src/main/java/net/javadiscord/BotInitializer.java

@@ -7,13 +7,17 @@
 import lombok.extern.slf4j.Slf4j;
 import net.javadiscord.command.CommandRegistry;
 import net.javadiscord.command.HelpCommand;
+import net.javadiscord.command.analytics.CustomQueryCommand;
 import net.javadiscord.command.analytics.JoinCountCommand;
 import net.javadiscord.command.analytics.MessageCountCommand;
 import net.javadiscord.data.GuildEventRecorderService;
 import org.springframework.boot.CommandLineRunner;
 import org.springframework.stereotype.Service;
 
 import java.time.Duration;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 /**
@@ -24,19 +28,27 @@
 @RequiredArgsConstructor
 @Slf4j
 public class BotInitializer implements CommandLineRunner {
+	public static final Set<Long> ADMIN_IDS = new HashSet<>();
+
 	private final GuildEventRecorderService recorderService;
 	private final CommandRegistry commandRegistry;
 
 	// Autowired commands (which require persistence components)
 	private final MessageCountCommand messageCountCommand;
 	private final JoinCountCommand joinCountCommand;
-	//private final CustomQueryCommand customQueryCommand;
+	private final CustomQueryCommand customQueryCommand;
 
 	@Override
 	public void run(String... args) {
-		if (args.length < 1 || args[0].isEmpty()) throw new IllegalArgumentException("Missing client token argument.");
+		String token = System.getenv("INSIGHTS_BOT_TOKEN");
+		if (token == null || token.trim().isEmpty()) throw new IllegalArgumentException("Missing client token argument.");
+		ADMIN_IDS.addAll(Arrays.stream(System.getenv("INSIGHTS_BOT_ADMINS").split("\\s*,\\s*")).map(Long::parseLong).collect(Collectors.toSet()));
+		if (!ADMIN_IDS.isEmpty()) {
+			log.info("Started with admin ids: {}", ADMIN_IDS.toString());
+		}
 		this.initializeCommands();
-		this.initializeBot(args[0]);
+		log.info("Initialized commands.");
+		this.initializeBot(token);
 	}
 
 	/**
@@ -46,7 +58,7 @@ private void initializeCommands() {
 		this.commandRegistry.register("help", new HelpCommand());
 		this.commandRegistry.register("messageCount", this.messageCountCommand);
 		this.commandRegistry.register("joinCount", this.joinCountCommand);
-		//this.commandRegistry.register("customQuery", this.customQueryCommand);
+		this.commandRegistry.register("customQuery", this.customQueryCommand);
 	}
 
 	/**

src/main/java/net/javadiscord/command/Command.java

@@ -2,6 +2,9 @@
 
 import discord4j.core.event.domain.message.MessageCreateEvent;
 import org.reactivestreams.Publisher;
+import org.springframework.lang.Nullable;
+
+import java.util.Set;
 
 /**
  * Commands must simply declare the logic for the {@link Command#handle} method,
@@ -16,4 +19,11 @@ public interface Command {
 	 * handled successfully.
 	 */
 	Publisher<?> handle(MessageCreateEvent event, String[] args);
+
+	/**
+	 * @return A list of ids of users that are allowed to use the command.
+	 */
+	default @Nullable Set<Long> getWhitelistedUserIds() {
+		return null;
+	}
 }

src/main/java/net/javadiscord/command/CommandHandler.java

@@ -1,11 +1,15 @@
 package net.javadiscord.command;
 
 import discord4j.core.event.domain.message.MessageCreateEvent;
+import discord4j.core.object.entity.User;
 import lombok.RequiredArgsConstructor;
 import org.reactivestreams.Publisher;
 import org.springframework.stereotype.Component;
 import reactor.core.publisher.Mono;
 
+import java.util.Arrays;
+import java.util.Optional;
+
 /**
  * Handles the process of preparing the contents of a command message for use
  * with {@link Command}s, by splitting the content by whitespace and extracting
@@ -28,9 +32,7 @@ public Publisher<?> handle(MessageCreateEvent event) {
 		if (words.length == 0) {
 			return Mono.empty();
 		}
-		String[] args = new String[words.length - 1];
-		System.arraycopy(words, 1, args, 0, words.length - 1);
-		return this.handle(event, args);
+		return this.handle(event, Arrays.copyOfRange(words, 1, words.length));
 	}
 
 	/**
@@ -43,9 +45,14 @@ public Publisher<?> handle(MessageCreateEvent event) {
 	public Publisher<?> handle(MessageCreateEvent event, String[] words) {
 		if (words.length > 0) {
 			String keyword = words[0].trim().toLowerCase();
-			String[] args = new String[words.length - 1];
-			System.arraycopy(words, 1, args, 0, words.length - 1);
-			return this.commandRegistry.get(keyword).orElse(unknownCommand).handle(event, args);
+			Command cmd = this.commandRegistry.get(keyword).orElse(unknownCommand);
+			if (cmd.getWhitelistedUserIds() != null && !cmd.getWhitelistedUserIds().isEmpty()) {
+				Optional<User> optionalUser = event.getMessage().getAuthor();
+				if (!optionalUser.isPresent() || !cmd.getWhitelistedUserIds().contains(optionalUser.get().getId().asLong())) {
+					return event.getMessage().getChannel().flatMap(c -> c.createMessage("Command not permitted."));
+				}
+			}
+			return cmd.handle(event, Arrays.copyOfRange(words, 1, words.length));
 		}
 		return Mono.empty();
 	}

src/main/java/net/javadiscord/command/analytics/CustomQueryCommand.java

@@ -3,12 +3,14 @@
 import discord4j.core.event.domain.message.MessageCreateEvent;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import net.javadiscord.BotInitializer;
 import net.javadiscord.command.Command;
 import org.reactivestreams.Publisher;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Component;
 
 import java.util.Arrays;
+import java.util.Set;
 
 @Component
 @RequiredArgsConstructor
@@ -49,4 +51,9 @@ public Publisher<?> handle(MessageCreateEvent event, String[] args) {
 		String finalMsg = msg;
 		return event.getMessage().getChannel().flatMap(c -> c.createMessage(finalMsg));
 	}
+
+	@Override
+	public Set<Long> getWhitelistedUserIds() {
+		return BotInitializer.ADMIN_IDS;
+	}
 }

src/main/java/net/javadiscord/data/GuildEventRecorderService.java

@@ -1,8 +1,10 @@
 package net.javadiscord.data;
 
 import discord4j.core.event.ReactiveEventAdapter;
+import discord4j.core.event.domain.guild.BanEvent;
 import discord4j.core.event.domain.guild.MemberJoinEvent;
 import discord4j.core.event.domain.guild.MemberLeaveEvent;
+import discord4j.core.event.domain.guild.UnbanEvent;
 import discord4j.core.event.domain.message.*;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -28,6 +30,7 @@ public class GuildEventRecorderService extends ReactiveEventAdapter {
 	private final GuildEventRepository guildEventRepository;
 	private final CommandHandler commandHandler;
 
+	// ---- MESSAGE EVENTS ----
 	@Override
 	public Publisher<?> onMessageCreate(MessageCreateEvent event) {
 		if (event.getGuildId().isPresent() && event.getMember().isPresent()) {
@@ -72,6 +75,7 @@ public Publisher<?> onReactionRemove(ReactionRemoveEvent event) {
 		return Mono.empty();
 	}
 
+	// ---- MEMBERSHIP EVENTS
 	@Override
 	public Publisher<?> onMemberJoin(MemberJoinEvent event) {
 		return Mono.just(this.guildEventRepository.save(new MembershipEvent(event)));
@@ -81,4 +85,14 @@ public Publisher<?> onMemberJoin(MemberJoinEvent event) {
 	public Publisher<?> onMemberLeave(MemberLeaveEvent event) {
 		return Mono.just(this.guildEventRepository.save(new MembershipEvent(event)));
 	}
+
+	@Override
+	public Publisher<?> onBan(BanEvent event) {
+		return Mono.just(this.guildEventRepository.save(new MembershipEvent(event)));
+	}
+
+	@Override
+	public Publisher<?> onUnban(UnbanEvent event) {
+		return Mono.just(this.guildEventRepository.save(new MembershipEvent(event)));
+	}
 }

src/main/java/net/javadiscord/data/aggregation/DailyAggregateGenerator.java

@@ -0,0 +1,61 @@
+package net.javadiscord.data.aggregation;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import net.javadiscord.data.dao.MessagesMetricRepository;
+import net.javadiscord.util.SqlHelper;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.sql.Timestamp;
+import java.time.*;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.Objects;
+
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class DailyAggregateGenerator {
+	private final JdbcTemplate jdbcTemplate;
+	private final TimeIntervalAggregationService intervalAggregationService;
+	private final MessagesMetricRepository messagesMetricRepository;
+
+	/**
+	 * Scheduled task that generates aggregate data for the previous day.
+	 */
+	@Scheduled(cron = "${insights-bot.tasks.daily-aggregate-generator}")
+	@Transactional
+	public void generateDayAggregates() {
+		LocalDateTime now = LocalDateTime.now(ZoneId.of("UTC"));
+		LocalDate yesterday = now.toLocalDate().minusDays(1);
+		log.info("Generating daily aggregate data for {}.", yesterday.format(DateTimeFormatter.ISO_LOCAL_DATE));
+		Instant start = yesterday.atStartOfDay().toInstant(ZoneOffset.UTC);
+		Instant end = yesterday.atTime(LocalTime.MAX).toInstant(ZoneOffset.UTC);
+		log.info("Start: {}", start.toString());
+		log.info("End: {}", end.toString());
+		long startMillis = System.currentTimeMillis();
+		List<Long> guildIds = this.jdbcTemplate.query(
+				Objects.requireNonNull(SqlHelper.load("sql/find_guild_ids.sql")),
+				(resultSet, i) -> resultSet.getLong(1),
+				Timestamp.from(start), Timestamp.from(end)
+		);
+		for (Long guildId : guildIds) {
+			log.info("Generating data for guild {}:", guildId);
+			if (this.messagesMetricRepository.existsByGuildIdEqualsAndStartTimestampEqualsAndEndTimestampEquals(guildId, start, end)) {
+				log.info("\tGuild already has messages metric for this day, skipping.");
+			} else {
+				this.intervalAggregationService.generateMessagesMetric(guildId, start, end);
+				log.info("\tGenerated messages metric.");
+			}
+		}
+		double runtimeSeconds = (System.currentTimeMillis() - startMillis) / 1000.0;
+		log.info(
+				"Daily aggregate data generation for {} completed in {} seconds.",
+				yesterday.format(DateTimeFormatter.ISO_LOCAL_DATE),
+				String.format("%.3f", runtimeSeconds)
+		);
+	}
+}

src/main/java/net/javadiscord/data/aggregation/TimeIntervalAggregationService.java

@@ -0,0 +1,45 @@
+package net.javadiscord.data.aggregation;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import net.javadiscord.data.dao.MetricRepository;
+import net.javadiscord.data.model.stats.time_interval_metrics.MessagesMetric;
+import net.javadiscord.util.SqlHelper;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.sql.Timestamp;
+import java.time.Instant;
+
+import static net.javadiscord.util.SqlHelper.wrapStr;
+
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class TimeIntervalAggregationService {
+	private final MetricRepository metricRepository;
+	private final JdbcTemplate jdbcTemplate;
+
+	@Transactional
+	public MessagesMetric generateMessagesMetric(long guildId, Instant start, Instant end) {
+		return this.jdbcTemplate.query(connection -> SqlHelper.loadMulti(
+				connection,
+				"sql/messages_metric.sql",
+				wrapStr(Timestamp.from(start).toString()),
+				wrapStr(Timestamp.from(end).toString()),
+				String.valueOf(guildId)
+		), r -> {
+			if (!r.next()) return null;
+			MessagesMetric mm = new MessagesMetric(guildId, start, end);
+			mm.setMessagesCreated(r.getLong("messages_created"));
+			mm.setMessagesDeleted(r.getLong("messages_deleted"));
+			mm.setMessagesUpdated(r.getLong("messages_updated"));
+			mm.setMessagesRetained(r.getLong("messages_retained"));
+			mm.setReactionsAdded(r.getLong("reactions_added"));
+			mm.setReactionsRemoved(r.getLong("reactions_removed"));
+			mm.setActiveUsers(r.getLong("active_users"));
+			return this.metricRepository.save(mm);
+		});
+	}
+}

src/main/java/net/javadiscord/data/dao/GuildEventRepository.java

@@ -1,6 +1,6 @@
 package net.javadiscord.data.dao;
 
-import net.javadiscord.data.model.GuildEvent;
+import net.javadiscord.data.model.events.GuildEvent;
 import org.springframework.stereotype.Repository;
 
 @Repository

src/main/java/net/javadiscord/data/dao/MessagesMetricRepository.java

@@ -0,0 +1,15 @@
+package net.javadiscord.data.dao;
+
+import net.javadiscord.data.model.stats.time_interval_metrics.MessagesMetric;
+import org.springframework.stereotype.Repository;
+
+import java.time.Instant;
+
+@Repository
+public interface MessagesMetricRepository extends BaseEntityRepository<MessagesMetric> {
+	boolean existsByGuildIdEqualsAndStartTimestampEqualsAndEndTimestampEquals(
+			long guildId,
+			Instant start,
+			Instant end
+	);
+}