Skip to content

Commit b0e9ef9

Browse files
duncte123duncte123
authored
Migrate to PostgreSQL (#284)
* Get started on implementing mysql instead of using the REST api * Use postresql instead & yeet custom command interface * Install liquibase * Add changelog file * Fill out with full guild setting table * Add custom commands to changelog * Add config support for psql * Bot can now boot from psql * Add a few more guild setting methods * re-use connection for same operations * Add purging of guild settings * Add updating of guild settings * Load patrons * Linter * Aa * When you duplicate the same entry causing you to rewrite everything * Creating of custom commands * Deleting custom commands * Lint * Updating custom commands * Adding to and clearing the blacklist * Finish patron system * Creating of temp bans * Insert warnings and mutes * Fetching warnings * Code for purging bans and mutes * Ban bypasses * Vc autorole * Make vc autoroles work * Tags :) * Inserting of reminders * Listing of reminders * SHowing and listing reminders * Add setting warn actions * Change all return types to futures * Convert postgres database engine * Convert web database engine * FUck sqlite * FIx codebase * CLeanup * Lint code a bit more * Rename adapter to database * Better errors * Don't store stuff tha's not needed * Add timers for psql * lint
1 parent 6e633fb commit b0e9ef9

File tree

66 files changed

+2440
-2632
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+2440
-2632
lines changed

‎.idea/sqldialects.xml‎

Lines changed: 1 addition & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎build.gradle.kts‎

Lines changed: 7 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,11 @@ plugins {
3030
application
3131

3232
kotlin("jvm") version "1.6.0"
33+
id("org.liquibase.gradle") version "2.0.4"
34+
// id("org.jmailen.kotlinter") version "3.9.0" // removes star imports :(
35+
id("org.jmailen.kotlinter") version "3.6.0"
3336
id("com.github.johnrengelman.shadow") version "7.1.2"
3437
id("com.github.breadmoirai.github-release") version "2.2.12"
35-
id("org.jmailen.kotlinter") version "3.5.1"
3638
}
3739

3840
val numberVersion = "3.106.8"
@@ -60,11 +62,6 @@ repositories {
6062
maven("https://jitpack.io")
6163
}
6264

63-
val devDependencies = arrayOf(
64-
// SQLite
65-
DependencyInfo(group = "org.xerial", name = "sqlite-jdbc", version = "3.32.3")
66-
)
67-
6865
dependencies {
6966
implementation(group = "com.dunctebot", name = "dunctebot-models", version = "0.1.22")
7067

@@ -106,8 +103,6 @@ dependencies {
106103

107104
// kotlin
108105
implementation(kotlin("stdlib-jdk8"))
109-
// implementation(kotlin("kotlinx-coroutines-core"))
110-
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.0")
111106

112107
// JDA utils
113108
implementation(group = "com.github.JDA-Applications", name = "JDA-Utilities", version = "804d58a") {
@@ -153,10 +148,10 @@ dependencies {
153148
// redis
154149
implementation(group = "redis.clients", name = "jedis", version = "3.7.0")
155150

156-
// dev deps
157-
devDependencies.forEach {
158-
implementation(group = it.group, name = it.name, version = it.version)
159-
}
151+
implementation(group = "com.zaxxer", name = "HikariCP", version = "5.0.0")
152+
implementation(group = "com.impossibl.pgjdbc-ng", name = "pgjdbc-ng", version = "0.8.9")
153+
implementation(group = "org.liquibase", name = "liquibase-core", version = "4.5.0")
154+
runtimeOnly(group = "com.mattbertolini", name = "liquibase-slf4j", version = "4.0.0")
160155
}
161156

162157
val compileKotlin: KotlinCompile by tasks
@@ -250,32 +245,12 @@ compileJava.apply {
250245
dependsOn(generateJavaSources)
251246
}
252247

253-
jar.apply {
254-
exclude(
255-
"**/SQLiteDatabaseConnectionManager.class",
256-
"**/AudioPlayerSenderHandler.class",
257-
"**/SqliteDatabaseAdapter**"
258-
)
259-
}
260-
261248
application {
262249
mainClass.set("ml.duncte123.skybot.SkyBot")
263250
}
264251

265252
shadowJar.apply {
266253
archiveClassifier.set("prod")
267-
268-
exclude(
269-
"**/SQLiteDatabaseConnectionManager.class",
270-
"**/SQLiteTimers.class",
271-
"**/SqliteDatabaseAdapter**"
272-
)
273-
274-
dependencies {
275-
devDependencies.forEach {
276-
exclude(dependency("${it.group}:${it.name}:${it.version}"))
277-
}
278-
}
279254
}
280255

281256
tasks.withType<Wrapper> {

‎docker-compose.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,6 @@ services:
4444
- WEBSOCKET_URL=
4545
- WEBSOCKET_ENABLE=false
4646

47-
# And finally
48-
- USE_DATABASE=false
47+
# Options are: psql, web
48+
- USE_DATABASE=psql
4949
- REDIS_HOST=localhost

‎gradle.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,6 @@
1717
#
1818

1919
#org.gradle.jvmargs=-Xmx1024M -XX:MaxPermSize=512m -Dkotlin.compiler.execution.strategy=�in-process� -Dkotlin.daemon.jvm.options=-Xmx512M
20-
org.gradle.jvmargs=-Xmx1g -XX:MaxPermSize=512m -Dkotlin.compiler.execution.strategy=�in-process�
20+
#org.gradle.jvmargs=-Xmx1g -XX:MaxPermSize=512m -Dkotlin.compiler.execution.strategy=�in-process�
2121

2222
kotlin.code.style=official

‎src/main/java/ml/duncte123/skybot/CommandManager.java‎

Lines changed: 28 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import com.jagrosh.jagtag.Parser;
2222
import gnu.trove.map.TObjectLongMap;
2323
import io.sentry.Sentry;
24-
import kotlin.Triple;
2524
import me.duncte123.botcommons.messaging.MessageConfig;
2625
import ml.duncte123.skybot.commands.admin.BlackListCommand;
2726
import ml.duncte123.skybot.commands.admin.VcAutoRoleCommand;
@@ -57,10 +56,7 @@
5756
import ml.duncte123.skybot.commands.utils.EnlargeCommand;
5857
import ml.duncte123.skybot.commands.utils.RoleInfoCommand;
5958
import ml.duncte123.skybot.commands.weeb.*;
60-
import ml.duncte123.skybot.objects.command.CommandCategory;
61-
import ml.duncte123.skybot.objects.command.CommandContext;
62-
import ml.duncte123.skybot.objects.command.ICommand;
63-
import ml.duncte123.skybot.objects.command.custom.CustomCommand;
59+
import ml.duncte123.skybot.objects.command.*;
6460
import ml.duncte123.skybot.objects.pairs.LongLongPair;
6561
import ml.duncte123.skybot.utils.CommandUtils;
6662
import ml.duncte123.skybot.utils.MapUtils;
@@ -562,59 +558,54 @@ private void runCustomCommand(ICommand cmd, String invoke, List<String> args, Gu
562558
}
563559

564560
private void loadCustomCommands() {
565-
this.variables.getDatabaseAdapter().getCustomCommands(
566-
(loadedCommands) -> {
567-
loadedCommands.forEach(
568-
(command) -> addCustomCommand(command, false, false)
569-
);
570-
571-
return null;
572-
}
573-
);
561+
this.variables.getDatabase().getCustomCommands().thenAccept((loadedCommands) -> {
562+
loadedCommands.forEach(
563+
(command) -> addCustomCommand(command, false, false)
564+
);
565+
});
574566
}
575567

576568
public boolean editCustomCommand(CustomCommand cmd) {
577-
return addCustomCommand(cmd, true, true).getFirst();
569+
return addCustomCommand(cmd, true, true) == CommandResult.SUCCESS;
578570
}
579571

580-
public Triple<Boolean, Boolean, Boolean> registerCustomCommand(CustomCommand cmd) {
572+
public CommandResult registerCustomCommand(CustomCommand cmd) {
581573
return addCustomCommand(cmd, true, false);
582574
}
583575

584-
public Triple<Boolean, Boolean, Boolean> addCustomCommand(CustomCommand command, boolean insertInDb, boolean isEdit) {
576+
public CommandResult addCustomCommand(CustomCommand command, boolean insertInDb, boolean isEdit) {
585577
if (command.getName().contains(" ")) {
586578
throw new IllegalArgumentException("Name can't have spaces!");
587579
}
588580

589-
final boolean commandFound = this.customCommands.stream()
590-
.anyMatch((cmd) -> cmd.getName().equalsIgnoreCase(command.getName()) && cmd.getGuildId() == command.getGuildId()) && !isEdit;
591-
final boolean limitReached = this.customCommands.stream().filter((cmd) -> cmd.getGuildId() == command.getGuildId()).count() >= 50 && !isEdit;
581+
final boolean commandFound = !isEdit && this.customCommands.stream()
582+
.anyMatch((cmd) -> cmd.getName().equalsIgnoreCase(command.getName()) && cmd.getGuildId() == command.getGuildId());
592583

593-
if (commandFound || limitReached) {
594-
return new Triple<>(false, commandFound, limitReached);
584+
if (commandFound) {
585+
return CommandResult.COMMAND_EXISTS;
586+
}
587+
588+
final boolean limitReached = !isEdit && this.customCommands.stream().filter((cmd) -> cmd.getGuildId() == command.getGuildId()).count() >= 50;
589+
590+
if (limitReached) {
591+
return CommandResult.LIMIT_REACHED;
595592
}
596593

597594
if (insertInDb) {
598595
try {
599-
final CompletableFuture<Triple<Boolean, Boolean, Boolean>> future = new CompletableFuture<>();
596+
final CompletableFuture<CommandResult> future;
600597

601598
if (isEdit) {
602-
this.variables.getDatabaseAdapter()
603-
.updateCustomCommand(command.getGuildId(), command.getName(), command.getMessage(), command.isAutoResponse(), (triple) -> {
604-
future.complete(triple);
605-
return null;
606-
});
599+
future = this.variables.getDatabase()
600+
.updateCustomCommand(command.getGuildId(), command.getName(), command.getMessage(), command.isAutoResponse());
607601
} else {
608-
this.variables.getDatabaseAdapter()
609-
.createCustomCommand(command.getGuildId(), command.getName(), command.getMessage(), (triple) -> {
610-
future.complete(triple);
611-
return null;
612-
});
602+
future = this.variables.getDatabase()
603+
.createCustomCommand(command.getGuildId(), command.getName(), command.getMessage());
613604
}
614605

615-
final Triple<Boolean, Boolean, Boolean> res = future.get();
606+
final CommandResult res = future.get();
616607

617-
if (res != null && !res.getFirst()) {
608+
if (res != null && res != CommandResult.SUCCESS) {
618609
return res;
619610
}
620611
}
@@ -630,7 +621,7 @@ public Triple<Boolean, Boolean, Boolean> addCustomCommand(CustomCommand command,
630621

631622
this.customCommands.add(command);
632623

633-
return new Triple<>(true, false, false);
624+
return CommandResult.SUCCESS;
634625
}
635626

636627
public boolean removeCustomCommand(String name, long guildId) {
@@ -650,9 +641,7 @@ public boolean removeCustomCommand(String name, long guildId, boolean updateDB)
650641
}
651642

652643
try {
653-
final CompletableFuture<Boolean> future = new CompletableFuture<>();
654-
this.variables.getDatabaseAdapter().deleteCustomCommand(guildId, name, future::complete);
655-
644+
final CompletableFuture<Boolean> future = this.variables.getDatabase().deleteCustomCommand(guildId, name);
656645
final boolean result = future.get();
657646

658647
if (result) {

‎src/main/java/ml/duncte123/skybot/SkyBot.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import fredboat.audio.player.LavalinkManager;
2222
import me.duncte123.botcommons.messaging.EmbedUtils;
2323
import me.duncte123.botcommons.messaging.MessageUtils;
24-
import me.duncte123.botcommons.text.TextColor;
2524
import me.duncte123.botcommons.web.WebUtils;
2625
import ml.duncte123.skybot.objects.config.DunctebotConfig;
2726
import ml.duncte123.skybot.objects.pairs.LongLongPair;
@@ -70,16 +69,20 @@ private SkyBot() throws LoginException {
7069
this.configureDefaults(variables);
7170

7271
final DunctebotConfig config = variables.getConfig();
73-
final CommandManager commandManager = variables.getCommandManager();
7472
final Logger logger = LoggerFactory.getLogger(SkyBot.class);
7573

7674
Settings.PREFIX = config.discord.prefix;
7775

78-
if (variables.useApi()) {
79-
logger.info(TextColor.GREEN + "Using api for all connections" + TextColor.RESET);
76+
final String useDatabase = variables.getConfig().useDatabase;
77+
78+
if ("psql".equals(useDatabase)) {
79+
logger.info("Using PostgreSQL as database impl");
80+
} else if ("web".equals(useDatabase)) {
81+
logger.warn("Using web api for all connections, please migrate to PostgreSQL");
8082
} else {
81-
logger.warn("Using SQLite as the database");
82-
logger.warn("Please note that it is not recommended for production");
83+
shardManager = null; // for compiling n' stuff
84+
logger.error("Unknown database driver \"{}\", exiting!", useDatabase);
85+
return;
8386
}
8487

8588
//Load the settings before loading the bot
@@ -89,6 +92,7 @@ private SkyBot() throws LoginException {
8992
final String token = config.discord.token;
9093
//But this time we are going to shard it
9194
final int totalShards = config.discord.totalShards;
95+
final CommandManager commandManager = variables.getCommandManager();
9296
final LongLongPair commandCount = commandManager.getCommandCount();
9397

9498
logger.info("{} commands with {} aliases loaded.", commandCount.getFirst(), commandCount.getSecond());

‎src/main/java/ml/duncte123/skybot/Variables.java‎

Lines changed: 20 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,15 @@
2323
import com.fasterxml.jackson.databind.DeserializationFeature;
2424
import com.fasterxml.jackson.databind.json.JsonMapper;
2525
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
26-
import com.sedmelluq.discord.lavaplayer.tools.ExceptionTools;
2726
import gnu.trove.map.TLongLongMap;
2827
import gnu.trove.map.TLongObjectMap;
2928
import io.sentry.Sentry;
3029
import me.duncte123.weebJava.WeebApiBuilder;
3130
import me.duncte123.weebJava.models.WeebApi;
3231
import me.duncte123.weebJava.types.TokenType;
33-
import ml.duncte123.skybot.adapters.DatabaseAdapter;
34-
import ml.duncte123.skybot.adapters.WebDatabaseAdapter;
32+
import ml.duncte123.skybot.database.AbstractDatabase;
33+
import ml.duncte123.skybot.database.PostgreDatabase;
34+
import ml.duncte123.skybot.database.WebDatabase;
3535
import ml.duncte123.skybot.objects.DBMap;
3636
import ml.duncte123.skybot.objects.api.DuncteApis;
3737
import ml.duncte123.skybot.objects.apis.BlargBot;
@@ -43,10 +43,11 @@
4343
import net.jodah.expiringmap.ExpirationPolicy;
4444
import net.jodah.expiringmap.ExpiringMap;
4545
import net.notfab.caching.client.CacheClient;
46-
import org.slf4j.LoggerFactory;
4746

48-
import java.lang.reflect.InvocationTargetException;
49-
import java.util.concurrent.*;
47+
import java.util.concurrent.ExecutionException;
48+
import java.util.concurrent.Executors;
49+
import java.util.concurrent.TimeUnit;
50+
import java.util.concurrent.TimeoutException;
5051

5152
public final class Variables {
5253
private final JsonMapper mapper = JsonMapper.builder()
@@ -67,20 +68,16 @@ public final class Variables {
6768
private final WeebApi weebApi;
6869
private final DunctebotConfig config;
6970
private final CacheClient youtubeCache;
70-
private DatabaseAdapter databaseAdapter;
71+
private AbstractDatabase database;
7172
@SuppressWarnings("PMD.UseConcurrentHashMap")
7273
private final DBMap<Long, GuildSetting> guildSettingsCache = new DBMap<>(ExpiringMap.builder()
7374
.expirationPolicy(ExpirationPolicy.ACCESSED)
7475
.expiration(12, TimeUnit.HOURS)
7576
.entryLoader((EntryLoader<Long, GuildSetting>) guildId -> {
76-
final CompletableFuture<GuildSetting> future = new CompletableFuture<>();
77-
getDatabaseAdapter().loadGuildSetting(guildId, (setting) -> {
78-
future.complete(setting);
79-
return null;
80-
});
81-
8277
try {
83-
return future.get(20L, TimeUnit.SECONDS);
78+
return getDatabase()
79+
.loadGuildSetting(guildId)
80+
.get(20L, TimeUnit.SECONDS);
8481
} catch (ExecutionException | TimeoutException | InterruptedException e) {
8582
return null;
8683
}
@@ -162,10 +159,6 @@ public WeebApi getWeebApi() {
162159
return this.weebApi;
163160
}
164161

165-
public boolean useApi() {
166-
return this.config.useDatabase;
167-
}
168-
169162
public Alexflipnote getAlexflipnote() {
170163
return this.alexflipnote;
171164
}
@@ -182,23 +175,17 @@ public DuncteApis getApis() {
182175
return this.apis;
183176
}
184177

185-
public DatabaseAdapter getDatabaseAdapter() {
186-
try {
187-
if (this.databaseAdapter == null) {
188-
this.databaseAdapter = this.useApi() ?
189-
new WebDatabaseAdapter(this.getApis(), this.getJackson()) :
190-
(DatabaseAdapter) (Class.forName("ml.duncte123.skybot.adapters.SqliteDatabaseAdapter")
191-
.getDeclaredConstructor().newInstance());
178+
public AbstractDatabase getDatabase() {
179+
if (this.database == null) {
180+
if ("psql".equals(this.config.useDatabase)) {
181+
this.database = new PostgreDatabase();
182+
} else if ("web".equals(this.config.useDatabase)) {
183+
this.database = new WebDatabase(this.getApis(), this.getJackson());
184+
} else {
185+
throw new IllegalArgumentException("Unknown database engine: " + this.config.useDatabase);
192186
}
193187
}
194-
catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException |
195-
InstantiationException | InvocationTargetException e) {
196-
LoggerFactory.getLogger(Variables.class).error("Could not load database class.\n" +
197-
"Are you a developer?", e);
198-
199-
throw ExceptionTools.wrapUnfriendlyExceptions(e);
200-
}
201188

202-
return this.databaseAdapter;
189+
return this.database;
203190
}
204191
}

0 commit comments

Comments
 (0)