Skip to content

Commit 4a1d29a

Browse files
BenCodezBenCodez
committed
Add streaming iteration methods for user data across storage types
1 parent fab3850 commit 4a1d29a

File tree

3 files changed

+281
-5
lines changed

3 files changed

+281
-5
lines changed

‎AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/UserManager.java‎

Lines changed: 73 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
import java.util.List;
77
import java.util.Map.Entry;
88
import java.util.UUID;
9+
import java.util.function.BiConsumer;
10+
import java.util.function.Consumer;
911

1012
import org.bukkit.OfflinePlayer;
1113
import org.bukkit.entity.Player;
@@ -132,6 +134,72 @@ public ArrayList<String> getAllPlayerNames() {
132134

133135
}
134136

137+
/**
138+
* Storage-agnostic streaming iteration over all users + their column data.
139+
*
140+
* MYSQL: uses plugin.getMysql().forEachUser(...) SQLITE: uses
141+
* plugin.getSQLiteUserTable().forEachUser(...) FLAT: iterates UUID files and
142+
* builds columns per user (no giant map).
143+
*
144+
* @param perUser called for each user with (uuid, columns)
145+
* @param onFinished called once at the end with the number of users processed
146+
*/
147+
public void forEachUserKeys(BiConsumer<UUID, ArrayList<Column>> perUser, Consumer<Integer> onFinished) {
148+
UserStorage storage = plugin.getStorageType();
149+
150+
if (storage == UserStorage.MYSQL) {
151+
plugin.getMysql().forEachUser((uuid, cols) -> {
152+
perUser.accept(uuid, cols);
153+
}, (count) -> {
154+
if (onFinished != null)
155+
onFinished.accept(count);
156+
});
157+
return;
158+
}
159+
160+
if (storage == UserStorage.SQLITE) {
161+
plugin.getSQLiteUserTable().forEachUser((uuid, cols) -> {
162+
perUser.accept(uuid, cols);
163+
}, (count) -> {
164+
if (onFinished != null)
165+
onFinished.accept(count);
166+
});
167+
return;
168+
}
169+
170+
// FLAT fallback (stream-like; no giant HashMap)
171+
int processed = 0;
172+
try {
173+
for (String uuidStr : getAllUUIDs(UserStorage.FLAT)) {
174+
if (uuidStr == null || uuidStr.isEmpty() || "null".equalsIgnoreCase(uuidStr)) {
175+
continue;
176+
}
177+
178+
UUID uuid;
179+
try {
180+
uuid = UUID.fromString(uuidStr);
181+
} catch (IllegalArgumentException ignored) {
182+
continue;
183+
}
184+
185+
AdvancedCoreUser user = getUser(uuid);
186+
user.dontCache();
187+
188+
// Build per-user columns from current values
189+
ArrayList<Column> colList = new ArrayList<>();
190+
for (Entry<String, DataValue> entry : user.getData().getValues().entrySet()) {
191+
colList.add(new Column(entry.getKey(), entry.getValue()));
192+
}
193+
194+
processed++;
195+
perUser.accept(uuid, colList);
196+
}
197+
} finally {
198+
if (onFinished != null)
199+
onFinished.accept(processed);
200+
}
201+
}
202+
135203
public ArrayList<String> getAllUUIDs() {
136204
return ArrayUtils.removeDuplicates(getAllUUIDs(plugin.getStorageType()));
137205
}
@@ -214,11 +282,11 @@ public String getProperName(String name) {
214282
return s;
215283
}
216284
}
217-
218-
for (String s : getAllPlayerNames()) {
219-
if (s.equalsIgnoreCase(name)) {
220-
return s;
221-
}
285+
286+
for (String s : getAllPlayerNames()) {
287+
if (s.equalsIgnoreCase(name)) {
288+
return s;
289+
}
222290
}
223291
return name;
224292
}

‎AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/userstorage/mysql/MySQL.java‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
import java.util.Set;
1212
import java.util.UUID;
1313
import java.util.concurrent.ConcurrentHashMap;
14+
import java.util.function.BiConsumer;
15+
import java.util.function.Consumer;
1416

1517
import org.bukkit.configuration.ConfigurationSection;
1618

@@ -174,6 +176,121 @@ public boolean containsUUID(String uuid) {
174176
return uuids.contains(uuid);
175177
}
176178

179+
/**
180+
* Streams every user row from the table and calls the consumer for each UUID +
181+
* column list. This avoids building a giant in-memory map like
182+
* {@link #getAllQuery()}.
183+
*
184+
* @param perUser called for each row (uuid, columns)
185+
* @param onFinished called once at the end with the number of rows processed
186+
* (even if 0)
187+
*/
188+
public void forEachUser(BiConsumer<UUID, ArrayList<Column>> perUser, Consumer<Integer> onFinished) {
189+
int processed = 0;
190+
191+
// NOTE: Selecting * means we still read every column, but we DON'T store all
192+
// rows at once.
193+
final String sqlStr = "SELECT * FROM " + qi(tableName) + ";";
194+
plugin.devDebug("DB QUERY: " + sqlStr);
195+
196+
try (Connection conn = mysql.getConnectionManager().getConnection();
197+
PreparedStatement ps = conn.prepareStatement(sqlStr, ResultSet.TYPE_FORWARD_ONLY,
198+
ResultSet.CONCUR_READ_ONLY)) {
199+
200+
// Streaming hints:
201+
// - MySQL: Integer.MIN_VALUE triggers row-by-row streaming for some drivers
202+
// - Postgres: a positive fetch size enables cursor-based fetching
203+
try {
204+
if (dbType == DbType.POSTGRESQL) {
205+
conn.setAutoCommit(false); // required for cursor fetch in pg
206+
ps.setFetchSize(500);
207+
} else {
208+
ps.setFetchSize(Integer.MIN_VALUE);
209+
}
210+
} catch (Exception ignore) {
211+
// Driver may not support the hint; safe to ignore.
212+
}
213+
214+
try (ResultSet rs = ps.executeQuery()) {
215+
final int colCount = rs.getMetaData().getColumnCount();
216+
217+
while (rs.next()) {
218+
UUID uuid = null;
219+
ArrayList<Column> cols = new ArrayList<>(colCount);
220+
221+
for (int i = 1; i <= colCount; i++) {
222+
String columnName = rs.getMetaData().getColumnLabel(i);
223+
Column rCol;
224+
225+
if (plugin.getUserManager().getDataManager().isInt(columnName)) {
226+
rCol = new Column(columnName, DataType.INTEGER);
227+
try {
228+
rCol.setValue(new DataValueInt(rs.getInt(i)));
229+
} catch (Exception e) {
230+
String data = rs.getString(i);
231+
if (data != null) {
232+
try {
233+
rCol.setValue(new DataValueInt(Integer.parseInt(data)));
234+
} catch (NumberFormatException ex) {
235+
rCol.setValue(new DataValueInt(0));
236+
}
237+
} else {
238+
rCol.setValue(new DataValueInt(0));
239+
}
240+
}
241+
} else if (plugin.getUserManager().getDataManager().isBoolean(columnName)) {
242+
rCol = new Column(columnName, DataType.BOOLEAN);
243+
rCol.setValue(new DataValueBoolean(Boolean.valueOf(rs.getString(i))));
244+
} else {
245+
rCol = new Column(columnName, DataType.STRING);
246+
247+
if ("uuid".equalsIgnoreCase(columnName)) {
248+
// Extract UUID efficiently and correctly for pg uuid type
249+
if (dbType == DbType.POSTGRESQL) {
250+
Object obj = rs.getObject(i);
251+
if (obj instanceof java.util.UUID) {
252+
uuid = (java.util.UUID) obj;
253+
rCol.setValue(new DataValueString(uuid.toString()));
254+
} else {
255+
String s = rs.getString(i);
256+
if (s != null && !s.isEmpty() && !"null".equalsIgnoreCase(s)) {
257+
uuid = UUID.fromString(s);
258+
rCol.setValue(new DataValueString(s));
259+
} else {
260+
rCol.setValue(new DataValueString(s));
261+
}
262+
}
263+
} else {
264+
String s = rs.getString(i);
265+
if (s != null && !s.isEmpty() && !"null".equalsIgnoreCase(s)) {
266+
uuid = UUID.fromString(s);
267+
}
268+
rCol.setValue(new DataValueString(s));
269+
}
270+
} else {
271+
rCol.setValue(new DataValueString(rs.getString(i)));
272+
}
273+
}
274+
275+
cols.add(rCol);
276+
}
277+
278+
if (uuid != null) {
279+
processed++;
280+
perUser.accept(uuid, cols);
281+
}
282+
// allow GC of cols for each row after callback returns
283+
}
284+
}
285+
} catch (SQLException e) {
286+
debug(e);
287+
} finally {
288+
if (onFinished != null) {
289+
onFinished.accept(processed);
290+
}
291+
}
292+
}
293+
177294
// -------------------------
178295
// Keep existing methods (getUuids / getUUID / etc.)
179296
// -------------------------

‎AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/userstorage/sql/UserTable.java‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import java.util.HashMap;
1010
import java.util.List;
1111
import java.util.UUID;
12+
import java.util.function.BiConsumer;
13+
import java.util.function.Consumer;
1214

1315
import com.bencodez.advancedcore.AdvancedCorePlugin;
1416
import com.bencodez.advancedcore.api.messages.PlaceholderUtils;
@@ -46,6 +48,95 @@ public UserTable(AdvancedCorePlugin plugin, String name, Collection<Column> colu
4648
this.plugin = plugin;
4749
}
4850

51+
/**
52+
* Streams every user row from SQLite without building a giant in-memory map.
53+
*
54+
* @param perUser called once per row with (uuid, columns)
55+
* @param onFinished called once at the end with the number of rows processed
56+
*/
57+
public void forEachUser(BiConsumer<UUID, ArrayList<Column>> perUser, Consumer<Integer> onFinished) {
58+
int processed = 0;
59+
String query = "SELECT * FROM " + getName() + ";";
60+
61+
PreparedStatement s = null;
62+
ResultSet rs = null;
63+
64+
try {
65+
s = sqLite.getSQLConnection().prepareStatement(query);
66+
rs = s.executeQuery();
67+
68+
while (rs.next()) {
69+
ArrayList<Column> cols = new ArrayList<>();
70+
UUID uuid = null;
71+
72+
for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) {
73+
String columnName = rs.getMetaData().getColumnLabel(i);
74+
Column rCol;
75+
76+
if (plugin.getUserManager().getDataManager().isInt(columnName)) {
77+
rCol = new Column(columnName, DataType.INTEGER);
78+
try {
79+
rCol.setValue(new DataValueInt(rs.getInt(i)));
80+
} catch (Exception e) {
81+
String data = rs.getString(i);
82+
if (data != null) {
83+
try {
84+
rCol.setValue(new DataValueInt(Integer.parseInt(data)));
85+
} catch (NumberFormatException ex) {
86+
rCol.setValue(new DataValueInt(0));
87+
}
88+
} else {
89+
rCol.setValue(new DataValueInt(0));
90+
}
91+
}
92+
} else if (plugin.getUserManager().getDataManager().isBoolean(columnName)) {
93+
rCol = new Column(columnName, DataType.BOOLEAN);
94+
rCol.setValue(new DataValueBoolean(Boolean.valueOf(rs.getString(i))));
95+
} else {
96+
rCol = new Column(columnName, DataType.STRING);
97+
String val = rs.getString(i);
98+
rCol.setValue(new DataValueString(val));
99+
100+
if ("uuid".equalsIgnoreCase(columnName)) {
101+
if (val != null && !val.isEmpty() && !"null".equalsIgnoreCase(val)) {
102+
try {
103+
uuid = UUID.fromString(val);
104+
} catch (IllegalArgumentException ignored) {
105+
// bad uuid in db; skip row below
106+
}
107+
}
108+
}
109+
}
110+
111+
cols.add(rCol);
112+
}
113+
114+
if (uuid != null) {
115+
processed++;
116+
perUser.accept(uuid, cols); // <-- per row
117+
}
118+
// cols becomes eligible for GC after this loop iteration
119+
}
120+
} catch (SQLException e) {
121+
e.printStackTrace();
122+
} finally {
123+
try {
124+
if (rs != null)
125+
rs.close();
126+
} catch (SQLException ignored) {
127+
}
128+
try {
129+
if (s != null)
130+
s.close();
131+
} catch (SQLException ignored) {
132+
}
133+
134+
if (onFinished != null) {
135+
onFinished.accept(processed); // <-- once
136+
}
137+
}
138+
}
139+
49140
public UserTable(AdvancedCorePlugin plugin, String name, Column... columns) {
50141
this.name = name;
51142
for (Column column : columns) {

0 commit comments

Comments
 (0)