Skip to content

Commit 337e8a4

Browse files
kingzacko1kingzacko1
authored
Add ability to preview entries in lookup tables (#23046)
* Add ability to preview entries in lookup tables * cl * Fix forbidden APIs * Fix ApiOperation description
1 parent 0d0e2c5 commit 337e8a4

File tree

10 files changed

+242
-10
lines changed

10 files changed

+242
-10
lines changed

‎changelog/unreleased/pr-23046.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
type = "a"
2+
message = "Added data preview feature for lookup tables with supported data adapters."
3+
4+
issues = ["graylog-plugin-enterprise#11114"]
5+
pulls = ["23046"]

‎graylog2-server/src/main/java/org/graylog2/lookup/LookupTable.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.graylog2.plugin.lookup.LookupCache;
2323
import org.graylog2.plugin.lookup.LookupCacheKey;
2424
import org.graylog2.plugin.lookup.LookupDataAdapter;
25+
import org.graylog2.plugin.lookup.LookupPreview;
2526
import org.graylog2.plugin.lookup.LookupResult;
2627

2728
import javax.annotation.Nonnull;
@@ -133,6 +134,14 @@ public LookupResult assignTtl(@Nonnull Object key, @Nonnull Long ttlSec) {
133134
return result;
134135
}
135136

137+
public boolean supportsPreview() {
138+
return dataAdapter().supportsPreview();
139+
}
140+
141+
public LookupPreview getPreview(int size) {
142+
return dataAdapter().getPreview(size);
143+
}
144+
136145
@AutoValue.Builder
137146
public abstract static class Builder {
138147
public abstract Builder id(String id);

‎graylog2-server/src/main/java/org/graylog2/lookup/LookupTableService.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.graylog2.lookup.events.LookupTablesUpdated;
3939
import org.graylog2.plugin.lookup.LookupCache;
4040
import org.graylog2.plugin.lookup.LookupDataAdapter;
41+
import org.graylog2.plugin.lookup.LookupPreview;
4142
import org.graylog2.plugin.lookup.LookupResult;
4243
import org.graylog2.system.SystemEntity;
4344
import org.graylog2.utilities.LoggingServiceListener;
@@ -732,5 +733,21 @@ public LookupResult assignTtl(Object key, Long ttlSec) {
732733
}
733734
return lookupTable.assignTtl(requireValidKey(key), ttlSec);
734735
}
736+
737+
public boolean supportsPreview() {
738+
final LookupTable lookupTable = lookupTableService.getTable(lookupTableName);
739+
if (lookupTable == null) {
740+
return false;
741+
}
742+
return lookupTable.supportsPreview();
743+
}
744+
745+
public LookupPreview getPreview(int size) {
746+
final LookupTable lookupTable = lookupTableService.getTable(lookupTableName);
747+
if (lookupTable == null) {
748+
return LookupPreview.empty();
749+
}
750+
return lookupTable.getPreview(size);
751+
}
735752
}
736753
}

‎graylog2-server/src/main/java/org/graylog2/lookup/adapters/CSVFileDataAdapter.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.graylog2.plugin.lookup.LookupCachePurge;
4242
import org.graylog2.plugin.lookup.LookupDataAdapter;
4343
import org.graylog2.plugin.lookup.LookupDataAdapterConfiguration;
44+
import org.graylog2.plugin.lookup.LookupPreview;
4445
import org.graylog2.plugin.lookup.LookupResult;
4546
import org.graylog2.plugin.utilities.FileInfo;
4647
import org.graylog2.utilities.CIDRPatriciaTrie;
@@ -60,6 +61,7 @@
6061
import java.nio.file.Files;
6162
import java.nio.file.Path;
6263
import java.nio.file.Paths;
64+
import java.util.HashMap;
6365
import java.util.Locale;
6466
import java.util.Map;
6567
import java.util.Optional;
@@ -291,7 +293,29 @@ public LookupResult doGet(Object key) {
291293
return LookupResult.single(value);
292294
}
293295

294-
public LookupResult getResultForCIDRRange(Object ip) {
296+
@Override
297+
public boolean supportsPreview() {
298+
return true;
299+
}
300+
301+
@Override
302+
public LookupPreview getPreview(int size) {
303+
if (config.isCidrLookup()) {
304+
return cidrLookupRef.get().getPreview(size);
305+
} else {
306+
final Map<Object, Object> result = new HashMap<>();
307+
final Map<String, String> lookup = lookupRef.get();
308+
for (Map.Entry<String, String> entries : lookup.entrySet()) {
309+
if (result.size() == size) {
310+
break;
311+
}
312+
result.put(entries.getKey(), entries.getValue());
313+
}
314+
return new LookupPreview(lookup.size(), result);
315+
}
316+
}
317+
318+
private LookupResult getResultForCIDRRange(Object ip) {
295319
LookupResult result = getEmptyResult();
296320
try {
297321
final String resultValue = cidrLookupRef.get().longestPrefixRangeLookup(String.valueOf(ip));

‎graylog2-server/src/main/java/org/graylog2/lookup/adapters/DSVHTTPDataAdapter.java‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,18 @@
2626
import com.google.common.collect.Multimap;
2727
import com.google.common.primitives.Ints;
2828
import com.google.inject.assistedinject.Assisted;
29+
import jakarta.inject.Inject;
30+
import jakarta.validation.constraints.Min;
31+
import jakarta.validation.constraints.NotEmpty;
32+
import jakarta.validation.constraints.Size;
2933
import okhttp3.HttpUrl;
3034
import org.graylog.autovalue.WithBeanGetter;
3135
import org.graylog2.lookup.adapters.dsvhttp.DSVParser;
3236
import org.graylog2.lookup.adapters.dsvhttp.HTTPFileRetriever;
3337
import org.graylog2.plugin.lookup.LookupCachePurge;
3438
import org.graylog2.plugin.lookup.LookupDataAdapter;
3539
import org.graylog2.plugin.lookup.LookupDataAdapterConfiguration;
40+
import org.graylog2.plugin.lookup.LookupPreview;
3641
import org.graylog2.plugin.lookup.LookupResult;
3742
import org.graylog2.system.urlwhitelist.UrlNotWhitelistedException;
3843
import org.graylog2.system.urlwhitelist.UrlWhitelistNotificationService;
@@ -41,13 +46,8 @@
4146
import org.slf4j.Logger;
4247
import org.slf4j.LoggerFactory;
4348

44-
import jakarta.inject.Inject;
45-
46-
import jakarta.validation.constraints.Min;
47-
import jakarta.validation.constraints.NotEmpty;
48-
import jakarta.validation.constraints.Size;
49-
5049
import java.util.Collections;
50+
import java.util.HashMap;
5151
import java.util.Locale;
5252
import java.util.Map;
5353
import java.util.Optional;
@@ -164,6 +164,24 @@ public LookupResult doGet(Object key) {
164164
return LookupResult.single(value);
165165
}
166166

167+
@Override
168+
public boolean supportsPreview() {
169+
return true;
170+
}
171+
172+
@Override
173+
public LookupPreview getPreview(int size) {
174+
final Map<Object, Object> result = new HashMap<>();
175+
final Map<String, String> lookup = lookupRef.get();
176+
for (Map.Entry<String, String> entries : lookup.entrySet()) {
177+
if (result.size() == size) {
178+
break;
179+
}
180+
result.put(entries.getKey(), entries.getValue());
181+
}
182+
return new LookupPreview(lookup.size(), result);
183+
}
184+
167185
@Override
168186
public void set(Object key, Object value) {
169187
throw new UnsupportedOperationException();

‎graylog2-server/src/main/java/org/graylog2/plugin/lookup/LookupDataAdapter.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,14 @@ public LookupResult assignTtl(Object key, Long ttlSec) {
237237
return resultWithError;
238238
}
239239

240+
public boolean supportsPreview() {
241+
return false;
242+
}
243+
244+
public LookupPreview getPreview(int size) {
245+
return LookupPreview.empty();
246+
}
247+
240248
public LookupDataAdapterConfiguration getConfig() {
241249
return config;
242250
}

‎graylog2-server/src/main/java/org/graylog2/plugin/lookup/LookupPreview.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* Copyright (C) 2020 Graylog, Inc.
3+
*
4+
* This program is free software: you can redistribute it and/or modify
5+
* it under the terms of the Server Side Public License, version 1,
6+
* as published by MongoDB, Inc.
7+
*
8+
* This program is distributed in the hope that it will be useful,
9+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
10+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11+
* Server Side Public License for more details.
12+
*
13+
* You should have received a copy of the Server Side Public License
14+
* along with this program. If not, see
15+
* <http://www.mongodb.com/licensing/server-side-public-license>.
16+
*/
17+
package org.graylog2.plugin.lookup;
18+
19+
import java.util.Map;
20+
21+
public record LookupPreview(long total, Map<Object, Object> results) {
22+
public static LookupPreview empty() {
23+
return new LookupPreview(0, Map.of());
24+
}
25+
}

‎graylog2-server/src/main/java/org/graylog2/rest/resources/system/lookup/LookupTableResource.java‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
import org.graylog2.lookup.dto.LookupTableDto;
7070
import org.graylog2.plugin.lookup.LookupCache;
7171
import org.graylog2.plugin.lookup.LookupDataAdapter;
72+
import org.graylog2.plugin.lookup.LookupPreview;
7273
import org.graylog2.plugin.lookup.LookupResult;
7374
import org.graylog2.plugin.rest.ValidationResult;
7475
import org.graylog2.rest.models.SortOrder;
@@ -227,7 +228,7 @@ public LookupResult performLookup(@ApiParam(name = "name") @PathParam("name") @N
227228
public void performPurge(@ApiParam(name = "idOrName") @PathParam("idOrName") @NotEmpty String idOrName,
228229
@ApiParam(name = "key") @QueryParam("key") String key) {
229230
final Optional<LookupTableDto> lookupTableDto = dbTableService.get(idOrName);
230-
if (!lookupTableDto.isPresent()) {
231+
if (lookupTableDto.isEmpty()) {
231232
throw new NotFoundException("Lookup table <" + idOrName + "> not found");
232233
}
233234

@@ -304,7 +305,7 @@ public LookupTablePage get(@ApiParam(name = "idOrName") @PathParam("idOrName") @
304305
@ApiParam(name = "resolve") @QueryParam("resolve") @DefaultValue("false") boolean resolveObjects) {
305306

306307
Optional<LookupTableDto> lookupTableDto = dbTableService.get(idOrName);
307-
if (!lookupTableDto.isPresent()) {
308+
if (lookupTableDto.isEmpty()) {
308309
throw new NotFoundException();
309310
}
310311
LookupTableDto tableDto = lookupTableDto.get();
@@ -369,7 +370,7 @@ public LookupTableApi updateTable(@ApiParam(name = "idOrName") @PathParam("idOrN
369370
public LookupTableApi removeTable(@ApiParam(name = "idOrName") @PathParam("idOrName") @NotEmpty String idOrName) {
370371
// TODO validate that table isn't in use, how?
371372
Optional<LookupTableDto> lookupTableDto = dbTableService.get(idOrName);
372-
if (!lookupTableDto.isPresent()) {
373+
if (lookupTableDto.isEmpty()) {
373374
throw new NotFoundException();
374375
}
375376
checkPermission(RestPermissions.LOOKUP_TABLES_DELETE, lookupTableDto.get().id());
@@ -414,6 +415,25 @@ public ValidationResult validateTable(@ApiParam LookupTableApi toValidate) {
414415
return validation;
415416
}
416417

418+
@GET
419+
@Path("tables/preview/{idOrName}")
420+
@ApiOperation(value = "Preview the entries in a lookup table.")
421+
public LookupPreview getPreview(@ApiParam(name = "idOrName") @PathParam("idOrName") @NotEmpty String idOrName,
422+
@ApiParam(name = "size") @QueryParam("size") @DefaultValue("5") int size) {
423+
Optional<LookupTableDto> lookupTableDto = dbTableService.get(idOrName);
424+
if (lookupTableDto.isEmpty()) {
425+
throw new NotFoundException();
426+
}
427+
LookupTableDto tableDto = lookupTableDto.get();
428+
429+
checkPermission(RestPermissions.LOOKUP_TABLES_READ, tableDto.id());
430+
final var service = lookupTableService.newBuilder().lookupTable(tableDto.name()).build();
431+
if (!service.supportsPreview()) {
432+
throw new UnsupportedOperationException("Backing data adapter does not support preview.");
433+
}
434+
return service.getPreview(size);
435+
}
436+
417437
@JsonAutoDetect
418438
public static class LookupTablePage {
419439

‎graylog2-server/src/main/java/org/graylog2/utilities/CIDRPatriciaTrie.java‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,11 @@
1818

1919
import com.google.common.annotations.VisibleForTesting;
2020
import org.apache.commons.collections4.trie.PatriciaTrie;
21+
import org.graylog2.plugin.lookup.LookupPreview;
2122
import org.joda.time.DateTime;
2223
import org.joda.time.DateTimeZone;
2324

25+
import java.util.HashMap;
2426
import java.util.Locale;
2527
import java.util.Map;
2628

@@ -190,6 +192,20 @@ public void recalculateShortestPrefix(boolean isIPV6) {
190192
}
191193
}
192194

195+
// Gets a preview of the nodes in the trie. Since initial CIDR ranges have been converted to binary strings, the
196+
// keys in the result map need to be converted back to their original format.
197+
public LookupPreview getPreview(int size) {
198+
final Map<Object, Object> result = new HashMap<>();
199+
for (Map.Entry<String, Node> entry : trie.entrySet()) {
200+
if (result.size() == size) {
201+
break;
202+
}
203+
final String cidrRange = fromBinaryString(entry.getKey(), entry.getValue().rangeIsIPv6());
204+
result.put(cidrRange, entry.getValue().rangeName);
205+
}
206+
return new LookupPreview(trie.size(), result);
207+
}
208+
193209
// Convert an IP address to a binary string (supports both IPv4 and IPv6)
194210
static String toBinaryString(String ip, int prefixLength) {
195211
final boolean isIPv6 = ip.contains(":");
@@ -234,6 +250,85 @@ static String toBinaryString(String ip, int prefixLength) {
234250
}
235251
}
236252

253+
static String fromBinaryString(String binary, boolean isIPv6) {
254+
if (isIPv6) {
255+
return fromBinaryIPv6(binary);
256+
} else {
257+
return fromBinaryIPv4(binary);
258+
}
259+
}
260+
261+
private static String fromBinaryIPv4(String binary) {
262+
final int prefixLength = binary.length();
263+
// Pad to 32 bits
264+
binary = String.format(Locale.ROOT, "%-32s", binary).replace(' ', '0');
265+
StringBuilder ip = new StringBuilder();
266+
for (int i = 0; i < 32; i += 8) {
267+
String octet = binary.substring(i, i + 8);
268+
ip.append(Integer.parseInt(octet, 2));
269+
if (i < 24) ip.append('.');
270+
}
271+
if (prefixLength == 32) {
272+
return ip.toString();
273+
}
274+
return ip + "/" + prefixLength;
275+
}
276+
277+
private static String fromBinaryIPv6(String binary) {
278+
final int prefixLength = binary.length();
279+
// Pad to 128 bits
280+
binary = String.format(Locale.ROOT, "%-128s", binary).replace(' ', '0');
281+
StringBuilder ip = new StringBuilder();
282+
for (int i = 0; i < 128; i += 16) {
283+
String hextet = binary.substring(i, i + 16);
284+
ip.append(Integer.toHexString(Integer.parseInt(hextet, 2)));
285+
if (i < 112) ip.append(':');
286+
}
287+
288+
// Compress using "::" for longest run of zeros
289+
String compressed = compressIPv6(ip.toString());
290+
if (prefixLength == 128) {
291+
return compressed;
292+
}
293+
return compressed + "/" + prefixLength;
294+
}
295+
296+
// Utility to compress IPv6 address (e.g., turn "2001:0:0:0:0:0:0:1" into "2001::1")
297+
private static String compressIPv6(String fullAddress) {
298+
String[] parts = fullAddress.split(":");
299+
int bestStart = -1, bestLen = 0;
300+
int currStart = -1, currLen = 0;
301+
302+
for (int i = 0; i <= parts.length; i++) {
303+
if (i < parts.length && parts[i].equals("0")) {
304+
if (currStart == -1) currStart = i;
305+
currLen++;
306+
} else {
307+
if (currLen > bestLen) {
308+
bestStart = currStart;
309+
bestLen = currLen;
310+
}
311+
currStart = -1;
312+
currLen = 0;
313+
}
314+
}
315+
316+
if (bestLen > 1) {
317+
StringBuilder sb = new StringBuilder();
318+
for (int i = 0; i < bestStart; i++) {
319+
sb.append(parts[i]).append(":");
320+
}
321+
sb.append("::");
322+
for (int i = bestStart + bestLen; i < parts.length; i++) {
323+
sb.append(parts[i]);
324+
if (i < parts.length - 1) sb.append(":");
325+
}
326+
return sb.toString().replaceAll("(^:)|(:$)", "");
327+
}
328+
329+
return fullAddress;
330+
}
331+
237332
// Used only for testing purposes.
238333
@VisibleForTesting
239334
Node getNode(String key) {

0 commit comments

Comments
 (0)