From 88370e4d72b98c10a412187252518ef4bc16d3bd Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 23 Jan 2025 15:13:47 -0500 Subject: [PATCH 1/2] ESQL: Expand HeapAttack for LOOKUP This expands the heap attack tests for LOOKUP. Now there are three flavors: 1. LOOKUP a single geo_point - about 30 bytes or so. 2. LOOKUP a one mb string. 3. LOOKUP no fields - just JOIN to alter cardinality. Fetching a geo_point is fine with about 500 repeated docs before it circuit breaks which works out to about 256mb of buffered results. That's sensible on our 512mb heap and likely to work ok for most folks. We'll flip to a streaming method eventually and this won't be a problem any more. But for now, we buffer. The no lookup fields is fine with like 7500 matches per incoming row. That's quite a lot, really. The 1mb string is trouble! We circuit break properly which is great and safe, but if you join 1mb worth of columns in LOOKUP you are going to need bigger heaps than our test. Again, we'll move from buffering these results to streaming them and it'll work better, but for now we buffer. --- .../xpack/esql/heap_attack/HeapAttackIT.java | 86 ++++++++++++++++++- 1 file changed, 83 insertions(+), 3 deletions(-) diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java index 2e68c094492fe..fc567c22bb51e 100644 --- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java +++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java @@ -624,7 +624,7 @@ private Response fetchMvLongs() throws IOException { } public void testLookupExplosion() throws IOException { - int sensorDataCount = 7500; + int sensorDataCount = 500; int lookupEntries = 10000; Map map = lookupExplosion(sensorDataCount, lookupEntries); assertMap(map, matchesMap().extraOk().entry("values", List.of(List.of(sensorDataCount * lookupEntries)))); @@ -632,16 +632,71 @@ public void testLookupExplosion() throws IOException { public void testLookupExplosionManyMatches() throws IOException { assertCircuitBreaks(() -> { - Map result = lookupExplosion(8500, 10000); + Map result = lookupExplosion(900, 10000); + logger.error("should have failed but got {}", result); + }); + } + + public void testLookupExplosionNoFetch() throws IOException { + int sensorDataCount = 7500; + int lookupEntries = 10000; + Map map = lookupExplosionNoFetch(sensorDataCount, lookupEntries); + assertMap(map, matchesMap().extraOk().entry("values", List.of(List.of(sensorDataCount * lookupEntries)))); + } + + public void testLookupExplosionNoFetchManyMatches() throws IOException { + assertCircuitBreaks(() -> { + Map result = lookupExplosionNoFetch(8500, 10000); + logger.error("should have failed but got {}", result); + }); + } + + public void testLookupExplosionBigString() throws IOException { + int sensorDataCount = 250; + int lookupEntries = 1; + Map map = lookupExplosionBigString(sensorDataCount, lookupEntries); + assertMap(map, matchesMap().extraOk().entry("values", List.of(List.of(sensorDataCount * lookupEntries)))); + } + + public void testLookupExplosionBigStringManyMatches() throws IOException { + assertCircuitBreaks(() -> { + Map result = lookupExplosionBigString(500, 1); logger.error("should have failed but got {}", result); }); } private Map lookupExplosion(int sensorDataCount, int lookupEntries) throws IOException { + lookupExplosionData(sensorDataCount, lookupEntries); + StringBuilder query = startQuery(); + query.append("FROM sensor_data | LOOKUP JOIN sensor_lookup ON id | STATS COUNT(location)\"}"); + return responseAsMap(query(query.toString(), null)); + } + + private Map lookupExplosionNoFetch(int sensorDataCount, int lookupEntries) throws IOException { + lookupExplosionData(sensorDataCount, lookupEntries); + StringBuilder query = startQuery(); + query.append("FROM sensor_data | LOOKUP JOIN sensor_lookup ON id | STATS COUNT(*)\"}"); + return responseAsMap(query(query.toString(), null)); + } + + private void lookupExplosionData(int sensorDataCount, int lookupEntries) throws IOException { initSensorData(sensorDataCount, 1); initSensorLookup(lookupEntries, 1, i -> "73.9857 40.7484"); + } + + private Map lookupExplosionBigString(int sensorDataCount, int lookupEntries) throws IOException { + initSensorData(sensorDataCount, 1); + initSensorLookupString(lookupEntries, 1, i -> { + int target = Math.toIntExact(ByteSizeValue.ofMb(1).getBytes()); + StringBuilder str = new StringBuilder(Math.toIntExact(ByteSizeValue.ofMb(2).getBytes())); + while (str.length() < target) { + str.append("Lorem ipsum dolor sit amet, consectetur adipiscing elit."); + } + logger.info("big string is {} characters", str.length()); + return str.toString(); + }); StringBuilder query = startQuery(); - query.append("FROM sensor_data | LOOKUP JOIN sensor_lookup ON id | STATS COUNT(*)\"}"); + query.append("FROM sensor_data | LOOKUP JOIN sensor_lookup ON id | STATS COUNT(string)\"}"); return responseAsMap(query(query.toString(), null)); } @@ -830,6 +885,31 @@ private void initSensorLookup(int lookupEntries, int sensorCount, IntFunction string) throws IOException { + logger.info("loading sensor lookup with huge strings"); + createIndex("sensor_lookup", Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOOKUP.getName()).build(), """ + { + "properties": { + "id": { "type": "long" }, + "string": { "type": "text" } + } + }"""); + int docsPerBulk = 10; + StringBuilder data = new StringBuilder(); + for (int i = 0; i < lookupEntries; i++) { + int sensor = i % sensorCount; + data.append(String.format(Locale.ROOT, """ + {"create":{}} + {"id": %d, "string": "%s"} + """, sensor, string.apply(sensor))); + if (i % docsPerBulk == docsPerBulk - 1) { + bulk("sensor_lookup", data.toString()); + data.setLength(0); + } + } + initIndex("sensor_lookup", data.toString()); + } + private void initSensorEnrich(int lookupEntries, int sensorCount, IntFunction location) throws IOException { initSensorLookup(lookupEntries, sensorCount, location); logger.info("loading sensor enrich"); From 788255fa5e493a35396011168064e127b94e2fc7 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 29 Jan 2025 15:13:17 -0500 Subject: [PATCH 2/2] updates --- .../elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java index bd85b0cd2b454..59ce3c2dbb1cf 100644 --- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java +++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java @@ -636,7 +636,7 @@ public void testLookupExplosion() throws IOException { public void testLookupExplosionManyMatches() throws IOException { assertCircuitBreaks(() -> { - Map result = lookupExplosion(900, 10000); + Map result = lookupExplosion(1500, 10000); logger.error("should have failed but got {}", result); }); } @@ -656,7 +656,7 @@ public void testLookupExplosionNoFetchManyMatches() throws IOException { } public void testLookupExplosionBigString() throws IOException { - int sensorDataCount = 250; + int sensorDataCount = 150; int lookupEntries = 1; Map map = lookupExplosionBigString(sensorDataCount, lookupEntries); assertMap(map, matchesMap().extraOk().entry("values", List.of(List.of(sensorDataCount * lookupEntries))));