Skip to content

Avoid excessive memory requests in singleton builders #133777

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Avoid excessive memory requests in singleton builders #133777

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions ...l/compute/src/main/java/org/elasticsearch/compute/lucene/read/SingletonDoubleBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

package org.elasticsearch.compute.lucene.read;

import org.apache.lucene.util.RamUsageEstimator;
import org.elasticsearch.compute.data.Block;
import org.elasticsearch.compute.data.BlockFactory;
import org.elasticsearch.core.Releasable;
Expand All @@ -23,10 +24,13 @@ public final class SingletonDoubleBuilder implements BlockLoader.SingletonDouble
private final BlockFactory blockFactory;

private int count;
private long memoryUsed;

public SingletonDoubleBuilder(int expectedCount, BlockFactory blockFactory) {
this.blockFactory = blockFactory;
blockFactory.adjustBreaker(valuesSize(expectedCount));
long memory = valuesSize(expectedCount);
blockFactory.adjustBreaker(memory);
memoryUsed += memory;
this.values = new double[expectedCount];
}

Expand Down Expand Up @@ -69,7 +73,9 @@ public Block build() {
if (values.length != count) {
throw new IllegalStateException("expected " + values.length + " values but got " + count);
}
return blockFactory.newDoubleArrayVector(values, count).asBlock();
var result = blockFactory.newDoubleArrayVector(values, count, memoryUsed).asBlock();
memoryUsed = 0;
return result;
}

@Override
Expand All @@ -83,10 +89,11 @@ public BlockLoader.SingletonDoubleBuilder appendLongs(BlockDocValuesReader.ToDou

@Override
public void close() {
blockFactory.adjustBreaker(-valuesSize(values.length));
blockFactory.adjustBreaker(-memoryUsed);
}

static long valuesSize(int count) {
return (long) count * Double.BYTES;
return RamUsageEstimator.alignObjectSize((long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long) Double.BYTES * count)
+ Block.PAGE_MEM_OVERHEAD_PER_BLOCK;
}
}
15 changes: 11 additions & 4 deletions ...sql/compute/src/main/java/org/elasticsearch/compute/lucene/read/SingletonLongBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

package org.elasticsearch.compute.lucene.read;

import org.apache.lucene.util.RamUsageEstimator;
import org.elasticsearch.compute.data.Block;
import org.elasticsearch.compute.data.BlockFactory;
import org.elasticsearch.core.Releasable;
Expand All @@ -20,12 +21,15 @@ public final class SingletonLongBuilder implements BlockLoader.SingletonLongBuil

private final long[] values;
private final BlockFactory blockFactory;
private long memoryUsed;

private int count;

public SingletonLongBuilder(int expectedCount, BlockFactory blockFactory) {
this.blockFactory = blockFactory;
blockFactory.adjustBreaker(valuesSize(expectedCount));
final long memory = valuesSize(expectedCount);
blockFactory.adjustBreaker(memory);
memoryUsed += memory;
this.values = new long[expectedCount];
}

Expand Down Expand Up @@ -68,7 +72,9 @@ public Block build() {
if (values.length != count) {
throw new IllegalStateException("expected [" + values.length + "] values but got [" + count + "]");
}
return blockFactory.newLongArrayVector(values, count).asBlock();
var result = blockFactory.newLongArrayVector(values, count, memoryUsed).asBlock();
memoryUsed = 0;
return result;
}

@Override
Expand All @@ -86,10 +92,11 @@ public BlockLoader.SingletonLongBuilder appendLongs(long[] values, int from, int

@Override
public void close() {
blockFactory.adjustBreaker(-valuesSize(values.length));
blockFactory.adjustBreaker(-memoryUsed);
}

static long valuesSize(int count) {
return (long) count * Long.BYTES;
return RamUsageEstimator.alignObjectSize((long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long) Long.BYTES * count)
+ Block.PAGE_MEM_OVERHEAD_PER_BLOCK;
}
}