Skip to content

Hash ids for modular distributed compiled mode metadata #1203

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kevin-m-knight-gs merged 2 commits into from
Feb 26, 2026
Merged

Hash ids for modular distributed compiled mode metadata #1203

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5510f52
Use base 32 encoding for distributed metadata id hashing
kevin-m-knight-gs Feb 25, 2026
161b824
Hash ids for modular distributed metadata
kevin-m-knight-gs Feb 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions ...d/pure/runtime/java/compiled/serialization/binary/DistributedBinaryGraphDeserializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -468,6 +468,12 @@ public RichIterable<String> getClassifierInstanceIds(String classifierId)
return (classifierIndex == null) ? Lists.immutable.empty() : classifierIndex.getInstanceIds();
}

@Override
public String processId(String id)
{
return (this.metadataName == null) ? id : super.processId(id);
}

@Override
public String processEnumId(String enumerationName, String enumName)
{
Expand Down
8 changes: 7 additions & 1 deletion ...end/pure/runtime/java/compiled/serialization/binary/DistributedBinaryGraphSerializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ protected DistributedBinaryGraphSerializer(PureRuntime runtime, DistributedMetad
this.metadataSpecification = metadataSpecification;
this.runtime = runtime;
this.processorSupport = runtime.getProcessorSupport();
this.idBuilder = DistributedMetadataHelper.possiblyHashIds(newIdBuilder(this.metadataSpecification, this.processorSupport));
this.idBuilder = newPossiblyHashedIdBuilder(this.metadataSpecification, this.processorSupport);
this.classifierCaches = new GraphSerializer.ClassifierCaches(this.processorSupport);
}

Expand Down Expand Up @@ -302,6 +302,12 @@ private static IdBuilder newIdBuilder_internal(String metadataName, ProcessorSup
.build();
}

private static IdBuilder newPossiblyHashedIdBuilder(DistributedMetadataSpecification metadataSpec, ProcessorSupport processorSupport)
{
IdBuilder idBuilder = newIdBuilder(metadataSpec, processorSupport);
return (metadataSpec == null) ? idBuilder : DistributedMetadataHelper.possiblyHashIds(idBuilder);
}

protected class SerializationCollector
{
private final MutableMap<String, MutableList<CoreInstance>> instancesForSerialization = Maps.mutable.empty();
Expand Down
10 changes: 3 additions & 7 deletions ...nos/legend/pure/runtime/java/compiled/serialization/binary/DistributedMetadataHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,11 @@
import org.finos.legend.pure.m4.coreinstance.CoreInstance;
import org.finos.legend.pure.runtime.java.compiled.generation.processors.IdBuilder;

import java.nio.ByteBuffer;
import java.util.Base64;
import java.util.Objects;

public class DistributedMetadataHelper
{
private static final boolean HASH_IDS = Boolean.parseBoolean(System.getProperty("legend.pure.runtime.java.compiled.serialization.binary.distributed.hashids", "false"));
private static final boolean HASH_IDS = Boolean.parseBoolean(System.getProperty("legend.pure.runtime.java.compiled.serialization.binary.distributed.hashids", "true"));

private static final String META_DATA_DIRNAME = "metadata/";
private static final String SPECS_DIRNAME = META_DATA_DIRNAME + "specs/";
Expand Down Expand Up @@ -209,10 +207,8 @@ public static String hashId(String id)
hash = SpreadFunctions.longSpreadOne(hash) + (codePoint = id.codePointAt(i));
}

// convert to base64 string
byte[] bytes = new byte[8];
ByteBuffer.wrap(bytes).putLong(hash);
return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes);
// convert to base 32 string
return Long.toUnsignedString(hash, 32);
}

public static IdBuilder hashIds(IdBuilder idBuilder)
Expand Down
2 changes: 1 addition & 1 deletion ...e/java/compiled/runtime/serialization/binary/TestDistributedBinaryGraphSerialization.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public void testModularSerialization() throws IOException
PureRuntime runtime = buildRuntime(repo);

DistributedBinaryGraphSerializer.newSerializer(runtime, repo).serialize(getFileWriter());
ListIterable<Obj> expectedObjs = getExpectedObjsFromRuntime(runtime, IdBuilder.builder(runtime.getProcessorSupport()).withDefaultIdPrefix(DistributedMetadataHelper.getMetadataIdPrefix(repo)).build());
ListIterable<Obj> expectedObjs = getExpectedObjsFromRuntime(runtime, DistributedMetadataHelper.possiblyHashIds(IdBuilder.builder(runtime.getProcessorSupport()).withDefaultIdPrefix(DistributedMetadataHelper.getMetadataIdPrefix(repo)).build()));
testSerialization(expectedObjs, Lists.immutable.with(repo), false);
}

Expand Down
Loading