Skip to content

Commit 36b9c2f

Browse files
kevin-m-knight-gskevin-m-knight-gs
authored
Add module dependencies to ModuleManifest (#1204)
* Add module dependencies to ModuleManifest * Ensure ModuleMetadataGenerator generates metadata for all modules, even those without elements * Ensure PureCompilerSerializer generates metadata for all modules, even those without elements
1 parent 1cd1465 commit 36b9c2f

File tree

24 files changed

+1599
-576
lines changed

24 files changed

+1599
-576
lines changed

‎legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/PureCompilerSerializer.java‎

Lines changed: 65 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,45 @@ private Serializer newSerializer(ZipOutputStream stream)
127127

128128
private void serializeAll(Serializer serializer, boolean includeRootModule)
129129
{
130-
serializeModules(serializer, includeRootModule ? m -> true : ModuleHelper::isNonRootModule);
130+
long start = System.nanoTime();
131+
LOGGER.info("Serializing all modules (include root: {})", includeRootModule);
132+
try
133+
{
134+
MutableSet<String> moduleNames = Sets.mutable.empty();
135+
long eltStart = System.nanoTime();
136+
LOGGER.info("Starting element serialization");
137+
int[] eltCount = {0};
138+
try
139+
{
140+
GraphTools.getTopLevelAndPackagedElements(this.processorSupport).forEach(e ->
141+
{
142+
String moduleName = ModuleHelper.getElementModule(e);
143+
if ((moduleName != null) && (includeRootModule || ModuleHelper.isNonRootModule(moduleName)))
144+
{
145+
moduleNames.add(moduleName);
146+
serializer.serializeElement(e);
147+
eltCount[0]++;
148+
}
149+
});
150+
}
151+
finally
152+
{
153+
long eltEnd = System.nanoTime();
154+
LOGGER.info("Finished serializing {} elements in {}s", eltCount[0], (eltEnd - eltStart) / 1_000_000_000.0);
155+
}
156+
157+
generateAndSerializeAllModuleMetadata(serializer, includeRootModule);
158+
}
159+
catch (Throwable t)
160+
{
161+
LOGGER.error("Error serializing all modules", t);
162+
throw t;
163+
}
164+
finally
165+
{
166+
long end = System.nanoTime();
167+
LOGGER.info("Finished serializing all modules in {}s", (end - start) / 1_000_000_000.0);
168+
}
131169
}
132170

133171
private void serializeModule(Serializer serializer, String moduleName)
@@ -323,6 +361,24 @@ private void serializeModules(Serializer serializer, Predicate<? super String> m
323361
}
324362
}
325363

364+
private void generateAndSerializeAllModuleMetadata(Serializer serializer, boolean includeRootModule)
365+
{
366+
long modMetaGenStart = System.nanoTime();
367+
MutableList<ModuleMetadata> moduleMetadata;
368+
LOGGER.info("Starting module metadata generation");
369+
try
370+
{
371+
moduleMetadata = this.moduleMetadataGenerator.generateAllModuleMetadata(includeRootModule);
372+
}
373+
finally
374+
{
375+
long modMetaGenEnd = System.nanoTime();
376+
LOGGER.info("Finished module metadata generation in {}s", (modMetaGenEnd - modMetaGenStart) / 1_000_000_000.0);
377+
}
378+
379+
serializeModuleMetadata(serializer, moduleMetadata);
380+
}
381+
326382
private void generateAndSerializeModuleMetadata(Serializer serializer, SetIterable<? extends String> moduleNames)
327383
{
328384
long modMetaGenStart = System.nanoTime();
@@ -338,18 +394,23 @@ private void generateAndSerializeModuleMetadata(Serializer serializer, SetIterab
338394
LOGGER.info("Finished module metadata generation in {}s", (modMetaGenEnd - modMetaGenStart) / 1_000_000_000.0);
339395
}
340396

397+
serializeModuleMetadata(serializer, moduleMetadata);
398+
}
399+
400+
private void serializeModuleMetadata(Serializer serializer, MutableList<ModuleMetadata> moduleMetadata)
401+
{
341402
moduleMetadata.forEach(modMeta ->
342403
{
343-
long modMetaSerStart = System.nanoTime();
404+
long start = System.nanoTime();
344405
LOGGER.info("Starting serialization of module metadata for {}", modMeta.getName());
345406
try
346407
{
347408
serializer.serializeModuleMetadata(modMeta);
348409
}
349410
finally
350411
{
351-
long modMetaSerEnd = System.nanoTime();
352-
LOGGER.info("Finished serialization of module metadata for {} in {}s", modMeta.getName(), (modMetaSerEnd - modMetaSerStart) / 1_000_000_000.0);
412+
long end = System.nanoTime();
413+
LOGGER.info("Finished serialization of module metadata for {} in {}s", modMeta.getName(), (end - start) / 1_000_000_000.0);
353414
}
354415
});
355416
}

‎legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/metadata/ModuleBackReferenceMetadata.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ private Builder()
117117
private Builder(ModuleBackReferenceMetadata metadata)
118118
{
119119
this.moduleName = metadata.moduleName;
120+
this.referenceIdVersion = metadata.referenceIdVersion;
120121
this.elementBackReferences = Maps.mutable.ofInitialCapacity(metadata.elementBackReferences.size());
121122
metadata.elementBackReferences.forEach(er -> this.elementBackReferences.put(er.getElementPath(), ElementBackReferenceMetadata.builder(er)));
122123
}

‎legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/metadata/ModuleExternalReferenceMetadata.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ private Builder(int elementExternalReferenceCount)
126126
private Builder(ModuleExternalReferenceMetadata metadata)
127127
{
128128
this.moduleName = metadata.moduleName;
129+
this.referenceIdVersion = metadata.referenceIdVersion;
129130
this.elementExternalReferences = Lists.mutable.withAll(metadata.elementExternalReferences);
130131
}
131132

‎legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/metadata/ModuleManifest.java‎

Lines changed: 139 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,13 @@
3535
public class ModuleManifest
3636
{
3737
private final String name;
38+
private final ImmutableList<String> dependencies;
3839
private final ImmutableList<ConcreteElementMetadata> elements;
3940

40-
private ModuleManifest(String name, ImmutableList<ConcreteElementMetadata> elements)
41+
private ModuleManifest(String name, ImmutableList<String> dependencies, ImmutableList<ConcreteElementMetadata> elements)
4142
{
4243
this.name = name;
44+
this.dependencies = dependencies;
4345
this.elements = elements;
4446
}
4547

@@ -48,6 +50,11 @@ public String getModuleName()
4850
return this.name;
4951
}
5052

53+
public ImmutableList<String> getDependencies()
54+
{
55+
return this.dependencies;
56+
}
57+
5158
public int getElementCount()
5259
{
5360
return this.elements.size();
@@ -78,6 +85,7 @@ public boolean equals(Object other)
7885

7986
ModuleManifest that = (ModuleManifest) other;
8087
return this.name.equals(that.name) &&
88+
this.dependencies.equals(that.dependencies) &&
8189
this.elements.equals(that.elements);
8290
}
8391

@@ -91,7 +99,9 @@ public int hashCode()
9199
public String toString()
92100
{
93101
StringBuilder builder = new StringBuilder("<").append(getClass().getSimpleName())
94-
.append(" moduleName='").append(this.name).append("' elements=[");
102+
.append(" moduleName='").append(this.name).append("' ");
103+
this.dependencies.appendString(builder, "dependencies=[", ", ", "] ");
104+
builder.append("elements=[");
95105
if (this.elements.notEmpty())
96106
{
97107
this.elements.forEach(e -> e.appendString(builder.append('{')).append("}, "));
@@ -175,9 +185,15 @@ public static Builder builder(String name)
175185
return builder().withModuleName(name);
176186
}
177187

188+
@Deprecated
178189
public static Builder builder(int elementCount)
179190
{
180-
return new Builder(elementCount);
191+
return builder(0, elementCount);
192+
}
193+
194+
public static Builder builder(int dependencyCount, int elementCount)
195+
{
196+
return new Builder(dependencyCount, elementCount);
181197
}
182198

183199
public static Builder builder(ModuleManifest metadata)
@@ -188,21 +204,25 @@ public static Builder builder(ModuleManifest metadata)
188204
public static class Builder
189205
{
190206
private String name;
207+
private final MutableList<String> dependencies;
191208
private final MutableList<ConcreteElementMetadata> elements;
192209

193210
private Builder()
194211
{
212+
this.dependencies = Lists.mutable.empty();
195213
this.elements = Lists.mutable.empty();
196214
}
197215

198-
private Builder(int elementCount)
216+
private Builder(int dependencyCount, int elementCount)
199217
{
218+
this.dependencies = Lists.mutable.ofInitialCapacity(dependencyCount);
200219
this.elements = Lists.mutable.ofInitialCapacity(elementCount);
201220
}
202221

203222
private Builder(ModuleManifest metadata)
204223
{
205224
this.name = metadata.getModuleName();
225+
this.dependencies = Lists.mutable.withAll(metadata.getDependencies());
206226
this.elements = Lists.mutable.withAll(metadata.getElements());
207227
}
208228

@@ -211,6 +231,46 @@ public void setModuleName(String name)
211231
this.name = name;
212232
}
213233

234+
public void addDependency(String dependency)
235+
{
236+
this.dependencies.add(Objects.requireNonNull(dependency, "dependency may not be null"));
237+
}
238+
239+
public void addDependencies(Iterable<? extends String> dependencies)
240+
{
241+
dependencies.forEach(this::addDependency);
242+
}
243+
244+
public void addDependencies(String... dependencies)
245+
{
246+
addDependencies(Arrays.asList(dependencies));
247+
}
248+
249+
public boolean removeDependency(String toRemove)
250+
{
251+
return removeDependencies(Sets.immutable.with(toRemove));
252+
}
253+
254+
public boolean removeDependencies(Iterable<? extends String> toRemove)
255+
{
256+
return removeDependencies(getRemoveDependencyPredicate(toRemove));
257+
}
258+
259+
public boolean removeDependencies(String... toRemove)
260+
{
261+
return removeDependencies(Sets.immutable.with(toRemove));
262+
}
263+
264+
public boolean removeDependencies(Predicate<? super String> toRemove)
265+
{
266+
return (toRemove != null) && this.dependencies.removeIf(toRemove::test);
267+
}
268+
269+
public void clearDependencies()
270+
{
271+
this.dependencies.clear();
272+
}
273+
214274
public void addElement(ConcreteElementMetadata element)
215275
{
216276
this.elements.add(Objects.requireNonNull(element, "element metadata may not be null"));
@@ -299,6 +359,54 @@ public Builder withModuleName(String name)
299359
return this;
300360
}
301361

362+
public Builder withDependency(String dependency)
363+
{
364+
addDependency(dependency);
365+
return this;
366+
}
367+
368+
public Builder withDependencies(Iterable<? extends String> dependencies)
369+
{
370+
addDependencies(dependencies);
371+
return this;
372+
}
373+
374+
public Builder withDependencies(String... dependencies)
375+
{
376+
addDependencies(dependencies);
377+
return this;
378+
}
379+
380+
public Builder withoutDependency(String toRemove)
381+
{
382+
removeDependency(toRemove);
383+
return this;
384+
}
385+
386+
public Builder withoutDependencies(Iterable<? extends String> toRemove)
387+
{
388+
removeDependencies(toRemove);
389+
return this;
390+
}
391+
392+
public Builder withoutDependencies(String... toRemove)
393+
{
394+
removeDependencies(toRemove);
395+
return this;
396+
}
397+
398+
public Builder withoutDependencies(Predicate<? super String> toRemove)
399+
{
400+
removeDependencies(toRemove);
401+
return this;
402+
}
403+
404+
public Builder withNoDependencies()
405+
{
406+
clearDependencies();
407+
return this;
408+
}
409+
302410
public Builder withElement(ConcreteElementMetadata element)
303411
{
304412
return withElement(element, false);
@@ -367,6 +475,7 @@ public Builder withoutElements(String... toRemove)
367475
public ModuleManifest build()
368476
{
369477
Objects.requireNonNull(this.name, "module name may not be null");
478+
ListHelper.sortAndRemoveDuplicates(this.dependencies);
370479
ListHelper.sortAndRemoveDuplicates(this.elements,
371480
Comparator.comparing(PackageableElementMetadata::getPath),
372481
(previous, current) ->
@@ -387,7 +496,7 @@ public ModuleManifest build()
387496

388497
private ModuleManifest buildNoValidation()
389498
{
390-
return new ModuleManifest(this.name, this.elements.toImmutable());
499+
return new ModuleManifest(this.name, this.dependencies.toImmutable(), this.elements.toImmutable());
391500
}
392501
}
393502

@@ -435,4 +544,29 @@ private static Predicate<ConcreteElementMetadata> getRemoveElementPredicate(Iter
435544
}
436545
}
437546
}
547+
548+
private static Predicate<String> getRemoveDependencyPredicate(Iterable<? extends String> toRemove)
549+
{
550+
if (toRemove == null)
551+
{
552+
return null;
553+
}
554+
Set<? extends String> set = (toRemove instanceof Set) ? (Set<? extends String>) toRemove : Sets.mutable.withAll(toRemove);
555+
switch (set.size())
556+
{
557+
case 0:
558+
{
559+
return null;
560+
}
561+
case 1:
562+
{
563+
String dependencyToRemove = Iterate.getFirst(set);
564+
return (dependencyToRemove == null) ? null : dependencyToRemove::equals;
565+
}
566+
default:
567+
{
568+
return set::contains;
569+
}
570+
}
571+
}
438572
}

0 commit comments

Comments
 (0)