Skip to content

Commit 2f5cd58

Browse files
ArnauPratArnauPrat
committed
Added scale factors for graphalytics
Changed the way configuration is parsed in order to guarantee that scalfactor options are loaded first
1 parent d6071c7 commit 2f5cd58

File tree

2 files changed

+120
-8
lines changed

2 files changed

+120
-8
lines changed

‎src/main/java/ldbc/snb/datagen/util/ConfigParser.java

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ public static Configuration initialize() {
6969
String value = property.getElementsByTagName("value").item(0).getTextContent();
7070
scaleFactor.properties.put(name,value);
7171
}
72-
System.out.println("Loaded scale factor configuration set "+scaleFactorName);
72+
System.out.println("Available scale factor configuration set "+scaleFactorName);
7373
scaleFactors.put(scaleFactorName, scaleFactor);
7474
}
7575
}
@@ -86,17 +86,21 @@ public static Configuration readConfig(Configuration conf, String paramsFile) {
8686
try {
8787
Properties properties = new Properties();
8888
properties.load(new InputStreamReader(new FileInputStream(paramsFile), "UTF-8"));
89+
String val = (String) properties.get("ldbc.snb.datagen.generator.scaleFactor");
90+
if( val != null ) {
91+
ScaleFactor scaleFactor = scaleFactors.get(val);
92+
System.out.println("Applied configuration of scale factor " + val);
93+
for( Map.Entry<String,String> e : scaleFactor.properties.entrySet()) {
94+
conf.set(e.getKey(), e.getValue());
95+
}
96+
}
97+
8998
for( String s : properties.stringPropertyNames()) {
90-
if(s.compareTo("ldbc.snb.datagen.generator.scaleFactor") == 0) {
91-
ScaleFactor scaleFactor = scaleFactors.get(properties.get(s));
92-
System.out.println("Applied configuration of scale factor " + properties.get(s));
93-
for( Map.Entry<String,String> e : scaleFactor.properties.entrySet()) {
94-
conf.set(e.getKey(), e.getValue());
95-
}
96-
} else {
99+
if(s.compareTo("ldbc.snb.datagen.generator.scaleFactor") != 0) {
97100
conf.set(s, properties.getProperty(s));
98101
}
99102
}
103+
100104
if (conf.get("fs.defaultFS").compareTo("file:///") == 0) {
101105
System.out.println("Running in standalone mode. Setting numThreads to 1");
102106
conf.set("ldbc.snb.datagen.generator.numThreads", "1");

‎src/main/resources/scale_factors.xml

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,4 +132,112 @@
132132
<value>ldbc.snb.datagen.generator.distribution.FacebookDegreeDistribution</value>
133133
</property>
134134
</scale_factor>
135+
136+
<scale_factor name="graphalytics.10" >
137+
<property>
138+
<name>ldbc.snb.datagen.generator.numPersons</name>
139+
<value>250000</value>
140+
</property>
141+
<property>
142+
<name>ldbc.snb.datagen.generator.startYear</name>
143+
<value>2010</value>
144+
</property>
145+
<property>
146+
<name>ldbc.snb.datagen.generator.numYears</name>
147+
<value>1</value>
148+
</property>
149+
<property>
150+
<name>ldbc.snb.datagen.generator.distribution.degreeDistribution</name>
151+
<value>ldbc.snb.datagen.generator.distribution.FacebookDegreeDistribution</value>
152+
</property>
153+
<property>
154+
<name>ldbc.snb.datagen.generator.activity</name>
155+
<value>false</value>
156+
</property>
157+
<property>
158+
<name>ldbc.snb.datagen.serializer.updateStreams</name>
159+
<value>false</value>
160+
</property>
161+
</scale_factor>
162+
163+
<scale_factor name="graphalytics.30" >
164+
<property>
165+
<name>ldbc.snb.datagen.generator.numPersons</name>
166+
<value>650000</value>
167+
</property>
168+
<property>
169+
<name>ldbc.snb.datagen.generator.startYear</name>
170+
<value>2010</value>
171+
</property>
172+
<property>
173+
<name>ldbc.snb.datagen.generator.numYears</name>
174+
<value>1</value>
175+
</property>
176+
<property>
177+
<name>ldbc.snb.datagen.generator.distribution.degreeDistribution</name>
178+
<value>ldbc.snb.datagen.generator.distribution.FacebookDegreeDistribution</value>
179+
</property>
180+
<property>
181+
<name>ldbc.snb.datagen.generator.activity</name>
182+
<value>false</value>
183+
</property>
184+
<property>
185+
<name>ldbc.snb.datagen.serializer.updateStreams</name>
186+
<value>false</value>
187+
</property>
188+
</scale_factor>
189+
190+
<scale_factor name="graphalytics.100" >
191+
<property>
192+
<name>ldbc.snb.datagen.generator.numPersons</name>
193+
<value>1800000</value>
194+
</property>
195+
<property>
196+
<name>ldbc.snb.datagen.generator.startYear</name>
197+
<value>2010</value>
198+
</property>
199+
<property>
200+
<name>ldbc.snb.datagen.generator.numYears</name>
201+
<value>1</value>
202+
</property>
203+
<property>
204+
<name>ldbc.snb.datagen.generator.distribution.degreeDistribution</name>
205+
<value>ldbc.snb.datagen.generator.distribution.FacebookDegreeDistribution</value>
206+
</property>
207+
<property>
208+
<name>ldbc.snb.datagen.generator.activity</name>
209+
<value>false</value>
210+
</property>
211+
<property>
212+
<name>ldbc.snb.datagen.serializer.updateStreams</name>
213+
<value>false</value>
214+
</property>
215+
</scale_factor>
216+
217+
<scale_factor name="graphalytics.300" >
218+
<property>
219+
<name>ldbc.snb.datagen.generator.numPersons</name>
220+
<value>4800000</value>
221+
</property>
222+
<property>
223+
<name>ldbc.snb.datagen.generator.startYear</name>
224+
<value>2010</value>
225+
</property>
226+
<property>
227+
<name>ldbc.snb.datagen.generator.numYears</name>
228+
<value>1</value>
229+
</property>
230+
<property>
231+
<name>ldbc.snb.datagen.generator.distribution.degreeDistribution</name>
232+
<value>ldbc.snb.datagen.generator.distribution.FacebookDegreeDistribution</value>
233+
</property>
234+
<property>
235+
<name>ldbc.snb.datagen.generator.activity</name>
236+
<value>false</value>
237+
</property>
238+
<property>
239+
<name>ldbc.snb.datagen.serializer.updateStreams</name>
240+
<value>false</value>
241+
</property>
242+
</scale_factor>
135243
</scale_factors>

0 commit comments

Comments
 (0)