nothing

Author: alexaverbuch

.project

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>ldbc_snb_datagen</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.m2e.core.maven2Builder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+		<nature>org.eclipse.m2e.core.maven2Nature</nature>
+	</natures>
+</projectDescription>

paramgenerator/discoverparams.pyc

@@ -34,8 +34,6 @@ def findNameParameters(names, amount = 100):
 		i -= 1
 	return res
 
-
-
 class CSVSerializer:
 	def __init__(self):
 		self.handlers = []
@@ -130,19 +128,23 @@ def main(argv=None):
 		return 1
 
 	indir = argv[1]+"/"
-	factorFiles=[]
+	activityFactorFiles=[]
+	personFactorFiles=[]
 	friendsFiles = []
 	outdir = argv[2]+"/"
 	random.seed(SEED)
 
+
 	for file in os.listdir(indir):
-		if file.endswith("factors.txt"):
-			factorFiles.append(indir+file)
+		if file.endswith("activityFactors.txt"):
+			activityFactorFiles.append(indir+file)
+		if file.endswith("personFactors.txt"):
+			personFactorFiles.append(indir+file)
 		if file.startswith("m0friendList"):
 			friendsFiles.append(indir+file)
 
 	# read precomputed counts from files	
-	(personFactors, countryFactors, tagFactors, tagClassFactors, nameFactors, givenNames,  ts, postHisto) = readfactors.load(factorFiles, friendsFiles)
+	(personFactors, countryFactors, tagFactors, tagClassFactors, nameFactors, givenNames,  ts, postHisto) = readfactors.load(personFactorFiles, activityFactorFiles, friendsFiles)
 
 	# find person parameters
 	print "find parameter bindings for Persons"
@@ -234,7 +236,7 @@ def main(argv=None):
 	}
 
 	print "find parameter bindings for Timestamps"
-	selectedTimeParams = findTimeParams(timeSelectionInput, factorFiles, friendsFiles, ts[1])
+	selectedTimeParams = findTimeParams(timeSelectionInput, personFactorFiles, activityFactorFiles, friendsFiles, ts[1])
 	# Query 11 takes WorksFrom timestamp
 	selectedTimeParams[11] = [random.randint(ts[2], ts[3]) for j in range(len(selectedPersonParams[11]))]
 

paramgenerator/generateparams.py

@@ -299,18 +299,21 @@ def main(argv=None):
       return 1
 
    indir = argv[1]+"/"
-   factorFiles=[]
+   activityFactorFiles=[]
+   personFactorFiles=[]
    friendsFiles = []
    outdir = argv[2]+"/"
 
    for file in os.listdir(indir):
-      if file.endswith("factors.txt"):
-         factorFiles.append(indir+file)
+      if file.endswith("activityFactors.txt"):
+         activityFactorFiles.append(indir+file)
+      if file.endswith("personFactors.txt"):
+         personFactorFiles.append(indir+file)
       if file.startswith("m0friendList"):
          friendsFiles.append(indir+file)
 
    # read precomputed counts from files   
-   (personFactors, countryFactors, tagFactors, tagClassFactors, nameFactors, givenNames,  ts, postsHisto) = readfactors.load(factorFiles, friendsFiles)
+   (personFactors, countryFactors, tagFactors, tagClassFactors, nameFactors, givenNames,  ts, postsHisto) = readfactors.load(personFactorFiles,activityFactorFiles, friendsFiles)
    week_posts = convert_posts_histo(postsHisto)
 
    persons = []

paramgenerator/generateparamsbi.py

@@ -53,7 +53,7 @@ def setValue(self, person, value):
 	def getValue(self, person):
 		return self.values[person]
 
-def load(factorFiles, friendFiles):
+def load(personFactorFiles,activityFactorFiles, friendFiles):
 	print "loading input for parameter generation"
 	results = Factors()
 	countries = Factors()
@@ -65,12 +65,10 @@ def load(factorFiles, friendFiles):
 	names = {}
 	timestamp = [0,0,0,0]
 
-	for inputFileName in factorFiles:
-		with codecs.open(inputFileName, "r", "utf-8") as f:
-			line = f.readline()
-			personCount = int(line)
-			for i in range(personCount):
-				line = f.readline().split(",")
+	for inputfileName in personFactorFiles:
+		with codecs.open(inputfileName, "r", "utf-8") as f:
+			for line in f.readlines():
+				line = line.split(",")
 				person = int(line[0])
 				if not results.existParam(person):
 					results.addNewParam(person)
@@ -88,6 +86,8 @@ def load(factorFiles, friendFiles):
 						postsHisto.addNewParam(i)
 					postsHisto.addValue(i, "p", int(line[9+i]))
 
+	for inputFileName in activityFactorFiles:
+		with codecs.open(inputFileName, "r", "utf-8") as f:
 			countryCount = int(f.readline())
 			for i in range(countryCount):
 				line = f.readline().split(",")

paramgenerator/readfactors.py

@@ -143,18 +143,17 @@ def computeTimeMedians(factors, lastmonthcount = LAST_MONTHS):
 
 	return (medianFirstMonth, medianLastMonth, median)	
 
-def readTimeParams(persons, factorFiles, friendFiles):
+def readTimeParams(persons, personFactorFiles, activityFactorFiles, friendFiles):
 
 	postCounts = {}
 	groupCounts = {}
 	offset = 8
 	monthcount = 12*3 + 1
 
-	for inputFactorFile in factorFiles:
+	for inputFactorFile in personFactorFiles:
 		with open(inputFactorFile, 'r') as f:
-			personCount = int(f.readline())
-			for i in range(personCount):
-				line = f.readline().split(",")
+			for line in f.readlines():
+				line = line.split(",")
 				person = int(line[0])
 				localPostCounts = map(int,line[offset:offset+monthcount])
 				localGroupCounts = map(int, line[offset+monthcount:])
@@ -209,15 +208,15 @@ def readTimeParams(persons, factorFiles, friendFiles):
 
 
 
-def findTimeParams(input, factorFiles, friendFiles, startYear):
+def findTimeParams(input, personFactorFiles, activityFactorFiles, friendFiles, startYear):
 	START_YEAR = startYear
 	fPostCount = {}
 	ffPostCount = {}
 	persons = []
 	for queryId in input:
 		persons += input[queryId][0]
 
-	(fPostCount, ffPostCount, ffGroupCount) = readTimeParams(set(persons),factorFiles, friendFiles)
+	(fPostCount, ffPostCount, ffGroupCount) = readTimeParams(set(persons),personFactorFiles, activityFactorFiles, friendFiles)
 
 	mapParam = {
 		"f" : fPostCount,

paramgenerator/readfactors.pyc

@@ -1,8 +1,4 @@
-# ldbc.snb.datagen.generator.scaleFactor:snb.interactive.1
-
-ldbc.snb.datagen.generator.numPersons:3000
-ldbc.snb.datagen.generator.numYears:1
-ldbc.snb.datagen.generator.startYear:2010
+ldbc.snb.datagen.generator.scaleFactor:snb.interactive.1
 
 ldbc.snb.datagen.serializer.compressed:false
 

paramgenerator/timeparameters.py

@@ -32,8 +32,10 @@ then
 	mkdir -p substitution_parameters
 	python paramgenerator/generateparams.py $LDBC_SNB_DATAGEN_HOME substitution_parameters/
 	python paramgenerator/generateparamsbi.py $LDBC_SNB_DATAGEN_HOME substitution_parameters/
-  rm -f m*factors*
-  rm -f .m*factors*
+  rm -f m*personFactors*
+  rm -f .m*personFactors*
+  rm -f m*activityFactors*
+  rm -f .m*activityFactors*
   rm -f m0friendList*
   rm -f .m0friendList*
 fi