Full graph generation with tunable clustering coefficient

Author: ArnauPrat

params.ini

@@ -1,15 +1,21 @@
 
-ldbc.snb.datagen.generator.scaleFactor:snb.interactive.1
+ldbc.snb.datagen.generator.numPersons:10000
+ldbc.snb.datagen.generator.startYear:2010
+ldbc.snb.datagen.generator.numYears:1
 
 ldbc.snb.datagen.serializer.compressed:false
 
-ldbc.snb.datagen.serializer.personSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVPersonSerializer
+ldbc.snb.datagen.serializer.personSerializer:ldbc.snb.datagen.serializer.graphalytics.CSVPersonSerializer
 
-ldbc.snb.datagen.serializer.invariantSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVInvariantSerializer
+ldbc.snb.datagen.serializer.invariantSerializer:ldbc.snb.datagen.serializer.empty.EmptyInvariantSerializer
 
-ldbc.snb.datagen.serializer.personActivitySerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVPersonActivitySerializer
+ldbc.snb.datagen.serializer.personActivitySerializer:ldbc.snb.datagen.serializer.empty.EmptyPersonActivitySerializer
+
+ldbc.snb.datagen.generator.ClusteringKnowsGenerator.clusteringCoefficient:0.1
 
 ldbc.snb.datagen.generator.numThreads:1
 
-ldbc.snb.datagen.serializer.updateStreams:true
+ldbc.snb.datagen.serializer.updateStreams:false
+
+ldbc.snb.datagen.generator.activity:false
 

src/main/java/ldbc/snb/datagen/generator/ClusteringKnowsGenerator.java

@@ -56,6 +56,7 @@ private class ClusteringInfo {
         public ArrayList<Long> community_core_stubs_ = new ArrayList<Long>();
         public ArrayList<Float> community_core_probs_ = new ArrayList<Float>();
         public ArrayList<Integer> core_nodes_ = new ArrayList<Integer>();
+        public ArrayList<Integer> community_id_ = new ArrayList<Integer>();
         public float sumProbs = 0.0f;
         public int numCommunities = 0;
 
@@ -67,17 +68,26 @@ private class ClusteringInfo {
                 core_node_expected_external_degree_.add(0.0);
                 is_core_.add(false);
                 clustering_coefficient_.add(0.0);
+                community_id_.add(0);
             }
             for( int i = 0; i < communities.size(); ++i) {
                 community_core_stubs_.add(0L);
                 community_core_probs_.add(0.0f);
             }
 
+            int index = 0;
             for( Community c: communities) {
                 for( PersonInfo pI : c.core_) {
                     core_nodes_.add(pI.index_);
                     is_core_.set(pI.index_, true);
+                    community_id_.set(pI.index_,index );
                 }
+
+                for( PersonInfo pI : c.periphery_) {
+                    is_core_.set(pI.index_, false);
+                    community_id_.set(pI.index_,index );
+                }
+                index++;
             }
 
             numCommunities = communities.size();
@@ -97,7 +107,7 @@ private Community findSolution( ArrayList<Person> persons, int begin, int last)
             PersonInfo pInfo = new PersonInfo();
             pInfo.index_ = i;
             pInfo.degree_ = Knows.target_edges(p,percentages,stepIndex);
-            pInfo.original_degree_ = (long)(p.maxNumKnows()*0.9);
+            pInfo.original_degree_ = (long)(p.maxNumKnows());
             nodes.add(pInfo);
         }
 
@@ -151,6 +161,12 @@ private Community checkBudget(ArrayList<Person> persons, ArrayList<PersonInfo> c
         return community;
     }
 
+    private void testCommunity(Community c) {
+        for(PersonInfo pI : c.core_ ) {
+            if(pI.degree_ < (c.core_.size() - 1))  System.out.println("Error in building communities\n");
+        }
+    }
+
     private  ArrayList<Community> generateCommunities( ArrayList<Person> persons) {
         ArrayList<Community> communities = new ArrayList<Community>();
         int last = 0;
@@ -173,6 +189,7 @@ private  ArrayList<Community> generateCommunities( ArrayList<Person> persons) {
             }
             bestCommunity.id_ = communities.size();
             communities.add(bestCommunity);
+            testCommunity(bestCommunity);
 
             last = best + 1;
             begin = last;
@@ -246,21 +263,23 @@ private void estimateCCCommunity( ClusteringInfo cInfo, Community c, float prob
         float probTwoConnected  = 0.0f;
         for( Integer i : cInfo.core_nodes_ ) {
             double degree1 = cInfo.core_node_expected_external_degree_.get(i);
-            if(degree1 > 0) {
+            if(degree1 >= 1) {
                 for (Integer ii : cInfo.core_nodes_) {
-                    double degree2 = cInfo.core_node_expected_external_degree_.get(ii);
-                    if(degree2 > 0)
-                        probTwoConnected += degree1 * degree2 / (float) (2 * sumStubs * sumStubs);
+                    if(cInfo.community_id_.get(i) != cInfo.community_id_.get(i)) {
+                        double degree2 = cInfo.core_node_expected_external_degree_.get(ii);
+                        if (degree2 >= 1)
+                            probTwoConnected += degree1 * degree2 / (float) (2 * sumStubs * sumStubs);
+                    }
                 }
             }
         }
 
         // Computing clustering coefficient of periphery nodes
         for (PersonInfo pI: c.periphery_) {
             if(pI.degree_ > 1) {
-                //cInfo.clustering_coefficient_.set(pI.index_, (double)pI.degree_*(pI.degree_-1)*prob/(pI.original_degree_*(pI.original_degree_-1)));
+                cInfo.clustering_coefficient_.set(pI.index_, (double)pI.degree_*(pI.degree_-1)*prob/(pI.original_degree_*(pI.original_degree_-1)));
                 //cInfo.clustering_coefficient_.set(pI.index_, (double)prob);
-                cInfo.clustering_coefficient_.set(pI.index_, 0.0);
+                //cInfo.clustering_coefficient_.set(pI.index_, 0.0);
             }
         }
 
@@ -278,32 +297,11 @@ private void estimateCCCommunity( ClusteringInfo cInfo, Community c, float prob
                 // core core triangles
                 double internalTriangles = 0.0;
                 double internalDegree = cInfo.core_node_expected_core_degree_.get(pI.index_);
-                /*if(size > 2) {
-                    internalTriangles = ((size - 1) * (size - 2) * Math.pow(prob, 3));
-                }*/
-                if(internalDegree > 1.0) {
+
+                if(internalDegree >= 2.0) {
                     internalTriangles = (internalDegree * (internalDegree - 1) * prob);
                 }
                 boolean enteredOffset = false;
-                /*if( internalDegree >= 2.0 ) {
-                    internalTriangles = (internalDegree * (internalDegree - 1) * prob);
-                } else if( internalDegree > 1.0) {
-                    double offset = internalDegree - 1.0;
-                    double p = rand.nextDouble();
-                    if(p < offset) {
-                        internalTriangles = prob;
-                        enteredOffset = true;
-                    }
-                }*/
-                /*long finalInternalDegree = (long)internalDegree;
-                if( internalDegree > 1 ) {
-                    double offset = internalDegree - 1.0;
-                    double p = rand.nextDouble();
-                    if(p < offset) {
-                        finalInternalDegree++;
-                    }
-                }*/
-                //internalTriangles = (finalInternalDegree * (finalInternalDegree - 1) * prob);
 
                 // core periphery triangles
                 double peripheryTriangles = 0;
@@ -325,21 +323,18 @@ private void estimateCCCommunity( ClusteringInfo cInfo, Community c, float prob
                     external_triangles += cInfo.core_node_expected_external_degree_.get(pI.index_) * (cInfo.core_node_expected_external_degree_.get(pI.index_) - 1) * (1 - probSameCommunity) * probTwoConnected;
                 }
 
+
                 //double degree = finalInternalDegree;
-                double degree = (cInfo.core_node_expected_core_degree_.get(pI.index_) /*+
+                /*double degree = (cInfo.core_node_expected_core_degree_.get(pI.index_) +
                         cInfo.core_node_expected_periphery_degree_.get(pI.index_) +
-                        cInfo.core_node_expected_external_degree_.get(pI.index_)*/);
+                        cInfo.core_node_expected_external_degree_.get(pI.index_));*/
 
-                //double degree = pI.original_degree_;
+                double degree = pI.original_degree_;
 
                 //System.out.println("Internal Triangles: "+internalTriangles+" , degree: "+degree);
-                if( degree > 1.0 ) {
-                    cInfo.clustering_coefficient_.set(pI.index_, (internalTriangles/*+peripheryTriangles+external_triangles*/)/(degree*(degree-1)));
+                if( degree >= 2.0 ) {
+                    cInfo.clustering_coefficient_.set(pI.index_, (internalTriangles+peripheryTriangles+external_triangles)/(degree*(degree-1)));
                 }
-                //else if( degree > 1.0 && enteredOffset ) {
-                //    degree = 2.0;
-                //    cInfo.clustering_coefficient_.set(pI.index_, (internalTriangles+peripheryTriangles+external_triangles)/(double)(degree*(degree-1)));
-                //}
             }
         }
     }
@@ -377,7 +372,7 @@ void refineCommunities( ClusteringInfo cInfo, ArrayList<Community> communities,
         int lookAhead = 5;
         int tries = 0;
         while( Math.abs(currentCC - targetCC)  > 0.001 && tries <= lookAhead) {
-            System.out.println(currentCC);
+         //   System.out.println(currentCC);
             boolean found = false;
             tries+=1;
             if( currentCC < targetCC ) {
@@ -537,6 +532,7 @@ public void generateKnows( ArrayList<Person> persons, int seed, ArrayList<Float>
 
         for( Community c : communities ) {
             c.p_ = 1.0f;//rand.nextFloat();
+            //c.p_ = rand.nextFloat();
             estimateCCCommunity(cInfo, c, c.p_ );
         }
 
@@ -551,12 +547,12 @@ public void generateKnows( ArrayList<Person> persons, int seed, ArrayList<Float>
             System.out.println("Creating graph");
             for(Community c : communities ) {
                 createEdgesCommunityCore(persons, c);
-                //createEdgesCommunityPeriphery(cInfo, persons,c);
+                createEdgesCommunityPeriphery(cInfo, persons,c);
             }
-            //fillGraphWithRemainingEdges(cInfo, communities, persons);
+            fillGraphWithRemainingEdges(cInfo, communities, persons);
             graph = new PersonGraph(persons);
             System.out.println("Computing clustering coefficient");
-            /*double finalCC = 0;
+            double finalCC = 0;
             ArrayList<Double> clusteringCoefficient = GraphUtils.ClusteringCoefficientList(graph);
             int i = 0;
             for( Person p : persons) {
@@ -567,16 +563,16 @@ public void generateKnows( ArrayList<Person> persons, int seed, ArrayList<Float>
                 i++;
             }
             finalCC /= persons.size();
-            */
-            double finalCC = GraphUtils.ClusteringCoefficient(graph);
+            //double finalCC = GraphUtils.ClusteringCoefficient(graph);
 
             System.out.println("Clustering coefficient of the generated graph: "+finalCC);
             double delta = targetCC - finalCC;
             if( Math.abs( delta ) > 0.001 ) {
+                resetStatistics();
                 for(Person person: persons) {
                     person.knows().clear();
                 }
-                fakeTargetCC +=  delta;
+                fakeTargetCC +=  delta*0.8f;
                 System.out.println("New Fake targetCC: "+fakeTargetCC );
                 iterate = true;
             }
@@ -617,7 +613,14 @@ public void generateKnows( ArrayList<Person> persons, int seed, ArrayList<Float>
     public void initialize( Configuration conf ) {
         targetCC = conf.getFloat("ldbc.snb.datagen.generator.ClusteringKnowsGenerator.clusteringCoefficient", 0.1f);
         System.out.println("Initialized clustering coefficient to "+targetCC);
-        //targetCC /= 2.0f;
+        targetCC /= 2.0f;
+    }
+
+    public void resetStatistics() {
+        numCoreCoreEdges = 0;
+        numCorePeripheryEdges = 0;
+        numCoreExternalEdges = 0;
+        numMisses = 0;
     }
 
     public void printStatistics() {

src/main/java/ldbc/snb/datagen/generator/LDBCDatagen.java

@@ -98,7 +98,7 @@ public int runGenerateJob(Configuration conf) throws Exception {
         knowsGenerator.run(hadoopPrefix+"/persons",hadoopPrefix+"/universityEdges");
         long endUniversity = System.currentTimeMillis();
 
-        /*
+
         printProgress("Creating main interest correlated edges");
         long startInterest= System.currentTimeMillis();
 
@@ -113,6 +113,7 @@ public int runGenerateJob(Configuration conf) throws Exception {
         long endInterest = System.currentTimeMillis();
 
 
+
         printProgress("Creating random correlated edges");
         long startRandom= System.currentTimeMillis();
 
@@ -125,16 +126,16 @@ public int runGenerateJob(Configuration conf) throws Exception {
 
         knowsGenerator.run(hadoopPrefix+"/persons",hadoopPrefix+"/randomEdges");
         long endRandom= System.currentTimeMillis();
-        */
+
 
 
 
         fs.delete(new Path(DatagenParams.hadoopDir + "/persons"), true);
         printProgress("Merging the different edge files");
         ArrayList<String> edgeFileNames = new ArrayList<String>();
         edgeFileNames.add(hadoopPrefix+"/universityEdges");
-        //edgeFileNames.add(hadoopPrefix+"/interestEdges");
-        //edgeFileNames.add(hadoopPrefix+"/randomEdges");
+        edgeFileNames.add(hadoopPrefix+"/interestEdges");
+        edgeFileNames.add(hadoopPrefix+"/randomEdges");
         long startMerge = System.currentTimeMillis();
         HadoopMergeFriendshipFiles merger = new HadoopMergeFriendshipFiles(conf,"ldbc.snb.datagen.hadoop.RandomKeySetter");
         merger.run(hadoopPrefix+"/mergedPersons", edgeFileNames);