some methods for dataset analysis added

Author: domkowald

src/common/Bookmark.java

@@ -113,6 +113,38 @@ public List<Integer> getTags() {
 
 	// Statics ----------------------------------------------------------------------------------
 
+	public static double getAvgNumberOfTopics(List<Bookmark> lines) {
+		double sum = 0.0;
+		for (Bookmark line : lines) {
+			sum += line.getCategories().size();
+		}
+		return sum / lines.size();
+	}
+	
+	public static double getBookmarkDiversity(List<Bookmark> lines) {
+		if (lines.size() == 0) {
+			return 0.0;
+		}
+		if (lines.size() == 1) {
+			return 1.0;
+		}
+		
+		double diversity = 0.0;
+		int size = 0;
+		for (int i = 0; i < lines.size() - 1; i++) {
+			for (int j = i + 1; j < lines.size(); j++) {
+				Bookmark srcBookmark = lines.get(i);
+				Bookmark destBookmark = lines.get(j);
+				diversity += Utilities.getJaccardSimLists(srcBookmark.getCategories(), destBookmark.getCategories());
+				size++;
+			}
+		}
+		if (size == 0) {
+			return 0.0;
+		}
+		return diversity / size;
+	}
+	
 	public static Bookmark getUserData(List<Bookmark> lines, int userID, int resID) {
 		Bookmark returnData = null;
 		for (Bookmark data : lines) {

src/common/Utilities.java

@@ -424,6 +424,8 @@ public static double getJaccardSimLists(List<Integer> targetMap, List<Integer> n
 		Set<Integer> intersectSet = new HashSet<Integer>(targetMap);
 		unionSet.addAll(nMap);
 		intersectSet.retainAll(nMap);
+		if (intersectSet.size() == 0 || unionSet.size() == 0)
+			return 0.0;
 		return (double)intersectSet.size() / (double)unionSet.size();
 	}
 
@@ -432,6 +434,8 @@ public static double getJaccardFloatSim(Map<Integer, Double> targetMap, Map<Inte
 		Set<Integer> intersectSet = new HashSet<Integer>(targetMap.keySet());
 		unionSet.addAll(nMap.keySet());
 		intersectSet.retainAll(nMap.keySet());
+		if (intersectSet.size() == 0 || unionSet.size() == 0)
+			return 0.0;
 		return (double)intersectSet.size() / (double)unionSet.size();
 	}
 
@@ -442,6 +446,8 @@ public static double getCosineSim(Map<Integer, Integer> targetMap, Map<Integer,
         for (int k : both) scalar += (targetMap.get(k) * nMap.get(k));
         for (int k : targetMap.keySet()) norm1 += (targetMap.get(k) * targetMap.get(k));
         for (int k : nMap.keySet()) norm2 += (nMap.get(k) * nMap.get(k));
+        if (Math.sqrt(norm1 * norm2) == 0.0)
+        	return 0.0;
         return scalar / Math.sqrt(norm1 * norm2);
 	}
 
@@ -454,6 +460,8 @@ public static double getCosineSimList(List<Integer> targetList, List<Integer> nL
         for (int k : both) scalar += (targetMap.get(k) * nMap.get(k));
         for (int k : targetMap.keySet()) norm1 += (targetMap.get(k) * targetMap.get(k));
         for (int k : nMap.keySet()) norm2 += (nMap.get(k) * nMap.get(k));
+        if (Math.sqrt(norm1 * norm2) == 0.0)
+        	return 0.0;
         return scalar / Math.sqrt(norm1 * norm2);
 	}
 
@@ -464,6 +472,8 @@ public static double getCosineFloatSim(Map<Integer, Double> targetMap, Map<Integ
         for (int k : both) scalar += (targetMap.get(k) * nMap.get(k));
         for (int k : targetMap.keySet()) norm1 += (targetMap.get(k) * targetMap.get(k));
         for (int k : nMap.keySet()) norm2 += (nMap.get(k) * nMap.get(k));
+        if (Math.sqrt(norm1 * norm2) == 0.0)
+        	return 0.0;
         return scalar / Math.sqrt(norm1 * norm2);
 	}
 

src/test/Pipeline.java

@@ -24,12 +24,19 @@ License, or (at your option) any later version.
 import itemrecommendations.Resource3LTCalculator;
 import itemrecommendations.ZhengCalculator;
 
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 
+import common.Bookmark;
 import common.CalculationType;
 import common.Features;
+import common.Utilities;
 import processing.BLLCalculator;
 import processing.BM25Calculator;
 import processing.BaselineCalculator;
@@ -65,9 +72,9 @@ public class Pipeline {
 	// set for categorizer/describer split (true is describer, false is categorizer - null for nothing)
 	private final static Boolean DESCRIBER = null;
 	// placeholder for the topic posfix
-	private static String TOPIC_NAME = null;
+	private static String TOPIC_NAME = "lda_500";
 	// placeholder for the used dataset
-	private final static String DATASET = "cul";
+	private final static String DATASET = "lastfm";
 
 	public static void main(String[] args) {
 		System.out.println("TagRecommender:\n" + "" +
@@ -88,7 +95,7 @@ public static void main(String[] args) {
 		// Resource-Recommender testing
 		String dir = DATASET + "_core";
 		String path = dir + "/" + DATASET + "_sample";
-		//getStatistics(path);
+		//try { getStatistics(path, true); } catch (IOException e) { e.printStackTrace(); }
 		//writeTensorFiles(path, false);
 		//evaluate(dir, path, "wrmf_500_mml", TOPIC_NAME, false, true);
 		//createLdaSamples(path, 1, 500, false);
@@ -477,19 +484,57 @@ private static List<Integer> getBetaValues(int betaUpperBound) {
 		return betaValues;
 	}
 
-	private static void getStatistics(String dataset) {
+	private static void getStatistics(String dataset, boolean writeAll) throws IOException {
+		if (TOPIC_NAME != null) {
+			dataset += ("_" + TOPIC_NAME);
+		}
 		BookmarkReader reader = new BookmarkReader(0, false);
 		reader.readFile(dataset);
+		
 		int bookmarks = reader.getBookmarks().size();
-		System.out.println("Bookmarks: " + bookmarks);
+		System.out.println("Posts: " + bookmarks);
 		int users = reader.getUsers().size();
 		System.out.println("Users: " + users);
 		int resources = reader.getResources().size();
 		System.out.println("Resources: " + resources);
 		int tags = reader.getTags().size();
 		System.out.println("Tags: " + tags);
 		int tagAssignments = reader.getTagAssignmentsCount();
-		System.out.println("Tag-Assignments: " + tagAssignments);
+		System.out.println("Tag-Assignments: " + tagAssignments);		
+		int categories = reader.getCategories().size();
+		System.out.println("Topics: " + categories);
+		double avgBookmarksPerUser = (double)bookmarks / users;
+		System.out.println("Avg. resources/posts per user: " + avgBookmarksPerUser);
+		double avgBookmarksPerResource = (double)bookmarks / resources;
+		System.out.println("Avg. users/posts per resource: " + avgBookmarksPerResource);
+		
+		if (writeAll) {
+			getTrainTestSize(dataset);
+			FileWriter userWriter = new FileWriter(new File("./data/metrics/" + dataset + "_userStats.txt"));
+			BufferedWriter userBW = new BufferedWriter(userWriter);
+			userBW.write("UserID| NoOfResources| NoOfTopics| Topic-Similarity\n");
+			List<Bookmark> trainList = reader.getBookmarks().subList(0, TRAIN_SIZE);
+			List<Integer> testUsers = reader.getUniqueUserListFromTestSet(TRAIN_SIZE);
+			System.out.println();
+
+			double avgTopicsPerUser = 0.0;
+			double avgTopicDiversityPerUser = 0.0;
+			List<Map<Integer, Double>> userTopics = Utilities.getRelativeTopicMaps(trainList, false);
+			List<List<Bookmark>> userBookmarks = Utilities.getBookmarks(trainList, false);
+			for (int userID : testUsers) {
+				Map<Integer, Double> topicsOfUser = userTopics.get(userID);
+				double topicDiversityOfUser = Bookmark.getBookmarkDiversity(userBookmarks.get(userID));
+				userBW.write(userID + "| " + reader.getUserCounts().get(userID) + "| " + topicsOfUser.keySet().size() + "| " + topicDiversityOfUser + "\n");
+				avgTopicsPerUser += topicsOfUser.keySet().size();
+				avgTopicDiversityPerUser += topicDiversityOfUser;
+			}
+			System.out.println("Avg. topics per user: " + avgTopicsPerUser / testUsers.size());
+			System.out.println("Avg. topic-similarity per user: " + avgTopicDiversityPerUser / testUsers.size());
+			double avgTopicsPerResource = Bookmark.getAvgNumberOfTopics(trainList);
+			System.out.println("Avg. topics per resource: " + avgTopicsPerResource);
+			userBW.flush();
+			userBW.close();
+		}
 	}
 
 	private static void getTrainTestSize(String sample) {