fixed 3L for bits and pieces study

domkowald · domkowald · commit bac74d39b07c · 2016-02-25T10:02:57.000+01:00
diff --git a/src/common/SolrConnector.java b/src/common/SolrConnector.java
@@ -17,6 +17,8 @@
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.params.MoreLikeThisParams;
 
+import processing.hashtag.Tweet;
+
 public class SolrConnector {
 	
 	private SolrServer server;
@@ -47,6 +49,27 @@ public Map<String, Set<String>> getTweets() {
 		return tweets;
 	}
 	
+	@SuppressWarnings("unchecked")
+	public List<Tweet> getTweetObjects() {
+		List<Tweet> tweetObjects = new ArrayList<Tweet>();
+		
+		SolrQuery solrParams = new SolrQuery();
+		solrParams.set("q", "*:*");
+		solrParams.set("rows", Integer.MAX_VALUE);
+		QueryResponse r = null;
+		try {
+			r = this.server.query(solrParams);
+			SolrDocumentList docs = r.getResults();
+			for (SolrDocument d : docs) {
+				tweetObjects.add(new Tweet((String) d.get("id"), (String) d.get("userid"), (String) d.get("text"), (String) d.get("timestamp"), new HashSet<String>((List<String>) d.get("hashtags"))));
+			}
+		} catch (SolrServerException e) {
+			e.printStackTrace();
+		}
+		
+		return tweetObjects;
+	}
+	
 	@SuppressWarnings("unchecked")
 	public Map<String, Set<String>> getUserIDs() {
 		Map<String, Set<String>> tweetIDs = new LinkedHashMap<String, Set<String>>();
diff --git a/src/common/TimeUtil.java b/src/common/TimeUtil.java
@@ -1,4 +1,4 @@
-package processing;
+package common;
 
 public class TimeUtil {
     
diff --git a/src/engine/EngineUtils.java b/src/engine/EngineUtils.java
@@ -39,7 +39,7 @@ public class EngineUtils {
 
 	public static BookmarkReader getSortedBookmarkReader(String path, String filename) {
 		BookmarkReader reader = new BookmarkReader(0, false);
-		reader.readFile(path, filename);
+		reader.readFile(path, filename);		
 		Collections.sort(reader.getBookmarks());
 		
 		String sortedFile = filename + "_sorted";
diff --git a/src/engine/ThreeLayersCollectiveEngine.java b/src/engine/ThreeLayersCollectiveEngine.java
@@ -30,7 +30,8 @@ public synchronized Map<String, Double> getEntitiesWithLikelihood(String user, S
 		List<Integer> topicIDs = new ArrayList<>();
 		if (topics != null) {
 			for (String t : topics) {
-				int tID = this.reader.getCategories().indexOf(t);
+				List<String> categories = this.reader.getCategories();
+				int tID = categories.indexOf(t.toLowerCase());
 				if (tID != -1) {
 					topicIDs.add(tID);
 				}
diff --git a/src/file/BookmarkReader.java b/src/file/BookmarkReader.java
@@ -131,11 +131,11 @@ private boolean doReadFile(String path, String filename) {
 				if (lineParts.length > 4) { // are there categories
 					for (String cat : lineParts[4].replace("\"", "").split(",")) {
 						if (!cat.isEmpty()) {
-							if (cat.contains("_")) {
-								categories.add(cat.substring(0, cat.indexOf("_")).toLowerCase());
-							} else {
+							//if (cat.contains("_")) {
+							//	categories.add(cat.substring(0, cat.indexOf("_")).toLowerCase());
+							//} else {
 								categories.add(cat.toLowerCase());
-							}
+							//}
 						}
 					}
 				}
@@ -290,6 +290,10 @@ public List<Integer> getTagCounts() {
 		return this.tagCounts;
 	}
 	
+	public Map<String, Integer> getTagMap() {
+		return this.tagMap;
+	}
+	
 	public List<String> getResources() {
 		return this.resources;
 	}
@@ -306,6 +310,10 @@ public List<Integer> getUserCounts() {
 		return this.userCounts;
 	}
 	
+	public Map<String, Integer> getUserMap() {
+		return this.userMap;
+	}
+	
 	public int getCountLimit() {
 		return this.countLimit;
 	}
diff --git a/src/file/BookmarkWriter.java b/src/file/BookmarkWriter.java
@@ -51,7 +51,8 @@ private static boolean doWriteSample(BookmarkReader reader, List<Bookmark> userS
 				i = 0;
 				for (int cat : userCats) {
 					//bw.write(URLEncoder.encode((catPredictions == null ? reader.getCategories().get(cat).replace("\"", "") : reader.getTags().get(cat)).replace("\"", ""), "UTF-8"));
-					bw.write("t" + cat);
+					String catName = (realValues ? reader.getCategories().get(cat).replace("\"", "") : "t" + cat);
+					bw.write(catName);
 					if (++i < userCats.size()) {
 						bw.write(',');
 					}					
diff --git a/src/processing/MetricsCalculator.java b/src/processing/MetricsCalculator.java
@@ -70,6 +70,7 @@ public MetricsCalculator(PredictionFileReader reader, String outputFile, int k,
 		}
 		BufferedWriter bw = null;
 		//TODO: Enable if you need data for statistical tests
+		/*
 		if ((recommTags && (k == 5 || k == 10)) || (!recommTags && k == 20)) {
 			try {
 				FileWriter writer = new FileWriter(new File(outputFile + "_" + k + ".txt"), true);
@@ -78,6 +79,7 @@ public MetricsCalculator(PredictionFileReader reader, String outputFile, int k,
 				e.printStackTrace();
 			}
 		}
+		*/
 		
 		//double count = this.reader.getPredictionCount(); // only user where there are recommendations
 		double count = this.reader.getPredictionData().size();		 // all users
diff --git a/src/processing/ThreeLTCalculator.java b/src/processing/ThreeLTCalculator.java
@@ -179,14 +179,24 @@ public Map<Integer, Double> getRankedTagList(int userID, int resID, List<Integer
 	
 	public Map<Integer, Double> getCollectiveRankedTagList(List<Integer> testCats, double testTimestamp, int limit, boolean tagBLL, boolean topicBLL) {
 		Map<Integer, Double> collectiveTagMap = new LinkedHashMap<Integer, Double>();
-		for (int id = 0; id < this.reader.getUsers().size(); id++) {
-			Map<Integer, Double> tagMap = getRankedTagList(id, -1, testCats, testTimestamp, limit, tagBLL, topicBLL, false);
-			for (Map.Entry<Integer, Double> entry : tagMap.entrySet()) {
-				Double val = collectiveTagMap.get(entry.getKey());
-				collectiveTagMap.put(entry.getKey(), val == null ? entry.getValue() : val.doubleValue() + entry.getValue());
+		
+		List<Bookmark> bookmarks = this.reader.getBookmarks();
+		List<Map<Integer, Integer>> resTopics = Utilities.getResTopics(bookmarks);
+		for (Bookmark b : bookmarks) {
+			if (b.getResourceID() < resTopics.size()) {
+				double sim = Utilities.getCosineSimList(testCats, new ArrayList<Integer>(resTopics.get(b.getResourceID()).keySet()));
+				Double ajhid = Math.pow(sim, 3);
+				if (ajhid.isNaN() || ajhid.isInfinite()) {
+					ajhid = 0.0;
+					System.out.println("Cos - NAN");
+				}
+				for (int t : b.getTags()) {
+					Double tVal = collectiveTagMap.get(t);
+					collectiveTagMap.put(t, tVal == null ? ajhid : tVal.doubleValue() + ajhid);
+				}
 			}
 		}
-		
+				
 		Map<Integer, Double> sortedResultMap = new TreeMap<Integer, Double>(new DoubleMapComparator(collectiveTagMap));
 		sortedResultMap.putAll(collectiveTagMap);
 		Map<Integer, Double> returnMap = new LinkedHashMap<Integer, Double>();
diff --git a/src/processing/hashtag/ProcessFrequencyRecency.java b/src/processing/hashtag/ProcessFrequencyRecency.java
@@ -1,4 +1,4 @@
-package processing;
+package processing.hashtag;
 
 import java.io.BufferedWriter;
 import java.io.File;
@@ -10,6 +10,8 @@
 import java.util.HashSet;
 import java.util.List;
 
+import common.TimeUtil;
+
 /**
  * @author spujari
  *
diff --git a/src/processing/hashtag/ProcessFrequencyRecencySocial.java b/src/processing/hashtag/ProcessFrequencyRecencySocial.java
@@ -1,4 +1,4 @@
-package processing;
+package processing.hashtag;
 
 import java.io.BufferedWriter;
 import java.io.File;
@@ -11,6 +11,8 @@
 import java.util.Collections;
 import java.util.HashMap;
 
+import common.TimeUtil;
+
 public class ProcessFrequencyRecencySocial {
     
 	private String sampleDir;
diff --git a/src/processing/hashtag/SocialCalculator.java b/src/processing/hashtag/SocialCalculator.java
@@ -1,4 +1,4 @@
-package processing;
+package processing.hashtag;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -15,6 +15,8 @@
 import java.util.Map.Entry;
 import java.util.TreeMap;
 
+import processing.BLLCalculator;
+
 import com.google.common.primitives.Ints;
 
 import common.Bookmark;
@@ -282,11 +284,11 @@ private Map<Integer, Double> getRankedTagListSocialFrequencyHybrid(int userID, L
         return sortedResultMap;
     } 
     
-    private Map<Integer, Double> getRankedTagListSocialBLLHybrid(int userID, Long timesString, double beta, double exponentSocial){
-        Map<Integer, Double> rankedList = new HashMap<Integer, Double>();
+    private Map<Integer, Double> getRankedTagListSocialBLLHybrid(int userID, Long timesString, double beta, double exponentSocial, boolean sort) {
+        Map<Integer, Double> rankedList = new LinkedHashMap<Integer, Double>();
         String user = this.users.get(userID);
         List<String> friendList = network.get(user);
-        HashMap <Integer, Double> tagRank = new HashMap<Integer, Double>();
+        HashMap <Integer, Double> tagRank = new LinkedHashMap<Integer, Double>();
         if (friendList != null){
             for(String friend : friendList){
                 HashMap<Integer, ArrayList<Long>> tagTimestampMap = userTagTimes.get(friend);
@@ -331,16 +333,21 @@ private Map<Integer, Double> getRankedTagListSocialBLLHybrid(int userID, Long ti
             Double val = resultMap.get(entry.getKey());    
             resultMap.put(entry.getKey(), val == null ? (beta) * entry.getValue().doubleValue() : (1-beta) * val.doubleValue() + (beta) * entry.getValue().doubleValue());
         }
-        Map<Integer, Double>sortedResultMap = new TreeMap<Integer, Double>(new DoubleMapComparator(resultMap));
-        sortedResultMap.putAll(resultMap);
-        return sortedResultMap;
+        
+        if (sort) {
+	        Map<Integer, Double>sortedResultMap = new TreeMap<Integer, Double>(new DoubleMapComparator(resultMap));
+	        sortedResultMap.putAll(resultMap);
+	        return sortedResultMap;
+        } else {
+        	return resultMap;
+        }
     }
     
     /**
      * @param sampleSize
      * @return
      */
-    private List<Map<Integer, Double>> calculateSocialTagScore(double beta, double exponentSocial, String algorithm) {
+    private List<Map<Integer, Double>> calculateSocialTagScore(double beta, double exponentSocial, String algorithm, boolean sort) {
         
         List<Map<Integer, Double>> results = new ArrayList<Map<Integer, Double>>();
         for (int i = trainSize; i < reader.getBookmarks().size(); i++) { // the test-set
@@ -351,7 +358,7 @@ private List<Map<Integer, Double>> calculateSocialTagScore(double beta, double e
             }else if(algorithm.equals("social")){
                 map = getRankedTagListSocial(data.getUserID(), data.getTimestampAsLong(), exponentSocial);
             }else if (algorithm.equals("hybrid")) {
-                map = getRankedTagListSocialBLLHybrid(data.getUserID(), data.getTimestampAsLong(), beta, exponentSocial);
+                map = getRankedTagListSocialBLLHybrid(data.getUserID(), data.getTimestampAsLong(), beta, exponentSocial, sort);
             }else if (algorithm.equals("hybrid_freq")){
             	map = getRankedTagListSocialFrequencyHybrid(data.getUserID(), data.getTimestampAsLong(), beta);
             }
@@ -363,8 +370,15 @@ private List<Map<Integer, Double>> calculateSocialTagScore(double beta, double e
     /**
      * @return
      */
-    public BookmarkReader predictSample(double exponentSocial, double beta, String algorithm) {
-        List<Map<Integer, Double>> actValues = calculateSocialTagScore(beta, exponentSocial, algorithm);
+    public BookmarkReader predictSample(double exponentSocial, double beta, String algorithm, List<Map<Integer, Double>> contentBasedValues) {
+    	List<Map<Integer, Double>> actValues = null;
+    	if (contentBasedValues == null) {
+    		actValues = calculateSocialTagScore(beta, exponentSocial, algorithm, true);        
+    	} else {
+    		actValues = calculateSocialTagScore(beta, exponentSocial, algorithm, false);
+    		// TODO: map with content-based results
+    	}
+        
         List<int[]> predictionValues = new ArrayList<int[]>();
         for (int i = 0; i < actValues.size(); i++) {
             Map<Integer, Double> modelVal = actValues.get(i);
diff --git a/src/processing/hashtag/SolrHashtagCalculator.java b/src/processing/hashtag/SolrHashtagCalculator.java
@@ -1,6 +1,7 @@
-package processing;
+package processing.hashtag;
 
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -17,6 +18,7 @@ public class SolrHashtagCalculator {
 	private final static int LIMIT = 10;
 	
 	// Statics ----------------------------------------------------------------------------------------------------------------------		
+		
 	public static String predictSample(String sampleDir, String filename, String solrUrl) {		
 		List<Set<String>> predictionValues = new ArrayList<Set<String>>();
 		List<Set<String>> realValues = new ArrayList<Set<String>>();
@@ -73,4 +75,27 @@ public static String predictTrainSample(String sampleDir, String filename, Strin
 		PredictionFileWriter.writeSimplePredictions(predictionValues, realValues, sampleDir + "/" + filename + "_" + suffix);
 		return suffix;
 	}
+	
+	public static Map<Integer, Map<Integer, Double>> getNormalizedHashtagPredictions(String filename, String solrUrl, BookmarkReader reader) {		
+		SolrConnector trainConnector = new SolrConnector(solrUrl, filename + "_train");
+		SolrConnector testConnector = new SolrConnector(solrUrl, filename + "_test");
+		Map<Integer, Map<Integer, Double>> hashtagMaps = new LinkedHashMap<Integer, Map<Integer, Double>>();
+		
+		List<Tweet> tweets = testConnector.getTweetObjects();
+		for (Tweet tweet : tweets) {
+			Map<Integer, Double> normalizedIntResult = new LinkedHashMap<Integer, Double>();
+			Map<String, Double> stringResult = trainConnector.getTopHashtagsForTweetText(tweet.getText(), 100);
+			double denom = 0.0;
+			for (Map.Entry<String, Double> e : stringResult.entrySet()) {
+				denom += Math.exp(e.getValue());
+			}
+			for (Map.Entry<String, Double> e : stringResult.entrySet()) {
+				normalizedIntResult.put(reader.getTagMap().get(e.getKey()), Math.exp(e.getValue()) / denom);
+			}
+			
+			hashtagMaps.put(reader.getUserMap().get(tweet.getUserid()), normalizedIntResult);
+		}
+		
+		return hashtagMaps;
+	}
 }
diff --git a/src/processing/hashtag/Tweet.java b/src/processing/hashtag/Tweet.java
@@ -0,0 +1,60 @@
+package processing.hashtag;
+
+import java.util.Set;
+
+public class Tweet {
+
+	private String id;
+	private String userid;
+	private String text;
+	private String timestamp;
+	private Set<String> hashtags;
+	
+	public Tweet(String id, String userid, String text, String timestamp, Set<String> hashtags) {
+		this.id = id;
+		this.userid = userid;
+		this.text = text;
+		this.timestamp = timestamp;
+		this.hashtags = hashtags;
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(String id) {
+		this.id = id;
+	}
+
+	public String getUserid() {
+		return userid;
+	}
+
+	public void setUserid(String userid) {
+		this.userid = userid;
+	}
+
+	public String getText() {
+		return text;
+	}
+
+	public void setText(String text) {
+		this.text = text;
+	}
+
+	public String getTimestamp() {
+		return timestamp;
+	}
+
+	public void setTimestamp(String timestamp) {
+		this.timestamp = timestamp;
+	}
+
+	public Set<String> getHashtags() {
+		return hashtags;
+	}
+
+	public void setHashtags(Set<String> hashtags) {
+		this.hashtags = hashtags;
+	}
+}
diff --git a/src/test/Pipeline.java b/src/test/Pipeline.java

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-package processing;`
	`1`	`+package common;`
`2`	`2`
`3`	`3`	`public class TimeUtil {`
`4`	`4`
Original file line number	Diff line number	Diff line change
`@@ -131,11 +131,11 @@ private boolean doReadFile(String path, String filename) {`
`131`	`131`	`if (lineParts.length > 4) { // are there categories`
`132`	`132`	`for (String cat : lineParts[4].replace("\"", "").split(",")) {`
`133`	`133`	`if (!cat.isEmpty()) {`
`134`		`- if (cat.contains("_")) {`
`135`		`- categories.add(cat.substring(0, cat.indexOf("_")).toLowerCase());`
`136`		`- } else {`
	`134`	`+ //if (cat.contains("_")) {`
	`135`	`+ // categories.add(cat.substring(0, cat.indexOf("_")).toLowerCase());`
	`136`	`+ //} else {`
`137`	`137`	`categories.add(cat.toLowerCase());`
`138`		`- }`
	`138`	`+ //}`
`139`	`139`	`}`
`140`	`140`	`}`
`141`	`141`	`}`
`@@ -290,6 +290,10 @@ public List<Integer> getTagCounts() {`
`290`	`290`	`return this.tagCounts;`
`291`	`291`	`}`
`292`	`292`
	`293`	`+ public Map<String, Integer> getTagMap() {`
	`294`	`+ return this.tagMap;`
	`295`	`+ }`
	`296`	`+`
`293`	`297`	`public List<String> getResources() {`
`294`	`298`	`return this.resources;`
`295`	`299`	`}`
`@@ -306,6 +310,10 @@ public List<Integer> getUserCounts() {`
`306`	`310`	`return this.userCounts;`
`307`	`311`	`}`
`308`	`312`
	`313`	`+ public Map<String, Integer> getUserMap() {`
	`314`	`+ return this.userMap;`
	`315`	`+ }`
	`316`	`+`
`309`	`317`	`public int getCountLimit() {`
`310`	`318`	`return this.countLimit;`
`311`	`319`	`}`
Original file line number	Diff line number	Diff line change
`@@ -70,6 +70,7 @@ public MetricsCalculator(PredictionFileReader reader, String outputFile, int k,`
`70`	`70`	`}`
`71`	`71`	`BufferedWriter bw = null;`
`72`	`72`	`//TODO: Enable if you need data for statistical tests`
	`73`	`+ /*`
`73`	`74`	`if ((recommTags && (k == 5 \|\| k == 10)) \|\| (!recommTags && k == 20)) {`
`74`	`75`	`try {`
`75`	`76`	`FileWriter writer = new FileWriter(new File(outputFile + "_" + k + ".txt"), true);`
`@@ -78,6 +79,7 @@ public MetricsCalculator(PredictionFileReader reader, String outputFile, int k,`
`78`	`79`	`e.printStackTrace();`
`79`	`80`	`}`
`80`	`81`	`}`
	`82`	`+ */`
`81`	`83`
`82`	`84`	`//double count = this.reader.getPredictionCount(); // only user where there are recommendations`
`83`	`85`	`double count = this.reader.getPredictionData().size(); // all users`