cbt argument added

Author: domkowald

src/processing/MalletCalculator.java

@@ -19,9 +19,7 @@ License, or (at your option) any later version.
  */
 package processing;
 
-import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -30,7 +28,6 @@ License, or (at your option) any later version.
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.TimeUnit;
-import java.util.regex.Pattern;
 
 import com.google.common.base.Stopwatch;
 import com.google.common.primitives.Doubles;
@@ -39,23 +36,12 @@ License, or (at your option) any later version.
 import common.DoubleMapComparator;
 import common.Bookmark;
 import common.Utilities;
-import cc.mallet.pipe.Array2FeatureVector;
-import cc.mallet.pipe.CharSequence2TokenSequence;
-import cc.mallet.pipe.CharSequenceArray2TokenSequence;
-import cc.mallet.pipe.CharSequenceLowercase;
-import cc.mallet.pipe.Pipe;
-import cc.mallet.pipe.PrintInputAndTarget;
-import cc.mallet.pipe.SerialPipes;
 import cc.mallet.pipe.StringList2FeatureSequence;
-import cc.mallet.pipe.TokenSequence2FeatureSequence;
-import cc.mallet.pipe.TokenSequenceLowercase;
 import cc.mallet.topics.ParallelTopicModel;
 import cc.mallet.types.Alphabet;
-import cc.mallet.types.FeatureSequence;
 import cc.mallet.types.IDSorter;
 import cc.mallet.types.Instance;
 import cc.mallet.types.InstanceList;
-import cc.mallet.types.TokenSequence;
 import file.PredictionFileWriter;
 import file.BookmarkReader;
 import file.BookmarkSplitter;
@@ -64,7 +50,7 @@ public class MalletCalculator {
 
 	private final static int MAX_RECOMMENDATIONS = 10;
 	private final static int MAX_TERMS = 100;
-	private final static int NUM_THREADS = 10;
+	//private final static int NUM_THREADS = 10;
 	private final static int NUM_ITERATIONS = 2000;
 	private final static double ALPHA = 0.01;
 	private final static double BETA = 0.01;
@@ -105,10 +91,10 @@ private List<Map<Integer, Double>> getMaxTopicsByDocs(ParallelTopicModel LDA, in
         	double[] topicProbs = LDA.getTopicProbabilities(doc);
         	//double probSum = 0.0;
         	for (int topic = 0; topic < topicProbs.length && topic < maxTopicsPerDoc; topic++) {
-        		//if (topicProbs[topic] > 0.01) { // TODO
+        		if (topicProbs[topic] > TOPIC_THRESHOLD) { // TODO
         			topicList.put(topic, topicProbs[topic]);
         			//probSum += topicProbs[topic];
-        		//}
+        		}
         	}
 			//System.out.println("Topic Sum: " + probSum);
         	Map<Integer, Double> sortedTopicList = new TreeMap<Integer, Double>(new DoubleMapComparator(topicList));
@@ -151,7 +137,7 @@ private List<Map<Integer, Double>> getMaxTermsByTopics(ParallelTopicModel LDA, i
     	return topicList;
 	}
 
-	public void predictValuesProbs() {
+	public void predictValuesProbs(boolean topicCreation) {
 		ParallelTopicModel LDA = new ParallelTopicModel(this.numTopics, ALPHA * this.numTopics, BETA); // TODO
 		LDA.addInstances(this.instances);
 		LDA.setNumThreads(1);
@@ -164,20 +150,25 @@ public void predictValuesProbs() {
 		}
 		this.docList = getMaxTopicsByDocs(LDA, this.numTopics);
 		System.out.println("Fetched Doc-List");
-		this.topicList = getMaxTermsByTopics(LDA, MAX_TERMS);
+		this.topicList = !topicCreation ? getMaxTermsByTopics(LDA, MAX_TERMS) : null;
 		System.out.println("Fetched Topic-List");
 	}
 
 	public Map<Integer, Double> getValueProbsForID(int id, boolean topicCreation) {
 		Map<Integer, Double> terms = null;
 		if (id < this.docList.size()) {
-			terms = new LinkedHashMap<Integer, Double>();
 			Map<Integer, Double> docVals = this.docList.get(id);
+			if (this.topicList == null) {
+				return docVals;
+			}
+			terms = new LinkedHashMap<Integer, Double>();
+
 			for (Map.Entry<Integer, Double> topic : docVals.entrySet()) { // look at each assigned topic
 				Set<Entry<Integer, Double>> entrySet = this.topicList.get(topic.getKey()).entrySet();
 				double topicProb = topic.getValue();
 				for (Map.Entry<Integer, Double> entry : entrySet) { // and its terms
 					if (topicCreation) {
+						// DEPRECATED
 						if (topicProb > TOPIC_THRESHOLD) {
 							terms.put(entry.getKey(), topicProb);
 							break; // only use first tag as topic-name with the topic probability
@@ -301,7 +292,7 @@ public static List<Map<Integer, Double>> startLdaCreation(BookmarkReader reader,
 		if (userBased) {
 			userMaps = Utilities.getUserMaps(reader.getBookmarks().subList(0, trainSize));
 			userCalc = new MalletCalculator(userMaps, numTopics);
-			userCalc.predictValuesProbs();
+			userCalc.predictValuesProbs(topicCreation);
 			//userDenoms = getDenoms(userPredictionValues);
 			System.out.println("User-Training finished");
 		}
@@ -311,7 +302,7 @@ public static List<Map<Integer, Double>> startLdaCreation(BookmarkReader reader,
 		if (resBased) {
 			resMaps = Utilities.getResMaps(reader.getBookmarks().subList(0, trainSize));
 			resCalc = new MalletCalculator(resMaps, numTopics);
-			resCalc.predictValuesProbs();
+			resCalc.predictValuesProbs(topicCreation);
 			//resDenoms = getDenoms(resPredictionValues);
 			System.out.println("Res-Training finished");
 		}

src/test/Pipeline.java

@@ -65,9 +65,9 @@ public class Pipeline {
 	// set for categorizer/describer split (true is describer, false is categorizer - null for nothing)
 	private final static Boolean DESCRIBER = null;
 	// placeholder for the topic posfix
-	private final static String TOPIC_NAME = null;
+	private static String TOPIC_NAME = null;
 	// placeholder for the used dataset
-	private final static String DATASET = "ml";
+	private final static String DATASET = "cul";
 
 	public static void main(String[] args) {
 		System.out.println("TagRecommender:\n" + "" +
@@ -90,10 +90,11 @@ public static void main(String[] args) {
 		String path = dir + "/" + DATASET + "_sample";
 		//getStatistics(path);
 		//writeTensorFiles(path, false);
-		//evaluate(dir, path, "wrmf_mml", null, false, true);
-		//createLdaSamples(path, 1, 100, false);
+		//evaluate(dir, path, "wrmf_500_mml", TOPIC_NAME, false, true);
+		//createLdaSamples(path, 1, 500, false);
 		//startCfResourceCalculator(dir, path, 1, 20, true, false, false, false, Features.ENTITIES);
 		//startCfResourceCalculator(dir, path, 1, 20, false, true, true, false, Features.ENTITIES);
+		//startCfResourceCalculator(dir, path, 1, 20, false, true, false, false, Features.TOPICS);
 		//startResourceCIRTTCalculator(dir, path, "", 1, 20, Features.ENTITIES, false, true, false, true);
 		//startBaselineCalculatorForResources(dir, path, 1, false);
 
@@ -253,6 +254,9 @@ public static void main(String[] args) {
 		} else if (op.equals("item_cfb")) {
 			boolean userBased = true, resourceBased = false, allResources  = false;
 			startCfResourceCalculator(sampleDir, samplePath, sampleCount, 20, userBased, resourceBased, allResources, false, Features.ENTITIES);
+		} else if (op.equals("item_cbt")) {
+			TOPIC_NAME = "lda_500";
+			startCfResourceCalculator(dir, path, 1, 20, false, true, false, false, Features.TOPICS);
 		} else if (op.equals("item_zheng")) {
 			startZhengResourceCalculator(sampleDir, samplePath, sampleCount);
 		} else if (op.equals("item_huang")) {
@@ -501,9 +505,9 @@ private static void getTrainTestSize(String sample) {
 
 	// passing the trainSize means that MyMediaLite files will be evaluated
 	private static void evaluate(String sampleDir, String sampleName, String prefix, String postfix, boolean calcTags, boolean mymedialite) {
-		getTrainTestSize(sampleName);
+		getTrainTestSize(sampleName + (postfix != null ? "_" + postfix : ""));
 		BookmarkReader reader = new BookmarkReader(TRAIN_SIZE, false);
-		reader.readFile(sampleName);
+		reader.readFile(sampleName + (postfix != null ? "_" + postfix : ""));
 		if (calcTags) {
 			writeMetrics(sampleDir, sampleName, prefix, 1, 10, postfix, reader);
 		} else {