ensure non-negativitty

Author: js1010

cpp/include/culda/cuda_lda_kernels.cuh

@@ -30,7 +30,7 @@ __global__ void EstepKernel(
   const int num_cols, const int num_indptr,
   const int num_topics, const int num_iters,
   float* gamma, float* new_gamma, float* phi,
-  float* alpha, float* beta,
+  const float* alpha, const float* beta,
   float* grad_alpha, float* new_beta, float* train_losses, float* vali_losses) {
 
   // storage for block
@@ -76,12 +76,12 @@ __global__ void EstepKernel(
           __syncthreads();
         }
         if (j + 1 == num_iters) {
-          float p = ReduceSum(_phi, num_topics);
+          float p = fmaxf(EPS, ReduceSum(_phi, num_topics));
           if (threadIdx.x == 0) {
             if (_vali)
-              vali_losses[blockIdx.x] += logf(p + EPS);
+              vali_losses[blockIdx.x] += logf(p);
             else
-              train_losses[blockIdx.x] += logf(p + EPS);
+              train_losses[blockIdx.x] += logf(p);
           } 
         }
         __syncthreads();

cpp/src/utils/ioutils.cc

@@ -37,10 +37,10 @@ void IoUtils::ParseLineImpl(std::string line, std::vector<std::string>& ret) {
   int n = line.size();
   std::string element;
   for (int i = 0; i < n; ++i) {
-    if (line[i] == ' ' or line[i] == ',') {
+    if (line[i] == ' ') {
       ret.push_back(element);
       element.clear();
-    } else if (line[i] != '"') {
+    } else {
       element += std::tolower(line[i]);
     }
   }

cusim/culda/pyculda.py

@@ -137,6 +137,7 @@ def _train_m_step(self):
     self.obj.pull()
 
     # update beta
+    self.new_beta[:, :] = np.maximum(self.new_beta, EPS)
     self.beta[:, :] = self.new_beta / np.sum(self.new_beta, axis=0)[None, :]
     self.new_beta[:, :] = 0
 
@@ -151,6 +152,15 @@ def _train_m_step(self):
     c_0 = c_nume / c_deno
     delta = (gvec - c_0) / hvec
     self.alpha -= delta
+    self.alpha[:] = np.maximum(self.alpha, EPS)
     self.grad_alpha[:,:] = 0
 
     self.obj.push()
+
+  def save_model(self, model_path):
+    self.logger.info("save model path: %s", model_path)
+    h5f = h5py.File(model_path, "w")
+    h5f.create_dataset("alpha", data=self.alpha)
+    h5f.create_dataset("beta", data=self.beta)
+    h5f.create_dataset("keys", data=np.array(self.words))
+    h5f.close()

examples/example1.py

@@ -9,6 +9,8 @@
 import subprocess
 import fire
 
+import h5py
+import numpy as np
 from gensim import downloader as api
 from cusim import aux, IoUtils, CuLDA
 
@@ -43,11 +45,23 @@ def run_lda():
     "data_path": DATA_PATH,
     "data_dir": DATA_PATH2,
     # "skip_preprocess": True,
-    "c_log_level": 3,
+    # "c_log_level": 3,
   }
   lda = CuLDA(opt)
   lda.train_model()
-
+  lda.save_model("res/lda.h5")
+  h5f = h5py.File("res/lda.h5", "r")
+  beta = h5f["beta"][:]
+  for i in range(lda.opt.num_topics):
+    print("=" * 50)
+    print(f"topic {i + 1}")
+    words = np.argsort(-beta.T[i])[:10]
+    print("-" * 50)
+    for j in range(10):
+      word = lda.words[words[j]].decode("utf8")
+      prob = beta[words[j], i]
+      print(f"rank {j + 1}. word: {word}, prob: {prob}")
+  h5f.close()
 
 if __name__ == "__main__":
   fire.Fire()