add files

Author: js1010

cpp/include/ioutils.hpp

@@ -0,0 +1,36 @@
+// Copyright (c) 2021 Jisang Yoon
+// All rights reserved.
+//
+// This source code is licensed under the Apache 2.0 license found in the
+// LICENSE file in the root directory of this source tree.
+#pragma once
+
+#include <set>
+#include <random>
+#include <memory>
+#include <string>
+#include <fstream>
+#include <utility>
+#include <queue>
+#include <deque>
+#include <functional>
+#include <vector>
+#include <cmath>
+#include <chrono> // NOLINT
+#include <iostream>
+#include <unordered_map>
+
+namespace cusim {
+
+class IoUtils {
+ public:
+  IoUtils();
+  ~IoUtils();
+  void LoadGensimVocab(std::string filepath, int min_count);
+ private:
+  std::vector<std::string> parse_line(std::string line);
+  std::unordered_map<std::string, int> word_idmap_;
+  std::vector<std::string> word_list_;
+};  // class IoUtils
+
+} // namespace cusim

cpp/include/log.hpp

@@ -0,0 +1,44 @@
+// Copyright (c) 2020 Jisang Yoon
+// All rights reserved.
+//
+// This source code is licensed under the Apache 2.0 license found in the
+// LICENSE file in the root directory of this source tree.
+
+// reference: https://github.com/kakao/buffalo/blob/5f571c2c7d8227e6625c6e538da929e4db11b66d/lib/misc/log.cc
+#pragma once
+#include <memory>
+
+#define SPDLOG_EOL ""
+#define SPDLOG_TRACE_ON
+#include "spdlog/spdlog.h"
+#include "spdlog/sinks/stdout_color_sinks.h"
+
+#define __FILENAME__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__)
+
+#define INFO(x, ...) logger_->info("[{}:{}] " x "\n", __FILENAME__, __LINE__, __VA_ARGS__);
+#define DEBUG(x, ...) logger_->debug("[{}:{}] " x "\n", __FILENAME__, __LINE__, __VA_ARGS__);
+#define WARN(x, ...) logger_->warn("[{}:{}] " x "\n", __FILENAME__, __LINE__, __VA_ARGS__);
+#define TRACE(x, ...) logger_->trace("[{}:{}] " x "\n", __FILENAME__, __LINE__, __VA_ARGS__);
+#define CRITICAL(x, ...) logger_->critical("[{}:{}] " x "\n", __FILENAME__, __LINE__, __VA_ARGS__);
+
+#define INFO0(x) logger_->info("[{}:{}] " x "\n", __FILENAME__, __LINE__);
+#define DEBUG0(x) logger_->debug("[{}:{}] " x "\n", __FILENAME__, __LINE__);
+#define WARN0(x) logger_->warn("[{}:{}] " x "\n", __FILENAME__, __LINE__);
+#define TRACE0(x) logger_->trace("[{}:{}] " x "\n", __FILENAME__, __LINE__);
+#define CRITICAL0(x) logger_->critical("[{}:{}] " x "\n", __FILENAME__, __LINE__);
+
+namespace cusim {
+
+class CuHNSWLogger {
+ public:
+  CuHNSWLogger();
+  std::shared_ptr<spdlog::logger>& get_logger();
+  void set_log_level(int level);
+  int get_log_level();
+
+ private:
+  static int global_logging_level_;
+  std::shared_ptr<spdlog::logger> logger_;
+};  // class CuHNSWLogger
+
+}  // namespace cusim

cpp/include/types.hpp

@@ -39,20 +39,3 @@
 #endif
 
 #define WARP_SIZE 32
-
-struct Neighbor {
-  cuda_scalar distance;
-  int nodeid;
-  bool checked;
-};
-
-// to manage the compatibility with hnswlib
-typedef unsigned int tableint;
-typedef unsigned int sizeint;
-typedef float scalar;
-typedef size_t labeltype;
-
-enum DIST_TYPE {
-  DOT,
-  L2,
-};

cpp/src/culda.cu

@@ -0,0 +1,16 @@
+// Copyright (c) 2020 Jisang Yoon
+// All rights reserved.
+//
+// This source code is licensed under the Apache 2.0 license found in the
+// LICENSE file in the root directory of this source tree.
+#include "culda.cuh"
+
+namespace cusim {
+
+CuLDA::CuLDA() {
+  logger_ = CuSimLogger().get_logger();
+}
+
+CuLDA::~CuLDA() {}
+
+} // namespace cusim

cpp/src/ioutils.cc

@@ -0,0 +1,59 @@
+// Copyright (c) 2021 Jisang Yoon
+// All rights reserved.
+//
+// This source code is licensed under the Apache 2.0 license found in the
+// LICENSE file in the root directory of this source tree.
+#include "ioutils.hpp"
+
+namespace cusim {
+
+IoUtils::IoUtils() {
+  logger_ = CuSimLogger().get_logger();
+}
+
+IoUtils::~IoUtils() {}
+
+std::vector<std::string> IoUtils::parse_line(std::string line) {
+  int n = line.size();
+  std::vector<std::string> ret;
+  std::string element;
+  for (int i = 0; i < n; ++i) {
+    if (line[i] == ' ') {
+      ret.push_back(element);
+      element.clear();
+    } else {
+      element += line[i];
+    }
+  }
+  if (element.size() > 0) {
+    ret.push_back(element);
+  }
+  return ret;
+}
+
+void IoUtils::LoadGensimVocab(std::string filepath, int min_count) {
+  INFO("read gensim file to generate vocabulary: {}, min_count: {}", filepath, min_count);
+  std::ifstream fin(filepath.c_str());
+  std::unordered_map<std::string, int> word_count;
+  while (not fin.eof()) {
+    std::string line;
+    getline(fin, line);
+    std::vector<std::string> line_vec = parse_line(line);
+    for (auto& word: line_vec) {
+      if (not word_count.count(word)) word_count[word] = 0;
+      word_count[word]++;
+    }
+  }
+  INFO("number of raw words: {}", word_count.size());
+  word_idmap_.clear();
+  word_list_.clear();
+  for (auto& it: word_count) {
+    if (it.second >= min_count) {
+      word_idmap_[it.first] = vocab_.size();
+      word_list_.push_back(it.first);
+    }
+  }
+  INFO("number of words after filtering: {}", word_list_.size());
+}
+
+}  // namespace cusim

cpp/src/log.cc

@@ -0,0 +1,38 @@
+// Copyright (c) 2020 Jisang Yoon
+// All rights reserved.
+//
+// This source code is licensed under the Apache 2.0 license found in the
+// LICENSE file in the root directory of this source tree.
+
+// reference: https://github.com/kakao/buffalo/blob/5f571c2c7d8227e6625c6e538da929e4db11b66d/lib/misc/log.cc
+#include "log.hpp"
+
+
+namespace cusim {
+int CuSimLogger::global_logging_level_ = 2;
+
+CuSimLogger::CuHNSWLogger() {
+  spdlog::set_pattern("[%^%-8l%$] %Y-%m-%d %H:%M:%S %v");
+  logger_ = spdlog::default_logger();
+}
+
+std::shared_ptr<spdlog::logger>& CuHNSWLogger::get_logger() {
+  return logger_;
+}
+
+void CuSimLogger::set_log_level(int level) {
+  global_logging_level_ = level;
+  switch (level) {
+    case 0: spdlog::set_level(spdlog::level::off); break;
+    case 1: spdlog::set_level(spdlog::level::warn); break;
+    case 2: spdlog::set_level(spdlog::level::info); break;
+    case 3: spdlog::set_level(spdlog::level::debug); break;
+    default: spdlog::set_level(spdlog::level::trace); break;
+  }
+}
+
+int CuSimLogger::get_log_level() {
+  return global_logging_level_;
+}
+
+}  // namespace cusim