diff --git a/index/rocksdb/CMakeLists.txt b/index/rocksdb/CMakeLists.txt new file mode 100644 index 0000000..1044d69 --- /dev/null +++ b/index/rocksdb/CMakeLists.txt @@ -0,0 +1,58 @@ +cmake_minimum_required(VERSION 3.10) +project(rocksdb_bench LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -frtti") + +# RocksDB +include_directories(/usr/local/include) +link_directories(/usr/local/lib) + +include(FetchContent) +FetchContent_Declare( + googletest + + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG release-1.12.1 +) +FetchContent_MakeAvailable(googletest) +enable_testing() + +add_executable(rocksdb_ts_test + rocksdb_test.cc + rocksdb_factory.cpp +) + +add_executable(rocksdb_load + rocksdb_load.cpp + rocksdb_factory.cpp +) + +target_link_libraries(rocksdb_ts_test + gtest + gtest_main + rocksdb + pthread + z + snappy + lz4 + zstd + bz2 +) + +target_link_libraries(rocksdb_load + gtest + gtest_main + rocksdb + pthread + z + snappy + lz4 + zstd + bz2 +) + +include(GoogleTest) +gtest_discover_tests(rocksdb_ts_test) +add_test(NAME rocksdb_ts_test_TESTS COMMAND rocksdb_old_ts_test) + diff --git a/index/rocksdb/config.h b/index/rocksdb/config.h new file mode 100644 index 0000000..6bbbd25 --- /dev/null +++ b/index/rocksdb/config.h @@ -0,0 +1,81 @@ +// +// Created by antio2 on 2025/10/21. +// + +#ifndef ROCKSDB_BENCH_CONFIG_H +#define ROCKSDB_BENCH_CONFIG_H + +#pragma once +#include +#include + +enum class ts_type_t { + embed_asc, + embed_desc, + udt +}; +inline std::ostream& operator<<(std::ostream& os, ts_type_t t) { + switch (t) { + case ts_type_t::embed_asc: os << "embed_asc"; break; + case ts_type_t::embed_desc: os << "embed_desc"; break; + case ts_type_t::udt: os << "udt"; break; + default: os << "unknown"; break; + } + return os; +} + +enum class Preset { + Udt10m, + Asc10m, + Desc10m, + Udt1k, + Asc1k, + Desc1k +}; + +struct Config { +public: + using scale_t = int; + + const std::string db_path_prefix = "/home/ubuntu/disk1"; // hard code + std::string db_path_; + int thread_num_; + int id_range_; + int ops_per_thread_; + bool destroy_before_start_; + ts_type_t ts_type_; + + Config() = delete; + + explicit Config(Preset preset) + : Config(getScaleAndType(preset)) + {} + + Config(scale_t scale, ts_type_t ts_type) { + thread_num_ = 16; + ops_per_thread_ = 1000000; + id_range_ = scale; + destroy_before_start_ = true; + ts_type_ = ts_type; + std::ostringstream oss; + oss << db_path_prefix << "/test_" << id_range_ << "_" << ts_type_; + db_path_ = oss.str(); + } +private: + static std::pair getScaleAndType(Preset preset) { + switch (preset) { + case Preset::Udt10m: return {10000000, ts_type_t::udt}; + case Preset::Asc10m: return {10000000, ts_type_t::embed_asc}; + case Preset::Desc10m: return {10000000, ts_type_t::embed_desc}; + case Preset::Udt1k: return {1000, ts_type_t::udt}; + case Preset::Asc1k: return {1000, ts_type_t::embed_asc}; + case Preset::Desc1k: return {1000, ts_type_t::embed_desc}; + } + return {0, ts_type_t::embed_asc}; + } + + explicit Config(const std::pair& p) + : Config(p.first, p.second) + {} +}; +#endif //ROCKSDB_BENCH_CONFIG_H diff --git a/index/rocksdb/cpp.gitignore b/index/rocksdb/cpp.gitignore new file mode 100644 index 0000000..a5cccb3 --- /dev/null +++ b/index/rocksdb/cpp.gitignore @@ -0,0 +1,1047 @@ +##### Windows +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +##### Linux +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +##### MacOS +# General +.DS_Store +.AppleDouble +.LSOverride + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +##### Android +# Built application files +*.apk +*.ap_ +*.aab + +# Files for the ART/Dalvik VM +*.dex + +# Java class files +*.class + +# Generated files +bin/ +gen/ +out/ +# Uncomment the following line in case you need and you don't have the release build type files in your app +# release/ + +# Gradle files +.gradle/ +build/ + +# Local configuration file (sdk path, etc) +local.properties + +# Proguard folder generated by Eclipse +proguard/ + +# Log Files +*.log + +# Android Studio Navigation editor temp files +.navigation/ + +# Android Studio captures folder +captures/ + +# IntelliJ +*.iml +.idea/workspace.xml +.idea/tasks.xml +.idea/gradle.xml +.idea/assetWizardSettings.xml +.idea/dictionaries +.idea/libraries +# Android Studio 3 in .gitignore file. +.idea/caches +.idea/modules.xml +# Comment next line if keeping position of elements in Navigation Editor is relevant for you +.idea/navEditor.xml + +# Keystore files +# Uncomment the following lines if you do not want to check your keystore files in. +#*.jks +#*.keystore + +# External native build folder generated in Android Studio 2.2 and later +.externalNativeBuild + +# Google Services (e.g. APIs or Firebase) +# google-services.json + +# Freeline +freeline.py +freeline/ +freeline_project_description.json + +# fastlane +fastlane/report.xml +fastlane/Preview.html +fastlane/screenshots +fastlane/test_output +fastlane/readme.md + +# Version control +vcs.xml + +# lint +lint/intermediates/ +lint/generated/ +lint/outputs/ +lint/tmp/ +# lint/reports/ + +##### Backup +*.bak +*.gho +*.ori +*.orig +*.tmp + +##### GPG +secring.* + +##### Dropbox +# Dropbox settings and caches +.dropbox +.dropbox.attr +.dropbox.cache + +##### SynopsysVCS +# Waveform formats +*.vcd +*.vpd +*.evcd +*.fsdb + +# Default name of the simulation executable. A different name can be +# specified with this switch (the associated daidir database name is +# also taken from here): -o / +simv + +# Generated for Verilog and VHDL top configs +simv.daidir/ +simv.db.dir/ + +# Infrastructure necessary to co-simulate SystemC models with +# Verilog/VHDL models. An alternate directory may be specified with this +# switch: -Mdir= +csrc/ + +# Log file - the following switch allows to specify the file that will be +# used to write all messages from simulation: -l +*.log + +# Coverage results (generated with urg) and database location. The +# following switch can also be used: urg -dir .vdb +simv.vdb/ +urgReport/ + +# DVE and UCLI related files. +DVEfiles/ +ucli.key + +# When the design is elaborated for DirectC, the following file is created +# with declarations for C/C++ functions. +vc_hdrs.h + +##### SVN +.svn/ + +##### Mercurial +.hg/ +.hgignore +.hgsigs +.hgsub +.hgsubstate +.hgtags + +##### Bazaar +.bzr/ +.bzrignore + +##### CVS +/CVS/* +**/CVS/* +.cvsignore +*/.cvsignore + +##### TortoiseGit +# Project-level settings +/.tgitconfig + +##### PuTTY +# Private key +*.ppk + +##### Vim +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +##### Emacs +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data + +##### SublimeText +# Cache files for Sublime Text +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache + +# Workspace files are user-specific +*.sublime-workspace + +# Project files should be checked into the repository, unless a significant +# proportion of contributors will probably not be using Sublime Text +# *.sublime-project + +# SFTP configuration file +sftp-config.json +sftp-config-alt*.json + +# Package control specific files +Package Control.last-run +Package Control.ca-list +Package Control.ca-bundle +Package Control.system-ca-bundle +Package Control.cache/ +Package Control.ca-certs/ +Package Control.merged-ca-bundle +Package Control.user-ca-bundle +oscrypto-ca-bundle.crt +bh_unicode_properties.cache + +# Sublime-github package stores a github token in this file +# https://packagecontrol.io/packages/sublime-github +GitHub.sublime-settings + +##### Notepad++ +# Notepad++ backups # +*.bak + +##### TextMate +*.tmproj +*.tmproject +tmtags + +##### VisualStudioCode +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +##### NetBeans +**/nbproject/private/ +**/nbproject/Makefile-*.mk +**/nbproject/Package-*.bash +build/ +nbbuild/ +dist/ +nbdist/ +.nb-gradle/ + +##### JetBrains +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +##### Eclipse +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + +# Annotation Processing +.apt_generated/ +.apt_generated_test/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet + +# Uncomment this line if you wish to ignore the project description file. +# Typically, this file would be tracked if it contains build/dependency configurations: +#.project + +##### Qt +# C++ objects and libs +*.slo +*.lo +*.o +*.a +*.la +*.lai +*.so +*.so.* +*.dll +*.dylib + +# Qt-es +object_script.*.Release +object_script.*.Debug +*_plugin_import.cpp +/.qmake.cache +/.qmake.stash +*.pro.user +*.pro.user.* +*.qbs.user +*.qbs.user.* +*.moc +moc_*.cpp +moc_*.h +qrc_*.cpp +ui_*.h +*.qmlc +*.jsc +Makefile* +*build-* +*.qm +*.prl + +# Qt unit tests +target_wrapper.* + +# QtCreator +*.autosave + +# QtCreator Qml +*.qmlproject.user +*.qmlproject.user.* + +# QtCreator CMake +CMakeLists.txt.user* + +# QtCreator 4.8< compilation database +compile_commands.json + +# QtCreator local machine specific files for imported projects +*creator.user* + +##### VisualStudio +##### VisualStudio +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*[.json, .xml, .info] + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +##### Gradle +.gradle +**/build/ +!src/**/build/ + +# Ignore Gradle GUI config +gradle-app.setting + +# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +!gradle-wrapper.jar + +# Cache of project +.gradletasknamecache + +# # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 +# gradle/wrapper/gradle-wrapper.properties + +##### CMake +CMakeLists.txt.user +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +Makefile +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +_deps + +##### C++ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# C/C++ binary extension file +*.bin + +##### C +# Prerequisites +*.d + +# Object files +*.o +*.ko +*.obj +*.elf + +# Linker output +*.ilk +*.map +*.exp + +# Precompiled Headers +*.gch +*.pch + +# Libraries +*.lib +*.a +*.la +*.lo + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex + +# Debug files +*.dSYM/ +*.su +*.idb +*.pdb + +# Kernel Module Compile Results +*.mod* +*.cmd +.tmp_versions/ +modules.order +Module.symvers +Mkfile.old +dkms.conf + +# Raspberry Pi Pico Object file +*.uf2 +# Raspberry Pi Pico disassembler file +*.dis +cmake-build-debug/** +cmake-build-release/** \ No newline at end of file diff --git a/index/rocksdb/rocksdb_factory.cpp b/index/rocksdb/rocksdb_factory.cpp new file mode 100644 index 0000000..2172bba --- /dev/null +++ b/index/rocksdb/rocksdb_factory.cpp @@ -0,0 +1,92 @@ +// +// Created by antio2 on 2025/10/21. +// + + + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include "rocksdb_factory.h" +#include "config.h" +#include +#include +#include +#include + +using namespace rocksdb; + +std::unique_ptr initRocksDB( + const Config& cfg, + std::vector>& handles) +{ + Options options; + options.create_if_missing = true; + return createRocksDB(cfg, handles); +} + +std::unique_ptr createRocksDB( + const Config& cfg, + std::vector>& handles) +{ + std::vector existingColumnFamilies; + Status s = DB::ListColumnFamilies(DBOptions(), cfg.db_path_, &existingColumnFamilies); + + if (!s.ok()) { + existingColumnFamilies = {kDefaultColumnFamilyName}; + } + + bool hasDefault = false; + for (const auto& n : existingColumnFamilies) + if (n == kDefaultColumnFamilyName) hasDefault = true; + if (!hasDefault) + existingColumnFamilies.push_back(kDefaultColumnFamilyName); + + std::vector descriptors; + for (const auto& name : existingColumnFamilies) { + ColumnFamilyOptions cfOptions; + if (cfg.ts_type_ == ts_type_t::udt) { + cfOptions.comparator = rocksdb::BytewiseComparatorWithU64Ts(); + } else { + } + + if(cfg.ts_type_ == ts_type_t::embed_desc) + { + rocksdb::BlockBasedTableOptions table_options; + table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false)); + table_options.whole_key_filtering = false; + + cfOptions.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(8)); + cfOptions.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); + } + + cfOptions.write_buffer_size = (size_t)6 << 30; + descriptors.emplace_back(name, cfOptions); + } + + DBOptions dbOptions; + dbOptions.create_if_missing = true; + dbOptions.create_missing_column_families = true; + + DB* db = nullptr; + std::vector rawHandles; + s = DB::Open(dbOptions, cfg.db_path_, descriptors, &rawHandles, &db); + if (!s.ok()) { + throw std::runtime_error("Failed to open RocksDB: " + s.ToString()); + } + + handles.clear(); + for (auto* h : rawHandles) + handles.emplace_back(h); + + std::cout << "Opened RocksDB: " << cfg.db_path_ + << ", CF=" << handles.size() + << ", ts_type=" << (cfg.ts_type_ == ts_type_t::udt ? "UDT" : "Retina") << std::endl; + + return std::unique_ptr(db); +} diff --git a/index/rocksdb/rocksdb_factory.h b/index/rocksdb/rocksdb_factory.h new file mode 100644 index 0000000..a1d0d72 --- /dev/null +++ b/index/rocksdb/rocksdb_factory.h @@ -0,0 +1,20 @@ +// +// Created by antio2 on 2025/10/21. +// + +#ifndef ROCKSDB_BENCH_ROCKSDB_FACTORY_H +#define ROCKSDB_BENCH_ROCKSDB_FACTORY_H + + +#include +#include +#include +#include "config.h" +std::unique_ptr initRocksDB( + const Config& cfg, + std::vector>& handles); +std::unique_ptr createRocksDB( + const Config& cfg, + std::vector>& handles); + +#endif //ROCKSDB_BENCH_ROCKSDB_FACTORY_H diff --git a/index/rocksdb/rocksdb_load.cpp b/index/rocksdb/rocksdb_load.cpp new file mode 100644 index 0000000..1134c42 --- /dev/null +++ b/index/rocksdb/rocksdb_load.cpp @@ -0,0 +1,134 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "rocksdb_factory.h" +#include "util.h" + +using namespace rocksdb; + +class RocksDBSequentialLoadTest : public ::testing::Test { +protected: + std::unique_ptr cfg; + std::unique_ptr db; + std::vector> handles; + + void test(Preset preset) + { + cfg = std::make_unique(preset); + OpenDb(); + fillSequentialData(); + } + + void TearDown() override { + handles.clear(); + db.reset(); + } + +private: + void OpenDb() { + Options options; + options.create_if_missing = true; + + if (cfg->destroy_before_start_) { + DestroyDB(cfg->db_path_, options); + } + + // Open DB with ts_type-aware factory + db = createRocksDB(*cfg, handles); + } + // Sequentially fill DB from 0 to max-1 + void fillSequentialData() { + std::vector threads; + std::atomic counter{0}; + long long timestamp = 0; + std::string tsBuffer(8, '\0'); + const Slice& tsSlice = EncodeU64Ts(timestamp, &tsBuffer); + for (int t = 0; t < cfg->thread_num_; ++t) { + threads.emplace_back([this, t, &counter, timestamp, &tsSlice]() + { + WriteOptions wopt; + for (int key = 0; key < cfg->id_range_; ++key) { + std::string k = encodeKey(*cfg, t, key, timestamp); + std::string v = encodeValue(key); + if (cfg->ts_type_ == ts_type_t::udt) { + db->Put(wopt, k, tsSlice, v); + } else { + db->Put(wopt, k, v); + } + counter.fetch_add(1, std::memory_order_relaxed); + } + } + ); + } + for (auto& th : threads) th.join(); + std::cout << "Total loaded: " << counter.load() << " entries" << std::endl; + } +}; + + +TEST_F(RocksDBSequentialLoadTest, SequentialLoadUdt10m) { + test(Preset::Udt10m); +} + +TEST_F(RocksDBSequentialLoadTest, SequentialLoadAsc10m) { + test(Preset::Asc10m); +} + +TEST_F(RocksDBSequentialLoadTest, SequentialLoadDesc10m) { + test(Preset::Desc10m); +} + +TEST_F(RocksDBSequentialLoadTest, SequentialLoadUdt1k) { + test(Preset::Udt1k); +} + +TEST_F(RocksDBSequentialLoadTest, SequentialLoadAsc1k) { + test(Preset::Asc1k); +} + +TEST_F(RocksDBSequentialLoadTest, SequentialLoadDesc1k) { + test(Preset::Desc1k); +} + +TEST(RocksDBSmallTest, UDT) { + Preset preset = Preset::Udt1k; + std::vector> handles; + auto cfg = std::make_unique(preset); + { + Options options; + options.create_if_missing = true; + if (cfg->destroy_before_start_) { + DestroyDB(cfg->db_path_, options); + } + } + + auto db = createRocksDB(*cfg, handles); + + std::string tsBuf = std::string(8, '\0'); + long long timestamp = 1761139362806988526; + auto ts = EncodeU64Ts(timestamp, &tsBuf); + + WriteOptions wopt; + std::string key = "abc"; + std::string value = "ray"; + db->Put(wopt, key, ts, value); + + ReadOptions ropt; + long long timestamp2 = 1761139362806988525; + auto ts2 = EncodeU64Ts(timestamp2, &tsBuf); + ropt.timestamp = &ts2; + std::string rvalue = std::string(8, '\0'); + auto s = db->Get(ropt, key, &rvalue); + ASSERT_TRUE(s.ok()); + std::cout << rvalue << std::endl; + handles.clear(); + db = nullptr; +} diff --git a/index/rocksdb/rocksdb_test.cc b/index/rocksdb/rocksdb_test.cc new file mode 100644 index 0000000..8a6890f --- /dev/null +++ b/index/rocksdb/rocksdb_test.cc @@ -0,0 +1,150 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "rocksdb_factory.h" +#include "util.h" + +using namespace rocksdb; + +class RocksDBPerfTest : public ::testing::Test { +protected: + std::unique_ptr cfg; + std::vector> handles; + std::unique_ptr db; + void test(Preset preset) + { + cfg = std::make_unique(preset); + db = initRocksDB(*cfg, handles); + + std::cerr << "Start update test" << std::endl; + std::atomic totalOps = 0; + auto start = std::chrono::high_resolution_clock::now(); + + std::vector threads; + for (int i = 0; i < cfg->thread_num_; ++i) { + threads.emplace_back(&RocksDBPerfTest::threadWorker, this, i, std::ref(totalOps)); + } + + for (auto& t : threads) t.join(); + + auto end = std::chrono::high_resolution_clock::now(); + double sec = std::chrono::duration(end - start).count(); + + std::cout << "Total ops: " << totalOps + << ", time: " << sec << "s, throughput: " + << (long long)(totalOps / sec) << " ops/s" << std::endl; + + ASSERT_GT(totalOps, 0); + } + + void TearDown() override { + handles.clear(); + db.reset(); + } + + long long getUniqueRowId(int indexId, int key, long long timestamp) { + ReadOptions ropt; + std::string val_str(8, '\0'); + std::string k = encodeKey(*cfg, indexId, key, timestamp); + std::string ts_buf(8, '\0'); + switch (cfg->ts_type_) { + case ts_type_t::embed_asc: + { + std::unique_ptr it(db->NewIterator(ropt)); + it->SeekForPrev(k); + if (!it->Valid()) return -1; + + const Slice val = it->value(); + long long rowId; + memcpy(&rowId, val.data(), sizeof(rowId)); + return rowId; + } + case ts_type_t::embed_desc: + { + ropt.prefix_same_as_start = true; + std::unique_ptr it(db->NewIterator(ropt)); + it->Seek(k); + if (!it->Valid()) return -1; + + const Slice val = it->value(); + long long rowId; + memcpy(&rowId, val.data(), sizeof(rowId)); + return rowId; + } + case ts_type_t::udt: + { + Slice tsSlice = EncodeU64Ts(timestamp, &ts_buf); + ropt.timestamp = &tsSlice; + Status s = db->Get(ropt, k, &val_str); + if (!s.ok()) { + if (s.IsNotFound()) + { + return -1; + } + std::cerr << "DB Get failed: " << s.ToString() << std::endl; + return -1; + } + long long rowId; + memcpy(&rowId, val_str.data(), sizeof(rowId)); + return rowId; + } + } + } + +public: + void threadWorker(int indexId, std::atomic& counter) { + std::mt19937_64 rng(std::random_device{}()); + + for (int i = 0; i < cfg->ops_per_thread_; ++i) { + int key = i % cfg->id_range_; + uint64_t ts = std::chrono::system_clock::now().time_since_epoch().count(); + long long oldRow = getUniqueRowId(indexId, key, ts); + ASSERT_GE(oldRow, 0); + long long newRow = (oldRow == -1) ? rng() : oldRow + 1; + std::string k = encodeKey(*cfg, indexId, key, ts); + std::string v = encodeValue(newRow); + WriteOptions wopt; + if (cfg->ts_type_ == ts_type_t::udt) { + std::string ts_buf; + Slice tsSlice = EncodeU64Ts(ts, &ts_buf); + db->Put(wopt, k, tsSlice, v); + } else { + db->Put(wopt, k, v); + } + counter.fetch_add(1, std::memory_order_relaxed); + } + } +}; + +TEST_F(RocksDBPerfTest, MultiThreadPerf_Udt10m) { + test(Preset::Udt10m); +} + +TEST_F(RocksDBPerfTest, MultiThreadPerf_Asc10m) { + test(Preset::Asc10m); +} + +TEST_F(RocksDBPerfTest, MultiThreadPerf_Desc10m) { + test(Preset::Desc10m); +} + +TEST_F(RocksDBPerfTest, MultiThreadPerf_Udt1k) { + test(Preset::Udt1k); +} + +TEST_F(RocksDBPerfTest, MultiThreadPerf_Asc1k) { + test(Preset::Asc1k); +} + +TEST_F(RocksDBPerfTest, MultiThreadPerf_Desc1k) { + test(Preset::Desc1k); +} diff --git a/index/rocksdb/util.h b/index/rocksdb/util.h new file mode 100644 index 0000000..bca21c8 --- /dev/null +++ b/index/rocksdb/util.h @@ -0,0 +1,42 @@ +// +// Created by antio2 on 2025/10/21. +// + +#ifndef ROCKSDB_BENCH_UTIL_H +#define ROCKSDB_BENCH_UTIL_H +#include +#include + +std::string encodeKey(Config cfg, int indexId, int key, long long timestamp) { + ts_type_t ts_type = cfg.ts_type_; + if (ts_type == ts_type_t::embed_asc) { + std::string s(4 + 4 + 8, '\0'); + memcpy(&s[0], &indexId, 4); + memcpy(&s[4], &key, 4); + memcpy(&s[8], ×tamp, 8); + return s; + } else if (ts_type == ts_type_t::udt) { + // UDT key format: indexId(4) + key(4) + timestamp_udt(8) + std::string s(4 + 4, '\0'); + memcpy(&s[0], &indexId, 4); + memcpy(&s[4], &key, 4); + return s; + } else if (ts_type == ts_type_t::embed_desc) + { + long long new_ts = LONG_LONG_MAX - timestamp; + std::string s(4 + 4 + 8, '\0'); + memcpy(&s[0], &indexId, 4); + memcpy(&s[4], &key, 4); + memcpy(&s[8], &new_ts, 8); + return s; + } + return {}; +} + +std::string encodeValue(long long rowId) { + std::string s(8, '\0'); + memcpy(&s[0], &rowId, 8); + return s; +} + +#endif //ROCKSDB_BENCH_UTIL_H