Skip to content

csdb: add RocksDB backend with runtime backend selection#73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
akaitrade wants to merge 1 commit into
base:master
Choose a base branch
from
Open

csdb: add RocksDB backend with runtime backend selection #73

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,6 +23,11 @@ option(LEVELDB_BUILD_TESTS "" OFF)
option(LEVELDB_AUTORUN_TESTS "" OFF)
option(DBSQL "Enable PostgeSQL server connection" OFF)

# The node binary always carries both DB backends; runtime selection is via
# config.ini `db_backend` (berkeleydb default, rocksdb opt-in).
set(CSDB_BACKEND "both" CACHE INTERNAL "csdb storage backend (locked to 'both')")
message(STATUS "csdb backend: both (locked)")

IF(DBSQL)
ADD_DEFINITIONS(-DDBSQL)
ENDIF(DBSQL)
Expand DownExpand Up@@ -88,6 +93,10 @@ endif()
# This large list is necessary to prevent thrift from generating unused stuff
add_definitions(-DUSE_STD_THREAD)

# Disable Boost's MSVC auto-link pragmas; we link Boost targets explicitly via
# CMake. Without this the linker hunts for libboost_*-vc142-mt-s-x64-*.lib.
add_definitions(-DBOOST_ALL_NO_LIB)

if (MSVC)
MARK_AS_ADVANCED(
CMAKE_CXX_FLAGS_RELMONITOR
Expand Down
5 changes: 5 additions & 0 deletions client/config/config.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -962,6 +962,11 @@ void Config::readStorageData(const boost::property_tree::ptree& config) {
storageData_.checkpointEvery = 1000;
}
checkAndSaveValue(data, block, PARAM_NAME_STORAGE_CHECKPOINT_EVERY_MINUTES, storageData_.checkpointEveryMinutes);
checkAndSaveValue(data, block, "async_write_queue_size", storageData_.asyncWriteQueueSize);
checkAndSaveValue(data, block, "write_batch_size", storageData_.writeBatchSize);
checkAndSaveValue(data, block, "db_backend", storageData_.dbBackend);
checkAndSaveValue(data, block, "rocksdb_block_cache_mb", storageData_.rocksdbBlockCacheMb);
checkAndSaveValue(data, block, "rocksdb_memtable_mb", storageData_.rocksdbMemtableMb);
}

void Config::readApiData(const boost::property_tree::ptree& config) {
Expand Down
5 changes: 5 additions & 0 deletions client/config/config.hpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -62,6 +62,11 @@ struct StorageData {
size_t checkpointKeep = 5; // retained periodic checkpoints (qs/0 always kept on top)
size_t checkpointEvery = 500'000; // blocks between periodic checkpoints (rolling history depth = checkpointEvery * checkpointKeep)
size_t checkpointEveryMinutes = 0; // wall-clock fallback: also save if this many minutes elapsed since last save (0 = disabled; opt-in for slow networks)
size_t asyncWriteQueueSize = 5000; // bounded queue for the async DB writer
size_t writeBatchSize = 100; // pools coalesced into one DB write
std::string dbBackend = "berkeleydb"; // runtime DB backend (CSDB_BACKEND=both builds); set to "rocksdb" to opt in
size_t rocksdbBlockCacheMb = 1024; // RocksDB shared block cache (MiB); 0 = built-in default
size_t rocksdbMemtableMb = 256; // RocksDB write_buffer_size (MiB); 0 = built-in default
};

struct ApiData {
Expand Down
26 changes: 23 additions & 3 deletions csdb/CMakeLists.txt
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -10,6 +10,16 @@ option(CSDB_BUILD_BENCHMARK "Bulid benchmark" OFF)
include (TestBigEndian)
TEST_BIG_ENDIAN(CSDB_PLATFORM_IS_BIG_ENDIAN)

if(CSDB_BACKEND STREQUAL "both")
set(CSDB_BACKEND_SOURCES
src/database_berkeleydb.cpp include/csdb/database_berkeleydb.hpp
src/database_rocksdb.cpp include/csdb/database_rocksdb.hpp)
elseif(CSDB_BACKEND STREQUAL "rocksdb")
set(CSDB_BACKEND_SOURCES src/database_rocksdb.cpp include/csdb/database_rocksdb.hpp)
else()
set(CSDB_BACKEND_SOURCES src/database_berkeleydb.cpp include/csdb/database_berkeleydb.hpp)
endif()

add_library(${PROJECT_NAME} STATIC
src/csdb.cpp
src/amount.cpp
Expand All@@ -29,7 +39,7 @@ add_library(${PROJECT_NAME} STATIC
src/priv_crypto.cpp
src/priv_crypto.hpp
src/database.cpp
src/database_berkeleydb.cpp
${CSDB_BACKEND_SOURCES}
src/user_field.cpp
include/csdb/internal/shared_data.hpp
include/csdb/internal/shared_data_ptr_implementation.hpp
Expand All@@ -48,7 +58,6 @@ add_library(${PROJECT_NAME} STATIC
include/csdb/wallet.hpp
include/csdb/storage.hpp
include/csdb/database.hpp
include/csdb/database_berkeleydb.hpp
include/csdb/user_field.hpp
)

Expand All@@ -72,13 +81,24 @@ target_include_directories(
)

include_directories(${Boost_INCLUDE_DIR})
set(CSDB_BACKEND_LIBS "")
if(CSDB_BACKEND STREQUAL "both")
list(APPEND CSDB_BACKEND_LIBS BerkeleyDB rocksdb)
target_compile_definitions(${PROJECT_NAME} PRIVATE CSDB_USE_ROCKSDB CSDB_USE_BERKELEYDB)
elseif(CSDB_BACKEND STREQUAL "rocksdb")
list(APPEND CSDB_BACKEND_LIBS rocksdb)
target_compile_definitions(${PROJECT_NAME} PRIVATE CSDB_USE_ROCKSDB)
else()
list(APPEND CSDB_BACKEND_LIBS BerkeleyDB)
endif()

target_link_libraries(
${PROJECT_NAME}
cscrypto
Boost::system
Boost::filesystem
Boost::disable_autolinking
BerkeleyDB
${CSDB_BACKEND_LIBS}
lz4
lib
)
Expand Down
11 changes: 11 additions & 0 deletions csdb/include/csdb/database.hpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,13 +41,24 @@ class Database {
virtual bool remove(const cs::Bytes& key) = 0;
virtual bool seq_no(const cs::Bytes& key, uint32_t* value) = 0; // sequence from block hash

// Vectorized put. Backends may override to coalesce writes (e.g. RocksDB WriteBatch).
struct PendingWrite {
cs::Bytes hash_key;
uint32_t seq_no;
cs::Bytes payload;
};
virtual bool put_batch(const std::vector<PendingWrite>& items);

using Item = std::pair<cs::Bytes, cs::Bytes>;
using ItemList = std::vector<Item>;
virtual bool write_batch(const ItemList& items) = 0;

virtual bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) = 0;
virtual bool getContractData(const cs::Bytes& key, cs::Bytes& data) = 0;

// anchor buffered writes to disk at checkpoint boundaries; default no-op (BDB has its own loop)
virtual bool flush() { return true; }

class Iterator {
protected:
Iterator();
Expand Down
71 changes: 71 additions & 0 deletions csdb/include/csdb/database_rocksdb.hpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
// RocksDB-backed csdb::Database. CFs: blocks, seq_no, contracts.
// blocks keyed BE(seq+1). Selected via -DCSDB_BACKEND=rocksdb.

#ifndef _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_
#define _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_

#include <memory>
#include <string>

#include <csdb/database.hpp>

namespace rocksdb {
class DB;
class ColumnFamilyHandle;
class Status;
} // namespace rocksdb

namespace csdb {

class DatabaseRocksDB : public Database {
public:
DatabaseRocksDB();
~DatabaseRocksDB() override;

public:
// Bulk-load mode (before open): no auto-compaction, disableWAL; caller must compact_full() before close.
void set_bulk_load(bool yes);
// Per-write fsync toggle (default off; durability anchored at checkpoint via flush()).
void set_sync_writes(bool v);

// Flush + full-range compaction across all CFs; call once after a bulk-load run.
bool compact_full();

// Override RocksDB cache/memtable budget (before open); 0 keeps defaults (1 GiB / 256 MiB).
void set_tuning(uint64_t block_cache_bytes, uint64_t memtable_bytes);

bool open(const std::string& path);

private:
bool is_open() const final;
bool put(const cs::Bytes& key, uint32_t seq_no, const cs::Bytes& value) final;
bool put_batch(const std::vector<PendingWrite>& items) final;
bool get(const cs::Bytes& key, cs::Bytes* value) final;
bool get(const uint32_t seq_no, cs::Bytes* value) final;
bool remove(const cs::Bytes&) final;
bool seq_no(const cs::Bytes& key, uint32_t* value) final;
bool write_batch(const ItemList&) final;
IteratorPtr new_iterator() final;

bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) override;
bool getContractData(const cs::Bytes& key, cs::Bytes& data) override;
bool flush() override;

private:
class Iterator;
void set_last_error_from_status(const rocksdb::Status& s);

private:
std::unique_ptr<rocksdb::DB> db_;
rocksdb::ColumnFamilyHandle* cf_blocks_ = nullptr; // default CF
rocksdb::ColumnFamilyHandle* cf_seq_no_ = nullptr;
rocksdb::ColumnFamilyHandle* cf_contracts_ = nullptr;
bool bulk_load_ = false;
bool sync_writes_ = false; // matches BDB's DB_TXN_NOSYNC; durability anchored at checkpoint via flush()
uint64_t block_cache_bytes_ = 1ULL << 30; // 1 GiB
uint64_t memtable_bytes_ = 256ULL << 20; // 256 MiB
};

} // namespace csdb

#endif // _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_
13 changes: 12 additions & 1 deletion csdb/include/csdb/storage.hpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -99,6 +99,9 @@ class Storage final {
::std::shared_ptr<Database> db;
::cs::Sequence newBlockchainTop = ::cs::kWrongSequence;
::cs::Sequence startSequence = 0;
// 0 = leave Storage defaults (5000 / 100) untouched.
size_t asyncWriteQueueMax = 0;
size_t writeBatchSize = 0;
};

struct OpenProgress {
Expand DownExpand Up@@ -140,7 +143,12 @@ class Storage final {
bool open(const ::std::string& path_to_base = ::std::string{},
OpenCallback callback = nullptr,
cs::Sequence newBlockchainTop = cs::kWrongSequence,
cs::Sequence startReadFrom = 0);
cs::Sequence startReadFrom = 0,
size_t asyncWriteQueueMax = 5000,
size_t writeBatchSize = 100,
uint64_t rocksDbBlockCacheBytes = 0,
uint64_t rocksDbMemtableBytes = 0,
const std::string& dbBackend = std::string{});

/**
* @brief Creating the storage using the parameters set.
Expand DownExpand Up@@ -170,6 +178,9 @@ class Storage final {
*/
void close();

// Anchor buffered writes to disk (RocksDB SyncWAL; BerkeleyDB no-op). Call at checkpoint boundaries.
bool flush();

/**
* @brief Last block hash
* @return Last block hash
Expand Down
9 changes: 9 additions & 0 deletions csdb/src/database.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,15 @@ Database::Iterator::Iterator() = default;

Database::Iterator::~Iterator() = default;

bool Database::put_batch(const std::vector<PendingWrite>& items) {
for (const auto& item : items) {
if (!put(item.hash_key, item.seq_no, item.payload)) {
return false;
}
}
return true;
}

Database::Error Database::last_error() const {
return last_error_map(this).last_error_;
}
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
csdb: add RocksDB backend with runtime backend selection by akaitrade · Pull Request #73 · CREDITSCOM/node · GitHub
Skip to content
csdb: add RocksDB backend with runtime backend selection#73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

akaitrade
wants to merge
1
commit into

base:master
Choose a base branch

from

 Open


csdb: add RocksDB backend with runtime backend selection
#73
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 9 additions & 0 deletions
CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,6 +23,11 @@ option(LEVELDB_BUILD_TESTS "" OFF)
option(LEVELDB_AUTORUN_TESTS "" OFF)
option(DBSQL "Enable PostgeSQL server connection" OFF)

# The node binary always carries both DB backends; runtime selection is via
# config.ini `db_backend` (berkeleydb default, rocksdb opt-in).
set(CSDB_BACKEND "both" CACHE INTERNAL "csdb storage backend (locked to 'both')")
message(STATUS "csdb backend: both (locked)")

IF(DBSQL)
 ADD_DEFINITIONS(-DDBSQL)
ENDIF(DBSQL)
Expand DownExpand Up@@ -88,6 +93,10 @@ endif()
# This large list is necessary to prevent thrift from generating unused stuff
add_definitions(-DUSE_STD_THREAD)

# Disable Boost's MSVC auto-link pragmas; we link Boost targets explicitly via
# CMake. Without this the linker hunts for libboost_*-vc142-mt-s-x64-*.lib.
add_definitions(-DBOOST_ALL_NO_LIB)

if (MSVC)
 MARK_AS_ADVANCED(
 CMAKE_CXX_FLAGS_RELMONITOR
Expand Down

5 changes: 5 additions & 0 deletions
client/config/config.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -962,6 +962,11 @@ void Config::readStorageData(const boost::property_tree::ptree& config) {
 storageData_.checkpointEvery = 1000;
 }
 checkAndSaveValue(data, block, PARAM_NAME_STORAGE_CHECKPOINT_EVERY_MINUTES, storageData_.checkpointEveryMinutes);
 checkAndSaveValue(data, block, "async_write_queue_size", storageData_.asyncWriteQueueSize);
 checkAndSaveValue(data, block, "write_batch_size", storageData_.writeBatchSize);
 checkAndSaveValue(data, block, "db_backend", storageData_.dbBackend);
 checkAndSaveValue(data, block, "rocksdb_block_cache_mb", storageData_.rocksdbBlockCacheMb);
 checkAndSaveValue(data, block, "rocksdb_memtable_mb", storageData_.rocksdbMemtableMb);
}

void Config::readApiData(const boost::property_tree::ptree& config) {
Expand Down

5 changes: 5 additions & 0 deletions
client/config/config.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -62,6 +62,11 @@ struct StorageData {
 size_t checkpointKeep = 5; // retained periodic checkpoints (qs/0 always kept on top)
 size_t checkpointEvery = 500'000; // blocks between periodic checkpoints (rolling history depth = checkpointEvery * checkpointKeep)
 size_t checkpointEveryMinutes = 0; // wall-clock fallback: also save if this many minutes elapsed since last save (0 = disabled; opt-in for slow networks)
 size_t asyncWriteQueueSize = 5000; // bounded queue for the async DB writer
 size_t writeBatchSize = 100; // pools coalesced into one DB write
 std::string dbBackend = "berkeleydb"; // runtime DB backend (CSDB_BACKEND=both builds); set to "rocksdb" to opt in
 size_t rocksdbBlockCacheMb = 1024; // RocksDB shared block cache (MiB); 0 = built-in default
 size_t rocksdbMemtableMb = 256; // RocksDB write_buffer_size (MiB); 0 = built-in default
};

struct ApiData {
Expand Down

26 changes: 23 additions & 3 deletions
csdb/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -10,6 +10,16 @@ option(CSDB_BUILD_BENCHMARK "Bulid benchmark" OFF)
include (TestBigEndian)
TEST_BIG_ENDIAN(CSDB_PLATFORM_IS_BIG_ENDIAN)

if(CSDB_BACKEND STREQUAL "both")
 set(CSDB_BACKEND_SOURCES
 src/database_berkeleydb.cpp include/csdb/database_berkeleydb.hpp
 src/database_rocksdb.cpp include/csdb/database_rocksdb.hpp)
elseif(CSDB_BACKEND STREQUAL "rocksdb")
 set(CSDB_BACKEND_SOURCES src/database_rocksdb.cpp include/csdb/database_rocksdb.hpp)
else()
 set(CSDB_BACKEND_SOURCES src/database_berkeleydb.cpp include/csdb/database_berkeleydb.hpp)
endif()

add_library(${PROJECT_NAME} STATIC
 src/csdb.cpp
 src/amount.cpp
Expand All@@ -29,7 +39,7 @@ add_library(${PROJECT_NAME} STATIC
 src/priv_crypto.cpp
 src/priv_crypto.hpp
 src/database.cpp
src/database_berkeleydb.cpp
${CSDB_BACKEND_SOURCES}
 src/user_field.cpp
 include/csdb/internal/shared_data.hpp
 include/csdb/internal/shared_data_ptr_implementation.hpp
Expand All@@ -48,7 +58,6 @@ add_library(${PROJECT_NAME} STATIC
 include/csdb/wallet.hpp
 include/csdb/storage.hpp
 include/csdb/database.hpp
 include/csdb/database_berkeleydb.hpp
 include/csdb/user_field.hpp
 )

Expand All@@ -72,13 +81,24 @@ target_include_directories(
)

include_directories(${Boost_INCLUDE_DIR})
set(CSDB_BACKEND_LIBS "")
if(CSDB_BACKEND STREQUAL "both")
 list(APPEND CSDB_BACKEND_LIBS BerkeleyDB rocksdb)
 target_compile_definitions(${PROJECT_NAME} PRIVATE CSDB_USE_ROCKSDB CSDB_USE_BERKELEYDB)
elseif(CSDB_BACKEND STREQUAL "rocksdb")
 list(APPEND CSDB_BACKEND_LIBS rocksdb)
 target_compile_definitions(${PROJECT_NAME} PRIVATE CSDB_USE_ROCKSDB)
else()
 list(APPEND CSDB_BACKEND_LIBS BerkeleyDB)
endif()

target_link_libraries(
 ${PROJECT_NAME}
 cscrypto
 Boost::system
 Boost::filesystem
 Boost::disable_autolinking
BerkeleyDB
${CSDB_BACKEND_LIBS}
 lz4
 lib
)
Expand Down

11 changes: 11 additions & 0 deletions
csdb/include/csdb/database.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,13 +41,24 @@ class Database {
 virtual bool remove(const cs::Bytes& key) = 0;
 virtual bool seq_no(const cs::Bytes& key, uint32_t* value) = 0; // sequence from block hash

 // Vectorized put. Backends may override to coalesce writes (e.g. RocksDB WriteBatch).
 struct PendingWrite {
 cs::Bytes hash_key;
 uint32_t seq_no;
 cs::Bytes payload;
 };
 virtual bool put_batch(const std::vector<PendingWrite>& items);

 using Item = std::pair<cs::Bytes, cs::Bytes>;
 using ItemList = std::vector<Item>;
 virtual bool write_batch(const ItemList& items) = 0;

 virtual bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) = 0;
 virtual bool getContractData(const cs::Bytes& key, cs::Bytes& data) = 0;

 // anchor buffered writes to disk at checkpoint boundaries; default no-op (BDB has its own loop)
 virtual bool flush() { return true; }

 class Iterator {
 protected:
 Iterator();
Expand Down

71 changes: 71 additions & 0 deletions
csdb/include/csdb/database_rocksdb.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
// RocksDB-backed csdb::Database. CFs: blocks, seq_no, contracts.
// blocks keyed BE(seq+1). Selected via -DCSDB_BACKEND=rocksdb.

#ifndef _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_
#define _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_

#include <memory>
#include <string>

#include <csdb/database.hpp>

namespace rocksdb {
class DB;
class ColumnFamilyHandle;
class Status;
} // namespace rocksdb

namespace csdb {

class DatabaseRocksDB : public Database {
public:
 DatabaseRocksDB();
 ~DatabaseRocksDB() override;

public:
 // Bulk-load mode (before open): no auto-compaction, disableWAL; caller must compact_full() before close.
 void set_bulk_load(bool yes);
 // Per-write fsync toggle (default off; durability anchored at checkpoint via flush()).
 void set_sync_writes(bool v);

 // Flush + full-range compaction across all CFs; call once after a bulk-load run.
 bool compact_full();

 // Override RocksDB cache/memtable budget (before open); 0 keeps defaults (1 GiB / 256 MiB).
 void set_tuning(uint64_t block_cache_bytes, uint64_t memtable_bytes);

 bool open(const std::string& path);

private:
 bool is_open() const final;
 bool put(const cs::Bytes& key, uint32_t seq_no, const cs::Bytes& value) final;
 bool put_batch(const std::vector<PendingWrite>& items) final;
 bool get(const cs::Bytes& key, cs::Bytes* value) final;
 bool get(const uint32_t seq_no, cs::Bytes* value) final;
 bool remove(const cs::Bytes&) final;
 bool seq_no(const cs::Bytes& key, uint32_t* value) final;
 bool write_batch(const ItemList&) final;
 IteratorPtr new_iterator() final;

 bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) override;
 bool getContractData(const cs::Bytes& key, cs::Bytes& data) override;
 bool flush() override;

private:
 class Iterator;
 void set_last_error_from_status(const rocksdb::Status& s);

private:
 std::unique_ptr<rocksdb::DB> db_;
 rocksdb::ColumnFamilyHandle* cf_blocks_ = nullptr; // default CF
 rocksdb::ColumnFamilyHandle* cf_seq_no_ = nullptr;
 rocksdb::ColumnFamilyHandle* cf_contracts_ = nullptr;
 bool bulk_load_ = false;
 bool sync_writes_ = false; // matches BDB's DB_TXN_NOSYNC; durability anchored at checkpoint via flush()
 uint64_t block_cache_bytes_ = 1ULL << 30; // 1 GiB
 uint64_t memtable_bytes_ = 256ULL << 20; // 256 MiB
};

} // namespace csdb

#endif // _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_

13 changes: 12 additions & 1 deletion
csdb/include/csdb/storage.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,6 +99,9 @@ class Storage final {
 ::std::shared_ptr<Database> db;
 ::cs::Sequence newBlockchainTop = ::cs::kWrongSequence;
 ::cs::Sequence startSequence = 0;
 // 0 = leave Storage defaults (5000 / 100) untouched.
 size_t asyncWriteQueueMax = 0;
 size_t writeBatchSize = 0;
 };

 struct OpenProgress {
Expand DownExpand Up@@ -140,7 +143,12 @@ class Storage final {
 bool open(const ::std::string& path_to_base = ::std::string{},
 OpenCallback callback = nullptr,
 cs::Sequence newBlockchainTop = cs::kWrongSequence,
 cs::Sequence startReadFrom = 0);
 cs::Sequence startReadFrom = 0,
 size_t asyncWriteQueueMax = 5000,
 size_t writeBatchSize = 100,
 uint64_t rocksDbBlockCacheBytes = 0,
 uint64_t rocksDbMemtableBytes = 0,
 const std::string& dbBackend = std::string{});

 /**
 * @brief Creating the storage using the parameters set.
Expand DownExpand Up@@ -170,6 +178,9 @@ class Storage final {
 */
 void close();

 // Anchor buffered writes to disk (RocksDB SyncWAL; BerkeleyDB no-op). Call at checkpoint boundaries.
 bool flush();

 /**
 * @brief Last block hash
 * @return Last block hash
Expand Down

9 changes: 9 additions & 0 deletions
csdb/src/database.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,15 @@ Database::Iterator::Iterator() = default;

Database::Iterator::~Iterator() = default;

bool Database::put_batch(const std::vector<PendingWrite>& items) {
 for (const auto& item : items) {
 if (!put(item.hash_key, item.seq_no, item.payload)) {
 return false;
 }
 }
 return true;
}

Database::Error Database::last_error() const {
 return last_error_map(this).last_error_;
}
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
csdb: add RocksDB backend with runtime backend selection by akaitrade · Pull Request #73 · CREDITSCOM/node · GitHub
Skip to content
csdb: add RocksDB backend with runtime backend selection#73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

akaitrade
wants to merge
1
commit into

base:master
Choose a base branch

from

 Open


csdb: add RocksDB backend with runtime backend selection
#73
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 9 additions & 0 deletions
CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,6 +23,11 @@ option(LEVELDB_BUILD_TESTS "" OFF)
option(LEVELDB_AUTORUN_TESTS "" OFF)
option(DBSQL "Enable PostgeSQL server connection" OFF)

# The node binary always carries both DB backends; runtime selection is via
# config.ini `db_backend` (berkeleydb default, rocksdb opt-in).
set(CSDB_BACKEND "both" CACHE INTERNAL "csdb storage backend (locked to 'both')")
message(STATUS "csdb backend: both (locked)")

IF(DBSQL)
 ADD_DEFINITIONS(-DDBSQL)
ENDIF(DBSQL)
Expand DownExpand Up@@ -88,6 +93,10 @@ endif()
# This large list is necessary to prevent thrift from generating unused stuff
add_definitions(-DUSE_STD_THREAD)

# Disable Boost's MSVC auto-link pragmas; we link Boost targets explicitly via
# CMake. Without this the linker hunts for libboost_*-vc142-mt-s-x64-*.lib.
add_definitions(-DBOOST_ALL_NO_LIB)

if (MSVC)
 MARK_AS_ADVANCED(
 CMAKE_CXX_FLAGS_RELMONITOR
Expand Down

5 changes: 5 additions & 0 deletions
client/config/config.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -962,6 +962,11 @@ void Config::readStorageData(const boost::property_tree::ptree& config) {
 storageData_.checkpointEvery = 1000;
 }
 checkAndSaveValue(data, block, PARAM_NAME_STORAGE_CHECKPOINT_EVERY_MINUTES, storageData_.checkpointEveryMinutes);
 checkAndSaveValue(data, block, "async_write_queue_size", storageData_.asyncWriteQueueSize);
 checkAndSaveValue(data, block, "write_batch_size", storageData_.writeBatchSize);
 checkAndSaveValue(data, block, "db_backend", storageData_.dbBackend);
 checkAndSaveValue(data, block, "rocksdb_block_cache_mb", storageData_.rocksdbBlockCacheMb);
 checkAndSaveValue(data, block, "rocksdb_memtable_mb", storageData_.rocksdbMemtableMb);
}

void Config::readApiData(const boost::property_tree::ptree& config) {
Expand Down

5 changes: 5 additions & 0 deletions
client/config/config.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -62,6 +62,11 @@ struct StorageData {
 size_t checkpointKeep = 5; // retained periodic checkpoints (qs/0 always kept on top)
 size_t checkpointEvery = 500'000; // blocks between periodic checkpoints (rolling history depth = checkpointEvery * checkpointKeep)
 size_t checkpointEveryMinutes = 0; // wall-clock fallback: also save if this many minutes elapsed since last save (0 = disabled; opt-in for slow networks)
 size_t asyncWriteQueueSize = 5000; // bounded queue for the async DB writer
 size_t writeBatchSize = 100; // pools coalesced into one DB write
 std::string dbBackend = "berkeleydb"; // runtime DB backend (CSDB_BACKEND=both builds); set to "rocksdb" to opt in
 size_t rocksdbBlockCacheMb = 1024; // RocksDB shared block cache (MiB); 0 = built-in default
 size_t rocksdbMemtableMb = 256; // RocksDB write_buffer_size (MiB); 0 = built-in default
};

struct ApiData {
Expand Down

26 changes: 23 additions & 3 deletions
csdb/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -10,6 +10,16 @@ option(CSDB_BUILD_BENCHMARK "Bulid benchmark" OFF)
include (TestBigEndian)
TEST_BIG_ENDIAN(CSDB_PLATFORM_IS_BIG_ENDIAN)

if(CSDB_BACKEND STREQUAL "both")
 set(CSDB_BACKEND_SOURCES
 src/database_berkeleydb.cpp include/csdb/database_berkeleydb.hpp
 src/database_rocksdb.cpp include/csdb/database_rocksdb.hpp)
elseif(CSDB_BACKEND STREQUAL "rocksdb")
 set(CSDB_BACKEND_SOURCES src/database_rocksdb.cpp include/csdb/database_rocksdb.hpp)
else()
 set(CSDB_BACKEND_SOURCES src/database_berkeleydb.cpp include/csdb/database_berkeleydb.hpp)
endif()

add_library(${PROJECT_NAME} STATIC
 src/csdb.cpp
 src/amount.cpp
Expand All@@ -29,7 +39,7 @@ add_library(${PROJECT_NAME} STATIC
 src/priv_crypto.cpp
 src/priv_crypto.hpp
 src/database.cpp
src/database_berkeleydb.cpp
${CSDB_BACKEND_SOURCES}
 src/user_field.cpp
 include/csdb/internal/shared_data.hpp
 include/csdb/internal/shared_data_ptr_implementation.hpp
Expand All@@ -48,7 +58,6 @@ add_library(${PROJECT_NAME} STATIC
 include/csdb/wallet.hpp
 include/csdb/storage.hpp
 include/csdb/database.hpp
 include/csdb/database_berkeleydb.hpp
 include/csdb/user_field.hpp
 )

Expand All@@ -72,13 +81,24 @@ target_include_directories(
)

include_directories(${Boost_INCLUDE_DIR})
set(CSDB_BACKEND_LIBS "")
if(CSDB_BACKEND STREQUAL "both")
 list(APPEND CSDB_BACKEND_LIBS BerkeleyDB rocksdb)
 target_compile_definitions(${PROJECT_NAME} PRIVATE CSDB_USE_ROCKSDB CSDB_USE_BERKELEYDB)
elseif(CSDB_BACKEND STREQUAL "rocksdb")
 list(APPEND CSDB_BACKEND_LIBS rocksdb)
 target_compile_definitions(${PROJECT_NAME} PRIVATE CSDB_USE_ROCKSDB)
else()
 list(APPEND CSDB_BACKEND_LIBS BerkeleyDB)
endif()

target_link_libraries(
 ${PROJECT_NAME}
 cscrypto
 Boost::system
 Boost::filesystem
 Boost::disable_autolinking
BerkeleyDB
${CSDB_BACKEND_LIBS}
 lz4
 lib
)
Expand Down

11 changes: 11 additions & 0 deletions
csdb/include/csdb/database.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,13 +41,24 @@ class Database {
 virtual bool remove(const cs::Bytes& key) = 0;
 virtual bool seq_no(const cs::Bytes& key, uint32_t* value) = 0; // sequence from block hash

 // Vectorized put. Backends may override to coalesce writes (e.g. RocksDB WriteBatch).
 struct PendingWrite {
 cs::Bytes hash_key;
 uint32_t seq_no;
 cs::Bytes payload;
 };
 virtual bool put_batch(const std::vector<PendingWrite>& items);

 using Item = std::pair<cs::Bytes, cs::Bytes>;
 using ItemList = std::vector<Item>;
 virtual bool write_batch(const ItemList& items) = 0;

 virtual bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) = 0;
 virtual bool getContractData(const cs::Bytes& key, cs::Bytes& data) = 0;

 // anchor buffered writes to disk at checkpoint boundaries; default no-op (BDB has its own loop)
 virtual bool flush() { return true; }

 class Iterator {
 protected:
 Iterator();
Expand Down

71 changes: 71 additions & 0 deletions
csdb/include/csdb/database_rocksdb.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
// RocksDB-backed csdb::Database. CFs: blocks, seq_no, contracts.
// blocks keyed BE(seq+1). Selected via -DCSDB_BACKEND=rocksdb.

#ifndef _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_
#define _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_

#include <memory>
#include <string>

#include <csdb/database.hpp>

namespace rocksdb {
class DB;
class ColumnFamilyHandle;
class Status;
} // namespace rocksdb

namespace csdb {

class DatabaseRocksDB : public Database {
public:
 DatabaseRocksDB();
 ~DatabaseRocksDB() override;

public:
 // Bulk-load mode (before open): no auto-compaction, disableWAL; caller must compact_full() before close.
 void set_bulk_load(bool yes);
 // Per-write fsync toggle (default off; durability anchored at checkpoint via flush()).
 void set_sync_writes(bool v);

 // Flush + full-range compaction across all CFs; call once after a bulk-load run.
 bool compact_full();

 // Override RocksDB cache/memtable budget (before open); 0 keeps defaults (1 GiB / 256 MiB).
 void set_tuning(uint64_t block_cache_bytes, uint64_t memtable_bytes);

 bool open(const std::string& path);

private:
 bool is_open() const final;
 bool put(const cs::Bytes& key, uint32_t seq_no, const cs::Bytes& value) final;
 bool put_batch(const std::vector<PendingWrite>& items) final;
 bool get(const cs::Bytes& key, cs::Bytes* value) final;
 bool get(const uint32_t seq_no, cs::Bytes* value) final;
 bool remove(const cs::Bytes&) final;
 bool seq_no(const cs::Bytes& key, uint32_t* value) final;
 bool write_batch(const ItemList&) final;
 IteratorPtr new_iterator() final;

 bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) override;
 bool getContractData(const cs::Bytes& key, cs::Bytes& data) override;
 bool flush() override;

private:
 class Iterator;
 void set_last_error_from_status(const rocksdb::Status& s);

private:
 std::unique_ptr<rocksdb::DB> db_;
 rocksdb::ColumnFamilyHandle* cf_blocks_ = nullptr; // default CF
 rocksdb::ColumnFamilyHandle* cf_seq_no_ = nullptr;
 rocksdb::ColumnFamilyHandle* cf_contracts_ = nullptr;
 bool bulk_load_ = false;
 bool sync_writes_ = false; // matches BDB's DB_TXN_NOSYNC; durability anchored at checkpoint via flush()
 uint64_t block_cache_bytes_ = 1ULL << 30; // 1 GiB
 uint64_t memtable_bytes_ = 256ULL << 20; // 256 MiB
};

} // namespace csdb

#endif // _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_

13 changes: 12 additions & 1 deletion
csdb/include/csdb/storage.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,6 +99,9 @@ class Storage final {
 ::std::shared_ptr<Database> db;
 ::cs::Sequence newBlockchainTop = ::cs::kWrongSequence;
 ::cs::Sequence startSequence = 0;
 // 0 = leave Storage defaults (5000 / 100) untouched.
 size_t asyncWriteQueueMax = 0;
 size_t writeBatchSize = 0;
 };

 struct OpenProgress {
Expand DownExpand Up@@ -140,7 +143,12 @@ class Storage final {
 bool open(const ::std::string& path_to_base = ::std::string{},
 OpenCallback callback = nullptr,
 cs::Sequence newBlockchainTop = cs::kWrongSequence,
 cs::Sequence startReadFrom = 0);
 cs::Sequence startReadFrom = 0,
 size_t asyncWriteQueueMax = 5000,
 size_t writeBatchSize = 100,
 uint64_t rocksDbBlockCacheBytes = 0,
 uint64_t rocksDbMemtableBytes = 0,
 const std::string& dbBackend = std::string{});

 /**
 * @brief Creating the storage using the parameters set.
Expand DownExpand Up@@ -170,6 +178,9 @@ class Storage final {
 */
 void close();

 // Anchor buffered writes to disk (RocksDB SyncWAL; BerkeleyDB no-op). Call at checkpoint boundaries.
 bool flush();

 /**
 * @brief Last block hash
 * @return Last block hash
Expand Down

9 changes: 9 additions & 0 deletions
csdb/src/database.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,15 @@ Database::Iterator::Iterator() = default;

Database::Iterator::~Iterator() = default;

bool Database::put_batch(const std::vector<PendingWrite>& items) {
 for (const auto& item : items) {
 if (!put(item.hash_key, item.seq_no, item.payload)) {
 return false;
 }
 }
 return true;
}

Database::Error Database::last_error() const {
 return last_error_map(this).last_error_;
}
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
csdb: add RocksDB backend with runtime backend selection by akaitrade · Pull Request #73 · CREDITSCOM/node · GitHub
Skip to content
csdb: add RocksDB backend with runtime backend selection#73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

akaitrade
wants to merge
1
commit into

base:master
Choose a base branch

from

 Open


csdb: add RocksDB backend with runtime backend selection
#73
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 9 additions & 0 deletions
CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,6 +23,11 @@ option(LEVELDB_BUILD_TESTS "" OFF)
option(LEVELDB_AUTORUN_TESTS "" OFF)
option(DBSQL "Enable PostgeSQL server connection" OFF)

# The node binary always carries both DB backends; runtime selection is via
# config.ini `db_backend` (berkeleydb default, rocksdb opt-in).
set(CSDB_BACKEND "both" CACHE INTERNAL "csdb storage backend (locked to 'both')")
message(STATUS "csdb backend: both (locked)")

IF(DBSQL)
 ADD_DEFINITIONS(-DDBSQL)
ENDIF(DBSQL)
Expand DownExpand Up@@ -88,6 +93,10 @@ endif()
# This large list is necessary to prevent thrift from generating unused stuff
add_definitions(-DUSE_STD_THREAD)

# Disable Boost's MSVC auto-link pragmas; we link Boost targets explicitly via
# CMake. Without this the linker hunts for libboost_*-vc142-mt-s-x64-*.lib.
add_definitions(-DBOOST_ALL_NO_LIB)

if (MSVC)
 MARK_AS_ADVANCED(
 CMAKE_CXX_FLAGS_RELMONITOR
Expand Down

5 changes: 5 additions & 0 deletions
client/config/config.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -962,6 +962,11 @@ void Config::readStorageData(const boost::property_tree::ptree& config) {
 storageData_.checkpointEvery = 1000;
 }
 checkAndSaveValue(data, block, PARAM_NAME_STORAGE_CHECKPOINT_EVERY_MINUTES, storageData_.checkpointEveryMinutes);
 checkAndSaveValue(data, block, "async_write_queue_size", storageData_.asyncWriteQueueSize);
 checkAndSaveValue(data, block, "write_batch_size", storageData_.writeBatchSize);
 checkAndSaveValue(data, block, "db_backend", storageData_.dbBackend);
 checkAndSaveValue(data, block, "rocksdb_block_cache_mb", storageData_.rocksdbBlockCacheMb);
 checkAndSaveValue(data, block, "rocksdb_memtable_mb", storageData_.rocksdbMemtableMb);
}

void Config::readApiData(const boost::property_tree::ptree& config) {
Expand Down

5 changes: 5 additions & 0 deletions
client/config/config.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -62,6 +62,11 @@ struct StorageData {
 size_t checkpointKeep = 5; // retained periodic checkpoints (qs/0 always kept on top)
 size_t checkpointEvery = 500'000; // blocks between periodic checkpoints (rolling history depth = checkpointEvery * checkpointKeep)
 size_t checkpointEveryMinutes = 0; // wall-clock fallback: also save if this many minutes elapsed since last save (0 = disabled; opt-in for slow networks)
 size_t asyncWriteQueueSize = 5000; // bounded queue for the async DB writer
 size_t writeBatchSize = 100; // pools coalesced into one DB write
 std::string dbBackend = "berkeleydb"; // runtime DB backend (CSDB_BACKEND=both builds); set to "rocksdb" to opt in
 size_t rocksdbBlockCacheMb = 1024; // RocksDB shared block cache (MiB); 0 = built-in default
 size_t rocksdbMemtableMb = 256; // RocksDB write_buffer_size (MiB); 0 = built-in default
};

struct ApiData {
Expand Down

26 changes: 23 additions & 3 deletions
csdb/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -10,6 +10,16 @@ option(CSDB_BUILD_BENCHMARK "Bulid benchmark" OFF)
include (TestBigEndian)
TEST_BIG_ENDIAN(CSDB_PLATFORM_IS_BIG_ENDIAN)

if(CSDB_BACKEND STREQUAL "both")
 set(CSDB_BACKEND_SOURCES
 src/database_berkeleydb.cpp include/csdb/database_berkeleydb.hpp
 src/database_rocksdb.cpp include/csdb/database_rocksdb.hpp)
elseif(CSDB_BACKEND STREQUAL "rocksdb")
 set(CSDB_BACKEND_SOURCES src/database_rocksdb.cpp include/csdb/database_rocksdb.hpp)
else()
 set(CSDB_BACKEND_SOURCES src/database_berkeleydb.cpp include/csdb/database_berkeleydb.hpp)
endif()

add_library(${PROJECT_NAME} STATIC
 src/csdb.cpp
 src/amount.cpp
Expand All@@ -29,7 +39,7 @@ add_library(${PROJECT_NAME} STATIC
 src/priv_crypto.cpp
 src/priv_crypto.hpp
 src/database.cpp
src/database_berkeleydb.cpp
${CSDB_BACKEND_SOURCES}
 src/user_field.cpp
 include/csdb/internal/shared_data.hpp
 include/csdb/internal/shared_data_ptr_implementation.hpp
Expand All@@ -48,7 +58,6 @@ add_library(${PROJECT_NAME} STATIC
 include/csdb/wallet.hpp
 include/csdb/storage.hpp
 include/csdb/database.hpp
 include/csdb/database_berkeleydb.hpp
 include/csdb/user_field.hpp
 )

Expand All@@ -72,13 +81,24 @@ target_include_directories(
)

include_directories(${Boost_INCLUDE_DIR})
set(CSDB_BACKEND_LIBS "")
if(CSDB_BACKEND STREQUAL "both")
 list(APPEND CSDB_BACKEND_LIBS BerkeleyDB rocksdb)
 target_compile_definitions(${PROJECT_NAME} PRIVATE CSDB_USE_ROCKSDB CSDB_USE_BERKELEYDB)
elseif(CSDB_BACKEND STREQUAL "rocksdb")
 list(APPEND CSDB_BACKEND_LIBS rocksdb)
 target_compile_definitions(${PROJECT_NAME} PRIVATE CSDB_USE_ROCKSDB)
else()
 list(APPEND CSDB_BACKEND_LIBS BerkeleyDB)
endif()

target_link_libraries(
 ${PROJECT_NAME}
 cscrypto
 Boost::system
 Boost::filesystem
 Boost::disable_autolinking
BerkeleyDB
${CSDB_BACKEND_LIBS}
 lz4
 lib
)
Expand Down

11 changes: 11 additions & 0 deletions
csdb/include/csdb/database.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,13 +41,24 @@ class Database {
 virtual bool remove(const cs::Bytes& key) = 0;
 virtual bool seq_no(const cs::Bytes& key, uint32_t* value) = 0; // sequence from block hash

 // Vectorized put. Backends may override to coalesce writes (e.g. RocksDB WriteBatch).
 struct PendingWrite {
 cs::Bytes hash_key;
 uint32_t seq_no;
 cs::Bytes payload;
 };
 virtual bool put_batch(const std::vector<PendingWrite>& items);

 using Item = std::pair<cs::Bytes, cs::Bytes>;
 using ItemList = std::vector<Item>;
 virtual bool write_batch(const ItemList& items) = 0;

 virtual bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) = 0;
 virtual bool getContractData(const cs::Bytes& key, cs::Bytes& data) = 0;

 // anchor buffered writes to disk at checkpoint boundaries; default no-op (BDB has its own loop)
 virtual bool flush() { return true; }

 class Iterator {
 protected:
 Iterator();
Expand Down

71 changes: 71 additions & 0 deletions
csdb/include/csdb/database_rocksdb.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
// RocksDB-backed csdb::Database. CFs: blocks, seq_no, contracts.
// blocks keyed BE(seq+1). Selected via -DCSDB_BACKEND=rocksdb.

#ifndef _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_
#define _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_

#include <memory>
#include <string>

#include <csdb/database.hpp>

namespace rocksdb {
class DB;
class ColumnFamilyHandle;
class Status;
} // namespace rocksdb

namespace csdb {

class DatabaseRocksDB : public Database {
public:
 DatabaseRocksDB();
 ~DatabaseRocksDB() override;

public:
 // Bulk-load mode (before open): no auto-compaction, disableWAL; caller must compact_full() before close.
 void set_bulk_load(bool yes);
 // Per-write fsync toggle (default off; durability anchored at checkpoint via flush()).
 void set_sync_writes(bool v);

 // Flush + full-range compaction across all CFs; call once after a bulk-load run.
 bool compact_full();

 // Override RocksDB cache/memtable budget (before open); 0 keeps defaults (1 GiB / 256 MiB).
 void set_tuning(uint64_t block_cache_bytes, uint64_t memtable_bytes);

 bool open(const std::string& path);

private:
 bool is_open() const final;
 bool put(const cs::Bytes& key, uint32_t seq_no, const cs::Bytes& value) final;
 bool put_batch(const std::vector<PendingWrite>& items) final;
 bool get(const cs::Bytes& key, cs::Bytes* value) final;
 bool get(const uint32_t seq_no, cs::Bytes* value) final;
 bool remove(const cs::Bytes&) final;
 bool seq_no(const cs::Bytes& key, uint32_t* value) final;
 bool write_batch(const ItemList&) final;
 IteratorPtr new_iterator() final;

 bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) override;
 bool getContractData(const cs::Bytes& key, cs::Bytes& data) override;
 bool flush() override;

private:
 class Iterator;
 void set_last_error_from_status(const rocksdb::Status& s);

private:
 std::unique_ptr<rocksdb::DB> db_;
 rocksdb::ColumnFamilyHandle* cf_blocks_ = nullptr; // default CF
 rocksdb::ColumnFamilyHandle* cf_seq_no_ = nullptr;
 rocksdb::ColumnFamilyHandle* cf_contracts_ = nullptr;
 bool bulk_load_ = false;
 bool sync_writes_ = false; // matches BDB's DB_TXN_NOSYNC; durability anchored at checkpoint via flush()
 uint64_t block_cache_bytes_ = 1ULL << 30; // 1 GiB
 uint64_t memtable_bytes_ = 256ULL << 20; // 256 MiB
};

} // namespace csdb

#endif // _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_

13 changes: 12 additions & 1 deletion
csdb/include/csdb/storage.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,6 +99,9 @@ class Storage final {
 ::std::shared_ptr<Database> db;
 ::cs::Sequence newBlockchainTop = ::cs::kWrongSequence;
 ::cs::Sequence startSequence = 0;
 // 0 = leave Storage defaults (5000 / 100) untouched.
 size_t asyncWriteQueueMax = 0;
 size_t writeBatchSize = 0;
 };

 struct OpenProgress {
Expand DownExpand Up@@ -140,7 +143,12 @@ class Storage final {
 bool open(const ::std::string& path_to_base = ::std::string{},
 OpenCallback callback = nullptr,
 cs::Sequence newBlockchainTop = cs::kWrongSequence,
 cs::Sequence startReadFrom = 0);
 cs::Sequence startReadFrom = 0,
 size_t asyncWriteQueueMax = 5000,
 size_t writeBatchSize = 100,
 uint64_t rocksDbBlockCacheBytes = 0,
 uint64_t rocksDbMemtableBytes = 0,
 const std::string& dbBackend = std::string{});

 /**
 * @brief Creating the storage using the parameters set.
Expand DownExpand Up@@ -170,6 +178,9 @@ class Storage final {
 */
 void close();

 // Anchor buffered writes to disk (RocksDB SyncWAL; BerkeleyDB no-op). Call at checkpoint boundaries.
 bool flush();

 /**
 * @brief Last block hash
 * @return Last block hash
Expand Down

9 changes: 9 additions & 0 deletions
csdb/src/database.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,15 @@ Database::Iterator::Iterator() = default;

Database::Iterator::~Iterator() = default;

bool Database::put_batch(const std::vector<PendingWrite>& items) {
 for (const auto& item : items) {
 if (!put(item.hash_key, item.seq_no, item.payload)) {
 return false;
 }
 }
 return true;
}

Database::Error Database::last_error() const {
 return last_error_map(this).last_error_;
}
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
csdb: add RocksDB backend with runtime backend selection by akaitrade · Pull Request #73 · CREDITSCOM/node · GitHub
Skip to content
csdb: add RocksDB backend with runtime backend selection#73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

akaitrade
wants to merge
1
commit into

base:master
Choose a base branch

from

 Open


csdb: add RocksDB backend with runtime backend selection
#73
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 9 additions & 0 deletions
CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,6 +23,11 @@ option(LEVELDB_BUILD_TESTS "" OFF)
option(LEVELDB_AUTORUN_TESTS "" OFF)
option(DBSQL "Enable PostgeSQL server connection" OFF)

# The node binary always carries both DB backends; runtime selection is via
# config.ini `db_backend` (berkeleydb default, rocksdb opt-in).
set(CSDB_BACKEND "both" CACHE INTERNAL "csdb storage backend (locked to 'both')")
message(STATUS "csdb backend: both (locked)")

IF(DBSQL)
 ADD_DEFINITIONS(-DDBSQL)
ENDIF(DBSQL)
Expand DownExpand Up@@ -88,6 +93,10 @@ endif()
# This large list is necessary to prevent thrift from generating unused stuff
add_definitions(-DUSE_STD_THREAD)

# Disable Boost's MSVC auto-link pragmas; we link Boost targets explicitly via
# CMake. Without this the linker hunts for libboost_*-vc142-mt-s-x64-*.lib.
add_definitions(-DBOOST_ALL_NO_LIB)

if (MSVC)
 MARK_AS_ADVANCED(
 CMAKE_CXX_FLAGS_RELMONITOR
Expand Down

5 changes: 5 additions & 0 deletions
client/config/config.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -962,6 +962,11 @@ void Config::readStorageData(const boost::property_tree::ptree& config) {
 storageData_.checkpointEvery = 1000;
 }
 checkAndSaveValue(data, block, PARAM_NAME_STORAGE_CHECKPOINT_EVERY_MINUTES, storageData_.checkpointEveryMinutes);
 checkAndSaveValue(data, block, "async_write_queue_size", storageData_.asyncWriteQueueSize);
 checkAndSaveValue(data, block, "write_batch_size", storageData_.writeBatchSize);
 checkAndSaveValue(data, block, "db_backend", storageData_.dbBackend);
 checkAndSaveValue(data, block, "rocksdb_block_cache_mb", storageData_.rocksdbBlockCacheMb);
 checkAndSaveValue(data, block, "rocksdb_memtable_mb", storageData_.rocksdbMemtableMb);
}

void Config::readApiData(const boost::property_tree::ptree& config) {
Expand Down

5 changes: 5 additions & 0 deletions
client/config/config.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -62,6 +62,11 @@ struct StorageData {
 size_t checkpointKeep = 5; // retained periodic checkpoints (qs/0 always kept on top)
 size_t checkpointEvery = 500'000; // blocks between periodic checkpoints (rolling history depth = checkpointEvery * checkpointKeep)
 size_t checkpointEveryMinutes = 0; // wall-clock fallback: also save if this many minutes elapsed since last save (0 = disabled; opt-in for slow networks)
 size_t asyncWriteQueueSize = 5000; // bounded queue for the async DB writer
 size_t writeBatchSize = 100; // pools coalesced into one DB write
 std::string dbBackend = "berkeleydb"; // runtime DB backend (CSDB_BACKEND=both builds); set to "rocksdb" to opt in
 size_t rocksdbBlockCacheMb = 1024; // RocksDB shared block cache (MiB); 0 = built-in default
 size_t rocksdbMemtableMb = 256; // RocksDB write_buffer_size (MiB); 0 = built-in default
};

struct ApiData {
Expand Down

26 changes: 23 additions & 3 deletions
csdb/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -10,6 +10,16 @@ option(CSDB_BUILD_BENCHMARK "Bulid benchmark" OFF)
include (TestBigEndian)
TEST_BIG_ENDIAN(CSDB_PLATFORM_IS_BIG_ENDIAN)

if(CSDB_BACKEND STREQUAL "both")
 set(CSDB_BACKEND_SOURCES
 src/database_berkeleydb.cpp include/csdb/database_berkeleydb.hpp
 src/database_rocksdb.cpp include/csdb/database_rocksdb.hpp)
elseif(CSDB_BACKEND STREQUAL "rocksdb")
 set(CSDB_BACKEND_SOURCES src/database_rocksdb.cpp include/csdb/database_rocksdb.hpp)
else()
 set(CSDB_BACKEND_SOURCES src/database_berkeleydb.cpp include/csdb/database_berkeleydb.hpp)
endif()

add_library(${PROJECT_NAME} STATIC
 src/csdb.cpp
 src/amount.cpp
Expand All@@ -29,7 +39,7 @@ add_library(${PROJECT_NAME} STATIC
 src/priv_crypto.cpp
 src/priv_crypto.hpp
 src/database.cpp
src/database_berkeleydb.cpp
${CSDB_BACKEND_SOURCES}
 src/user_field.cpp
 include/csdb/internal/shared_data.hpp
 include/csdb/internal/shared_data_ptr_implementation.hpp
Expand All@@ -48,7 +58,6 @@ add_library(${PROJECT_NAME} STATIC
 include/csdb/wallet.hpp
 include/csdb/storage.hpp
 include/csdb/database.hpp
 include/csdb/database_berkeleydb.hpp
 include/csdb/user_field.hpp
 )

Expand All@@ -72,13 +81,24 @@ target_include_directories(
)

include_directories(${Boost_INCLUDE_DIR})
set(CSDB_BACKEND_LIBS "")
if(CSDB_BACKEND STREQUAL "both")
 list(APPEND CSDB_BACKEND_LIBS BerkeleyDB rocksdb)
 target_compile_definitions(${PROJECT_NAME} PRIVATE CSDB_USE_ROCKSDB CSDB_USE_BERKELEYDB)
elseif(CSDB_BACKEND STREQUAL "rocksdb")
 list(APPEND CSDB_BACKEND_LIBS rocksdb)
 target_compile_definitions(${PROJECT_NAME} PRIVATE CSDB_USE_ROCKSDB)
else()
 list(APPEND CSDB_BACKEND_LIBS BerkeleyDB)
endif()

target_link_libraries(
 ${PROJECT_NAME}
 cscrypto
 Boost::system
 Boost::filesystem
 Boost::disable_autolinking
BerkeleyDB
${CSDB_BACKEND_LIBS}
 lz4
 lib
)
Expand Down

11 changes: 11 additions & 0 deletions
csdb/include/csdb/database.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,13 +41,24 @@ class Database {
 virtual bool remove(const cs::Bytes& key) = 0;
 virtual bool seq_no(const cs::Bytes& key, uint32_t* value) = 0; // sequence from block hash

 // Vectorized put. Backends may override to coalesce writes (e.g. RocksDB WriteBatch).
 struct PendingWrite {
 cs::Bytes hash_key;
 uint32_t seq_no;
 cs::Bytes payload;
 };
 virtual bool put_batch(const std::vector<PendingWrite>& items);

 using Item = std::pair<cs::Bytes, cs::Bytes>;
 using ItemList = std::vector<Item>;
 virtual bool write_batch(const ItemList& items) = 0;

 virtual bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) = 0;
 virtual bool getContractData(const cs::Bytes& key, cs::Bytes& data) = 0;

 // anchor buffered writes to disk at checkpoint boundaries; default no-op (BDB has its own loop)
 virtual bool flush() { return true; }

 class Iterator {
 protected:
 Iterator();
Expand Down

71 changes: 71 additions & 0 deletions
csdb/include/csdb/database_rocksdb.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
// RocksDB-backed csdb::Database. CFs: blocks, seq_no, contracts.
// blocks keyed BE(seq+1). Selected via -DCSDB_BACKEND=rocksdb.

#ifndef _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_
#define _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_

#include <memory>
#include <string>

#include <csdb/database.hpp>

namespace rocksdb {
class DB;
class ColumnFamilyHandle;
class Status;
} // namespace rocksdb

namespace csdb {

class DatabaseRocksDB : public Database {
public:
 DatabaseRocksDB();
 ~DatabaseRocksDB() override;

public:
 // Bulk-load mode (before open): no auto-compaction, disableWAL; caller must compact_full() before close.
 void set_bulk_load(bool yes);
 // Per-write fsync toggle (default off; durability anchored at checkpoint via flush()).
 void set_sync_writes(bool v);

 // Flush + full-range compaction across all CFs; call once after a bulk-load run.
 bool compact_full();

 // Override RocksDB cache/memtable budget (before open); 0 keeps defaults (1 GiB / 256 MiB).
 void set_tuning(uint64_t block_cache_bytes, uint64_t memtable_bytes);

 bool open(const std::string& path);

private:
 bool is_open() const final;
 bool put(const cs::Bytes& key, uint32_t seq_no, const cs::Bytes& value) final;
 bool put_batch(const std::vector<PendingWrite>& items) final;
 bool get(const cs::Bytes& key, cs::Bytes* value) final;
 bool get(const uint32_t seq_no, cs::Bytes* value) final;
 bool remove(const cs::Bytes&) final;
 bool seq_no(const cs::Bytes& key, uint32_t* value) final;
 bool write_batch(const ItemList&) final;
 IteratorPtr new_iterator() final;

 bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) override;
 bool getContractData(const cs::Bytes& key, cs::Bytes& data) override;
 bool flush() override;

private:
 class Iterator;
 void set_last_error_from_status(const rocksdb::Status& s);

private:
 std::unique_ptr<rocksdb::DB> db_;
 rocksdb::ColumnFamilyHandle* cf_blocks_ = nullptr; // default CF
 rocksdb::ColumnFamilyHandle* cf_seq_no_ = nullptr;
 rocksdb::ColumnFamilyHandle* cf_contracts_ = nullptr;
 bool bulk_load_ = false;
 bool sync_writes_ = false; // matches BDB's DB_TXN_NOSYNC; durability anchored at checkpoint via flush()
 uint64_t block_cache_bytes_ = 1ULL << 30; // 1 GiB
 uint64_t memtable_bytes_ = 256ULL << 20; // 256 MiB
};

} // namespace csdb

#endif // _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_

13 changes: 12 additions & 1 deletion
csdb/include/csdb/storage.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,6 +99,9 @@ class Storage final {
 ::std::shared_ptr<Database> db;
 ::cs::Sequence newBlockchainTop = ::cs::kWrongSequence;
 ::cs::Sequence startSequence = 0;
 // 0 = leave Storage defaults (5000 / 100) untouched.
 size_t asyncWriteQueueMax = 0;
 size_t writeBatchSize = 0;
 };

 struct OpenProgress {
Expand DownExpand Up@@ -140,7 +143,12 @@ class Storage final {
 bool open(const ::std::string& path_to_base = ::std::string{},
 OpenCallback callback = nullptr,
 cs::Sequence newBlockchainTop = cs::kWrongSequence,
 cs::Sequence startReadFrom = 0);
 cs::Sequence startReadFrom = 0,
 size_t asyncWriteQueueMax = 5000,
 size_t writeBatchSize = 100,
 uint64_t rocksDbBlockCacheBytes = 0,
 uint64_t rocksDbMemtableBytes = 0,
 const std::string& dbBackend = std::string{});

 /**
 * @brief Creating the storage using the parameters set.
Expand DownExpand Up@@ -170,6 +178,9 @@ class Storage final {
 */
 void close();

 // Anchor buffered writes to disk (RocksDB SyncWAL; BerkeleyDB no-op). Call at checkpoint boundaries.
 bool flush();

 /**
 * @brief Last block hash
 * @return Last block hash
Expand Down

9 changes: 9 additions & 0 deletions
csdb/src/database.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,15 @@ Database::Iterator::Iterator() = default;

Database::Iterator::~Iterator() = default;

bool Database::put_batch(const std::vector<PendingWrite>& items) {
 for (const auto& item : items) {
 if (!put(item.hash_key, item.seq_no, item.payload)) {
 return false;
 }
 }
 return true;
}

Database::Error Database::last_error() const {
 return last_error_map(this).last_error_;
}
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
csdb: add RocksDB backend with runtime backend selection by akaitrade · Pull Request #73 · CREDITSCOM/node · GitHub
Skip to content
csdb: add RocksDB backend with runtime backend selection#73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

akaitrade
wants to merge
1
commit into

base:master
Choose a base branch

from

 Open


csdb: add RocksDB backend with runtime backend selection
#73
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 9 additions & 0 deletions
CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,6 +23,11 @@ option(LEVELDB_BUILD_TESTS "" OFF)
option(LEVELDB_AUTORUN_TESTS "" OFF)
option(DBSQL "Enable PostgeSQL server connection" OFF)

# The node binary always carries both DB backends; runtime selection is via
# config.ini `db_backend` (berkeleydb default, rocksdb opt-in).
set(CSDB_BACKEND "both" CACHE INTERNAL "csdb storage backend (locked to 'both')")
message(STATUS "csdb backend: both (locked)")

IF(DBSQL)
 ADD_DEFINITIONS(-DDBSQL)
ENDIF(DBSQL)
Expand DownExpand Up@@ -88,6 +93,10 @@ endif()
# This large list is necessary to prevent thrift from generating unused stuff
add_definitions(-DUSE_STD_THREAD)

# Disable Boost's MSVC auto-link pragmas; we link Boost targets explicitly via
# CMake. Without this the linker hunts for libboost_*-vc142-mt-s-x64-*.lib.
add_definitions(-DBOOST_ALL_NO_LIB)

if (MSVC)
 MARK_AS_ADVANCED(
 CMAKE_CXX_FLAGS_RELMONITOR
Expand Down

5 changes: 5 additions & 0 deletions
client/config/config.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -962,6 +962,11 @@ void Config::readStorageData(const boost::property_tree::ptree& config) {
 storageData_.checkpointEvery = 1000;
 }
 checkAndSaveValue(data, block, PARAM_NAME_STORAGE_CHECKPOINT_EVERY_MINUTES, storageData_.checkpointEveryMinutes);
 checkAndSaveValue(data, block, "async_write_queue_size", storageData_.asyncWriteQueueSize);
 checkAndSaveValue(data, block, "write_batch_size", storageData_.writeBatchSize);
 checkAndSaveValue(data, block, "db_backend", storageData_.dbBackend);
 checkAndSaveValue(data, block, "rocksdb_block_cache_mb", storageData_.rocksdbBlockCacheMb);
 checkAndSaveValue(data, block, "rocksdb_memtable_mb", storageData_.rocksdbMemtableMb);
}

void Config::readApiData(const boost::property_tree::ptree& config) {
Expand Down

5 changes: 5 additions & 0 deletions
client/config/config.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -62,6 +62,11 @@ struct StorageData {
 size_t checkpointKeep = 5; // retained periodic checkpoints (qs/0 always kept on top)
 size_t checkpointEvery = 500'000; // blocks between periodic checkpoints (rolling history depth = checkpointEvery * checkpointKeep)
 size_t checkpointEveryMinutes = 0; // wall-clock fallback: also save if this many minutes elapsed since last save (0 = disabled; opt-in for slow networks)
 size_t asyncWriteQueueSize = 5000; // bounded queue for the async DB writer
 size_t writeBatchSize = 100; // pools coalesced into one DB write
 std::string dbBackend = "berkeleydb"; // runtime DB backend (CSDB_BACKEND=both builds); set to "rocksdb" to opt in
 size_t rocksdbBlockCacheMb = 1024; // RocksDB shared block cache (MiB); 0 = built-in default
 size_t rocksdbMemtableMb = 256; // RocksDB write_buffer_size (MiB); 0 = built-in default
};

struct ApiData {
Expand Down

26 changes: 23 additions & 3 deletions
csdb/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -10,6 +10,16 @@ option(CSDB_BUILD_BENCHMARK "Bulid benchmark" OFF)
include (TestBigEndian)
TEST_BIG_ENDIAN(CSDB_PLATFORM_IS_BIG_ENDIAN)

if(CSDB_BACKEND STREQUAL "both")
 set(CSDB_BACKEND_SOURCES
 src/database_berkeleydb.cpp include/csdb/database_berkeleydb.hpp
 src/database_rocksdb.cpp include/csdb/database_rocksdb.hpp)
elseif(CSDB_BACKEND STREQUAL "rocksdb")
 set(CSDB_BACKEND_SOURCES src/database_rocksdb.cpp include/csdb/database_rocksdb.hpp)
else()
 set(CSDB_BACKEND_SOURCES src/database_berkeleydb.cpp include/csdb/database_berkeleydb.hpp)
endif()

add_library(${PROJECT_NAME} STATIC
 src/csdb.cpp
 src/amount.cpp
Expand All@@ -29,7 +39,7 @@ add_library(${PROJECT_NAME} STATIC
 src/priv_crypto.cpp
 src/priv_crypto.hpp
 src/database.cpp
src/database_berkeleydb.cpp
${CSDB_BACKEND_SOURCES}
 src/user_field.cpp
 include/csdb/internal/shared_data.hpp
 include/csdb/internal/shared_data_ptr_implementation.hpp
Expand All@@ -48,7 +58,6 @@ add_library(${PROJECT_NAME} STATIC
 include/csdb/wallet.hpp
 include/csdb/storage.hpp
 include/csdb/database.hpp
 include/csdb/database_berkeleydb.hpp
 include/csdb/user_field.hpp
 )

Expand All@@ -72,13 +81,24 @@ target_include_directories(
)

include_directories(${Boost_INCLUDE_DIR})
set(CSDB_BACKEND_LIBS "")
if(CSDB_BACKEND STREQUAL "both")
 list(APPEND CSDB_BACKEND_LIBS BerkeleyDB rocksdb)
 target_compile_definitions(${PROJECT_NAME} PRIVATE CSDB_USE_ROCKSDB CSDB_USE_BERKELEYDB)
elseif(CSDB_BACKEND STREQUAL "rocksdb")
 list(APPEND CSDB_BACKEND_LIBS rocksdb)
 target_compile_definitions(${PROJECT_NAME} PRIVATE CSDB_USE_ROCKSDB)
else()
 list(APPEND CSDB_BACKEND_LIBS BerkeleyDB)
endif()

target_link_libraries(
 ${PROJECT_NAME}
 cscrypto
 Boost::system
 Boost::filesystem
 Boost::disable_autolinking
BerkeleyDB
${CSDB_BACKEND_LIBS}
 lz4
 lib
)
Expand Down

11 changes: 11 additions & 0 deletions
csdb/include/csdb/database.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,13 +41,24 @@ class Database {
 virtual bool remove(const cs::Bytes& key) = 0;
 virtual bool seq_no(const cs::Bytes& key, uint32_t* value) = 0; // sequence from block hash

 // Vectorized put. Backends may override to coalesce writes (e.g. RocksDB WriteBatch).
 struct PendingWrite {
 cs::Bytes hash_key;
 uint32_t seq_no;
 cs::Bytes payload;
 };
 virtual bool put_batch(const std::vector<PendingWrite>& items);

 using Item = std::pair<cs::Bytes, cs::Bytes>;
 using ItemList = std::vector<Item>;
 virtual bool write_batch(const ItemList& items) = 0;

 virtual bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) = 0;
 virtual bool getContractData(const cs::Bytes& key, cs::Bytes& data) = 0;

 // anchor buffered writes to disk at checkpoint boundaries; default no-op (BDB has its own loop)
 virtual bool flush() { return true; }

 class Iterator {
 protected:
 Iterator();
Expand Down

71 changes: 71 additions & 0 deletions
csdb/include/csdb/database_rocksdb.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
// RocksDB-backed csdb::Database. CFs: blocks, seq_no, contracts.
// blocks keyed BE(seq+1). Selected via -DCSDB_BACKEND=rocksdb.

#ifndef _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_
#define _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_

#include <memory>
#include <string>

#include <csdb/database.hpp>

namespace rocksdb {
class DB;
class ColumnFamilyHandle;
class Status;
} // namespace rocksdb

namespace csdb {

class DatabaseRocksDB : public Database {
public:
 DatabaseRocksDB();
 ~DatabaseRocksDB() override;

public:
 // Bulk-load mode (before open): no auto-compaction, disableWAL; caller must compact_full() before close.
 void set_bulk_load(bool yes);
 // Per-write fsync toggle (default off; durability anchored at checkpoint via flush()).
 void set_sync_writes(bool v);

 // Flush + full-range compaction across all CFs; call once after a bulk-load run.
 bool compact_full();

 // Override RocksDB cache/memtable budget (before open); 0 keeps defaults (1 GiB / 256 MiB).
 void set_tuning(uint64_t block_cache_bytes, uint64_t memtable_bytes);

 bool open(const std::string& path);

private:
 bool is_open() const final;
 bool put(const cs::Bytes& key, uint32_t seq_no, const cs::Bytes& value) final;
 bool put_batch(const std::vector<PendingWrite>& items) final;
 bool get(const cs::Bytes& key, cs::Bytes* value) final;
 bool get(const uint32_t seq_no, cs::Bytes* value) final;
 bool remove(const cs::Bytes&) final;
 bool seq_no(const cs::Bytes& key, uint32_t* value) final;
 bool write_batch(const ItemList&) final;
 IteratorPtr new_iterator() final;

 bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) override;
 bool getContractData(const cs::Bytes& key, cs::Bytes& data) override;
 bool flush() override;

private:
 class Iterator;
 void set_last_error_from_status(const rocksdb::Status& s);

private:
 std::unique_ptr<rocksdb::DB> db_;
 rocksdb::ColumnFamilyHandle* cf_blocks_ = nullptr; // default CF
 rocksdb::ColumnFamilyHandle* cf_seq_no_ = nullptr;
 rocksdb::ColumnFamilyHandle* cf_contracts_ = nullptr;
 bool bulk_load_ = false;
 bool sync_writes_ = false; // matches BDB's DB_TXN_NOSYNC; durability anchored at checkpoint via flush()
 uint64_t block_cache_bytes_ = 1ULL << 30; // 1 GiB
 uint64_t memtable_bytes_ = 256ULL << 20; // 256 MiB
};

} // namespace csdb

#endif // _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_

13 changes: 12 additions & 1 deletion
csdb/include/csdb/storage.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,6 +99,9 @@ class Storage final {
 ::std::shared_ptr<Database> db;
 ::cs::Sequence newBlockchainTop = ::cs::kWrongSequence;
 ::cs::Sequence startSequence = 0;
 // 0 = leave Storage defaults (5000 / 100) untouched.
 size_t asyncWriteQueueMax = 0;
 size_t writeBatchSize = 0;
 };

 struct OpenProgress {
Expand DownExpand Up@@ -140,7 +143,12 @@ class Storage final {
 bool open(const ::std::string& path_to_base = ::std::string{},
 OpenCallback callback = nullptr,
 cs::Sequence newBlockchainTop = cs::kWrongSequence,
 cs::Sequence startReadFrom = 0);
 cs::Sequence startReadFrom = 0,
 size_t asyncWriteQueueMax = 5000,
 size_t writeBatchSize = 100,
 uint64_t rocksDbBlockCacheBytes = 0,
 uint64_t rocksDbMemtableBytes = 0,
 const std::string& dbBackend = std::string{});

 /**
 * @brief Creating the storage using the parameters set.
Expand DownExpand Up@@ -170,6 +178,9 @@ class Storage final {
 */
 void close();

 // Anchor buffered writes to disk (RocksDB SyncWAL; BerkeleyDB no-op). Call at checkpoint boundaries.
 bool flush();

 /**
 * @brief Last block hash
 * @return Last block hash
Expand Down

9 changes: 9 additions & 0 deletions
csdb/src/database.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,15 @@ Database::Iterator::Iterator() = default;

Database::Iterator::~Iterator() = default;

bool Database::put_batch(const std::vector<PendingWrite>& items) {
 for (const auto& item : items) {
 if (!put(item.hash_key, item.seq_no, item.payload)) {
 return false;
 }
 }
 return true;
}

Database::Error Database::last_error() const {
 return last_error_map(this).last_error_;
}
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
csdb: add RocksDB backend with runtime backend selection by akaitrade · Pull Request #73 · CREDITSCOM/node · GitHub
Skip to content
csdb: add RocksDB backend with runtime backend selection#73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

akaitrade
wants to merge
1
commit into

base:master
Choose a base branch

from

 Open


csdb: add RocksDB backend with runtime backend selection
#73
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 9 additions & 0 deletions
CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,6 +23,11 @@ option(LEVELDB_BUILD_TESTS "" OFF)
option(LEVELDB_AUTORUN_TESTS "" OFF)
option(DBSQL "Enable PostgeSQL server connection" OFF)

# The node binary always carries both DB backends; runtime selection is via
# config.ini `db_backend` (berkeleydb default, rocksdb opt-in).
set(CSDB_BACKEND "both" CACHE INTERNAL "csdb storage backend (locked to 'both')")
message(STATUS "csdb backend: both (locked)")

IF(DBSQL)
 ADD_DEFINITIONS(-DDBSQL)
ENDIF(DBSQL)
Expand DownExpand Up@@ -88,6 +93,10 @@ endif()
# This large list is necessary to prevent thrift from generating unused stuff
add_definitions(-DUSE_STD_THREAD)

# Disable Boost's MSVC auto-link pragmas; we link Boost targets explicitly via
# CMake. Without this the linker hunts for libboost_*-vc142-mt-s-x64-*.lib.
add_definitions(-DBOOST_ALL_NO_LIB)

if (MSVC)
 MARK_AS_ADVANCED(
 CMAKE_CXX_FLAGS_RELMONITOR
Expand Down

5 changes: 5 additions & 0 deletions
client/config/config.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -962,6 +962,11 @@ void Config::readStorageData(const boost::property_tree::ptree& config) {
 storageData_.checkpointEvery = 1000;
 }
 checkAndSaveValue(data, block, PARAM_NAME_STORAGE_CHECKPOINT_EVERY_MINUTES, storageData_.checkpointEveryMinutes);
 checkAndSaveValue(data, block, "async_write_queue_size", storageData_.asyncWriteQueueSize);
 checkAndSaveValue(data, block, "write_batch_size", storageData_.writeBatchSize);
 checkAndSaveValue(data, block, "db_backend", storageData_.dbBackend);
 checkAndSaveValue(data, block, "rocksdb_block_cache_mb", storageData_.rocksdbBlockCacheMb);
 checkAndSaveValue(data, block, "rocksdb_memtable_mb", storageData_.rocksdbMemtableMb);
}

void Config::readApiData(const boost::property_tree::ptree& config) {
Expand Down

5 changes: 5 additions & 0 deletions
client/config/config.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -62,6 +62,11 @@ struct StorageData {
 size_t checkpointKeep = 5; // retained periodic checkpoints (qs/0 always kept on top)
 size_t checkpointEvery = 500'000; // blocks between periodic checkpoints (rolling history depth = checkpointEvery * checkpointKeep)
 size_t checkpointEveryMinutes = 0; // wall-clock fallback: also save if this many minutes elapsed since last save (0 = disabled; opt-in for slow networks)
 size_t asyncWriteQueueSize = 5000; // bounded queue for the async DB writer
 size_t writeBatchSize = 100; // pools coalesced into one DB write
 std::string dbBackend = "berkeleydb"; // runtime DB backend (CSDB_BACKEND=both builds); set to "rocksdb" to opt in
 size_t rocksdbBlockCacheMb = 1024; // RocksDB shared block cache (MiB); 0 = built-in default
 size_t rocksdbMemtableMb = 256; // RocksDB write_buffer_size (MiB); 0 = built-in default
};

struct ApiData {
Expand Down

26 changes: 23 additions & 3 deletions
csdb/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -10,6 +10,16 @@ option(CSDB_BUILD_BENCHMARK "Bulid benchmark" OFF)
include (TestBigEndian)
TEST_BIG_ENDIAN(CSDB_PLATFORM_IS_BIG_ENDIAN)

if(CSDB_BACKEND STREQUAL "both")
 set(CSDB_BACKEND_SOURCES
 src/database_berkeleydb.cpp include/csdb/database_berkeleydb.hpp
 src/database_rocksdb.cpp include/csdb/database_rocksdb.hpp)
elseif(CSDB_BACKEND STREQUAL "rocksdb")
 set(CSDB_BACKEND_SOURCES src/database_rocksdb.cpp include/csdb/database_rocksdb.hpp)
else()
 set(CSDB_BACKEND_SOURCES src/database_berkeleydb.cpp include/csdb/database_berkeleydb.hpp)
endif()

add_library(${PROJECT_NAME} STATIC
 src/csdb.cpp
 src/amount.cpp
Expand All@@ -29,7 +39,7 @@ add_library(${PROJECT_NAME} STATIC
 src/priv_crypto.cpp
 src/priv_crypto.hpp
 src/database.cpp
src/database_berkeleydb.cpp
${CSDB_BACKEND_SOURCES}
 src/user_field.cpp
 include/csdb/internal/shared_data.hpp
 include/csdb/internal/shared_data_ptr_implementation.hpp
Expand All@@ -48,7 +58,6 @@ add_library(${PROJECT_NAME} STATIC
 include/csdb/wallet.hpp
 include/csdb/storage.hpp
 include/csdb/database.hpp
 include/csdb/database_berkeleydb.hpp
 include/csdb/user_field.hpp
 )

Expand All@@ -72,13 +81,24 @@ target_include_directories(
)

include_directories(${Boost_INCLUDE_DIR})
set(CSDB_BACKEND_LIBS "")
if(CSDB_BACKEND STREQUAL "both")
 list(APPEND CSDB_BACKEND_LIBS BerkeleyDB rocksdb)
 target_compile_definitions(${PROJECT_NAME} PRIVATE CSDB_USE_ROCKSDB CSDB_USE_BERKELEYDB)
elseif(CSDB_BACKEND STREQUAL "rocksdb")
 list(APPEND CSDB_BACKEND_LIBS rocksdb)
 target_compile_definitions(${PROJECT_NAME} PRIVATE CSDB_USE_ROCKSDB)
else()
 list(APPEND CSDB_BACKEND_LIBS BerkeleyDB)
endif()

target_link_libraries(
 ${PROJECT_NAME}
 cscrypto
 Boost::system
 Boost::filesystem
 Boost::disable_autolinking
BerkeleyDB
${CSDB_BACKEND_LIBS}
 lz4
 lib
)
Expand Down

11 changes: 11 additions & 0 deletions
csdb/include/csdb/database.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,13 +41,24 @@ class Database {
 virtual bool remove(const cs::Bytes& key) = 0;
 virtual bool seq_no(const cs::Bytes& key, uint32_t* value) = 0; // sequence from block hash

 // Vectorized put. Backends may override to coalesce writes (e.g. RocksDB WriteBatch).
 struct PendingWrite {
 cs::Bytes hash_key;
 uint32_t seq_no;
 cs::Bytes payload;
 };
 virtual bool put_batch(const std::vector<PendingWrite>& items);

 using Item = std::pair<cs::Bytes, cs::Bytes>;
 using ItemList = std::vector<Item>;
 virtual bool write_batch(const ItemList& items) = 0;

 virtual bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) = 0;
 virtual bool getContractData(const cs::Bytes& key, cs::Bytes& data) = 0;

 // anchor buffered writes to disk at checkpoint boundaries; default no-op (BDB has its own loop)
 virtual bool flush() { return true; }

 class Iterator {
 protected:
 Iterator();
Expand Down

71 changes: 71 additions & 0 deletions
csdb/include/csdb/database_rocksdb.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
// RocksDB-backed csdb::Database. CFs: blocks, seq_no, contracts.
// blocks keyed BE(seq+1). Selected via -DCSDB_BACKEND=rocksdb.

#ifndef _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_
#define _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_

#include <memory>
#include <string>

#include <csdb/database.hpp>

namespace rocksdb {
class DB;
class ColumnFamilyHandle;
class Status;
} // namespace rocksdb

namespace csdb {

class DatabaseRocksDB : public Database {
public:
 DatabaseRocksDB();
 ~DatabaseRocksDB() override;

public:
 // Bulk-load mode (before open): no auto-compaction, disableWAL; caller must compact_full() before close.
 void set_bulk_load(bool yes);
 // Per-write fsync toggle (default off; durability anchored at checkpoint via flush()).
 void set_sync_writes(bool v);

 // Flush + full-range compaction across all CFs; call once after a bulk-load run.
 bool compact_full();

 // Override RocksDB cache/memtable budget (before open); 0 keeps defaults (1 GiB / 256 MiB).
 void set_tuning(uint64_t block_cache_bytes, uint64_t memtable_bytes);

 bool open(const std::string& path);

private:
 bool is_open() const final;
 bool put(const cs::Bytes& key, uint32_t seq_no, const cs::Bytes& value) final;
 bool put_batch(const std::vector<PendingWrite>& items) final;
 bool get(const cs::Bytes& key, cs::Bytes* value) final;
 bool get(const uint32_t seq_no, cs::Bytes* value) final;
 bool remove(const cs::Bytes&) final;
 bool seq_no(const cs::Bytes& key, uint32_t* value) final;
 bool write_batch(const ItemList&) final;
 IteratorPtr new_iterator() final;

 bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) override;
 bool getContractData(const cs::Bytes& key, cs::Bytes& data) override;
 bool flush() override;

private:
 class Iterator;
 void set_last_error_from_status(const rocksdb::Status& s);

private:
 std::unique_ptr<rocksdb::DB> db_;
 rocksdb::ColumnFamilyHandle* cf_blocks_ = nullptr; // default CF
 rocksdb::ColumnFamilyHandle* cf_seq_no_ = nullptr;
 rocksdb::ColumnFamilyHandle* cf_contracts_ = nullptr;
 bool bulk_load_ = false;
 bool sync_writes_ = false; // matches BDB's DB_TXN_NOSYNC; durability anchored at checkpoint via flush()
 uint64_t block_cache_bytes_ = 1ULL << 30; // 1 GiB
 uint64_t memtable_bytes_ = 256ULL << 20; // 256 MiB
};

} // namespace csdb

#endif // _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_

13 changes: 12 additions & 1 deletion
csdb/include/csdb/storage.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,6 +99,9 @@ class Storage final {
 ::std::shared_ptr<Database> db;
 ::cs::Sequence newBlockchainTop = ::cs::kWrongSequence;
 ::cs::Sequence startSequence = 0;
 // 0 = leave Storage defaults (5000 / 100) untouched.
 size_t asyncWriteQueueMax = 0;
 size_t writeBatchSize = 0;
 };

 struct OpenProgress {
Expand DownExpand Up@@ -140,7 +143,12 @@ class Storage final {
 bool open(const ::std::string& path_to_base = ::std::string{},
 OpenCallback callback = nullptr,
 cs::Sequence newBlockchainTop = cs::kWrongSequence,
 cs::Sequence startReadFrom = 0);
 cs::Sequence startReadFrom = 0,
 size_t asyncWriteQueueMax = 5000,
 size_t writeBatchSize = 100,
 uint64_t rocksDbBlockCacheBytes = 0,
 uint64_t rocksDbMemtableBytes = 0,
 const std::string& dbBackend = std::string{});

 /**
 * @brief Creating the storage using the parameters set.
Expand DownExpand Up@@ -170,6 +178,9 @@ class Storage final {
 */
 void close();

 // Anchor buffered writes to disk (RocksDB SyncWAL; BerkeleyDB no-op). Call at checkpoint boundaries.
 bool flush();

 /**
 * @brief Last block hash
 * @return Last block hash
Expand Down

9 changes: 9 additions & 0 deletions
csdb/src/database.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,15 @@ Database::Iterator::Iterator() = default;

Database::Iterator::~Iterator() = default;

bool Database::put_batch(const std::vector<PendingWrite>& items) {
 for (const auto& item : items) {
 if (!put(item.hash_key, item.seq_no, item.payload)) {
 return false;
 }
 }
 return true;
}

Database::Error Database::last_error() const {
 return last_error_map(this).last_error_;
}
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();
csdb: add RocksDB backend with runtime backend selection by akaitrade · Pull Request #73 · CREDITSCOM/node · GitHub
Skip to content
csdb: add RocksDB backend with runtime backend selection#73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Open

akaitrade
wants to merge
1
commit into

base:master
Choose a base branch

from

 Open


csdb: add RocksDB backend with runtime backend selection
#73
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 9 additions & 0 deletions
CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -23,6 +23,11 @@ option(LEVELDB_BUILD_TESTS "" OFF)
option(LEVELDB_AUTORUN_TESTS "" OFF)
option(DBSQL "Enable PostgeSQL server connection" OFF)

# The node binary always carries both DB backends; runtime selection is via
# config.ini `db_backend` (berkeleydb default, rocksdb opt-in).
set(CSDB_BACKEND "both" CACHE INTERNAL "csdb storage backend (locked to 'both')")
message(STATUS "csdb backend: both (locked)")

IF(DBSQL)
 ADD_DEFINITIONS(-DDBSQL)
ENDIF(DBSQL)
Expand DownExpand Up@@ -88,6 +93,10 @@ endif()
# This large list is necessary to prevent thrift from generating unused stuff
add_definitions(-DUSE_STD_THREAD)

# Disable Boost's MSVC auto-link pragmas; we link Boost targets explicitly via
# CMake. Without this the linker hunts for libboost_*-vc142-mt-s-x64-*.lib.
add_definitions(-DBOOST_ALL_NO_LIB)

if (MSVC)
 MARK_AS_ADVANCED(
 CMAKE_CXX_FLAGS_RELMONITOR
Expand Down

5 changes: 5 additions & 0 deletions
client/config/config.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -962,6 +962,11 @@ void Config::readStorageData(const boost::property_tree::ptree& config) {
 storageData_.checkpointEvery = 1000;
 }
 checkAndSaveValue(data, block, PARAM_NAME_STORAGE_CHECKPOINT_EVERY_MINUTES, storageData_.checkpointEveryMinutes);
 checkAndSaveValue(data, block, "async_write_queue_size", storageData_.asyncWriteQueueSize);
 checkAndSaveValue(data, block, "write_batch_size", storageData_.writeBatchSize);
 checkAndSaveValue(data, block, "db_backend", storageData_.dbBackend);
 checkAndSaveValue(data, block, "rocksdb_block_cache_mb", storageData_.rocksdbBlockCacheMb);
 checkAndSaveValue(data, block, "rocksdb_memtable_mb", storageData_.rocksdbMemtableMb);
}

void Config::readApiData(const boost::property_tree::ptree& config) {
Expand Down

5 changes: 5 additions & 0 deletions
client/config/config.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -62,6 +62,11 @@ struct StorageData {
 size_t checkpointKeep = 5; // retained periodic checkpoints (qs/0 always kept on top)
 size_t checkpointEvery = 500'000; // blocks between periodic checkpoints (rolling history depth = checkpointEvery * checkpointKeep)
 size_t checkpointEveryMinutes = 0; // wall-clock fallback: also save if this many minutes elapsed since last save (0 = disabled; opt-in for slow networks)
 size_t asyncWriteQueueSize = 5000; // bounded queue for the async DB writer
 size_t writeBatchSize = 100; // pools coalesced into one DB write
 std::string dbBackend = "berkeleydb"; // runtime DB backend (CSDB_BACKEND=both builds); set to "rocksdb" to opt in
 size_t rocksdbBlockCacheMb = 1024; // RocksDB shared block cache (MiB); 0 = built-in default
 size_t rocksdbMemtableMb = 256; // RocksDB write_buffer_size (MiB); 0 = built-in default
};

struct ApiData {
Expand Down

26 changes: 23 additions & 3 deletions
csdb/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -10,6 +10,16 @@ option(CSDB_BUILD_BENCHMARK "Bulid benchmark" OFF)
include (TestBigEndian)
TEST_BIG_ENDIAN(CSDB_PLATFORM_IS_BIG_ENDIAN)

if(CSDB_BACKEND STREQUAL "both")
 set(CSDB_BACKEND_SOURCES
 src/database_berkeleydb.cpp include/csdb/database_berkeleydb.hpp
 src/database_rocksdb.cpp include/csdb/database_rocksdb.hpp)
elseif(CSDB_BACKEND STREQUAL "rocksdb")
 set(CSDB_BACKEND_SOURCES src/database_rocksdb.cpp include/csdb/database_rocksdb.hpp)
else()
 set(CSDB_BACKEND_SOURCES src/database_berkeleydb.cpp include/csdb/database_berkeleydb.hpp)
endif()

add_library(${PROJECT_NAME} STATIC
 src/csdb.cpp
 src/amount.cpp
Expand All@@ -29,7 +39,7 @@ add_library(${PROJECT_NAME} STATIC
 src/priv_crypto.cpp
 src/priv_crypto.hpp
 src/database.cpp
src/database_berkeleydb.cpp
${CSDB_BACKEND_SOURCES}
 src/user_field.cpp
 include/csdb/internal/shared_data.hpp
 include/csdb/internal/shared_data_ptr_implementation.hpp
Expand All@@ -48,7 +58,6 @@ add_library(${PROJECT_NAME} STATIC
 include/csdb/wallet.hpp
 include/csdb/storage.hpp
 include/csdb/database.hpp
 include/csdb/database_berkeleydb.hpp
 include/csdb/user_field.hpp
 )

Expand All@@ -72,13 +81,24 @@ target_include_directories(
)

include_directories(${Boost_INCLUDE_DIR})
set(CSDB_BACKEND_LIBS "")
if(CSDB_BACKEND STREQUAL "both")
 list(APPEND CSDB_BACKEND_LIBS BerkeleyDB rocksdb)
 target_compile_definitions(${PROJECT_NAME} PRIVATE CSDB_USE_ROCKSDB CSDB_USE_BERKELEYDB)
elseif(CSDB_BACKEND STREQUAL "rocksdb")
 list(APPEND CSDB_BACKEND_LIBS rocksdb)
 target_compile_definitions(${PROJECT_NAME} PRIVATE CSDB_USE_ROCKSDB)
else()
 list(APPEND CSDB_BACKEND_LIBS BerkeleyDB)
endif()

target_link_libraries(
 ${PROJECT_NAME}
 cscrypto
 Boost::system
 Boost::filesystem
 Boost::disable_autolinking
BerkeleyDB
${CSDB_BACKEND_LIBS}
 lz4
 lib
)
Expand Down

11 changes: 11 additions & 0 deletions
csdb/include/csdb/database.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,13 +41,24 @@ class Database {
 virtual bool remove(const cs::Bytes& key) = 0;
 virtual bool seq_no(const cs::Bytes& key, uint32_t* value) = 0; // sequence from block hash

 // Vectorized put. Backends may override to coalesce writes (e.g. RocksDB WriteBatch).
 struct PendingWrite {
 cs::Bytes hash_key;
 uint32_t seq_no;
 cs::Bytes payload;
 };
 virtual bool put_batch(const std::vector<PendingWrite>& items);

 using Item = std::pair<cs::Bytes, cs::Bytes>;
 using ItemList = std::vector<Item>;
 virtual bool write_batch(const ItemList& items) = 0;

 virtual bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) = 0;
 virtual bool getContractData(const cs::Bytes& key, cs::Bytes& data) = 0;

 // anchor buffered writes to disk at checkpoint boundaries; default no-op (BDB has its own loop)
 virtual bool flush() { return true; }

 class Iterator {
 protected:
 Iterator();
Expand Down

71 changes: 71 additions & 0 deletions
csdb/include/csdb/database_rocksdb.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
// RocksDB-backed csdb::Database. CFs: blocks, seq_no, contracts.
// blocks keyed BE(seq+1). Selected via -DCSDB_BACKEND=rocksdb.

#ifndef _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_
#define _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_

#include <memory>
#include <string>

#include <csdb/database.hpp>

namespace rocksdb {
class DB;
class ColumnFamilyHandle;
class Status;
} // namespace rocksdb

namespace csdb {

class DatabaseRocksDB : public Database {
public:
 DatabaseRocksDB();
 ~DatabaseRocksDB() override;

public:
 // Bulk-load mode (before open): no auto-compaction, disableWAL; caller must compact_full() before close.
 void set_bulk_load(bool yes);
 // Per-write fsync toggle (default off; durability anchored at checkpoint via flush()).
 void set_sync_writes(bool v);

 // Flush + full-range compaction across all CFs; call once after a bulk-load run.
 bool compact_full();

 // Override RocksDB cache/memtable budget (before open); 0 keeps defaults (1 GiB / 256 MiB).
 void set_tuning(uint64_t block_cache_bytes, uint64_t memtable_bytes);

 bool open(const std::string& path);

private:
 bool is_open() const final;
 bool put(const cs::Bytes& key, uint32_t seq_no, const cs::Bytes& value) final;
 bool put_batch(const std::vector<PendingWrite>& items) final;
 bool get(const cs::Bytes& key, cs::Bytes* value) final;
 bool get(const uint32_t seq_no, cs::Bytes* value) final;
 bool remove(const cs::Bytes&) final;
 bool seq_no(const cs::Bytes& key, uint32_t* value) final;
 bool write_batch(const ItemList&) final;
 IteratorPtr new_iterator() final;

 bool updateContractData(const cs::Bytes& key, const cs::Bytes& data) override;
 bool getContractData(const cs::Bytes& key, cs::Bytes& data) override;
 bool flush() override;

private:
 class Iterator;
 void set_last_error_from_status(const rocksdb::Status& s);

private:
 std::unique_ptr<rocksdb::DB> db_;
 rocksdb::ColumnFamilyHandle* cf_blocks_ = nullptr; // default CF
 rocksdb::ColumnFamilyHandle* cf_seq_no_ = nullptr;
 rocksdb::ColumnFamilyHandle* cf_contracts_ = nullptr;
 bool bulk_load_ = false;
 bool sync_writes_ = false; // matches BDB's DB_TXN_NOSYNC; durability anchored at checkpoint via flush()
 uint64_t block_cache_bytes_ = 1ULL << 30; // 1 GiB
 uint64_t memtable_bytes_ = 256ULL << 20; // 256 MiB
};

} // namespace csdb

#endif // _CREDITS_CSDB_DATABASE_ROCKSDB_H_INCLUDED_

13 changes: 12 additions & 1 deletion
csdb/include/csdb/storage.hpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,6 +99,9 @@ class Storage final {
 ::std::shared_ptr<Database> db;
 ::cs::Sequence newBlockchainTop = ::cs::kWrongSequence;
 ::cs::Sequence startSequence = 0;
 // 0 = leave Storage defaults (5000 / 100) untouched.
 size_t asyncWriteQueueMax = 0;
 size_t writeBatchSize = 0;
 };

 struct OpenProgress {
Expand DownExpand Up@@ -140,7 +143,12 @@ class Storage final {
 bool open(const ::std::string& path_to_base = ::std::string{},
 OpenCallback callback = nullptr,
 cs::Sequence newBlockchainTop = cs::kWrongSequence,
 cs::Sequence startReadFrom = 0);
 cs::Sequence startReadFrom = 0,
 size_t asyncWriteQueueMax = 5000,
 size_t writeBatchSize = 100,
 uint64_t rocksDbBlockCacheBytes = 0,
 uint64_t rocksDbMemtableBytes = 0,
 const std::string& dbBackend = std::string{});

 /**
 * @brief Creating the storage using the parameters set.
Expand DownExpand Up@@ -170,6 +178,9 @@ class Storage final {
 */
 void close();

 // Anchor buffered writes to disk (RocksDB SyncWAL; BerkeleyDB no-op). Call at checkpoint boundaries.
 bool flush();

 /**
 * @brief Last block hash
 * @return Last block hash
Expand Down

9 changes: 9 additions & 0 deletions
csdb/src/database.cpp

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,15 @@ Database::Iterator::Iterator() = default;

Database::Iterator::~Iterator() = default;

bool Database::put_batch(const std::vector<PendingWrite>& items) {
 for (const auto& item : items) {
 if (!put(item.hash_key, item.seq_no, item.payload)) {
 return false;
 }
 }
 return true;
}

Database::Error Database::last_error() const {
 return last_error_map(this).last_error_;
}
Expand Down
Loading