From d434dfb3a56afde239023685ca0a8d191355314b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timo=20K=C3=B6sters?= <timo@koesters.xyz>
Date: Fri, 14 Jan 2022 11:40:49 +0100
Subject: [PATCH] feat: config option for rocksdb max open files
---
src/database.rs | 6 ++++++
src/database/abstraction/rocksdb.rs | 29 ++++++++++++++++++++++-------
2 files changed, 28 insertions(+), 7 deletions(-)
diff --git a/src/database.rs b/src/database.rs
index d688ff9fc..fd7a14515 100644
--- a/src/database.rs
+++ b/src/database.rs
@@ -49,6 +49,8 @@ pub struct Config {
database_path: String,
#[serde(default = "default_db_cache_capacity_mb")]
db_cache_capacity_mb: f64,
+ #[serde(default = "default_rocksdb_max_open_files")]
+ rocksdb_max_open_files: i32,
#[serde(default = "default_pdu_cache_capacity")]
pdu_cache_capacity: u32,
#[serde(default = "default_cleanup_second_interval")]
@@ -127,6 +129,10 @@ fn default_db_cache_capacity_mb() -> f64 {
10.0
}
+fn default_rocksdb_max_open_files() -> i32 {
+ 512
+}
+
fn default_pdu_cache_capacity() -> u32 {
1_000_000
}
diff --git a/src/database/abstraction/rocksdb.rs b/src/database/abstraction/rocksdb.rs
index 79a3d82a1..adda67875 100644
--- a/src/database/abstraction/rocksdb.rs
+++ b/src/database/abstraction/rocksdb.rs
@@ -5,6 +5,7 @@
pub struct Engine {
rocks: rocksdb::DBWithThreadMode<rocksdb::MultiThreaded>,
cache_capacity_bytes: usize,
+ max_open_files: i32,
cache: rocksdb::Cache,
old_cfs: Vec<String>,
}
@@ -16,7 +17,11 @@ pub struct RocksDbEngineTree<'a> {
write_lock: RwLock<()>,
}
-fn db_options(cache_capacity_bytes: usize, rocksdb_cache: &rocksdb::Cache) -> rocksdb::Options {
+fn db_options(
+ cache_capacity_bytes: usize,
+ max_open_files: i32,
+ rocksdb_cache: &rocksdb::Cache,
+) -> rocksdb::Options {
let mut block_based_options = rocksdb::BlockBasedOptions::default();
block_based_options.set_block_cache(rocksdb_cache);
@@ -36,7 +41,7 @@ fn db_options(cache_capacity_bytes: usize, rocksdb_cache: &rocksdb::Cache) -> ro
//db_opts.set_use_direct_io_for_flush_and_compaction(true);
db_opts.create_if_missing(true);
db_opts.increase_parallelism(num_cpus::get() as i32);
- db_opts.set_max_open_files(512);
+ db_opts.set_max_open_files(max_open_files);
db_opts.set_compression_type(rocksdb::DBCompressionType::Zstd);
db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
db_opts.optimize_level_style_compaction(cache_capacity_bytes);
@@ -52,7 +57,11 @@ fn open(config: &Config) -> Result<Self> {
let cache_capacity_bytes = (config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize;
let rocksdb_cache = rocksdb::Cache::new_lru_cache(cache_capacity_bytes).unwrap();
- let db_opts = db_options(cache_capacity_bytes, &rocksdb_cache);
+ let db_opts = db_options(
+ cache_capacity_bytes,
+ config.rocksdb_max_open_files,
+ &rocksdb_cache,
+ );
let cfs = rocksdb::DBWithThreadMode::<rocksdb::MultiThreaded>::list_cf(
&db_opts,
@@ -66,7 +75,11 @@ fn open(config: &Config) -> Result<Self> {
cfs.iter().map(|name| {
rocksdb::ColumnFamilyDescriptor::new(
name,
- db_options(cache_capacity_bytes, &rocksdb_cache),
+ db_options(
+ cache_capacity_bytes,
+ config.rocksdb_max_open_files,
+ &rocksdb_cache,
+ ),
)
}),
)?;
@@ -74,6 +87,7 @@ fn open(config: &Config) -> Result<Self> {
Ok(Arc::new(Engine {
rocks: db,
cache_capacity_bytes,
+ max_open_files: config.rocksdb_max_open_files,
cache: rocksdb_cache,
old_cfs: cfs,
}))
@@ -82,9 +96,10 @@ fn open(config: &Config) -> Result<Self> {
fn open_tree(&self, name: &'static str) -> Result<Arc<dyn Tree>> {
if !self.old_cfs.contains(&name.to_owned()) {
// Create if it didn't exist
- let _ = self
- .rocks
- .create_cf(name, &db_options(self.cache_capacity_bytes, &self.cache));
+ let _ = self.rocks.create_cf(
+ name,
+ &db_options(self.cache_capacity_bytes, self.max_open_files, &self.cache),
+ );
}
Ok(Arc::new(RocksDbEngineTree {
--
GitLab