mirror of
https://github.com/rustmailer/bichon.git
synced 2026-08-31 01:52:30 +00:00
Compare commits
18 Commits
1.6.2
...
2.0.0-alph
| Author | SHA1 | Date | |
|---|---|---|---|
|
022813a17c | ||
|
|
281a256582 | ||
|
|
02e9864343 | ||
|
|
eae26d3e97 | ||
|
|
de2a2b5d47 | ||
|
|
664ac2fe55 | ||
|
|
c468f8be41 | ||
|
|
a88b5c84f3 | ||
|
|
c6da79cdb0 | ||
|
|
eac19ce695 | ||
|
|
6bb88d37fa | ||
|
|
951901ac0b | ||
|
|
b7e757dbf7 | ||
|
|
85987e39fb | ||
|
|
adff34940c | ||
|
|
a8f2740973 | ||
|
|
36692e2091 | ||
|
|
4cdf3ee5f1 |
1
.github/workflows/release.yml
vendored
1
.github/workflows/release.yml
vendored
@@ -4,6 +4,7 @@ on:
|
||||
push:
|
||||
tags:
|
||||
- '[0-9]+.[0-9]+.[0-9]+'
|
||||
- '[0-9]+.[0-9]+.[0-9]+-*'
|
||||
env:
|
||||
BINARY_NAME: bichon-server
|
||||
BINARY_CLI: bichon-cli
|
||||
|
||||
83
Cargo.lock
generated
83
Cargo.lock
generated
@@ -299,31 +299,43 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "bichon-admin"
|
||||
version = "1.6.2"
|
||||
version = "2.0.0-alpha.1"
|
||||
dependencies = [
|
||||
"bichon-blob",
|
||||
"bichon-core",
|
||||
"bichon-memdb",
|
||||
"bytes 1.12.0",
|
||||
"chrono",
|
||||
"console",
|
||||
"dialoguer",
|
||||
"fjall",
|
||||
"hex",
|
||||
"indicatif",
|
||||
"itertools 0.15.0",
|
||||
"mail-parser",
|
||||
"native_db",
|
||||
"native_model",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"snafu",
|
||||
"tantivy",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bichon-blob"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"bincode_reloaded",
|
||||
"crc32fast",
|
||||
"criterion",
|
||||
"fs2",
|
||||
"lz4_flex",
|
||||
"rand 0.10.2",
|
||||
"redb 4.1.0",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
@@ -334,7 +346,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bichon-cli"
|
||||
version = "1.6.2"
|
||||
version = "2.0.0-alpha.1"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bichon-core",
|
||||
@@ -356,10 +368,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bichon-core"
|
||||
version = "1.6.2"
|
||||
version = "2.0.0-alpha.1"
|
||||
dependencies = [
|
||||
"async-imap",
|
||||
"base64 0.22.1",
|
||||
"bichon-blob",
|
||||
"bichon-memdb",
|
||||
"blake3",
|
||||
"bytes 1.12.0",
|
||||
@@ -372,7 +385,6 @@ dependencies = [
|
||||
"deunicode",
|
||||
"email_address",
|
||||
"encoding_rs",
|
||||
"fjall",
|
||||
"futures",
|
||||
"governor",
|
||||
"hex",
|
||||
@@ -380,7 +392,7 @@ dependencies = [
|
||||
"html2text",
|
||||
"itertools 0.15.0",
|
||||
"itoa",
|
||||
"lru 0.18.0",
|
||||
"lru 0.18.1",
|
||||
"mail-parser",
|
||||
"mail-send",
|
||||
"memmap2",
|
||||
@@ -432,7 +444,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bichon-server"
|
||||
version = "1.6.2"
|
||||
version = "2.0.0-alpha.1"
|
||||
dependencies = [
|
||||
"bichon-core",
|
||||
"bichon-smtp",
|
||||
@@ -457,7 +469,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bichon-smtp"
|
||||
version = "1.6.2"
|
||||
version = "2.0.0-alpha.1"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bichon-core",
|
||||
@@ -484,6 +496,16 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bincode_reloaded"
|
||||
version = "3.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c6f3fe39960aac27f7de2e5f41be9fa576be3fbffddf86a06b967b579f603b8"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"unty",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bit-vec"
|
||||
version = "0.9.1"
|
||||
@@ -1482,6 +1504,16 @@ dependencies = [
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fs2"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fs4"
|
||||
version = "0.13.1"
|
||||
@@ -2491,9 +2523,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.18.0"
|
||||
version = "0.18.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9"
|
||||
checksum = "0b6180140927ee907000b0aa540091f6ea512ead4447c92b8fc35bc72788a5a6"
|
||||
dependencies = [
|
||||
"hashbrown 0.17.0",
|
||||
]
|
||||
@@ -3570,9 +3602,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "quick_cache"
|
||||
version = "0.6.21"
|
||||
version = "0.6.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a70b1b8b47e31d0498ecbc3c5470bb931399a8bfed1fd79d1717a61ce7f96e3"
|
||||
checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown 0.16.1",
|
||||
@@ -3797,6 +3829,15 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redb"
|
||||
version = "4.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e925444704b5f17d32bf42f5b6e2df050bceebc3dcd6e71cc73dafe8092e839"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.5.18"
|
||||
@@ -3808,9 +3849,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.12.4"
|
||||
version = "1.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
|
||||
checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
@@ -5279,6 +5320,12 @@ version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
|
||||
|
||||
[[package]]
|
||||
name = "unty"
|
||||
version = "0.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7dbe8d477efbc6c70a1dea1f9f1e0482168983a9147dee6b3a619f666f3aeac6"
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.5.8"
|
||||
@@ -5353,9 +5400,9 @@ checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
|
||||
|
||||
[[package]]
|
||||
name = "varint-rs"
|
||||
version = "2.2.0"
|
||||
version = "2.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
|
||||
checksum = "bfa6c38708f6257f1ec2ca7e5a11f9bbf58a27d7060078b6b333624968183d96"
|
||||
|
||||
[[package]]
|
||||
name = "vcpkg"
|
||||
@@ -6037,9 +6084,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "xxhash-rust"
|
||||
version = "0.8.15"
|
||||
version = "0.8.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
|
||||
checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576"
|
||||
|
||||
[[package]]
|
||||
name = "yasna"
|
||||
|
||||
14
Cargo.toml
14
Cargo.toml
@@ -13,7 +13,7 @@ members = [
|
||||
resolver = "2"
|
||||
|
||||
[workspace.package]
|
||||
version = "1.6.2"
|
||||
version = "2.0.0-alpha.1"
|
||||
edition = "2021"
|
||||
|
||||
[workspace.dependencies]
|
||||
@@ -41,11 +41,11 @@ reqwest = { version = "0.12.24", default-features = false, features = [
|
||||
] }
|
||||
tokio-socks = "0.5.3"
|
||||
http = "1.4.2"
|
||||
regex = "1.12.4"
|
||||
regex = "1.13"
|
||||
email_address = "0.2.9"
|
||||
futures = "0.3.32"
|
||||
utf7-imap = "0.3.2"
|
||||
mail-parser = { version = '0.11.4', features = ["serde"] }
|
||||
mail-parser = { version = '0.11', features = ["serde"] }
|
||||
# mail-send = "0.5.2"
|
||||
tokio-rustls = { version = "0.26.4", default-features = false, features = [
|
||||
"ring",
|
||||
@@ -54,7 +54,7 @@ tokio-rustls = { version = "0.26.4", default-features = false, features = [
|
||||
timeago = "0.6.1"
|
||||
oauth2 = { version = "5.0.0", features = ["reqwest-blocking"] }
|
||||
url = { version = "2.5.8", features = ["serde"] }
|
||||
sysinfo = "0.39.4"
|
||||
sysinfo = "0.39"
|
||||
num_cpus = "1.17.0"
|
||||
rand = "0.10.2"
|
||||
encoding_rs = "0.8.35"
|
||||
@@ -64,7 +64,7 @@ rustls-pki-types = "1.15.0"
|
||||
tokio-io-timeout = "1.2.1"
|
||||
semver = "1.0.28"
|
||||
governor = "0.10.4"
|
||||
lru = "0.18.0"
|
||||
lru = "0.18.1"
|
||||
mime_guess = "2.0.5"
|
||||
hex = "0.4.3"
|
||||
time = { version = "0.3.53", features = [
|
||||
@@ -72,14 +72,14 @@ time = { version = "0.3.53", features = [
|
||||
"parsing",
|
||||
"local-offset",
|
||||
] }
|
||||
rust-embed = "8.12.0"
|
||||
rust-embed = "8.12"
|
||||
murmur3 = "0.5.2"
|
||||
urlencoding = "2.1.3"
|
||||
dashmap = "6.2.1"
|
||||
gethostname = "1.1.0"
|
||||
itoa = "1.0.18"
|
||||
html2text = "0.17.1"
|
||||
bytes = "1.12.0"
|
||||
bytes = "1.12"
|
||||
dialoguer = "0.12.0"
|
||||
console = "0.16.4"
|
||||
mail-send = "0.6.1"
|
||||
|
||||
@@ -17,4 +17,16 @@ serde_json.workspace = true
|
||||
|
||||
itertools.workspace = true
|
||||
snafu.workspace = true
|
||||
bichon-memdb.workspace = true
|
||||
bichon-memdb.workspace = true
|
||||
fjall.workspace = true
|
||||
hex.workspace = true
|
||||
bichon-blob.workspace = true
|
||||
mail-parser.workspace = true
|
||||
bytes.workspace = true
|
||||
uuid.workspace = true
|
||||
tantivy = { version = "0.26.1", features = ["zstd-compression", "quickwit"] }
|
||||
chrono.workspace = true
|
||||
tracing.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
@@ -1,6 +1,6 @@
|
||||
use tantivy::schema::{FacetOptions, Field, Schema, FAST, INDEXED, STORED, STRING, TEXT};
|
||||
|
||||
use crate::migrate::legacy::fields::{EmlFields, EnvelopeFields, *};
|
||||
use crate::legacy::fields::{EmlFields, EnvelopeFields, *};
|
||||
|
||||
pub struct SchemaTools;
|
||||
|
||||
@@ -19,10 +19,13 @@
|
||||
use console::style;
|
||||
use dialoguer::{theme::ColorfulTheme, Select};
|
||||
|
||||
use crate::{migrate::handle_migration, reset::handle_reset_password};
|
||||
use crate::{migrate_v037::handle_migration_v037, migrate_v1::handle_migrate_v1, reset::handle_reset_password};
|
||||
|
||||
pub mod legacy;
|
||||
pub mod meta;
|
||||
pub mod migrate;
|
||||
pub mod migrate_store_v2;
|
||||
pub mod migrate_v037;
|
||||
pub mod migrate_v1;
|
||||
pub mod reset;
|
||||
|
||||
|
||||
@@ -40,7 +43,8 @@ async fn run_interactive() {
|
||||
|
||||
let main_options = vec![
|
||||
"Reset Admin Password",
|
||||
"Migrate Legacy v0.3.7 Storage to v1.x",
|
||||
"Migrate Legacy v0.3.7 Storage to v2.x (bichon-blob)",
|
||||
"Migrate v1.x Storage to v2.x (Fjall → bichon-blob)",
|
||||
"Exit",
|
||||
];
|
||||
|
||||
@@ -53,7 +57,8 @@ async fn run_interactive() {
|
||||
|
||||
match selection {
|
||||
0 => handle_reset_password(&theme),
|
||||
1 => handle_migration(&theme),
|
||||
1 => handle_migration_v037(&theme),
|
||||
2 => handle_migrate_v1(&theme),
|
||||
_ => {
|
||||
println!("{}", style("Exiting...").dim());
|
||||
}
|
||||
|
||||
@@ -255,7 +255,6 @@ impl From<AccountV3> for AccountModel {
|
||||
created_at: value.created_at,
|
||||
updated_at: value.updated_at,
|
||||
created_by: value.created_by,
|
||||
use_proxy: value.use_proxy,
|
||||
use_dangerous: value.use_dangerous,
|
||||
pgp_key: value.pgp_key,
|
||||
imap_quota_window: None,
|
||||
|
||||
@@ -3,20 +3,17 @@ use std::{path::PathBuf, time::Instant};
|
||||
use bytes::Bytes;
|
||||
use mail_parser::MimeHeaders;
|
||||
|
||||
use crate::{
|
||||
use bichon_core::{
|
||||
envelope::extractor::extract_references, message::content::AttachmentInfo,
|
||||
store::tantivy::tokenizers::EuroTokenizer, utils::compute_content_hash,
|
||||
};
|
||||
|
||||
use fjall::{
|
||||
config::{BlockSizePolicy, CompressionPolicy},
|
||||
CompressionType, Database, Keyspace, KeyspaceCreateOptions, KvSeparationOptions,
|
||||
};
|
||||
use bichon_blob::{Codec, Config, Engine};
|
||||
use mail_parser::MessageParser;
|
||||
use tantivy::{indexer::NoMergePolicy, Index, IndexWriter, TantivyDocument};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{
|
||||
use bichon_core::{
|
||||
common::AddrVec,
|
||||
envelope::extractor::{compute_thread_id, generate_message_id},
|
||||
error::{code::ErrorCode, BichonResult},
|
||||
@@ -64,6 +61,17 @@ pub struct DetachOutput {
|
||||
pub blobs: Vec<(String, Bytes)>,
|
||||
}
|
||||
|
||||
fn hex_to_raw_key(hex: &str) -> BichonResult<[u8; 32]> {
|
||||
let mut key = [0u8; 32];
|
||||
hex::decode_to_slice(hex, &mut key).map_err(|e| {
|
||||
raise_error!(
|
||||
format!("invalid content hash: {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
pub fn detach_attachments_standalone(
|
||||
original_body: &[u8],
|
||||
message: &mail_parser::Message<'_>,
|
||||
@@ -132,19 +140,16 @@ pub fn detach_attachments_standalone(
|
||||
(stripped_eml, DetachOutput { infos, blobs })
|
||||
}
|
||||
|
||||
pub struct NewIndexWriter {
|
||||
pub struct NewIndexWriterV2 {
|
||||
pub envelope_writer: Option<IndexWriter>,
|
||||
pub attachment_writer: Option<IndexWriter>,
|
||||
pub email_ks: Keyspace,
|
||||
pub attachment_ks: Keyspace,
|
||||
pub engine: Engine,
|
||||
pending: usize,
|
||||
email_buf: Vec<(String, Vec<u8>)>,
|
||||
attachment_buf: Vec<(String, Vec<u8>)>,
|
||||
email_buf: Vec<([u8; 32], Vec<u8>)>,
|
||||
attachment_buf: Vec<([u8; 32], Vec<u8>)>,
|
||||
}
|
||||
|
||||
//const COMMIT_THRESHOLD: usize = 500;
|
||||
|
||||
impl NewIndexWriter {
|
||||
impl NewIndexWriterV2 {
|
||||
pub fn open(dirs: NewDirs) -> BichonResult<Self> {
|
||||
// ── envelope index ──────────────────────────────────────────────
|
||||
std::fs::create_dir_all(&dirs.envelope_dir)
|
||||
@@ -170,11 +175,6 @@ impl NewIndexWriter {
|
||||
.writer_with_num_threads(3, 256 * 1024 * 1024)
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
|
||||
// let mut merge_policy = LogMergePolicy::default();
|
||||
// merge_policy.set_min_num_segments(25);
|
||||
// merge_policy.set_min_layer_size(10_000);
|
||||
// merge_policy.set_max_docs_before_merge(100_000);
|
||||
|
||||
envelope_writer.set_merge_policy(Box::new(NoMergePolicy));
|
||||
// ── attachment index ─────────────────────────────────────────────
|
||||
std::fs::create_dir_all(&dirs.attachment_dir)
|
||||
@@ -199,58 +199,26 @@ impl NewIndexWriter {
|
||||
.writer_with_num_threads(3, 256 * 1024 * 1024)
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
|
||||
// let mut merge_policy = LogMergePolicy::default();
|
||||
// merge_policy.set_min_num_segments(25);
|
||||
// merge_policy.set_min_layer_size(10_000);
|
||||
// merge_policy.set_max_docs_before_merge(100_000);
|
||||
|
||||
attachment_writer.set_merge_policy(Box::new(NoMergePolicy));
|
||||
|
||||
// ── blob store ───────────────────────────────────────────────────
|
||||
// ── blob store (bichon-blob, not fjall) ───────────────────────────
|
||||
std::fs::create_dir_all(&dirs.storage_dir)
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
let db = Database::builder(&dirs.storage_dir)
|
||||
.cache_size(8 * 1024 * 1024)
|
||||
.journal_compression(CompressionType::None)
|
||||
.max_journaling_size(64 * 1024 * 1024)
|
||||
.open()
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
let blob_dir = dirs.storage_dir.join("blobs");
|
||||
|
||||
let email_ks = db
|
||||
.keyspace("email", || {
|
||||
KeyspaceCreateOptions::default()
|
||||
.max_memtable_size(4 * 1024 * 1024)
|
||||
.data_block_size_policy(BlockSizePolicy::all(4 * 1024))
|
||||
.data_block_compression_policy(CompressionPolicy::all(CompressionType::Lz4))
|
||||
.with_kv_separation(Some(
|
||||
KvSeparationOptions::default()
|
||||
.separation_threshold(1024)
|
||||
.compression(CompressionType::Lz4)
|
||||
.file_target_size(512 * 1024 * 1024),
|
||||
))
|
||||
})
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
let mut config = Config::default();
|
||||
config.default_codec = Codec::Zstd;
|
||||
config.compress_threshold = 1024;
|
||||
config.flush_interval_secs = 0;
|
||||
config.gc_interval_secs = 0;
|
||||
|
||||
let attachment_ks = db
|
||||
.keyspace("attachments", || {
|
||||
KeyspaceCreateOptions::default()
|
||||
.max_memtable_size(4 * 1024 * 1024)
|
||||
.data_block_size_policy(BlockSizePolicy::all(4 * 1024))
|
||||
.data_block_compression_policy(CompressionPolicy::all(CompressionType::Lz4))
|
||||
.with_kv_separation(Some(
|
||||
KvSeparationOptions::default()
|
||||
.separation_threshold(1024)
|
||||
.compression(CompressionType::Lz4)
|
||||
.file_target_size(512 * 1024 * 1024),
|
||||
))
|
||||
})
|
||||
let engine = Engine::open(&blob_dir, config)
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
|
||||
Ok(Self {
|
||||
envelope_writer: Some(envelope_writer),
|
||||
attachment_writer: Some(attachment_writer),
|
||||
email_ks,
|
||||
attachment_ks,
|
||||
engine,
|
||||
pending: 0,
|
||||
email_buf: Vec::new(),
|
||||
attachment_buf: Vec::new(),
|
||||
@@ -266,6 +234,7 @@ impl NewIndexWriter {
|
||||
internal_date: i64,
|
||||
) -> BichonResult<()> {
|
||||
let email_content_hash = compute_content_hash(eml_bytes);
|
||||
let email_raw_key = hex_to_raw_key(&email_content_hash)?;
|
||||
|
||||
let message = MessageParser::new()
|
||||
.parse(eml_bytes)
|
||||
@@ -284,7 +253,7 @@ impl NewIndexWriter {
|
||||
.or_else(|| {
|
||||
message
|
||||
.body_html(0)
|
||||
.map(|html| crate::utils::html::extract_text(html.into_owned()))
|
||||
.map(|html| bichon_core::utils::html::extract_text(html.into_owned()))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let text = text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
@@ -336,11 +305,13 @@ impl NewIndexWriter {
|
||||
// ── detach attachments → blob ──────────────────────────────────────
|
||||
let (stripped_eml, attachment_output) = detach_attachments_standalone(eml_bytes, &message);
|
||||
|
||||
// Buffer for bulk ingestion — sorted + flushed later.
|
||||
// Buffer for bulk write — sorted + flushed later.
|
||||
// Key is the raw 32-byte hash (not the hex string).
|
||||
self.email_buf
|
||||
.push((email_content_hash.clone(), stripped_eml));
|
||||
.push((email_raw_key, stripped_eml));
|
||||
for (hash, data) in &attachment_output.blobs {
|
||||
self.attachment_buf.push((hash.clone(), data.to_vec()));
|
||||
let raw_key = hex_to_raw_key(hash)?;
|
||||
self.attachment_buf.push((raw_key, data.to_vec()));
|
||||
}
|
||||
|
||||
// ── build envelope doc ────────────────────────────────────────────
|
||||
@@ -430,9 +401,6 @@ impl NewIndexWriter {
|
||||
}
|
||||
|
||||
self.pending += 1;
|
||||
// if self.pending >= COMMIT_THRESHOLD {
|
||||
// self.commit()?;
|
||||
// }
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -473,7 +441,7 @@ impl NewIndexWriter {
|
||||
let seg_ids = writer
|
||||
.index()
|
||||
.searchable_segment_ids()
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
println!("merging {} {} segments...", seg_ids.len(), name);
|
||||
if seg_ids.len() > 1 {
|
||||
let _ = writer.merge(&seg_ids);
|
||||
@@ -491,70 +459,111 @@ impl NewIndexWriter {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sort buffered (hash, data) pairs, dedup, and write via Fjall's
|
||||
/// ingestion API — writes SSTables directly, bypassing memtable and WAL.
|
||||
/// Write buffered blobs to the bichon-blob engine.
|
||||
/// Also commits the Tantivy writers to bound their in-memory state.
|
||||
pub fn flush_fjall_buffers(&mut self) -> BichonResult<()> {
|
||||
pub fn flush_blob_buffers(&mut self) -> BichonResult<()> {
|
||||
self.commit_tantivy()?;
|
||||
|
||||
if !self.email_buf.is_empty() {
|
||||
self.email_buf.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
self.email_buf.dedup_by(|a, b| a.0 == b.0);
|
||||
let mut buf = std::mem::take(&mut self.email_buf);
|
||||
buf.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
buf.dedup_by(|a, b| a.0 == b.0);
|
||||
|
||||
let mut ingestion = self.email_ks.start_ingestion().map_err(|e| {
|
||||
raise_error!(
|
||||
format!("email ingestion start: {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
for (hash, data) in &self.email_buf {
|
||||
ingestion
|
||||
.write(hash.as_bytes(), data.as_slice())
|
||||
.map_err(|e| {
|
||||
raise_error!(
|
||||
format!("email ingestion write: {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
let count = buf.len();
|
||||
let mut skipped = 0usize;
|
||||
let mut batch: Vec<([u8; 32], Vec<u8>, Codec)> = Vec::with_capacity(buf.len());
|
||||
for (key, data) in buf {
|
||||
if data.len() > 100 * 1024 * 1024 {
|
||||
eprintln!(
|
||||
"{}",
|
||||
console::style(format!(
|
||||
"WARN: skipping oversized email blob key={} ({} bytes)",
|
||||
hex::encode(key),
|
||||
data.len()
|
||||
))
|
||||
.yellow()
|
||||
);
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
batch.push((key, data, Codec::Zstd));
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
self.engine.put_batch(&batch).map_err(|e| {
|
||||
raise_error!(
|
||||
format!("blob engine put_batch error: {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
}
|
||||
println!("flushed {} email blobs to engine", count - skipped);
|
||||
if skipped > 0 {
|
||||
eprintln!(
|
||||
"{}",
|
||||
console::style(format!("skipped {} oversized email blobs", skipped)).yellow()
|
||||
);
|
||||
}
|
||||
ingestion.finish().map_err(|e| {
|
||||
raise_error!(
|
||||
format!("email ingestion finish: {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
self.email_buf.clear();
|
||||
}
|
||||
|
||||
if !self.attachment_buf.is_empty() {
|
||||
self.attachment_buf.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
self.attachment_buf.dedup_by(|a, b| a.0 == b.0);
|
||||
let mut buf = std::mem::take(&mut self.attachment_buf);
|
||||
buf.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
buf.dedup_by(|a, b| a.0 == b.0);
|
||||
|
||||
let mut ingestion = self.attachment_ks.start_ingestion().map_err(|e| {
|
||||
raise_error!(
|
||||
format!("attachment ingestion start: {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
for (hash, data) in &self.attachment_buf {
|
||||
ingestion
|
||||
.write(hash.as_bytes(), data.as_slice())
|
||||
.map_err(|e| {
|
||||
raise_error!(
|
||||
format!("attachment ingestion write: {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
let count = buf.len();
|
||||
let mut skipped = 0usize;
|
||||
let mut batch: Vec<([u8; 32], Vec<u8>, Codec)> = Vec::with_capacity(buf.len());
|
||||
for (key, data) in buf {
|
||||
if data.len() > 100 * 1024 * 1024 {
|
||||
eprintln!(
|
||||
"{}",
|
||||
console::style(format!(
|
||||
"WARN: skipping oversized attachment blob key={} ({} bytes)",
|
||||
hex::encode(key),
|
||||
data.len()
|
||||
))
|
||||
.yellow()
|
||||
);
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
batch.push((key, data, Codec::Zstd));
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
self.engine.put_batch(&batch).map_err(|e| {
|
||||
raise_error!(
|
||||
format!("blob engine put_batch error: {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
}
|
||||
println!("flushed {} attachment blobs to engine", count - skipped);
|
||||
if skipped > 0 {
|
||||
eprintln!(
|
||||
"{}",
|
||||
console::style(format!("skipped {} oversized attachment blobs", skipped))
|
||||
.yellow()
|
||||
);
|
||||
}
|
||||
ingestion.finish().map_err(|e| {
|
||||
raise_error!(
|
||||
format!("attachment ingestion finish: {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
self.attachment_buf.clear();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush and shutdown the blob engine (called once at the very end).
|
||||
pub fn shutdown_engine(&mut self) -> BichonResult<()> {
|
||||
self.engine.flush().map_err(|e| {
|
||||
raise_error!(
|
||||
format!("engine flush error: {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
self.engine.shutdown().map_err(|e| {
|
||||
raise_error!(
|
||||
format!("engine shutdown error: {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,17 +1,62 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::{collections::HashMap, path::PathBuf};
|
||||
|
||||
use bichon_core::migrate::{
|
||||
count_eml_segments, do_migrate_segment, is_tantivy_index_dir,
|
||||
store::{LegacyDirs, NewDirs, NewIndexWriter},
|
||||
use bichon_core::{
|
||||
error::{code::ErrorCode, BichonResult},
|
||||
migrate::{is_tantivy_index_dir, write_storage_version},
|
||||
raise_error,
|
||||
};
|
||||
use console::style;
|
||||
use dialoguer::{theme::ColorfulTheme, Confirm, Input};
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
use tantivy::{
|
||||
collector::TopDocs,
|
||||
columnar::Column,
|
||||
query::TermQuery,
|
||||
schema::{IndexRecordOption, Value},
|
||||
DocAddress, Index, TantivyDocument, Term,
|
||||
};
|
||||
|
||||
pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
use crate::legacy::schema::SchemaTools;
|
||||
use crate::migrate_store_v2::{NewDirs, NewIndexWriterV2};
|
||||
|
||||
pub struct LegacyDirs {
|
||||
pub envelope_dir: PathBuf,
|
||||
pub eml_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl LegacyDirs {
|
||||
pub fn new(index: PathBuf, data: PathBuf) -> Self {
|
||||
Self {
|
||||
envelope_dir: index,
|
||||
eml_dir: data,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_legacy_data_layout_with_paths(
|
||||
envelope_dir: &PathBuf,
|
||||
eml_dir: &PathBuf,
|
||||
) -> std::io::Result<bool> {
|
||||
let envelope_result = is_tantivy_index_dir(envelope_dir)?;
|
||||
let eml_result = is_tantivy_index_dir(eml_dir)?;
|
||||
Ok(envelope_result || eml_result)
|
||||
}
|
||||
|
||||
/// Return the number of segments in the legacy EML Tantivy index.
|
||||
pub fn count_eml_segments(legacy: &LegacyDirs) -> BichonResult<usize> {
|
||||
let eml_index = Index::open_in_dir(&legacy.eml_dir)
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
let reader = eml_index
|
||||
.reader()
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
let searcher = reader.searcher();
|
||||
Ok(searcher.segment_readers().len())
|
||||
}
|
||||
|
||||
pub fn handle_migration_v037(theme: &ColorfulTheme) {
|
||||
println!(
|
||||
"\n{}",
|
||||
style("MIGRATION: Bichon v0.3.7 Storage Architecture → v1.x")
|
||||
style("MIGRATION: Bichon v0.3.7 Storage → v2.x (bichon-blob)")
|
||||
.bold()
|
||||
.yellow()
|
||||
);
|
||||
@@ -20,8 +65,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
"{}",
|
||||
style(
|
||||
"This tool migrates data from the legacy v0.3.7 Tantivy-based storage \
|
||||
architecture to the new v1.x \
|
||||
separated index and Fjall-backed storage format."
|
||||
architecture directly to the v2.x bichon-blob storage format."
|
||||
)
|
||||
.dim()
|
||||
);
|
||||
@@ -32,11 +76,11 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
"Legacy v0.3.7 architecture:\n\
|
||||
• envelope metadata stored in Tantivy\n\
|
||||
• message data stored in Tantivy\n\n\
|
||||
New v1.x architecture:\n\
|
||||
New v2.x architecture:\n\
|
||||
• mail indexes stored in Tantivy\n\
|
||||
• attachment indexes stored in Tantivy\n\
|
||||
• raw message data stored in Fjall\n\
|
||||
• attachment blobs stored in Fjall"
|
||||
• raw message data stored in bichon-blob engine\n\
|
||||
• attachment blobs stored in bichon-blob engine"
|
||||
)
|
||||
.dim()
|
||||
);
|
||||
@@ -54,7 +98,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
let root_dir_str: String = Input::with_theme(theme)
|
||||
.with_prompt("Enter --bichon-root-dir (same value used by the old server)")
|
||||
.validate_with(|input: &String| -> Result<(), &str> {
|
||||
let path = Path::new(input);
|
||||
let path = PathBuf::from(input);
|
||||
if !path.is_absolute() {
|
||||
return Err("Path must be absolute.");
|
||||
}
|
||||
@@ -81,7 +125,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
if input.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let path = Path::new(input);
|
||||
let path = PathBuf::from(input);
|
||||
if !path.is_absolute() {
|
||||
return Err("Path must be absolute.");
|
||||
}
|
||||
@@ -119,7 +163,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
if input.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let path = Path::new(input);
|
||||
let path = PathBuf::from(input);
|
||||
if !path.is_absolute() {
|
||||
return Err("Path must be absolute.");
|
||||
}
|
||||
@@ -172,7 +216,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
println!(
|
||||
"{} {}",
|
||||
style("✔").green(),
|
||||
style("Legacy v0.3.7 Tantivy-based storage detected. Migration to v1.x is required.")
|
||||
style("Legacy v0.3.7 Tantivy-based storage detected. Migration to v2.x is required.")
|
||||
.yellow()
|
||||
);
|
||||
}
|
||||
@@ -186,7 +230,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
println!(
|
||||
"{}",
|
||||
style(
|
||||
"The selected directories may already be using the v1.x storage architecture."
|
||||
"The selected directories may already be using a newer storage architecture."
|
||||
)
|
||||
.dim()
|
||||
);
|
||||
@@ -326,7 +370,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
.progress_chars("#>-"),
|
||||
);
|
||||
|
||||
let mut writer = match NewIndexWriter::open(NewDirs::new(
|
||||
let mut writer = match NewIndexWriterV2::open(NewDirs::new(
|
||||
new_index_path.clone(),
|
||||
new_data_path.clone(),
|
||||
)) {
|
||||
@@ -346,7 +390,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
|
||||
pb.set_message(format!("Segment {}/{}", seg_idx + 1, total_segments));
|
||||
let legacy = LegacyDirs::new(index_path.clone(), data_path.clone());
|
||||
match do_migrate_segment(
|
||||
match do_migrate_segment_v2(
|
||||
batch_size,
|
||||
legacy,
|
||||
&mut writer,
|
||||
@@ -426,6 +470,24 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
return;
|
||||
}
|
||||
|
||||
pb.set_message(style("Shutting down blob engine...").dim().to_string());
|
||||
if let Err(e) = writer.shutdown_engine() {
|
||||
pb.finish_with_message(format!("{}", style("Migration failed.").red()));
|
||||
eprintln!("\n{} {:?}", style("✘").red().bold(), e);
|
||||
return;
|
||||
}
|
||||
|
||||
// Write STORAGE_VERSION = 2 to mark the data as v2.x compatible
|
||||
if let Err(e) = write_storage_version(&root_path, 2) {
|
||||
pb.finish_with_message(format!("{}", style("Migration failed.").red()));
|
||||
eprintln!(
|
||||
"\n{} Failed to write STORAGE_VERSION: {:?}",
|
||||
style("✘").red().bold(),
|
||||
e
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
pb.finish_with_message(format!(
|
||||
"Migration finished. Total: {}, Skipped: {}",
|
||||
grand_total_migrated, grand_total_skipped
|
||||
@@ -434,16 +496,196 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
println!(
|
||||
"{} {}",
|
||||
style("✔").green(),
|
||||
style("Migration completed successfully!").bold()
|
||||
style("Migration to v2.x completed successfully!").bold()
|
||||
);
|
||||
}
|
||||
|
||||
pub fn is_legacy_data_layout_with_paths(
|
||||
envelope_dir: &PathBuf,
|
||||
eml_dir: &PathBuf,
|
||||
) -> std::io::Result<bool> {
|
||||
let envelope_result = is_tantivy_index_dir(envelope_dir)?;
|
||||
let eml_result = is_tantivy_index_dir(eml_dir)?;
|
||||
/// Migrate all documents from a single EML segment to the v2.x storage layout.
|
||||
fn do_migrate_segment_v2<F>(
|
||||
batch_size: u32,
|
||||
legacy: LegacyDirs,
|
||||
writer: &mut NewIndexWriterV2,
|
||||
segment_index: usize,
|
||||
mut on_progress: F,
|
||||
) -> BichonResult<()>
|
||||
where
|
||||
F: FnMut(&str),
|
||||
{
|
||||
// ── open legacy indices ────────────────────────────────────────────
|
||||
let envelope_index = Index::open_in_dir(&legacy.envelope_dir)
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
let eml_index = Index::open_in_dir(&legacy.eml_dir)
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
|
||||
Ok(envelope_result || eml_result)
|
||||
let envelope_reader = envelope_index
|
||||
.reader()
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
let eml_reader = eml_index
|
||||
.reader()
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
|
||||
let envelope_searcher = envelope_reader.searcher();
|
||||
let eml_searcher = eml_reader.searcher();
|
||||
|
||||
let ef = SchemaTools::envelope_fields();
|
||||
let mf = SchemaTools::eml_fields();
|
||||
|
||||
let eml_segments = eml_searcher.segment_readers();
|
||||
let eml_segment = eml_segments.get(segment_index).ok_or_else(|| {
|
||||
raise_error!(
|
||||
format!(
|
||||
"segment index {} out of range ({} segments)",
|
||||
segment_index,
|
||||
eml_segments.len()
|
||||
),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
|
||||
let num_docs = eml_segment.num_docs();
|
||||
if num_docs == 0 {
|
||||
on_progress("TOTAL:0");
|
||||
on_progress("DONE:0:0");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
on_progress(&format!("TOTAL:{}", num_docs));
|
||||
|
||||
let max_doc = eml_segment.max_doc();
|
||||
let ff = eml_segment.fast_fields();
|
||||
let f_id_col: Column<u64> = ff.u64("id").map_err(|e| {
|
||||
raise_error!(
|
||||
format!("failed to open f_id fast field: {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
|
||||
// ── Phase 1: build eid → (uid, internal_date) from envelope, then drop it ──
|
||||
let mut envelope_map: HashMap<u64, (u32, i64)> = HashMap::with_capacity(num_docs as usize);
|
||||
|
||||
let mut env_scanned = 0u32;
|
||||
let mut env_skipped = 0u32;
|
||||
for doc_id in 0..max_doc {
|
||||
if eml_segment.is_deleted(doc_id) {
|
||||
continue;
|
||||
}
|
||||
let eid = f_id_col.values.get_val(doc_id);
|
||||
|
||||
let term = Term::from_field_u64(ef.f_id, eid);
|
||||
let query = TermQuery::new(term, IndexRecordOption::Basic);
|
||||
let hits: Vec<(_, DocAddress)> = envelope_searcher
|
||||
.search(&query, &TopDocs::with_limit(1).order_by_score())
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
|
||||
if let Some((_, addr)) = hits.first() {
|
||||
let env_doc: TantivyDocument = envelope_searcher
|
||||
.doc(*addr)
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
let uid = env_doc
|
||||
.get_first(ef.f_uid)
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(0) as u32;
|
||||
let internal_date = env_doc
|
||||
.get_first(ef.f_internal_date)
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(0);
|
||||
envelope_map.insert(eid, (uid, internal_date));
|
||||
env_scanned += 1;
|
||||
} else {
|
||||
env_skipped += 1;
|
||||
}
|
||||
|
||||
if env_scanned % 10 == 0 {
|
||||
on_progress(&format!(
|
||||
"PHASE1:{}/{} skipped:{}",
|
||||
env_scanned, max_doc, env_skipped
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Free the envelope index before the heavy EML processing.
|
||||
drop(envelope_searcher);
|
||||
drop(envelope_reader);
|
||||
drop(envelope_index);
|
||||
|
||||
// ── Phase 2: process EML docs, streaming one at a time ─────────────
|
||||
let mut total_migrated = 0usize;
|
||||
let mut total_skipped = 0usize;
|
||||
|
||||
let mut chunk_start = 0u32;
|
||||
|
||||
while chunk_start < max_doc {
|
||||
let chunk_end = (chunk_start + batch_size).min(max_doc);
|
||||
let store_reader = eml_segment
|
||||
.get_store_reader(2)
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
|
||||
for doc_id in chunk_start..chunk_end {
|
||||
if eml_segment.is_deleted(doc_id) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let eid = f_id_col.values.get_val(doc_id);
|
||||
|
||||
let (uid, internal_date) = match envelope_map.get(&eid) {
|
||||
Some(v) => *v,
|
||||
None => {
|
||||
on_progress(&format!("WARN: eid {} envelope not found", eid));
|
||||
total_skipped += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let eml_doc: TantivyDocument = store_reader
|
||||
.get(doc_id)
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
|
||||
let account_id = match eml_doc.get_first(mf.f_account_id).and_then(|v| v.as_u64()) {
|
||||
Some(v) => v,
|
||||
None => {
|
||||
on_progress(&format!("WARN: eid {} account_id missing", eid));
|
||||
total_skipped += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let mailbox_id = eml_doc
|
||||
.get_first(mf.f_mailbox_id)
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(0);
|
||||
|
||||
let eml_bytes = match eml_doc.get_first(mf.f_eml).and_then(|v| v.as_bytes()) {
|
||||
Some(b) => b,
|
||||
None => {
|
||||
on_progress(&format!("WARN: eid {} eml bytes missing", eid));
|
||||
total_skipped += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(e) = writer.ingest(eml_bytes, account_id, mailbox_id, uid, internal_date) {
|
||||
on_progress(&format!(
|
||||
"ERROR: Account {} eid {} ingest failed: {}",
|
||||
account_id, eid, e
|
||||
));
|
||||
total_skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
total_migrated += 1;
|
||||
|
||||
if total_migrated % 10 == 0 || total_migrated as u32 == num_docs {
|
||||
on_progress(&format!("PROGRESS:{}:{}", total_migrated, num_docs));
|
||||
}
|
||||
}
|
||||
|
||||
drop(store_reader);
|
||||
|
||||
// Flush blob buffers to bichon-blob engine.
|
||||
writer.flush_blob_buffers()?;
|
||||
|
||||
chunk_start = chunk_end;
|
||||
}
|
||||
|
||||
on_progress(&format!("DONE:{}:{}", total_migrated, total_skipped));
|
||||
Ok(())
|
||||
}
|
||||
460
crates/admin/src/migrate_v1.rs
Normal file
460
crates/admin/src/migrate_v1.rs
Normal file
@@ -0,0 +1,460 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use bichon_blob::{Codec, Config, Engine};
|
||||
use bichon_core::{
|
||||
error::{code::ErrorCode, BichonResult},
|
||||
migrate::write_storage_version,
|
||||
raise_error,
|
||||
};
|
||||
use console::style;
|
||||
use dialoguer::{theme::ColorfulTheme, Input};
|
||||
use fjall::{Config as FjallConfig, Database};
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
|
||||
fn hex_key_to_raw(hex_bytes: &[u8]) -> BichonResult<[u8; 32]> {
|
||||
let hex_str = std::str::from_utf8(hex_bytes).map_err(|e| {
|
||||
raise_error!(
|
||||
format!("invalid UTF-8 in fjall key: {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
let mut raw = [0u8; 32];
|
||||
hex::decode_to_slice(hex_str, &mut raw).map_err(|e| {
|
||||
raise_error!(
|
||||
format!("invalid hex in fjall key '{hex_str}': {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
Ok(raw)
|
||||
}
|
||||
|
||||
fn migrate_keyspace(
|
||||
engine: &Engine,
|
||||
db: &Database,
|
||||
ks_name: &str,
|
||||
label: &str,
|
||||
batch_size: usize,
|
||||
) -> BichonResult<u64> {
|
||||
|
||||
let ks = db
|
||||
.keyspace(ks_name, || {
|
||||
panic!("{ks_name} keyspace not found in fjall database")
|
||||
})
|
||||
.map_err(|e| {
|
||||
raise_error!(
|
||||
format!("failed to open fjall keyspace '{ks_name}': {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
|
||||
let pb = ProgressBar::new_spinner();
|
||||
pb.set_style(
|
||||
ProgressStyle::with_template("{spinner:.cyan} {msg} [{elapsed_precise}]")
|
||||
.unwrap(),
|
||||
);
|
||||
pb.set_message(format!("Scanning {label} blobs..."));
|
||||
|
||||
let mut count: u64 = 0;
|
||||
let mut batch: Vec<([u8; 32], Vec<u8>, Codec)> = Vec::with_capacity(batch_size);
|
||||
|
||||
for item in ks.iter() {
|
||||
let (key_bytes, value) = item.into_inner().map_err(|e| {
|
||||
raise_error!(
|
||||
format!("fjall iter error in '{ks_name}': {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
|
||||
if value.is_empty() {
|
||||
continue;
|
||||
}
|
||||
// MAX_VALUE_SIZE = 100 MB (bichon_blob::types)
|
||||
if value.len() > 100 * 1024 * 1024 {
|
||||
let raw_key = hex_key_to_raw(&key_bytes)?;
|
||||
eprintln!(
|
||||
"{}",
|
||||
console::style(format!(
|
||||
"WARN: skipping oversized blob key={} ({} bytes)",
|
||||
hex::encode(raw_key),
|
||||
value.len()
|
||||
))
|
||||
.yellow()
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let raw_key = hex_key_to_raw(&key_bytes)?;
|
||||
batch.push((raw_key, value.to_vec(), Codec::Zstd));
|
||||
|
||||
if batch.len() >= batch_size {
|
||||
engine.put_batch(&batch).map_err(|e| {
|
||||
raise_error!(format!("{e:#?}"), ErrorCode::InternalError)
|
||||
})?;
|
||||
count += batch.len() as u64;
|
||||
pb.set_message(format!("{label}: {} blobs migrated...", count));
|
||||
batch.clear();
|
||||
}
|
||||
}
|
||||
|
||||
if !batch.is_empty() {
|
||||
engine.put_batch(&batch).map_err(|e| {
|
||||
raise_error!(format!("{e:#?}"), ErrorCode::InternalError)
|
||||
})?;
|
||||
count += batch.len() as u64;
|
||||
}
|
||||
|
||||
pb.finish_with_message(format!("{label}: {} blobs migrated", count));
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
pub fn handle_migrate_v1(theme: &ColorfulTheme) {
|
||||
println!(
|
||||
"\n{}",
|
||||
style("MIGRATION: Bichon v1.x Storage → v2.x (Fjall → bichon-blob)")
|
||||
.bold()
|
||||
.yellow()
|
||||
);
|
||||
println!(
|
||||
"{}\n",
|
||||
style("This migrates blob storage from the fjall engine to bichon-blob.\n\
|
||||
Tantivy indexes and metadata (memdb) are NOT affected.")
|
||||
.dim()
|
||||
);
|
||||
|
||||
let root_dir: String = Input::with_theme(theme)
|
||||
.with_prompt("Enter --bichon-root-dir (same value used by the old server)")
|
||||
.validate_with(|input: &String| -> Result<(), &str> {
|
||||
let path = PathBuf::from(input);
|
||||
if !path.is_absolute() {
|
||||
return Err("Path must be absolute.");
|
||||
}
|
||||
if !path.exists() {
|
||||
return Err("Directory does not exist.");
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.interact_text()
|
||||
.unwrap();
|
||||
let root_dir = PathBuf::from(root_dir.trim());
|
||||
|
||||
let data_base = {
|
||||
let input: String = Input::with_theme(theme)
|
||||
.with_prompt("Enter --bichon-data-dir (leave blank to use root directory)")
|
||||
.allow_empty(true)
|
||||
.interact_text()
|
||||
.unwrap();
|
||||
if input.trim().is_empty() {
|
||||
root_dir.clone()
|
||||
} else {
|
||||
let path = PathBuf::from(input.trim());
|
||||
if !path.exists() {
|
||||
eprintln!(
|
||||
"{}",
|
||||
style(format!("Data directory does not exist: {}", path.display())).red()
|
||||
);
|
||||
return;
|
||||
}
|
||||
path
|
||||
}
|
||||
};
|
||||
|
||||
let fjall_path = data_base.join("bichon-storage");
|
||||
let blob_path = fjall_path.join("blobs");
|
||||
|
||||
if !fjall_path.exists() {
|
||||
println!(
|
||||
"{}",
|
||||
style(format!(
|
||||
"Fjall database not found at '{}'. Is this really a v1.x install?",
|
||||
fjall_path.display()
|
||||
))
|
||||
.red()
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if blob_path.exists() {
|
||||
println!(
|
||||
"{}",
|
||||
style(format!(
|
||||
"Target blob directory '{}' already exists.\n\
|
||||
If you have already migrated, you can remove the old fjall files manually.\n\
|
||||
Otherwise, delete this directory and re-run the migration.",
|
||||
blob_path.display()
|
||||
))
|
||||
.yellow()
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let batch_size: usize = {
|
||||
let input: String = Input::with_theme(theme)
|
||||
.with_prompt("Enter batch size (affects memory usage, higher = faster but uses more RAM)")
|
||||
.default("1000".to_string())
|
||||
.validate_with(|s: &String| match s.trim().parse::<usize>() {
|
||||
Ok(n) if n > 0 => Ok(()),
|
||||
_ => Err("Please enter a valid positive number"),
|
||||
})
|
||||
.interact_text()
|
||||
.unwrap_or("1000".to_string());
|
||||
input.trim().parse::<usize>().unwrap_or(1000)
|
||||
};
|
||||
|
||||
println!(
|
||||
"{} Using batch size: {}\n",
|
||||
style("✓").green(),
|
||||
style(batch_size).cyan().bold()
|
||||
);
|
||||
|
||||
// Open old Fjall database (read-only by nature of the iter API)
|
||||
println!("\n{}", style("Opening fjall database...").dim());
|
||||
let db = match Database::open(FjallConfig::new(&fjall_path)) {
|
||||
Ok(db) => db,
|
||||
Err(e) => {
|
||||
println!(
|
||||
"{}",
|
||||
style(format!("Failed to open fjall database: {e:#?}")).red()
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Open new bichon-blob engine
|
||||
println!("{}", style("Initializing bichon-blob engine...").dim());
|
||||
let mut config = Config::default();
|
||||
config.default_codec = Codec::Zstd;
|
||||
config.compress_threshold = 1024;
|
||||
config.flush_interval_secs = 0;
|
||||
config.gc_interval_secs = 0;
|
||||
|
||||
let engine = match Engine::open(&blob_path, config) {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
println!(
|
||||
"{}",
|
||||
style(format!("Failed to open bichon-blob engine: {e:#?}")).red()
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Migrate email blobs
|
||||
let email_count = match migrate_keyspace(&engine, &db, "email", "Email", batch_size) {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
println!("{}", style(format!("Email migration failed: {e:#?}")).red());
|
||||
let _ = engine.shutdown();
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Migrate attachment blobs
|
||||
let attach_count =
|
||||
match migrate_keyspace(&engine, &db, "attachments", "Attachment", batch_size) {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
println!(
|
||||
"{}",
|
||||
style(format!("Attachment migration failed: {e:#?}")).red()
|
||||
);
|
||||
let _ = engine.shutdown();
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Flush and shutdown
|
||||
println!("\n{}", style("Flushing and shutting down blob engine...").dim());
|
||||
if let Err(e) = engine.flush() {
|
||||
println!("{}", style(format!("flush warning: {e:#?}")).yellow());
|
||||
}
|
||||
if let Err(e) = engine.shutdown() {
|
||||
println!(
|
||||
"{}",
|
||||
style(format!("shutdown error: {e:#?}")).red()
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Write STORAGE_VERSION = 2
|
||||
if let Err(e) = write_storage_version(&root_dir, 2) {
|
||||
println!(
|
||||
"{}",
|
||||
style(format!("Failed to write STORAGE_VERSION: {e:#?}")).red()
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
println!(
|
||||
"\n{}",
|
||||
style(format!(
|
||||
"Migration complete!\n Email blobs: {}\n Attachment blobs: {}\n Total: {}\n\n\
|
||||
The old fjall database at '{}' is no longer used.\n\
|
||||
You may delete it to free disk space after verifying everything works.",
|
||||
email_count,
|
||||
attach_count,
|
||||
email_count + attach_count,
|
||||
fjall_path.display()
|
||||
))
|
||||
.green()
|
||||
.bold()
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use bichon_core::utils::compute_content_hash;
|
||||
|
||||
// ── hex_key_to_raw ────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn hex_key_to_raw_valid() {
|
||||
// "hello" blake3 hex = 64 chars
|
||||
let hash_hex = compute_content_hash(b"hello");
|
||||
assert_eq!(hash_hex.len(), 64);
|
||||
|
||||
let raw = hex_key_to_raw(hash_hex.as_bytes()).unwrap();
|
||||
// Decoding 64 hex chars → 32 bytes
|
||||
assert_eq!(raw.len(), 32);
|
||||
// Round-trip: raw → hex should match original
|
||||
assert_eq!(hex::encode(raw), hash_hex);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hex_key_to_raw_invalid_utf8() {
|
||||
// 0xFF is not valid UTF-8
|
||||
let invalid = vec![0xFFu8; 64];
|
||||
let err = hex_key_to_raw(&invalid).unwrap_err();
|
||||
assert!(err.to_string().contains("invalid UTF-8"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hex_key_to_raw_invalid_hex() {
|
||||
// "zz" is valid UTF-8 but not valid hex
|
||||
let invalid = b"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz";
|
||||
let err = hex_key_to_raw(invalid).unwrap_err();
|
||||
assert!(err.to_string().contains("invalid hex"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hex_key_to_raw_wrong_length() {
|
||||
let short = b"abcd";
|
||||
let err = hex_key_to_raw(short).unwrap_err();
|
||||
assert!(err.to_string().contains("invalid hex"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hex_key_to_raw_different_content() {
|
||||
let a = hex_key_to_raw(compute_content_hash(b"a").as_bytes()).unwrap();
|
||||
let b = hex_key_to_raw(compute_content_hash(b"b").as_bytes()).unwrap();
|
||||
assert_ne!(a, b);
|
||||
}
|
||||
|
||||
// ── migrate_keyspace integration ──────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn migrate_keyspace_end_to_end() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let fjall_path = tmp.path().join("fjall");
|
||||
let blob_path = tmp.path().join("blobs");
|
||||
|
||||
use fjall::KeyspaceCreateOptions;
|
||||
|
||||
// --- Setup: create a Fjall database with test blobs ---
|
||||
let fjall_db = Database::open(FjallConfig::new(&fjall_path)).unwrap();
|
||||
let ks = fjall_db
|
||||
.keyspace("test_ks", KeyspaceCreateOptions::default)
|
||||
.unwrap();
|
||||
|
||||
// Insert test blobs using hex string keys (matching v1.x convention)
|
||||
let mut expected: Vec<(String, Vec<u8>)> = Vec::new();
|
||||
for i in 0..10 {
|
||||
let data = format!("blob data {}", i).into_bytes();
|
||||
let hash = compute_content_hash(&data);
|
||||
ks.insert(hash.as_bytes(), data.clone()).unwrap();
|
||||
expected.push((hash, data));
|
||||
}
|
||||
|
||||
// --- Setup: create bichon-blob engine and run migration ---
|
||||
{
|
||||
let mut config = Config::default();
|
||||
config.flush_interval_secs = 0;
|
||||
config.gc_interval_secs = 0;
|
||||
let engine = Engine::open(&blob_path, config).unwrap();
|
||||
|
||||
let count = migrate_keyspace(&engine, &fjall_db, "test_ks", "Test", 100).unwrap();
|
||||
assert_eq!(count, expected.len() as u64);
|
||||
|
||||
engine.flush().unwrap();
|
||||
engine.shutdown().unwrap();
|
||||
// engine dropped here → LOCK released
|
||||
}
|
||||
|
||||
// --- Verify: re-open engine and check all blobs ---
|
||||
let mut config = Config::default();
|
||||
config.flush_interval_secs = 0;
|
||||
config.gc_interval_secs = 0;
|
||||
let engine2 = Engine::open(&blob_path, config).unwrap();
|
||||
|
||||
for (hex_hash, expected_data) in &expected {
|
||||
let mut raw_key = [0u8; 32];
|
||||
hex::decode_to_slice(hex_hash, &mut raw_key).unwrap();
|
||||
let got = engine2.get(&raw_key).unwrap();
|
||||
assert_eq!(
|
||||
got.as_deref(),
|
||||
Some(expected_data.as_slice()),
|
||||
"mismatch for key {}",
|
||||
hex_hash
|
||||
);
|
||||
}
|
||||
|
||||
// Verify non-existent key returns None
|
||||
let fake_hash = compute_content_hash(b"nonexistent");
|
||||
let mut fake_key = [0u8; 32];
|
||||
hex::decode_to_slice(&fake_hash, &mut fake_key).unwrap();
|
||||
assert!(engine2.get(&fake_key).unwrap().is_none());
|
||||
|
||||
engine2.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migrate_empty_keyspace() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let fjall_path = tmp.path().join("fjall");
|
||||
let blob_path = tmp.path().join("blobs");
|
||||
|
||||
let fjall_db = Database::open(FjallConfig::new(&fjall_path)).unwrap();
|
||||
let _ks = fjall_db
|
||||
.keyspace("empty_ks", fjall::KeyspaceCreateOptions::default)
|
||||
.unwrap();
|
||||
|
||||
let mut config = Config::default();
|
||||
config.flush_interval_secs = 0;
|
||||
config.gc_interval_secs = 0;
|
||||
let engine = Engine::open(&blob_path, config).unwrap();
|
||||
|
||||
let count = migrate_keyspace(&engine, &fjall_db, "empty_ks", "Empty", 100).unwrap();
|
||||
assert_eq!(count, 0);
|
||||
|
||||
engine.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hex_key_roundtrip_with_real_content_hash() {
|
||||
// Simulate the exact data flow from v1.x to v2.x
|
||||
let eml_data = b"From: sender@example.com\r\nSubject: Test\r\n\r\nHello world";
|
||||
let hash_hex = compute_content_hash(eml_data); // 64-char hex string
|
||||
|
||||
// v1.x: key stored as hash_hex.as_bytes()
|
||||
let fjall_key = hash_hex.as_bytes().to_vec();
|
||||
assert_eq!(fjall_key.len(), 64);
|
||||
|
||||
// Migration: hex decode → raw 32 bytes
|
||||
let raw_key = hex_key_to_raw(&fjall_key).unwrap();
|
||||
assert_eq!(raw_key.len(), 32);
|
||||
|
||||
// v2.x: engine.put(raw_key, data)
|
||||
// Verify round-trip: raw_key → hex → compare
|
||||
let hex_roundtrip = hex::encode(raw_key);
|
||||
assert_eq!(hex_roundtrip, hash_hex);
|
||||
}
|
||||
}
|
||||
@@ -10,13 +10,15 @@ zstd = "0.13"
|
||||
lz4_flex = "0.13.1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
bincode = "1"
|
||||
bincode = { package = "bincode_reloaded", version = "3.1.10", default-features = false, features = ["serde", "alloc"] }
|
||||
tracing = "0.1"
|
||||
thiserror = "2"
|
||||
redb = "4.1"
|
||||
fs2 = "0.4"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
rand = "0.10.1"
|
||||
rand = "0.10.2"
|
||||
criterion = { version = "0.6", features = ["html_reports"] }
|
||||
|
||||
[[bench]]
|
||||
|
||||
177
crates/blob/README.md
Normal file
177
crates/blob/README.md
Normal file
@@ -0,0 +1,177 @@
|
||||
# bichon-blob
|
||||
|
||||
Content-addressable blob store for [Bichon](https://github.com/rustmailer/bichon) email archival.
|
||||
|
||||
All data is keyed by a 32-byte content hash — identical content is stored only once. The caller (Bichon) strips attachments from emails and stores the "holed" raw email + attachments into this store, tracking reference counts in its own schema.
|
||||
|
||||
## On-disk layout
|
||||
|
||||
```
|
||||
<root>/
|
||||
├── meta.bin # global metadata (bincode + CRC32)
|
||||
├── index.redb # key → (segment_id, offset, size) index (redb B-tree)
|
||||
├── segments/
|
||||
│ ├── 00000001.seg # append-only segment files (≤ 1 GB each)
|
||||
│ ├── 00000002.seg
|
||||
│ └── ...
|
||||
```
|
||||
|
||||
### Segment entry format (50-byte fixed header + variable data)
|
||||
|
||||
```
|
||||
magic(4) crc32(4) flags(1) codec(1) key(32) raw_size(4) data_size(4) data(*)
|
||||
```
|
||||
|
||||
- `magic`: `0xB3DB_0001` — entry boundary validation
|
||||
- `crc32`: covers everything after this field
|
||||
- `flags`: `0` = live, `1` = tombstone
|
||||
- `codec`: `0` = none, `1` = Zstd, `2` = Lz4
|
||||
- `key`: 32-byte content hash (BLAKE3 / SHA-256)
|
||||
- `raw_size`: original uncompressed size
|
||||
- `data_size`: on-disk data size (after compression)
|
||||
|
||||
### Index store
|
||||
|
||||
The key → (segment_id, offset, data_size, flags) mapping is stored in a single [redb](https://github.com/cberner/redb) database (`index.redb`). redb provides:
|
||||
|
||||
- **B-tree + mmap**: O(log N) point lookups with zero heap allocation — pages are faulted in on demand.
|
||||
- **ACID transactions**: every index write is durable and atomic.
|
||||
- **Crash recovery**: handled transparently by redb's WAL — no manual reload or rebuild logic.
|
||||
- **O(1) startup**: only the B-tree root page is read at open time.
|
||||
|
||||
Records are stored as fixed-size 56-byte blobs, each carrying an internal CRC32 checksum.
|
||||
|
||||
## Read path
|
||||
|
||||
```
|
||||
get(key)
|
||||
→ index_store.get(key) # redb B-tree lookup, zero-copy
|
||||
→ IndexRecord CRC32 verify # (segment_id, offset, data_size, flags)
|
||||
→ pread entry from segment file
|
||||
→ entry CRC32 verify → decompress → return value
|
||||
```
|
||||
|
||||
The index record and the segment entry carry independent CRC32 checksums. Corruption in one record or entry is contained — it never affects other keys.
|
||||
|
||||
## Write path
|
||||
|
||||
```
|
||||
put(key, value, codec)
|
||||
→ compress value (Zstd/Lz4 if ≥ 4 KB, else store raw)
|
||||
→ append entry to active segment file
|
||||
→ insert IndexRecord into redb (single write txn)
|
||||
→ update metadata (indexed_up_to_offset)
|
||||
→ if segment ≥ 1 GB → seal it, create new segment
|
||||
```
|
||||
|
||||
## Delete
|
||||
|
||||
Deletes are **tombstones** — an entry with `flags=1` and empty data is appended to the active segment. Before writing the tombstone, the existing index record is consulted to increment `deleted_bytes` on the **original** segment (the one that holds the live data). This drives the GC threshold.
|
||||
|
||||
```
|
||||
delete(key)
|
||||
→ index_store.get(key) → find original (segment_id, data_size)
|
||||
→ original_segment.deleted_bytes += data_size
|
||||
→ recompute deleted_ratio on original segment
|
||||
→ append tombstone entry to active segment
|
||||
→ insert tombstone IndexRecord into redb
|
||||
→ index_store.get(key) now returns None
|
||||
```
|
||||
|
||||
## GC
|
||||
|
||||
**Segment GC:** two-phase, driven by the `deleted_ratio` tracked per segment.
|
||||
|
||||
### Trigger
|
||||
|
||||
- **Background**: the `blob-gc` thread wakes up every `gc_interval_secs` (default 300s), checks whether any sealed segment's `deleted_ratio ≥ gc_deleted_ratio` (default 0.30), and runs GC on the worst segment if so.
|
||||
- **Manual**: `engine.gc_if_needed()` or `engine.gc()`.
|
||||
|
||||
### Phase 1 — scan & compact (read-only, no write lock)
|
||||
|
||||
1. Pick the sealed segment with the highest `deleted_ratio`.
|
||||
2. Scan only that segment's entries.
|
||||
3. For each entry, ask the index: "is this entry still the latest version for its key?"
|
||||
- If the index points to this exact `(segment_id, offset)` → **keep**, write to a temp segment file.
|
||||
- If the index points elsewhere (overwritten by a later segment, or a tombstone) → **skip** (stale).
|
||||
4. Fsync the temp file.
|
||||
|
||||
Phase 1 holds only the read lock — `put` / `delete` continue uninterrupted.
|
||||
|
||||
### Phase 2 — commit & update index (write lock)
|
||||
|
||||
1. Atomically rename the temp file over the original segment.
|
||||
2. Batch-insert new `IndexRecord`s (now at new offsets) into redb. Old records with the same key are naturally overwritten.
|
||||
3. Reset the segment's `deleted_bytes` and `deleted_ratio` to zero.
|
||||
4. Persist metadata.
|
||||
|
||||
Phase 2 holds the write lock, but is fast — no full segment scan, no full index rebuild.
|
||||
|
||||
## Data integrity
|
||||
|
||||
Every record on disk is independently checksummed:
|
||||
|
||||
| Layer | Format | Protection |
|
||||
|---|---|---|
|
||||
| Segment entry | 50-byte header + data | CRC32 covers all fields + data |
|
||||
| Index record | 56 bytes | CRC32 covers key + segment_id + offset + data_size + flags |
|
||||
| Global metadata | bincode blob | CRC32 + version header |
|
||||
|
||||
Corruption is **contained** — a bad segment entry or index record produces an error for that key only. Recovery and GC skip corrupt records (with a warning) rather than aborting. The index is backed by redb's B-tree which maintains its own internal integrity.
|
||||
|
||||
## Crash recovery
|
||||
|
||||
- Temp files from interrupted GC are cleaned up on open.
|
||||
- Any segment data beyond `indexed_up_to_offset` is scanned and inserted into the index.
|
||||
- Partial writes at the tail of a segment (detected via CRC32 mismatch near EOF) are truncated.
|
||||
- redb's WAL ensures the index is always consistent — no manual reload or rebuild needed.
|
||||
|
||||
## Background threads
|
||||
|
||||
Set `Config.flush_interval_secs` and `Config.gc_interval_secs` to positive values to enable periodic background work:
|
||||
|
||||
| Thread | Config | Default | What it does |
|
||||
|---|---|---|---|
|
||||
| `blob-flush` | `flush_interval_secs` | `0` (off) | Fsync the active segment and save metadata |
|
||||
| `blob-gc` | `gc_interval_secs` | `0` (off) | Check deleted-ratio, compact one segment if needed |
|
||||
|
||||
The two threads are independent — a long GC run never blocks fsync.
|
||||
|
||||
## Config
|
||||
|
||||
| Field | Default | Notes |
|
||||
|---|---|---|
|
||||
| `compress_threshold` | 4096 | Bytes; smaller values stored uncompressed |
|
||||
| `default_codec` | Zstd | Also supports Lz4 |
|
||||
| `compression_level` | 0 | Zstd compression level |
|
||||
| `gc_deleted_ratio` | 0.30 | Trigger GC when a sealed segment exceeds this |
|
||||
| `flush_interval_secs` | 0 | 0 = disabled; ≥ 5 for periodic background fsync |
|
||||
| `gc_interval_secs` | 0 | 0 = disabled; ≥ 10 for periodic background GC |
|
||||
|
||||
## Basic usage
|
||||
|
||||
```rust
|
||||
use bichon_blob::{Codec, Config, Engine};
|
||||
|
||||
let engine = Engine::open(path, Config::default())?;
|
||||
|
||||
// Store
|
||||
let hash = blake3::hash(b"email body").into();
|
||||
engine.put(hash, b"email body", Codec::Zstd)?;
|
||||
|
||||
// Retrieve
|
||||
let value = engine.get(&hash)?; // Some(Vec<u8>) or None
|
||||
|
||||
// Delete (caller must track refcounts)
|
||||
engine.delete(&hash)?;
|
||||
|
||||
// Batch operations
|
||||
engine.put_batch(&[(hash1, data1, Codec::Zstd), (hash2, data2, Codec::Lz4)])?;
|
||||
engine.delete_batch(&[hash1, hash2])?;
|
||||
|
||||
// GC
|
||||
engine.gc_if_needed()?;
|
||||
|
||||
// Clean shutdown
|
||||
engine.shutdown()?;
|
||||
```
|
||||
@@ -12,7 +12,6 @@ fn make_key(seed: u64) -> [u8; 32] {
|
||||
|
||||
fn make_value(size: usize) -> Vec<u8> {
|
||||
let mut v = Vec::with_capacity(size);
|
||||
// Fill with somewhat realistic text-like data so compression works
|
||||
let pattern = b"The quick brown fox jumps over the lazy dog. ";
|
||||
while v.len() < size {
|
||||
let rem = size - v.len();
|
||||
@@ -29,7 +28,6 @@ pub fn bench_write_small(c: &mut Criterion) {
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(1024); // 1 KB
|
||||
let mut counter = 0u64;
|
||||
@@ -41,9 +39,7 @@ pub fn bench_write_small(c: &mut Criterion) {
|
||||
(make_key(counter), value.clone())
|
||||
},
|
||||
|(key, val)| {
|
||||
engine
|
||||
.write("bench", key, &val, Codec::Zstd)
|
||||
.unwrap()
|
||||
engine.put(key, &val, Codec::Zstd).unwrap()
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
@@ -58,7 +54,6 @@ pub fn bench_write_medium(c: &mut Criterion) {
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(64 * 1024); // 64 KB
|
||||
let mut counter = 0u64;
|
||||
@@ -70,9 +65,7 @@ pub fn bench_write_medium(c: &mut Criterion) {
|
||||
(make_key(counter), value.clone())
|
||||
},
|
||||
|(key, val)| {
|
||||
engine
|
||||
.write("bench", key, &val, Codec::Zstd)
|
||||
.unwrap()
|
||||
engine.put(key, &val, Codec::Zstd).unwrap()
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
@@ -87,7 +80,6 @@ pub fn bench_write_large(c: &mut Criterion) {
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(1024 * 1024); // 1 MB
|
||||
let mut counter = 0u64;
|
||||
@@ -99,9 +91,7 @@ pub fn bench_write_large(c: &mut Criterion) {
|
||||
(make_key(counter), value.clone())
|
||||
},
|
||||
|(key, val)| {
|
||||
engine
|
||||
.write("bench", key, &val, Codec::Zstd)
|
||||
.unwrap()
|
||||
engine.put(key, &val, Codec::Zstd).unwrap()
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
@@ -109,59 +99,55 @@ pub fn bench_write_large(c: &mut Criterion) {
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_read_cache_hit(c: &mut Criterion) {
|
||||
pub fn bench_read_hot(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("read");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
// Pre-populate: 10 keys, all in same bucket → cache hit after first read
|
||||
// Pre-populate: 10 keys
|
||||
let value = make_value(4096);
|
||||
for i in 0..10u64 {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::Zstd)
|
||||
.put(make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut counter = 0u64;
|
||||
group.bench_function("cache_hit", |b| {
|
||||
group.bench_function("hot", |b| {
|
||||
b.iter(|| {
|
||||
let key = make_key(counter % 10);
|
||||
counter += 1;
|
||||
std::hint::black_box(engine.read("bench", &key).unwrap());
|
||||
std::hint::black_box(engine.get(&key).unwrap());
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_read_cache_miss(c: &mut Criterion) {
|
||||
pub fn bench_read_cold(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("read");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut config = Config::default();
|
||||
config.lru_bucket_count = 8; // Small cache to force misses
|
||||
let engine = Engine::open(dir.path(), config).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
|
||||
let value = make_value(4096);
|
||||
// Write 1000 keys spread across all 16 buckets — small LRU will thrash
|
||||
for i in 0..1000u64 {
|
||||
// Write 5000 keys across all 256 buckets — mmap page faults will occur
|
||||
for i in 0..5000u64 {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::Zstd)
|
||||
.put(make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut counter = 0u64;
|
||||
group.bench_function("cache_miss", |b| {
|
||||
group.bench_function("cold", |b| {
|
||||
b.iter(|| {
|
||||
let key = make_key(counter % 1000);
|
||||
let key = make_key(counter % 5000);
|
||||
counter += 1;
|
||||
std::hint::black_box(engine.read("bench", &key).unwrap());
|
||||
std::hint::black_box(engine.get(&key).unwrap());
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
@@ -174,21 +160,20 @@ pub fn bench_read_large_value(c: &mut Criterion) {
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(1024 * 1024); // 1 MB
|
||||
for i in 0..5u64 {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::Zstd)
|
||||
.put(make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut counter = 0u64;
|
||||
group.bench_function("1MB_cache_hit", |b| {
|
||||
group.bench_function("1MB", |b| {
|
||||
b.iter(|| {
|
||||
let key = make_key(counter % 5);
|
||||
counter += 1;
|
||||
std::hint::black_box(engine.read("bench", &key).unwrap());
|
||||
std::hint::black_box(engine.get(&key).unwrap());
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
@@ -202,7 +187,6 @@ pub fn bench_delete(c: &mut Criterion) {
|
||||
group.bench_function("delete", |b| {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(4096);
|
||||
let mut counter = 0u64;
|
||||
@@ -211,13 +195,11 @@ pub fn bench_delete(c: &mut Criterion) {
|
||||
|| {
|
||||
counter += 1;
|
||||
let key = make_key(counter);
|
||||
engine
|
||||
.write("bench", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
engine.put(key, &value, Codec::Zstd).unwrap();
|
||||
key
|
||||
},
|
||||
|key| {
|
||||
engine.delete("bench", &key).unwrap();
|
||||
engine.delete(&key).unwrap();
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
@@ -232,13 +214,12 @@ pub fn bench_mixed_workload(c: &mut Criterion) {
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
// Pre-populate with 500 entries
|
||||
let value = make_value(8192);
|
||||
for i in 0..500u64 {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::Zstd)
|
||||
.put(make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
@@ -249,20 +230,19 @@ pub fn bench_mixed_workload(c: &mut Criterion) {
|
||||
let op = counter % 100;
|
||||
match op {
|
||||
0..=79 => {
|
||||
// 80% writes
|
||||
let key = make_key(counter);
|
||||
let val = make_value(4096);
|
||||
engine.write("bench", key, &val, Codec::Zstd).unwrap();
|
||||
engine.put(key, &val, Codec::Zstd).unwrap();
|
||||
}
|
||||
80..=94 => {
|
||||
// 15% reads
|
||||
std::hint::black_box(engine.read("bench", &make_key(counter % 500)).unwrap());
|
||||
std::hint::black_box(
|
||||
engine.get(&make_key(counter % 500)).unwrap(),
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
// 5% deletes
|
||||
if counter % 2 == 0 {
|
||||
let key = make_key(counter % 500);
|
||||
let _ = engine.delete("bench", &key);
|
||||
let _ = engine.delete(&key);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -279,23 +259,20 @@ pub fn bench_gc(c: &mut Criterion) {
|
||||
group.bench_function("gc_30pct_deleted", |b| {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
// Fill a segment with ~1000 entries, then delete 30%
|
||||
let value = make_value(200_000); // 200KB each → ~1000 entries to fill 256MB
|
||||
let value = make_value(200_000);
|
||||
let n = 1200u64;
|
||||
for i in 0..n {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::None)
|
||||
.put(make_key(i), &value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
// Delete ~30%
|
||||
for i in (0..n).step_by(3) {
|
||||
engine.delete("bench", &make_key(i)).unwrap();
|
||||
engine.delete(&make_key(i)).unwrap();
|
||||
}
|
||||
|
||||
b.iter(|| {
|
||||
engine.gc("bench").unwrap();
|
||||
engine.gc().unwrap();
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
@@ -306,8 +283,8 @@ criterion_group!(
|
||||
bench_write_small,
|
||||
bench_write_medium,
|
||||
bench_write_large,
|
||||
bench_read_cache_hit,
|
||||
bench_read_cache_miss,
|
||||
bench_read_hot,
|
||||
bench_read_cold,
|
||||
bench_read_large_value,
|
||||
bench_delete,
|
||||
bench_mixed_workload,
|
||||
|
||||
125
crates/blob/examples/email_archive.rs
Normal file
125
crates/blob/examples/email_archive.rs
Normal file
@@ -0,0 +1,125 @@
|
||||
/// bichon-blob usage example: email archival with content-addressable storage.
|
||||
///
|
||||
/// This example simulates a mail archival system where multiple accounts
|
||||
/// may receive the same email (e.g. CC'd or forwarded). The blob store
|
||||
/// deduplicates by content hash, and the upper application layer tracks
|
||||
/// which accounts reference each hash.
|
||||
///
|
||||
/// Run: cargo run --example email_archive
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use bichon_blob::{Codec, Config, Engine};
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// ── Setup ──────────────────────────────────────────────────────────
|
||||
|
||||
let store_path = std::path::Path::new("target/example_blob_store");
|
||||
let _ = std::fs::remove_dir_all(store_path); // clean up from previous run
|
||||
let engine = Engine::open(store_path, Config::default())?;
|
||||
|
||||
println!("Store opened at: {:?}\n", store_path);
|
||||
|
||||
// ── Simulated upper-layer reference tracker ────────────────────────
|
||||
let mut refs: HashMap<String, Vec<[u8; 32]>> = HashMap::new();
|
||||
|
||||
// ── 1. Store emails ────────────────────────────────────────────────
|
||||
|
||||
// Simulate three emails arriving. Email #2 is a newsletter that
|
||||
// both alice and bob received — identical content, same hash.
|
||||
let emails = vec![
|
||||
("alice", "Welcome to Bichon Mail!"),
|
||||
("alice", "Weekly Newsletter: Rust Edition"),
|
||||
("bob", "Weekly Newsletter: Rust Edition"), // same content as above
|
||||
];
|
||||
|
||||
for (account, body) in &emails {
|
||||
let hash = mock_content_hash(body.as_bytes());
|
||||
let account_refs = refs.entry(account.to_string()).or_default();
|
||||
|
||||
// Check if this content already exists in the blob store
|
||||
if engine.exists(&hash)? {
|
||||
println!(
|
||||
"[dedup] {}: hash {:02x?}... already stored, skipping",
|
||||
account,
|
||||
&hash[..4]
|
||||
);
|
||||
} else {
|
||||
engine.put(hash, body.as_bytes(), Codec::Zstd)?;
|
||||
println!(
|
||||
"[store] {}: hash {:02x?}..., {} bytes",
|
||||
account,
|
||||
&hash[..4],
|
||||
body.len()
|
||||
);
|
||||
}
|
||||
|
||||
account_refs.push(hash);
|
||||
}
|
||||
|
||||
println!();
|
||||
|
||||
// ── 2. Read back emails ────────────────────────────────────────────
|
||||
|
||||
let hash = mock_content_hash(b"Weekly Newsletter: Rust Edition");
|
||||
let stored = engine.get(&hash)?;
|
||||
println!(
|
||||
"Read newsletter: {:?}",
|
||||
stored.map(|v| String::from_utf8_lossy(&v).to_string())
|
||||
);
|
||||
|
||||
// ── 3. Batch store attachments ─────────────────────────────────────
|
||||
|
||||
let attachments: Vec<([u8; 32], Vec<u8>, Codec)> = (0..10)
|
||||
.map(|i| {
|
||||
let body = format!("Attachment #{}: {}", i, "X".repeat(5000));
|
||||
let hash = mock_content_hash(body.as_bytes());
|
||||
(hash, body.into_bytes(), Codec::Zstd)
|
||||
})
|
||||
.collect();
|
||||
|
||||
engine.put_batch(&attachments)?;
|
||||
println!("\nBatch-stored {} attachments", attachments.len());
|
||||
|
||||
// ── 4. Stats ───────────────────────────────────────────────────────
|
||||
|
||||
let stats = engine.stats()?;
|
||||
println!(
|
||||
"Stats: {} keys, {} bytes, {} segments",
|
||||
stats.total_keys, stats.total_bytes, stats.segment_count
|
||||
);
|
||||
|
||||
// ── 5. Delete an email (simulating: last reference removed) ─────────
|
||||
|
||||
// In production, before calling engine.delete(), you'd check:
|
||||
// SELECT COUNT(*) FROM email_refs WHERE content_hash = ? AND account_id != ?
|
||||
// If count == 0, it's safe to delete from blob.
|
||||
|
||||
let welcome_hash = mock_content_hash(b"Welcome to Bichon Mail!");
|
||||
engine.delete(&welcome_hash)?;
|
||||
println!("\nDeleted welcome email, still exists? {}", engine.exists(&welcome_hash)?);
|
||||
|
||||
// ── 6. Shutdown ────────────────────────────────────────────────────
|
||||
|
||||
engine.shutdown()?;
|
||||
println!("\nClean shutdown complete.");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// In production, use BLAKE3 or SHA-256.
|
||||
/// Here we use a trivial hash for demonstration.
|
||||
fn mock_content_hash(data: &[u8]) -> [u8; 32] {
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::hash::{Hash, Hasher};
|
||||
|
||||
let mut hasher = DefaultHasher::new();
|
||||
data.hash(&mut hasher);
|
||||
let h = hasher.finish();
|
||||
|
||||
let mut key = [0u8; 32];
|
||||
key[0..8].copy_from_slice(&h.to_le_bytes());
|
||||
// Mix in the length so different-sized content gets different hashes
|
||||
key[8..16].copy_from_slice(&(data.len() as u64).to_le_bytes());
|
||||
key
|
||||
}
|
||||
@@ -1,284 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex, RwLock};
|
||||
|
||||
use crate::bucket::{self, BucketFile, IndexRecord};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::file_pool::FilePool;
|
||||
use crate::meta::{AccountMeta, SegmentStats};
|
||||
use crate::segment::{self, SegmentReader, SegmentWriter};
|
||||
use crate::types::Codec;
|
||||
|
||||
// ── AccountHandle ──────────────────────────────────────────────────────────
|
||||
|
||||
pub struct AccountHandle {
|
||||
id: String,
|
||||
dir: PathBuf,
|
||||
inner: RwLock<AccountInner>,
|
||||
pub(crate) write_mutex: Mutex<()>,
|
||||
file_pool: FilePool,
|
||||
}
|
||||
|
||||
impl AccountHandle {
|
||||
pub fn id(&self) -> &str {
|
||||
&self.id
|
||||
}
|
||||
|
||||
pub fn dir(&self) -> &Path {
|
||||
&self.dir
|
||||
}
|
||||
|
||||
/// Open an existing account.
|
||||
pub fn open(store_root: &Path, account_id: &str) -> Result<Arc<Self>> {
|
||||
let dir = store_root.join("accounts").join(account_id);
|
||||
if !dir.exists() {
|
||||
return Err(Error::AccountNotFound(account_id.to_string()));
|
||||
}
|
||||
let inner = AccountInner::open(&dir)?;
|
||||
Ok(Arc::new(Self {
|
||||
id: account_id.to_string(),
|
||||
dir,
|
||||
inner: RwLock::new(inner),
|
||||
write_mutex: Mutex::new(()),
|
||||
file_pool: FilePool::new(8),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Create a new account.
|
||||
pub fn create(store_root: &Path, account_id: &str) -> Result<Arc<Self>> {
|
||||
let dir = store_root.join("accounts").join(account_id);
|
||||
if dir.exists() {
|
||||
return Err(Error::AccountAlreadyExists(account_id.to_string()));
|
||||
}
|
||||
let inner = AccountInner::create(&dir, account_id)?;
|
||||
Ok(Arc::new(Self {
|
||||
id: account_id.to_string(),
|
||||
dir,
|
||||
inner: RwLock::new(inner),
|
||||
write_mutex: Mutex::new(()),
|
||||
file_pool: FilePool::new(8),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Lock the inner state for reading.
|
||||
pub fn read(&self) -> std::sync::RwLockReadGuard<'_, AccountInner> {
|
||||
self.inner.read().unwrap()
|
||||
}
|
||||
|
||||
/// Lock the inner state for writing.
|
||||
pub fn write(&self) -> std::sync::RwLockWriteGuard<'_, AccountInner> {
|
||||
self.inner.write().unwrap()
|
||||
}
|
||||
|
||||
/// Get a cached file handle for a segment.
|
||||
pub fn get_segment_file(&self, seg_id: u32, path: &Path) -> Result<Arc<Mutex<std::fs::File>>> {
|
||||
self.file_pool.get(seg_id, path)
|
||||
}
|
||||
|
||||
/// Invalidate cached file handles for a segment (after GC).
|
||||
pub fn invalidate_file_cache(&self, seg_id: u32) {
|
||||
self.file_pool.invalidate(seg_id);
|
||||
}
|
||||
}
|
||||
|
||||
// ── AccountInner ───────────────────────────────────────────────────────────
|
||||
|
||||
pub struct AccountInner {
|
||||
dir: PathBuf,
|
||||
meta: AccountMeta,
|
||||
active_writer: SegmentWriter,
|
||||
readers: HashMap<u32, SegmentReader>,
|
||||
}
|
||||
|
||||
impl AccountInner {
|
||||
fn open(dir: &Path) -> Result<Self> {
|
||||
let meta = AccountMeta::load(dir)?;
|
||||
|
||||
let seg_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(meta.active_segment_id));
|
||||
let active_writer = if seg_path.exists() {
|
||||
SegmentWriter::open_append(seg_path, meta.active_segment_id)?
|
||||
} else {
|
||||
fs::create_dir_all(dir.join("segments"))?;
|
||||
SegmentWriter::create(seg_path, meta.active_segment_id)?
|
||||
};
|
||||
|
||||
let mut readers = HashMap::new();
|
||||
for (&seg_id, stats) in &meta.segments {
|
||||
if stats.sealed {
|
||||
let seg_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(seg_id));
|
||||
if seg_path.exists() {
|
||||
readers.insert(seg_id, SegmentReader::open(seg_path, seg_id)?);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
dir: dir.to_path_buf(),
|
||||
meta,
|
||||
active_writer,
|
||||
readers,
|
||||
})
|
||||
}
|
||||
|
||||
fn create(dir: &Path, account_id: &str) -> Result<Self> {
|
||||
fs::create_dir_all(dir.join("segments"))?;
|
||||
BucketFile::ensure_dir(dir)?;
|
||||
|
||||
let meta = AccountMeta::new(account_id.to_string(), 1);
|
||||
|
||||
let seg_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(1));
|
||||
let active_writer = SegmentWriter::create(seg_path, 1)?;
|
||||
|
||||
meta.save(dir)?;
|
||||
|
||||
Ok(Self {
|
||||
dir: dir.to_path_buf(),
|
||||
meta,
|
||||
active_writer,
|
||||
readers: HashMap::new(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn meta(&self) -> &AccountMeta {
|
||||
&self.meta
|
||||
}
|
||||
|
||||
/// Mark the segment as indexed up to the given offset and persist meta.
|
||||
pub fn mark_indexed(&mut self, segment_id: u32, indexed_up_to_offset: u64) -> Result<()> {
|
||||
if let Some(stats) = self.meta.segments.get_mut(&segment_id) {
|
||||
if indexed_up_to_offset > stats.indexed_up_to_offset {
|
||||
stats.indexed_up_to_offset = indexed_up_to_offset;
|
||||
}
|
||||
}
|
||||
self.meta.save(&self.dir)
|
||||
}
|
||||
|
||||
/// Append an entry without fsync.
|
||||
pub fn append_entry(
|
||||
&mut self,
|
||||
key: [u8; 32],
|
||||
data: &[u8],
|
||||
flags: u8,
|
||||
codec: Codec,
|
||||
) -> Result<(u32, u64, u32)> {
|
||||
if self.active_writer.is_full() {
|
||||
self.seal_active()?;
|
||||
}
|
||||
|
||||
use crate::segment::Entry;
|
||||
let entry = if flags == 1 {
|
||||
Entry::tombstone(key)
|
||||
} else {
|
||||
Entry::new(key, data, flags, codec)
|
||||
};
|
||||
|
||||
let data_size = entry.data.len() as u32;
|
||||
let segment_id = self.active_writer.id();
|
||||
let offset = self.active_writer.append(&entry)?;
|
||||
|
||||
let stats = self
|
||||
.meta
|
||||
.segments
|
||||
.entry(segment_id)
|
||||
.or_insert_with(|| SegmentStats::new(segment_id));
|
||||
stats.total_bytes += data_size as u64;
|
||||
if flags == 1 {
|
||||
stats.deleted_bytes += entry.raw_size as u64;
|
||||
}
|
||||
stats.recompute_ratio();
|
||||
|
||||
Ok((segment_id, offset, data_size))
|
||||
}
|
||||
|
||||
/// Fsync the active segment and persist meta.
|
||||
pub fn flush_active(&mut self) -> Result<()> {
|
||||
self.active_writer.fsync()?;
|
||||
self.meta.save(&self.dir)
|
||||
}
|
||||
|
||||
/// Write an entry with fsync.
|
||||
pub fn write_entry(
|
||||
&mut self,
|
||||
key: [u8; 32],
|
||||
data: &[u8],
|
||||
flags: u8,
|
||||
codec: Codec,
|
||||
) -> Result<(u32, u64, u32)> {
|
||||
let result = self.append_entry(key, data, flags, codec)?;
|
||||
self.flush_active()?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn seal_active(&mut self) -> Result<()> {
|
||||
let old_id = self.active_writer.id();
|
||||
let old_stats = self
|
||||
.meta
|
||||
.segments
|
||||
.entry(old_id)
|
||||
.or_insert_with(|| SegmentStats::new(old_id));
|
||||
old_stats.sealed = true;
|
||||
|
||||
let seg_path = self
|
||||
.dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(old_id));
|
||||
self.readers
|
||||
.insert(old_id, SegmentReader::open(seg_path, old_id)?);
|
||||
|
||||
let new_id = old_id + 1;
|
||||
self.meta.active_segment_id = new_id;
|
||||
let new_path = self
|
||||
.dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(new_id));
|
||||
self.active_writer = SegmentWriter::create(new_path, new_id)?;
|
||||
self.meta.save(&self.dir)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the on-disk path for a segment.
|
||||
pub fn segment_path(&self, segment_id: u32) -> Result<PathBuf> {
|
||||
let filename = segment::segment_filename(segment_id);
|
||||
let path = self.dir.join("segments").join(&filename);
|
||||
if path.exists() {
|
||||
Ok(path)
|
||||
} else {
|
||||
Err(Error::SegmentNotFound(segment_id))
|
||||
}
|
||||
}
|
||||
|
||||
/// Append index record to the appropriate bucket file.
|
||||
pub fn append_index(&self, record: &IndexRecord) -> Result<()> {
|
||||
let bucket_id = bucket::bucket_id(&record.key);
|
||||
let bf = BucketFile::open(&self.dir, bucket_id);
|
||||
bf.append(record)
|
||||
}
|
||||
|
||||
/// Return list of sealed segment IDs.
|
||||
pub fn sealed_segments(&self) -> Vec<u32> {
|
||||
self.meta
|
||||
.segments
|
||||
.iter()
|
||||
.filter(|(_, s)| s.sealed)
|
||||
.map(|(id, _)| *id)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// All segment IDs (including active).
|
||||
pub fn all_segment_ids(&self) -> Vec<u32> {
|
||||
let mut ids: Vec<u32> = self.meta.segments.keys().copied().collect();
|
||||
if !ids.contains(&self.meta.active_segment_id) {
|
||||
ids.push(self.meta.active_segment_id);
|
||||
}
|
||||
ids.sort_unstable();
|
||||
ids
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,14 @@
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::path::Path;
|
||||
|
||||
use redb::{Database, ReadableTable, TableDefinition};
|
||||
use redb::ReadableDatabase;
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::types::{BUCKET_COUNT, INDEX_RECORD_SIZE};
|
||||
use crate::types::INDEX_RECORD_SIZE;
|
||||
|
||||
/// On-disk format: 52 bytes per record.
|
||||
// ── IndexRecord ──────────────────────────────────────────────────────────────
|
||||
|
||||
/// On-disk format: 52 bytes per record + 4 bytes CRC = 56 bytes total.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct IndexRecord {
|
||||
pub key: [u8; 32],
|
||||
@@ -37,293 +40,218 @@ impl IndexRecord {
|
||||
buf[36..44].copy_from_slice(&self.offset.to_le_bytes());
|
||||
buf[44..48].copy_from_slice(&self.data_size.to_le_bytes());
|
||||
buf[48] = self.flags;
|
||||
// bytes 49..52 are padding (keep zero)
|
||||
let crc = crate::checksum::crc32(&buf[..52]);
|
||||
buf[52..56].copy_from_slice(&crc.to_le_bytes());
|
||||
buf
|
||||
}
|
||||
|
||||
pub fn decode(buf: &[u8; INDEX_RECORD_SIZE]) -> Self {
|
||||
pub fn decode(buf: &[u8; INDEX_RECORD_SIZE]) -> crate::error::Result<Self> {
|
||||
let stored_crc = u32::from_le_bytes(buf[52..56].try_into().unwrap());
|
||||
let computed = crate::checksum::crc32(&buf[..52]);
|
||||
if stored_crc != computed {
|
||||
return Err(crate::error::Error::BucketIndexCorrupt {
|
||||
path: std::path::PathBuf::new(),
|
||||
reason: format!(
|
||||
"CRC mismatch: stored=0x{:08X} computed=0x{:08X}",
|
||||
stored_crc, computed
|
||||
),
|
||||
});
|
||||
}
|
||||
let mut key = [0u8; 32];
|
||||
key.copy_from_slice(&buf[0..32]);
|
||||
let segment_id = u32::from_le_bytes(buf[32..36].try_into().unwrap());
|
||||
let offset = u64::from_le_bytes(buf[36..44].try_into().unwrap());
|
||||
let data_size = u32::from_le_bytes(buf[44..48].try_into().unwrap());
|
||||
let flags = buf[48];
|
||||
Self {
|
||||
Ok(Self {
|
||||
key,
|
||||
segment_id,
|
||||
offset,
|
||||
data_size,
|
||||
flags,
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Represents a loaded and deduplicated bucket in memory.
|
||||
pub struct BucketIndex {
|
||||
pub bucket_id: u16,
|
||||
/// Records sorted by key, deduplicated (one record per key, latest wins).
|
||||
pub records: Vec<IndexRecord>,
|
||||
// ── redb Value impl for fixed-size record bytes ──────────────────────────────
|
||||
|
||||
/// Newtype wrapper so we can implement `redb::Value` for `[u8; INDEX_RECORD_SIZE]`.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct RecordBytes([u8; INDEX_RECORD_SIZE]);
|
||||
|
||||
impl redb::Value for RecordBytes {
|
||||
type SelfType<'a> = RecordBytes;
|
||||
type AsBytes<'a> = [u8; INDEX_RECORD_SIZE];
|
||||
|
||||
fn fixed_width() -> Option<usize> {
|
||||
Some(INDEX_RECORD_SIZE)
|
||||
}
|
||||
|
||||
fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>
|
||||
where
|
||||
Self: 'a
|
||||
{
|
||||
let mut arr = [0u8; INDEX_RECORD_SIZE];
|
||||
arr.copy_from_slice(data);
|
||||
RecordBytes(arr)
|
||||
}
|
||||
|
||||
fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> {
|
||||
value.0
|
||||
}
|
||||
|
||||
fn type_name() -> redb::TypeName {
|
||||
redb::TypeName::new("IndexRecord")
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketIndex {
|
||||
/// Build from raw records: sort by key, dedup keeping the one with max offset.
|
||||
pub fn from_records(mut records: Vec<IndexRecord>, bucket_id: u16) -> Self {
|
||||
records.sort_by_key(|a| a.key);
|
||||
// Dedup: keep last (max offset) for each key
|
||||
let mut deduped = Vec::with_capacity(records.len());
|
||||
let mut i = 0;
|
||||
while i < records.len() {
|
||||
let mut best = i;
|
||||
let mut j = i + 1;
|
||||
while j < records.len() && records[j].key == records[i].key {
|
||||
if records[j].offset > records[best].offset {
|
||||
best = j;
|
||||
}
|
||||
j += 1;
|
||||
// ── IndexStore ───────────────────────────────────────────────────────────────
|
||||
|
||||
const INDEX_TABLE: TableDefinition<[u8; 32], RecordBytes> = TableDefinition::new("blob_index");
|
||||
|
||||
/// Zero-heap key-value index backed by redb.
|
||||
///
|
||||
/// B-tree + mmap + ACID. No manual compact / sort / dedup / per-bucket mmap
|
||||
/// management. Startup is O(1) — redb reads only its root page.
|
||||
pub struct IndexStore {
|
||||
db: Database,
|
||||
}
|
||||
|
||||
impl IndexStore {
|
||||
/// Open (or create) the index database at `dir/index.redb`.
|
||||
pub fn open(dir: &Path) -> Result<Self> {
|
||||
let path = dir.join("index.redb");
|
||||
let db = Database::create(&path)
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("failed to create index: {}", e)))?;
|
||||
// Ensure the table exists so reads on a fresh database don't fail.
|
||||
{
|
||||
let txn = db
|
||||
.begin_write()
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("init write txn: {}", e)))?;
|
||||
txn.open_table(INDEX_TABLE)
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("init table: {}", e)))?;
|
||||
txn.commit()
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("init commit: {}", e)))?;
|
||||
}
|
||||
Ok(Self { db })
|
||||
}
|
||||
|
||||
/// Look up a key. Returns the latest IndexRecord, or None if absent/tombstone.
|
||||
pub fn get(&self, key: &[u8; 32]) -> Result<Option<IndexRecord>> {
|
||||
let txn = self
|
||||
.db
|
||||
.begin_read()
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("read txn: {}", e)))?;
|
||||
let table = txn
|
||||
.open_table(INDEX_TABLE)
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("open table: {}", e)))?;
|
||||
|
||||
match table
|
||||
.get(key)
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("get: {}", e)))?
|
||||
{
|
||||
Some(guard) => {
|
||||
let record = IndexRecord::decode(&guard.value().0)?;
|
||||
Ok(if record.is_tombstone() { None } else { Some(record) })
|
||||
}
|
||||
deduped.push(records[best].clone());
|
||||
i = j;
|
||||
}
|
||||
Self {
|
||||
bucket_id,
|
||||
records: deduped,
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Binary search for a key. Returns the record if found.
|
||||
pub fn find(&self, key: &[u8; 32]) -> Option<&IndexRecord> {
|
||||
match self.records.binary_search_by(|r| r.key.cmp(key)) {
|
||||
Ok(idx) => Some(&self.records[idx]),
|
||||
Err(_) => None,
|
||||
/// Check whether a key exists (non-tombstone) in the store.
|
||||
pub fn exists(&self, key: &[u8; 32]) -> Result<bool> {
|
||||
self.get(key).map(|r| r.is_some())
|
||||
}
|
||||
|
||||
/// Insert or update a record for a key. Committed in a single write txn.
|
||||
pub fn insert(&self, record: &IndexRecord) -> Result<()> {
|
||||
let txn = self
|
||||
.db
|
||||
.begin_write()
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("write txn: {}", e)))?;
|
||||
{
|
||||
let mut table = txn
|
||||
.open_table(INDEX_TABLE)
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("open table: {}", e)))?;
|
||||
table
|
||||
.insert(&record.key, RecordBytes(record.encode()))
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("insert: {}", e)))?;
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a new record and maintain sorted order.
|
||||
pub fn insert(&mut self, record: IndexRecord) {
|
||||
match self.records.binary_search_by(|r| r.key.cmp(&record.key)) {
|
||||
Ok(idx) => {
|
||||
// Replace if newer (larger offset)
|
||||
if record.offset > self.records[idx].offset {
|
||||
self.records[idx] = record;
|
||||
}
|
||||
}
|
||||
Err(idx) => {
|
||||
self.records.insert(idx, record);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.records.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.records.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// Manages a bucket index file on disk.
|
||||
pub struct BucketFile {
|
||||
path: PathBuf,
|
||||
bucket_id: u16,
|
||||
}
|
||||
|
||||
impl BucketFile {
|
||||
pub fn path_for(account_dir: &Path, bucket_id: u16) -> PathBuf {
|
||||
account_dir.join("buckets").join(format!("{:02x}.idx", bucket_id))
|
||||
}
|
||||
|
||||
pub fn open(account_dir: &Path, bucket_id: u16) -> Self {
|
||||
Self {
|
||||
path: Self::path_for(account_dir, bucket_id),
|
||||
bucket_id,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub fn bucket_id(&self) -> u16 {
|
||||
self.bucket_id
|
||||
}
|
||||
|
||||
/// Ensure the buckets directory exists.
|
||||
pub fn ensure_dir(account_dir: &Path) -> Result<()> {
|
||||
let dir = account_dir.join("buckets");
|
||||
std::fs::create_dir_all(&dir)?;
|
||||
txn.commit()
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("commit: {}", e)))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append a single record to the bucket file.
|
||||
pub fn append(&self, record: &IndexRecord) -> Result<()> {
|
||||
let mut file = OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&self.path)?;
|
||||
file.write_all(&record.encode())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append multiple records at once.
|
||||
pub fn append_batch(&self, records: &[IndexRecord]) -> Result<()> {
|
||||
/// Batch insert multiple records in a single write transaction.
|
||||
pub fn insert_batch(&self, records: &[IndexRecord]) -> Result<()> {
|
||||
if records.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut file = OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&self.path)?;
|
||||
for r in records {
|
||||
file.write_all(&r.encode())?;
|
||||
let txn = self
|
||||
.db
|
||||
.begin_write()
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("write txn: {}", e)))?;
|
||||
{
|
||||
let mut table = txn
|
||||
.open_table(INDEX_TABLE)
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("open table: {}", e)))?;
|
||||
for record in records {
|
||||
table
|
||||
.insert(&record.key, RecordBytes(record.encode()))
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("insert: {}", e)))?;
|
||||
}
|
||||
}
|
||||
txn.commit()
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("commit: {}", e)))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Load all records from the bucket file.
|
||||
/// If the file size is not a multiple of INDEX_RECORD_SIZE (partial write),
|
||||
/// the trailing bytes are silently ignored.
|
||||
pub fn load_all(&self) -> Result<Vec<IndexRecord>> {
|
||||
if !self.path.exists() {
|
||||
return Ok(Vec::new());
|
||||
/// Remove keys from the index in a single write transaction.
|
||||
pub fn delete_batch(&self, keys: &[[u8; 32]]) -> Result<()> {
|
||||
if keys.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let data = std::fs::read(&self.path)?;
|
||||
let remainder = data.len() % INDEX_RECORD_SIZE;
|
||||
let count = data.len() / INDEX_RECORD_SIZE;
|
||||
let mut records = Vec::with_capacity(count);
|
||||
for i in 0..count {
|
||||
let start = i * INDEX_RECORD_SIZE;
|
||||
let end = start + INDEX_RECORD_SIZE;
|
||||
let buf: &[u8; INDEX_RECORD_SIZE] = data[start..end]
|
||||
.try_into()
|
||||
.map_err(|_| crate::error::Error::BucketIndexCorrupt {
|
||||
path: self.path.clone(),
|
||||
reason: format!("unexpected file size {}, not a multiple of {}", data.len(), INDEX_RECORD_SIZE),
|
||||
})?;
|
||||
records.push(IndexRecord::decode(buf));
|
||||
}
|
||||
if remainder > 0 {
|
||||
tracing::warn!(
|
||||
"Bucket file {:?} has {} trailing bytes (expected multiple of {}), ignoring",
|
||||
self.path, remainder, INDEX_RECORD_SIZE
|
||||
);
|
||||
}
|
||||
Ok(records)
|
||||
}
|
||||
|
||||
/// Load all records, sort, and deduplicate into a BucketIndex.
|
||||
pub fn load_index(&self) -> Result<BucketIndex> {
|
||||
let records = self.load_all()?;
|
||||
Ok(BucketIndex::from_records(records, self.bucket_id))
|
||||
}
|
||||
|
||||
/// Rewrite the bucket file with a sorted, deduplicated set of records.
|
||||
/// Uses atomic temp+rename to be safe on NFS.
|
||||
pub fn rewrite(&self, records: &[IndexRecord]) -> Result<()> {
|
||||
let mut buf = Vec::with_capacity(records.len() * INDEX_RECORD_SIZE);
|
||||
for r in records {
|
||||
buf.extend_from_slice(&r.encode());
|
||||
}
|
||||
crate::fs::create_atomic(&self.path, &buf)
|
||||
}
|
||||
|
||||
/// Delete the bucket file.
|
||||
pub fn delete(&self) -> Result<()> {
|
||||
if self.path.exists() {
|
||||
std::fs::remove_file(&self.path)?;
|
||||
let txn = self
|
||||
.db
|
||||
.begin_write()
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("write txn: {}", e)))?;
|
||||
{
|
||||
let mut table = txn
|
||||
.open_table(INDEX_TABLE)
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("open table: {}", e)))?;
|
||||
for key in keys {
|
||||
table
|
||||
.remove(key)
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("remove: {}", e)))?;
|
||||
}
|
||||
}
|
||||
txn.commit()
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("commit: {}", e)))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute bucket_id from a key's first 2 bytes.
|
||||
pub fn bucket_id(key: &[u8; 32]) -> u16 {
|
||||
u16::from_be_bytes([key[0], key[1]]) % BUCKET_COUNT
|
||||
}
|
||||
/// Total number of live (non-tombstone) keys.
|
||||
pub fn total_keys(&self) -> Result<usize> {
|
||||
let txn = self
|
||||
.db
|
||||
.begin_read()
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("read txn: {}", e)))?;
|
||||
let table = txn
|
||||
.open_table(INDEX_TABLE)
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("open table: {}", e)))?;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_index_record_encode_decode() {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&[1, 2, 3, 4]);
|
||||
let rec = IndexRecord::new(key, 5, 12345, 500, 0);
|
||||
let encoded = rec.encode();
|
||||
let decoded = IndexRecord::decode(&encoded);
|
||||
assert_eq!(rec, decoded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_id_deterministic() {
|
||||
let mut key = [0u8; 32];
|
||||
key[0] = 0x00;
|
||||
key[1] = 0x0F;
|
||||
assert_eq!(bucket_id(&key), 15);
|
||||
key[0] = 0x00;
|
||||
key[1] = 0x10;
|
||||
assert_eq!(bucket_id(&key), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_append_and_load() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let bucket = BucketFile::open(dir.path(), 0);
|
||||
BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
|
||||
let r1 = IndexRecord::new([1u8; 32], 1, 100, 50, 0);
|
||||
let r2 = IndexRecord::new([2u8; 32], 1, 200, 60, 0);
|
||||
|
||||
bucket.append(&r1).unwrap();
|
||||
bucket.append(&r2).unwrap();
|
||||
|
||||
let loaded = bucket.load_all().unwrap();
|
||||
assert_eq!(loaded.len(), 2);
|
||||
assert_eq!(loaded[0].key, [1u8; 32]);
|
||||
assert_eq!(loaded[1].key, [2u8; 32]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_index_dedup() {
|
||||
let recs = vec![
|
||||
IndexRecord::new([1u8; 32], 1, 100, 50, 0),
|
||||
IndexRecord::new([1u8; 32], 2, 200, 50, 0), // newer offset wins
|
||||
IndexRecord::new([2u8; 32], 1, 300, 60, 0),
|
||||
];
|
||||
let idx = BucketIndex::from_records(recs, 0);
|
||||
assert_eq!(idx.len(), 2);
|
||||
let found = idx.find(&[1u8; 32]).unwrap();
|
||||
assert_eq!(found.segment_id, 2);
|
||||
assert_eq!(found.offset, 200);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_index_find_missing() {
|
||||
let recs = vec![IndexRecord::new([1u8; 32], 1, 100, 50, 0)];
|
||||
let idx = BucketIndex::from_records(recs, 0);
|
||||
assert!(idx.find(&[99u8; 32]).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_rewrite() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let bucket = BucketFile::open(dir.path(), 0);
|
||||
BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
|
||||
let r1 = IndexRecord::new([3u8; 32], 1, 300, 70, 0);
|
||||
let r2 = IndexRecord::new([1u8; 32], 1, 100, 50, 0);
|
||||
bucket.append(&r1).unwrap();
|
||||
bucket.append(&r2).unwrap();
|
||||
|
||||
// Rewrite sorted
|
||||
let sorted = vec![r2.clone(), r1.clone()];
|
||||
bucket.rewrite(&sorted).unwrap();
|
||||
|
||||
let loaded = bucket.load_all().unwrap();
|
||||
assert_eq!(loaded.len(), 2);
|
||||
assert_eq!(loaded[0].key, [1u8; 32]);
|
||||
assert_eq!(loaded[1].key, [3u8; 32]);
|
||||
let mut count = 0usize;
|
||||
let iter = table
|
||||
.iter()
|
||||
.map_err(|e| crate::error::Error::IndexDb(format!("iter: {}", e)))?;
|
||||
for item in iter {
|
||||
let (_, guard) =
|
||||
item.map_err(|e| crate::error::Error::IndexDb(format!("iter next: {}", e)))?;
|
||||
let record = IndexRecord::decode(&guard.value().0)?;
|
||||
if !record.is_tombstone() {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
Ok(count)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,219 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use crate::bucket::{BucketFile, BucketIndex, IndexRecord};
|
||||
use crate::error::Result;
|
||||
|
||||
type CacheKey = (String, u16);
|
||||
|
||||
/// Thread-safe LRU bucket cache with single-lock interior.
|
||||
/// Eliminates the TOCTOU race in the old two-Mutex design.
|
||||
pub struct BucketCache {
|
||||
inner: Mutex<CacheInner>,
|
||||
}
|
||||
|
||||
struct CacheInner {
|
||||
max_entries: usize,
|
||||
entries: Vec<CacheEntry>,
|
||||
index: HashMap<CacheKey, usize>,
|
||||
}
|
||||
|
||||
struct CacheEntry {
|
||||
key: CacheKey,
|
||||
index: BucketIndex,
|
||||
}
|
||||
|
||||
impl BucketCache {
|
||||
pub fn new(max_entries: usize) -> Self {
|
||||
Self {
|
||||
inner: Mutex::new(CacheInner {
|
||||
max_entries: max_entries.max(1),
|
||||
entries: Vec::new(),
|
||||
index: HashMap::new(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get or load a bucket index. Eliminates TOCTOU via double-checked locking.
|
||||
pub fn get_or_load(
|
||||
&self,
|
||||
account: &str,
|
||||
bucket_id: u16,
|
||||
account_dir: &Path,
|
||||
) -> Result<Vec<IndexRecord>> {
|
||||
let key: CacheKey = (account.to_string(), bucket_id);
|
||||
|
||||
// Check cache
|
||||
{
|
||||
let inner = self.inner.lock().unwrap();
|
||||
if let Some(&pos) = inner.index.get(&key) {
|
||||
return Ok(inner.entries[pos].index.records.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Load from disk
|
||||
let bucket_file = BucketFile::open(account_dir, bucket_id);
|
||||
let bucket_index = bucket_file.load_index()?;
|
||||
let records = bucket_index.records.clone();
|
||||
|
||||
// Insert with double-check (another thread might have beaten us)
|
||||
{
|
||||
let mut inner = self.inner.lock().unwrap();
|
||||
if let Some(&pos) = inner.index.get(&key) {
|
||||
return Ok(inner.entries[pos].index.records.clone());
|
||||
}
|
||||
// Evict if full
|
||||
if inner.entries.len() >= inner.max_entries {
|
||||
if let Some(evicted) = inner.entries.pop() {
|
||||
inner.index.remove(&evicted.key);
|
||||
}
|
||||
}
|
||||
// Insert at front
|
||||
inner.entries.insert(0, CacheEntry {
|
||||
key: key.clone(),
|
||||
index: bucket_index,
|
||||
});
|
||||
// Rebuild index
|
||||
inner.index.clear();
|
||||
for i in 0..inner.entries.len() {
|
||||
let key = inner.entries[i].key.clone();
|
||||
inner.index.insert(key, i);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(records)
|
||||
}
|
||||
|
||||
/// Insert or update a single record in a cached bucket.
|
||||
pub fn update_record(&self, account: &str, bucket_id: u16, record: IndexRecord) {
|
||||
let key: CacheKey = (account.to_string(), bucket_id);
|
||||
let mut inner = self.inner.lock().unwrap();
|
||||
|
||||
if let Some(&pos) = inner.index.get(&key) {
|
||||
inner.entries[pos].index.insert(record);
|
||||
// Move to front
|
||||
let entry = inner.entries.remove(pos);
|
||||
inner.entries.insert(0, entry);
|
||||
// Rebuild index
|
||||
inner.index.clear();
|
||||
for i in 0..inner.entries.len() {
|
||||
let entry_key = inner.entries[i].key.clone();
|
||||
inner.index.insert(entry_key, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Invalidate a cached bucket (after GC rewrites bucket files).
|
||||
pub fn invalidate(&self, account: &str, bucket_id: u16) {
|
||||
let key: CacheKey = (account.to_string(), bucket_id);
|
||||
let mut inner = self.inner.lock().unwrap();
|
||||
|
||||
if let Some(&pos) = inner.index.get(&key) {
|
||||
inner.entries.remove(pos);
|
||||
inner.index.clear();
|
||||
for i in 0..inner.entries.len() {
|
||||
let entry_key = inner.entries[i].key.clone();
|
||||
inner.index.insert(entry_key, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.inner.lock().unwrap().entries.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.inner.lock().unwrap().entries.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::IndexRecord;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_cache_miss_loads_from_disk() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
let bf = BucketFile::open(dir.path(), 0);
|
||||
bf.append(&IndexRecord::new([1u8; 32], 1, 100, 50, 0))
|
||||
.unwrap();
|
||||
|
||||
let cache = BucketCache::new(10);
|
||||
let records = cache
|
||||
.get_or_load("test", 0, dir.path())
|
||||
.unwrap();
|
||||
assert_eq!(records.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_hit() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
let bf = BucketFile::open(dir.path(), 0);
|
||||
bf.append(&IndexRecord::new([2u8; 32], 1, 200, 60, 0))
|
||||
.unwrap();
|
||||
|
||||
let cache = BucketCache::new(10);
|
||||
let _ = cache.get_or_load("test", 0, dir.path()).unwrap();
|
||||
let records = cache
|
||||
.get_or_load("test", 0, dir.path())
|
||||
.unwrap();
|
||||
assert_eq!(records.len(), 1);
|
||||
assert_eq!(cache.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_eviction() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
let cache = BucketCache::new(2);
|
||||
|
||||
for b in 0..4 {
|
||||
let bf = BucketFile::open(dir.path(), b);
|
||||
bf.append(&IndexRecord::new([b as u8; 32], 1, 100, 50, 0))
|
||||
.unwrap();
|
||||
let _ = cache.get_or_load("test", b, dir.path()).unwrap();
|
||||
}
|
||||
|
||||
assert!(cache.len() <= 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_concurrent_get_or_load_no_deadlock() {
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
let bf = BucketFile::open(dir.path(), 0);
|
||||
for i in 0..10u8 {
|
||||
bf.append(&IndexRecord::new([i; 32], 1, i as u64 * 100, 50, 0))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let cache = Arc::new(BucketCache::new(10));
|
||||
let dir_path = dir.path().to_path_buf();
|
||||
|
||||
let mut handles = vec![];
|
||||
for _ in 0..4 {
|
||||
let cache = cache.clone();
|
||||
let dir_path = dir_path.clone();
|
||||
handles.push(thread::spawn(move || {
|
||||
for _ in 0..100 {
|
||||
let records = cache
|
||||
.get_or_load("test", 0, &dir_path)
|
||||
.unwrap();
|
||||
assert_eq!(records.len(), 10);
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,28 +1,61 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::fs::File;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex, RwLock};
|
||||
use std::thread::{self, JoinHandle};
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::account::AccountHandle;
|
||||
use crate::bucket::{self, IndexRecord};
|
||||
use crate::cache::BucketCache;
|
||||
use fs2::FileExt;
|
||||
|
||||
use crate::bucket::{IndexRecord, IndexStore};
|
||||
use crate::compress;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::file_pool::FilePool;
|
||||
use crate::gc::{self, GcStats};
|
||||
use crate::meta::GlobalMeta;
|
||||
use crate::segment::SegmentReader;
|
||||
use crate::meta::{GlobalMeta, SegmentStats};
|
||||
use crate::segment::{self, SegmentReader, SegmentWriter};
|
||||
use crate::types::{Codec, Config, ENTRY_HEADER_SIZE};
|
||||
|
||||
/// Global content-addressable blob store.
|
||||
///
|
||||
/// All data is keyed by a 32-byte content hash. Identical content is stored
|
||||
/// only once. The caller is responsible for tracking which entities reference
|
||||
/// which content hashes — this crate is a pure content-addressable KV engine.
|
||||
pub struct Engine {
|
||||
root: PathBuf,
|
||||
shared: Arc<EngineShared>,
|
||||
flush_handle: Mutex<Option<FlushHandle>>,
|
||||
gc_handle: Mutex<Option<GcHandle>>,
|
||||
}
|
||||
|
||||
struct EngineShared {
|
||||
config: Config,
|
||||
cache: BucketCache,
|
||||
accounts: RwLock<HashMap<String, Arc<AccountHandle>>>,
|
||||
inner: RwLock<EngineInner>,
|
||||
index_store: IndexStore,
|
||||
write_mutex: Mutex<()>,
|
||||
file_pool: FilePool,
|
||||
#[allow(dead_code)]
|
||||
lock_file: File,
|
||||
}
|
||||
|
||||
struct FlushHandle {
|
||||
handle: JoinHandle<()>,
|
||||
stop: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
struct GcHandle {
|
||||
handle: JoinHandle<()>,
|
||||
stop: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
struct EngineInner {
|
||||
root: PathBuf,
|
||||
meta: GlobalMeta,
|
||||
active_writer: SegmentWriter,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AccountStats {
|
||||
pub account_id: String,
|
||||
pub struct Stats {
|
||||
pub total_keys: u64,
|
||||
pub total_bytes: u64,
|
||||
pub deleted_bytes: u64,
|
||||
@@ -32,255 +65,346 @@ pub struct AccountStats {
|
||||
impl Engine {
|
||||
pub fn open(path: &Path, config: Config) -> Result<Self> {
|
||||
config.validate()?;
|
||||
|
||||
fs::create_dir_all(path)?;
|
||||
fs::create_dir_all(path.join("accounts"))?;
|
||||
fs::create_dir_all(path.join("segments"))?;
|
||||
|
||||
let mut global = GlobalMeta::load(path)?;
|
||||
global.save(path)?;
|
||||
// Acquire an exclusive file lock so that no two processes can open
|
||||
// the same database directory concurrently.
|
||||
let lock_path = path.join("LOCK");
|
||||
let lock_file = File::create(&lock_path)?;
|
||||
lock_file.try_lock_exclusive().map_err(|_| Error::AlreadyOpen {
|
||||
path: path.display().to_string(),
|
||||
})?;
|
||||
|
||||
let cache = BucketCache::new(config.lru_bucket_count);
|
||||
let (index_store, index_rebuilt) = match IndexStore::open(path) {
|
||||
Ok(s) => (s, false),
|
||||
Err(e) => {
|
||||
tracing::warn!("Index database unreadable, rebuilding from segments: {}", e);
|
||||
let index_path = path.join("index.redb");
|
||||
let _ = std::fs::remove_file(&index_path);
|
||||
(IndexStore::open(path)?, true)
|
||||
}
|
||||
};
|
||||
|
||||
let accounts_dir = path.join("accounts");
|
||||
let mut accounts = HashMap::new();
|
||||
let mut meta = GlobalMeta::load(path)?;
|
||||
|
||||
if accounts_dir.exists() {
|
||||
for entry in fs::read_dir(&accounts_dir)? {
|
||||
// Discover existing segments on disk
|
||||
let seg_dir = path.join("segments");
|
||||
let mut disk_segments: Vec<u32> = Vec::new();
|
||||
if seg_dir.exists() {
|
||||
for entry in fs::read_dir(&seg_dir)? {
|
||||
let entry = entry?;
|
||||
if entry.file_type()?.is_dir() {
|
||||
let account_name = entry.file_name().to_string_lossy().into_owned();
|
||||
|
||||
let _ = crate::recovery::cleanup_temp_files(&entry.path());
|
||||
|
||||
match crate::recovery::recover_account(&entry.path()) {
|
||||
Ok(_meta) => {
|
||||
match AccountHandle::open(path, &account_name) {
|
||||
Ok(handle) => {
|
||||
accounts.insert(account_name, handle);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Failed to open account {}: {}",
|
||||
account_name,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Failed to recover account {}: {}",
|
||||
account_name,
|
||||
e
|
||||
);
|
||||
let name = entry.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
if name_str.ends_with(".seg") && !name_str.contains("temp_") {
|
||||
if let Some(id_str) = name_str.strip_suffix(".seg") {
|
||||
if let Ok(id) = id_str.parse::<u32>() {
|
||||
disk_segments.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
disk_segments.sort_unstable();
|
||||
|
||||
global.accounts = accounts.keys().cloned().collect();
|
||||
global.save(path)?;
|
||||
for &seg_id in &disk_segments {
|
||||
if !meta.segments.contains_key(&seg_id) {
|
||||
meta.segments.insert(seg_id, SegmentStats::new(seg_id));
|
||||
}
|
||||
}
|
||||
|
||||
let max_disk_id = disk_segments.last().copied().unwrap_or(0);
|
||||
if max_disk_id > meta.active_segment_id {
|
||||
meta.active_segment_id = max_disk_id;
|
||||
}
|
||||
if meta.active_segment_id == 0 {
|
||||
meta.active_segment_id = 1;
|
||||
}
|
||||
|
||||
crate::recovery::cleanup_temp_files(path)?;
|
||||
let recovered_records = if index_rebuilt {
|
||||
crate::recovery::rebuild_index(path, &mut meta)?
|
||||
} else {
|
||||
crate::recovery::recover(path, &mut meta)?
|
||||
};
|
||||
index_store.insert_batch(&recovered_records)?;
|
||||
|
||||
let seg_path = path
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(meta.active_segment_id));
|
||||
let active_writer = if seg_path.exists() {
|
||||
SegmentWriter::open_append(seg_path, meta.active_segment_id)?
|
||||
} else {
|
||||
SegmentWriter::create(seg_path, meta.active_segment_id)?
|
||||
};
|
||||
|
||||
meta.save(path)?;
|
||||
|
||||
let shared = Arc::new(EngineShared {
|
||||
config: config.clone(),
|
||||
inner: RwLock::new(EngineInner {
|
||||
root: path.to_path_buf(),
|
||||
meta,
|
||||
active_writer,
|
||||
}),
|
||||
index_store,
|
||||
write_mutex: Mutex::new(()),
|
||||
file_pool: FilePool::new(8),
|
||||
lock_file,
|
||||
});
|
||||
|
||||
let flush_handle = if config.flush_interval_secs > 0 {
|
||||
let shared2 = Arc::clone(&shared);
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let stop2 = Arc::clone(&stop);
|
||||
let interval = Duration::from_secs(config.flush_interval_secs);
|
||||
|
||||
let handle = thread::Builder::new()
|
||||
.name("blob-flush".into())
|
||||
.spawn(move || {
|
||||
while !stop2.load(Ordering::Acquire) {
|
||||
thread::park_timeout(interval);
|
||||
if stop2.load(Ordering::Acquire) {
|
||||
break;
|
||||
}
|
||||
let _lock = shared2.write_mutex.lock().unwrap();
|
||||
let mut inner = shared2.inner.write().unwrap();
|
||||
if let Err(e) = inner.flush_active() {
|
||||
tracing::error!("background flush failed: {}", e);
|
||||
}
|
||||
}
|
||||
})
|
||||
.expect("failed to spawn blob-flush thread");
|
||||
|
||||
Some(FlushHandle { handle, stop })
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let gc_handle = if config.gc_interval_secs > 0 {
|
||||
let shared3 = Arc::clone(&shared);
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let stop2 = Arc::clone(&stop);
|
||||
let interval = Duration::from_secs(config.gc_interval_secs);
|
||||
|
||||
let handle = thread::Builder::new()
|
||||
.name("blob-gc".into())
|
||||
.spawn(move || {
|
||||
while !stop2.load(Ordering::Acquire) {
|
||||
thread::park_timeout(interval);
|
||||
if stop2.load(Ordering::Acquire) {
|
||||
break;
|
||||
}
|
||||
// Seal the active segment first so that the data from
|
||||
// this GC cycle becomes eligible for compaction.
|
||||
// Without this, the active segment never seals until
|
||||
// it reaches SEGMENT_MAX_SIZE (1 GB), and GC would
|
||||
// have no candidates on small databases.
|
||||
if let Err(e) = shared3.ensure_sealed() {
|
||||
tracing::error!("background GC: seal failed: {}", e);
|
||||
}
|
||||
match shared3.gc_if_needed() {
|
||||
Ok(Some(stats)) => {
|
||||
tracing::info!(
|
||||
"GC compacted segment {}: {} → {} bytes (kept {}, skipped {})",
|
||||
stats.segment_id,
|
||||
stats.bytes_before,
|
||||
stats.bytes_after,
|
||||
stats.entries_kept,
|
||||
stats.entries_skipped,
|
||||
);
|
||||
}
|
||||
Ok(None) => {
|
||||
tracing::debug!("GC check: no segment exceeds deleted-ratio threshold");
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("background GC failed: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.expect("failed to spawn blob-gc thread");
|
||||
|
||||
Some(GcHandle { handle, stop })
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
root: path.to_path_buf(),
|
||||
config,
|
||||
cache,
|
||||
accounts: RwLock::new(accounts),
|
||||
shared,
|
||||
flush_handle: Mutex::new(flush_handle),
|
||||
gc_handle: Mutex::new(gc_handle),
|
||||
})
|
||||
}
|
||||
|
||||
// ── Account management ──────────────────────────────────────────────
|
||||
|
||||
pub fn create_account(&self, account_id: &str) -> Result<()> {
|
||||
let mut accounts = self.accounts.write().unwrap();
|
||||
if accounts.contains_key(account_id) {
|
||||
return Err(Error::AccountAlreadyExists(account_id.to_string()));
|
||||
}
|
||||
let handle = AccountHandle::create(&self.root, account_id)?;
|
||||
accounts.insert(account_id.to_string(), handle);
|
||||
|
||||
let mut global = GlobalMeta::load(&self.root)?;
|
||||
global.accounts = accounts.keys().cloned().collect();
|
||||
global.save(&self.root)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn delete_account(&self, account_id: &str) -> Result<()> {
|
||||
let mut accounts = self.accounts.write().unwrap();
|
||||
let handle = accounts
|
||||
.remove(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?;
|
||||
|
||||
let account_dir = handle.dir().to_path_buf();
|
||||
drop(handle);
|
||||
fs::remove_dir_all(&account_dir)?;
|
||||
|
||||
let mut global = GlobalMeta::load(&self.root)?;
|
||||
global.accounts = accounts.keys().cloned().collect();
|
||||
global.save(&self.root)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn list_accounts(&self) -> Vec<String> {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts.keys().cloned().collect()
|
||||
}
|
||||
|
||||
// ── Read / Write / Delete ───────────────────────────────────────────
|
||||
|
||||
pub fn write(
|
||||
&self,
|
||||
account_id: &str,
|
||||
key: [u8; 32],
|
||||
value: &[u8],
|
||||
codec: Codec,
|
||||
) -> Result<()> {
|
||||
pub fn put(&self, key: [u8; 32], value: &[u8], codec: Codec) -> Result<()> {
|
||||
if value.len() > crate::types::MAX_VALUE_SIZE {
|
||||
return Err(Error::ValueTooLarge { size: value.len() });
|
||||
}
|
||||
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
let _write_lock = self.shared.write_mutex.lock().unwrap();
|
||||
let mut inner = self.shared.inner.write().unwrap();
|
||||
|
||||
let _write_lock = handle.write_mutex.lock().unwrap();
|
||||
let mut inner = handle.write();
|
||||
|
||||
let (data, actual_codec) =
|
||||
compress::compress(value, codec, self.config.compress_threshold, self.config.compression_level);
|
||||
let (data, actual_codec) = compress::compress(
|
||||
value,
|
||||
codec,
|
||||
self.shared.config.compress_threshold,
|
||||
self.shared.config.compression_level,
|
||||
);
|
||||
|
||||
let original_len = value.len() as u32;
|
||||
let (segment_id, offset, data_size) =
|
||||
inner.write_entry(key, &data, 0, actual_codec)?;
|
||||
inner.append_entry(key, &data, original_len, 0, actual_codec)?;
|
||||
|
||||
let record = IndexRecord::new(key, segment_id, offset, data_size, 0);
|
||||
inner.append_index(&record)?;
|
||||
self.shared.index_store.insert(&record)?;
|
||||
|
||||
let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
inner.mark_indexed(segment_id, entry_end)?;
|
||||
|
||||
let bucket_id = bucket::bucket_id(&key);
|
||||
self.cache.update_record(account_id, bucket_id, record);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn read(&self, account_id: &str, key: &[u8; 32]) -> Result<Option<Vec<u8>>> {
|
||||
let bucket_id = bucket::bucket_id(key);
|
||||
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
pub fn get(&self, key: &[u8; 32]) -> Result<Option<Vec<u8>>> {
|
||||
let record = match self.shared.index_store.get(key)? {
|
||||
Some(r) => r,
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
let (record, seg_path): (IndexRecord, PathBuf) = {
|
||||
let inner = handle.read();
|
||||
let records = self
|
||||
.cache
|
||||
.get_or_load(account_id, bucket_id, handle.dir())?;
|
||||
match records.binary_search_by(|r| r.key.cmp(key)) {
|
||||
Ok(idx) => {
|
||||
let r = records[idx].clone();
|
||||
if r.is_tombstone() {
|
||||
return Ok(None);
|
||||
}
|
||||
let seg_path = inner.segment_path(r.segment_id)?;
|
||||
(r, seg_path)
|
||||
}
|
||||
Err(_) => return Ok(None),
|
||||
}
|
||||
};
|
||||
let inner = self.shared.inner.read().unwrap();
|
||||
let seg_path = inner.segment_path(record.segment_id)?;
|
||||
|
||||
if !seg_path.exists() {
|
||||
return Err(Error::SegmentNotFound(record.segment_id));
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(seg_path.clone(), record.segment_id)?;
|
||||
let file = handle.get_segment_file(record.segment_id, &seg_path)?;
|
||||
let file = self.shared.file_pool.get(record.segment_id, &seg_path)?;
|
||||
let (entry, _) = reader.read_entry_at_file(record.offset, &file)?;
|
||||
|
||||
let value = compress::decompress(&entry.data, entry.codec, entry.raw_size as usize)?;
|
||||
|
||||
Ok(Some(value))
|
||||
}
|
||||
|
||||
pub fn delete(&self, account_id: &str, key: &[u8; 32]) -> Result<()> {
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
pub fn delete(&self, key: &[u8; 32]) -> Result<()> {
|
||||
let _write_lock = self.shared.write_mutex.lock().unwrap();
|
||||
let mut inner = self.shared.inner.write().unwrap();
|
||||
|
||||
let _write_lock = handle.write_mutex.lock().unwrap();
|
||||
let mut inner = handle.write();
|
||||
// Look up the existing record so we can account deleted_bytes on the
|
||||
// segment that holds the original data — this is what drives GC.
|
||||
if let Some(rec) = self.shared.index_store.get(key)? {
|
||||
if !rec.is_tombstone() {
|
||||
if let Some(stats) = inner.meta.segments.get_mut(&rec.segment_id) {
|
||||
stats.deleted_bytes += rec.data_size as u64;
|
||||
stats.recompute_ratio();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let (segment_id, offset, data_size) =
|
||||
inner.write_entry(*key, &[], 1, Codec::None)?;
|
||||
inner.append_entry(*key, &[], 0, 1, Codec::None)?;
|
||||
|
||||
let record = IndexRecord::new(*key, segment_id, offset, data_size, 1);
|
||||
inner.append_index(&record)?;
|
||||
self.shared.index_store.insert(&record)?;
|
||||
|
||||
let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
inner.mark_indexed(segment_id, entry_end)?;
|
||||
|
||||
let bucket_id = bucket::bucket_id(key);
|
||||
self.cache.update_record(account_id, bucket_id, record);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn exists(&self, key: &[u8; 32]) -> Result<bool> {
|
||||
self.shared.index_store.exists(key)
|
||||
}
|
||||
|
||||
// ── Batch delete ─────────────────────────────────────────────────────
|
||||
|
||||
pub fn delete_batch(&self, keys: &[[u8; 32]]) -> Result<()> {
|
||||
if keys.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let _write_lock = self.shared.write_mutex.lock().unwrap();
|
||||
let mut inner = self.shared.inner.write().unwrap();
|
||||
|
||||
let mut records: Vec<IndexRecord> = Vec::with_capacity(keys.len());
|
||||
let mut ends: Vec<(u32, u64)> = Vec::with_capacity(keys.len());
|
||||
|
||||
for key in keys {
|
||||
// Track deleted_bytes for GC threshold on the original segment.
|
||||
if let Some(rec) = self.shared.index_store.get(key)? {
|
||||
if !rec.is_tombstone() {
|
||||
if let Some(stats) = inner.meta.segments.get_mut(&rec.segment_id) {
|
||||
stats.deleted_bytes += rec.data_size as u64;
|
||||
stats.recompute_ratio();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let (segment_id, offset, data_size) =
|
||||
inner.append_entry(*key, &[], 0, 1, Codec::None)?;
|
||||
|
||||
let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
records.push(IndexRecord::new(*key, segment_id, offset, data_size, 1));
|
||||
ends.push((segment_id, entry_end));
|
||||
}
|
||||
|
||||
inner.flush_active()?;
|
||||
|
||||
self.shared.index_store.insert_batch(&records)?;
|
||||
|
||||
for (segment_id, entry_end) in &ends {
|
||||
inner.mark_indexed(*segment_id, *entry_end)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── Batch write ─────────────────────────────────────────────────────
|
||||
|
||||
pub fn write_batch(&self, account_id: &str, entries: &[([u8; 32], Vec<u8>, Codec)]) -> Result<()> {
|
||||
pub fn put_batch(&self, entries: &[([u8; 32], Vec<u8>, Codec)]) -> Result<()> {
|
||||
if entries.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
let _write_lock = self.shared.write_mutex.lock().unwrap();
|
||||
let mut inner = self.shared.inner.write().unwrap();
|
||||
|
||||
let _write_lock = handle.write_mutex.lock().unwrap();
|
||||
let mut inner = handle.write();
|
||||
let mut records: Vec<IndexRecord> = Vec::with_capacity(entries.len());
|
||||
let mut ends: Vec<(u32, u64)> = Vec::with_capacity(entries.len());
|
||||
|
||||
let mut pending: Vec<(IndexRecord, u64)> = Vec::with_capacity(entries.len());
|
||||
for (key, value, codec) in entries {
|
||||
if value.len() > crate::types::MAX_VALUE_SIZE {
|
||||
return Err(Error::ValueTooLarge { size: value.len() });
|
||||
}
|
||||
let (data, actual_codec) =
|
||||
compress::compress(value, *codec, self.config.compress_threshold, self.config.compression_level);
|
||||
let (data, actual_codec) = compress::compress(
|
||||
value,
|
||||
*codec,
|
||||
self.shared.config.compress_threshold,
|
||||
self.shared.config.compression_level,
|
||||
);
|
||||
|
||||
let original_len = value.len() as u32;
|
||||
let (segment_id, offset, data_size) =
|
||||
inner.append_entry(*key, &data, 0, actual_codec)?;
|
||||
inner.append_entry(*key, &data, original_len, 0, actual_codec)?;
|
||||
|
||||
let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
let record = IndexRecord::new(*key, segment_id, offset, data_size, 0);
|
||||
pending.push((record, entry_end));
|
||||
records.push(IndexRecord::new(*key, segment_id, offset, data_size, 0));
|
||||
ends.push((segment_id, entry_end));
|
||||
}
|
||||
|
||||
inner.flush_active()?;
|
||||
|
||||
for (record, entry_end) in &pending {
|
||||
inner.append_index(record)?;
|
||||
inner.mark_indexed(record.segment_id, *entry_end)?;
|
||||
self.shared.index_store.insert_batch(&records)?;
|
||||
|
||||
let bucket_id = bucket::bucket_id(&record.key);
|
||||
self.cache.update_record(account_id, bucket_id, record.clone());
|
||||
// Deduplicate: keep only the max offset per segment.
|
||||
ends.sort_by(|a, b| a.0.cmp(&b.0).then(b.1.cmp(&a.1)));
|
||||
ends.dedup_by(|a, b| a.0 == b.0);
|
||||
for (segment_id, entry_end) in &ends {
|
||||
inner.mark_indexed(*segment_id, *entry_end)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -288,87 +412,42 @@ impl Engine {
|
||||
|
||||
// ── GC ──────────────────────────────────────────────────────────────
|
||||
|
||||
pub fn gc(&self, account_id: &str) -> Result<Option<GcStats>> {
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
// Hold write_mutex to prevent concurrent writes from racing
|
||||
// with GC's bucket rebuild phase.
|
||||
let _write_lock = handle.write_mutex.lock().unwrap();
|
||||
|
||||
let result = gc::gc_account(handle.dir(), self.config.gc_deleted_ratio)?;
|
||||
|
||||
// Invalidate FilePool for GC'd segments (they were rewritten via rename)
|
||||
if let Some(ref stats) = result {
|
||||
handle.invalidate_file_cache(stats.segment_id);
|
||||
}
|
||||
|
||||
for bid in 0..crate::types::BUCKET_COUNT {
|
||||
self.cache.invalidate(account_id, bid);
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
pub fn gc(&self) -> Result<Option<GcStats>> {
|
||||
self.shared.gc()
|
||||
}
|
||||
|
||||
pub fn compact_buckets(&self, account_id: &str) -> Result<()> {
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
// Hold write_mutex — compact rewrites all bucket files.
|
||||
let _write_lock = handle.write_mutex.lock().unwrap();
|
||||
|
||||
gc::compact_buckets(handle.dir())?;
|
||||
|
||||
for bid in 0..crate::types::BUCKET_COUNT {
|
||||
self.cache.invalidate(account_id, bid);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
/// Run GC only if some segment exceeds the configured deleted-ratio threshold.
|
||||
/// Returns `Ok(None)` immediately without acquiring the write lock when no
|
||||
/// segment qualifies.
|
||||
pub fn gc_if_needed(&self) -> Result<Option<GcStats>> {
|
||||
self.shared.gc_if_needed()
|
||||
}
|
||||
|
||||
// ── Stats / Shutdown ────────────────────────────────────────────────
|
||||
// ── Flush / Stats / Shutdown ────────────────────────────────────────
|
||||
|
||||
pub fn stats(&self, account_id: &str) -> Result<AccountStats> {
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
/// Fsync the active segment and save metadata without compacting buckets.
|
||||
/// Lightweight checkpoint suitable for periodic calls from the background
|
||||
/// flush thread or external schedulers.
|
||||
pub fn flush(&self) -> Result<()> {
|
||||
let _write_lock = self.shared.write_mutex.lock().unwrap();
|
||||
let mut inner = self.shared.inner.write().unwrap();
|
||||
inner.flush_active()
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> Result<Stats> {
|
||||
let inner = self.shared.inner.read().unwrap();
|
||||
let meta = &inner.meta;
|
||||
|
||||
let inner = handle.read();
|
||||
let meta = inner.meta();
|
||||
let mut total_bytes = 0u64;
|
||||
let mut deleted_bytes = 0u64;
|
||||
|
||||
for seg in meta.segments.values() {
|
||||
total_bytes += seg.total_bytes;
|
||||
deleted_bytes += seg.deleted_bytes;
|
||||
}
|
||||
|
||||
let mut total_keys = 0u64;
|
||||
for bid in 0..crate::types::BUCKET_COUNT {
|
||||
if let Ok(records) =
|
||||
self.cache
|
||||
.get_or_load(account_id, bid, handle.dir())
|
||||
{
|
||||
total_keys += records.iter().filter(|r| !r.is_tombstone()).count() as u64;
|
||||
}
|
||||
}
|
||||
let total_keys = self.shared.index_store.total_keys()? as u64;
|
||||
|
||||
Ok(AccountStats {
|
||||
account_id: account_id.to_string(),
|
||||
Ok(Stats {
|
||||
total_keys,
|
||||
total_bytes,
|
||||
deleted_bytes,
|
||||
@@ -376,14 +455,33 @@ impl Engine {
|
||||
})
|
||||
}
|
||||
|
||||
/// Seal the active segment, forcing it to become a GC candidate.
|
||||
#[doc(hidden)]
|
||||
pub fn seal_active_segment(&self) -> Result<u32> {
|
||||
let _write_lock = self.shared.write_mutex.lock().unwrap();
|
||||
let mut inner = self.shared.inner.write().unwrap();
|
||||
let id = inner.active_writer.id();
|
||||
inner.seal_active()?;
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
pub fn shutdown(&self) -> Result<()> {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
for (_, handle) in accounts.iter() {
|
||||
let mut inner = handle.write();
|
||||
inner.flush_active()?;
|
||||
// Stop background threads first
|
||||
if let Some(fh) = self.flush_handle.lock().unwrap().take() {
|
||||
fh.stop.store(true, Ordering::Release);
|
||||
fh.handle.thread().unpark();
|
||||
let _ = fh.handle.join();
|
||||
}
|
||||
let global = GlobalMeta::load(&self.root)?;
|
||||
global.save(&self.root)?;
|
||||
if let Some(gh) = self.gc_handle.lock().unwrap().take() {
|
||||
gh.stop.store(true, Ordering::Release);
|
||||
gh.handle.thread().unpark();
|
||||
let _ = gh.handle.join();
|
||||
}
|
||||
|
||||
let mut inner = self.shared.inner.write().unwrap();
|
||||
inner.flush_active()?;
|
||||
|
||||
inner.meta.save(&inner.root)?;
|
||||
tracing::info!("bichon-blob shut down cleanly");
|
||||
Ok(())
|
||||
}
|
||||
@@ -391,8 +489,209 @@ impl Engine {
|
||||
|
||||
impl Drop for Engine {
|
||||
fn drop(&mut self) {
|
||||
if let Err(e) = self.shutdown() {
|
||||
tracing::error!("bichon-blob shutdown error: {}", e);
|
||||
// Best-effort shutdown that is panic-safe: only signal threads to
|
||||
// stop — don't try to acquire write_mutex or inner.write(), which
|
||||
// would deadlock if we're unwinding from a panic that happened while
|
||||
// one of those locks was held.
|
||||
if let Some(fh) = self.flush_handle.lock().ok().and_then(|mut g| g.take()) {
|
||||
fh.stop.store(true, Ordering::Release);
|
||||
fh.handle.thread().unpark();
|
||||
let _ = fh.handle.join();
|
||||
}
|
||||
if let Some(gh) = self.gc_handle.lock().ok().and_then(|mut g| g.take()) {
|
||||
gh.stop.store(true, Ordering::Release);
|
||||
gh.handle.thread().unpark();
|
||||
let _ = gh.handle.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── EngineShared ────────────────────────────────────────────────────────────
|
||||
|
||||
impl EngineShared {
|
||||
/// Seal the active segment if its deleted-ratio exceeds the GC threshold,
|
||||
/// so that the upcoming GC pass can compact it. Called periodically by
|
||||
/// the background GC thread.
|
||||
fn ensure_sealed(&self) -> Result<()> {
|
||||
let mut inner = self.inner.write().unwrap();
|
||||
let active_id = inner.active_writer.id();
|
||||
if let Some(stats) = inner.meta.segments.get(&active_id) {
|
||||
if stats.deleted_ratio >= self.config.gc_deleted_ratio {
|
||||
inner.seal_active()?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn gc_if_needed(&self) -> Result<Option<GcStats>> {
|
||||
let inner = self.inner.read().unwrap();
|
||||
let needs_gc = inner
|
||||
.meta
|
||||
.segments
|
||||
.values()
|
||||
.any(|s| s.sealed && s.deleted_ratio >= self.config.gc_deleted_ratio);
|
||||
drop(inner);
|
||||
|
||||
if needs_gc {
|
||||
self.gc()
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
fn gc(&self) -> Result<Option<GcStats>> {
|
||||
// Phase 1: scan only the target segment, consult bucket index per entry.
|
||||
// Read-only with respect to Engine state — no write_mutex needed.
|
||||
let prep = {
|
||||
let inner = self.inner.read().unwrap();
|
||||
gc::gc_prepare(
|
||||
&inner.root,
|
||||
&inner.meta,
|
||||
self.config.gc_deleted_ratio,
|
||||
&self.index_store,
|
||||
)?
|
||||
};
|
||||
|
||||
let mut prep = match prep {
|
||||
Some(p) => p,
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
// Phase 2: rename temp file + update bucket index.
|
||||
let _write_lock = self.write_mutex.lock().unwrap();
|
||||
|
||||
let kept_records = std::mem::take(&mut prep.kept_records);
|
||||
let deleted_keys = std::mem::take(&mut prep.deleted_keys);
|
||||
let stats = gc::gc_finish(prep)?;
|
||||
self.file_pool.invalidate(stats.segment_id);
|
||||
|
||||
// Insert new index records with updated offsets for kept entries.
|
||||
if !kept_records.is_empty() {
|
||||
self.index_store.insert_batch(&kept_records)?;
|
||||
}
|
||||
|
||||
// Remove tombstone IndexRecords that pointed to entries in this
|
||||
// compacted segment — they are gone now and would accumulate forever.
|
||||
if !deleted_keys.is_empty() {
|
||||
self.index_store.delete_batch(&deleted_keys)?;
|
||||
}
|
||||
|
||||
{
|
||||
let mut inner = self.inner.write().unwrap();
|
||||
|
||||
if stats.bytes_after == 0 {
|
||||
// Segment was completely emptied — remove it from meta first,
|
||||
// then delete the file. If we crash between the two steps the
|
||||
// orphaned file is rediscovered on next open and retried.
|
||||
inner.meta.segments.remove(&stats.segment_id);
|
||||
inner.meta.save(&inner.root)?;
|
||||
drop(inner);
|
||||
|
||||
let seg_path = self.inner.read().unwrap()
|
||||
.root
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(stats.segment_id));
|
||||
let _ = fs::remove_file(&seg_path);
|
||||
} else {
|
||||
// Update segment stats: now smaller and clean.
|
||||
if let Some(seg_stats) = inner.meta.segments.get_mut(&stats.segment_id) {
|
||||
seg_stats.total_bytes = stats.bytes_after;
|
||||
seg_stats.deleted_bytes = 0;
|
||||
seg_stats.deleted_ratio = 0.0;
|
||||
seg_stats.indexed_up_to_offset = stats.bytes_after;
|
||||
}
|
||||
inner.meta.save(&inner.root)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Some(stats))
|
||||
}
|
||||
}
|
||||
|
||||
// ── EngineInner ───────────────────────────────────────────────────────────
|
||||
|
||||
impl EngineInner {
|
||||
fn append_entry(
|
||||
&mut self,
|
||||
key: [u8; 32],
|
||||
data: &[u8],
|
||||
raw_size: u32,
|
||||
flags: u8,
|
||||
codec: Codec,
|
||||
) -> Result<(u32, u64, u32)> {
|
||||
if self.active_writer.is_full() {
|
||||
self.seal_active()?;
|
||||
}
|
||||
|
||||
use crate::segment::Entry;
|
||||
let entry = if flags == 1 {
|
||||
Entry::tombstone(key)
|
||||
} else {
|
||||
Entry::new(key, data, raw_size, flags, codec)
|
||||
};
|
||||
|
||||
let data_size = entry.data.len() as u32;
|
||||
let segment_id = self.active_writer.id();
|
||||
let offset = self.active_writer.append(&entry)?;
|
||||
|
||||
let stats = self
|
||||
.meta
|
||||
.segments
|
||||
.entry(segment_id)
|
||||
.or_insert_with(|| SegmentStats::new(segment_id));
|
||||
stats.total_bytes += data_size as u64;
|
||||
if flags == 1 {
|
||||
stats.deleted_bytes += entry.raw_size as u64;
|
||||
}
|
||||
stats.recompute_ratio();
|
||||
|
||||
Ok((segment_id, offset, data_size))
|
||||
}
|
||||
|
||||
fn flush_active(&mut self) -> Result<()> {
|
||||
self.active_writer.fsync()?;
|
||||
self.meta.save(&self.root)
|
||||
}
|
||||
|
||||
fn mark_indexed(&mut self, segment_id: u32, offset: u64) -> Result<()> {
|
||||
if let Some(stats) = self.meta.segments.get_mut(&segment_id) {
|
||||
if offset > stats.indexed_up_to_offset {
|
||||
stats.indexed_up_to_offset = offset;
|
||||
}
|
||||
}
|
||||
self.meta.save(&self.root)
|
||||
}
|
||||
|
||||
fn seal_active(&mut self) -> Result<()> {
|
||||
let old_id = self.active_writer.id();
|
||||
let old_stats = self
|
||||
.meta
|
||||
.segments
|
||||
.entry(old_id)
|
||||
.or_insert_with(|| SegmentStats::new(old_id));
|
||||
old_stats.sealed = true;
|
||||
|
||||
let new_id = old_id + 1;
|
||||
self.meta.active_segment_id = new_id;
|
||||
let new_path = self
|
||||
.root
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(new_id));
|
||||
self.active_writer = SegmentWriter::create(new_path, new_id)?;
|
||||
self.meta.save(&self.root)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn segment_path(&self, segment_id: u32) -> Result<PathBuf> {
|
||||
let path = self
|
||||
.root
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(segment_id));
|
||||
if path.exists() {
|
||||
Ok(path)
|
||||
} else {
|
||||
Err(Error::SegmentNotFound(segment_id))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,12 +20,6 @@ pub enum Error {
|
||||
reason: String,
|
||||
},
|
||||
|
||||
#[error("Account not found: {0}")]
|
||||
AccountNotFound(String),
|
||||
|
||||
#[error("Account already exists: {0}")]
|
||||
AccountAlreadyExists(String),
|
||||
|
||||
#[error("Segment not found: {0}")]
|
||||
SegmentNotFound(u32),
|
||||
|
||||
@@ -56,4 +50,10 @@ pub enum Error {
|
||||
|
||||
#[error("Unsupported metadata version {version} in {path}")]
|
||||
UnsupportedMetaVersion { path: PathBuf, version: u32 },
|
||||
|
||||
#[error("Index database error: {0}")]
|
||||
IndexDb(String),
|
||||
|
||||
#[error("Database is already open by another process at {path}")]
|
||||
AlreadyOpen { path: String },
|
||||
}
|
||||
|
||||
@@ -7,10 +7,10 @@ use crate::error::Result;
|
||||
use crate::fs as fs_util;
|
||||
|
||||
/// Simple LRU pool of open file handles, keyed by segment_id.
|
||||
/// Uses Arc<Mutex<File>> to allow safe concurrent reads from the same segment.
|
||||
/// Uses pread-based reads so a single `Arc<File>` supports concurrent access.
|
||||
pub struct FilePool {
|
||||
max_entries: usize,
|
||||
entries: Mutex<VecDeque<(u32, Arc<Mutex<File>>)>>,
|
||||
entries: Mutex<VecDeque<(u32, Arc<File>)>>,
|
||||
}
|
||||
|
||||
impl FilePool {
|
||||
@@ -22,7 +22,7 @@ impl FilePool {
|
||||
}
|
||||
|
||||
/// Get an open File for the given segment. Reuses cached handle if available.
|
||||
pub fn get(&self, seg_id: u32, path: &Path) -> Result<Arc<Mutex<File>>> {
|
||||
pub fn get(&self, seg_id: u32, path: &Path) -> Result<Arc<File>> {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
|
||||
// Check for existing entry
|
||||
@@ -35,7 +35,7 @@ impl FilePool {
|
||||
}
|
||||
|
||||
// Open new file
|
||||
let file = Arc::new(Mutex::new(fs_util::open_read(path)?));
|
||||
let file = Arc::new(fs_util::open_read(path)?);
|
||||
|
||||
// Evict oldest if full
|
||||
if entries.len() >= self.max_entries {
|
||||
|
||||
@@ -86,6 +86,31 @@ pub fn truncate(path: &Path, size: u64) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Positional read: read `buf.len()` bytes at `offset` from `file`.
|
||||
/// Uses platform-specific pread so `&File` (shared ref) suffices —
|
||||
/// no Mutex needed for concurrent reads.
|
||||
pub fn pread_exact(file: &File, offset: u64, buf: &mut [u8]) -> Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::FileExt;
|
||||
file.read_exact_at(buf, offset)?;
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::fs::FileExt;
|
||||
file.seek_read(buf, offset)?;
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
{
|
||||
// Fallback: seek+read (requires &mut, so this is best-effort on exotic platforms)
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
let mut tmp = file.try_clone()?;
|
||||
tmp.seek(SeekFrom::Start(offset))?;
|
||||
tmp.read_exact(buf)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -1,12 +1,9 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::bucket::{self, BucketFile, BucketIndex, IndexRecord};
|
||||
use crate::bucket::{IndexRecord, IndexStore};
|
||||
use crate::error::Result;
|
||||
#[cfg(test)]
|
||||
use crate::meta::SegmentStats;
|
||||
use crate::meta::GlobalMeta;
|
||||
use crate::segment::{self, SegmentReader, SegmentWriter};
|
||||
|
||||
/// Result of a GC run.
|
||||
@@ -19,15 +16,34 @@ pub struct GcStats {
|
||||
pub entries_skipped: usize,
|
||||
}
|
||||
|
||||
/// Run GC on an account: pick the sealed segment with highest deleted_ratio,
|
||||
/// rewrite it without deleted/overwritten entries, then rebuild all bucket files.
|
||||
pub fn gc_account(
|
||||
account_dir: &Path,
|
||||
deleted_ratio_threshold: f64,
|
||||
) -> Result<Option<GcStats>> {
|
||||
let meta = crate::meta::AccountMeta::load(account_dir)?;
|
||||
/// Prepared GC result — the compacted segment has been written to a temp file
|
||||
/// but not yet renamed over the original. `gc_finish` must be called to commit.
|
||||
pub struct GcPrepare {
|
||||
pub segment_id: u32,
|
||||
pub bytes_before: u64,
|
||||
pub bytes_after: u64,
|
||||
pub entries_kept: usize,
|
||||
pub entries_skipped: usize,
|
||||
/// Index records for kept entries with their new offsets in the compacted segment.
|
||||
pub kept_records: Vec<IndexRecord>,
|
||||
/// Keys whose tombstone IndexRecord should be removed from redb after
|
||||
/// this segment is compacted (the tombstone entries they pointed to are gone).
|
||||
pub deleted_keys: Vec<[u8; 32]>,
|
||||
temp_path: PathBuf,
|
||||
seg_path: PathBuf,
|
||||
}
|
||||
|
||||
// Find the best candidate
|
||||
/// Phase 1: pick the sealed segment with the highest deleted_ratio, then for
|
||||
/// each entry in that segment consult the bucket index to decide whether it is
|
||||
/// still the latest version. Live entries are written to a temp file; stale
|
||||
/// entries and tombstones are skipped. Does NOT rename — the caller should
|
||||
/// hold the write lock only during `gc_finish`.
|
||||
pub fn gc_prepare(
|
||||
store_root: &Path,
|
||||
meta: &GlobalMeta,
|
||||
deleted_ratio_threshold: f64,
|
||||
index_store: &IndexStore,
|
||||
) -> Result<Option<GcPrepare>> {
|
||||
let candidate = meta
|
||||
.segments
|
||||
.values()
|
||||
@@ -39,229 +55,94 @@ pub fn gc_account(
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
let seg_path = account_dir
|
||||
let seg_path = store_root
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(target.segment_id));
|
||||
let reader = SegmentReader::open(seg_path.clone(), target.segment_id)?;
|
||||
|
||||
// Build a global view: for each key, which entry (segment_id + offset) is the latest?
|
||||
let mut latest_key: HashMap<[u8; 32], (u32, u64)> = HashMap::new();
|
||||
|
||||
for &seg_id in meta.segments.keys() {
|
||||
let rpath = account_dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(seg_id));
|
||||
if !rpath.exists() {
|
||||
continue;
|
||||
}
|
||||
let r = SegmentReader::open(rpath, seg_id)?;
|
||||
let _ = r.scan_entries(0, |entry, offset| {
|
||||
match latest_key.get(&entry.key) {
|
||||
Some((existing_seg, existing_off)) => {
|
||||
if seg_id > *existing_seg
|
||||
|| (seg_id == *existing_seg && offset > *existing_off)
|
||||
{
|
||||
latest_key.insert(entry.key, (seg_id, offset));
|
||||
}
|
||||
}
|
||||
None => {
|
||||
latest_key.insert(entry.key, (seg_id, offset));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
// Guard against stale meta entries: if the segment file was deleted
|
||||
// (e.g. after a prior GC emptied it), skip this candidate.
|
||||
if !seg_path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Create temp segment with a unique name
|
||||
let timestamp = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
let reader = SegmentReader::open(seg_path.clone(), target.segment_id)?;
|
||||
|
||||
// Create temp segment (not renamed yet)
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
let temp_name = format!("temp_{:016x}.seg", timestamp);
|
||||
let temp_path = account_dir.join("segments").join(&temp_name);
|
||||
let temp_path = store_root.join("segments").join(&temp_name);
|
||||
let mut writer = SegmentWriter::create(temp_path.clone(), target.segment_id)?;
|
||||
|
||||
let mut kept_records: Vec<IndexRecord> = Vec::new();
|
||||
let mut deleted_keys: Vec<[u8; 32]> = Vec::new();
|
||||
let mut bytes_after: u64 = 0;
|
||||
let mut entries_kept: usize = 0;
|
||||
let mut entries_skipped: usize = 0;
|
||||
|
||||
reader.scan_entries(0, |entry, offset| {
|
||||
// Skip tombstones
|
||||
if entry.is_tombstone() {
|
||||
// If this tombstone is the latest version in the index, the key
|
||||
// must be removed from redb after compaction — otherwise it
|
||||
// accumulates forever.
|
||||
match index_store.get(&entry.key)? {
|
||||
Some(rec)
|
||||
if rec.segment_id == target.segment_id && rec.offset == offset =>
|
||||
{
|
||||
deleted_keys.push(entry.key);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
entries_skipped += 1;
|
||||
return Ok(());
|
||||
}
|
||||
// Skip if this key has a newer entry in another segment
|
||||
if let Some((latest_seg, latest_off)) = latest_key.get(&entry.key) {
|
||||
if *latest_seg != target.segment_id || *latest_off != offset {
|
||||
|
||||
// Ask the bucket index whether this entry is still the latest version.
|
||||
match index_store.get(&entry.key)? {
|
||||
Some(rec) if rec.segment_id == target.segment_id && rec.offset == offset => {
|
||||
let new_offset = writer.append(entry)?;
|
||||
kept_records.push(IndexRecord::new(
|
||||
entry.key,
|
||||
target.segment_id,
|
||||
new_offset,
|
||||
entry.data.len() as u32,
|
||||
entry.flags,
|
||||
));
|
||||
bytes_after += entry.data.len() as u64;
|
||||
entries_kept += 1;
|
||||
}
|
||||
_ => {
|
||||
entries_skipped += 1;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
// Keep this entry
|
||||
writer.append(entry)?;
|
||||
bytes_after += entry.data.len() as u64;
|
||||
entries_kept += 1;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
writer.fsync()?;
|
||||
|
||||
// Atomic rename: replace old segment with new one
|
||||
fs::rename(&temp_path, &seg_path)?;
|
||||
|
||||
// Rebuild all bucket files
|
||||
rebuild_buckets(account_dir, &meta)?;
|
||||
|
||||
// Update meta
|
||||
let mut meta = crate::meta::AccountMeta::load(account_dir)?;
|
||||
if let Some(stats) = meta.segments.get_mut(&target.segment_id) {
|
||||
stats.total_bytes = bytes_after;
|
||||
stats.deleted_bytes = 0;
|
||||
stats.recompute_ratio();
|
||||
}
|
||||
meta.save(account_dir)?;
|
||||
|
||||
Ok(Some(GcStats {
|
||||
Ok(Some(GcPrepare {
|
||||
segment_id: target.segment_id,
|
||||
bytes_before: target.total_bytes,
|
||||
bytes_after,
|
||||
entries_kept,
|
||||
entries_skipped,
|
||||
kept_records,
|
||||
deleted_keys,
|
||||
temp_path,
|
||||
seg_path,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Rebuild all 16 bucket files from scratch by scanning all segments.
|
||||
fn rebuild_buckets(account_dir: &Path, meta: &crate::meta::AccountMeta) -> Result<()> {
|
||||
let mut bucket_records: HashMap<u16, Vec<IndexRecord>> = HashMap::new();
|
||||
for i in 0..crate::types::BUCKET_COUNT {
|
||||
bucket_records.insert(i, Vec::new());
|
||||
}
|
||||
|
||||
for &seg_id in meta.segments.keys() {
|
||||
let seg_path = account_dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(seg_id));
|
||||
if !seg_path.exists() {
|
||||
continue;
|
||||
}
|
||||
let reader = SegmentReader::open(seg_path, seg_id)?;
|
||||
reader.scan_entries(0, |entry, offset| {
|
||||
let bid = bucket::bucket_id(&entry.key);
|
||||
let rec = IndexRecord::new(
|
||||
entry.key,
|
||||
seg_id,
|
||||
offset,
|
||||
entry.data.len() as u32,
|
||||
entry.flags,
|
||||
);
|
||||
bucket_records.entry(bid).or_default().push(rec);
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
|
||||
for (bid, records) in &bucket_records {
|
||||
let index = BucketIndex::from_records(records.clone(), *bid);
|
||||
let bf = BucketFile::open(account_dir, *bid);
|
||||
bf.rewrite(&index.records)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compact bucket files: load, dedup, rewrite.
|
||||
pub fn compact_buckets(account_dir: &Path) -> Result<()> {
|
||||
for bid in 0..crate::types::BUCKET_COUNT {
|
||||
let bf = BucketFile::open(account_dir, bid);
|
||||
if bf.path().exists() {
|
||||
let index = bf.load_index()?;
|
||||
bf.rewrite(&index.records)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::segment::Entry;
|
||||
use crate::types::Codec;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn setup_account(dir: &Path) {
|
||||
fs::create_dir_all(dir.join("segments")).unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir).unwrap();
|
||||
|
||||
let seg_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(1));
|
||||
let mut writer = SegmentWriter::create(seg_path, 1).unwrap();
|
||||
|
||||
// Write 5 entries
|
||||
for i in 0..5u8 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0] = i;
|
||||
let entry = Entry::new(key, &vec![i; 1000], 0, Codec::None);
|
||||
writer.append(&entry).unwrap();
|
||||
}
|
||||
|
||||
// Tombstone entry 2
|
||||
let mut key2 = [0u8; 32];
|
||||
key2[0] = 2;
|
||||
let tomb = Entry::tombstone(key2);
|
||||
writer.append(&tomb).unwrap();
|
||||
|
||||
writer.fsync().unwrap();
|
||||
|
||||
// Save meta
|
||||
let mut meta = crate::meta::AccountMeta::new("test".into(), 2);
|
||||
meta.segments.insert(
|
||||
1,
|
||||
SegmentStats {
|
||||
segment_id: 1,
|
||||
total_bytes: 6000,
|
||||
deleted_bytes: 1000,
|
||||
deleted_ratio: 1000.0 / 6000.0,
|
||||
sealed: true,
|
||||
indexed_up_to_offset: 0,
|
||||
},
|
||||
);
|
||||
// Make segment 2 active so segment 1 is sealed
|
||||
let seg2_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(2));
|
||||
SegmentWriter::create(seg2_path, 2).unwrap();
|
||||
meta.save(dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gc_removes_tombstones() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
setup_account(dir.path());
|
||||
|
||||
let result = gc_account(dir.path(), 0.01).unwrap();
|
||||
assert!(result.is_some());
|
||||
|
||||
// Verify segment 1 no longer has the tombstone'd entry
|
||||
let seg_path = dir
|
||||
.path()
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(1));
|
||||
let reader = SegmentReader::open(seg_path, 1).unwrap();
|
||||
let mut count = 0;
|
||||
reader.scan_entries(0, |entry, _offset| {
|
||||
count += 1;
|
||||
assert!(entry.key[0] != 2);
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
assert_eq!(count, 4); // 5 original - 1 tombstoned
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compact_buckets() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
setup_account(dir.path());
|
||||
compact_buckets(dir.path()).unwrap();
|
||||
// Should not panic
|
||||
}
|
||||
/// Phase 2: atomically replace the old segment with the compacted one.
|
||||
pub fn gc_finish(prep: GcPrepare) -> Result<GcStats> {
|
||||
fs::rename(&prep.temp_path, &prep.seg_path)?;
|
||||
Ok(GcStats {
|
||||
segment_id: prep.segment_id,
|
||||
bytes_before: prep.bytes_before,
|
||||
bytes_after: prep.bytes_after,
|
||||
entries_kept: prep.entries_kept,
|
||||
entries_skipped: prep.entries_skipped,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
pub mod account;
|
||||
pub mod bucket;
|
||||
pub mod cache;
|
||||
pub mod checksum;
|
||||
pub mod compress;
|
||||
pub mod engine;
|
||||
@@ -13,7 +11,6 @@ pub mod recovery;
|
||||
pub mod segment;
|
||||
pub mod types;
|
||||
|
||||
pub use account::AccountHandle;
|
||||
pub use engine::{AccountStats, Engine};
|
||||
pub use engine::{Engine, Stats};
|
||||
pub use error::{Error, Result};
|
||||
pub use types::{Codec, Config};
|
||||
|
||||
@@ -5,14 +5,15 @@ use crate::checksum;
|
||||
use crate::error::Result;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
const META_VERSION: u32 = 1;
|
||||
const META_VERSION: u32 = 2;
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
fn write_bin<T: Serialize>(path: &Path, value: &T) -> Result<()> {
|
||||
let payload = bincode::serialize(value).map_err(|e| {
|
||||
crate::error::Error::CorruptMeta(format!("{}: bincode encode: {}", path.display(), e))
|
||||
})?;
|
||||
let payload =
|
||||
bincode::serde::encode_to_vec(value, bincode::config::standard()).map_err(|e| {
|
||||
crate::error::Error::CorruptMeta(format!("{}: bincode encode: {}", path.display(), e))
|
||||
})?;
|
||||
let crc = checksum::crc32(&payload);
|
||||
let mut buf = Vec::with_capacity(8 + payload.len());
|
||||
buf.extend_from_slice(&crc.to_le_bytes());
|
||||
@@ -40,53 +41,11 @@ fn read_bin<T: for<'de> Deserialize<'de>>(path: &Path) -> Result<T> {
|
||||
if stored_crc != computed {
|
||||
return Err(crate::error::Error::CorruptMeta(path.display().to_string()));
|
||||
}
|
||||
bincode::deserialize(&data[8..]).map_err(|e| {
|
||||
crate::error::Error::CorruptMeta(format!("{}: bincode decode: {}", path.display(), e))
|
||||
})
|
||||
}
|
||||
|
||||
// ── GlobalMeta ─────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GlobalMeta {
|
||||
pub version: u32,
|
||||
pub accounts: Vec<String>,
|
||||
}
|
||||
|
||||
impl Default for GlobalMeta {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
version: META_VERSION,
|
||||
accounts: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl GlobalMeta {
|
||||
pub fn load(store_root: &Path) -> Result<Self> {
|
||||
let bin_path = store_root.join("global_meta.bin");
|
||||
if bin_path.exists() {
|
||||
return read_bin(&bin_path);
|
||||
}
|
||||
// Migration from JSON
|
||||
let json_path = store_root.join("global_meta.json");
|
||||
if json_path.exists() {
|
||||
let data = std::fs::read_to_string(&json_path)?;
|
||||
let mut meta: Self = serde_json::from_str(&data)?;
|
||||
meta.accounts.sort();
|
||||
write_bin(&bin_path, &meta)?;
|
||||
let _ = std::fs::remove_file(&json_path);
|
||||
return Ok(meta);
|
||||
}
|
||||
Ok(Self::default())
|
||||
}
|
||||
|
||||
pub fn save(&self, store_root: &Path) -> Result<()> {
|
||||
let path = store_root.join("global_meta.bin");
|
||||
let mut meta = self.clone();
|
||||
meta.accounts.sort();
|
||||
write_bin(&path, &meta)
|
||||
}
|
||||
bincode::serde::decode_from_slice(&data[8..], bincode::config::standard())
|
||||
.map(|(v, _)| v)
|
||||
.map_err(|e| {
|
||||
crate::error::Error::CorruptMeta(format!("{}: bincode decode: {}", path.display(), e))
|
||||
})
|
||||
}
|
||||
|
||||
// ── SegmentStats ───────────────────────────────────────────────────────────
|
||||
@@ -99,8 +58,10 @@ pub struct SegmentStats {
|
||||
pub deleted_ratio: f64,
|
||||
pub sealed: bool,
|
||||
/// Byte offset up to which entries have been indexed in bucket files.
|
||||
/// Recovery starts scanning from here instead of 0.
|
||||
pub indexed_up_to_offset: u64,
|
||||
/// Number of compacted (sorted, deduped) records in each bucket file for this segment.
|
||||
/// Used by BucketStore on recovery to know where the clean portion ends.
|
||||
pub bucket_compacted: u64,
|
||||
}
|
||||
|
||||
impl SegmentStats {
|
||||
@@ -112,6 +73,7 @@ impl SegmentStats {
|
||||
deleted_ratio: 0.0,
|
||||
sealed: false,
|
||||
indexed_up_to_offset: 0,
|
||||
bucket_compacted: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,45 +86,41 @@ impl SegmentStats {
|
||||
}
|
||||
}
|
||||
|
||||
// ── AccountMeta ────────────────────────────────────────────────────────────
|
||||
// ── GlobalMeta ─────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AccountMeta {
|
||||
pub account_id: String,
|
||||
pub struct GlobalMeta {
|
||||
pub version: u32,
|
||||
pub active_segment_id: u32,
|
||||
pub segments: BTreeMap<u32, SegmentStats>,
|
||||
}
|
||||
|
||||
impl AccountMeta {
|
||||
pub fn new(account_id: String, active_segment_id: u32) -> Self {
|
||||
impl GlobalMeta {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
active_segment_id,
|
||||
version: META_VERSION,
|
||||
active_segment_id: 1,
|
||||
segments: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load(account_dir: &Path) -> Result<Self> {
|
||||
let bin_path = account_dir.join("meta.bin");
|
||||
pub fn load(store_root: &Path) -> Result<Self> {
|
||||
let bin_path = store_root.join("meta.bin");
|
||||
if bin_path.exists() {
|
||||
return read_bin(&bin_path);
|
||||
}
|
||||
// Migration from JSON
|
||||
let json_path = account_dir.join("meta.json");
|
||||
if json_path.exists() {
|
||||
let data = std::fs::read_to_string(&json_path)?;
|
||||
let meta: Self = serde_json::from_str(&data)?;
|
||||
write_bin(&bin_path, &meta)?;
|
||||
let _ = std::fs::remove_file(&json_path);
|
||||
return Ok(meta);
|
||||
}
|
||||
Err(crate::error::Error::AccountNotFound(
|
||||
account_dir.to_string_lossy().into(),
|
||||
))
|
||||
Ok(Self::new())
|
||||
}
|
||||
|
||||
pub fn save(&self, account_dir: &Path) -> Result<()> {
|
||||
write_bin(&account_dir.join("meta.bin"), self)
|
||||
pub fn save(&self, store_root: &Path) -> Result<()> {
|
||||
let path = store_root.join("meta.bin");
|
||||
write_bin(&path, self)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for GlobalMeta {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,45 +130,9 @@ mod tests {
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_global_meta_bin_roundtrip() {
|
||||
fn test_global_meta_roundtrip() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut meta = GlobalMeta::default();
|
||||
meta.accounts.push("alice".into());
|
||||
meta.save(dir.path()).unwrap();
|
||||
|
||||
let loaded = GlobalMeta::load(dir.path()).unwrap();
|
||||
assert_eq!(loaded.accounts, vec!["alice"]);
|
||||
assert!(!dir.path().join("global_meta.json").exists());
|
||||
assert!(dir.path().join("global_meta.bin").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_global_meta_default_when_missing() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let meta = GlobalMeta::load(dir.path()).unwrap();
|
||||
assert!(meta.accounts.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_json_migration() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
// Write old JSON format
|
||||
let json = r#"{"version":1,"accounts":["bob","alice"]}"#;
|
||||
std::fs::write(dir.path().join("global_meta.json"), json).unwrap();
|
||||
|
||||
let meta = GlobalMeta::load(dir.path()).unwrap();
|
||||
// Should be sorted
|
||||
assert_eq!(meta.accounts, vec!["alice", "bob"]);
|
||||
// JSON should be removed
|
||||
assert!(!dir.path().join("global_meta.json").exists());
|
||||
// BIN should exist
|
||||
assert!(dir.path().join("global_meta.bin").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_account_meta_bin_roundtrip() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut meta = AccountMeta::new("alice".into(), 1);
|
||||
let mut meta = GlobalMeta::new();
|
||||
meta.segments.insert(
|
||||
1,
|
||||
SegmentStats {
|
||||
@@ -219,66 +141,40 @@ mod tests {
|
||||
deleted_bytes: 300,
|
||||
deleted_ratio: 0.3,
|
||||
sealed: false,
|
||||
indexed_up_to_offset: 0,
|
||||
indexed_up_to_offset: 500,
|
||||
bucket_compacted: 0,
|
||||
},
|
||||
);
|
||||
meta.save(dir.path()).unwrap();
|
||||
|
||||
let loaded = AccountMeta::load(dir.path()).unwrap();
|
||||
let loaded = GlobalMeta::load(dir.path()).unwrap();
|
||||
assert_eq!(loaded.active_segment_id, 1);
|
||||
assert_eq!(loaded.segments[&1].total_bytes, 1000);
|
||||
assert_eq!(loaded.segments[&1].indexed_up_to_offset, 500);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_global_meta_default_when_missing() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let meta = GlobalMeta::load(dir.path()).unwrap();
|
||||
assert_eq!(meta.active_segment_id, 1);
|
||||
assert!(meta.segments.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_corrupt_bin_detected() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("meta.bin"), vec![0xFFu8; 100]).unwrap();
|
||||
let result = AccountMeta::load(dir.path());
|
||||
let result = GlobalMeta::load(dir.path());
|
||||
assert!(result.is_err());
|
||||
// 0xFFFFFFFF version triggers UnsupportedMetaVersion
|
||||
assert!(matches!(result.unwrap_err(), crate::error::Error::UnsupportedMetaVersion { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crc_corruption_detected() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
// Write a well-formed header (version=1) but with wrong CRC bytes
|
||||
let mut buf = Vec::new();
|
||||
buf.extend_from_slice(&0xDEADBEEFu32.to_le_bytes()); // wrong CRC
|
||||
buf.extend_from_slice(&1u32.to_le_bytes()); // version = 1 (OK)
|
||||
buf.extend_from_slice(b"some payload bytes"); // payload
|
||||
std::fs::write(dir.path().join("meta.bin"), &buf).unwrap();
|
||||
let result = AccountMeta::load(dir.path());
|
||||
assert!(matches!(result.unwrap_err(), crate::error::Error::CorruptMeta(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_account_json_migration() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
// Write old JSON format for AccountMeta
|
||||
let json = r#"{"account_id":"alice","active_segment_id":5,"segments":{}}"#;
|
||||
std::fs::write(dir.path().join("meta.json"), json).unwrap();
|
||||
|
||||
let meta = AccountMeta::load(dir.path()).unwrap();
|
||||
assert_eq!(meta.account_id, "alice");
|
||||
assert_eq!(meta.active_segment_id, 5);
|
||||
// JSON should be removed
|
||||
assert!(!dir.path().join("meta.json").exists());
|
||||
// BIN should exist
|
||||
assert!(dir.path().join("meta.bin").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bin_sorted_keys() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut meta = AccountMeta::new("test".into(), 1);
|
||||
meta.segments.insert(3, SegmentStats::new(3));
|
||||
meta.segments.insert(1, SegmentStats::new(1));
|
||||
meta.segments.insert(2, SegmentStats::new(2));
|
||||
meta.save(dir.path()).unwrap();
|
||||
|
||||
let loaded = AccountMeta::load(dir.path()).unwrap();
|
||||
let keys: Vec<u32> = loaded.segments.keys().copied().collect();
|
||||
assert_eq!(keys, vec![1, 2, 3]);
|
||||
fn test_segment_stats_recompute() {
|
||||
let mut s = SegmentStats::new(1);
|
||||
s.total_bytes = 1000;
|
||||
s.deleted_bytes = 250;
|
||||
s.recompute_ratio();
|
||||
assert!((s.deleted_ratio - 0.25).abs() < 0.001);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,45 +1,50 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::bucket::{self, BucketFile, IndexRecord};
|
||||
use crate::bucket::IndexRecord;
|
||||
use crate::error::Result;
|
||||
use crate::meta::{AccountMeta, SegmentStats};
|
||||
use crate::meta::{GlobalMeta, SegmentStats};
|
||||
use crate::segment::{self, SegmentReader};
|
||||
|
||||
/// Recover an account after a crash: scan segments, repair indices, update stats.
|
||||
pub fn recover_account(account_dir: &Path) -> Result<AccountMeta> {
|
||||
let meta_bin = account_dir.join("meta.bin");
|
||||
let meta_json = account_dir.join("meta.json");
|
||||
let meta_exists = meta_bin.exists() || meta_json.exists();
|
||||
let mut meta = if meta_exists {
|
||||
AccountMeta::load(account_dir).unwrap_or_else(|_| {
|
||||
AccountMeta::new(
|
||||
account_dir
|
||||
.file_name()
|
||||
.unwrap_or_default()
|
||||
.to_string_lossy()
|
||||
.into(),
|
||||
1,
|
||||
)
|
||||
})
|
||||
} else {
|
||||
return Ok(AccountMeta::new(
|
||||
account_dir
|
||||
.file_name()
|
||||
.unwrap_or_default()
|
||||
.to_string_lossy()
|
||||
.into(),
|
||||
1,
|
||||
));
|
||||
};
|
||||
/// Recover after a crash: scan any unindexed portions of segments,
|
||||
/// update segment stats, and return newly discovered index records.
|
||||
///
|
||||
/// The caller is responsible for inserting the returned records into
|
||||
/// the index store.
|
||||
pub fn recover(store_root: &Path, meta: &mut GlobalMeta) -> Result<Vec<IndexRecord>> {
|
||||
scan_segments(store_root, meta, false)
|
||||
}
|
||||
|
||||
// Discover all segment files on disk
|
||||
let seg_dir = account_dir.join("segments");
|
||||
/// Rebuild the entire index from scratch by scanning all segments from
|
||||
/// offset 0. Used when the index database is corrupted or lost.
|
||||
///
|
||||
/// Resets all segment stats and returns every entry found on disk.
|
||||
/// The caller should replace the index database before calling this.
|
||||
pub fn rebuild_index(store_root: &Path, meta: &mut GlobalMeta) -> Result<Vec<IndexRecord>> {
|
||||
// Reset all segment stats — they'll be recomputed during the scan.
|
||||
// Also reset indexed_up_to_offset so we scan from 0.
|
||||
for stats in meta.segments.values_mut() {
|
||||
stats.total_bytes = 0;
|
||||
stats.deleted_bytes = 0;
|
||||
stats.deleted_ratio = 0.0;
|
||||
stats.indexed_up_to_offset = 0;
|
||||
}
|
||||
scan_segments(store_root, meta, true)
|
||||
}
|
||||
|
||||
/// Common implementation: scan segments and collect index records.
|
||||
/// When `full_scan` is true, every segment is scanned from offset 0.
|
||||
fn scan_segments(
|
||||
store_root: &Path,
|
||||
meta: &mut GlobalMeta,
|
||||
full_scan: bool,
|
||||
) -> Result<Vec<IndexRecord>> {
|
||||
let seg_dir = store_root.join("segments");
|
||||
if !seg_dir.exists() {
|
||||
fs::create_dir_all(&seg_dir)?;
|
||||
}
|
||||
|
||||
// Discover all segment files on disk
|
||||
let mut disk_segments: Vec<u32> = Vec::new();
|
||||
if seg_dir.exists() {
|
||||
for entry in fs::read_dir(&seg_dir)? {
|
||||
@@ -57,53 +62,45 @@ pub fn recover_account(account_dir: &Path) -> Result<AccountMeta> {
|
||||
}
|
||||
disk_segments.sort_unstable();
|
||||
|
||||
if disk_segments.is_empty() {
|
||||
meta.active_segment_id = 1;
|
||||
} else {
|
||||
let max_id = *disk_segments.last().unwrap();
|
||||
meta.active_segment_id = max_id;
|
||||
}
|
||||
let mut all_records: Vec<IndexRecord> = Vec::new();
|
||||
|
||||
// Ensure buckets directory exists
|
||||
let buckets_dir = account_dir.join("buckets");
|
||||
fs::create_dir_all(&buckets_dir)?;
|
||||
|
||||
// For each segment, scan only the unindexed tail and update stats incrementally
|
||||
// For each segment, scan unindexed portions
|
||||
for &seg_id in &disk_segments {
|
||||
let seg_path = seg_dir.join(segment::segment_filename(seg_id));
|
||||
let file_size = fs::metadata(&seg_path)?.len();
|
||||
|
||||
// Preserve existing stats; start fresh if this is a newly discovered segment
|
||||
let mut stats = meta.segments.remove(&seg_id).unwrap_or_else(|| SegmentStats::new(seg_id));
|
||||
let mut stats = meta
|
||||
.segments
|
||||
.remove(&seg_id)
|
||||
.unwrap_or_else(|| SegmentStats::new(seg_id));
|
||||
let is_sealed = seg_id != meta.active_segment_id;
|
||||
stats.sealed = is_sealed;
|
||||
|
||||
// Scan start: from last indexed offset. Clamp defensively.
|
||||
let scan_start = if stats.indexed_up_to_offset <= file_size {
|
||||
// Determine scan range.
|
||||
let scan_start = if full_scan {
|
||||
0
|
||||
} else if stats.indexed_up_to_offset <= file_size {
|
||||
stats.indexed_up_to_offset
|
||||
} else {
|
||||
// Segment was replaced (interrupted GC) — full rescan needed.
|
||||
0
|
||||
};
|
||||
|
||||
// If fully indexed, skip scanning entirely
|
||||
if scan_start >= file_size {
|
||||
meta.segments.insert(seg_id, stats);
|
||||
continue;
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(seg_path.clone(), seg_id)?;
|
||||
let mut new_records: HashMap<u16, Vec<IndexRecord>> = HashMap::new();
|
||||
|
||||
let truncation_point = reader.scan_entries(scan_start, |entry, offset| {
|
||||
let bid = bucket::bucket_id(&entry.key);
|
||||
let rec = IndexRecord::new(
|
||||
all_records.push(IndexRecord::new(
|
||||
entry.key,
|
||||
seg_id,
|
||||
offset,
|
||||
entry.data.len() as u32,
|
||||
entry.flags,
|
||||
);
|
||||
new_records.entry(bid).or_default().push(rec);
|
||||
));
|
||||
|
||||
stats.total_bytes += entry.data.len() as u64;
|
||||
if entry.is_tombstone() {
|
||||
@@ -113,12 +110,6 @@ pub fn recover_account(account_dir: &Path) -> Result<AccountMeta> {
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
// Merge new records into bucket files (only the newly discovered ones)
|
||||
for (bid, records) in &new_records {
|
||||
let bf = BucketFile::open(account_dir, *bid);
|
||||
bf.append_batch(records)?;
|
||||
}
|
||||
|
||||
// Truncate if tail corruption found
|
||||
if truncation_point < file_size {
|
||||
segment::truncate_segment(&seg_path, truncation_point)?;
|
||||
@@ -129,14 +120,14 @@ pub fn recover_account(account_dir: &Path) -> Result<AccountMeta> {
|
||||
meta.segments.insert(seg_id, stats);
|
||||
}
|
||||
|
||||
meta.save(account_dir)?;
|
||||
meta.save(store_root)?;
|
||||
|
||||
Ok(meta)
|
||||
Ok(all_records)
|
||||
}
|
||||
|
||||
/// Clean up leftover temp files from interrupted GC.
|
||||
pub fn cleanup_temp_files(account_dir: &Path) -> Result<()> {
|
||||
let seg_dir = account_dir.join("segments");
|
||||
pub fn cleanup_temp_files(store_root: &Path) -> Result<()> {
|
||||
let seg_dir = store_root.join("segments");
|
||||
if seg_dir.exists() {
|
||||
for entry in fs::read_dir(&seg_dir)? {
|
||||
let entry = entry?;
|
||||
@@ -149,54 +140,5 @@ pub fn cleanup_temp_files(account_dir: &Path) -> Result<()> {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Also cleanup temp bucket files
|
||||
let buckets_dir = account_dir.join("buckets");
|
||||
if buckets_dir.exists() {
|
||||
for entry in fs::read_dir(&buckets_dir)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
if name_str.ends_with(".tmp") {
|
||||
let path = entry.path();
|
||||
tracing::warn!("Removing leftover temp bucket file: {:?}", path);
|
||||
fs::remove_file(&path)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_recover_fresh_account() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let account_dir = dir.path().join("test");
|
||||
fs::create_dir_all(&account_dir).unwrap();
|
||||
|
||||
let meta = recover_account(&account_dir).unwrap();
|
||||
assert_eq!(meta.active_segment_id, 1);
|
||||
assert!(meta.segments.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cleanup_temp_files() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let account_dir = dir.path().join("test");
|
||||
fs::create_dir_all(account_dir.join("segments")).unwrap();
|
||||
fs::create_dir_all(account_dir.join("buckets")).unwrap();
|
||||
fs::write(
|
||||
account_dir.join("segments").join("temp_ABC123.seg"),
|
||||
b"garbage",
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(account_dir.join("buckets").join("00.idx.tmp"), b"garbage").unwrap();
|
||||
|
||||
cleanup_temp_files(&account_dir).unwrap();
|
||||
|
||||
assert!(!account_dir.join("segments").join("temp_ABC123.seg").exists());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
use std::fs::{self, File};
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use crate::checksum;
|
||||
use crate::error::{Error, Result};
|
||||
@@ -20,13 +19,14 @@ pub struct Entry {
|
||||
|
||||
impl Entry {
|
||||
/// Create a normal data entry.
|
||||
pub fn new(key: [u8; 32], raw_data: &[u8], flags: u8, codec: Codec) -> Self {
|
||||
/// `raw_size` is the original uncompressed size; `data` is what goes to disk.
|
||||
pub fn new(key: [u8; 32], data: &[u8], raw_size: u32, flags: u8, codec: Codec) -> Self {
|
||||
Self {
|
||||
flags,
|
||||
codec,
|
||||
key,
|
||||
raw_size: raw_data.len() as u32,
|
||||
data: raw_data.to_vec(),
|
||||
raw_size,
|
||||
data: data.to_vec(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -232,6 +232,14 @@ impl SegmentReader {
|
||||
file.read_exact(&mut data_size_buf)?;
|
||||
let data_size = u32::from_le_bytes(data_size_buf);
|
||||
|
||||
if data_size as usize > crate::types::MAX_VALUE_SIZE {
|
||||
return Err(Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
reason: format!("data_size {} exceeds max {}", data_size, crate::types::MAX_VALUE_SIZE),
|
||||
});
|
||||
}
|
||||
|
||||
// Read data
|
||||
let mut data = vec![0u8; data_size as usize];
|
||||
file.read_exact(&mut data)?;
|
||||
@@ -269,18 +277,18 @@ impl SegmentReader {
|
||||
))
|
||||
}
|
||||
|
||||
/// Read a single entry at the given offset using a pre-opened File (via Mutex).
|
||||
/// This avoids the per-read File::open cost for hot segments.
|
||||
pub fn read_entry_at_file(&self, offset: u64, file: &Mutex<File>) -> Result<(Entry, u64)> {
|
||||
/// Read a single entry at the given offset using a pre-opened File.
|
||||
/// Uses pread so concurrent reads on the same segment don't block each other.
|
||||
pub fn read_entry_at_file(&self, offset: u64, file: &File) -> Result<(Entry, u64)> {
|
||||
// Read header (50 bytes)
|
||||
let mut header = [0u8; ENTRY_HEADER_SIZE];
|
||||
fs_util::pread_exact(file, offset, &mut header)?;
|
||||
|
||||
let mut file = file.lock().unwrap();
|
||||
let mut pos = 0;
|
||||
|
||||
file.seek(SeekFrom::Start(offset))?;
|
||||
|
||||
// Read magic
|
||||
let mut magic_buf = [0u8; 4];
|
||||
file.read_exact(&mut magic_buf)?;
|
||||
let magic = u32::from_le_bytes(magic_buf);
|
||||
// Magic
|
||||
let magic = u32::from_le_bytes(header[pos..pos+4].try_into().unwrap());
|
||||
pos += 4;
|
||||
if magic != ENTRY_MAGIC {
|
||||
return Err(Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
@@ -289,41 +297,45 @@ impl SegmentReader {
|
||||
});
|
||||
}
|
||||
|
||||
// Read CRC32
|
||||
let mut crc_buf = [0u8; 4];
|
||||
file.read_exact(&mut crc_buf)?;
|
||||
let stored_crc = u32::from_le_bytes(crc_buf);
|
||||
// CRC32
|
||||
let stored_crc = u32::from_le_bytes(header[pos..pos+4].try_into().unwrap());
|
||||
pos += 4;
|
||||
|
||||
// Read flags, codec
|
||||
let mut flags_buf = [0u8; 1];
|
||||
file.read_exact(&mut flags_buf)?;
|
||||
let flags = flags_buf[0];
|
||||
|
||||
let mut codec_buf = [0u8; 1];
|
||||
file.read_exact(&mut codec_buf)?;
|
||||
let codec = Codec::from_u8(codec_buf[0]).ok_or_else(|| Error::CorruptEntry {
|
||||
// Flags, codec
|
||||
let flags = header[pos];
|
||||
pos += 1;
|
||||
let codec = Codec::from_u8(header[pos]).ok_or_else(|| Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
reason: format!("unknown codec: {}", codec_buf[0]),
|
||||
reason: format!("unknown codec: {}", header[pos]),
|
||||
})?;
|
||||
pos += 1;
|
||||
|
||||
// Read key, raw_size, data_size
|
||||
// Key
|
||||
let mut key = [0u8; 32];
|
||||
file.read_exact(&mut key)?;
|
||||
key.copy_from_slice(&header[pos..pos+32]);
|
||||
pos += 32;
|
||||
|
||||
let mut raw_size_buf = [0u8; 4];
|
||||
file.read_exact(&mut raw_size_buf)?;
|
||||
let raw_size = u32::from_le_bytes(raw_size_buf);
|
||||
// Raw size, data size
|
||||
let raw_size = u32::from_le_bytes(header[pos..pos+4].try_into().unwrap());
|
||||
pos += 4;
|
||||
let data_size = u32::from_le_bytes(header[pos..pos+4].try_into().unwrap());
|
||||
|
||||
let mut data_size_buf = [0u8; 4];
|
||||
file.read_exact(&mut data_size_buf)?;
|
||||
let data_size = u32::from_le_bytes(data_size_buf);
|
||||
// Defense against header corruption: refuse absurdly large allocations
|
||||
if data_size as usize > crate::types::MAX_VALUE_SIZE {
|
||||
return Err(Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
reason: format!("data_size {} exceeds max {}", data_size, crate::types::MAX_VALUE_SIZE),
|
||||
});
|
||||
}
|
||||
|
||||
// Read data
|
||||
let data_offset = offset + ENTRY_HEADER_SIZE as u64;
|
||||
let mut data = vec![0u8; data_size as usize];
|
||||
file.read_exact(&mut data)?;
|
||||
fs_util::pread_exact(file, data_offset, &mut data)?;
|
||||
|
||||
// Verify CRC32
|
||||
// Verify CRC32 (over everything after the crc32 field: flags+codec+key+raw_size+data_size+data)
|
||||
let computed_crc = {
|
||||
let mut hasher = crate::checksum::CrcWriter::new();
|
||||
hasher.update(&[flags]);
|
||||
@@ -449,7 +461,7 @@ mod tests {
|
||||
let key = [0xAAu8; 32];
|
||||
let data = b"hello world".to_vec();
|
||||
|
||||
let entry = Entry::new(key, &data, 0, Codec::None);
|
||||
let entry = Entry::new(key, &data, data.len() as u32, 0, Codec::None);
|
||||
{
|
||||
let mut writer = SegmentWriter::create(path.clone(), 1).unwrap();
|
||||
writer.append(&entry).unwrap();
|
||||
@@ -495,7 +507,7 @@ mod tests {
|
||||
.map(|i| {
|
||||
let mut key = [0u8; 32];
|
||||
key[0] = i;
|
||||
Entry::new(key, &vec![i; 100], 0, Codec::None)
|
||||
Entry::new(key, &vec![i; 100], 100, 0, Codec::None)
|
||||
})
|
||||
.collect();
|
||||
|
||||
|
||||
@@ -6,14 +6,11 @@ pub const ENTRY_MAGIC: u32 = 0xB3DB_0001;
|
||||
/// Fixed header size: magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4)
|
||||
pub const ENTRY_HEADER_SIZE: usize = 50;
|
||||
|
||||
/// Index record size: key(32) + segment_id(4) + offset(8) + data_size(4) + flags(1) + _pad(3)
|
||||
pub const INDEX_RECORD_SIZE: usize = 52;
|
||||
/// Index record size: key(32) + segment_id(4) + offset(8) + data_size(4) + flags(1) + _pad(3) + crc32(4)
|
||||
pub const INDEX_RECORD_SIZE: usize = 56;
|
||||
|
||||
/// Maximum segment size (256 MB)
|
||||
pub const SEGMENT_MAX_SIZE: u64 = 256 * 1024 * 1024;
|
||||
|
||||
/// Number of hash buckets per account
|
||||
pub const BUCKET_COUNT: u16 = 16;
|
||||
/// Maximum segment size (1 GB)
|
||||
pub const SEGMENT_MAX_SIZE: u64 = 1024 * 1024 * 1024;
|
||||
|
||||
/// Maximum value size (100 MB)
|
||||
pub const MAX_VALUE_SIZE: usize = 100 * 1024 * 1024;
|
||||
@@ -21,12 +18,12 @@ pub const MAX_VALUE_SIZE: usize = 100 * 1024 * 1024;
|
||||
/// Default compression threshold (4 KB)
|
||||
pub const DEFAULT_COMPRESS_THRESHOLD: usize = 4096;
|
||||
|
||||
/// Default LRU bucket cache size
|
||||
pub const DEFAULT_LRU_BUCKET_COUNT: usize = 256;
|
||||
|
||||
/// Default GC deleted ratio threshold
|
||||
pub const DEFAULT_GC_DELETED_RATIO: f64 = 0.30;
|
||||
|
||||
/// Default GC interval in seconds (5 minutes)
|
||||
pub const DEFAULT_GC_INTERVAL_SECS: u64 = 300;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum Codec {
|
||||
None = 0,
|
||||
@@ -50,8 +47,15 @@ pub struct Config {
|
||||
pub compress_threshold: usize,
|
||||
pub default_codec: Codec,
|
||||
pub compression_level: i32,
|
||||
pub lru_bucket_count: usize,
|
||||
pub gc_deleted_ratio: f64,
|
||||
/// Interval in seconds for periodic background flush (0 = disabled).
|
||||
/// When set, a background thread fsyncs the active segment and saves
|
||||
/// metadata at this interval, bounding recovery time after a crash.
|
||||
pub flush_interval_secs: u64,
|
||||
/// Interval in seconds for periodic background GC (0 = disabled).
|
||||
/// When set, a background thread checks whether any sealed segment
|
||||
/// exceeds the deleted-ratio threshold and compacts it if needed.
|
||||
pub gc_interval_secs: u64,
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
@@ -60,19 +64,15 @@ impl Default for Config {
|
||||
compress_threshold: DEFAULT_COMPRESS_THRESHOLD,
|
||||
default_codec: Codec::Zstd,
|
||||
compression_level: 0,
|
||||
lru_bucket_count: DEFAULT_LRU_BUCKET_COUNT,
|
||||
gc_deleted_ratio: DEFAULT_GC_DELETED_RATIO,
|
||||
flush_interval_secs: 0,
|
||||
gc_interval_secs: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn validate(&self) -> crate::error::Result<()> {
|
||||
if self.lru_bucket_count == 0 {
|
||||
return Err(crate::error::Error::InvalidConfig(
|
||||
"lru_bucket_count must be > 0".into(),
|
||||
));
|
||||
}
|
||||
if self.gc_deleted_ratio <= 0.0 || self.gc_deleted_ratio >= 1.0 {
|
||||
return Err(crate::error::Error::InvalidConfig(
|
||||
"gc_deleted_ratio must be in (0.0, 1.0)".into(),
|
||||
@@ -83,6 +83,16 @@ impl Config {
|
||||
"compression_level must be >= 0".into(),
|
||||
));
|
||||
}
|
||||
if self.flush_interval_secs > 0 && self.flush_interval_secs < 5 {
|
||||
return Err(crate::error::Error::InvalidConfig(
|
||||
"flush_interval_secs must be 0 (disabled) or >= 5".into(),
|
||||
));
|
||||
}
|
||||
if self.gc_interval_secs > 0 && self.gc_interval_secs < 10 {
|
||||
return Err(crate::error::Error::InvalidConfig(
|
||||
"gc_interval_secs must be 0 (disabled) or >= 10".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,9 +3,6 @@
|
||||
/// Since we can't kill the process mid-write in an inline test, we simulate crashes
|
||||
/// by dropping the Engine without calling any cleanup (close/drop is the "crash"),
|
||||
/// then re-opening and verifying recovery produced consistent state.
|
||||
///
|
||||
/// For true power-loss simulation, each test writes data, drops the engine abruptly,
|
||||
/// then reopens and verifies: no corruption, no lost committed data, no partial writes.
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
@@ -50,16 +47,13 @@ fn test_durability_single_write_survives_crash() {
|
||||
// Write
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
engine.put(key, &value, Codec::Zstd).unwrap();
|
||||
} // <-- Engine dropped = simulated crash
|
||||
|
||||
// Recover
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
let result = engine.get(&key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
}
|
||||
@@ -73,21 +67,23 @@ fn test_durability_many_writes_survive_crash() {
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
for i in 0..n {
|
||||
let key = make_key(i as u64);
|
||||
keys.push(key);
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
engine.put(key, &value, Codec::Zstd).unwrap();
|
||||
}
|
||||
} // crash
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for (i, key) in keys.iter().enumerate() {
|
||||
let result = engine.read("alice", key).unwrap();
|
||||
assert_eq!(result, Some(value.clone()), "missing key at index {}", i);
|
||||
let result = engine.get(key).unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
Some(value.clone()),
|
||||
"missing key at index {}",
|
||||
i
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -100,20 +96,17 @@ fn test_durability_delete_survives_crash() {
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
engine.put(key, &value, Codec::Zstd).unwrap();
|
||||
} // crash after write
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.delete("alice", &key).unwrap();
|
||||
engine.delete(&key).unwrap();
|
||||
} // crash after delete
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
let result = engine.get(&key).unwrap();
|
||||
assert_eq!(result, None, "delete should persist across crash");
|
||||
}
|
||||
}
|
||||
@@ -129,22 +122,20 @@ fn test_atomicity_no_partial_entries_after_crash() {
|
||||
// Write enough entries to fill part of a segment, then crash
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
let value = make_value(50_000); // big enough to notice
|
||||
let value = make_value(50_000);
|
||||
for i in 0..200u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &value, Codec::None)
|
||||
.put(make_key(i), &value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
} // crash
|
||||
|
||||
// Recovery should clean up any partial tail entries and all committed
|
||||
// entries should be readable
|
||||
// Recovery should clean up any partial tail entries
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let value = make_value(50_000);
|
||||
for i in 0..200u64 {
|
||||
let result = engine.read("alice", &make_key(i)).unwrap();
|
||||
let result = engine.get(&make_key(i)).unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
Some(value.clone()),
|
||||
@@ -162,20 +153,18 @@ fn test_atomicity_crash_during_segment_roll() {
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
// Write enough to cross at least one segment boundary (256 MB)
|
||||
for i in 0..140u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &big_value, Codec::None)
|
||||
.put(make_key(i), &big_value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
} // crash mid-way or after multiple segments
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
// All committed writes (that returned Ok) must be readable
|
||||
for i in 0..140u64 {
|
||||
let result = engine.read("alice", &make_key(i)).unwrap();
|
||||
let result = engine.get(&make_key(i)).unwrap();
|
||||
assert!(
|
||||
result.is_some(),
|
||||
"key {} should exist after segment roll recovery",
|
||||
@@ -197,16 +186,12 @@ fn test_consistency_crc_detects_corruption() {
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
engine.put(key, &value, Codec::Zstd).unwrap();
|
||||
}
|
||||
|
||||
// Corrupt the segment file by flipping a byte
|
||||
let seg_path = find_first_segment(dir.path(), "alice");
|
||||
let seg_path = find_first_segment(dir.path());
|
||||
let mut data = fs::read(&seg_path).unwrap();
|
||||
// Flip a byte in the data portion, not the header
|
||||
let flip_pos = data.len() - 100;
|
||||
data[flip_pos] ^= 0xFF;
|
||||
fs::write(&seg_path, &data).unwrap();
|
||||
@@ -214,16 +199,14 @@ fn test_consistency_crc_detects_corruption() {
|
||||
// Reading should detect CRC mismatch
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.read("alice", &key);
|
||||
// Either error or None is acceptable — never silently wrong data
|
||||
let result = engine.get(&key);
|
||||
match result {
|
||||
Err(_) => {} // CRC mismatch detected — good
|
||||
Err(_) => {} // CRC mismatch detected - good
|
||||
Ok(None) => {} // index may point to truncated/removed data
|
||||
Ok(Some(v)) => {
|
||||
if v == value {
|
||||
panic!("CRC corruption was NOT detected — silent data corruption!");
|
||||
panic!("CRC corruption was NOT detected - silent data corruption!");
|
||||
}
|
||||
// If value differs, index pointed elsewhere after recovery
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -235,35 +218,39 @@ fn test_consistency_corrupt_magic_truncated_on_recovery() {
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
for i in 0..10u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &make_value(4096), Codec::Zstd)
|
||||
.put(make_key(i), &make_value(4096), Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
// Append garbage to the segment file (simulating partial write from crash)
|
||||
let seg_path = find_first_segment(dir.path(), "alice");
|
||||
let seg_path = find_first_segment(dir.path());
|
||||
let mut data = fs::read(&seg_path).unwrap();
|
||||
let orig_len = data.len();
|
||||
// Append garbage that doesn't start with the magic number
|
||||
data.extend_from_slice(&[0xFF; 200]);
|
||||
fs::write(&seg_path, &data).unwrap();
|
||||
|
||||
// Recovery should truncate the garbage
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
// Verify committed data is still intact
|
||||
for i in 0..10u64 {
|
||||
let result = engine.read("alice", &make_key(i)).unwrap();
|
||||
assert!(result.is_some(), "committed key {} should survive tail truncation", i);
|
||||
let result = engine.get(&make_key(i)).unwrap();
|
||||
assert!(
|
||||
result.is_some(),
|
||||
"committed key {} should survive tail truncation",
|
||||
i
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify file was actually truncated
|
||||
let truncated_len = fs::metadata(&seg_path).unwrap().len();
|
||||
assert!(truncated_len <= orig_len as u64, "garbage should have been truncated");
|
||||
assert!(
|
||||
truncated_len <= orig_len as u64,
|
||||
"garbage should have been truncated"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -274,22 +261,17 @@ fn test_consistency_corrupt_magic_truncated_on_recovery() {
|
||||
fn test_isolation_reader_sees_snapshot_not_partial_write() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
// Pre-populate a known key
|
||||
let original_value = make_value(4096);
|
||||
let key = make_key(100);
|
||||
engine
|
||||
.write("alice", key, &original_value, Codec::Zstd)
|
||||
.unwrap();
|
||||
engine.put(key, &original_value, Codec::Zstd).unwrap();
|
||||
|
||||
let running = Arc::new(AtomicBool::new(true));
|
||||
let writer_done = Arc::new(AtomicBool::new(false));
|
||||
|
||||
// Spawn a writer that continuously overwrites the same key
|
||||
let writer_engine = engine.clone();
|
||||
let writer_running = running.clone();
|
||||
let writer_done_flag = writer_done.clone();
|
||||
let writer_key = key;
|
||||
|
||||
let writer = thread::spawn(move || {
|
||||
@@ -299,11 +281,10 @@ fn test_isolation_reader_sees_snapshot_not_partial_write() {
|
||||
}
|
||||
let val = make_value(4096 + (i as usize % 100));
|
||||
writer_engine
|
||||
.write("alice", writer_key, &val, Codec::Zstd)
|
||||
.put(writer_key, &val, Codec::Zstd)
|
||||
.unwrap();
|
||||
thread::yield_now();
|
||||
}
|
||||
writer_done_flag.store(true, Ordering::SeqCst);
|
||||
});
|
||||
|
||||
// Concurrent reader: reads should never panic or hang
|
||||
@@ -315,11 +296,10 @@ fn test_isolation_reader_sees_snapshot_not_partial_write() {
|
||||
if !reader_running.load(Ordering::Relaxed) && reads > 0 {
|
||||
break;
|
||||
}
|
||||
let result = reader_engine.read("alice", &key);
|
||||
let result = reader_engine.get(&key);
|
||||
match result {
|
||||
Ok(Some(_)) | Ok(None) => {} // OK
|
||||
Ok(Some(_)) | Ok(None) => {}
|
||||
Err(e) => {
|
||||
// Accept transient errors but report them
|
||||
eprintln!("reader saw error: {:?}", e);
|
||||
}
|
||||
}
|
||||
@@ -332,8 +312,7 @@ fn test_isolation_reader_sees_snapshot_not_partial_write() {
|
||||
running.store(false, Ordering::SeqCst);
|
||||
writer.join().unwrap();
|
||||
|
||||
// Final read should see the last committed value (not partial)
|
||||
let final_result = engine.read("alice", &key).unwrap();
|
||||
let final_result = engine.get(&key).unwrap();
|
||||
assert!(final_result.is_some(), "final read should find a value");
|
||||
}
|
||||
|
||||
@@ -347,21 +326,18 @@ fn test_crash_during_gc_leaves_data_intact() {
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let value = make_value(500_000); // 500 KB each
|
||||
// Write enough entries and delete some to create GC candidate
|
||||
let value = make_value(500_000);
|
||||
for i in 0..500u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &value, Codec::None)
|
||||
.put(make_key(i), &value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
// Delete ~40%
|
||||
for i in (0..500u64).step_by(5) {
|
||||
engine.delete("alice", &make_key(i)).unwrap();
|
||||
engine.delete(&make_key(i)).unwrap();
|
||||
}
|
||||
// Single GC run (may or may not trigger)
|
||||
let _ = engine.gc("alice");
|
||||
let _ = engine.gc();
|
||||
} // crash after GC
|
||||
|
||||
// All non-deleted entries must still be readable
|
||||
@@ -370,9 +346,8 @@ fn test_crash_during_gc_leaves_data_intact() {
|
||||
let value = make_value(500_000);
|
||||
for i in 0..500u64 {
|
||||
let key = make_key(i);
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
let result = engine.get(&key).unwrap();
|
||||
if i % 5 == 0 {
|
||||
// Deleted keys
|
||||
assert_eq!(result, None, "key {} should be deleted", i);
|
||||
} else {
|
||||
assert_eq!(
|
||||
@@ -401,11 +376,8 @@ fn test_multiple_crash_reopen_cycles() {
|
||||
// Populate and crash
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
for i in 0..50u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
engine.put(make_key(i), &value, Codec::Zstd).unwrap();
|
||||
alive.insert(i);
|
||||
}
|
||||
}
|
||||
@@ -414,11 +386,11 @@ fn test_multiple_crash_reopen_cycles() {
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for &k in &alive {
|
||||
assert!(engine.read("alice", &make_key(k)).unwrap().is_some());
|
||||
assert!(engine.get(&make_key(k)).unwrap().is_some());
|
||||
}
|
||||
for i in 100..150u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &value, Codec::Zstd)
|
||||
.put(make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
alive.insert(i);
|
||||
}
|
||||
@@ -428,10 +400,10 @@ fn test_multiple_crash_reopen_cycles() {
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for &k in &alive {
|
||||
assert!(engine.read("alice", &make_key(k)).unwrap().is_some());
|
||||
assert!(engine.get(&make_key(k)).unwrap().is_some());
|
||||
}
|
||||
for i in 0..10u64 {
|
||||
engine.delete("alice", &make_key(i)).unwrap();
|
||||
engine.delete(&make_key(i)).unwrap();
|
||||
alive.remove(&i);
|
||||
}
|
||||
}
|
||||
@@ -440,44 +412,88 @@ fn test_multiple_crash_reopen_cycles() {
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for &k in &alive {
|
||||
assert!(engine.read("alice", &make_key(k)).unwrap().is_some(),
|
||||
"key {} should exist", k);
|
||||
assert!(
|
||||
engine.get(&make_key(k)).unwrap().is_some(),
|
||||
"key {} should exist",
|
||||
k
|
||||
);
|
||||
}
|
||||
for i in 0..10u64 {
|
||||
assert_eq!(engine.read("alice", &make_key(i)).unwrap(), None,
|
||||
"key {} should be deleted", i);
|
||||
assert_eq!(
|
||||
engine.get(&make_key(i)).unwrap(),
|
||||
None,
|
||||
"key {} should be deleted",
|
||||
i
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Account-level isolation
|
||||
// 7. Global dedup: same content stored once
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_account_isolation_crash_one_account_does_not_affect_others() {
|
||||
fn test_global_dedup_after_crash() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let key = make_key(42);
|
||||
let value = make_value(8192);
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine.create_account("bob").unwrap();
|
||||
|
||||
engine
|
||||
.write("alice", make_key(1), &make_value(4096), Codec::Zstd)
|
||||
.unwrap();
|
||||
engine
|
||||
.write("bob", make_key(1), &make_value(8192), Codec::Zstd)
|
||||
.unwrap();
|
||||
// Write same key twice (simulating two sources with same content)
|
||||
engine.put(key, &value, Codec::Zstd).unwrap();
|
||||
engine.put(key, &value, Codec::Zstd).unwrap();
|
||||
}
|
||||
|
||||
// Delete alice's account dir partially to simulate corruption
|
||||
// Then verify bob is intact
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
// Bob should be fine
|
||||
let result = engine.read("bob", &make_key(1)).unwrap();
|
||||
assert!(result.is_some(), "bob should be unaffected");
|
||||
let result = engine.get(&key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 8. Recovery + mmap consistency: entries re-indexed by recovery are readable
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_recovery_reloads_bucket_mmaps() {
|
||||
use bichon_blob::meta::GlobalMeta;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let value = make_value(8192);
|
||||
let n = 50u64;
|
||||
|
||||
// Phase 1: write data with clean shutdown, then corrupt meta to force
|
||||
// re-indexing on next open (simulates crash where mark_indexed didn't run).
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for i in 0..n {
|
||||
engine.put(make_key(i), &value, Codec::None).unwrap();
|
||||
}
|
||||
} // clean shutdown — everything is indexed and fsynced
|
||||
|
||||
// Corrupt meta: zero out indexed_up_to_offset so recovery re-scans.
|
||||
let mut meta = GlobalMeta::load(dir.path()).expect("failed to load meta");
|
||||
for seg in meta.segments.values_mut() {
|
||||
seg.indexed_up_to_offset = 0;
|
||||
}
|
||||
meta.save(dir.path()).expect("failed to save corrupted meta");
|
||||
|
||||
// Phase 2: reopen — recovery must re-scan the segment and append to bucket
|
||||
// files. After the fix, reload_all() ensures the mmaps include recovered data.
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for i in 0..n {
|
||||
let result = engine.get(&make_key(i)).unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
Some(value.clone()),
|
||||
"key {} should be readable after recovery reload",
|
||||
i
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -485,8 +501,8 @@ fn test_account_isolation_crash_one_account_does_not_affect_others() {
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn find_first_segment(store_root: &Path, account: &str) -> std::path::PathBuf {
|
||||
let seg_dir = store_root.join("accounts").join(account).join("segments");
|
||||
fn find_first_segment(store_root: &Path) -> std::path::PathBuf {
|
||||
let seg_dir = store_root.join("segments");
|
||||
for entry in fs::read_dir(&seg_dir).unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
|
||||
380
crates/blob/tests/fuzz_test.rs
Normal file
380
crates/blob/tests/fuzz_test.rs
Normal file
@@ -0,0 +1,380 @@
|
||||
/// Fuzz-style tests: randomized operation sequences, corruption injection,
|
||||
/// and crash-recovery stress testing.
|
||||
use bichon_blob::{Codec, Config, Engine};
|
||||
use rand::RngExt;
|
||||
use std::collections::HashMap;
|
||||
use tempfile::TempDir;
|
||||
|
||||
/// Number of iterations for each randomized test.
|
||||
const FUZZ_OPS: usize = 2000;
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
fn make_key(i: u64) -> [u8; 32] {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..8].copy_from_slice(&i.to_le_bytes());
|
||||
key
|
||||
}
|
||||
|
||||
fn make_value(rng: &mut impl RngExt) -> Vec<u8> {
|
||||
let size = match rng.random_range(0..100) {
|
||||
0..=4 => rng.random_range(0..64), // tiny
|
||||
5..=9 => 0, // empty
|
||||
10..=79 => rng.random_range(64..4096), // small
|
||||
80..=89 => rng.random_range(4096..65536), // medium
|
||||
90..=94 => rng.random_range(65536..500_000), // large
|
||||
_ => rng.random_range(500_000..2_000_000), // xl (near max)
|
||||
};
|
||||
let mut v = vec![0u8; size];
|
||||
rng.fill(&mut v[..]);
|
||||
v
|
||||
}
|
||||
|
||||
fn random_codec(rng: &mut impl RngExt) -> Codec {
|
||||
match rng.random_range(0..4) {
|
||||
0 => Codec::None,
|
||||
1 => Codec::Zstd,
|
||||
_ => Codec::Lz4,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Fuzz: random operation sequence ─────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn fuzz_random_ops() {
|
||||
let mut rng = rand::rng();
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut config = Config::default();
|
||||
config.flush_interval_secs = 0; // manual flush only
|
||||
config.gc_interval_secs = 0;
|
||||
|
||||
let engine = Engine::open(dir.path(), config).unwrap();
|
||||
|
||||
// Oracle: track expected values in memory
|
||||
let mut oracle: HashMap<[u8; 32], Vec<u8>> = HashMap::new();
|
||||
let mut next_key = 0u64;
|
||||
|
||||
for _ in 0..FUZZ_OPS {
|
||||
match rng.random_range(0..100) {
|
||||
// 45%: put new key
|
||||
0..=44 => {
|
||||
let key = make_key(next_key);
|
||||
next_key += 1;
|
||||
let value = make_value(&mut rng);
|
||||
let codec = random_codec(&mut rng);
|
||||
engine.put(key, &value, codec).unwrap();
|
||||
oracle.insert(key, value);
|
||||
}
|
||||
// 20%: put overwrite existing key
|
||||
45..=64 => {
|
||||
if oracle.is_empty() { continue; }
|
||||
let idx = rng.random_range(0..oracle.len());
|
||||
let key = *oracle.iter().nth(idx).unwrap().0;
|
||||
let value = make_value(&mut rng);
|
||||
let codec = random_codec(&mut rng);
|
||||
engine.put(key, &value, codec).unwrap();
|
||||
oracle.insert(key, value);
|
||||
}
|
||||
// 15%: read and verify
|
||||
65..=79 => {
|
||||
if oracle.is_empty() { continue; }
|
||||
let idx = rng.random_range(0..oracle.len());
|
||||
let key = *oracle.iter().nth(idx).unwrap().0;
|
||||
let expected = oracle.get(&key).unwrap();
|
||||
let got = engine.get(&key).unwrap();
|
||||
assert_eq!(got.as_ref(), Some(expected), "key mismatch on read");
|
||||
}
|
||||
// 10%: delete
|
||||
80..=89 => {
|
||||
if oracle.is_empty() { continue; }
|
||||
let idx = rng.random_range(0..oracle.len());
|
||||
let key = *oracle.iter().nth(idx).unwrap().0;
|
||||
engine.delete(&key).unwrap();
|
||||
oracle.remove(&key);
|
||||
let got = engine.get(&key).unwrap();
|
||||
assert_eq!(got, None, "deleted key should return None");
|
||||
}
|
||||
// 5%: read non-existent key
|
||||
90..=94 => {
|
||||
let key = make_key(next_key + rng.random_range(1000u64..10000));
|
||||
let got = engine.get(&key).unwrap();
|
||||
assert_eq!(got, None, "non-existent key should return None");
|
||||
}
|
||||
// 5%: flush
|
||||
_ => {
|
||||
engine.flush().unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Final verification: all oracle entries must match
|
||||
for (key, expected) in &oracle {
|
||||
let got = engine.get(key).unwrap();
|
||||
assert_eq!(got.as_ref(), Some(expected), "final verification: key mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
// ── Fuzz: crash + reopen cycle ──────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn fuzz_crash_reopen_cycles() {
|
||||
let mut rng = rand::rng();
|
||||
let dir = TempDir::new().unwrap();
|
||||
let dir_path = dir.path().to_path_buf();
|
||||
|
||||
let mut oracle: HashMap<[u8; 32], Vec<u8>> = HashMap::new();
|
||||
let mut next_key = 0u64;
|
||||
let cycles = 20;
|
||||
|
||||
for _cycle in 0..cycles {
|
||||
// Open database
|
||||
let mut config = Config::default();
|
||||
config.flush_interval_secs = 0;
|
||||
config.gc_interval_secs = 0;
|
||||
let engine = Engine::open(&dir_path, config).unwrap();
|
||||
|
||||
// Do some work
|
||||
let ops = rng.random_range(50..200);
|
||||
for _ in 0..ops {
|
||||
match rng.random_range(0..100) {
|
||||
0..=50 => {
|
||||
let key = make_key(next_key);
|
||||
next_key += 1;
|
||||
let value = make_value(&mut rng);
|
||||
if engine.put(key, &value, Codec::Zstd).is_ok() {
|
||||
oracle.insert(key, value);
|
||||
}
|
||||
}
|
||||
51..=65 => {
|
||||
if oracle.is_empty() { continue; }
|
||||
let idx = rng.random_range(0..oracle.len());
|
||||
let key = *oracle.iter().nth(idx).unwrap().0;
|
||||
engine.delete(&key).unwrap();
|
||||
oracle.remove(&key);
|
||||
}
|
||||
66..=85 => {
|
||||
if oracle.is_empty() { continue; }
|
||||
let idx = rng.random_range(0..oracle.len());
|
||||
let key = *oracle.iter().nth(idx).unwrap().0;
|
||||
let expected = oracle.get(&key).unwrap();
|
||||
if let Ok(Some(got)) = engine.get(&key) {
|
||||
assert_eq!(&got, expected, "pre-crash read mismatch");
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
let _ = engine.flush();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Simulate crash: drop without shutdown
|
||||
drop(engine);
|
||||
}
|
||||
|
||||
// Final reopen: all oracle entries must be intact
|
||||
let config = Config::default();
|
||||
let engine = Engine::open(&dir_path, config).unwrap();
|
||||
for (key, expected) in &oracle {
|
||||
let got = engine.get(key).unwrap();
|
||||
assert_eq!(got.as_ref(), Some(expected), "after {} crash cycles", cycles);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Fuzz: batch operations ──────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn fuzz_batch_ops() {
|
||||
let mut rng = rand::rng();
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
let config = Config::default();
|
||||
let engine = Engine::open(dir.path(), config).unwrap();
|
||||
|
||||
let mut oracle: HashMap<[u8; 32], Vec<u8>> = HashMap::new();
|
||||
let mut next_key = 0u64;
|
||||
|
||||
for _ in 0..200 {
|
||||
match rng.random_range(0..100) {
|
||||
// 50%: batch write
|
||||
0..=49 => {
|
||||
let batch_size = rng.random_range(1..30);
|
||||
let entries: Vec<_> = (0..batch_size)
|
||||
.map(|_| {
|
||||
let key = make_key(next_key);
|
||||
next_key += 1;
|
||||
let value = make_value(&mut rng);
|
||||
oracle.insert(key, value.clone());
|
||||
(key, value, Codec::Zstd)
|
||||
})
|
||||
.collect();
|
||||
engine.put_batch(&entries).unwrap();
|
||||
}
|
||||
// 30%: verify random subset
|
||||
50..=79 => {
|
||||
if oracle.is_empty() { continue; }
|
||||
let n = rng.random_range(1..=20.min(oracle.len()));
|
||||
for _ in 0..n {
|
||||
let idx = rng.random_range(0..oracle.len());
|
||||
let (key, expected) = oracle.iter().nth(idx).unwrap();
|
||||
let got = engine.get(key).unwrap();
|
||||
assert_eq!(got.as_ref(), Some(expected));
|
||||
}
|
||||
}
|
||||
// 20%: batch delete
|
||||
_ => {
|
||||
if oracle.is_empty() { continue; }
|
||||
let n = rng.random_range(1..=20.min(oracle.len()));
|
||||
let keys: Vec<[u8; 32]> = (0..n)
|
||||
.map(|_| {
|
||||
let idx = rng.random_range(0..oracle.len());
|
||||
let key = *oracle.iter().nth(idx).unwrap().0;
|
||||
oracle.remove(&key);
|
||||
key
|
||||
})
|
||||
.collect();
|
||||
engine.delete_batch(&keys).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (key, expected) in &oracle {
|
||||
let got = engine.get(key).unwrap();
|
||||
assert_eq!(got.as_ref(), Some(expected));
|
||||
}
|
||||
}
|
||||
|
||||
// ── Fuzz: GC stress ─────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn fuzz_gc_stress() {
|
||||
let mut rng = rand::rng();
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
let mut config = Config::default();
|
||||
config.gc_deleted_ratio = 0.1; // aggressive GC trigger
|
||||
let engine = Engine::open(dir.path(), config).unwrap();
|
||||
|
||||
let mut oracle: HashMap<[u8; 32], Vec<u8>> = HashMap::new();
|
||||
let mut next_key = 0u64;
|
||||
|
||||
for round in 0..10 {
|
||||
// Write a batch of keys
|
||||
let n = rng.random_range(50..150);
|
||||
let mut round_keys: Vec<[u8; 32]> = Vec::new();
|
||||
let value_size = rng.random_range(100..10000);
|
||||
let value: Vec<u8> = (0..value_size).map(|_| rng.random::<u8>()).collect();
|
||||
|
||||
for _ in 0..n {
|
||||
let key = make_key(next_key);
|
||||
next_key += 1;
|
||||
engine.put(key, &value, Codec::None).unwrap();
|
||||
oracle.insert(key, value.clone());
|
||||
round_keys.push(key);
|
||||
}
|
||||
|
||||
// Delete some fraction
|
||||
let delete_frac: f64 = rng.random_range(0.2..0.8);
|
||||
let delete_count = (round_keys.len() as f64 * delete_frac) as usize;
|
||||
for _ in 0..delete_count {
|
||||
let idx = rng.random_range(0..round_keys.len());
|
||||
let key = round_keys.swap_remove(idx);
|
||||
engine.delete(&key).unwrap();
|
||||
oracle.remove(&key);
|
||||
}
|
||||
|
||||
// Seal and run GC
|
||||
engine.seal_active_segment().unwrap();
|
||||
let _ = engine.gc().unwrap();
|
||||
|
||||
// Verify all remaining oracle entries
|
||||
for (key, expected) in &oracle {
|
||||
let got = engine.get(key).unwrap();
|
||||
assert_eq!(
|
||||
got.as_ref(),
|
||||
Some(expected),
|
||||
"GC round {}: key mismatch",
|
||||
round
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Fuzz: bitflip corruption resilience ─────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn fuzz_corruption_resilience() {
|
||||
let mut rng = rand::rng();
|
||||
let dir = TempDir::new().unwrap();
|
||||
let dir_path = dir.path().to_path_buf();
|
||||
|
||||
let config = Config::default();
|
||||
let engine = Engine::open(&dir_path, config).unwrap();
|
||||
|
||||
// Write known data
|
||||
let mut good_keys: Vec<[u8; 32]> = Vec::new();
|
||||
let n = 100u64;
|
||||
for i in 0..n {
|
||||
let key = make_key(i);
|
||||
let value = vec![i as u8; 2048];
|
||||
engine.put(key, &value, Codec::None).unwrap();
|
||||
good_keys.push(key);
|
||||
}
|
||||
engine.flush().unwrap();
|
||||
|
||||
// Seal so segment file is durable on disk
|
||||
engine.seal_active_segment().unwrap();
|
||||
engine.shutdown().unwrap();
|
||||
drop(engine);
|
||||
|
||||
// Find segment files and corrupt random bytes
|
||||
let seg_dir = dir_path.join("segments");
|
||||
let mut seg_files: Vec<_> = std::fs::read_dir(&seg_dir)
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| {
|
||||
e.file_name()
|
||||
.to_string_lossy()
|
||||
.ends_with(".seg")
|
||||
})
|
||||
.collect();
|
||||
seg_files.sort_by_key(|e| e.file_name());
|
||||
|
||||
// Corrupt 3 random bytes in the first segment
|
||||
if let Some(seg) = seg_files.first() {
|
||||
let path = seg.path();
|
||||
let mut data = std::fs::read(&path).unwrap();
|
||||
if data.len() > 100 {
|
||||
for _ in 0..3 {
|
||||
let pos = rng.random_range(50..data.len());
|
||||
data[pos] ^= 0xFF; // flip all bits
|
||||
}
|
||||
std::fs::write(&path, &data).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
// Reopen: recovery must succeed (not panic), even if some keys are lost
|
||||
let config = Config::default();
|
||||
let engine = Engine::open(&dir_path, config).unwrap();
|
||||
|
||||
// At least some uncorrupted keys should still be readable
|
||||
let mut readable = 0;
|
||||
let mut corrupted = 0;
|
||||
for key in &good_keys {
|
||||
match engine.get(key) {
|
||||
Ok(Some(_)) => readable += 1,
|
||||
Ok(None) => { /* key might be lost due to corruption */ }
|
||||
Err(_) => corrupted += 1,
|
||||
}
|
||||
}
|
||||
// The vast majority should still be ok (corruption only hit 3 bytes in one segment)
|
||||
assert!(
|
||||
readable + corrupted > 0,
|
||||
"at least some outcomes should be observable"
|
||||
);
|
||||
assert!(
|
||||
readable > n as usize / 2,
|
||||
"majority of keys should survive localized corruption ({} of {})",
|
||||
readable,
|
||||
n
|
||||
);
|
||||
}
|
||||
@@ -1,33 +1,17 @@
|
||||
use bichon_blob::{Codec, Config, Engine};
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_create_and_list_accounts() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
|
||||
engine.create_account("alice").unwrap();
|
||||
engine.create_account("bob").unwrap();
|
||||
|
||||
let accounts = engine.list_accounts();
|
||||
assert!(accounts.contains(&"alice".to_string()));
|
||||
assert!(accounts.contains(&"bob".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_write_and_read() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xAA; 32];
|
||||
let value = b"Hello, this is a test email!".to_vec();
|
||||
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
engine.put(key, &value, Codec::Zstd).unwrap();
|
||||
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
let result = engine.get(&key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
|
||||
@@ -35,10 +19,9 @@ fn test_write_and_read() {
|
||||
fn test_read_missing_key() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xFF; 32];
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
let result = engine.get(&key).unwrap();
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
|
||||
@@ -46,45 +29,43 @@ fn test_read_missing_key() {
|
||||
fn test_delete() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xBB; 32];
|
||||
let value = b"Some email content".to_vec();
|
||||
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
engine.delete("alice", &key).unwrap();
|
||||
engine.put(key, &value, Codec::Zstd).unwrap();
|
||||
engine.delete(&key).unwrap();
|
||||
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
let result = engine.get(&key).unwrap();
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_account() {
|
||||
fn test_exists() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine.delete_account("alice").unwrap();
|
||||
|
||||
let accounts = engine.list_accounts();
|
||||
assert!(!accounts.contains(&"alice".to_string()));
|
||||
let key = [0xCC; 32];
|
||||
assert!(!engine.exists(&key).unwrap());
|
||||
|
||||
engine.put(key, b"data", Codec::None).unwrap();
|
||||
assert!(engine.exists(&key).unwrap());
|
||||
|
||||
engine.delete(&key).unwrap();
|
||||
assert!(!engine.exists(&key).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_small_value_not_compressed() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xCC; 32];
|
||||
let value = b"hi"; // Smaller than 4KB threshold
|
||||
|
||||
engine
|
||||
.write("alice", key, value, Codec::Zstd)
|
||||
.unwrap();
|
||||
engine.put(key, value, Codec::Zstd).unwrap();
|
||||
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
let result = engine.get(&key).unwrap();
|
||||
assert_eq!(result, Some(value.to_vec()));
|
||||
}
|
||||
|
||||
@@ -92,16 +73,13 @@ fn test_small_value_not_compressed() {
|
||||
fn test_large_value() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xDD; 32];
|
||||
let value = vec![b'X'; 100_000]; // 100KB
|
||||
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
engine.put(key, &value, Codec::Zstd).unwrap();
|
||||
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
let result = engine.get(&key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
|
||||
@@ -109,22 +87,19 @@ fn test_large_value() {
|
||||
fn test_multiple_keys() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let n = 100;
|
||||
for i in 0..n {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let value = format!("email number {}", i).into_bytes();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
engine.put(key, &value, Codec::Zstd).unwrap();
|
||||
}
|
||||
|
||||
for i in 0..n {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
let result = engine.get(&key).unwrap();
|
||||
assert_eq!(result, Some(format!("email number {}", i).into_bytes()));
|
||||
}
|
||||
}
|
||||
@@ -133,7 +108,6 @@ fn test_multiple_keys() {
|
||||
fn test_gc() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
// Write many entries
|
||||
let value = vec![b'Y'; 5000];
|
||||
@@ -142,26 +116,24 @@ fn test_gc() {
|
||||
for i in 0..n {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
engine
|
||||
.write("alice", key, &value, Codec::None)
|
||||
.unwrap();
|
||||
engine.put(key, &value, Codec::None).unwrap();
|
||||
}
|
||||
|
||||
// Delete even-numbered keys
|
||||
for i in (0..n).step_by(2) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
engine.delete("alice", &key).unwrap();
|
||||
engine.delete(&key).unwrap();
|
||||
}
|
||||
|
||||
// Run GC
|
||||
let _result = engine.gc("alice").unwrap();
|
||||
let _result = engine.gc().unwrap();
|
||||
|
||||
// Verify remaining keys still readable
|
||||
for i in (1..n).step_by(2) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
let result = engine.get(&key).unwrap();
|
||||
assert_eq!(result, Some(value.clone()));
|
||||
}
|
||||
|
||||
@@ -169,7 +141,7 @@ fn test_gc() {
|
||||
for i in (0..n).step_by(2) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
let result = engine.get(&key).unwrap();
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
}
|
||||
@@ -182,16 +154,13 @@ fn test_reopen_persistence() {
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
engine.put(key, &value, Codec::Zstd).unwrap();
|
||||
}
|
||||
|
||||
// Reopen
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
let result = engine.get(&key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
}
|
||||
@@ -200,21 +169,18 @@ fn test_reopen_persistence() {
|
||||
fn test_stats() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
engine
|
||||
.write("alice", [1u8; 32], b"hello", Codec::None)
|
||||
.unwrap();
|
||||
engine.put([1u8; 32], b"hello", Codec::None).unwrap();
|
||||
|
||||
let stats = engine.stats("alice").unwrap();
|
||||
let stats = engine.stats().unwrap();
|
||||
assert!(stats.total_bytes > 0);
|
||||
assert!(stats.total_keys > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_batch_write() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let n = 50;
|
||||
let entries: Vec<_> = (0..n)
|
||||
@@ -226,10 +192,10 @@ fn test_batch_write() {
|
||||
})
|
||||
.collect();
|
||||
|
||||
engine.write_batch("alice", &entries).unwrap();
|
||||
engine.put_batch(&entries).unwrap();
|
||||
|
||||
for (key, value, _) in &entries {
|
||||
let result = engine.read("alice", key).unwrap();
|
||||
let result = engine.get(key).unwrap();
|
||||
assert_eq!(result.as_ref(), Some(value));
|
||||
}
|
||||
}
|
||||
@@ -247,14 +213,13 @@ fn test_batch_write_persistence() {
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine.write_batch("alice", &entries).unwrap();
|
||||
engine.put_batch(&entries).unwrap();
|
||||
}
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for (key, value, _) in &entries {
|
||||
let result = engine.read("alice", key).unwrap();
|
||||
let result = engine.get(key).unwrap();
|
||||
assert_eq!(result.as_ref(), Some(value));
|
||||
}
|
||||
}
|
||||
@@ -264,7 +229,7 @@ fn test_batch_write_persistence() {
|
||||
fn test_invalid_config_rejected() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut config = Config::default();
|
||||
config.lru_bucket_count = 0;
|
||||
config.compression_level = -1;
|
||||
assert!(Engine::open(dir.path(), config).is_err());
|
||||
|
||||
let mut config = Config::default();
|
||||
@@ -279,13 +244,14 @@ fn test_concurrent_reads() {
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
// Write some data
|
||||
for i in 0..50u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
engine.write("alice", key, &vec![i as u8; 1024], Codec::None).unwrap();
|
||||
engine
|
||||
.put(key, &vec![i as u8; 1024], Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Spawn 4 threads, each reading a different subset
|
||||
@@ -296,7 +262,7 @@ fn test_concurrent_reads() {
|
||||
for i in (t * 12)..((t + 1) * 12) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let read = engine.read("alice", &key).unwrap();
|
||||
let read = engine.get(&key).unwrap();
|
||||
assert!(read.is_some(), "key {} should exist", i);
|
||||
}
|
||||
}));
|
||||
@@ -307,43 +273,25 @@ fn test_concurrent_reads() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_concurrent_writes_different_accounts() {
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
|
||||
fn test_global_dedup() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
|
||||
for name in &["alice", "bob", "carol"] {
|
||||
engine.create_account(name).unwrap();
|
||||
}
|
||||
let key = [0x42; 32];
|
||||
let value = b"same content across what would be accounts".to_vec();
|
||||
|
||||
let mut handles = vec![];
|
||||
for (t, name) in ["alice", "bob", "carol"].iter().enumerate() {
|
||||
let engine = engine.clone();
|
||||
let account_name = name.to_string();
|
||||
handles.push(thread::spawn(move || {
|
||||
for i in 0..20 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&((t * 100 + i) as u32).to_le_bytes());
|
||||
let value = vec![(t * 100 + i) as u8; 512];
|
||||
engine.write(&account_name, key, &value, Codec::None).unwrap();
|
||||
}
|
||||
}));
|
||||
}
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
// Write same key twice (simulating two accounts ingesting the same email)
|
||||
engine.put(key, &value, Codec::Zstd).unwrap();
|
||||
engine.put(key, &value, Codec::Zstd).unwrap();
|
||||
|
||||
// Verify all writes persisted
|
||||
for (t, name) in ["alice", "bob", "carol"].iter().enumerate() {
|
||||
for i in 0..20 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&((t * 100 + i) as u32).to_le_bytes());
|
||||
let read = engine.read(name, &key).unwrap();
|
||||
assert!(read.is_some(), "account {} key {} should exist", name, i);
|
||||
}
|
||||
}
|
||||
// Should still be readable
|
||||
let result = engine.get(&key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
|
||||
// Stats should reflect dedup (not double count)
|
||||
let stats = engine.stats().unwrap();
|
||||
// The key appears once in the bucket store
|
||||
assert!(stats.total_keys > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -354,144 +302,144 @@ fn test_crash_recovery() {
|
||||
// Phase 1: write data, then drop without shutdown (simulates crash)
|
||||
{
|
||||
let engine = Engine::open(&dir_path, Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
for i in 0..50u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
engine.write("alice", key, &vec![i as u8; 512], Codec::None).unwrap();
|
||||
engine
|
||||
.put(key, &vec![i as u8; 512], Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
// Engine dropped here without calling shutdown()
|
||||
}
|
||||
|
||||
// Phase 2: reopen — recovery should run, data should be intact
|
||||
// Phase 2: reopen - recovery should run, data should be intact
|
||||
let engine = Engine::open(&dir_path, Config::default()).unwrap();
|
||||
let stats = engine.stats("alice").unwrap();
|
||||
let stats = engine.stats().unwrap();
|
||||
assert!(stats.total_keys > 0, "recovery should preserve data");
|
||||
|
||||
// Verify reads work
|
||||
for i in 0..50u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
let read = engine.read("alice", &key).unwrap();
|
||||
let read = engine.get(&key).unwrap();
|
||||
assert!(read.is_some(), "key {} should survive crash recovery", i);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_meta_bin_durability() {
|
||||
// Verify meta.bin has valid CRC and can be read after a write cycle.
|
||||
let dir = TempDir::new().unwrap();
|
||||
let dir_path = dir.path().to_path_buf();
|
||||
|
||||
{
|
||||
let engine = Engine::open(&dir_path, Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0x42u8; 32];
|
||||
engine.write("alice", key, b"durable", Codec::None).unwrap();
|
||||
engine.put(key, b"durable", Codec::None).unwrap();
|
||||
}
|
||||
// Engine dropped → shutdown() called → meta saved via write_bin (with fsync)
|
||||
// Engine dropped -> shutdown() called -> meta saved
|
||||
|
||||
// Verify meta.bin exists and has valid CRC
|
||||
let meta_path = dir_path
|
||||
.join("accounts")
|
||||
.join("alice")
|
||||
.join("meta.bin");
|
||||
// Verify meta.bin exists
|
||||
let meta_path = dir_path.join("meta.bin");
|
||||
assert!(meta_path.exists(), "meta.bin should exist after clean shutdown");
|
||||
|
||||
let data = std::fs::read(&meta_path).unwrap();
|
||||
assert!(data.len() >= 8, "meta.bin should have at least 8 bytes (crc + version)");
|
||||
assert!(
|
||||
data.len() >= 8,
|
||||
"meta.bin should have at least 8 bytes"
|
||||
);
|
||||
|
||||
let stored_crc = u32::from_le_bytes(data[0..4].try_into().unwrap());
|
||||
assert_ne!(stored_crc, 0, "stored CRC should be non-zero");
|
||||
|
||||
// Reopen and verify data is intact
|
||||
let engine = Engine::open(&dir_path, Config::default()).unwrap();
|
||||
let read = engine.read("alice", &[0x42u8; 32]).unwrap();
|
||||
let read = engine.get(&[0x42u8; 32]).unwrap();
|
||||
assert_eq!(read, Some(b"durable".to_vec()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gc_concurrent_with_writes() {
|
||||
// GC should not lose entries that are written concurrently.
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::thread;
|
||||
|
||||
fn test_gc_deletes_empty_segment() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
|
||||
engine.create_account("alice").unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
|
||||
// Pre-fill: write enough to trigger eventual GC
|
||||
let big_value = vec![b'X'; 8192];
|
||||
for i in 0..500u32 {
|
||||
// Write data, seal, then delete everything so the sealed segment
|
||||
// becomes 100% garbage. GC should remove the segment file entirely.
|
||||
let n = 50u64;
|
||||
for i in 0..n {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
engine.write("alice", key, &big_value, Codec::None).unwrap();
|
||||
key[0..8].copy_from_slice(&i.to_le_bytes());
|
||||
engine.put(key, &vec![b'X'; 4096], Codec::None).unwrap();
|
||||
}
|
||||
|
||||
// Delete some to create GC candidates
|
||||
for i in 0..250u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
engine.delete("alice", &key).unwrap();
|
||||
}
|
||||
// Force seal so the segment becomes a GC candidate.
|
||||
let sealed_id = engine.seal_active_segment().unwrap();
|
||||
|
||||
let running = Arc::new(AtomicBool::new(true));
|
||||
let engine_gc = engine.clone();
|
||||
let running_gc = running.clone();
|
||||
|
||||
// Thread 1: run GC in a loop
|
||||
let gc_handle = thread::spawn(move || {
|
||||
while running_gc.load(Ordering::Relaxed) {
|
||||
let _ = engine_gc.gc("alice");
|
||||
thread::sleep(std::time::Duration::from_millis(10));
|
||||
}
|
||||
});
|
||||
|
||||
// Thread 2: keep writing new entries
|
||||
let engine_write = engine.clone();
|
||||
let running_write = running.clone();
|
||||
let write_handle = thread::spawn(move || {
|
||||
let mut counter = 10000u32;
|
||||
while running_write.load(Ordering::Relaxed) {
|
||||
// Delete all keys — the sealed segment is now entirely garbage.
|
||||
let keys: Vec<[u8; 32]> = (0..n)
|
||||
.map(|i| {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&counter.to_le_bytes());
|
||||
engine_write
|
||||
.write("alice", key, &vec![counter as u8; 256], Codec::None)
|
||||
.unwrap();
|
||||
counter += 1;
|
||||
}
|
||||
counter
|
||||
});
|
||||
key[0..8].copy_from_slice(&i.to_le_bytes());
|
||||
key
|
||||
})
|
||||
.collect();
|
||||
engine.delete_batch(&keys).unwrap();
|
||||
|
||||
// Let them race for a bit
|
||||
thread::sleep(std::time::Duration::from_millis(500));
|
||||
running.store(false, Ordering::Relaxed);
|
||||
// Run GC — this should empty and then delete the sealed segment.
|
||||
let result = engine.gc().unwrap();
|
||||
assert!(result.is_some(), "GC should have found a candidate");
|
||||
let stats = result.unwrap();
|
||||
assert_eq!(stats.segment_id, sealed_id);
|
||||
assert_eq!(stats.bytes_after, 0);
|
||||
|
||||
gc_handle.join().unwrap();
|
||||
let final_counter = write_handle.join().unwrap();
|
||||
// Segment file must be deleted.
|
||||
let seg_path = dir
|
||||
.path()
|
||||
.join("segments")
|
||||
.join(format!("{:08}.seg", sealed_id));
|
||||
assert!(
|
||||
!seg_path.exists(),
|
||||
"emptied segment file should have been deleted, but {:?} exists",
|
||||
seg_path
|
||||
);
|
||||
|
||||
// All written entries must be readable
|
||||
let mut missing = 0;
|
||||
for i in 0..500u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
if engine.read("alice", &key).unwrap().is_none() {
|
||||
// Entries 0..250 were deleted, they should be gone
|
||||
if i >= 250 {
|
||||
missing += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert_eq!(missing, 0, "pre-existing entries should survive concurrent GC");
|
||||
// Subsequent reads / writes must still work (no corruption).
|
||||
let new_key = [0x99u8; 32];
|
||||
engine
|
||||
.put(new_key, b"post-gc data", Codec::None)
|
||||
.unwrap();
|
||||
let result = engine.get(&new_key).unwrap();
|
||||
assert_eq!(result, Some(b"post-gc data".to_vec()));
|
||||
|
||||
// Entries written during the race should be readable
|
||||
for i in 10000..final_counter {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
let read = engine.read("alice", &key).unwrap();
|
||||
assert!(read.is_some(), "concurrently written key {} should exist after GC", i);
|
||||
// Engine stats should be consistent.
|
||||
let stats = engine.stats().unwrap();
|
||||
assert_eq!(stats.total_keys, 1);
|
||||
|
||||
// Shutdown and reopen — persistence must be intact.
|
||||
drop(engine);
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.get(&new_key).unwrap();
|
||||
assert_eq!(result, Some(b"post-gc data".to_vec()));
|
||||
let result = engine.get(&keys[0]).unwrap();
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_file_lock_prevents_concurrent_open() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let _engine1 = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = Engine::open(dir.path(), Config::default());
|
||||
assert!(result.is_err(), "second open on same directory must fail");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_file_lock_released_after_close() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
{
|
||||
let _engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
}
|
||||
// Lock should be released after engine is dropped
|
||||
let engine = Engine::open(dir.path(), Config::default());
|
||||
assert!(engine.is_ok(), "reopen after close must succeed");
|
||||
}
|
||||
|
||||
@@ -63,7 +63,7 @@ bytes.workspace = true
|
||||
mail-send.workspace = true
|
||||
blake3.workspace = true
|
||||
uuid.workspace = true
|
||||
fjall.workspace = true
|
||||
bichon-blob.workspace = true
|
||||
tracing-log.workspace = true
|
||||
tokio-util.workspace = true
|
||||
whichlang = "0.1.1"
|
||||
|
||||
@@ -300,7 +300,6 @@ pub struct Account {
|
||||
pub created_at: i64,
|
||||
pub updated_at: i64,
|
||||
pub created_by: u64, //user id
|
||||
pub use_proxy: Option<u64>,
|
||||
pub use_dangerous: bool,
|
||||
pub pgp_key: Option<String>,
|
||||
pub imap_quota_bytes: Option<u64>,
|
||||
@@ -345,7 +344,6 @@ impl Account {
|
||||
download_interval_min: request.download_interval_min,
|
||||
created_at: utc_now!(),
|
||||
updated_at: utc_now!(),
|
||||
use_proxy: request.use_proxy,
|
||||
use_dangerous: request.use_dangerous,
|
||||
pgp_key: request.pgp_key,
|
||||
created_by: user_id,
|
||||
@@ -657,10 +655,6 @@ impl Account {
|
||||
if let Some(max_email_size_bytes) = request.max_email_size_bytes {
|
||||
new.max_email_size_bytes = Some(max_email_size_bytes);
|
||||
}
|
||||
|
||||
if let Some(use_proxy) = request.use_proxy {
|
||||
new.use_proxy = Some(use_proxy);
|
||||
}
|
||||
}
|
||||
|
||||
if matches!(old.account_type, AccountType::NoSync) {
|
||||
|
||||
@@ -51,7 +51,6 @@ pub struct AccountCreateRequest {
|
||||
)]
|
||||
pub download_batch_size: Option<u32>,
|
||||
pub max_email_size_bytes: Option<u64>,
|
||||
pub use_proxy: Option<u64>,
|
||||
pub use_dangerous: bool,
|
||||
pub pgp_key: Option<String>,
|
||||
pub imap_quota_bytes: Option<u64>,
|
||||
@@ -188,11 +187,6 @@ pub struct AccountUpdateRequest {
|
||||
)]
|
||||
pub download_batch_size: Option<u32>,
|
||||
pub max_email_size_bytes: Option<u64>,
|
||||
/// Optional proxy ID for establishing the connection to external APIs (e.g., Gmail, Outlook).
|
||||
/// - If `None` or not provided, the client will connect directly to the API server.
|
||||
/// - If `Some(proxy_id)`, the client will use the pre-configured proxy with the given ID for API requests.
|
||||
pub use_proxy: Option<u64>,
|
||||
|
||||
pub use_dangerous: Option<bool>,
|
||||
|
||||
pub pgp_key: Option<String>,
|
||||
|
||||
@@ -51,7 +51,6 @@ pub struct AccountResp {
|
||||
pub created_by: u64, //user id
|
||||
pub created_user_name: String,
|
||||
pub created_user_email: String,
|
||||
pub use_proxy: Option<u64>,
|
||||
pub use_dangerous: bool,
|
||||
pub pgp_key: Option<String>,
|
||||
pub imap_quota_bytes: Option<u64>,
|
||||
@@ -90,7 +89,6 @@ impl AccountResp {
|
||||
created_user_email: user
|
||||
.map(|u| u.email.clone())
|
||||
.unwrap_or_else(|| "N/A".to_string()),
|
||||
use_proxy: account.use_proxy,
|
||||
use_dangerous: account.use_dangerous,
|
||||
pgp_key: account.pgp_key,
|
||||
imap_quota_bytes: account.imap_quota_bytes,
|
||||
|
||||
@@ -1,24 +1,24 @@
|
||||
use std::{collections::HashMap, path::PathBuf};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::{
|
||||
error::{code::ErrorCode, BichonResult},
|
||||
migrate::{
|
||||
legacy::schema::SchemaTools,
|
||||
store::{LegacyDirs, NewIndexWriter},
|
||||
},
|
||||
raise_error,
|
||||
settings::cli::SETTINGS,
|
||||
};
|
||||
use tantivy::{
|
||||
collector::TopDocs,
|
||||
columnar::Column,
|
||||
query::TermQuery,
|
||||
schema::{IndexRecordOption, Value},
|
||||
DocAddress, Index, TantivyDocument, Term,
|
||||
};
|
||||
use crate::settings::cli::SETTINGS;
|
||||
|
||||
pub mod legacy;
|
||||
pub mod store;
|
||||
/// Current storage layout version.
|
||||
/// - 1: fjall-based blob storage (post v0.3.7 migration)
|
||||
/// - 2: bichon-blob based storage
|
||||
pub const CURRENT_STORAGE_VERSION: u32 = 2;
|
||||
|
||||
const VERSION_FILE: &str = "STORAGE_VERSION";
|
||||
|
||||
/// Read the storage layout version from `root_dir/STORAGE_VERSION`.
|
||||
pub fn read_storage_version(root_dir: &Path) -> Option<u32> {
|
||||
let content = std::fs::read_to_string(root_dir.join(VERSION_FILE)).ok()?;
|
||||
content.trim().parse().ok()
|
||||
}
|
||||
|
||||
/// Write the storage layout version to `root_dir/STORAGE_VERSION`.
|
||||
pub fn write_storage_version(root_dir: &Path, version: u32) -> std::io::Result<()> {
|
||||
std::fs::write(root_dir.join(VERSION_FILE), format!("{}\n", version))
|
||||
}
|
||||
|
||||
pub fn is_tantivy_index_dir(dir: &PathBuf) -> std::io::Result<bool> {
|
||||
if !dir.exists() || !dir.is_dir() {
|
||||
@@ -47,28 +47,17 @@ pub fn is_tantivy_index_dir(dir: &PathBuf) -> std::io::Result<bool> {
|
||||
Ok(has_meta_json && match_count >= 3)
|
||||
}
|
||||
|
||||
/// Return the number of segments in the legacy EML Tantivy index.
|
||||
/// Each segment can be passed to `do_migrate_segment` for bounded-memory batch migration.
|
||||
pub fn count_eml_segments(legacy: &LegacyDirs) -> BichonResult<usize> {
|
||||
let eml_index = Index::open_in_dir(&legacy.eml_dir)
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
let reader = eml_index
|
||||
.reader()
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
let searcher = reader.searcher();
|
||||
Ok(searcher.segment_readers().len())
|
||||
}
|
||||
|
||||
/// Check whether the data layout is compatible with the current server.
|
||||
/// Returns `false` when legacy data (v0.3.7 or v1.x) is detected and migration is required.
|
||||
pub fn check_data_status() -> std::io::Result<bool> {
|
||||
let root_dir = PathBuf::from(&SETTINGS.bichon_root_dir);
|
||||
|
||||
let new_indices_base = SETTINGS
|
||||
.bichon_index_dir
|
||||
.as_ref()
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| root_dir.clone());
|
||||
let new_indices_path = new_indices_base.join("bichon-indices");
|
||||
// 1. Version file takes precedence
|
||||
if let Some(version) = read_storage_version(&root_dir) {
|
||||
return Ok(version >= CURRENT_STORAGE_VERSION);
|
||||
}
|
||||
|
||||
// 2. No version file — check for existing v1.x-style storage (fjall era)
|
||||
let new_data_base = SETTINGS
|
||||
.bichon_data_dir
|
||||
.as_ref()
|
||||
@@ -76,14 +65,13 @@ pub fn check_data_status() -> std::io::Result<bool> {
|
||||
.unwrap_or_else(|| root_dir.clone());
|
||||
let new_storage_path = new_data_base.join("bichon-storage");
|
||||
|
||||
let has_new_indices = is_tantivy_index_dir(&new_indices_path.join("attachment_metadata"))?
|
||||
&& is_tantivy_index_dir(&new_indices_path.join("mail_metadata"))?;
|
||||
let has_new_storage = is_dir_not_empty(&new_storage_path)?;
|
||||
|
||||
if has_new_indices && has_new_storage {
|
||||
return Ok(true);
|
||||
if is_dir_not_empty(&new_storage_path)? {
|
||||
// Existing v1.x install predates version file — mark it as v1
|
||||
let _ = write_storage_version(&root_dir, 1);
|
||||
return Ok(false); // Needs migration: v1.x → v2.x
|
||||
}
|
||||
|
||||
// 3. Check for legacy v0.3.7 Tantivy layout
|
||||
let legacy_index_root = SETTINGS
|
||||
.bichon_index_dir
|
||||
.as_ref()
|
||||
@@ -99,9 +87,9 @@ pub fn check_data_status() -> std::io::Result<bool> {
|
||||
let has_legacy_data = is_tantivy_index_dir(&legacy_data_root)?;
|
||||
|
||||
if has_legacy_index || has_legacy_data {
|
||||
Ok(false)
|
||||
Ok(false) // Needs migration
|
||||
} else {
|
||||
Ok(true)
|
||||
Ok(true) // Fresh install
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,213 +100,3 @@ fn is_dir_not_empty(path: &PathBuf) -> std::io::Result<bool> {
|
||||
let mut entries = std::fs::read_dir(path)?;
|
||||
Ok(entries.next().is_some())
|
||||
}
|
||||
|
||||
/// Migrate all documents from a single EML segment to the new storage layout.
|
||||
///
|
||||
/// This is the core of the batch migration strategy: each Process B invocation
|
||||
/// handles exactly one EML segment, so peak memory is bounded by that segment's
|
||||
/// size regardless of the total archive size.
|
||||
pub fn do_migrate_segment<F>(
|
||||
batch_size: u32,
|
||||
legacy: LegacyDirs,
|
||||
writer: &mut NewIndexWriter,
|
||||
segment_index: usize,
|
||||
mut on_progress: F,
|
||||
) -> BichonResult<()>
|
||||
where
|
||||
F: FnMut(&str),
|
||||
{
|
||||
// ── open legacy indices ────────────────────────────────────────────
|
||||
let envelope_index = Index::open_in_dir(&legacy.envelope_dir)
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
let eml_index = Index::open_in_dir(&legacy.eml_dir)
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
|
||||
let envelope_reader = envelope_index
|
||||
.reader()
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
let eml_reader = eml_index
|
||||
.reader()
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
|
||||
let envelope_searcher = envelope_reader.searcher();
|
||||
let eml_searcher = eml_reader.searcher();
|
||||
|
||||
let ef = SchemaTools::envelope_fields();
|
||||
let mf = SchemaTools::eml_fields();
|
||||
|
||||
let eml_segments = eml_searcher.segment_readers();
|
||||
let eml_segment = eml_segments.get(segment_index).ok_or_else(|| {
|
||||
raise_error!(
|
||||
format!(
|
||||
"segment index {} out of range ({} segments)",
|
||||
segment_index,
|
||||
eml_segments.len()
|
||||
),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
|
||||
let num_docs = eml_segment.num_docs();
|
||||
if num_docs == 0 {
|
||||
on_progress("TOTAL:0");
|
||||
on_progress("DONE:0:0");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
on_progress(&format!("TOTAL:{}", num_docs));
|
||||
|
||||
let max_doc = eml_segment.max_doc();
|
||||
let ff = eml_segment.fast_fields();
|
||||
let f_id_col: Column<u64> = ff.u64("id").map_err(|e| {
|
||||
raise_error!(
|
||||
format!("failed to open f_id fast field: {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
|
||||
// ── Phase 1: build eid → (uid, internal_date) from envelope, then drop it ──
|
||||
let mut envelope_map: HashMap<u64, (u32, i64)> = HashMap::with_capacity(num_docs as usize);
|
||||
|
||||
let mut env_scanned = 0u32;
|
||||
let mut env_skipped = 0u32;
|
||||
for doc_id in 0..max_doc {
|
||||
if eml_segment.is_deleted(doc_id) {
|
||||
continue;
|
||||
}
|
||||
let eid = f_id_col.values.get_val(doc_id);
|
||||
|
||||
let term = Term::from_field_u64(ef.f_id, eid);
|
||||
let query = TermQuery::new(term, IndexRecordOption::Basic);
|
||||
let hits: Vec<(_, DocAddress)> = envelope_searcher
|
||||
.search(&query, &TopDocs::with_limit(1).order_by_score())
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
|
||||
if let Some((_, addr)) = hits.first() {
|
||||
let env_doc: TantivyDocument = envelope_searcher
|
||||
.doc(*addr)
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
let uid = env_doc
|
||||
.get_first(ef.f_uid)
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(0) as u32;
|
||||
let internal_date = env_doc
|
||||
.get_first(ef.f_internal_date)
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(0);
|
||||
envelope_map.insert(eid, (uid, internal_date));
|
||||
env_scanned += 1;
|
||||
} else {
|
||||
env_skipped += 1;
|
||||
}
|
||||
|
||||
if env_scanned % 10 == 0 {
|
||||
on_progress(&format!(
|
||||
"PHASE1:{}/{} skipped:{}",
|
||||
env_scanned, max_doc, env_skipped
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Free the envelope index before the heavy EML processing.
|
||||
drop(envelope_searcher);
|
||||
drop(envelope_reader);
|
||||
drop(envelope_index);
|
||||
|
||||
// ── Phase 2: process EML docs, streaming one at a time ─────────────
|
||||
let mut total_migrated = 0usize;
|
||||
let mut total_skipped = 0usize;
|
||||
|
||||
// Recreate the StoreReader periodically to bound any internal caches.
|
||||
//const CHUNK_SIZE: u32 = 3000;
|
||||
let mut chunk_start = 0u32;
|
||||
|
||||
while chunk_start < max_doc {
|
||||
let chunk_end = (chunk_start + batch_size).min(max_doc);
|
||||
let store_reader = eml_segment
|
||||
.get_store_reader(2)
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
|
||||
for doc_id in chunk_start..chunk_end {
|
||||
if eml_segment.is_deleted(doc_id) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let eid = f_id_col.values.get_val(doc_id);
|
||||
|
||||
let (uid, internal_date) = match envelope_map.get(&eid) {
|
||||
Some(v) => *v,
|
||||
None => {
|
||||
on_progress(&format!("WARN: eid {} envelope not found", eid));
|
||||
total_skipped += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let eml_doc: TantivyDocument = store_reader
|
||||
.get(doc_id)
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
|
||||
let account_id = match eml_doc.get_first(mf.f_account_id).and_then(|v| v.as_u64()) {
|
||||
Some(v) => v,
|
||||
None => {
|
||||
on_progress(&format!("WARN: eid {} account_id missing", eid));
|
||||
total_skipped += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let mailbox_id = eml_doc
|
||||
.get_first(mf.f_mailbox_id)
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(0);
|
||||
|
||||
// Borrow directly from eml_doc — no .to_vec() clone.
|
||||
let eml_bytes = match eml_doc.get_first(mf.f_eml).and_then(|v| v.as_bytes()) {
|
||||
Some(b) => b,
|
||||
None => {
|
||||
on_progress(&format!("WARN: eid {} eml bytes missing", eid));
|
||||
total_skipped += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(e) = writer.ingest(eml_bytes, account_id, mailbox_id, uid, internal_date) {
|
||||
on_progress(&format!(
|
||||
"ERROR: Account {} eid {} ingest failed: {}",
|
||||
account_id, eid, e
|
||||
));
|
||||
total_skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
total_migrated += 1;
|
||||
|
||||
if total_migrated % 10 == 0 || total_migrated as u32 == num_docs {
|
||||
on_progress(&format!("PROGRESS:{}:{}", total_migrated, num_docs));
|
||||
}
|
||||
}
|
||||
|
||||
drop(store_reader);
|
||||
|
||||
// Flush Fjall buffers via ingestion API — bypasses memtable/WAL.
|
||||
writer.flush_fjall_buffers()?;
|
||||
|
||||
chunk_start = chunk_end;
|
||||
}
|
||||
|
||||
on_progress(&format!("DONE:{}:{}", total_migrated, total_skipped));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_real_tantivy_dir() {
|
||||
let path = PathBuf::from(r"D:\test-data\envelope");
|
||||
let result = is_tantivy_index_dir(&path).unwrap();
|
||||
println!("is tantivy index dir: {}", result);
|
||||
assert!(result);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -265,7 +265,7 @@ pub struct Settings {
|
||||
Ok(s.to_string())
|
||||
})
|
||||
)]
|
||||
pub bichon_smtp_tls_key_path: Option<String>,
|
||||
pub bichon_tls_key_path: Option<String>,
|
||||
|
||||
#[clap(
|
||||
long,
|
||||
@@ -282,7 +282,7 @@ pub struct Settings {
|
||||
Ok(s.to_string())
|
||||
})
|
||||
)]
|
||||
pub bichon_smtp_tls_cert_path: Option<String>,
|
||||
pub bichon_tls_cert_path: Option<String>,
|
||||
|
||||
#[clap(
|
||||
long,
|
||||
@@ -299,7 +299,7 @@ pub struct Settings {
|
||||
default_value = "starttls",
|
||||
help = "Set the encryption mode for SMTP: 'none', 'starttls', or 'tls'"
|
||||
)]
|
||||
pub bichon_smtp_encryption: SmtpEncryptionMode,
|
||||
pub bichon_smtp_encryption: EncryptionMode,
|
||||
|
||||
#[clap(
|
||||
long,
|
||||
@@ -309,6 +309,42 @@ pub struct Settings {
|
||||
)]
|
||||
pub bichon_smtp_auth_required: bool,
|
||||
|
||||
/// Enable the built-in IMAP server for read-only email access via standard
|
||||
/// email clients (Thunderbird, Outlook, Apple Mail, etc.).
|
||||
#[clap(
|
||||
long,
|
||||
default_value = "false",
|
||||
env,
|
||||
help = "Enable the embedded IMAP server"
|
||||
)]
|
||||
pub bichon_enable_imap: bool,
|
||||
|
||||
#[clap(
|
||||
long,
|
||||
default_value = "10143",
|
||||
env,
|
||||
help = "Set the IMAP port (STARTTLS or plaintext)",
|
||||
value_parser = clap::value_parser!(u16).range(1..)
|
||||
)]
|
||||
pub bichon_imap_port: u16,
|
||||
|
||||
#[clap(
|
||||
long,
|
||||
default_value = "10993",
|
||||
env,
|
||||
help = "Set the IMAPS port (implicit TLS)",
|
||||
value_parser = clap::value_parser!(u16).range(1..)
|
||||
)]
|
||||
pub bichon_imaps_port: u16,
|
||||
|
||||
#[clap(
|
||||
long,
|
||||
env,
|
||||
default_value = "none",
|
||||
help = "Set the encryption mode for IMAP: 'none', 'starttls', or 'tls'"
|
||||
)]
|
||||
pub bichon_imap_encryption: EncryptionMode,
|
||||
|
||||
/// Enable OIDC-based Single Sign-On (Pro/Enterprise feature).
|
||||
#[clap(long, default_value = "false", env, help = "Enable OpenID Connect SSO")]
|
||||
pub bichon_oidc_enabled: bool,
|
||||
@@ -417,7 +453,7 @@ impl fmt::Display for CompressionAlgorithm {
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, ValueEnum)]
|
||||
pub enum SmtpEncryptionMode {
|
||||
pub enum EncryptionMode {
|
||||
#[clap(name = "none")]
|
||||
None,
|
||||
#[clap(name = "starttls")]
|
||||
@@ -426,12 +462,12 @@ pub enum SmtpEncryptionMode {
|
||||
Tls,
|
||||
}
|
||||
|
||||
impl fmt::Display for SmtpEncryptionMode {
|
||||
impl fmt::Display for EncryptionMode {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
SmtpEncryptionMode::None => write!(f, "none"),
|
||||
SmtpEncryptionMode::Starttls => write!(f, "starttls"),
|
||||
SmtpEncryptionMode::Tls => write!(f, "tls"),
|
||||
EncryptionMode::None => write!(f, "none"),
|
||||
EncryptionMode::Starttls => write!(f, "starttls"),
|
||||
EncryptionMode::Tls => write!(f, "tls"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use crate::context::Initialize;
|
||||
use crate::migrate::{write_storage_version, CURRENT_STORAGE_VERSION};
|
||||
use crate::settings::cli::SETTINGS;
|
||||
use crate::{
|
||||
error::{code::ErrorCode, BichonResult},
|
||||
@@ -62,6 +63,14 @@ impl Initialize for DataDirManager {
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
std::fs::create_dir_all(&DATA_DIR_MANAGER.storage_dir)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
// Write STORAGE_VERSION on fresh install (no existing data)
|
||||
let version_path = DATA_DIR_MANAGER.root_dir.join("STORAGE_VERSION");
|
||||
if !version_path.exists() && !DATA_DIR_MANAGER.storage_dir.join("blobs").exists() {
|
||||
write_storage_version(&DATA_DIR_MANAGER.root_dir, CURRENT_STORAGE_VERSION)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,8 +103,8 @@ impl From<&Settings> for SystemConfigurations {
|
||||
bichon_smtp_port: s.bichon_smtp_port,
|
||||
bichon_smtp_encryption: s.bichon_smtp_encryption.to_string(),
|
||||
bichon_smtp_auth_required: s.bichon_smtp_auth_required,
|
||||
bichon_smtp_tls_key_path: s.bichon_smtp_tls_key_path.clone(),
|
||||
bichon_smtp_tls_cert_path: s.bichon_smtp_tls_cert_path.clone(),
|
||||
bichon_smtp_tls_key_path: s.bichon_tls_key_path.clone(),
|
||||
bichon_smtp_tls_cert_path: s.bichon_tls_cert_path.clone(),
|
||||
bichon_oidc_enabled: s.bichon_oidc_enabled,
|
||||
bichon_oidc_issuer_url: s.bichon_oidc_issuer_url.clone(),
|
||||
bichon_oidc_client_id: s.bichon_oidc_client_id.clone(),
|
||||
|
||||
@@ -16,17 +16,17 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use crate::raise_error;
|
||||
use crate::{
|
||||
common::signal::SIGNAL_MANAGER,
|
||||
envelope::extractor::reattach_eml_content_self_healing,
|
||||
error::{code::ErrorCode, BichonResult},
|
||||
settings::dir::DATA_DIR_MANAGER,
|
||||
};
|
||||
use crate::raise_error;
|
||||
use bichon_blob::{Codec, Config, Engine};
|
||||
use bytes::Bytes;
|
||||
use fjall::{CompressionType, Database, Keyspace, KeyspaceCreateOptions, KvSeparationOptions, config::{BlockSizePolicy, CompressionPolicy}};
|
||||
|
||||
use std::{io::Cursor, sync::LazyLock};
|
||||
use std::{io::Cursor, sync::Arc, sync::LazyLock};
|
||||
use tokio::{
|
||||
sync::{mpsc, Mutex},
|
||||
task::{self, JoinHandle},
|
||||
@@ -41,33 +41,48 @@ pub struct DetachedEmail {
|
||||
|
||||
pub struct BlobManager {
|
||||
sender: mpsc::Sender<DetachedEmail>,
|
||||
db: Database,
|
||||
email_keyspace: Keyspace,
|
||||
attachments_keyspace: Keyspace,
|
||||
engine: Arc<Engine>,
|
||||
handle: Mutex<Option<JoinHandle<()>>>,
|
||||
}
|
||||
|
||||
fn hex_to_key(hex: &str) -> BichonResult<[u8; 32]> {
|
||||
let mut key = [0u8; 32];
|
||||
hex::decode_to_slice(hex, &mut key).map_err(|e| {
|
||||
raise_error!(
|
||||
format!("invalid content hash '{hex}': {e:#?}"),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
impl BlobManager {
|
||||
pub async fn shutdown(&self) {
|
||||
let mut guard = self.handle.lock().await;
|
||||
if let Some(handle) = guard.take() {
|
||||
let _ = handle.await;
|
||||
}
|
||||
if let Err(e) = self.engine.shutdown() {
|
||||
tracing::error!("blob engine shutdown error: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
fn process_detached_email(
|
||||
eml: DetachedEmail,
|
||||
email_ks: &Keyspace,
|
||||
attach_ks: &Keyspace,
|
||||
) {
|
||||
fn process_detached_email(eml: DetachedEmail, engine: &Engine) {
|
||||
let (email_hash, email_data) = eml.email;
|
||||
match email_ks.contains_key(&email_hash) {
|
||||
let email_key = match hex_to_key(&email_hash) {
|
||||
Ok(k) => k,
|
||||
Err(e) => {
|
||||
tracing::error!("{:#?}", e);
|
||||
return;
|
||||
}
|
||||
};
|
||||
match engine.exists(&email_key) {
|
||||
Ok(false) => {
|
||||
if let Err(e) = email_ks.insert(email_hash, email_data) {
|
||||
tracing::error!("CRITICAL: Failed to insert email: {:?}", e);
|
||||
if let Err(e) = engine.put(email_key, &email_data, Codec::Lz4) {
|
||||
tracing::error!("CRITICAL: Failed to insert email blob: {:?}", e);
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::error!("Fjall email_ks error: {:?}", e),
|
||||
Err(e) => tracing::error!("blob engine error: {:?}", e),
|
||||
Ok(true) => {
|
||||
tracing::debug!("Email blob already exists (dedup): {}", &email_hash);
|
||||
}
|
||||
@@ -75,13 +90,20 @@ impl BlobManager {
|
||||
|
||||
if let Some(attachments) = eml.attachments {
|
||||
for (a_hash, a_data) in attachments {
|
||||
match attach_ks.contains_key(&a_hash) {
|
||||
let a_key = match hex_to_key(&a_hash) {
|
||||
Ok(k) => k,
|
||||
Err(e) => {
|
||||
tracing::error!("{:#?}", e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
match engine.exists(&a_key) {
|
||||
Ok(false) => {
|
||||
if let Err(e) = attach_ks.insert(a_hash, a_data) {
|
||||
tracing::error!("CRITICAL: Failed to insert attachment: {:?}", e);
|
||||
if let Err(e) = engine.put(a_key, &a_data, Codec::Lz4) {
|
||||
tracing::error!("CRITICAL: Failed to insert attachment blob: {:?}", e);
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::error!("Fjall attach_ks error: {:?}", e),
|
||||
Err(e) => tracing::error!("blob engine error: {:?}", e),
|
||||
Ok(true) => {
|
||||
tracing::debug!("Attachment blob already exists (dedup): {}", &a_hash);
|
||||
}
|
||||
@@ -91,57 +113,22 @@ impl BlobManager {
|
||||
}
|
||||
|
||||
pub fn new() -> Self {
|
||||
let db = Database::builder(&DATA_DIR_MANAGER.storage_dir)
|
||||
.cache_size(64 * 1024 * 1024)
|
||||
.max_cached_files(Some(400))
|
||||
.journal_compression(CompressionType::None)
|
||||
.max_journaling_size(64 * 1024 * 1024)
|
||||
.open()
|
||||
.expect("Failed to initialize Fjall database: Check if the directory exists and has write permissions.");
|
||||
let blob_dir = DATA_DIR_MANAGER.storage_dir.join("blobs");
|
||||
|
||||
let mut config = Config::default();
|
||||
config.default_codec = Codec::Zstd;
|
||||
config.compress_threshold = 1024;
|
||||
config.flush_interval_secs = 60;
|
||||
config.gc_interval_secs = 300;
|
||||
|
||||
let engine = Engine::open(&blob_dir, config)
|
||||
.expect("Failed to initialize blob engine: Check disk space and permissions.");
|
||||
|
||||
let engine = Arc::new(engine);
|
||||
|
||||
let email_keyspace = db
|
||||
.keyspace("email", || {
|
||||
KeyspaceCreateOptions::default()
|
||||
.max_memtable_size(16 * 1024 * 1024)
|
||||
.data_block_size_policy(BlockSizePolicy::all(4 * 1024))
|
||||
.data_block_compression_policy(
|
||||
CompressionPolicy::all(CompressionType::Lz4)
|
||||
)
|
||||
.with_kv_separation(Some(
|
||||
KvSeparationOptions::default()
|
||||
.separation_threshold(1024)
|
||||
.compression(CompressionType::Lz4)
|
||||
.file_target_size(512 * 1024 * 1024)
|
||||
.staleness_threshold(0.5)
|
||||
.age_cutoff(0.6),
|
||||
))
|
||||
})
|
||||
.expect("Failed to open 'email' keyspace: The partition metadata might be corrupted or inaccessible.");
|
||||
|
||||
let attachments_keyspace = db
|
||||
.keyspace("attachments", || {
|
||||
KeyspaceCreateOptions::default()
|
||||
.data_block_size_policy(BlockSizePolicy::all(4 * 1024))
|
||||
.data_block_compression_policy(
|
||||
CompressionPolicy::all(CompressionType::Lz4)
|
||||
)
|
||||
.with_kv_separation(Some(
|
||||
KvSeparationOptions::default()
|
||||
.separation_threshold(1024)
|
||||
.compression(CompressionType::Lz4)
|
||||
.file_target_size(512 * 1024 * 1024)
|
||||
.staleness_threshold(0.5)
|
||||
.age_cutoff(0.6),
|
||||
))
|
||||
.max_memtable_size(16 * 1024 * 1024)
|
||||
})
|
||||
.expect("Failed to open 'attachments' keyspace: Check disk space for blob storage initialization.");
|
||||
|
||||
let (sender, mut receiver) = mpsc::channel::<DetachedEmail>(100);
|
||||
|
||||
let email_ks = email_keyspace.clone();
|
||||
let attach_ks = attachments_keyspace.clone();
|
||||
let engine_bg = Arc::clone(&engine);
|
||||
let handler = task::spawn(async move {
|
||||
let mut shutdown = SIGNAL_MANAGER.subscribe();
|
||||
loop {
|
||||
@@ -153,18 +140,17 @@ impl BlobManager {
|
||||
while let Ok(next_eml) = receiver.try_recv() {
|
||||
batch.push(next_eml);
|
||||
}
|
||||
let email_ks = email_ks.clone();
|
||||
let attach_ks = attach_ks.clone();
|
||||
let engine_bg = Arc::clone(&engine_bg);
|
||||
if let Err(e) = tokio::task::spawn_blocking(move || {
|
||||
for eml in batch {
|
||||
Self::process_detached_email(eml, &email_ks, &attach_ks);
|
||||
Self::process_detached_email(eml, &engine_bg);
|
||||
}
|
||||
}).await {
|
||||
tracing::error!("BlobManager: spawn_blocking join error: {:#?}", e);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
tracing::info!("BlobManager: All senders dropped, closing storage.");
|
||||
tracing::info!("BlobManager: All senders dropped, closing blob storage.");
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -180,17 +166,16 @@ impl BlobManager {
|
||||
remaining.len()
|
||||
);
|
||||
if !remaining.is_empty() {
|
||||
let email_ks = email_ks.clone();
|
||||
let attach_ks = attach_ks.clone();
|
||||
let engine_bg = Arc::clone(&engine_bg);
|
||||
if let Err(e) = tokio::task::spawn_blocking(move || {
|
||||
for eml in remaining {
|
||||
Self::process_detached_email(eml, &email_ks, &attach_ks);
|
||||
Self::process_detached_email(eml, &engine_bg);
|
||||
}
|
||||
}).await {
|
||||
tracing::error!("BlobManager: shutdown spawn_blocking join error: {:#?}", e);
|
||||
}
|
||||
}
|
||||
tracing::info!("BlobManager: All remaining tasks processed. Closing Fjall.");
|
||||
tracing::info!("BlobManager: All remaining tasks processed. Closing blob engine.");
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -199,30 +184,30 @@ impl BlobManager {
|
||||
|
||||
Self {
|
||||
sender,
|
||||
db,
|
||||
email_keyspace,
|
||||
attachments_keyspace,
|
||||
engine,
|
||||
handle: Mutex::new(Some(handler)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn queue(&self, email: DetachedEmail) {
|
||||
if let Err(e) = self.sender.send(email).await {
|
||||
tracing::error!("BlobManager channel closed, email lost: {:#?}", e);
|
||||
tracing::error!("BlobManager channel closed, email lost: {:#?}", e);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_email(&self, content_hash: &str) -> BichonResult<Option<Bytes>> {
|
||||
self.email_keyspace
|
||||
.get(content_hash)
|
||||
.map(|user_value| user_value.map(|s| s.into()))
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
|
||||
self.get(content_hash)
|
||||
}
|
||||
|
||||
pub fn get_attachment(&self, content_hash: &str) -> BichonResult<Option<Bytes>> {
|
||||
self.attachments_keyspace
|
||||
.get(content_hash)
|
||||
.map(|user_value| user_value.map(|s| s.into()))
|
||||
self.get(content_hash)
|
||||
}
|
||||
|
||||
fn get(&self, content_hash: &str) -> BichonResult<Option<Bytes>> {
|
||||
let key = hex_to_key(content_hash)?;
|
||||
self.engine
|
||||
.get(&key)
|
||||
.map(|v| v.map(Bytes::from))
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
|
||||
}
|
||||
|
||||
@@ -235,17 +220,23 @@ impl BlobManager {
|
||||
I1: IntoIterator,
|
||||
I1::Item: AsRef<str>,
|
||||
I2: IntoIterator,
|
||||
I2::Item: AsRef<str> {
|
||||
let mut batch = self.db.batch();
|
||||
for hash in email_content_hashes {
|
||||
batch.remove(&self.email_keyspace, hash.as_ref());
|
||||
I2::Item: AsRef<str>,
|
||||
{
|
||||
let mut keys: Vec<[u8; 32]> = email_content_hashes
|
||||
.into_iter()
|
||||
.map(|h| hex_to_key(h.as_ref()))
|
||||
.collect::<BichonResult<_>>()?;
|
||||
|
||||
for h in attachment_content_hashes {
|
||||
keys.push(hex_to_key(h.as_ref())?);
|
||||
}
|
||||
for hash in attachment_content_hashes {
|
||||
batch.remove(&self.attachments_keyspace, hash.as_ref());
|
||||
|
||||
if !keys.is_empty() {
|
||||
self.engine
|
||||
.delete_batch(&keys)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
}
|
||||
batch
|
||||
.commit()
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2529,4 +2529,138 @@ mod tests {
|
||||
]);
|
||||
assert_eq!(searcher.search(&q99, &Count).unwrap(), 0);
|
||||
}
|
||||
|
||||
// ── IMAP UID concept: order_by_fast_field(ingest_at) ─────────────
|
||||
|
||||
#[test]
|
||||
fn stable_uid_ordering_by_ingest_at() {
|
||||
let f = SchemaTools::email_fields();
|
||||
let index = Index::create_in_ram(SchemaTools::email_schema());
|
||||
index.tokenizers().register("euro", EuroTokenizer::new());
|
||||
|
||||
let mailbox_id = 42u64;
|
||||
|
||||
// 10 emails, some sharing the same ingest_at (simulating batch import)
|
||||
#[rustfmt::skip]
|
||||
let test_data: Vec<(u64, u64, i64)> = vec![
|
||||
// (account_id, uid, ingest_at)
|
||||
(1, 101, 1_700_000_000_000), // epoch #1
|
||||
(1, 102, 1_700_000_000_000), // epoch #1 — same ms, different uid
|
||||
(1, 201, 1_700_000_000_100), // epoch #2
|
||||
(1, 202, 1_700_000_000_100), // epoch #2 — same ms
|
||||
(1, 203, 1_700_000_000_100), // epoch #2 — 3 in same ms
|
||||
(1, 301, 1_700_000_000_200),
|
||||
(1, 401, 1_700_000_000_300),
|
||||
(2, 501, 1_700_000_000_300), // different account, same ms — different mailbox
|
||||
(1, 402, 1_700_000_000_400),
|
||||
(1, 501, 1_700_000_000_500),
|
||||
];
|
||||
|
||||
{
|
||||
let mut writer = index
|
||||
.writer_with_num_threads(1, 15_000_000)
|
||||
.expect("writer");
|
||||
|
||||
for (i, (account_id, uid, ingest_at)) in test_data.iter().enumerate() {
|
||||
let mut doc = TantivyDocument::new();
|
||||
doc.add_text(f.f_id, format!("eid-{}", i));
|
||||
doc.add_text(f.f_message_id, format!("<msg-{}@test>", i));
|
||||
doc.add_u64(f.f_account_id, *account_id);
|
||||
doc.add_u64(f.f_mailbox_id, mailbox_id);
|
||||
doc.add_u64(f.f_uid, *uid);
|
||||
doc.add_i64(f.f_ingest_at, *ingest_at);
|
||||
doc.add_i64(f.f_date, *ingest_at);
|
||||
doc.add_i64(f.f_internal_date, *ingest_at);
|
||||
doc.add_u64(f.f_size, 100);
|
||||
doc.add_text(f.f_subject, "test");
|
||||
doc.add_text(f.f_preview, "preview");
|
||||
doc.add_text(f.f_content_hash, format!("hash-{}", i));
|
||||
doc.add_text(f.f_from, "a@b.com");
|
||||
doc.add_text(f.f_thread_id, "t1");
|
||||
writer.add_document(doc).unwrap();
|
||||
}
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
let reader = index
|
||||
.reader()
|
||||
.expect("reader");
|
||||
let searcher = reader.searcher();
|
||||
|
||||
let query = TermQuery::new(
|
||||
Term::from_field_u64(f.f_mailbox_id, mailbox_id),
|
||||
IndexRecordOption::Basic,
|
||||
);
|
||||
|
||||
// Primary sort: Tantivy fast field on ingest_at
|
||||
let top_docs: Vec<(Option<i64>, DocAddress)> = searcher
|
||||
.search(
|
||||
&query,
|
||||
&tantivy::collector::TopDocs::with_limit(100)
|
||||
.order_by_fast_field::<i64>(F_INGEST_AT, tantivy::Order::Asc),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(top_docs.len(), 10, "all 10 docs should be returned");
|
||||
|
||||
// Read each doc, extract (ingest_at, uid) for app-level tie-break
|
||||
let mut results: Vec<(i64, u64)> = Vec::new();
|
||||
for (_, addr) in &top_docs {
|
||||
let doc: TantivyDocument = searcher.doc(*addr).unwrap();
|
||||
let ingest_at = doc
|
||||
.get_first(f.f_ingest_at)
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(0);
|
||||
let uid = doc
|
||||
.get_first(f.f_uid)
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(0);
|
||||
results.push((ingest_at, uid));
|
||||
}
|
||||
|
||||
// Verify primary sort by ingest_at is correct
|
||||
for w in results.windows(2) {
|
||||
assert!(
|
||||
w[0].0 <= w[1].0,
|
||||
"ingest_at must be non-decreasing"
|
||||
);
|
||||
}
|
||||
|
||||
// Verify deterministic: run again, same order
|
||||
let top_docs2: Vec<DocAddress> = searcher
|
||||
.search(
|
||||
&query,
|
||||
&tantivy::collector::TopDocs::with_limit(100)
|
||||
.order_by_fast_field::<i64>(F_INGEST_AT, tantivy::Order::Asc),
|
||||
)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|(_, addr)| addr)
|
||||
.collect();
|
||||
|
||||
for (i, addr) in top_docs.iter().map(|(_, a)| *a).enumerate() {
|
||||
assert_eq!(addr, top_docs2[i], "order must be stable across queries");
|
||||
}
|
||||
|
||||
// Demonstrate tie-break: same ingest_at groups MUST be ordered by uid
|
||||
let mut current_group: Vec<u64> = Vec::new();
|
||||
for w in results.windows(2) {
|
||||
current_group.push(w[0].1);
|
||||
if w[0].0 != w[1].0 {
|
||||
current_group.push(w[0].1);
|
||||
let mut sorted = current_group.clone();
|
||||
sorted.sort();
|
||||
assert_eq!(
|
||||
current_group, sorted,
|
||||
"within same ingest_at, uids must be ascending"
|
||||
);
|
||||
current_group = Vec::new();
|
||||
}
|
||||
}
|
||||
|
||||
println!("IMAP UID mapping (position → ingest_at, uid):");
|
||||
for (pos, (ingest_at, uid)) in results.iter().enumerate() {
|
||||
println!(" UID {} → (ingest_at={}, original_uid={})", pos + 1, ingest_at, uid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,9 +66,11 @@ pub async fn run() -> BichonResult<()> {
|
||||
Ok(false) => {
|
||||
error!("Incompatible data format detected.");
|
||||
error!("Your data was created by an older version of Bichon and must be migrated before use.");
|
||||
error!("Please stop the Bichon v0.3.7 service before migration.");
|
||||
error!("Please run: bichon-admin");
|
||||
error!("Documentation: https://github.com/rustmailer/bichon/wiki/Bichon-Data-Migration:-v0.3.7-%E2%86%92-v1.0");
|
||||
error!("Available migration options:");
|
||||
error!(" - Legacy v0.3.7 → v2.x (via v1.x)");
|
||||
error!(" - v1.x (Fjall) → v2.x (bichon-blob)");
|
||||
error!("Documentation: https://github.com/rustmailer/bichon/wiki");
|
||||
return Err(raise_error!(
|
||||
"Legacy data layout detected".into(),
|
||||
ErrorCode::InternalError
|
||||
|
||||
@@ -26,7 +26,7 @@ use bichon_core::cache::imap::mailbox::{Attribute, AttributeEnum};
|
||||
use bichon_core::common::signal::SIGNAL_MANAGER;
|
||||
use bichon_core::envelope::extractor::extract_envelope_from_smtp;
|
||||
use bichon_core::error::BichonResult;
|
||||
use bichon_core::settings::cli::{SmtpEncryptionMode, SETTINGS};
|
||||
use bichon_core::settings::cli::{EncryptionMode, SETTINGS};
|
||||
use bichon_core::utils::create_hash;
|
||||
use bichon_core::{
|
||||
account::migration::AccountModel,
|
||||
@@ -698,8 +698,8 @@ pub async fn start_smtp_server() -> std::io::Result<SmtpServer> {
|
||||
let smtp_port = SETTINGS.bichon_smtp_port;
|
||||
|
||||
let tls_acceptor: Option<TlsAcceptor> = match SETTINGS.bichon_smtp_encryption {
|
||||
SmtpEncryptionMode::None => None,
|
||||
SmtpEncryptionMode::Starttls | SmtpEncryptionMode::Tls => Some(create_acceptor().await?),
|
||||
EncryptionMode::None => None,
|
||||
EncryptionMode::Starttls | EncryptionMode::Tls => Some(create_acceptor().await?),
|
||||
};
|
||||
|
||||
let smtp_listener = TcpListener::bind((
|
||||
@@ -721,15 +721,15 @@ pub async fn start_smtp_server() -> std::io::Result<SmtpServer> {
|
||||
let smtp_config = SmtpConfig {
|
||||
whitelist: None,
|
||||
tls_acceptor: match SETTINGS.bichon_smtp_encryption {
|
||||
SmtpEncryptionMode::None | SmtpEncryptionMode::Tls => None,
|
||||
SmtpEncryptionMode::Starttls => tls_acceptor.clone(),
|
||||
EncryptionMode::None | EncryptionMode::Tls => None,
|
||||
EncryptionMode::Starttls => tls_acceptor.clone(),
|
||||
},
|
||||
auth_required: SETTINGS.bichon_smtp_auth_required,
|
||||
};
|
||||
|
||||
let smtp_shutdown = SIGNAL_MANAGER.subscribe();
|
||||
|
||||
let smtp_handle = if matches!(SETTINGS.bichon_smtp_encryption, SmtpEncryptionMode::Tls) {
|
||||
let smtp_handle = if matches!(SETTINGS.bichon_smtp_encryption, EncryptionMode::Tls) {
|
||||
let acceptor = tls_acceptor
|
||||
.clone()
|
||||
.expect("TLS acceptor required when tls=true");
|
||||
|
||||
@@ -29,8 +29,8 @@ use bichon_core::settings::cli::SETTINGS;
|
||||
|
||||
pub async fn create_acceptor() -> io::Result<TlsAcceptor> {
|
||||
let (certs, key) = if let (Some(key_path), Some(cert_path)) = (
|
||||
&SETTINGS.bichon_smtp_tls_key_path,
|
||||
&SETTINGS.bichon_smtp_tls_cert_path,
|
||||
&SETTINGS.bichon_tls_key_path,
|
||||
&SETTINGS.bichon_tls_cert_path,
|
||||
) {
|
||||
load_certs_from_files(key_path, cert_path).await?
|
||||
} else {
|
||||
|
||||
@@ -38,6 +38,6 @@ EXPOSE 15630
|
||||
WORKDIR /data
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \
|
||||
CMD curl -fs http://localhost:15630/api/status || exit 1
|
||||
CMD curl -fs http://localhost:${BICHON_HTTP_PORT:-15630}/api/status || exit 1
|
||||
|
||||
CMD ["/opt/bichon/bichon-server"]
|
||||
|
||||
@@ -152,7 +152,6 @@ export interface AccountModel {
|
||||
created_user_email: string;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
use_proxy?: number;
|
||||
use_dangerous: boolean;
|
||||
pgp_key?: string;
|
||||
imap_quota_window?: QuotaWindow;
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import React from 'react';
|
||||
import React, { useCallback, useRef, useState } from 'react';
|
||||
|
||||
interface EmailIframeProps {
|
||||
emailHtml: string;
|
||||
@@ -25,16 +25,36 @@ interface EmailIframeProps {
|
||||
}
|
||||
|
||||
const EmailIframe: React.FC<EmailIframeProps> = ({ emailHtml, height }) => {
|
||||
const encodedHtml = encodeURIComponent(emailHtml);
|
||||
const iframeSrc = `data:text/html;charset=utf-8,${encodedHtml}`;
|
||||
const iframeRef = useRef<HTMLIFrameElement>(null);
|
||||
const [iframeHeight, setIframeHeight] = useState<number>(height ?? 300);
|
||||
|
||||
const onLoad = useCallback(() => {
|
||||
try {
|
||||
const doc = iframeRef.current?.contentWindow?.document;
|
||||
if (doc?.body) {
|
||||
const h = Math.max(
|
||||
doc.body.scrollHeight,
|
||||
doc.body.offsetHeight,
|
||||
doc.documentElement.scrollHeight,
|
||||
doc.documentElement.offsetHeight,
|
||||
);
|
||||
if (h > 0) setIframeHeight(h + 20);
|
||||
}
|
||||
} catch {
|
||||
// sandbox prevents access — keep default height
|
||||
}
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<iframe
|
||||
src={iframeSrc}
|
||||
sandbox=""
|
||||
ref={iframeRef}
|
||||
srcDoc={emailHtml}
|
||||
sandbox="allow-same-origin"
|
||||
scrolling="no"
|
||||
className="w-full border-none"
|
||||
title="Email Content"
|
||||
style={{ height: height ?? '4000px' }}
|
||||
onLoad={onLoad}
|
||||
style={{ height: iframeHeight }}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -111,12 +111,14 @@ export function AccountNewPage() {
|
||||
|
||||
const onSubmit = useCallback(
|
||||
(data: AccountFormValues) => {
|
||||
const { use_proxy, ...imapRest } = data.imap;
|
||||
createMutation.mutate({
|
||||
email: data.email,
|
||||
account_name: data.account_name,
|
||||
login_name: data.login_name,
|
||||
imap: {
|
||||
...data.imap,
|
||||
...imapRest,
|
||||
use_proxy,
|
||||
auth: {
|
||||
...data.imap.auth,
|
||||
password: data.imap.auth.auth_type === 'OAuth2' ? undefined : data.imap.auth.password,
|
||||
|
||||
@@ -44,15 +44,11 @@ const emptyImap = {
|
||||
port: 0,
|
||||
encryption: "None" as const,
|
||||
auth: { auth_type: "Password" as const, password: undefined },
|
||||
use_proxy: undefined,
|
||||
};
|
||||
|
||||
function mapAccountToFormValues(account: AccountModel): AccountFormValues {
|
||||
const imap = { ...(account.imap ?? emptyImap) };
|
||||
imap.auth = { ...imap.auth, password: undefined };
|
||||
if ((imap as any).use_proxy === null) {
|
||||
(imap as any).use_proxy = undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
account_name: account.account_name ?? undefined,
|
||||
@@ -137,12 +133,14 @@ export function AccountSettingsPage({ accountId }: AccountSettingsPageProps) {
|
||||
|
||||
const onSubmit = useCallback(
|
||||
(data: AccountFormValues) => {
|
||||
const { use_proxy, ...imapRest } = data.imap;
|
||||
const payload: Record<string, any> = {
|
||||
email: data.email,
|
||||
account_name: data.account_name,
|
||||
login_name: data.login_name,
|
||||
imap: {
|
||||
...data.imap,
|
||||
...imapRest,
|
||||
use_proxy,
|
||||
auth: {
|
||||
...data.imap.auth,
|
||||
password: data.imap.auth.auth_type === 'OAuth2'
|
||||
|
||||
@@ -37,6 +37,7 @@ import {
|
||||
} from "@/components/ui/select";
|
||||
import { PasswordInput } from "@/components/password-input";
|
||||
import { AccountFormValues } from "./schema";
|
||||
import useProxyList from "@/hooks/use-proxy";
|
||||
|
||||
interface TabServerProps {
|
||||
isEdit?: boolean;
|
||||
@@ -46,6 +47,7 @@ export function TabServer({ isEdit }: TabServerProps) {
|
||||
const { t } = useTranslation();
|
||||
const { control, watch } = useFormContext<AccountFormValues>();
|
||||
const authType = watch('imap.auth.auth_type');
|
||||
const { proxyOptions } = useProxyList();
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
@@ -161,6 +163,38 @@ export function TabServer({ isEdit }: TabServerProps) {
|
||||
/>
|
||||
)}
|
||||
|
||||
<FormField
|
||||
control={control}
|
||||
name="imap.use_proxy"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.useProxyOptional')}</FormLabel>
|
||||
<Select
|
||||
onValueChange={(v) => field.onChange(v === 'none' ? undefined : Number(v))}
|
||||
defaultValue={field.value?.toString()}
|
||||
>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue placeholder={t('accounts.selectProxy')}/>
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem key="none" value="none">{t('accounts.useNoProxy')}</SelectItem>
|
||||
{proxyOptions.map((opt) => (
|
||||
<SelectItem key={opt.value} value={opt.value}>
|
||||
<span className="max-w-[280px] truncate block" title={opt.label}>
|
||||
{opt.label}
|
||||
</span>
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormDescription>{t('accounts.imapProxy')}</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={control}
|
||||
name="use_dangerous"
|
||||
|
||||
@@ -43,23 +43,24 @@ export function MailDisplayDrawer({ open, onOpenChange }: Props) {
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className='w-full md:max-w-6xl mx-auto h-full'>
|
||||
<DialogHeader className="p-4 pb-3 border-b shrink-0">
|
||||
<DialogTitle>{t('mail.emailViewer')}</DialogTitle>
|
||||
</DialogHeader>
|
||||
|
||||
<ScrollArea className="h-[calc(100vh-100px)]">
|
||||
<div className='m-5'>
|
||||
{isLoading ? (
|
||||
<div className="p-8 text-center text-muted-foreground">{t('common.loading')}...</div>
|
||||
) : error ? (
|
||||
<div className="p-8 text-center text-red-500">{t('attachment.emailMessageNotFound')}</div>
|
||||
) : envelope ? (
|
||||
<MailMessageView envelope={envelope} />
|
||||
) : (
|
||||
<div className="p-8 text-center text-muted-foreground">{t('mail.noMessageSelected')}</div>
|
||||
)}
|
||||
</div>
|
||||
</ScrollArea>
|
||||
<div className="flex flex-col h-full overflow-hidden -m-6">
|
||||
<DialogHeader className="p-4 pb-3 border-b shrink-0 m-0">
|
||||
<DialogTitle>{t('mail.emailViewer')}</DialogTitle>
|
||||
</DialogHeader>
|
||||
<ScrollArea className="flex-1">
|
||||
<div className='m-5'>
|
||||
{isLoading ? (
|
||||
<div className="p-8 text-center text-muted-foreground">{t('common.loading')}...</div>
|
||||
) : error ? (
|
||||
<div className="p-8 text-center text-red-500">{t('attachment.emailMessageNotFound')}</div>
|
||||
) : envelope ? (
|
||||
<MailMessageView envelope={envelope} />
|
||||
) : (
|
||||
<div className="p-8 text-center text-muted-foreground">{t('mail.noMessageSelected')}</div>
|
||||
)}
|
||||
</div>
|
||||
</ScrollArea>
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
)
|
||||
|
||||
@@ -221,7 +221,7 @@ export function MailMessageView({
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex flex-col h-full">
|
||||
<div className="flex flex-col">
|
||||
{showHeader && <div className="grid gap-1 text-xs">
|
||||
<div className="flex space-x-2">
|
||||
<span className="font-medium text-gray-400">{t('mail.account')}:</span>
|
||||
@@ -424,7 +424,7 @@ export function MailMessageView({
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
<div className="flex-1 overflow-auto">
|
||||
<div>
|
||||
{loading ? (
|
||||
<div className="flex justify-center items-center py-8">
|
||||
<Loader className="w-6 h-6 animate-spin" />
|
||||
|
||||
@@ -40,22 +40,24 @@ export function MailDisplayDrawer({ open, onOpenChange }: Props) {
|
||||
onOpenChange={onOpenChange}
|
||||
>
|
||||
<DialogContent className='w-full md:max-w-6xl mx-auto h-full'>
|
||||
<DialogHeader className="p-4 pb-3 border-b shrink-0">
|
||||
<div className="flex items-center justify-between">
|
||||
<DialogTitle className="flex items-center gap-2">
|
||||
{t('mail.emailViewer')}
|
||||
</DialogTitle>
|
||||
</div>
|
||||
</DialogHeader>
|
||||
<ScrollArea>
|
||||
<div className='m-5'>
|
||||
{currentEnvelope ? (
|
||||
<MailMessageView envelope={currentEnvelope} />
|
||||
) : (
|
||||
<div className="p-8 text-center text-muted-foreground">{t('mail.noMessageSelected')}</div>
|
||||
)}
|
||||
</div>
|
||||
</ScrollArea>
|
||||
<div className="flex flex-col h-full overflow-hidden -m-6">
|
||||
<DialogHeader className="p-4 pb-3 border-b shrink-0 m-0">
|
||||
<div className="flex items-center justify-between">
|
||||
<DialogTitle className="flex items-center gap-2">
|
||||
{t('mail.emailViewer')}
|
||||
</DialogTitle>
|
||||
</div>
|
||||
</DialogHeader>
|
||||
<ScrollArea className="flex-1">
|
||||
<div className='m-5'>
|
||||
{currentEnvelope ? (
|
||||
<MailMessageView envelope={currentEnvelope} />
|
||||
) : (
|
||||
<div className="p-8 text-center text-muted-foreground">{t('mail.noMessageSelected')}</div>
|
||||
)}
|
||||
</div>
|
||||
</ScrollArea>
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>)
|
||||
}
|
||||
@@ -231,7 +231,7 @@ export function MailMessageView({
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex flex-col h-full">
|
||||
<div className="flex flex-col">
|
||||
{showHeader && <div className="grid gap-1 text-xs">
|
||||
<div className="flex space-x-2">
|
||||
<span className="font-medium text-gray-400">{t('mail.account')}:</span>
|
||||
@@ -456,7 +456,7 @@ export function MailMessageView({
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
<div className="flex-1 overflow-auto">
|
||||
<div>
|
||||
{loading ? (
|
||||
<div className="flex justify-center items-center py-8">
|
||||
<Loader className="w-6 h-6 animate-spin" />
|
||||
|
||||
Reference in New Issue
Block a user