mirror of
https://github.com/rustmailer/bichon.git
synced 2026-08-31 01:52:30 +00:00
Compare commits
36 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5fe45795b9 | ||
|
|
4996dac1ea | ||
|
|
42ce36a439 | ||
|
|
40ae2d49d4 | ||
|
|
5034760517 | ||
|
|
875eb2e309 | ||
|
|
f0d0f202ba | ||
|
|
0d37825625 | ||
|
|
cfb8172607 | ||
|
|
8641bb4b56 | ||
|
|
cdf27f2dd4 | ||
|
|
db36272bae | ||
|
|
cad447275f | ||
|
|
8542ba0d28 | ||
|
|
af089958e9 | ||
|
|
bba1ea5cc7 | ||
|
|
9e55026f12 | ||
|
|
b0f229618c | ||
|
|
ff59d47a0d | ||
|
|
b6957c8ceb | ||
|
|
cd98c050b6 | ||
|
|
11e40750fe | ||
|
|
b0374517e8 | ||
|
|
4e12ae5457 | ||
|
|
495c91b7ae | ||
|
|
e442db4a2d | ||
|
|
cc54063669 | ||
|
|
04745458bc | ||
|
|
6fcc9e0e8e | ||
|
|
220aa268c1 | ||
|
|
e3fd9d2f29 | ||
|
|
41c3b84e65 | ||
|
|
89557700ae | ||
|
|
2f5de48c6a | ||
|
|
debb119d3d | ||
|
|
ce3f8944a3 |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -3,4 +3,5 @@
|
||||
.idea
|
||||
config.toml
|
||||
node_modules
|
||||
dedup_report.txt
|
||||
dedup_report.txt
|
||||
crates/*/target
|
||||
281
Cargo.lock
generated
281
Cargo.lock
generated
@@ -60,6 +60,12 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anes"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "1.0.0"
|
||||
@@ -293,14 +299,14 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "bichon-admin"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
dependencies = [
|
||||
"bichon-core",
|
||||
"bichon-memdb",
|
||||
"console",
|
||||
"dialoguer",
|
||||
"indicatif",
|
||||
"itertools",
|
||||
"memdb",
|
||||
"itertools 0.15.0",
|
||||
"native_db",
|
||||
"native_model",
|
||||
"serde",
|
||||
@@ -309,22 +315,35 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bichon-blob"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"crc32fast",
|
||||
"criterion",
|
||||
"lz4_flex",
|
||||
"rand 0.10.1",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"tracing",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bichon-cli"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bichon-core",
|
||||
"chrono",
|
||||
"clap",
|
||||
"codepage-strings",
|
||||
"compressed-rtf",
|
||||
"console",
|
||||
"dialoguer",
|
||||
"hex",
|
||||
"indicatif",
|
||||
"mail-parser",
|
||||
"mail-send",
|
||||
"memmap2",
|
||||
"outlook-pst",
|
||||
"reqwest",
|
||||
@@ -337,14 +356,17 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bichon-core"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
dependencies = [
|
||||
"async-imap",
|
||||
"base64 0.22.1",
|
||||
"bichon-memdb",
|
||||
"blake3",
|
||||
"bytes 1.11.1",
|
||||
"chrono",
|
||||
"clap",
|
||||
"codepage-strings",
|
||||
"compressed-rtf",
|
||||
"cron",
|
||||
"dashmap",
|
||||
"deunicode",
|
||||
@@ -353,17 +375,19 @@ dependencies = [
|
||||
"fjall",
|
||||
"futures",
|
||||
"governor",
|
||||
"hex",
|
||||
"hickory-resolver",
|
||||
"html2text",
|
||||
"itertools",
|
||||
"itertools 0.15.0",
|
||||
"itoa",
|
||||
"lru 0.18.0",
|
||||
"mail-parser",
|
||||
"mail-send",
|
||||
"memdb",
|
||||
"memmap2",
|
||||
"murmur3",
|
||||
"num_cpus",
|
||||
"oauth2",
|
||||
"outlook-pst",
|
||||
"poem-openapi",
|
||||
"quick-xml 0.40.0",
|
||||
"rand 0.10.1",
|
||||
@@ -394,14 +418,27 @@ dependencies = [
|
||||
"whichlang",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bichon-memdb"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"rand 0.9.2",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bichon-server"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
dependencies = [
|
||||
"bichon-core",
|
||||
"bichon-smtp",
|
||||
"chrono",
|
||||
"email_address",
|
||||
"futures",
|
||||
"governor",
|
||||
"http",
|
||||
"poem",
|
||||
@@ -420,7 +457,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bichon-smtp"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bichon-core",
|
||||
@@ -613,6 +650,12 @@ dependencies = [
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cast"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.60"
|
||||
@@ -668,6 +711,33 @@ dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ciborium"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
|
||||
dependencies = [
|
||||
"ciborium-io",
|
||||
"ciborium-ll",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ciborium-io"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
|
||||
|
||||
[[package]]
|
||||
name = "ciborium-ll"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
|
||||
dependencies = [
|
||||
"ciborium-io",
|
||||
"half",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.1"
|
||||
@@ -868,6 +938,39 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "criterion"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3bf7af66b0989381bd0be551bd7cc91912a655a58c6918420c9527b1fd8b4679"
|
||||
dependencies = [
|
||||
"anes",
|
||||
"cast",
|
||||
"ciborium",
|
||||
"clap",
|
||||
"criterion-plot",
|
||||
"itertools 0.13.0",
|
||||
"num-traits",
|
||||
"oorandom",
|
||||
"plotters",
|
||||
"rayon",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tinytemplate",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "criterion-plot"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
|
||||
dependencies = [
|
||||
"cast",
|
||||
"itertools 0.10.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "critical-section"
|
||||
version = "1.2.0"
|
||||
@@ -1060,7 +1163,6 @@ version = "0.5.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
|
||||
dependencies = [
|
||||
"powerfmt",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
@@ -1602,6 +1704,17 @@ dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "half"
|
||||
version = "2.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crunchy",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.14.5"
|
||||
@@ -2128,6 +2241,24 @@ dependencies = [
|
||||
"phf 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.10.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
|
||||
dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
|
||||
dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.14.0"
|
||||
@@ -2137,6 +2268,15 @@ dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
|
||||
dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
@@ -2338,9 +2478,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lz4_flex"
|
||||
version = "0.13.0"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a"
|
||||
checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
|
||||
dependencies = [
|
||||
"twox-hash",
|
||||
]
|
||||
@@ -2375,9 +2515,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mail-parser"
|
||||
version = "0.11.3"
|
||||
version = "0.11.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d8a2420e9ce11c2b0583ca97ddff7ab2398c8a613154e9b72e3bafdbf767f1d7"
|
||||
checksum = "f2c0e7e0704500930be5b6c629f30d23fd1dde4d1800e138e04b3fa302e64d51"
|
||||
dependencies = [
|
||||
"encoding_rs",
|
||||
"hashify",
|
||||
@@ -2445,18 +2585,6 @@ version = "2.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
||||
|
||||
[[package]]
|
||||
name = "memdb"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"rand 0.9.2",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memmap2"
|
||||
version = "0.9.10"
|
||||
@@ -2699,9 +2827,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.2.1"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
|
||||
checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
@@ -2858,6 +2986,12 @@ version = "0.1.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"
|
||||
|
||||
[[package]]
|
||||
name = "oorandom"
|
||||
version = "11.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
|
||||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.78"
|
||||
@@ -3090,6 +3224,34 @@ version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
|
||||
|
||||
[[package]]
|
||||
name = "plotters"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
"plotters-backend",
|
||||
"plotters-svg",
|
||||
"wasm-bindgen",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "plotters-backend"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
|
||||
|
||||
[[package]]
|
||||
name = "plotters-svg"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
|
||||
dependencies = [
|
||||
"plotters-backend",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "poem"
|
||||
version = "3.1.12"
|
||||
@@ -3159,7 +3321,7 @@ dependencies = [
|
||||
"email_address",
|
||||
"futures-util",
|
||||
"indexmap",
|
||||
"itertools",
|
||||
"itertools 0.14.0",
|
||||
"mime",
|
||||
"num-traits",
|
||||
"poem",
|
||||
@@ -3551,9 +3713,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.12.3"
|
||||
version = "1.12.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
|
||||
checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
@@ -3574,9 +3736,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.10"
|
||||
version = "0.8.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
|
||||
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
@@ -3732,9 +3894,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.40"
|
||||
version = "0.23.41"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
|
||||
checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"log",
|
||||
@@ -4278,9 +4440,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "sysinfo"
|
||||
version = "0.39.2"
|
||||
version = "0.39.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14311e7e9a03114cd4b65eedd54e8fed2945e17f08586ae97ef53bc0669f9581"
|
||||
checksum = "2c8bd2130a9b60bee2581bf82cfe89ee836424d1f37dcfa4ce21509611684673"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"memchr",
|
||||
@@ -4320,7 +4482,7 @@ dependencies = [
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"htmlescape",
|
||||
"itertools",
|
||||
"itertools 0.14.0",
|
||||
"levenshtein_automata",
|
||||
"log",
|
||||
"lru 0.16.4",
|
||||
@@ -4371,7 +4533,7 @@ checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc"
|
||||
dependencies = [
|
||||
"downcast-rs",
|
||||
"fastdivide",
|
||||
"itertools",
|
||||
"itertools 0.14.0",
|
||||
"serde",
|
||||
"tantivy-bitpacker",
|
||||
"tantivy-common",
|
||||
@@ -4423,7 +4585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"itertools",
|
||||
"itertools 0.14.0",
|
||||
"tantivy-bitpacker",
|
||||
"tantivy-common",
|
||||
"tantivy-fst",
|
||||
@@ -4523,12 +4685,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.47"
|
||||
version = "0.3.51"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c"
|
||||
checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327"
|
||||
dependencies = [
|
||||
"deranged",
|
||||
"itoa",
|
||||
"libc",
|
||||
"num-conv",
|
||||
"num_threads",
|
||||
@@ -4540,15 +4701,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "time-core"
|
||||
version = "0.1.8"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"
|
||||
checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
|
||||
|
||||
[[package]]
|
||||
name = "time-macros"
|
||||
version = "0.2.27"
|
||||
version = "0.2.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215"
|
||||
checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935"
|
||||
dependencies = [
|
||||
"num-conv",
|
||||
"time-core",
|
||||
@@ -4574,6 +4735,16 @@ dependencies = [
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinytemplate"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.11.0"
|
||||
@@ -5039,9 +5210,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.23.2"
|
||||
version = "1.23.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7"
|
||||
checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53"
|
||||
dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"js-sys",
|
||||
@@ -5261,9 +5432,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "1.0.7"
|
||||
version = "1.0.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d"
|
||||
checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
22
Cargo.toml
22
Cargo.toml
@@ -3,6 +3,7 @@
|
||||
members = [
|
||||
"crates/memdb",
|
||||
"crates/core",
|
||||
"crates/blob",
|
||||
"crates/server",
|
||||
"crates/cli",
|
||||
"crates/admin",
|
||||
@@ -12,14 +13,15 @@ members = [
|
||||
resolver = "2"
|
||||
|
||||
[workspace.package]
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
edition = "2021"
|
||||
|
||||
[workspace.dependencies]
|
||||
chrono = "0.4.45"
|
||||
clap = { version = "4.6.1", features = ["derive", "env"] }
|
||||
memdb = { path = "crates/memdb" }
|
||||
itertools = "0.14.0"
|
||||
bichon-memdb = { path = "crates/memdb" }
|
||||
bichon-blob = { path = "crates/blob" }
|
||||
itertools = "0.15.0"
|
||||
ring = { version = "0.17.14", features = ["std"] }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = "1.0.150"
|
||||
@@ -39,11 +41,11 @@ reqwest = { version = "0.12.24", default-features = false, features = [
|
||||
] }
|
||||
tokio-socks = "0.5.3"
|
||||
http = "1.4.2"
|
||||
regex = "1.12.3"
|
||||
regex = "1.12.4"
|
||||
email_address = "0.2.9"
|
||||
futures = "0.3.32"
|
||||
utf7-imap = "0.3.2"
|
||||
mail-parser = { version = '0.11.3', features = ["serde"] }
|
||||
mail-parser = { version = '0.11.4', features = ["serde"] }
|
||||
# mail-send = "0.5.2"
|
||||
tokio-rustls = { version = "0.26.4", default-features = false, features = [
|
||||
"ring",
|
||||
@@ -52,12 +54,12 @@ tokio-rustls = { version = "0.26.4", default-features = false, features = [
|
||||
timeago = "0.6.0"
|
||||
oauth2 = { version = "5.0.0", features = ["reqwest-blocking"] }
|
||||
url = { version = "2.5.8", features = ["serde"] }
|
||||
sysinfo = "0.39.2"
|
||||
sysinfo = "0.39.4"
|
||||
num_cpus = "1.17.0"
|
||||
rand = "0.10.1"
|
||||
encoding_rs = "0.8.35"
|
||||
webpki-roots = "1.0.7"
|
||||
rustls = { version = "0.23.40", default-features = false, features = ["ring"] }
|
||||
webpki-roots = "1.0.8"
|
||||
rustls = { version = "0.23.41", default-features = false, features = ["ring"] }
|
||||
rustls-pki-types = "1.14.1"
|
||||
tokio-io-timeout = "1.2.1"
|
||||
semver = "1.0.28"
|
||||
@@ -65,7 +67,7 @@ governor = "0.10.4"
|
||||
lru = "0.18.0"
|
||||
mime_guess = "2.0.5"
|
||||
hex = "0.4.3"
|
||||
time = { version = "0.3.47", features = [
|
||||
time = { version = "0.3.51", features = [
|
||||
"formatting",
|
||||
"parsing",
|
||||
"local-offset",
|
||||
@@ -84,7 +86,7 @@ mail-send = "0.6.0"
|
||||
rcgen = "0.14.8"
|
||||
rustls-pemfile = "2.2.0"
|
||||
blake3 = "1.8.5"
|
||||
uuid = { version = "1.23.2", features = ["v4", "serde"] }
|
||||
uuid = { version = "1.23.4", features = ["v4", "serde"] }
|
||||
fjall = { version = "3.1.5", features = ["lz4", "metrics", "bytes_1"] }
|
||||
tracing-log = "0.2.0"
|
||||
tokio-util = "0.7.18"
|
||||
|
||||
25
README.md
25
README.md
@@ -715,6 +715,31 @@ cargo test
|
||||
|
||||
Feel free to open an [Issue](https://github.com/rustmailer/bichon/issues) or join the [Discord](https://discord.gg/Bq4M2cDmF4) to discuss ideas.
|
||||
|
||||
#### Guidelines
|
||||
|
||||
1. **AI-assisted, not AI-authored.** Use AI to help analyze, debug, or draft code when unsure — but understand and review every change yourself before submitting. Don't submit unreviewed AI-generated content.
|
||||
2. **Keep PRs scoped to one issue.** Don't bundle unrelated changes (CI config, dependency bumps, fixes to other modules) into the same PR. Split them into separate PRs.
|
||||
3. **Frontend/backend changes go together.** If a change affects an API, data structure, or behavior with a frontend consumer, update the frontend in the same PR (or a clearly linked companion PR).
|
||||
4. **Unit tests are required.** New or fixed logic must include tests that reproduce the original issue and verify the fix. PRs without tests won't be merged.
|
||||
5. **Maintain backward compatibility.** Changes to data formats, protocols, configs, or APIs must state whether they're backward compatible. If not, include a migration plan.
|
||||
6. **State the blast radius.** PR descriptions must specify which modules/APIs/data are affected and any downstream impact.
|
||||
|
||||
### Commit Messages
|
||||
|
||||
Format: `<type>(<scope>): <subject>`
|
||||
|
||||
- **type**: `fix`, `feat`, `refactor`, `ci`, `test`, `docs`, `chore`
|
||||
- **scope**: affected module/component (e.g. `rustmailer#286`, `dedup_cache`)
|
||||
- **subject**: imperative, present tense, no period
|
||||
|
||||
Rules:
|
||||
- One logical change per commit — don't mix a fix with CI tweaks or unrelated module changes.
|
||||
- Reference the issue number when applicable (e.g. `fix(#286): ...`).
|
||||
- Body explains *why*, not just *what* — include root cause and how it was verified for non-trivial fixes.
|
||||
- Rebase before submitting — squash WIP/fixup commits into a clean, logical sequence.
|
||||
- No vague messages like `update`, `fix bug`, `wip`.
|
||||
|
||||
|
||||
## Tech Stack
|
||||
|
||||
| Layer | Technology |
|
||||
|
||||
@@ -17,4 +17,4 @@ serde_json.workspace = true
|
||||
|
||||
itertools.workspace = true
|
||||
snafu.workspace = true
|
||||
memdb.workspace = true
|
||||
bichon-memdb.workspace = true
|
||||
@@ -18,9 +18,9 @@ use bichon_core::{
|
||||
token::TokenType,
|
||||
users::{acl::AccessControl, role::RoleType},
|
||||
};
|
||||
use bichon_memdb::{Durability, MemDb};
|
||||
use console::style;
|
||||
use itertools::Itertools;
|
||||
use memdb::{Durability, MemDb};
|
||||
use native_db::*;
|
||||
use native_model::{native_model, Model};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -263,6 +263,8 @@ impl From<AccountV3> for AccountModel {
|
||||
auto_download_new_mailboxes: None,
|
||||
download_schedule: None,
|
||||
deleting: false,
|
||||
archive_rules: None,
|
||||
extraction_rules: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -574,6 +576,8 @@ impl From<BichonUserV2> for bichon_core::users::BichonUserV2 {
|
||||
acl: value.acl,
|
||||
theme: value.theme,
|
||||
language: value.language,
|
||||
sso_id: None,
|
||||
sso_provider: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
1105
crates/blob/Cargo.lock
generated
Normal file
1105
crates/blob/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
24
crates/blob/Cargo.toml
Normal file
24
crates/blob/Cargo.toml
Normal file
@@ -0,0 +1,24 @@
|
||||
[package]
|
||||
name = "bichon-blob"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Embedded KV storage engine for email"
|
||||
|
||||
[dependencies]
|
||||
crc32fast = "1.4"
|
||||
zstd = "0.13"
|
||||
lz4_flex = "0.13.1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
bincode = "1"
|
||||
tracing = "0.1"
|
||||
thiserror = "2"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
rand = "0.10.1"
|
||||
criterion = { version = "0.6", features = ["html_reports"] }
|
||||
|
||||
[[bench]]
|
||||
name = "benchmark"
|
||||
harness = false
|
||||
316
crates/blob/benches/benchmark.rs
Normal file
316
crates/blob/benches/benchmark.rs
Normal file
@@ -0,0 +1,316 @@
|
||||
use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput};
|
||||
use std::time::Duration;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use bichon_blob::{Codec, Config, Engine};
|
||||
|
||||
fn make_key(seed: u64) -> [u8; 32] {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..8].copy_from_slice(&seed.to_le_bytes());
|
||||
key
|
||||
}
|
||||
|
||||
fn make_value(size: usize) -> Vec<u8> {
|
||||
let mut v = Vec::with_capacity(size);
|
||||
// Fill with somewhat realistic text-like data so compression works
|
||||
let pattern = b"The quick brown fox jumps over the lazy dog. ";
|
||||
while v.len() < size {
|
||||
let rem = size - v.len();
|
||||
let n = rem.min(pattern.len());
|
||||
v.extend_from_slice(&pattern[..n]);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
pub fn bench_write_small(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("write");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(1024); // 1 KB
|
||||
let mut counter = 0u64;
|
||||
|
||||
group.bench_function("1KB", |b| {
|
||||
b.iter_batched(
|
||||
|| {
|
||||
counter += 1;
|
||||
(make_key(counter), value.clone())
|
||||
},
|
||||
|(key, val)| {
|
||||
engine
|
||||
.write("bench", key, &val, Codec::Zstd)
|
||||
.unwrap()
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_write_medium(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("write");
|
||||
group.throughput(Throughput::Bytes(64 * 1024));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(64 * 1024); // 64 KB
|
||||
let mut counter = 0u64;
|
||||
|
||||
group.bench_function("64KB", |b| {
|
||||
b.iter_batched(
|
||||
|| {
|
||||
counter += 1;
|
||||
(make_key(counter), value.clone())
|
||||
},
|
||||
|(key, val)| {
|
||||
engine
|
||||
.write("bench", key, &val, Codec::Zstd)
|
||||
.unwrap()
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_write_large(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("write");
|
||||
group.throughput(Throughput::Bytes(1024 * 1024));
|
||||
group.measurement_time(Duration::from_secs(15));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(1024 * 1024); // 1 MB
|
||||
let mut counter = 0u64;
|
||||
|
||||
group.bench_function("1MB", |b| {
|
||||
b.iter_batched(
|
||||
|| {
|
||||
counter += 1;
|
||||
(make_key(counter), value.clone())
|
||||
},
|
||||
|(key, val)| {
|
||||
engine
|
||||
.write("bench", key, &val, Codec::Zstd)
|
||||
.unwrap()
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_read_cache_hit(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("read");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
// Pre-populate: 10 keys, all in same bucket → cache hit after first read
|
||||
let value = make_value(4096);
|
||||
for i in 0..10u64 {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut counter = 0u64;
|
||||
group.bench_function("cache_hit", |b| {
|
||||
b.iter(|| {
|
||||
let key = make_key(counter % 10);
|
||||
counter += 1;
|
||||
std::hint::black_box(engine.read("bench", &key).unwrap());
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_read_cache_miss(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("read");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut config = Config::default();
|
||||
config.lru_bucket_count = 8; // Small cache to force misses
|
||||
let engine = Engine::open(dir.path(), config).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(4096);
|
||||
// Write 1000 keys spread across all 16 buckets — small LRU will thrash
|
||||
for i in 0..1000u64 {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut counter = 0u64;
|
||||
group.bench_function("cache_miss", |b| {
|
||||
b.iter(|| {
|
||||
let key = make_key(counter % 1000);
|
||||
counter += 1;
|
||||
std::hint::black_box(engine.read("bench", &key).unwrap());
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_read_large_value(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("read");
|
||||
group.throughput(Throughput::Bytes(1024 * 1024));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(1024 * 1024); // 1 MB
|
||||
for i in 0..5u64 {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut counter = 0u64;
|
||||
group.bench_function("1MB_cache_hit", |b| {
|
||||
b.iter(|| {
|
||||
let key = make_key(counter % 5);
|
||||
counter += 1;
|
||||
std::hint::black_box(engine.read("bench", &key).unwrap());
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_delete(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("delete");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
group.bench_function("delete", |b| {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(4096);
|
||||
let mut counter = 0u64;
|
||||
|
||||
b.iter_batched(
|
||||
|| {
|
||||
counter += 1;
|
||||
let key = make_key(counter);
|
||||
engine
|
||||
.write("bench", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
key
|
||||
},
|
||||
|key| {
|
||||
engine.delete("bench", &key).unwrap();
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_mixed_workload(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("mixed");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(15));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
// Pre-populate with 500 entries
|
||||
let value = make_value(8192);
|
||||
for i in 0..500u64 {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut counter: u64 = 500;
|
||||
group.bench_function("80w_15r_5d", |b| {
|
||||
b.iter(|| {
|
||||
counter += 1;
|
||||
let op = counter % 100;
|
||||
match op {
|
||||
0..=79 => {
|
||||
// 80% writes
|
||||
let key = make_key(counter);
|
||||
let val = make_value(4096);
|
||||
engine.write("bench", key, &val, Codec::Zstd).unwrap();
|
||||
}
|
||||
80..=94 => {
|
||||
// 15% reads
|
||||
std::hint::black_box(engine.read("bench", &make_key(counter % 500)).unwrap());
|
||||
}
|
||||
_ => {
|
||||
// 5% deletes
|
||||
if counter % 2 == 0 {
|
||||
let key = make_key(counter % 500);
|
||||
let _ = engine.delete("bench", &key);
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_gc(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("gc");
|
||||
group.measurement_time(Duration::from_secs(30));
|
||||
group.sample_size(10);
|
||||
|
||||
group.bench_function("gc_30pct_deleted", |b| {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
// Fill a segment with ~1000 entries, then delete 30%
|
||||
let value = make_value(200_000); // 200KB each → ~1000 entries to fill 256MB
|
||||
let n = 1200u64;
|
||||
for i in 0..n {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
// Delete ~30%
|
||||
for i in (0..n).step_by(3) {
|
||||
engine.delete("bench", &make_key(i)).unwrap();
|
||||
}
|
||||
|
||||
b.iter(|| {
|
||||
engine.gc("bench").unwrap();
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
benches,
|
||||
bench_write_small,
|
||||
bench_write_medium,
|
||||
bench_write_large,
|
||||
bench_read_cache_hit,
|
||||
bench_read_cache_miss,
|
||||
bench_read_large_value,
|
||||
bench_delete,
|
||||
bench_mixed_workload,
|
||||
bench_gc,
|
||||
);
|
||||
criterion_main!(benches);
|
||||
284
crates/blob/src/account.rs
Normal file
284
crates/blob/src/account.rs
Normal file
@@ -0,0 +1,284 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex, RwLock};
|
||||
|
||||
use crate::bucket::{self, BucketFile, IndexRecord};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::file_pool::FilePool;
|
||||
use crate::meta::{AccountMeta, SegmentStats};
|
||||
use crate::segment::{self, SegmentReader, SegmentWriter};
|
||||
use crate::types::Codec;
|
||||
|
||||
// ── AccountHandle ──────────────────────────────────────────────────────────
|
||||
|
||||
pub struct AccountHandle {
|
||||
id: String,
|
||||
dir: PathBuf,
|
||||
inner: RwLock<AccountInner>,
|
||||
pub(crate) write_mutex: Mutex<()>,
|
||||
file_pool: FilePool,
|
||||
}
|
||||
|
||||
impl AccountHandle {
|
||||
pub fn id(&self) -> &str {
|
||||
&self.id
|
||||
}
|
||||
|
||||
pub fn dir(&self) -> &Path {
|
||||
&self.dir
|
||||
}
|
||||
|
||||
/// Open an existing account.
|
||||
pub fn open(store_root: &Path, account_id: &str) -> Result<Arc<Self>> {
|
||||
let dir = store_root.join("accounts").join(account_id);
|
||||
if !dir.exists() {
|
||||
return Err(Error::AccountNotFound(account_id.to_string()));
|
||||
}
|
||||
let inner = AccountInner::open(&dir)?;
|
||||
Ok(Arc::new(Self {
|
||||
id: account_id.to_string(),
|
||||
dir,
|
||||
inner: RwLock::new(inner),
|
||||
write_mutex: Mutex::new(()),
|
||||
file_pool: FilePool::new(8),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Create a new account.
|
||||
pub fn create(store_root: &Path, account_id: &str) -> Result<Arc<Self>> {
|
||||
let dir = store_root.join("accounts").join(account_id);
|
||||
if dir.exists() {
|
||||
return Err(Error::AccountAlreadyExists(account_id.to_string()));
|
||||
}
|
||||
let inner = AccountInner::create(&dir, account_id)?;
|
||||
Ok(Arc::new(Self {
|
||||
id: account_id.to_string(),
|
||||
dir,
|
||||
inner: RwLock::new(inner),
|
||||
write_mutex: Mutex::new(()),
|
||||
file_pool: FilePool::new(8),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Lock the inner state for reading.
|
||||
pub fn read(&self) -> std::sync::RwLockReadGuard<'_, AccountInner> {
|
||||
self.inner.read().unwrap()
|
||||
}
|
||||
|
||||
/// Lock the inner state for writing.
|
||||
pub fn write(&self) -> std::sync::RwLockWriteGuard<'_, AccountInner> {
|
||||
self.inner.write().unwrap()
|
||||
}
|
||||
|
||||
/// Get a cached file handle for a segment.
|
||||
pub fn get_segment_file(&self, seg_id: u32, path: &Path) -> Result<Arc<Mutex<std::fs::File>>> {
|
||||
self.file_pool.get(seg_id, path)
|
||||
}
|
||||
|
||||
/// Invalidate cached file handles for a segment (after GC).
|
||||
pub fn invalidate_file_cache(&self, seg_id: u32) {
|
||||
self.file_pool.invalidate(seg_id);
|
||||
}
|
||||
}
|
||||
|
||||
// ── AccountInner ───────────────────────────────────────────────────────────
|
||||
|
||||
pub struct AccountInner {
|
||||
dir: PathBuf,
|
||||
meta: AccountMeta,
|
||||
active_writer: SegmentWriter,
|
||||
readers: HashMap<u32, SegmentReader>,
|
||||
}
|
||||
|
||||
impl AccountInner {
|
||||
fn open(dir: &Path) -> Result<Self> {
|
||||
let meta = AccountMeta::load(dir)?;
|
||||
|
||||
let seg_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(meta.active_segment_id));
|
||||
let active_writer = if seg_path.exists() {
|
||||
SegmentWriter::open_append(seg_path, meta.active_segment_id)?
|
||||
} else {
|
||||
fs::create_dir_all(dir.join("segments"))?;
|
||||
SegmentWriter::create(seg_path, meta.active_segment_id)?
|
||||
};
|
||||
|
||||
let mut readers = HashMap::new();
|
||||
for (&seg_id, stats) in &meta.segments {
|
||||
if stats.sealed {
|
||||
let seg_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(seg_id));
|
||||
if seg_path.exists() {
|
||||
readers.insert(seg_id, SegmentReader::open(seg_path, seg_id)?);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
dir: dir.to_path_buf(),
|
||||
meta,
|
||||
active_writer,
|
||||
readers,
|
||||
})
|
||||
}
|
||||
|
||||
fn create(dir: &Path, account_id: &str) -> Result<Self> {
|
||||
fs::create_dir_all(dir.join("segments"))?;
|
||||
BucketFile::ensure_dir(dir)?;
|
||||
|
||||
let meta = AccountMeta::new(account_id.to_string(), 1);
|
||||
|
||||
let seg_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(1));
|
||||
let active_writer = SegmentWriter::create(seg_path, 1)?;
|
||||
|
||||
meta.save(dir)?;
|
||||
|
||||
Ok(Self {
|
||||
dir: dir.to_path_buf(),
|
||||
meta,
|
||||
active_writer,
|
||||
readers: HashMap::new(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn meta(&self) -> &AccountMeta {
|
||||
&self.meta
|
||||
}
|
||||
|
||||
/// Mark the segment as indexed up to the given offset and persist meta.
|
||||
pub fn mark_indexed(&mut self, segment_id: u32, indexed_up_to_offset: u64) -> Result<()> {
|
||||
if let Some(stats) = self.meta.segments.get_mut(&segment_id) {
|
||||
if indexed_up_to_offset > stats.indexed_up_to_offset {
|
||||
stats.indexed_up_to_offset = indexed_up_to_offset;
|
||||
}
|
||||
}
|
||||
self.meta.save(&self.dir)
|
||||
}
|
||||
|
||||
/// Append an entry without fsync.
|
||||
pub fn append_entry(
|
||||
&mut self,
|
||||
key: [u8; 32],
|
||||
data: &[u8],
|
||||
flags: u8,
|
||||
codec: Codec,
|
||||
) -> Result<(u32, u64, u32)> {
|
||||
if self.active_writer.is_full() {
|
||||
self.seal_active()?;
|
||||
}
|
||||
|
||||
use crate::segment::Entry;
|
||||
let entry = if flags == 1 {
|
||||
Entry::tombstone(key)
|
||||
} else {
|
||||
Entry::new(key, data, flags, codec)
|
||||
};
|
||||
|
||||
let data_size = entry.data.len() as u32;
|
||||
let segment_id = self.active_writer.id();
|
||||
let offset = self.active_writer.append(&entry)?;
|
||||
|
||||
let stats = self
|
||||
.meta
|
||||
.segments
|
||||
.entry(segment_id)
|
||||
.or_insert_with(|| SegmentStats::new(segment_id));
|
||||
stats.total_bytes += data_size as u64;
|
||||
if flags == 1 {
|
||||
stats.deleted_bytes += entry.raw_size as u64;
|
||||
}
|
||||
stats.recompute_ratio();
|
||||
|
||||
Ok((segment_id, offset, data_size))
|
||||
}
|
||||
|
||||
/// Fsync the active segment and persist meta.
|
||||
pub fn flush_active(&mut self) -> Result<()> {
|
||||
self.active_writer.fsync()?;
|
||||
self.meta.save(&self.dir)
|
||||
}
|
||||
|
||||
/// Write an entry with fsync.
|
||||
pub fn write_entry(
|
||||
&mut self,
|
||||
key: [u8; 32],
|
||||
data: &[u8],
|
||||
flags: u8,
|
||||
codec: Codec,
|
||||
) -> Result<(u32, u64, u32)> {
|
||||
let result = self.append_entry(key, data, flags, codec)?;
|
||||
self.flush_active()?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn seal_active(&mut self) -> Result<()> {
|
||||
let old_id = self.active_writer.id();
|
||||
let old_stats = self
|
||||
.meta
|
||||
.segments
|
||||
.entry(old_id)
|
||||
.or_insert_with(|| SegmentStats::new(old_id));
|
||||
old_stats.sealed = true;
|
||||
|
||||
let seg_path = self
|
||||
.dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(old_id));
|
||||
self.readers
|
||||
.insert(old_id, SegmentReader::open(seg_path, old_id)?);
|
||||
|
||||
let new_id = old_id + 1;
|
||||
self.meta.active_segment_id = new_id;
|
||||
let new_path = self
|
||||
.dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(new_id));
|
||||
self.active_writer = SegmentWriter::create(new_path, new_id)?;
|
||||
self.meta.save(&self.dir)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the on-disk path for a segment.
|
||||
pub fn segment_path(&self, segment_id: u32) -> Result<PathBuf> {
|
||||
let filename = segment::segment_filename(segment_id);
|
||||
let path = self.dir.join("segments").join(&filename);
|
||||
if path.exists() {
|
||||
Ok(path)
|
||||
} else {
|
||||
Err(Error::SegmentNotFound(segment_id))
|
||||
}
|
||||
}
|
||||
|
||||
/// Append index record to the appropriate bucket file.
|
||||
pub fn append_index(&self, record: &IndexRecord) -> Result<()> {
|
||||
let bucket_id = bucket::bucket_id(&record.key);
|
||||
let bf = BucketFile::open(&self.dir, bucket_id);
|
||||
bf.append(record)
|
||||
}
|
||||
|
||||
/// Return list of sealed segment IDs.
|
||||
pub fn sealed_segments(&self) -> Vec<u32> {
|
||||
self.meta
|
||||
.segments
|
||||
.iter()
|
||||
.filter(|(_, s)| s.sealed)
|
||||
.map(|(id, _)| *id)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// All segment IDs (including active).
|
||||
pub fn all_segment_ids(&self) -> Vec<u32> {
|
||||
let mut ids: Vec<u32> = self.meta.segments.keys().copied().collect();
|
||||
if !ids.contains(&self.meta.active_segment_id) {
|
||||
ids.push(self.meta.active_segment_id);
|
||||
}
|
||||
ids.sort_unstable();
|
||||
ids
|
||||
}
|
||||
}
|
||||
329
crates/blob/src/bucket.rs
Normal file
329
crates/blob/src/bucket.rs
Normal file
@@ -0,0 +1,329 @@
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::types::{BUCKET_COUNT, INDEX_RECORD_SIZE};
|
||||
|
||||
/// On-disk format: 52 bytes per record.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct IndexRecord {
|
||||
pub key: [u8; 32],
|
||||
pub segment_id: u32,
|
||||
pub offset: u64,
|
||||
pub data_size: u32,
|
||||
pub flags: u8,
|
||||
}
|
||||
|
||||
impl IndexRecord {
|
||||
pub fn new(key: [u8; 32], segment_id: u32, offset: u64, data_size: u32, flags: u8) -> Self {
|
||||
Self {
|
||||
key,
|
||||
segment_id,
|
||||
offset,
|
||||
data_size,
|
||||
flags,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_tombstone(&self) -> bool {
|
||||
self.flags == 1
|
||||
}
|
||||
|
||||
pub fn encode(&self) -> [u8; INDEX_RECORD_SIZE] {
|
||||
let mut buf = [0u8; INDEX_RECORD_SIZE];
|
||||
buf[0..32].copy_from_slice(&self.key);
|
||||
buf[32..36].copy_from_slice(&self.segment_id.to_le_bytes());
|
||||
buf[36..44].copy_from_slice(&self.offset.to_le_bytes());
|
||||
buf[44..48].copy_from_slice(&self.data_size.to_le_bytes());
|
||||
buf[48] = self.flags;
|
||||
// bytes 49..52 are padding (keep zero)
|
||||
buf
|
||||
}
|
||||
|
||||
pub fn decode(buf: &[u8; INDEX_RECORD_SIZE]) -> Self {
|
||||
let mut key = [0u8; 32];
|
||||
key.copy_from_slice(&buf[0..32]);
|
||||
let segment_id = u32::from_le_bytes(buf[32..36].try_into().unwrap());
|
||||
let offset = u64::from_le_bytes(buf[36..44].try_into().unwrap());
|
||||
let data_size = u32::from_le_bytes(buf[44..48].try_into().unwrap());
|
||||
let flags = buf[48];
|
||||
Self {
|
||||
key,
|
||||
segment_id,
|
||||
offset,
|
||||
data_size,
|
||||
flags,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Represents a loaded and deduplicated bucket in memory.
|
||||
pub struct BucketIndex {
|
||||
pub bucket_id: u16,
|
||||
/// Records sorted by key, deduplicated (one record per key, latest wins).
|
||||
pub records: Vec<IndexRecord>,
|
||||
}
|
||||
|
||||
impl BucketIndex {
|
||||
/// Build from raw records: sort by key, dedup keeping the one with max offset.
|
||||
pub fn from_records(mut records: Vec<IndexRecord>, bucket_id: u16) -> Self {
|
||||
records.sort_by_key(|a| a.key);
|
||||
// Dedup: keep last (max offset) for each key
|
||||
let mut deduped = Vec::with_capacity(records.len());
|
||||
let mut i = 0;
|
||||
while i < records.len() {
|
||||
let mut best = i;
|
||||
let mut j = i + 1;
|
||||
while j < records.len() && records[j].key == records[i].key {
|
||||
if records[j].offset > records[best].offset {
|
||||
best = j;
|
||||
}
|
||||
j += 1;
|
||||
}
|
||||
deduped.push(records[best].clone());
|
||||
i = j;
|
||||
}
|
||||
Self {
|
||||
bucket_id,
|
||||
records: deduped,
|
||||
}
|
||||
}
|
||||
|
||||
/// Binary search for a key. Returns the record if found.
|
||||
pub fn find(&self, key: &[u8; 32]) -> Option<&IndexRecord> {
|
||||
match self.records.binary_search_by(|r| r.key.cmp(key)) {
|
||||
Ok(idx) => Some(&self.records[idx]),
|
||||
Err(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a new record and maintain sorted order.
|
||||
pub fn insert(&mut self, record: IndexRecord) {
|
||||
match self.records.binary_search_by(|r| r.key.cmp(&record.key)) {
|
||||
Ok(idx) => {
|
||||
// Replace if newer (larger offset)
|
||||
if record.offset > self.records[idx].offset {
|
||||
self.records[idx] = record;
|
||||
}
|
||||
}
|
||||
Err(idx) => {
|
||||
self.records.insert(idx, record);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.records.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.records.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// Manages a bucket index file on disk.
|
||||
pub struct BucketFile {
|
||||
path: PathBuf,
|
||||
bucket_id: u16,
|
||||
}
|
||||
|
||||
impl BucketFile {
|
||||
pub fn path_for(account_dir: &Path, bucket_id: u16) -> PathBuf {
|
||||
account_dir.join("buckets").join(format!("{:02x}.idx", bucket_id))
|
||||
}
|
||||
|
||||
pub fn open(account_dir: &Path, bucket_id: u16) -> Self {
|
||||
Self {
|
||||
path: Self::path_for(account_dir, bucket_id),
|
||||
bucket_id,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub fn bucket_id(&self) -> u16 {
|
||||
self.bucket_id
|
||||
}
|
||||
|
||||
/// Ensure the buckets directory exists.
|
||||
pub fn ensure_dir(account_dir: &Path) -> Result<()> {
|
||||
let dir = account_dir.join("buckets");
|
||||
std::fs::create_dir_all(&dir)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append a single record to the bucket file.
|
||||
pub fn append(&self, record: &IndexRecord) -> Result<()> {
|
||||
let mut file = OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&self.path)?;
|
||||
file.write_all(&record.encode())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append multiple records at once.
|
||||
pub fn append_batch(&self, records: &[IndexRecord]) -> Result<()> {
|
||||
if records.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut file = OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&self.path)?;
|
||||
for r in records {
|
||||
file.write_all(&r.encode())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Load all records from the bucket file.
|
||||
/// If the file size is not a multiple of INDEX_RECORD_SIZE (partial write),
|
||||
/// the trailing bytes are silently ignored.
|
||||
pub fn load_all(&self) -> Result<Vec<IndexRecord>> {
|
||||
if !self.path.exists() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let data = std::fs::read(&self.path)?;
|
||||
let remainder = data.len() % INDEX_RECORD_SIZE;
|
||||
let count = data.len() / INDEX_RECORD_SIZE;
|
||||
let mut records = Vec::with_capacity(count);
|
||||
for i in 0..count {
|
||||
let start = i * INDEX_RECORD_SIZE;
|
||||
let end = start + INDEX_RECORD_SIZE;
|
||||
let buf: &[u8; INDEX_RECORD_SIZE] = data[start..end]
|
||||
.try_into()
|
||||
.map_err(|_| crate::error::Error::BucketIndexCorrupt {
|
||||
path: self.path.clone(),
|
||||
reason: format!("unexpected file size {}, not a multiple of {}", data.len(), INDEX_RECORD_SIZE),
|
||||
})?;
|
||||
records.push(IndexRecord::decode(buf));
|
||||
}
|
||||
if remainder > 0 {
|
||||
tracing::warn!(
|
||||
"Bucket file {:?} has {} trailing bytes (expected multiple of {}), ignoring",
|
||||
self.path, remainder, INDEX_RECORD_SIZE
|
||||
);
|
||||
}
|
||||
Ok(records)
|
||||
}
|
||||
|
||||
/// Load all records, sort, and deduplicate into a BucketIndex.
|
||||
pub fn load_index(&self) -> Result<BucketIndex> {
|
||||
let records = self.load_all()?;
|
||||
Ok(BucketIndex::from_records(records, self.bucket_id))
|
||||
}
|
||||
|
||||
/// Rewrite the bucket file with a sorted, deduplicated set of records.
|
||||
/// Uses atomic temp+rename to be safe on NFS.
|
||||
pub fn rewrite(&self, records: &[IndexRecord]) -> Result<()> {
|
||||
let mut buf = Vec::with_capacity(records.len() * INDEX_RECORD_SIZE);
|
||||
for r in records {
|
||||
buf.extend_from_slice(&r.encode());
|
||||
}
|
||||
crate::fs::create_atomic(&self.path, &buf)
|
||||
}
|
||||
|
||||
/// Delete the bucket file.
|
||||
pub fn delete(&self) -> Result<()> {
|
||||
if self.path.exists() {
|
||||
std::fs::remove_file(&self.path)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute bucket_id from a key's first 2 bytes.
|
||||
pub fn bucket_id(key: &[u8; 32]) -> u16 {
|
||||
u16::from_be_bytes([key[0], key[1]]) % BUCKET_COUNT
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_index_record_encode_decode() {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&[1, 2, 3, 4]);
|
||||
let rec = IndexRecord::new(key, 5, 12345, 500, 0);
|
||||
let encoded = rec.encode();
|
||||
let decoded = IndexRecord::decode(&encoded);
|
||||
assert_eq!(rec, decoded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_id_deterministic() {
|
||||
let mut key = [0u8; 32];
|
||||
key[0] = 0x00;
|
||||
key[1] = 0x0F;
|
||||
assert_eq!(bucket_id(&key), 15);
|
||||
key[0] = 0x00;
|
||||
key[1] = 0x10;
|
||||
assert_eq!(bucket_id(&key), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_append_and_load() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let bucket = BucketFile::open(dir.path(), 0);
|
||||
BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
|
||||
let r1 = IndexRecord::new([1u8; 32], 1, 100, 50, 0);
|
||||
let r2 = IndexRecord::new([2u8; 32], 1, 200, 60, 0);
|
||||
|
||||
bucket.append(&r1).unwrap();
|
||||
bucket.append(&r2).unwrap();
|
||||
|
||||
let loaded = bucket.load_all().unwrap();
|
||||
assert_eq!(loaded.len(), 2);
|
||||
assert_eq!(loaded[0].key, [1u8; 32]);
|
||||
assert_eq!(loaded[1].key, [2u8; 32]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_index_dedup() {
|
||||
let recs = vec![
|
||||
IndexRecord::new([1u8; 32], 1, 100, 50, 0),
|
||||
IndexRecord::new([1u8; 32], 2, 200, 50, 0), // newer offset wins
|
||||
IndexRecord::new([2u8; 32], 1, 300, 60, 0),
|
||||
];
|
||||
let idx = BucketIndex::from_records(recs, 0);
|
||||
assert_eq!(idx.len(), 2);
|
||||
let found = idx.find(&[1u8; 32]).unwrap();
|
||||
assert_eq!(found.segment_id, 2);
|
||||
assert_eq!(found.offset, 200);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_index_find_missing() {
|
||||
let recs = vec![IndexRecord::new([1u8; 32], 1, 100, 50, 0)];
|
||||
let idx = BucketIndex::from_records(recs, 0);
|
||||
assert!(idx.find(&[99u8; 32]).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_rewrite() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let bucket = BucketFile::open(dir.path(), 0);
|
||||
BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
|
||||
let r1 = IndexRecord::new([3u8; 32], 1, 300, 70, 0);
|
||||
let r2 = IndexRecord::new([1u8; 32], 1, 100, 50, 0);
|
||||
bucket.append(&r1).unwrap();
|
||||
bucket.append(&r2).unwrap();
|
||||
|
||||
// Rewrite sorted
|
||||
let sorted = vec![r2.clone(), r1.clone()];
|
||||
bucket.rewrite(&sorted).unwrap();
|
||||
|
||||
let loaded = bucket.load_all().unwrap();
|
||||
assert_eq!(loaded.len(), 2);
|
||||
assert_eq!(loaded[0].key, [1u8; 32]);
|
||||
assert_eq!(loaded[1].key, [3u8; 32]);
|
||||
}
|
||||
}
|
||||
219
crates/blob/src/cache.rs
Normal file
219
crates/blob/src/cache.rs
Normal file
@@ -0,0 +1,219 @@
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use crate::bucket::{BucketFile, BucketIndex, IndexRecord};
|
||||
use crate::error::Result;
|
||||
|
||||
type CacheKey = (String, u16);
|
||||
|
||||
/// Thread-safe LRU bucket cache with single-lock interior.
|
||||
/// Eliminates the TOCTOU race in the old two-Mutex design.
|
||||
pub struct BucketCache {
|
||||
inner: Mutex<CacheInner>,
|
||||
}
|
||||
|
||||
struct CacheInner {
|
||||
max_entries: usize,
|
||||
entries: Vec<CacheEntry>,
|
||||
index: HashMap<CacheKey, usize>,
|
||||
}
|
||||
|
||||
struct CacheEntry {
|
||||
key: CacheKey,
|
||||
index: BucketIndex,
|
||||
}
|
||||
|
||||
impl BucketCache {
|
||||
pub fn new(max_entries: usize) -> Self {
|
||||
Self {
|
||||
inner: Mutex::new(CacheInner {
|
||||
max_entries: max_entries.max(1),
|
||||
entries: Vec::new(),
|
||||
index: HashMap::new(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get or load a bucket index. Eliminates TOCTOU via double-checked locking.
|
||||
pub fn get_or_load(
|
||||
&self,
|
||||
account: &str,
|
||||
bucket_id: u16,
|
||||
account_dir: &Path,
|
||||
) -> Result<Vec<IndexRecord>> {
|
||||
let key: CacheKey = (account.to_string(), bucket_id);
|
||||
|
||||
// Check cache
|
||||
{
|
||||
let inner = self.inner.lock().unwrap();
|
||||
if let Some(&pos) = inner.index.get(&key) {
|
||||
return Ok(inner.entries[pos].index.records.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Load from disk
|
||||
let bucket_file = BucketFile::open(account_dir, bucket_id);
|
||||
let bucket_index = bucket_file.load_index()?;
|
||||
let records = bucket_index.records.clone();
|
||||
|
||||
// Insert with double-check (another thread might have beaten us)
|
||||
{
|
||||
let mut inner = self.inner.lock().unwrap();
|
||||
if let Some(&pos) = inner.index.get(&key) {
|
||||
return Ok(inner.entries[pos].index.records.clone());
|
||||
}
|
||||
// Evict if full
|
||||
if inner.entries.len() >= inner.max_entries {
|
||||
if let Some(evicted) = inner.entries.pop() {
|
||||
inner.index.remove(&evicted.key);
|
||||
}
|
||||
}
|
||||
// Insert at front
|
||||
inner.entries.insert(0, CacheEntry {
|
||||
key: key.clone(),
|
||||
index: bucket_index,
|
||||
});
|
||||
// Rebuild index
|
||||
inner.index.clear();
|
||||
for i in 0..inner.entries.len() {
|
||||
let key = inner.entries[i].key.clone();
|
||||
inner.index.insert(key, i);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(records)
|
||||
}
|
||||
|
||||
/// Insert or update a single record in a cached bucket.
|
||||
pub fn update_record(&self, account: &str, bucket_id: u16, record: IndexRecord) {
|
||||
let key: CacheKey = (account.to_string(), bucket_id);
|
||||
let mut inner = self.inner.lock().unwrap();
|
||||
|
||||
if let Some(&pos) = inner.index.get(&key) {
|
||||
inner.entries[pos].index.insert(record);
|
||||
// Move to front
|
||||
let entry = inner.entries.remove(pos);
|
||||
inner.entries.insert(0, entry);
|
||||
// Rebuild index
|
||||
inner.index.clear();
|
||||
for i in 0..inner.entries.len() {
|
||||
let entry_key = inner.entries[i].key.clone();
|
||||
inner.index.insert(entry_key, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Invalidate a cached bucket (after GC rewrites bucket files).
|
||||
pub fn invalidate(&self, account: &str, bucket_id: u16) {
|
||||
let key: CacheKey = (account.to_string(), bucket_id);
|
||||
let mut inner = self.inner.lock().unwrap();
|
||||
|
||||
if let Some(&pos) = inner.index.get(&key) {
|
||||
inner.entries.remove(pos);
|
||||
inner.index.clear();
|
||||
for i in 0..inner.entries.len() {
|
||||
let entry_key = inner.entries[i].key.clone();
|
||||
inner.index.insert(entry_key, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.inner.lock().unwrap().entries.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.inner.lock().unwrap().entries.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::IndexRecord;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_cache_miss_loads_from_disk() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
let bf = BucketFile::open(dir.path(), 0);
|
||||
bf.append(&IndexRecord::new([1u8; 32], 1, 100, 50, 0))
|
||||
.unwrap();
|
||||
|
||||
let cache = BucketCache::new(10);
|
||||
let records = cache
|
||||
.get_or_load("test", 0, dir.path())
|
||||
.unwrap();
|
||||
assert_eq!(records.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_hit() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
let bf = BucketFile::open(dir.path(), 0);
|
||||
bf.append(&IndexRecord::new([2u8; 32], 1, 200, 60, 0))
|
||||
.unwrap();
|
||||
|
||||
let cache = BucketCache::new(10);
|
||||
let _ = cache.get_or_load("test", 0, dir.path()).unwrap();
|
||||
let records = cache
|
||||
.get_or_load("test", 0, dir.path())
|
||||
.unwrap();
|
||||
assert_eq!(records.len(), 1);
|
||||
assert_eq!(cache.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_eviction() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
let cache = BucketCache::new(2);
|
||||
|
||||
for b in 0..4 {
|
||||
let bf = BucketFile::open(dir.path(), b);
|
||||
bf.append(&IndexRecord::new([b as u8; 32], 1, 100, 50, 0))
|
||||
.unwrap();
|
||||
let _ = cache.get_or_load("test", b, dir.path()).unwrap();
|
||||
}
|
||||
|
||||
assert!(cache.len() <= 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_concurrent_get_or_load_no_deadlock() {
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
let bf = BucketFile::open(dir.path(), 0);
|
||||
for i in 0..10u8 {
|
||||
bf.append(&IndexRecord::new([i; 32], 1, i as u64 * 100, 50, 0))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let cache = Arc::new(BucketCache::new(10));
|
||||
let dir_path = dir.path().to_path_buf();
|
||||
|
||||
let mut handles = vec![];
|
||||
for _ in 0..4 {
|
||||
let cache = cache.clone();
|
||||
let dir_path = dir_path.clone();
|
||||
handles.push(thread::spawn(move || {
|
||||
for _ in 0..100 {
|
||||
let records = cache
|
||||
.get_or_load("test", 0, &dir_path)
|
||||
.unwrap();
|
||||
assert_eq!(records.len(), 10);
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
65
crates/blob/src/checksum.rs
Normal file
65
crates/blob/src/checksum.rs
Normal file
@@ -0,0 +1,65 @@
|
||||
use crc32fast::Hasher;
|
||||
|
||||
pub fn crc32(data: &[u8]) -> u32 {
|
||||
let mut h = Hasher::new();
|
||||
h.update(data);
|
||||
h.finalize()
|
||||
}
|
||||
|
||||
pub struct CrcWriter {
|
||||
hasher: Hasher,
|
||||
}
|
||||
|
||||
impl Default for CrcWriter {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl CrcWriter {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
hasher: Hasher::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update(&mut self, data: &[u8]) {
|
||||
self.hasher.update(data);
|
||||
}
|
||||
|
||||
pub fn finalize(self) -> u32 {
|
||||
self.hasher.finalize()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_crc32_deterministic() {
|
||||
let a = crc32(b"hello");
|
||||
let b = crc32(b"hello");
|
||||
assert_eq!(a, b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crc32_different() {
|
||||
let a = crc32(b"hello");
|
||||
let b = crc32(b"world");
|
||||
assert!(a != b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crc_writer_matches_crc32() {
|
||||
let mut w = CrcWriter::new();
|
||||
w.update(b"hello");
|
||||
w.update(b" world");
|
||||
assert_eq!(w.finalize(), crc32(b"hello world"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crc32_empty() {
|
||||
assert_eq!(crc32(b""), 0);
|
||||
}
|
||||
}
|
||||
97
crates/blob/src/compress.rs
Normal file
97
crates/blob/src/compress.rs
Normal file
@@ -0,0 +1,97 @@
|
||||
use crate::types::Codec;
|
||||
|
||||
pub fn compress(data: &[u8], codec: Codec, threshold: usize, level: i32) -> (Vec<u8>, Codec) {
|
||||
if data.len() < threshold {
|
||||
return (data.to_vec(), Codec::None);
|
||||
}
|
||||
let (compressed, actual_codec) = match codec {
|
||||
Codec::Zstd => {
|
||||
match zstd::encode_all(data, level) {
|
||||
Ok(out) => (out, Codec::Zstd),
|
||||
Err(e) => {
|
||||
tracing::warn!("zstd compression failed, storing uncompressed: {}", e);
|
||||
(data.to_vec(), Codec::None)
|
||||
}
|
||||
}
|
||||
}
|
||||
Codec::Lz4 => {
|
||||
let out = lz4_flex::compress(data);
|
||||
(out, Codec::Lz4)
|
||||
}
|
||||
Codec::None => (data.to_vec(), Codec::None),
|
||||
};
|
||||
// If compression made it larger, store uncompressed
|
||||
if compressed.len() >= data.len() {
|
||||
(data.to_vec(), Codec::None)
|
||||
} else {
|
||||
(compressed, actual_codec)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decompress(data: &[u8], codec: Codec, raw_size: usize) -> crate::error::Result<Vec<u8>> {
|
||||
match codec {
|
||||
Codec::None => Ok(data.to_vec()),
|
||||
Codec::Zstd => {
|
||||
zstd::decode_all(data)
|
||||
.map_err(|e| crate::error::Error::Compression(format!("zstd decompress: {}", e)))
|
||||
}
|
||||
Codec::Lz4 => {
|
||||
lz4_flex::decompress(data, raw_size)
|
||||
.map_err(|e| crate::error::Error::Compression(format!("lz4 decompress: {}", e)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_small_data_not_compressed() {
|
||||
let data = b"hi";
|
||||
let (out, codec) = compress(data, Codec::Zstd, 4096, 0);
|
||||
assert_eq!(out, b"hi");
|
||||
assert_eq!(codec, Codec::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_large_data_compressed_zstd() {
|
||||
let data = vec![b'A'; 5000];
|
||||
let (out, codec) = compress(&data, Codec::Zstd, 4096, 0);
|
||||
assert_eq!(codec, Codec::Zstd);
|
||||
assert!(out.len() < data.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_zstd() {
|
||||
let data = vec![b'B'; 10000];
|
||||
let (compressed, codec) = compress(&data, Codec::Zstd, 4096, 0);
|
||||
let decompressed = decompress(&compressed, codec, data.len()).unwrap();
|
||||
assert_eq!(decompressed, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_lz4() {
|
||||
let data = vec![b'C'; 10000];
|
||||
let (compressed, codec) = compress(&data, Codec::Lz4, 4096, 0);
|
||||
let decompressed = decompress(&compressed, codec, data.len()).unwrap();
|
||||
assert_eq!(decompressed, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_none() {
|
||||
let data = vec![b'D'; 100];
|
||||
let (compressed, codec) = compress(&data, Codec::None, 4096, 0);
|
||||
assert_eq!(codec, Codec::None);
|
||||
let decompressed = decompress(&compressed, codec, data.len()).unwrap();
|
||||
assert_eq!(decompressed, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_threshold_zero_always_compresses() {
|
||||
let data = vec![b'E'; 100];
|
||||
let (out, codec) = compress(&data, Codec::Zstd, 0, 0);
|
||||
assert_eq!(codec, Codec::Zstd);
|
||||
assert!(out.len() < data.len());
|
||||
}
|
||||
}
|
||||
398
crates/blob/src/engine.rs
Normal file
398
crates/blob/src/engine.rs
Normal file
@@ -0,0 +1,398 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
use crate::account::AccountHandle;
|
||||
use crate::bucket::{self, IndexRecord};
|
||||
use crate::cache::BucketCache;
|
||||
use crate::compress;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::gc::{self, GcStats};
|
||||
use crate::meta::GlobalMeta;
|
||||
use crate::segment::SegmentReader;
|
||||
use crate::types::{Codec, Config, ENTRY_HEADER_SIZE};
|
||||
|
||||
pub struct Engine {
|
||||
root: PathBuf,
|
||||
config: Config,
|
||||
cache: BucketCache,
|
||||
accounts: RwLock<HashMap<String, Arc<AccountHandle>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AccountStats {
|
||||
pub account_id: String,
|
||||
pub total_keys: u64,
|
||||
pub total_bytes: u64,
|
||||
pub deleted_bytes: u64,
|
||||
pub segment_count: usize,
|
||||
}
|
||||
|
||||
impl Engine {
|
||||
pub fn open(path: &Path, config: Config) -> Result<Self> {
|
||||
config.validate()?;
|
||||
fs::create_dir_all(path)?;
|
||||
fs::create_dir_all(path.join("accounts"))?;
|
||||
|
||||
let mut global = GlobalMeta::load(path)?;
|
||||
global.save(path)?;
|
||||
|
||||
let cache = BucketCache::new(config.lru_bucket_count);
|
||||
|
||||
let accounts_dir = path.join("accounts");
|
||||
let mut accounts = HashMap::new();
|
||||
|
||||
if accounts_dir.exists() {
|
||||
for entry in fs::read_dir(&accounts_dir)? {
|
||||
let entry = entry?;
|
||||
if entry.file_type()?.is_dir() {
|
||||
let account_name = entry.file_name().to_string_lossy().into_owned();
|
||||
|
||||
let _ = crate::recovery::cleanup_temp_files(&entry.path());
|
||||
|
||||
match crate::recovery::recover_account(&entry.path()) {
|
||||
Ok(_meta) => {
|
||||
match AccountHandle::open(path, &account_name) {
|
||||
Ok(handle) => {
|
||||
accounts.insert(account_name, handle);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Failed to open account {}: {}",
|
||||
account_name,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Failed to recover account {}: {}",
|
||||
account_name,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
global.accounts = accounts.keys().cloned().collect();
|
||||
global.save(path)?;
|
||||
|
||||
Ok(Self {
|
||||
root: path.to_path_buf(),
|
||||
config,
|
||||
cache,
|
||||
accounts: RwLock::new(accounts),
|
||||
})
|
||||
}
|
||||
|
||||
// ── Account management ──────────────────────────────────────────────
|
||||
|
||||
pub fn create_account(&self, account_id: &str) -> Result<()> {
|
||||
let mut accounts = self.accounts.write().unwrap();
|
||||
if accounts.contains_key(account_id) {
|
||||
return Err(Error::AccountAlreadyExists(account_id.to_string()));
|
||||
}
|
||||
let handle = AccountHandle::create(&self.root, account_id)?;
|
||||
accounts.insert(account_id.to_string(), handle);
|
||||
|
||||
let mut global = GlobalMeta::load(&self.root)?;
|
||||
global.accounts = accounts.keys().cloned().collect();
|
||||
global.save(&self.root)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn delete_account(&self, account_id: &str) -> Result<()> {
|
||||
let mut accounts = self.accounts.write().unwrap();
|
||||
let handle = accounts
|
||||
.remove(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?;
|
||||
|
||||
let account_dir = handle.dir().to_path_buf();
|
||||
drop(handle);
|
||||
fs::remove_dir_all(&account_dir)?;
|
||||
|
||||
let mut global = GlobalMeta::load(&self.root)?;
|
||||
global.accounts = accounts.keys().cloned().collect();
|
||||
global.save(&self.root)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn list_accounts(&self) -> Vec<String> {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts.keys().cloned().collect()
|
||||
}
|
||||
|
||||
// ── Read / Write / Delete ───────────────────────────────────────────
|
||||
|
||||
pub fn write(
|
||||
&self,
|
||||
account_id: &str,
|
||||
key: [u8; 32],
|
||||
value: &[u8],
|
||||
codec: Codec,
|
||||
) -> Result<()> {
|
||||
if value.len() > crate::types::MAX_VALUE_SIZE {
|
||||
return Err(Error::ValueTooLarge { size: value.len() });
|
||||
}
|
||||
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
let _write_lock = handle.write_mutex.lock().unwrap();
|
||||
let mut inner = handle.write();
|
||||
|
||||
let (data, actual_codec) =
|
||||
compress::compress(value, codec, self.config.compress_threshold, self.config.compression_level);
|
||||
|
||||
let (segment_id, offset, data_size) =
|
||||
inner.write_entry(key, &data, 0, actual_codec)?;
|
||||
|
||||
let record = IndexRecord::new(key, segment_id, offset, data_size, 0);
|
||||
inner.append_index(&record)?;
|
||||
|
||||
let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
inner.mark_indexed(segment_id, entry_end)?;
|
||||
|
||||
let bucket_id = bucket::bucket_id(&key);
|
||||
self.cache.update_record(account_id, bucket_id, record);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn read(&self, account_id: &str, key: &[u8; 32]) -> Result<Option<Vec<u8>>> {
|
||||
let bucket_id = bucket::bucket_id(key);
|
||||
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
let (record, seg_path): (IndexRecord, PathBuf) = {
|
||||
let inner = handle.read();
|
||||
let records = self
|
||||
.cache
|
||||
.get_or_load(account_id, bucket_id, handle.dir())?;
|
||||
match records.binary_search_by(|r| r.key.cmp(key)) {
|
||||
Ok(idx) => {
|
||||
let r = records[idx].clone();
|
||||
if r.is_tombstone() {
|
||||
return Ok(None);
|
||||
}
|
||||
let seg_path = inner.segment_path(r.segment_id)?;
|
||||
(r, seg_path)
|
||||
}
|
||||
Err(_) => return Ok(None),
|
||||
}
|
||||
};
|
||||
|
||||
if !seg_path.exists() {
|
||||
return Err(Error::SegmentNotFound(record.segment_id));
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(seg_path.clone(), record.segment_id)?;
|
||||
let file = handle.get_segment_file(record.segment_id, &seg_path)?;
|
||||
let (entry, _) = reader.read_entry_at_file(record.offset, &file)?;
|
||||
|
||||
let value = compress::decompress(&entry.data, entry.codec, entry.raw_size as usize)?;
|
||||
|
||||
Ok(Some(value))
|
||||
}
|
||||
|
||||
pub fn delete(&self, account_id: &str, key: &[u8; 32]) -> Result<()> {
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
let _write_lock = handle.write_mutex.lock().unwrap();
|
||||
let mut inner = handle.write();
|
||||
|
||||
let (segment_id, offset, data_size) =
|
||||
inner.write_entry(*key, &[], 1, Codec::None)?;
|
||||
|
||||
let record = IndexRecord::new(*key, segment_id, offset, data_size, 1);
|
||||
inner.append_index(&record)?;
|
||||
|
||||
let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
inner.mark_indexed(segment_id, entry_end)?;
|
||||
|
||||
let bucket_id = bucket::bucket_id(key);
|
||||
self.cache.update_record(account_id, bucket_id, record);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── Batch write ─────────────────────────────────────────────────────
|
||||
|
||||
pub fn write_batch(&self, account_id: &str, entries: &[([u8; 32], Vec<u8>, Codec)]) -> Result<()> {
|
||||
if entries.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
let _write_lock = handle.write_mutex.lock().unwrap();
|
||||
let mut inner = handle.write();
|
||||
|
||||
let mut pending: Vec<(IndexRecord, u64)> = Vec::with_capacity(entries.len());
|
||||
for (key, value, codec) in entries {
|
||||
if value.len() > crate::types::MAX_VALUE_SIZE {
|
||||
return Err(Error::ValueTooLarge { size: value.len() });
|
||||
}
|
||||
let (data, actual_codec) =
|
||||
compress::compress(value, *codec, self.config.compress_threshold, self.config.compression_level);
|
||||
|
||||
let (segment_id, offset, data_size) =
|
||||
inner.append_entry(*key, &data, 0, actual_codec)?;
|
||||
|
||||
let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
let record = IndexRecord::new(*key, segment_id, offset, data_size, 0);
|
||||
pending.push((record, entry_end));
|
||||
}
|
||||
|
||||
inner.flush_active()?;
|
||||
|
||||
for (record, entry_end) in &pending {
|
||||
inner.append_index(record)?;
|
||||
inner.mark_indexed(record.segment_id, *entry_end)?;
|
||||
|
||||
let bucket_id = bucket::bucket_id(&record.key);
|
||||
self.cache.update_record(account_id, bucket_id, record.clone());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── GC ──────────────────────────────────────────────────────────────
|
||||
|
||||
pub fn gc(&self, account_id: &str) -> Result<Option<GcStats>> {
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
// Hold write_mutex to prevent concurrent writes from racing
|
||||
// with GC's bucket rebuild phase.
|
||||
let _write_lock = handle.write_mutex.lock().unwrap();
|
||||
|
||||
let result = gc::gc_account(handle.dir(), self.config.gc_deleted_ratio)?;
|
||||
|
||||
// Invalidate FilePool for GC'd segments (they were rewritten via rename)
|
||||
if let Some(ref stats) = result {
|
||||
handle.invalidate_file_cache(stats.segment_id);
|
||||
}
|
||||
|
||||
for bid in 0..crate::types::BUCKET_COUNT {
|
||||
self.cache.invalidate(account_id, bid);
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn compact_buckets(&self, account_id: &str) -> Result<()> {
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
// Hold write_mutex — compact rewrites all bucket files.
|
||||
let _write_lock = handle.write_mutex.lock().unwrap();
|
||||
|
||||
gc::compact_buckets(handle.dir())?;
|
||||
|
||||
for bid in 0..crate::types::BUCKET_COUNT {
|
||||
self.cache.invalidate(account_id, bid);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── Stats / Shutdown ────────────────────────────────────────────────
|
||||
|
||||
pub fn stats(&self, account_id: &str) -> Result<AccountStats> {
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
let inner = handle.read();
|
||||
let meta = inner.meta();
|
||||
let mut total_bytes = 0u64;
|
||||
let mut deleted_bytes = 0u64;
|
||||
|
||||
for seg in meta.segments.values() {
|
||||
total_bytes += seg.total_bytes;
|
||||
deleted_bytes += seg.deleted_bytes;
|
||||
}
|
||||
|
||||
let mut total_keys = 0u64;
|
||||
for bid in 0..crate::types::BUCKET_COUNT {
|
||||
if let Ok(records) =
|
||||
self.cache
|
||||
.get_or_load(account_id, bid, handle.dir())
|
||||
{
|
||||
total_keys += records.iter().filter(|r| !r.is_tombstone()).count() as u64;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(AccountStats {
|
||||
account_id: account_id.to_string(),
|
||||
total_keys,
|
||||
total_bytes,
|
||||
deleted_bytes,
|
||||
segment_count: meta.segments.len(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn shutdown(&self) -> Result<()> {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
for (_, handle) in accounts.iter() {
|
||||
let mut inner = handle.write();
|
||||
inner.flush_active()?;
|
||||
}
|
||||
let global = GlobalMeta::load(&self.root)?;
|
||||
global.save(&self.root)?;
|
||||
tracing::info!("bichon-blob shut down cleanly");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Engine {
|
||||
fn drop(&mut self) {
|
||||
if let Err(e) = self.shutdown() {
|
||||
tracing::error!("bichon-blob shutdown error: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
59
crates/blob/src/error.rs
Normal file
59
crates/blob/src/error.rs
Normal file
@@ -0,0 +1,59 @@
|
||||
use std::{io, path::PathBuf};
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] io::Error),
|
||||
|
||||
#[error("JSON error: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
|
||||
#[error("CRC32 mismatch at {path}:{offset}")]
|
||||
CrcMismatch { path: PathBuf, offset: u64 },
|
||||
|
||||
#[error("Corrupt entry at {path}:{offset}: {reason}")]
|
||||
CorruptEntry {
|
||||
path: PathBuf,
|
||||
offset: u64,
|
||||
reason: String,
|
||||
},
|
||||
|
||||
#[error("Account not found: {0}")]
|
||||
AccountNotFound(String),
|
||||
|
||||
#[error("Account already exists: {0}")]
|
||||
AccountAlreadyExists(String),
|
||||
|
||||
#[error("Segment not found: {0}")]
|
||||
SegmentNotFound(u32),
|
||||
|
||||
#[error("Value too large: {size} bytes (max 100 MB)")]
|
||||
ValueTooLarge { size: usize },
|
||||
|
||||
#[error("Compression error: {0}")]
|
||||
Compression(String),
|
||||
|
||||
#[error("Disk full: {0}")]
|
||||
DiskFull(String),
|
||||
|
||||
#[error("Invalid config: {0}")]
|
||||
InvalidConfig(String),
|
||||
|
||||
#[error("Bucket index corrupt at {path}: {reason}")]
|
||||
BucketIndexCorrupt { path: PathBuf, reason: String },
|
||||
|
||||
#[error("Segment file truncated at {path}: expected {expected}, got {actual}")]
|
||||
SegmentTruncated {
|
||||
path: PathBuf,
|
||||
expected: u64,
|
||||
actual: u64,
|
||||
},
|
||||
|
||||
#[error("Corrupt metadata file: {0}")]
|
||||
CorruptMeta(String),
|
||||
|
||||
#[error("Unsupported metadata version {version} in {path}")]
|
||||
UnsupportedMetaVersion { path: PathBuf, version: u32 },
|
||||
}
|
||||
54
crates/blob/src/file_pool.rs
Normal file
54
crates/blob/src/file_pool.rs
Normal file
@@ -0,0 +1,54 @@
|
||||
use std::collections::VecDeque;
|
||||
use std::fs::File;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::fs as fs_util;
|
||||
|
||||
/// Simple LRU pool of open file handles, keyed by segment_id.
|
||||
/// Uses Arc<Mutex<File>> to allow safe concurrent reads from the same segment.
|
||||
pub struct FilePool {
|
||||
max_entries: usize,
|
||||
entries: Mutex<VecDeque<(u32, Arc<Mutex<File>>)>>,
|
||||
}
|
||||
|
||||
impl FilePool {
|
||||
pub fn new(max_entries: usize) -> Self {
|
||||
Self {
|
||||
max_entries: max_entries.max(1),
|
||||
entries: Mutex::new(VecDeque::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get an open File for the given segment. Reuses cached handle if available.
|
||||
pub fn get(&self, seg_id: u32, path: &Path) -> Result<Arc<Mutex<File>>> {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
|
||||
// Check for existing entry
|
||||
for (i, (id, _)) in entries.iter().enumerate() {
|
||||
if *id == seg_id {
|
||||
let (_, file) = entries.remove(i).unwrap();
|
||||
entries.push_front((seg_id, file.clone()));
|
||||
return Ok(file);
|
||||
}
|
||||
}
|
||||
|
||||
// Open new file
|
||||
let file = Arc::new(Mutex::new(fs_util::open_read(path)?));
|
||||
|
||||
// Evict oldest if full
|
||||
if entries.len() >= self.max_entries {
|
||||
entries.pop_back();
|
||||
}
|
||||
|
||||
entries.push_front((seg_id, file.clone()));
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
/// Remove a cached file handle (e.g. after GC rewrites a segment).
|
||||
pub fn invalidate(&self, seg_id: u32) {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
entries.retain(|(id, _)| *id != seg_id);
|
||||
}
|
||||
}
|
||||
142
crates/blob/src/fs.rs
Normal file
142
crates/blob/src/fs.rs
Normal file
@@ -0,0 +1,142 @@
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, Write};
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::error::Result;
|
||||
|
||||
/// Max retries for transient filesystem errors (NFS ESTALE, CIFS sharing violations, etc.)
|
||||
const MAX_RETRIES: u32 = 5;
|
||||
const RETRY_DELAY: Duration = Duration::from_millis(20);
|
||||
|
||||
/// Check if an I/O error is transient (retryable).
|
||||
fn is_transient(err: &io::Error) -> bool {
|
||||
use std::io::ErrorKind;
|
||||
matches!(
|
||||
err.kind(),
|
||||
ErrorKind::TimedOut
|
||||
| ErrorKind::Interrupted
|
||||
| ErrorKind::WouldBlock
|
||||
| ErrorKind::UnexpectedEof
|
||||
) || err.raw_os_error() == Some(116) // ESTALE on Linux
|
||||
}
|
||||
|
||||
/// Open an existing file for reading, with retry on transient errors (NFS ESTALE etc.).
|
||||
pub fn open_read(path: &Path) -> Result<File> {
|
||||
let mut last_err = None;
|
||||
for attempt in 0..MAX_RETRIES {
|
||||
match File::open(path) {
|
||||
Ok(f) => return Ok(f),
|
||||
Err(e) if is_transient(&e) => {
|
||||
last_err = Some(e);
|
||||
if attempt > 0 {
|
||||
std::thread::sleep(RETRY_DELAY * attempt);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
}
|
||||
Err(crate::error::Error::Io(last_err.unwrap()))
|
||||
}
|
||||
|
||||
/// Open an existing file for writing, with retry on transient errors.
|
||||
pub fn open_write(path: &Path) -> Result<File> {
|
||||
let mut last_err = None;
|
||||
for attempt in 0..MAX_RETRIES {
|
||||
match OpenOptions::new().write(true).open(path) {
|
||||
Ok(f) => return Ok(f),
|
||||
Err(e) if is_transient(&e) => {
|
||||
last_err = Some(e);
|
||||
if attempt > 0 {
|
||||
std::thread::sleep(RETRY_DELAY * attempt);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
}
|
||||
Err(crate::error::Error::Io(last_err.unwrap()))
|
||||
}
|
||||
|
||||
/// Create a new file atomically: write content to a temp file, fsync, then rename.
|
||||
/// Avoids `create_new(true)` which is racy on NFS.
|
||||
pub fn create_atomic(path: &Path, content: &[u8]) -> Result<()> {
|
||||
let tmp = path.with_extension(
|
||||
path.extension()
|
||||
.map(|e| format!("{}.tmp", e.to_string_lossy()))
|
||||
.unwrap_or_else(|| "tmp".to_string()),
|
||||
);
|
||||
|
||||
{
|
||||
let mut f = File::create(&tmp)?;
|
||||
f.write_all(content)?;
|
||||
f.sync_all()?;
|
||||
}
|
||||
|
||||
fs::rename(&tmp, path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Truncate an existing file to the given size, with retry.
|
||||
pub fn truncate(path: &Path, size: u64) -> Result<()> {
|
||||
let f = open_write(path)?;
|
||||
f.set_len(size)?;
|
||||
f.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_open_read_existing() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = dir.path().join("test.txt");
|
||||
std::fs::write(&path, b"hello").unwrap();
|
||||
|
||||
let mut f = open_read(&path).unwrap();
|
||||
let mut s = String::new();
|
||||
std::io::Read::read_to_string(&mut f, &mut s).unwrap();
|
||||
assert_eq!(s, "hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_open_read_missing() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let result = open_read(&dir.path().join("nope.txt"));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_create_atomic_success() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = dir.path().join("data.bin");
|
||||
create_atomic(&path, b"hello world").unwrap();
|
||||
|
||||
let content = std::fs::read(&path).unwrap();
|
||||
assert_eq!(content, b"hello world");
|
||||
// Temp file should not exist
|
||||
assert!(!dir.path().join("data.bin.tmp").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_create_atomic_overwrites() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = dir.path().join("data.bin");
|
||||
create_atomic(&path, b"first").unwrap();
|
||||
create_atomic(&path, b"second").unwrap();
|
||||
assert_eq!(std::fs::read(&path).unwrap(), b"second");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_truncate() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = dir.path().join("trunc.bin");
|
||||
std::fs::write(&path, b"1234567890").unwrap();
|
||||
truncate(&path, 5).unwrap();
|
||||
assert_eq!(std::fs::metadata(&path).unwrap().len(), 5);
|
||||
}
|
||||
}
|
||||
267
crates/blob/src/gc.rs
Normal file
267
crates/blob/src/gc.rs
Normal file
@@ -0,0 +1,267 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::bucket::{self, BucketFile, BucketIndex, IndexRecord};
|
||||
use crate::error::Result;
|
||||
#[cfg(test)]
|
||||
use crate::meta::SegmentStats;
|
||||
use crate::segment::{self, SegmentReader, SegmentWriter};
|
||||
|
||||
/// Result of a GC run.
|
||||
#[derive(Debug)]
|
||||
pub struct GcStats {
|
||||
pub segment_id: u32,
|
||||
pub bytes_before: u64,
|
||||
pub bytes_after: u64,
|
||||
pub entries_kept: usize,
|
||||
pub entries_skipped: usize,
|
||||
}
|
||||
|
||||
/// Run GC on an account: pick the sealed segment with highest deleted_ratio,
|
||||
/// rewrite it without deleted/overwritten entries, then rebuild all bucket files.
|
||||
pub fn gc_account(
|
||||
account_dir: &Path,
|
||||
deleted_ratio_threshold: f64,
|
||||
) -> Result<Option<GcStats>> {
|
||||
let meta = crate::meta::AccountMeta::load(account_dir)?;
|
||||
|
||||
// Find the best candidate
|
||||
let candidate = meta
|
||||
.segments
|
||||
.values()
|
||||
.filter(|s| s.sealed && s.deleted_ratio >= deleted_ratio_threshold)
|
||||
.max_by(|a, b| a.deleted_ratio.partial_cmp(&b.deleted_ratio).unwrap());
|
||||
|
||||
let target = match candidate {
|
||||
Some(s) => s.clone(),
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
let seg_path = account_dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(target.segment_id));
|
||||
let reader = SegmentReader::open(seg_path.clone(), target.segment_id)?;
|
||||
|
||||
// Build a global view: for each key, which entry (segment_id + offset) is the latest?
|
||||
let mut latest_key: HashMap<[u8; 32], (u32, u64)> = HashMap::new();
|
||||
|
||||
for &seg_id in meta.segments.keys() {
|
||||
let rpath = account_dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(seg_id));
|
||||
if !rpath.exists() {
|
||||
continue;
|
||||
}
|
||||
let r = SegmentReader::open(rpath, seg_id)?;
|
||||
let _ = r.scan_entries(0, |entry, offset| {
|
||||
match latest_key.get(&entry.key) {
|
||||
Some((existing_seg, existing_off)) => {
|
||||
if seg_id > *existing_seg
|
||||
|| (seg_id == *existing_seg && offset > *existing_off)
|
||||
{
|
||||
latest_key.insert(entry.key, (seg_id, offset));
|
||||
}
|
||||
}
|
||||
None => {
|
||||
latest_key.insert(entry.key, (seg_id, offset));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
|
||||
// Create temp segment with a unique name
|
||||
let timestamp = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
let temp_name = format!("temp_{:016x}.seg", timestamp);
|
||||
let temp_path = account_dir.join("segments").join(&temp_name);
|
||||
let mut writer = SegmentWriter::create(temp_path.clone(), target.segment_id)?;
|
||||
|
||||
let mut bytes_after: u64 = 0;
|
||||
let mut entries_kept: usize = 0;
|
||||
let mut entries_skipped: usize = 0;
|
||||
|
||||
reader.scan_entries(0, |entry, offset| {
|
||||
// Skip tombstones
|
||||
if entry.is_tombstone() {
|
||||
entries_skipped += 1;
|
||||
return Ok(());
|
||||
}
|
||||
// Skip if this key has a newer entry in another segment
|
||||
if let Some((latest_seg, latest_off)) = latest_key.get(&entry.key) {
|
||||
if *latest_seg != target.segment_id || *latest_off != offset {
|
||||
entries_skipped += 1;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
// Keep this entry
|
||||
writer.append(entry)?;
|
||||
bytes_after += entry.data.len() as u64;
|
||||
entries_kept += 1;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
writer.fsync()?;
|
||||
|
||||
// Atomic rename: replace old segment with new one
|
||||
fs::rename(&temp_path, &seg_path)?;
|
||||
|
||||
// Rebuild all bucket files
|
||||
rebuild_buckets(account_dir, &meta)?;
|
||||
|
||||
// Update meta
|
||||
let mut meta = crate::meta::AccountMeta::load(account_dir)?;
|
||||
if let Some(stats) = meta.segments.get_mut(&target.segment_id) {
|
||||
stats.total_bytes = bytes_after;
|
||||
stats.deleted_bytes = 0;
|
||||
stats.recompute_ratio();
|
||||
}
|
||||
meta.save(account_dir)?;
|
||||
|
||||
Ok(Some(GcStats {
|
||||
segment_id: target.segment_id,
|
||||
bytes_before: target.total_bytes,
|
||||
bytes_after,
|
||||
entries_kept,
|
||||
entries_skipped,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Rebuild all 16 bucket files from scratch by scanning all segments.
|
||||
fn rebuild_buckets(account_dir: &Path, meta: &crate::meta::AccountMeta) -> Result<()> {
|
||||
let mut bucket_records: HashMap<u16, Vec<IndexRecord>> = HashMap::new();
|
||||
for i in 0..crate::types::BUCKET_COUNT {
|
||||
bucket_records.insert(i, Vec::new());
|
||||
}
|
||||
|
||||
for &seg_id in meta.segments.keys() {
|
||||
let seg_path = account_dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(seg_id));
|
||||
if !seg_path.exists() {
|
||||
continue;
|
||||
}
|
||||
let reader = SegmentReader::open(seg_path, seg_id)?;
|
||||
reader.scan_entries(0, |entry, offset| {
|
||||
let bid = bucket::bucket_id(&entry.key);
|
||||
let rec = IndexRecord::new(
|
||||
entry.key,
|
||||
seg_id,
|
||||
offset,
|
||||
entry.data.len() as u32,
|
||||
entry.flags,
|
||||
);
|
||||
bucket_records.entry(bid).or_default().push(rec);
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
|
||||
for (bid, records) in &bucket_records {
|
||||
let index = BucketIndex::from_records(records.clone(), *bid);
|
||||
let bf = BucketFile::open(account_dir, *bid);
|
||||
bf.rewrite(&index.records)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compact bucket files: load, dedup, rewrite.
|
||||
pub fn compact_buckets(account_dir: &Path) -> Result<()> {
|
||||
for bid in 0..crate::types::BUCKET_COUNT {
|
||||
let bf = BucketFile::open(account_dir, bid);
|
||||
if bf.path().exists() {
|
||||
let index = bf.load_index()?;
|
||||
bf.rewrite(&index.records)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::segment::Entry;
|
||||
use crate::types::Codec;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn setup_account(dir: &Path) {
|
||||
fs::create_dir_all(dir.join("segments")).unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir).unwrap();
|
||||
|
||||
let seg_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(1));
|
||||
let mut writer = SegmentWriter::create(seg_path, 1).unwrap();
|
||||
|
||||
// Write 5 entries
|
||||
for i in 0..5u8 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0] = i;
|
||||
let entry = Entry::new(key, &vec![i; 1000], 0, Codec::None);
|
||||
writer.append(&entry).unwrap();
|
||||
}
|
||||
|
||||
// Tombstone entry 2
|
||||
let mut key2 = [0u8; 32];
|
||||
key2[0] = 2;
|
||||
let tomb = Entry::tombstone(key2);
|
||||
writer.append(&tomb).unwrap();
|
||||
|
||||
writer.fsync().unwrap();
|
||||
|
||||
// Save meta
|
||||
let mut meta = crate::meta::AccountMeta::new("test".into(), 2);
|
||||
meta.segments.insert(
|
||||
1,
|
||||
SegmentStats {
|
||||
segment_id: 1,
|
||||
total_bytes: 6000,
|
||||
deleted_bytes: 1000,
|
||||
deleted_ratio: 1000.0 / 6000.0,
|
||||
sealed: true,
|
||||
indexed_up_to_offset: 0,
|
||||
},
|
||||
);
|
||||
// Make segment 2 active so segment 1 is sealed
|
||||
let seg2_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(2));
|
||||
SegmentWriter::create(seg2_path, 2).unwrap();
|
||||
meta.save(dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gc_removes_tombstones() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
setup_account(dir.path());
|
||||
|
||||
let result = gc_account(dir.path(), 0.01).unwrap();
|
||||
assert!(result.is_some());
|
||||
|
||||
// Verify segment 1 no longer has the tombstone'd entry
|
||||
let seg_path = dir
|
||||
.path()
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(1));
|
||||
let reader = SegmentReader::open(seg_path, 1).unwrap();
|
||||
let mut count = 0;
|
||||
reader.scan_entries(0, |entry, _offset| {
|
||||
count += 1;
|
||||
assert!(entry.key[0] != 2);
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
assert_eq!(count, 4); // 5 original - 1 tombstoned
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compact_buckets() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
setup_account(dir.path());
|
||||
compact_buckets(dir.path()).unwrap();
|
||||
// Should not panic
|
||||
}
|
||||
}
|
||||
19
crates/blob/src/lib.rs
Normal file
19
crates/blob/src/lib.rs
Normal file
@@ -0,0 +1,19 @@
|
||||
pub mod account;
|
||||
pub mod bucket;
|
||||
pub mod cache;
|
||||
pub mod checksum;
|
||||
pub mod compress;
|
||||
pub mod engine;
|
||||
pub mod error;
|
||||
pub mod file_pool;
|
||||
pub mod fs;
|
||||
pub mod gc;
|
||||
pub mod meta;
|
||||
pub mod recovery;
|
||||
pub mod segment;
|
||||
pub mod types;
|
||||
|
||||
pub use account::AccountHandle;
|
||||
pub use engine::{AccountStats, Engine};
|
||||
pub use error::{Error, Result};
|
||||
pub use types::{Codec, Config};
|
||||
284
crates/blob/src/meta.rs
Normal file
284
crates/blob/src/meta.rs
Normal file
@@ -0,0 +1,284 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::checksum;
|
||||
use crate::error::Result;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
const META_VERSION: u32 = 1;
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
fn write_bin<T: Serialize>(path: &Path, value: &T) -> Result<()> {
|
||||
let payload = bincode::serialize(value).map_err(|e| {
|
||||
crate::error::Error::CorruptMeta(format!("{}: bincode encode: {}", path.display(), e))
|
||||
})?;
|
||||
let crc = checksum::crc32(&payload);
|
||||
let mut buf = Vec::with_capacity(8 + payload.len());
|
||||
buf.extend_from_slice(&crc.to_le_bytes());
|
||||
buf.extend_from_slice(&META_VERSION.to_le_bytes());
|
||||
buf.extend_from_slice(&payload);
|
||||
|
||||
crate::fs::create_atomic(path, &buf)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_bin<T: for<'de> Deserialize<'de>>(path: &Path) -> Result<T> {
|
||||
let data = std::fs::read(path)?;
|
||||
if data.len() < 8 {
|
||||
return Err(crate::error::Error::CorruptMeta(path.display().to_string()));
|
||||
}
|
||||
let stored_crc = u32::from_le_bytes(data[0..4].try_into().unwrap());
|
||||
let version = u32::from_le_bytes(data[4..8].try_into().unwrap());
|
||||
if version != META_VERSION {
|
||||
return Err(crate::error::Error::UnsupportedMetaVersion {
|
||||
path: path.to_path_buf(),
|
||||
version,
|
||||
});
|
||||
}
|
||||
let computed = checksum::crc32(&data[8..]);
|
||||
if stored_crc != computed {
|
||||
return Err(crate::error::Error::CorruptMeta(path.display().to_string()));
|
||||
}
|
||||
bincode::deserialize(&data[8..]).map_err(|e| {
|
||||
crate::error::Error::CorruptMeta(format!("{}: bincode decode: {}", path.display(), e))
|
||||
})
|
||||
}
|
||||
|
||||
// ── GlobalMeta ─────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GlobalMeta {
|
||||
pub version: u32,
|
||||
pub accounts: Vec<String>,
|
||||
}
|
||||
|
||||
impl Default for GlobalMeta {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
version: META_VERSION,
|
||||
accounts: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl GlobalMeta {
|
||||
pub fn load(store_root: &Path) -> Result<Self> {
|
||||
let bin_path = store_root.join("global_meta.bin");
|
||||
if bin_path.exists() {
|
||||
return read_bin(&bin_path);
|
||||
}
|
||||
// Migration from JSON
|
||||
let json_path = store_root.join("global_meta.json");
|
||||
if json_path.exists() {
|
||||
let data = std::fs::read_to_string(&json_path)?;
|
||||
let mut meta: Self = serde_json::from_str(&data)?;
|
||||
meta.accounts.sort();
|
||||
write_bin(&bin_path, &meta)?;
|
||||
let _ = std::fs::remove_file(&json_path);
|
||||
return Ok(meta);
|
||||
}
|
||||
Ok(Self::default())
|
||||
}
|
||||
|
||||
pub fn save(&self, store_root: &Path) -> Result<()> {
|
||||
let path = store_root.join("global_meta.bin");
|
||||
let mut meta = self.clone();
|
||||
meta.accounts.sort();
|
||||
write_bin(&path, &meta)
|
||||
}
|
||||
}
|
||||
|
||||
// ── SegmentStats ───────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SegmentStats {
|
||||
pub segment_id: u32,
|
||||
pub total_bytes: u64,
|
||||
pub deleted_bytes: u64,
|
||||
pub deleted_ratio: f64,
|
||||
pub sealed: bool,
|
||||
/// Byte offset up to which entries have been indexed in bucket files.
|
||||
/// Recovery starts scanning from here instead of 0.
|
||||
pub indexed_up_to_offset: u64,
|
||||
}
|
||||
|
||||
impl SegmentStats {
|
||||
pub fn new(segment_id: u32) -> Self {
|
||||
Self {
|
||||
segment_id,
|
||||
total_bytes: 0,
|
||||
deleted_bytes: 0,
|
||||
deleted_ratio: 0.0,
|
||||
sealed: false,
|
||||
indexed_up_to_offset: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn recompute_ratio(&mut self) {
|
||||
if self.total_bytes > 0 {
|
||||
self.deleted_ratio = self.deleted_bytes as f64 / self.total_bytes as f64;
|
||||
} else {
|
||||
self.deleted_ratio = 0.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── AccountMeta ────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AccountMeta {
|
||||
pub account_id: String,
|
||||
pub active_segment_id: u32,
|
||||
pub segments: BTreeMap<u32, SegmentStats>,
|
||||
}
|
||||
|
||||
impl AccountMeta {
|
||||
pub fn new(account_id: String, active_segment_id: u32) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
active_segment_id,
|
||||
segments: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load(account_dir: &Path) -> Result<Self> {
|
||||
let bin_path = account_dir.join("meta.bin");
|
||||
if bin_path.exists() {
|
||||
return read_bin(&bin_path);
|
||||
}
|
||||
// Migration from JSON
|
||||
let json_path = account_dir.join("meta.json");
|
||||
if json_path.exists() {
|
||||
let data = std::fs::read_to_string(&json_path)?;
|
||||
let meta: Self = serde_json::from_str(&data)?;
|
||||
write_bin(&bin_path, &meta)?;
|
||||
let _ = std::fs::remove_file(&json_path);
|
||||
return Ok(meta);
|
||||
}
|
||||
Err(crate::error::Error::AccountNotFound(
|
||||
account_dir.to_string_lossy().into(),
|
||||
))
|
||||
}
|
||||
|
||||
pub fn save(&self, account_dir: &Path) -> Result<()> {
|
||||
write_bin(&account_dir.join("meta.bin"), self)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_global_meta_bin_roundtrip() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut meta = GlobalMeta::default();
|
||||
meta.accounts.push("alice".into());
|
||||
meta.save(dir.path()).unwrap();
|
||||
|
||||
let loaded = GlobalMeta::load(dir.path()).unwrap();
|
||||
assert_eq!(loaded.accounts, vec!["alice"]);
|
||||
assert!(!dir.path().join("global_meta.json").exists());
|
||||
assert!(dir.path().join("global_meta.bin").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_global_meta_default_when_missing() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let meta = GlobalMeta::load(dir.path()).unwrap();
|
||||
assert!(meta.accounts.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_json_migration() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
// Write old JSON format
|
||||
let json = r#"{"version":1,"accounts":["bob","alice"]}"#;
|
||||
std::fs::write(dir.path().join("global_meta.json"), json).unwrap();
|
||||
|
||||
let meta = GlobalMeta::load(dir.path()).unwrap();
|
||||
// Should be sorted
|
||||
assert_eq!(meta.accounts, vec!["alice", "bob"]);
|
||||
// JSON should be removed
|
||||
assert!(!dir.path().join("global_meta.json").exists());
|
||||
// BIN should exist
|
||||
assert!(dir.path().join("global_meta.bin").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_account_meta_bin_roundtrip() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut meta = AccountMeta::new("alice".into(), 1);
|
||||
meta.segments.insert(
|
||||
1,
|
||||
SegmentStats {
|
||||
segment_id: 1,
|
||||
total_bytes: 1000,
|
||||
deleted_bytes: 300,
|
||||
deleted_ratio: 0.3,
|
||||
sealed: false,
|
||||
indexed_up_to_offset: 0,
|
||||
},
|
||||
);
|
||||
meta.save(dir.path()).unwrap();
|
||||
|
||||
let loaded = AccountMeta::load(dir.path()).unwrap();
|
||||
assert_eq!(loaded.active_segment_id, 1);
|
||||
assert_eq!(loaded.segments[&1].total_bytes, 1000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_corrupt_bin_detected() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("meta.bin"), vec![0xFFu8; 100]).unwrap();
|
||||
let result = AccountMeta::load(dir.path());
|
||||
assert!(result.is_err());
|
||||
// 0xFFFFFFFF version triggers UnsupportedMetaVersion
|
||||
assert!(matches!(result.unwrap_err(), crate::error::Error::UnsupportedMetaVersion { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crc_corruption_detected() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
// Write a well-formed header (version=1) but with wrong CRC bytes
|
||||
let mut buf = Vec::new();
|
||||
buf.extend_from_slice(&0xDEADBEEFu32.to_le_bytes()); // wrong CRC
|
||||
buf.extend_from_slice(&1u32.to_le_bytes()); // version = 1 (OK)
|
||||
buf.extend_from_slice(b"some payload bytes"); // payload
|
||||
std::fs::write(dir.path().join("meta.bin"), &buf).unwrap();
|
||||
let result = AccountMeta::load(dir.path());
|
||||
assert!(matches!(result.unwrap_err(), crate::error::Error::CorruptMeta(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_account_json_migration() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
// Write old JSON format for AccountMeta
|
||||
let json = r#"{"account_id":"alice","active_segment_id":5,"segments":{}}"#;
|
||||
std::fs::write(dir.path().join("meta.json"), json).unwrap();
|
||||
|
||||
let meta = AccountMeta::load(dir.path()).unwrap();
|
||||
assert_eq!(meta.account_id, "alice");
|
||||
assert_eq!(meta.active_segment_id, 5);
|
||||
// JSON should be removed
|
||||
assert!(!dir.path().join("meta.json").exists());
|
||||
// BIN should exist
|
||||
assert!(dir.path().join("meta.bin").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bin_sorted_keys() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut meta = AccountMeta::new("test".into(), 1);
|
||||
meta.segments.insert(3, SegmentStats::new(3));
|
||||
meta.segments.insert(1, SegmentStats::new(1));
|
||||
meta.segments.insert(2, SegmentStats::new(2));
|
||||
meta.save(dir.path()).unwrap();
|
||||
|
||||
let loaded = AccountMeta::load(dir.path()).unwrap();
|
||||
let keys: Vec<u32> = loaded.segments.keys().copied().collect();
|
||||
assert_eq!(keys, vec![1, 2, 3]);
|
||||
}
|
||||
}
|
||||
202
crates/blob/src/recovery.rs
Normal file
202
crates/blob/src/recovery.rs
Normal file
@@ -0,0 +1,202 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::bucket::{self, BucketFile, IndexRecord};
|
||||
use crate::error::Result;
|
||||
use crate::meta::{AccountMeta, SegmentStats};
|
||||
use crate::segment::{self, SegmentReader};
|
||||
|
||||
/// Recover an account after a crash: scan segments, repair indices, update stats.
|
||||
pub fn recover_account(account_dir: &Path) -> Result<AccountMeta> {
|
||||
let meta_bin = account_dir.join("meta.bin");
|
||||
let meta_json = account_dir.join("meta.json");
|
||||
let meta_exists = meta_bin.exists() || meta_json.exists();
|
||||
let mut meta = if meta_exists {
|
||||
AccountMeta::load(account_dir).unwrap_or_else(|_| {
|
||||
AccountMeta::new(
|
||||
account_dir
|
||||
.file_name()
|
||||
.unwrap_or_default()
|
||||
.to_string_lossy()
|
||||
.into(),
|
||||
1,
|
||||
)
|
||||
})
|
||||
} else {
|
||||
return Ok(AccountMeta::new(
|
||||
account_dir
|
||||
.file_name()
|
||||
.unwrap_or_default()
|
||||
.to_string_lossy()
|
||||
.into(),
|
||||
1,
|
||||
));
|
||||
};
|
||||
|
||||
// Discover all segment files on disk
|
||||
let seg_dir = account_dir.join("segments");
|
||||
if !seg_dir.exists() {
|
||||
fs::create_dir_all(&seg_dir)?;
|
||||
}
|
||||
|
||||
let mut disk_segments: Vec<u32> = Vec::new();
|
||||
if seg_dir.exists() {
|
||||
for entry in fs::read_dir(&seg_dir)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
if name_str.ends_with(".seg") && !name_str.contains("temp_") {
|
||||
if let Some(id_str) = name_str.strip_suffix(".seg") {
|
||||
if let Ok(id) = id_str.parse::<u32>() {
|
||||
disk_segments.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
disk_segments.sort_unstable();
|
||||
|
||||
if disk_segments.is_empty() {
|
||||
meta.active_segment_id = 1;
|
||||
} else {
|
||||
let max_id = *disk_segments.last().unwrap();
|
||||
meta.active_segment_id = max_id;
|
||||
}
|
||||
|
||||
// Ensure buckets directory exists
|
||||
let buckets_dir = account_dir.join("buckets");
|
||||
fs::create_dir_all(&buckets_dir)?;
|
||||
|
||||
// For each segment, scan only the unindexed tail and update stats incrementally
|
||||
for &seg_id in &disk_segments {
|
||||
let seg_path = seg_dir.join(segment::segment_filename(seg_id));
|
||||
let file_size = fs::metadata(&seg_path)?.len();
|
||||
|
||||
// Preserve existing stats; start fresh if this is a newly discovered segment
|
||||
let mut stats = meta.segments.remove(&seg_id).unwrap_or_else(|| SegmentStats::new(seg_id));
|
||||
let is_sealed = seg_id != meta.active_segment_id;
|
||||
stats.sealed = is_sealed;
|
||||
|
||||
// Scan start: from last indexed offset. Clamp defensively.
|
||||
let scan_start = if stats.indexed_up_to_offset <= file_size {
|
||||
stats.indexed_up_to_offset
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
// If fully indexed, skip scanning entirely
|
||||
if scan_start >= file_size {
|
||||
meta.segments.insert(seg_id, stats);
|
||||
continue;
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(seg_path.clone(), seg_id)?;
|
||||
let mut new_records: HashMap<u16, Vec<IndexRecord>> = HashMap::new();
|
||||
|
||||
let truncation_point = reader.scan_entries(scan_start, |entry, offset| {
|
||||
let bid = bucket::bucket_id(&entry.key);
|
||||
let rec = IndexRecord::new(
|
||||
entry.key,
|
||||
seg_id,
|
||||
offset,
|
||||
entry.data.len() as u32,
|
||||
entry.flags,
|
||||
);
|
||||
new_records.entry(bid).or_default().push(rec);
|
||||
|
||||
stats.total_bytes += entry.data.len() as u64;
|
||||
if entry.is_tombstone() {
|
||||
stats.deleted_bytes += entry.raw_size as u64;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
// Merge new records into bucket files (only the newly discovered ones)
|
||||
for (bid, records) in &new_records {
|
||||
let bf = BucketFile::open(account_dir, *bid);
|
||||
bf.append_batch(records)?;
|
||||
}
|
||||
|
||||
// Truncate if tail corruption found
|
||||
if truncation_point < file_size {
|
||||
segment::truncate_segment(&seg_path, truncation_point)?;
|
||||
}
|
||||
|
||||
stats.indexed_up_to_offset = truncation_point;
|
||||
stats.recompute_ratio();
|
||||
meta.segments.insert(seg_id, stats);
|
||||
}
|
||||
|
||||
meta.save(account_dir)?;
|
||||
|
||||
Ok(meta)
|
||||
}
|
||||
|
||||
/// Clean up leftover temp files from interrupted GC.
|
||||
pub fn cleanup_temp_files(account_dir: &Path) -> Result<()> {
|
||||
let seg_dir = account_dir.join("segments");
|
||||
if seg_dir.exists() {
|
||||
for entry in fs::read_dir(&seg_dir)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
if name_str.starts_with("temp_") {
|
||||
let path = entry.path();
|
||||
tracing::warn!("Removing leftover temp file: {:?}", path);
|
||||
fs::remove_file(&path)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Also cleanup temp bucket files
|
||||
let buckets_dir = account_dir.join("buckets");
|
||||
if buckets_dir.exists() {
|
||||
for entry in fs::read_dir(&buckets_dir)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
if name_str.ends_with(".tmp") {
|
||||
let path = entry.path();
|
||||
tracing::warn!("Removing leftover temp bucket file: {:?}", path);
|
||||
fs::remove_file(&path)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_recover_fresh_account() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let account_dir = dir.path().join("test");
|
||||
fs::create_dir_all(&account_dir).unwrap();
|
||||
|
||||
let meta = recover_account(&account_dir).unwrap();
|
||||
assert_eq!(meta.active_segment_id, 1);
|
||||
assert!(meta.segments.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cleanup_temp_files() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let account_dir = dir.path().join("test");
|
||||
fs::create_dir_all(account_dir.join("segments")).unwrap();
|
||||
fs::create_dir_all(account_dir.join("buckets")).unwrap();
|
||||
fs::write(
|
||||
account_dir.join("segments").join("temp_ABC123.seg"),
|
||||
b"garbage",
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(account_dir.join("buckets").join("00.idx.tmp"), b"garbage").unwrap();
|
||||
|
||||
cleanup_temp_files(&account_dir).unwrap();
|
||||
|
||||
assert!(!account_dir.join("segments").join("temp_ABC123.seg").exists());
|
||||
}
|
||||
}
|
||||
539
crates/blob/src/segment.rs
Normal file
539
crates/blob/src/segment.rs
Normal file
@@ -0,0 +1,539 @@
|
||||
use std::fs::{self, File};
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use crate::checksum;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::fs as fs_util;
|
||||
use crate::types::{Codec, ENTRY_HEADER_SIZE, ENTRY_MAGIC, SEGMENT_MAX_SIZE};
|
||||
|
||||
/// In-memory representation of a stored entry.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Entry {
|
||||
pub flags: u8,
|
||||
pub codec: Codec,
|
||||
pub key: [u8; 32],
|
||||
pub raw_size: u32,
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Entry {
|
||||
/// Create a normal data entry.
|
||||
pub fn new(key: [u8; 32], raw_data: &[u8], flags: u8, codec: Codec) -> Self {
|
||||
Self {
|
||||
flags,
|
||||
codec,
|
||||
key,
|
||||
raw_size: raw_data.len() as u32,
|
||||
data: raw_data.to_vec(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a tombstone entry.
|
||||
pub fn tombstone(key: [u8; 32]) -> Self {
|
||||
Self {
|
||||
flags: 1,
|
||||
codec: Codec::None,
|
||||
key,
|
||||
raw_size: 0,
|
||||
data: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_tombstone(&self) -> bool {
|
||||
self.flags == 1
|
||||
}
|
||||
|
||||
/// Total on-disk size: header + data
|
||||
pub fn disk_size(&self) -> usize {
|
||||
ENTRY_HEADER_SIZE + self.data.len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Write entries sequentially to a segment file.
|
||||
pub struct SegmentWriter {
|
||||
file: File,
|
||||
path: PathBuf,
|
||||
id: u32,
|
||||
bytes_written: u64,
|
||||
}
|
||||
|
||||
impl SegmentWriter {
|
||||
pub fn create(path: PathBuf, id: u32) -> Result<Self> {
|
||||
// Use create+truncate instead of create_new to avoid NFS O_EXCL issues.
|
||||
let file = File::create(&path)?;
|
||||
Ok(Self {
|
||||
file,
|
||||
path,
|
||||
id,
|
||||
bytes_written: 0,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn open_append(path: PathBuf, id: u32) -> Result<Self> {
|
||||
let mut file = fs_util::open_write(&path)?;
|
||||
file.seek(SeekFrom::End(0))?;
|
||||
let bytes_written = file.stream_position()?;
|
||||
Ok(Self {
|
||||
file,
|
||||
path,
|
||||
id,
|
||||
bytes_written,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn id(&self) -> u32 {
|
||||
self.id
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub fn bytes_written(&self) -> u64 {
|
||||
self.bytes_written
|
||||
}
|
||||
|
||||
pub fn is_full(&self) -> bool {
|
||||
self.bytes_written >= SEGMENT_MAX_SIZE
|
||||
}
|
||||
|
||||
/// Append an entry. Returns the offset where it was written.
|
||||
pub fn append(&mut self, entry: &Entry) -> Result<u64> {
|
||||
let offset = self.bytes_written;
|
||||
self.write_entry(entry)
|
||||
.map_err(|e| map_io_err(e, &self.path))?;
|
||||
Ok(offset)
|
||||
}
|
||||
|
||||
fn write_entry(&mut self, entry: &Entry) -> Result<()> {
|
||||
let data_size = entry.data.len() as u32;
|
||||
|
||||
// Write magic
|
||||
self.file.write_all(&ENTRY_MAGIC.to_le_bytes())?;
|
||||
|
||||
// CRC32 placeholder: write zeros, remember position
|
||||
let crc_pos = self.file.stream_position()?;
|
||||
self.file.write_all(&0u32.to_le_bytes())?;
|
||||
|
||||
// Write flags, codec, key, raw_size, data_size
|
||||
self.file.write_all(&[entry.flags])?;
|
||||
self.file.write_all(&[entry.codec as u8])?;
|
||||
self.file.write_all(&entry.key)?;
|
||||
self.file.write_all(&entry.raw_size.to_le_bytes())?;
|
||||
self.file.write_all(&data_size.to_le_bytes())?;
|
||||
|
||||
// Write data
|
||||
self.file.write_all(&entry.data)?;
|
||||
|
||||
// Calculate CRC32 over everything after the crc32 field
|
||||
let crc = {
|
||||
let mut hasher = checksum::CrcWriter::new();
|
||||
hasher.update(&[entry.flags]);
|
||||
hasher.update(&[entry.codec as u8]);
|
||||
hasher.update(&entry.key);
|
||||
hasher.update(&entry.raw_size.to_le_bytes());
|
||||
hasher.update(&data_size.to_le_bytes());
|
||||
hasher.update(&entry.data);
|
||||
hasher.finalize()
|
||||
};
|
||||
|
||||
// Seek back and write the real CRC32
|
||||
self.file.seek(SeekFrom::Start(crc_pos))?;
|
||||
self.file.write_all(&crc.to_le_bytes())?;
|
||||
|
||||
// Seek back to end
|
||||
self.file.seek(SeekFrom::End(0))?;
|
||||
|
||||
self.bytes_written += entry.disk_size() as u64;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn fsync(&self) -> Result<()> {
|
||||
self.file.sync_all().map_err(|e| {
|
||||
if e.kind() == std::io::ErrorKind::StorageFull {
|
||||
Error::DiskFull(format!("{}: {}", self.path.display(), e))
|
||||
} else {
|
||||
Error::Io(e)
|
||||
}
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Read entries from a segment file.
|
||||
pub struct SegmentReader {
|
||||
path: PathBuf,
|
||||
id: u32,
|
||||
}
|
||||
|
||||
impl SegmentReader {
|
||||
pub fn open(path: PathBuf, id: u32) -> Result<Self> {
|
||||
Ok(Self { path, id })
|
||||
}
|
||||
|
||||
pub fn id(&self) -> u32 {
|
||||
self.id
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub fn file_size(&self) -> Result<u64> {
|
||||
Ok(fs::metadata(&self.path)?.len())
|
||||
}
|
||||
|
||||
/// Read a single entry at the given offset. Returns the entry and the offset of the next entry.
|
||||
pub fn read_entry_at(&self, offset: u64) -> Result<(Entry, u64)> {
|
||||
let mut file = fs_util::open_read(&self.path)?;
|
||||
file.seek(SeekFrom::Start(offset))?;
|
||||
|
||||
// Read magic
|
||||
let mut magic_buf = [0u8; 4];
|
||||
file.read_exact(&mut magic_buf)?;
|
||||
let magic = u32::from_le_bytes(magic_buf);
|
||||
if magic != ENTRY_MAGIC {
|
||||
return Err(Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
reason: format!("bad magic: 0x{:08X}", magic),
|
||||
});
|
||||
}
|
||||
|
||||
// Read CRC32
|
||||
let mut crc_buf = [0u8; 4];
|
||||
file.read_exact(&mut crc_buf)?;
|
||||
let stored_crc = u32::from_le_bytes(crc_buf);
|
||||
|
||||
// Read flags, codec
|
||||
let mut flags_buf = [0u8; 1];
|
||||
file.read_exact(&mut flags_buf)?;
|
||||
let flags = flags_buf[0];
|
||||
|
||||
let mut codec_buf = [0u8; 1];
|
||||
file.read_exact(&mut codec_buf)?;
|
||||
let codec = Codec::from_u8(codec_buf[0]).ok_or_else(|| Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
reason: format!("unknown codec: {}", codec_buf[0]),
|
||||
})?;
|
||||
|
||||
// Read key, raw_size, data_size
|
||||
let mut key = [0u8; 32];
|
||||
file.read_exact(&mut key)?;
|
||||
|
||||
let mut raw_size_buf = [0u8; 4];
|
||||
file.read_exact(&mut raw_size_buf)?;
|
||||
let raw_size = u32::from_le_bytes(raw_size_buf);
|
||||
|
||||
let mut data_size_buf = [0u8; 4];
|
||||
file.read_exact(&mut data_size_buf)?;
|
||||
let data_size = u32::from_le_bytes(data_size_buf);
|
||||
|
||||
// Read data
|
||||
let mut data = vec![0u8; data_size as usize];
|
||||
file.read_exact(&mut data)?;
|
||||
|
||||
// Verify CRC32 (over everything after the crc32 field)
|
||||
let computed_crc = {
|
||||
let mut hasher = checksum::CrcWriter::new();
|
||||
hasher.update(&[flags]);
|
||||
hasher.update(&[codec as u8]);
|
||||
hasher.update(&key);
|
||||
hasher.update(&raw_size.to_le_bytes());
|
||||
hasher.update(&data_size.to_le_bytes());
|
||||
hasher.update(&data);
|
||||
hasher.finalize()
|
||||
};
|
||||
|
||||
if stored_crc != computed_crc {
|
||||
return Err(Error::CrcMismatch {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
});
|
||||
}
|
||||
|
||||
let next_offset = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
|
||||
Ok((
|
||||
Entry {
|
||||
flags,
|
||||
codec,
|
||||
key,
|
||||
raw_size,
|
||||
data,
|
||||
},
|
||||
next_offset,
|
||||
))
|
||||
}
|
||||
|
||||
/// Read a single entry at the given offset using a pre-opened File (via Mutex).
|
||||
/// This avoids the per-read File::open cost for hot segments.
|
||||
pub fn read_entry_at_file(&self, offset: u64, file: &Mutex<File>) -> Result<(Entry, u64)> {
|
||||
|
||||
let mut file = file.lock().unwrap();
|
||||
|
||||
file.seek(SeekFrom::Start(offset))?;
|
||||
|
||||
// Read magic
|
||||
let mut magic_buf = [0u8; 4];
|
||||
file.read_exact(&mut magic_buf)?;
|
||||
let magic = u32::from_le_bytes(magic_buf);
|
||||
if magic != ENTRY_MAGIC {
|
||||
return Err(Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
reason: format!("bad magic: 0x{:08X}", magic),
|
||||
});
|
||||
}
|
||||
|
||||
// Read CRC32
|
||||
let mut crc_buf = [0u8; 4];
|
||||
file.read_exact(&mut crc_buf)?;
|
||||
let stored_crc = u32::from_le_bytes(crc_buf);
|
||||
|
||||
// Read flags, codec
|
||||
let mut flags_buf = [0u8; 1];
|
||||
file.read_exact(&mut flags_buf)?;
|
||||
let flags = flags_buf[0];
|
||||
|
||||
let mut codec_buf = [0u8; 1];
|
||||
file.read_exact(&mut codec_buf)?;
|
||||
let codec = Codec::from_u8(codec_buf[0]).ok_or_else(|| Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
reason: format!("unknown codec: {}", codec_buf[0]),
|
||||
})?;
|
||||
|
||||
// Read key, raw_size, data_size
|
||||
let mut key = [0u8; 32];
|
||||
file.read_exact(&mut key)?;
|
||||
|
||||
let mut raw_size_buf = [0u8; 4];
|
||||
file.read_exact(&mut raw_size_buf)?;
|
||||
let raw_size = u32::from_le_bytes(raw_size_buf);
|
||||
|
||||
let mut data_size_buf = [0u8; 4];
|
||||
file.read_exact(&mut data_size_buf)?;
|
||||
let data_size = u32::from_le_bytes(data_size_buf);
|
||||
|
||||
// Read data
|
||||
let mut data = vec![0u8; data_size as usize];
|
||||
file.read_exact(&mut data)?;
|
||||
|
||||
// Verify CRC32
|
||||
let computed_crc = {
|
||||
let mut hasher = crate::checksum::CrcWriter::new();
|
||||
hasher.update(&[flags]);
|
||||
hasher.update(&[codec as u8]);
|
||||
hasher.update(&key);
|
||||
hasher.update(&raw_size.to_le_bytes());
|
||||
hasher.update(&data_size.to_le_bytes());
|
||||
hasher.update(&data);
|
||||
hasher.finalize()
|
||||
};
|
||||
|
||||
if stored_crc != computed_crc {
|
||||
return Err(Error::CrcMismatch {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
});
|
||||
}
|
||||
|
||||
let next_offset = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
|
||||
Ok((
|
||||
Entry {
|
||||
flags,
|
||||
codec,
|
||||
key,
|
||||
raw_size,
|
||||
data,
|
||||
},
|
||||
next_offset,
|
||||
))
|
||||
}
|
||||
|
||||
/// Read data portion of an entry (for pread-style reads when you already know offset + data_size).
|
||||
pub fn read_data(&self, offset: u64, data_size: u32) -> Result<Vec<u8>> {
|
||||
let mut file = fs_util::open_read(&self.path)?;
|
||||
// Skip magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4) = 50 bytes
|
||||
let data_start = offset + ENTRY_HEADER_SIZE as u64;
|
||||
file.seek(SeekFrom::Start(data_start))?;
|
||||
let mut buf = vec![0u8; data_size as usize];
|
||||
file.read_exact(&mut buf)?;
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
/// Read the full entry header + data for verification (used by recovery and GC).
|
||||
pub fn read_full_entry(&self, offset: u64, data_size: u32) -> Result<Vec<u8>> {
|
||||
let mut file = fs_util::open_read(&self.path)?;
|
||||
file.seek(SeekFrom::Start(offset))?;
|
||||
let total = ENTRY_HEADER_SIZE + data_size as usize;
|
||||
let mut buf = vec![0u8; total];
|
||||
file.read_exact(&mut buf)?;
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
/// Iterate over all valid entries in the segment, calling f for each.
|
||||
/// Stops when hitting a corrupt/incomplete entry at the tail.
|
||||
pub fn scan_entries<F>(&self, start_offset: u64, mut f: F) -> Result<u64>
|
||||
where
|
||||
F: FnMut(&Entry, u64) -> Result<()>,
|
||||
{
|
||||
let file_size = self.file_size()?;
|
||||
let mut offset = start_offset;
|
||||
|
||||
while offset + ENTRY_HEADER_SIZE as u64 <= file_size {
|
||||
match self.read_entry_at(offset) {
|
||||
Ok((entry, next)) => {
|
||||
f(&entry, offset)?;
|
||||
offset = next;
|
||||
}
|
||||
Err(Error::CrcMismatch { .. }) | Err(Error::CorruptEntry { .. }) => {
|
||||
// If near end of file (within one max entry), truncate
|
||||
if file_size - offset < ENTRY_HEADER_SIZE as u64 + 100 * 1024 * 1024 {
|
||||
// Likely a partial write at tail, stop here
|
||||
break;
|
||||
} else {
|
||||
return Err(Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
reason: "mid-file corruption detected".into(),
|
||||
});
|
||||
}
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(offset) // return the truncation point
|
||||
}
|
||||
}
|
||||
|
||||
/// Truncate a segment file to the given size.
|
||||
pub fn truncate_segment(path: &Path, size: u64) -> Result<()> {
|
||||
fs_util::truncate(path, size)
|
||||
}
|
||||
|
||||
/// Map an Error, converting Io(StorageFull) to DiskFull with path context.
|
||||
fn map_io_err(e: Error, path: &Path) -> Error {
|
||||
match e {
|
||||
Error::Io(io) if io.kind() == std::io::ErrorKind::StorageFull => {
|
||||
Error::DiskFull(format!("{}: {}", path.display(), io))
|
||||
}
|
||||
_ => e,
|
||||
}
|
||||
}
|
||||
|
||||
/// Segment file name from id: "00000001.seg"
|
||||
pub fn segment_filename(id: u32) -> String {
|
||||
format!("{:08}.seg", id)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn temp_segment_path(dir: &TempDir, id: u32) -> PathBuf {
|
||||
dir.path().join(segment_filename(id))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_write_and_read_entry() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = temp_segment_path(&dir, 1);
|
||||
let key = [0xAAu8; 32];
|
||||
let data = b"hello world".to_vec();
|
||||
|
||||
let entry = Entry::new(key, &data, 0, Codec::None);
|
||||
{
|
||||
let mut writer = SegmentWriter::create(path.clone(), 1).unwrap();
|
||||
writer.append(&entry).unwrap();
|
||||
writer.fsync().unwrap();
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(path, 1).unwrap();
|
||||
let (read_entry, next) = reader.read_entry_at(0).unwrap();
|
||||
|
||||
assert_eq!(read_entry.key, key);
|
||||
assert_eq!(read_entry.data, data);
|
||||
assert_eq!(read_entry.flags, 0);
|
||||
assert_eq!(read_entry.raw_size, 11);
|
||||
assert!(next > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tombstone_entry() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = temp_segment_path(&dir, 1);
|
||||
let key = [0xBBu8; 32];
|
||||
|
||||
let entry = Entry::tombstone(key);
|
||||
{
|
||||
let mut writer = SegmentWriter::create(path.clone(), 1).unwrap();
|
||||
writer.append(&entry).unwrap();
|
||||
writer.fsync().unwrap();
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(path, 1).unwrap();
|
||||
let (read_entry, _) = reader.read_entry_at(0).unwrap();
|
||||
|
||||
assert!(read_entry.is_tombstone());
|
||||
assert_eq!(read_entry.data.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_entries() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = temp_segment_path(&dir, 1);
|
||||
|
||||
let entries: Vec<_> = (0..10)
|
||||
.map(|i| {
|
||||
let mut key = [0u8; 32];
|
||||
key[0] = i;
|
||||
Entry::new(key, &vec![i; 100], 0, Codec::None)
|
||||
})
|
||||
.collect();
|
||||
|
||||
{
|
||||
let mut writer = SegmentWriter::create(path.clone(), 1).unwrap();
|
||||
for e in &entries {
|
||||
writer.append(e).unwrap();
|
||||
}
|
||||
writer.fsync().unwrap();
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(path, 1).unwrap();
|
||||
let mut offset = 0u64;
|
||||
for (i, expected) in entries.iter().enumerate() {
|
||||
let (entry, next) = reader.read_entry_at(offset).unwrap();
|
||||
assert_eq!(entry.key[0], i as u8);
|
||||
assert_eq!(entry.data, expected.data);
|
||||
offset = next;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bad_magic_detected() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = temp_segment_path(&dir, 1);
|
||||
// Write garbage
|
||||
std::fs::write(&path, vec![0xFFu8; 100]).unwrap();
|
||||
|
||||
let reader = SegmentReader::open(path, 1).unwrap();
|
||||
let result = reader.read_entry_at(0);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_full() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = temp_segment_path(&dir, 1);
|
||||
let writer = SegmentWriter::create(path, 1).unwrap();
|
||||
assert!(!writer.is_full());
|
||||
}
|
||||
}
|
||||
88
crates/blob/src/types.rs
Normal file
88
crates/blob/src/types.rs
Normal file
@@ -0,0 +1,88 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Magic number for entry identification
|
||||
pub const ENTRY_MAGIC: u32 = 0xB3DB_0001;
|
||||
|
||||
/// Fixed header size: magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4)
|
||||
pub const ENTRY_HEADER_SIZE: usize = 50;
|
||||
|
||||
/// Index record size: key(32) + segment_id(4) + offset(8) + data_size(4) + flags(1) + _pad(3)
|
||||
pub const INDEX_RECORD_SIZE: usize = 52;
|
||||
|
||||
/// Maximum segment size (256 MB)
|
||||
pub const SEGMENT_MAX_SIZE: u64 = 256 * 1024 * 1024;
|
||||
|
||||
/// Number of hash buckets per account
|
||||
pub const BUCKET_COUNT: u16 = 16;
|
||||
|
||||
/// Maximum value size (100 MB)
|
||||
pub const MAX_VALUE_SIZE: usize = 100 * 1024 * 1024;
|
||||
|
||||
/// Default compression threshold (4 KB)
|
||||
pub const DEFAULT_COMPRESS_THRESHOLD: usize = 4096;
|
||||
|
||||
/// Default LRU bucket cache size
|
||||
pub const DEFAULT_LRU_BUCKET_COUNT: usize = 256;
|
||||
|
||||
/// Default GC deleted ratio threshold
|
||||
pub const DEFAULT_GC_DELETED_RATIO: f64 = 0.30;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum Codec {
|
||||
None = 0,
|
||||
Zstd = 1,
|
||||
Lz4 = 2,
|
||||
}
|
||||
|
||||
impl Codec {
|
||||
pub fn from_u8(v: u8) -> Option<Self> {
|
||||
match v {
|
||||
0 => Some(Codec::None),
|
||||
1 => Some(Codec::Zstd),
|
||||
2 => Some(Codec::Lz4),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Config {
|
||||
pub compress_threshold: usize,
|
||||
pub default_codec: Codec,
|
||||
pub compression_level: i32,
|
||||
pub lru_bucket_count: usize,
|
||||
pub gc_deleted_ratio: f64,
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
compress_threshold: DEFAULT_COMPRESS_THRESHOLD,
|
||||
default_codec: Codec::Zstd,
|
||||
compression_level: 0,
|
||||
lru_bucket_count: DEFAULT_LRU_BUCKET_COUNT,
|
||||
gc_deleted_ratio: DEFAULT_GC_DELETED_RATIO,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn validate(&self) -> crate::error::Result<()> {
|
||||
if self.lru_bucket_count == 0 {
|
||||
return Err(crate::error::Error::InvalidConfig(
|
||||
"lru_bucket_count must be > 0".into(),
|
||||
));
|
||||
}
|
||||
if self.gc_deleted_ratio <= 0.0 || self.gc_deleted_ratio >= 1.0 {
|
||||
return Err(crate::error::Error::InvalidConfig(
|
||||
"gc_deleted_ratio must be in (0.0, 1.0)".into(),
|
||||
));
|
||||
}
|
||||
if self.compression_level < 0 {
|
||||
return Err(crate::error::Error::InvalidConfig(
|
||||
"compression_level must be >= 0".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
498
crates/blob/tests/acid_test.rs
Normal file
498
crates/blob/tests/acid_test.rs
Normal file
@@ -0,0 +1,498 @@
|
||||
/// Crash-consistency and ACID property tests for bichon-blob.
|
||||
///
|
||||
/// Since we can't kill the process mid-write in an inline test, we simulate crashes
|
||||
/// by dropping the Engine without calling any cleanup (close/drop is the "crash"),
|
||||
/// then re-opening and verifying recovery produced consistent state.
|
||||
///
|
||||
/// For true power-loss simulation, each test writes data, drops the engine abruptly,
|
||||
/// then reopens and verifies: no corruption, no lost committed data, no partial writes.
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::thread;
|
||||
|
||||
use bichon_blob::{Codec, Config, Engine};
|
||||
use tempfile::TempDir;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn make_key(seed: u64) -> [u8; 32] {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..8].copy_from_slice(&seed.to_le_bytes());
|
||||
key
|
||||
}
|
||||
|
||||
fn make_value(size: usize) -> Vec<u8> {
|
||||
let pattern = b"The quick brown fox jumps over the lazy dog. ";
|
||||
let mut v = Vec::with_capacity(size);
|
||||
while v.len() < size {
|
||||
let rem = size - v.len();
|
||||
let n = rem.min(pattern.len());
|
||||
v.extend_from_slice(&pattern[..n]);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Durability: committed data survives crash
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_durability_single_write_survives_crash() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let key = make_key(42);
|
||||
let value = make_value(8192);
|
||||
|
||||
// Write
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
} // <-- Engine dropped = simulated crash
|
||||
|
||||
// Recover
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_durability_many_writes_survive_crash() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let n = 500;
|
||||
let value = make_value(2048);
|
||||
let mut keys = Vec::new();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
for i in 0..n {
|
||||
let key = make_key(i as u64);
|
||||
keys.push(key);
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
} // crash
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for (i, key) in keys.iter().enumerate() {
|
||||
let result = engine.read("alice", key).unwrap();
|
||||
assert_eq!(result, Some(value.clone()), "missing key at index {}", i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_durability_delete_survives_crash() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let key = make_key(99);
|
||||
let value = make_value(4096);
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
} // crash after write
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.delete("alice", &key).unwrap();
|
||||
} // crash after delete
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, None, "delete should persist across crash");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Atomicity: no partial writes visible after crash
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_atomicity_no_partial_entries_after_crash() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
// Write enough entries to fill part of a segment, then crash
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
let value = make_value(50_000); // big enough to notice
|
||||
for i in 0..200u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
} // crash
|
||||
|
||||
// Recovery should clean up any partial tail entries and all committed
|
||||
// entries should be readable
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let value = make_value(50_000);
|
||||
for i in 0..200u64 {
|
||||
let result = engine.read("alice", &make_key(i)).unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
Some(value.clone()),
|
||||
"committed key {} should be intact",
|
||||
i
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_atomicity_crash_during_segment_roll() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let big_value = make_value(2 * 1024 * 1024); // 2 MB each entry
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
// Write enough to cross at least one segment boundary (256 MB)
|
||||
for i in 0..140u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &big_value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
} // crash mid-way or after multiple segments
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
// All committed writes (that returned Ok) must be readable
|
||||
for i in 0..140u64 {
|
||||
let result = engine.read("alice", &make_key(i)).unwrap();
|
||||
assert!(
|
||||
result.is_some(),
|
||||
"key {} should exist after segment roll recovery",
|
||||
i
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 3. Consistency: CRC detects corruption, no silent data loss
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_consistency_crc_detects_corruption() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let key = make_key(77);
|
||||
let value = make_value(8192);
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Corrupt the segment file by flipping a byte
|
||||
let seg_path = find_first_segment(dir.path(), "alice");
|
||||
let mut data = fs::read(&seg_path).unwrap();
|
||||
// Flip a byte in the data portion, not the header
|
||||
let flip_pos = data.len() - 100;
|
||||
data[flip_pos] ^= 0xFF;
|
||||
fs::write(&seg_path, &data).unwrap();
|
||||
|
||||
// Reading should detect CRC mismatch
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.read("alice", &key);
|
||||
// Either error or None is acceptable — never silently wrong data
|
||||
match result {
|
||||
Err(_) => {} // CRC mismatch detected — good
|
||||
Ok(None) => {} // index may point to truncated/removed data
|
||||
Ok(Some(v)) => {
|
||||
if v == value {
|
||||
panic!("CRC corruption was NOT detected — silent data corruption!");
|
||||
}
|
||||
// If value differs, index pointed elsewhere after recovery
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_consistency_corrupt_magic_truncated_on_recovery() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
for i in 0..10u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &make_value(4096), Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
// Append garbage to the segment file (simulating partial write from crash)
|
||||
let seg_path = find_first_segment(dir.path(), "alice");
|
||||
let mut data = fs::read(&seg_path).unwrap();
|
||||
let orig_len = data.len();
|
||||
// Append garbage that doesn't start with the magic number
|
||||
data.extend_from_slice(&[0xFF; 200]);
|
||||
fs::write(&seg_path, &data).unwrap();
|
||||
|
||||
// Recovery should truncate the garbage
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
// Verify committed data is still intact
|
||||
for i in 0..10u64 {
|
||||
let result = engine.read("alice", &make_key(i)).unwrap();
|
||||
assert!(result.is_some(), "committed key {} should survive tail truncation", i);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify file was actually truncated
|
||||
let truncated_len = fs::metadata(&seg_path).unwrap().len();
|
||||
assert!(truncated_len <= orig_len as u64, "garbage should have been truncated");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 4. Isolation: concurrent reader sees consistent snapshot
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_isolation_reader_sees_snapshot_not_partial_write() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
// Pre-populate a known key
|
||||
let original_value = make_value(4096);
|
||||
let key = make_key(100);
|
||||
engine
|
||||
.write("alice", key, &original_value, Codec::Zstd)
|
||||
.unwrap();
|
||||
|
||||
let running = Arc::new(AtomicBool::new(true));
|
||||
let writer_done = Arc::new(AtomicBool::new(false));
|
||||
|
||||
// Spawn a writer that continuously overwrites the same key
|
||||
let writer_engine = engine.clone();
|
||||
let writer_running = running.clone();
|
||||
let writer_done_flag = writer_done.clone();
|
||||
let writer_key = key;
|
||||
|
||||
let writer = thread::spawn(move || {
|
||||
for i in 0..1000u64 {
|
||||
if !writer_running.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
let val = make_value(4096 + (i as usize % 100));
|
||||
writer_engine
|
||||
.write("alice", writer_key, &val, Codec::Zstd)
|
||||
.unwrap();
|
||||
thread::yield_now();
|
||||
}
|
||||
writer_done_flag.store(true, Ordering::SeqCst);
|
||||
});
|
||||
|
||||
// Concurrent reader: reads should never panic or hang
|
||||
let reader_engine = engine.clone();
|
||||
let reader_running = running.clone();
|
||||
let reader = thread::spawn(move || {
|
||||
let mut reads = 0;
|
||||
while reads < 500 {
|
||||
if !reader_running.load(Ordering::Relaxed) && reads > 0 {
|
||||
break;
|
||||
}
|
||||
let result = reader_engine.read("alice", &key);
|
||||
match result {
|
||||
Ok(Some(_)) | Ok(None) => {} // OK
|
||||
Err(e) => {
|
||||
// Accept transient errors but report them
|
||||
eprintln!("reader saw error: {:?}", e);
|
||||
}
|
||||
}
|
||||
reads += 1;
|
||||
thread::yield_now();
|
||||
}
|
||||
});
|
||||
|
||||
reader.join().unwrap();
|
||||
running.store(false, Ordering::SeqCst);
|
||||
writer.join().unwrap();
|
||||
|
||||
// Final read should see the last committed value (not partial)
|
||||
let final_result = engine.read("alice", &key).unwrap();
|
||||
assert!(final_result.is_some(), "final read should find a value");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Crash during GC: old data intact, no corruption
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_crash_during_gc_leaves_data_intact() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let value = make_value(500_000); // 500 KB each
|
||||
// Write enough entries and delete some to create GC candidate
|
||||
for i in 0..500u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
// Delete ~40%
|
||||
for i in (0..500u64).step_by(5) {
|
||||
engine.delete("alice", &make_key(i)).unwrap();
|
||||
}
|
||||
// Single GC run (may or may not trigger)
|
||||
let _ = engine.gc("alice");
|
||||
} // crash after GC
|
||||
|
||||
// All non-deleted entries must still be readable
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let value = make_value(500_000);
|
||||
for i in 0..500u64 {
|
||||
let key = make_key(i);
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
if i % 5 == 0 {
|
||||
// Deleted keys
|
||||
assert_eq!(result, None, "key {} should be deleted", i);
|
||||
} else {
|
||||
assert_eq!(
|
||||
result,
|
||||
Some(value.clone()),
|
||||
"key {} should survive GC+crash",
|
||||
i
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Multiple crash-reopen cycles (torture test)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_multiple_crash_reopen_cycles() {
|
||||
use std::collections::HashSet;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let value = make_value(4096);
|
||||
let mut alive: HashSet<u64> = HashSet::new();
|
||||
|
||||
// Populate and crash
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
for i in 0..50u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
alive.insert(i);
|
||||
}
|
||||
}
|
||||
|
||||
// Reopen, verify all exist, write more, crash
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for &k in &alive {
|
||||
assert!(engine.read("alice", &make_key(k)).unwrap().is_some());
|
||||
}
|
||||
for i in 100..150u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
alive.insert(i);
|
||||
}
|
||||
}
|
||||
|
||||
// Reopen, verify all exist, delete some, crash
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for &k in &alive {
|
||||
assert!(engine.read("alice", &make_key(k)).unwrap().is_some());
|
||||
}
|
||||
for i in 0..10u64 {
|
||||
engine.delete("alice", &make_key(i)).unwrap();
|
||||
alive.remove(&i);
|
||||
}
|
||||
}
|
||||
|
||||
// Final reopen: survivors exist, deleted gone
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for &k in &alive {
|
||||
assert!(engine.read("alice", &make_key(k)).unwrap().is_some(),
|
||||
"key {} should exist", k);
|
||||
}
|
||||
for i in 0..10u64 {
|
||||
assert_eq!(engine.read("alice", &make_key(i)).unwrap(), None,
|
||||
"key {} should be deleted", i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Account-level isolation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_account_isolation_crash_one_account_does_not_affect_others() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine.create_account("bob").unwrap();
|
||||
|
||||
engine
|
||||
.write("alice", make_key(1), &make_value(4096), Codec::Zstd)
|
||||
.unwrap();
|
||||
engine
|
||||
.write("bob", make_key(1), &make_value(8192), Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Delete alice's account dir partially to simulate corruption
|
||||
// Then verify bob is intact
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
// Bob should be fine
|
||||
let result = engine.read("bob", &make_key(1)).unwrap();
|
||||
assert!(result.is_some(), "bob should be unaffected");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn find_first_segment(store_root: &Path, account: &str) -> std::path::PathBuf {
|
||||
let seg_dir = store_root.join("accounts").join(account).join("segments");
|
||||
for entry in fs::read_dir(&seg_dir).unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
if name.ends_with(".seg") && !name.contains("temp_") {
|
||||
return entry.path();
|
||||
}
|
||||
}
|
||||
panic!("no segment found in {:?}", seg_dir);
|
||||
}
|
||||
497
crates/blob/tests/integration_test.rs
Normal file
497
crates/blob/tests/integration_test.rs
Normal file
@@ -0,0 +1,497 @@
|
||||
use bichon_blob::{Codec, Config, Engine};
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_create_and_list_accounts() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
|
||||
engine.create_account("alice").unwrap();
|
||||
engine.create_account("bob").unwrap();
|
||||
|
||||
let accounts = engine.list_accounts();
|
||||
assert!(accounts.contains(&"alice".to_string()));
|
||||
assert!(accounts.contains(&"bob".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_write_and_read() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xAA; 32];
|
||||
let value = b"Hello, this is a test email!".to_vec();
|
||||
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_read_missing_key() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xFF; 32];
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xBB; 32];
|
||||
let value = b"Some email content".to_vec();
|
||||
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
engine.delete("alice", &key).unwrap();
|
||||
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_account() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine.delete_account("alice").unwrap();
|
||||
|
||||
let accounts = engine.list_accounts();
|
||||
assert!(!accounts.contains(&"alice".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_small_value_not_compressed() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xCC; 32];
|
||||
let value = b"hi"; // Smaller than 4KB threshold
|
||||
|
||||
engine
|
||||
.write("alice", key, value, Codec::Zstd)
|
||||
.unwrap();
|
||||
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value.to_vec()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_large_value() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xDD; 32];
|
||||
let value = vec![b'X'; 100_000]; // 100KB
|
||||
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_keys() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let n = 100;
|
||||
for i in 0..n {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let value = format!("email number {}", i).into_bytes();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
for i in 0..n {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(format!("email number {}", i).into_bytes()));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gc() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
// Write many entries
|
||||
let value = vec![b'Y'; 5000];
|
||||
let n = 100;
|
||||
|
||||
for i in 0..n {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
engine
|
||||
.write("alice", key, &value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Delete even-numbered keys
|
||||
for i in (0..n).step_by(2) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
engine.delete("alice", &key).unwrap();
|
||||
}
|
||||
|
||||
// Run GC
|
||||
let _result = engine.gc("alice").unwrap();
|
||||
|
||||
// Verify remaining keys still readable
|
||||
for i in (1..n).step_by(2) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value.clone()));
|
||||
}
|
||||
|
||||
// Deleted keys should not exist
|
||||
for i in (0..n).step_by(2) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reopen_persistence() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let key = [0xEE; 32];
|
||||
let value = b"persistent data".to_vec();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Reopen
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_stats() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
engine
|
||||
.write("alice", [1u8; 32], b"hello", Codec::None)
|
||||
.unwrap();
|
||||
|
||||
let stats = engine.stats("alice").unwrap();
|
||||
assert!(stats.total_bytes > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_batch_write() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let n = 50;
|
||||
let entries: Vec<_> = (0..n)
|
||||
.map(|i: u64| {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..8].copy_from_slice(&i.to_le_bytes());
|
||||
let value = format!("batch email {}", i).into_bytes();
|
||||
(key, value, Codec::Zstd)
|
||||
})
|
||||
.collect();
|
||||
|
||||
engine.write_batch("alice", &entries).unwrap();
|
||||
|
||||
for (key, value, _) in &entries {
|
||||
let result = engine.read("alice", key).unwrap();
|
||||
assert_eq!(result.as_ref(), Some(value));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_batch_write_persistence() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let entries: Vec<_> = (0..30u64)
|
||||
.map(|i| {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..8].copy_from_slice(&i.to_le_bytes());
|
||||
(key, format!("persist {}", i).into_bytes(), Codec::Zstd)
|
||||
})
|
||||
.collect();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine.write_batch("alice", &entries).unwrap();
|
||||
}
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for (key, value, _) in &entries {
|
||||
let result = engine.read("alice", key).unwrap();
|
||||
assert_eq!(result.as_ref(), Some(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_config_rejected() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut config = Config::default();
|
||||
config.lru_bucket_count = 0;
|
||||
assert!(Engine::open(dir.path(), config).is_err());
|
||||
|
||||
let mut config = Config::default();
|
||||
config.gc_deleted_ratio = 1.5;
|
||||
assert!(Engine::open(dir.path(), config).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_concurrent_reads() {
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
// Write some data
|
||||
for i in 0..50u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
engine.write("alice", key, &vec![i as u8; 1024], Codec::None).unwrap();
|
||||
}
|
||||
|
||||
// Spawn 4 threads, each reading a different subset
|
||||
let mut handles = vec![];
|
||||
for t in 0..4 {
|
||||
let engine = engine.clone();
|
||||
handles.push(thread::spawn(move || {
|
||||
for i in (t * 12)..((t + 1) * 12) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let read = engine.read("alice", &key).unwrap();
|
||||
assert!(read.is_some(), "key {} should exist", i);
|
||||
}
|
||||
}));
|
||||
}
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_concurrent_writes_different_accounts() {
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
|
||||
|
||||
for name in &["alice", "bob", "carol"] {
|
||||
engine.create_account(name).unwrap();
|
||||
}
|
||||
|
||||
let mut handles = vec![];
|
||||
for (t, name) in ["alice", "bob", "carol"].iter().enumerate() {
|
||||
let engine = engine.clone();
|
||||
let account_name = name.to_string();
|
||||
handles.push(thread::spawn(move || {
|
||||
for i in 0..20 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&((t * 100 + i) as u32).to_le_bytes());
|
||||
let value = vec![(t * 100 + i) as u8; 512];
|
||||
engine.write(&account_name, key, &value, Codec::None).unwrap();
|
||||
}
|
||||
}));
|
||||
}
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
|
||||
// Verify all writes persisted
|
||||
for (t, name) in ["alice", "bob", "carol"].iter().enumerate() {
|
||||
for i in 0..20 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&((t * 100 + i) as u32).to_le_bytes());
|
||||
let read = engine.read(name, &key).unwrap();
|
||||
assert!(read.is_some(), "account {} key {} should exist", name, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crash_recovery() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let dir_path = dir.path().to_path_buf();
|
||||
|
||||
// Phase 1: write data, then drop without shutdown (simulates crash)
|
||||
{
|
||||
let engine = Engine::open(&dir_path, Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
for i in 0..50u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
engine.write("alice", key, &vec![i as u8; 512], Codec::None).unwrap();
|
||||
}
|
||||
// Engine dropped here without calling shutdown()
|
||||
}
|
||||
|
||||
// Phase 2: reopen — recovery should run, data should be intact
|
||||
let engine = Engine::open(&dir_path, Config::default()).unwrap();
|
||||
let stats = engine.stats("alice").unwrap();
|
||||
assert!(stats.total_keys > 0, "recovery should preserve data");
|
||||
|
||||
// Verify reads work
|
||||
for i in 0..50u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
let read = engine.read("alice", &key).unwrap();
|
||||
assert!(read.is_some(), "key {} should survive crash recovery", i);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_meta_bin_durability() {
|
||||
// Verify meta.bin has valid CRC and can be read after a write cycle.
|
||||
let dir = TempDir::new().unwrap();
|
||||
let dir_path = dir.path().to_path_buf();
|
||||
|
||||
{
|
||||
let engine = Engine::open(&dir_path, Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0x42u8; 32];
|
||||
engine.write("alice", key, b"durable", Codec::None).unwrap();
|
||||
}
|
||||
// Engine dropped → shutdown() called → meta saved via write_bin (with fsync)
|
||||
|
||||
// Verify meta.bin exists and has valid CRC
|
||||
let meta_path = dir_path
|
||||
.join("accounts")
|
||||
.join("alice")
|
||||
.join("meta.bin");
|
||||
assert!(meta_path.exists(), "meta.bin should exist after clean shutdown");
|
||||
|
||||
let data = std::fs::read(&meta_path).unwrap();
|
||||
assert!(data.len() >= 8, "meta.bin should have at least 8 bytes (crc + version)");
|
||||
|
||||
let stored_crc = u32::from_le_bytes(data[0..4].try_into().unwrap());
|
||||
assert_ne!(stored_crc, 0, "stored CRC should be non-zero");
|
||||
|
||||
// Reopen and verify data is intact
|
||||
let engine = Engine::open(&dir_path, Config::default()).unwrap();
|
||||
let read = engine.read("alice", &[0x42u8; 32]).unwrap();
|
||||
assert_eq!(read, Some(b"durable".to_vec()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gc_concurrent_with_writes() {
|
||||
// GC should not lose entries that are written concurrently.
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::thread;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
// Pre-fill: write enough to trigger eventual GC
|
||||
let big_value = vec![b'X'; 8192];
|
||||
for i in 0..500u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
engine.write("alice", key, &big_value, Codec::None).unwrap();
|
||||
}
|
||||
|
||||
// Delete some to create GC candidates
|
||||
for i in 0..250u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
engine.delete("alice", &key).unwrap();
|
||||
}
|
||||
|
||||
let running = Arc::new(AtomicBool::new(true));
|
||||
let engine_gc = engine.clone();
|
||||
let running_gc = running.clone();
|
||||
|
||||
// Thread 1: run GC in a loop
|
||||
let gc_handle = thread::spawn(move || {
|
||||
while running_gc.load(Ordering::Relaxed) {
|
||||
let _ = engine_gc.gc("alice");
|
||||
thread::sleep(std::time::Duration::from_millis(10));
|
||||
}
|
||||
});
|
||||
|
||||
// Thread 2: keep writing new entries
|
||||
let engine_write = engine.clone();
|
||||
let running_write = running.clone();
|
||||
let write_handle = thread::spawn(move || {
|
||||
let mut counter = 10000u32;
|
||||
while running_write.load(Ordering::Relaxed) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&counter.to_le_bytes());
|
||||
engine_write
|
||||
.write("alice", key, &vec![counter as u8; 256], Codec::None)
|
||||
.unwrap();
|
||||
counter += 1;
|
||||
}
|
||||
counter
|
||||
});
|
||||
|
||||
// Let them race for a bit
|
||||
thread::sleep(std::time::Duration::from_millis(500));
|
||||
running.store(false, Ordering::Relaxed);
|
||||
|
||||
gc_handle.join().unwrap();
|
||||
let final_counter = write_handle.join().unwrap();
|
||||
|
||||
// All written entries must be readable
|
||||
let mut missing = 0;
|
||||
for i in 0..500u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
if engine.read("alice", &key).unwrap().is_none() {
|
||||
// Entries 0..250 were deleted, they should be gone
|
||||
if i >= 250 {
|
||||
missing += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert_eq!(missing, 0, "pre-existing entries should survive concurrent GC");
|
||||
|
||||
// Entries written during the race should be readable
|
||||
for i in 10000..final_counter {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
let read = engine.read("alice", &key).unwrap();
|
||||
assert!(read.is_some(), "concurrently written key {} should exist after GC", i);
|
||||
}
|
||||
}
|
||||
@@ -16,12 +16,8 @@ reqwest.workspace = true
|
||||
toml = "0.9.8"
|
||||
memmap2 = "0.9.10"
|
||||
outlook-pst = { git = "https://github.com/rustmailer/outlook-pst-rs.git", branch = "main" }
|
||||
compressed-rtf = "1.0.1"
|
||||
chrono.workspace = true
|
||||
mail-send.workspace = true
|
||||
base64.workspace = true
|
||||
codepage-strings = "1.0.2"
|
||||
hex = "0.4.3"
|
||||
sysinfo.workspace = true
|
||||
indicatif.workspace = true
|
||||
serde_json.workspace = true
|
||||
@@ -21,7 +21,7 @@ use std::path::PathBuf;
|
||||
|
||||
use crate::api::sender::send_batch_request;
|
||||
use crate::mbox::gmail::determine_folder;
|
||||
use crate::mbox::reader::MboxFile;
|
||||
use bichon_core::import::reader::MboxFile;
|
||||
use crate::BichonCliConfig;
|
||||
use bichon_core::base64_encode_url_safe;
|
||||
use bichon_core::envelope::meta::{parse_bichon_metadata, BichonMetadata};
|
||||
@@ -37,7 +37,6 @@ const MAX_EMAIL_BYTES: usize = 100 * 1024 * 1024;
|
||||
const MAX_BUFFER_BYTES: usize = 200 * 1024 * 1024;
|
||||
|
||||
pub mod gmail;
|
||||
pub mod reader;
|
||||
|
||||
pub async fn handle_mbox_single_file_import(
|
||||
config: &BichonCliConfig,
|
||||
|
||||
@@ -16,21 +16,12 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use chrono::{DateTime, TimeZone, Utc};
|
||||
use dialoguer::theme::ColorfulTheme;
|
||||
use dialoguer::Input;
|
||||
use mail_send::mail_builder::headers::text::Text;
|
||||
use mail_send::mail_builder::MessageBuilder;
|
||||
use outlook_pst::ltp::prop_context::PropertyValue;
|
||||
|
||||
use crate::api::sender::send_batch_request;
|
||||
use crate::pst::encoding::decode_subject;
|
||||
use crate::BichonCliConfig;
|
||||
use bichon_core::base64_encode_url_safe;
|
||||
use dialoguer::Confirm;
|
||||
use outlook_pst::messaging::attachment::AttachmentProperties;
|
||||
use bichon_core::import::pst::build_eml_base64;
|
||||
use dialoguer::theme::ColorfulTheme;
|
||||
use dialoguer::{Confirm, Input};
|
||||
use outlook_pst::messaging::folder::Folder;
|
||||
use outlook_pst::messaging::message::{Message, MessageProperties};
|
||||
use outlook_pst::ndb::node_id::NodeId;
|
||||
use reqwest::Client;
|
||||
use std::future::Future;
|
||||
@@ -38,28 +29,6 @@ use std::path::PathBuf;
|
||||
use std::pin::Pin;
|
||||
use std::rc::Rc;
|
||||
|
||||
mod encoding;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct EmailMetadata {
|
||||
pub message_id: Option<String>,
|
||||
pub subject: Option<String>,
|
||||
pub from: Option<String>,
|
||||
pub to: Option<Vec<String>>,
|
||||
pub cc: Option<Vec<String>>,
|
||||
pub bcc: Option<Vec<String>>,
|
||||
pub html: Option<String>,
|
||||
pub text: Option<String>,
|
||||
pub in_reply_to: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct EmailAttachment {
|
||||
pub name: Option<String>,
|
||||
pub mime_type: Option<String>,
|
||||
pub data: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
pub async fn handle_pst_import(config: &BichonCliConfig, account_id: u64, theme: &ColorfulTheme) {
|
||||
let path_str: String = Input::with_theme(theme)
|
||||
.with_prompt("Enter the path to your SINGLE .pst file")
|
||||
@@ -244,167 +213,6 @@ fn process_folder_recursively<'a>(
|
||||
})
|
||||
}
|
||||
|
||||
fn build_eml_base64(message: Rc<dyn Message>) -> Option<String> {
|
||||
let properties = message.properties();
|
||||
|
||||
let mut builder = MessageBuilder::new();
|
||||
if let Some(sub) = extract_subject(properties) {
|
||||
builder = builder.subject(sub);
|
||||
}
|
||||
if let Some(mid) = extract_string_property(properties, 0x1035) {
|
||||
builder = builder.message_id(mid);
|
||||
}
|
||||
if let Some(irt) = extract_string_property(properties, 0x1042) {
|
||||
builder = builder.in_reply_to(irt);
|
||||
}
|
||||
|
||||
if let Some(refs) = extract_string_property(properties, 0x1039) {
|
||||
builder = builder.header("References", Text::new(refs));
|
||||
}
|
||||
|
||||
if let Some(cid_val) = properties.get(0x3013) {
|
||||
if let PropertyValue::Binary(bin) = cid_val {
|
||||
builder = builder.header(
|
||||
"X-Bichon-Conversation-ID",
|
||||
Text::new(hex::encode(bin.buffer())),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let from = extract_string_property(properties, 0x5D01)
|
||||
.or_else(|| extract_string_property(properties, 0x5D02))
|
||||
.or_else(|| extract_string_property(properties, 0x0C1F));
|
||||
|
||||
if let Some(f) = from {
|
||||
builder = builder.from(f);
|
||||
}
|
||||
|
||||
if let Some(filetime) = extract_i64_property(properties, &[0x0039, 0x0E06]) {
|
||||
let dt = filetime_to_datetime(filetime).timestamp();
|
||||
builder = builder.date(dt);
|
||||
}
|
||||
|
||||
let (to, cc, bcc) = extract_recipients_list(&message);
|
||||
if !to.is_empty() {
|
||||
builder = builder.to(to.iter().map(|s| s.as_str()).collect::<Vec<_>>());
|
||||
}
|
||||
if !cc.is_empty() {
|
||||
builder = builder.cc(cc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
|
||||
}
|
||||
if !bcc.is_empty() {
|
||||
builder = builder.bcc(bcc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
if let Some(html) = extract_html(properties) {
|
||||
builder = builder.html_body(html);
|
||||
}
|
||||
|
||||
if let Some(text) = extract_text(properties) {
|
||||
builder = builder.text_body(text);
|
||||
}
|
||||
|
||||
if let Some(attachment_table) = message.attachment_table() {
|
||||
for row in attachment_table.rows_matrix() {
|
||||
let node_id = NodeId::from(u32::from(row.id()));
|
||||
if let Ok(attachment) = message.clone().read_attachment(node_id, None) {
|
||||
let att_props = attachment.properties();
|
||||
let name = extract_attachment_string_property(att_props, 0x3707);
|
||||
let mime = extract_attachment_string_property(att_props, 0x370E)
|
||||
.unwrap_or_else(|| "application/octet-stream".into());
|
||||
let cid = extract_attachment_string_property(att_props, 0x3712);
|
||||
let is_inline = att_props
|
||||
.get(0x3714)
|
||||
.and_then(|val| {
|
||||
if let PropertyValue::Integer32(f) = val {
|
||||
Some(f)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.map(|flag| (flag & 0x4) != 0)
|
||||
.unwrap_or(false);
|
||||
|
||||
if let Some(PropertyValue::Binary(bin)) = att_props.get(0x3701) {
|
||||
let data = bin.buffer().to_vec();
|
||||
let file_name = name.unwrap_or_else(|| "unnamed_attachment".to_string());
|
||||
|
||||
if is_inline && cid.is_some() {
|
||||
let content_id = cid.unwrap();
|
||||
builder = builder.inline(mime, content_id, data);
|
||||
} else {
|
||||
builder = builder.attachment(mime, file_name, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match builder.write_to_vec() {
|
||||
Ok(eml_vec) => Some(base64_encode_url_safe!(eml_vec)),
|
||||
Err(e) => {
|
||||
eprintln!("Failed to generate EML: {:?}", e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn filetime_to_datetime(filetime: i64) -> DateTime<Utc> {
|
||||
let unix_secs = (filetime / 10_000_000) - 11_644_473_600;
|
||||
let nsecs = (filetime % 10_000_000) * 100;
|
||||
Utc.timestamp_opt(unix_secs, nsecs as u32).unwrap()
|
||||
}
|
||||
|
||||
fn extract_recipients_list(message: &Rc<dyn Message>) -> (Vec<String>, Vec<String>, Vec<String>) {
|
||||
let mut to = Vec::new();
|
||||
let mut cc = Vec::new();
|
||||
let mut bcc = Vec::new();
|
||||
|
||||
let recipient_table = message.recipient_table();
|
||||
if let Some(recipient_table) = recipient_table {
|
||||
let context = recipient_table.context();
|
||||
for row in recipient_table.rows_matrix() {
|
||||
if let Ok(cols) = row.columns(context) {
|
||||
let mut r_type = 0;
|
||||
let mut email = String::new();
|
||||
|
||||
for (col, val) in context.columns().iter().zip(cols) {
|
||||
let prop_val = val
|
||||
.as_ref()
|
||||
.and_then(|v| recipient_table.read_column(v, col.prop_type()).ok());
|
||||
match col.prop_id() {
|
||||
0x0C15 => {
|
||||
if let Some(PropertyValue::Integer32(t)) = prop_val {
|
||||
r_type = t;
|
||||
}
|
||||
}
|
||||
0x39FE | 0x3003 => {
|
||||
if let Some(s) = prop_val.and_then(|v| extract_string(&v)) {
|
||||
email = s;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if !email.is_empty() {
|
||||
match r_type {
|
||||
1 => to.push(email),
|
||||
2 => cc.push(email),
|
||||
3 => bcc.push(email),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let receiver = extract_string_property(message.properties(), 0x0076);
|
||||
if let Some(receiver) = receiver {
|
||||
to.push(receiver);
|
||||
}
|
||||
}
|
||||
(to, cc, bcc)
|
||||
}
|
||||
|
||||
async fn send_to_bichon(
|
||||
client: &Client,
|
||||
config: &BichonCliConfig,
|
||||
@@ -414,69 +222,3 @@ async fn send_to_bichon(
|
||||
) {
|
||||
send_batch_request(client, config, account_id, folder_path, emls).await;
|
||||
}
|
||||
|
||||
fn extract_subject(props: &MessageProperties) -> Option<String> {
|
||||
props.get(0x0037).and_then(|val| decode_subject(val))
|
||||
}
|
||||
|
||||
fn extract_string_property(properties: &MessageProperties, prop_id: u16) -> Option<String> {
|
||||
properties
|
||||
.get(prop_id)
|
||||
.and_then(|value| extract_string(value))
|
||||
}
|
||||
|
||||
fn extract_attachment_string_property(
|
||||
properties: &AttachmentProperties,
|
||||
prop_id: u16,
|
||||
) -> Option<String> {
|
||||
properties
|
||||
.get(prop_id)
|
||||
.and_then(|value| extract_string(value))
|
||||
}
|
||||
|
||||
fn extract_string(value: &PropertyValue) -> Option<String> {
|
||||
match value {
|
||||
PropertyValue::String8(value) => Some(value.to_string()),
|
||||
PropertyValue::Unicode(value) => Some(value.to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_text(properties: &MessageProperties) -> Option<String> {
|
||||
properties.get(0x1000).and_then(extract_string).or_else(|| {
|
||||
properties.get(0x1009).and_then(|value| match value {
|
||||
PropertyValue::Binary(value) => encoding::decode_rtf_compressed(value.buffer()),
|
||||
_ => None,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_html(properties: &MessageProperties) -> Option<String> {
|
||||
properties.get(0x1013).and_then(|value| match value {
|
||||
PropertyValue::Binary(value) => {
|
||||
let code_page = properties
|
||||
.get(0x3FDE)
|
||||
.and_then(|v| {
|
||||
if let PropertyValue::Integer32(cpid) = v {
|
||||
Some(*cpid as u16)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or(65001);
|
||||
encoding::decode_html_body(value.buffer(), code_page)
|
||||
}
|
||||
PropertyValue::String8(value) => Some(value.to_string()),
|
||||
PropertyValue::Unicode(value) => Some(value.to_string()),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_i64_property(properties: &MessageProperties, prop_ids: &[u16]) -> Option<i64> {
|
||||
for &prop_id in prop_ids {
|
||||
if let Some(PropertyValue::Time(value)) = properties.get(prop_id) {
|
||||
return Some(*value);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ poem-openapi = { version = "5.1.16", features = [
|
||||
], optional = true }
|
||||
chrono.workspace = true
|
||||
clap.workspace = true
|
||||
memdb.workspace = true
|
||||
bichon-memdb.workspace = true
|
||||
itertools.workspace = true
|
||||
ring.workspace = true
|
||||
serde.workspace = true
|
||||
@@ -72,3 +72,8 @@ scopeguard = "1.2.0"
|
||||
cron = "0.15"
|
||||
quick-xml = { version = "0.40.0", features = ["serialize"] }
|
||||
hickory-resolver = "0.26.0-alpha.1"
|
||||
memmap2 = "0.9.10"
|
||||
outlook-pst = { git = "https://github.com/rustmailer/outlook-pst-rs.git", branch = "main" }
|
||||
compressed-rtf = "1.0.1"
|
||||
codepage-strings = "1.0.2"
|
||||
hex.workspace = true
|
||||
|
||||
@@ -64,6 +64,216 @@ pub enum QuotaWindow {
|
||||
Monthly,
|
||||
}
|
||||
|
||||
/// Include/exclude filter rule.
|
||||
///
|
||||
/// - `include` non-empty: only values matching these patterns pass.
|
||||
/// - `exclude` non-empty: values matching these patterns are rejected.
|
||||
/// - Both empty: all values pass.
|
||||
/// - Both set: include checked first, then exclude.
|
||||
///
|
||||
/// Extension patterns use case-insensitive exact match; all others use regex.
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct FilterRule {
|
||||
#[serde(default)]
|
||||
pub include: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub exclude: Vec<String>,
|
||||
}
|
||||
|
||||
impl FilterRule {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.include.is_empty() && self.exclude.is_empty()
|
||||
}
|
||||
|
||||
fn matches_exact(&self, value: &str) -> bool {
|
||||
if !self.include.is_empty() && !self.include.iter().any(|e| e.eq_ignore_ascii_case(value)) {
|
||||
return false;
|
||||
}
|
||||
if !self.exclude.is_empty() && self.exclude.iter().any(|e| e.eq_ignore_ascii_case(value)) {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn matches_regex(&self, value: &str) -> bool {
|
||||
if !self.include.is_empty() && !matches_any_regex(&self.include, value) {
|
||||
return false;
|
||||
}
|
||||
if !self.exclude.is_empty() && matches_any_regex(&self.exclude, value) {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn validate_regex(&self, field: &str) -> Result<(), String> {
|
||||
validate_patterns(&self.include, &format!("{field}.include"))?;
|
||||
validate_patterns(&self.exclude, &format!("{field}.exclude"))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct ExtractionRules {
|
||||
/// Type 0: Master switch.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Type 1: File extensions (exact match, e.g. `{"include": ["pdf","docx"]}`).
|
||||
#[serde(default)]
|
||||
pub extensions: FilterRule,
|
||||
/// Type 2: Folder patterns (regex, e.g. `{"include": ["^INBOX/Invoices"]}`).
|
||||
#[serde(default)]
|
||||
pub folders: FilterRule,
|
||||
/// Type 3: Attachment filename patterns (regex).
|
||||
#[serde(default)]
|
||||
pub attachment_names: FilterRule,
|
||||
/// Type 4: Sender patterns (regex).
|
||||
#[serde(default)]
|
||||
pub senders: FilterRule,
|
||||
}
|
||||
|
||||
impl ExtractionRules {
|
||||
/// Returns `true` if the attachment should be extracted under these rules.
|
||||
pub fn should_extract(
|
||||
&self,
|
||||
ext: &str,
|
||||
folder: Option<&str>,
|
||||
attachment_name: Option<&str>,
|
||||
sender: Option<&str>,
|
||||
) -> bool {
|
||||
if !self.enabled {
|
||||
return false;
|
||||
}
|
||||
if !self.extensions.matches_exact(ext) {
|
||||
return false;
|
||||
}
|
||||
if !self.folders.is_empty() {
|
||||
if let Some(folder) = folder {
|
||||
if !self.folders.matches_regex(folder) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !self.attachment_names.is_empty() {
|
||||
if let Some(name) = attachment_name {
|
||||
if !self.attachment_names.matches_regex(name) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !self.senders.is_empty() {
|
||||
if let Some(sender) = sender {
|
||||
if !self.senders.matches_regex(sender) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
self.folders.validate_regex("folders")?;
|
||||
self.attachment_names.validate_regex("attachment_names")?;
|
||||
self.senders.validate_regex("senders")?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Archive filtering rules — skip unwanted emails before storage.
|
||||
///
|
||||
/// Rule types:
|
||||
/// 0 — Master switch
|
||||
/// 1 — Sender filter (regex)
|
||||
/// 2 — Subject filter (regex)
|
||||
/// 3 — Skip emails larger than this (bytes)
|
||||
/// 4 — Skip emails with spam headers (X-Spam-Flag, X-Spam)
|
||||
///
|
||||
/// `None` = archive everything (backward compatible).
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct ArchiveRules {
|
||||
/// Type 0: Master switch. `false` = archive everything.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Type 1: Sender filter (regex, include/exclude).
|
||||
#[serde(default)]
|
||||
pub senders: FilterRule,
|
||||
/// Type 2: Subject filter (regex, include/exclude).
|
||||
#[serde(default)]
|
||||
pub subjects: FilterRule,
|
||||
/// Type 3: Skip emails larger than this (bytes). `None` = no size limit.
|
||||
#[serde(default)]
|
||||
pub skip_larger_than: Option<u64>,
|
||||
/// Type 4: Spam header names to check (e.g. `["X-Spam-Flag", "X-Spam"]`).
|
||||
/// When the value is `yes` or `true` (case-insensitive), the email is skipped.
|
||||
/// Empty = don't check. Common headers: `X-Spam-Flag` (SpamAssassin),
|
||||
/// `X-Spam` (rspamd), `X-MS-Exchange-Organization-SCL` (Exchange).
|
||||
#[serde(default)]
|
||||
pub spam_headers: Vec<String>,
|
||||
}
|
||||
|
||||
impl ArchiveRules {
|
||||
/// Returns `true` if the email should be archived under these rules.
|
||||
pub fn should_archive(
|
||||
&self,
|
||||
sender: Option<&str>,
|
||||
subject: Option<&str>,
|
||||
size: u32,
|
||||
is_spam: bool,
|
||||
) -> bool {
|
||||
if !self.enabled {
|
||||
return true;
|
||||
}
|
||||
if !self.senders.is_empty() {
|
||||
if let Some(sender) = sender {
|
||||
if !self.senders.matches_regex(sender) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !self.subjects.is_empty() {
|
||||
if let Some(subject) = subject {
|
||||
if !self.subjects.matches_regex(subject) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(limit) = self.skip_larger_than {
|
||||
if size as u64 > limit {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if !self.spam_headers.is_empty() && is_spam {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Validate all regex patterns are well-formed.
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
self.senders.validate_regex("senders")?;
|
||||
self.subjects.validate_regex("subjects")?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn matches_any_regex(patterns: &[String], value: &str) -> bool {
|
||||
patterns.iter().any(|p| {
|
||||
regex::Regex::new(p)
|
||||
.map(|re| re.is_match(value))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_patterns(patterns: &[String], field_name: &str) -> Result<(), String> {
|
||||
for p in patterns {
|
||||
regex::Regex::new(p)
|
||||
.map_err(|e| format!("{} pattern '{}' is invalid regex: {}", field_name, p, e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct Account {
|
||||
@@ -99,6 +309,14 @@ pub struct Account {
|
||||
pub download_schedule: Option<String>,
|
||||
#[serde(default)]
|
||||
pub deleting: bool,
|
||||
/// Email-level filtering rules (Pro feature).
|
||||
/// `None` = archive everything (backward compatible).
|
||||
#[serde(default)]
|
||||
pub archive_rules: Option<ArchiveRules>,
|
||||
/// Attachment text extraction rules (Pro feature).
|
||||
/// `None` = extract everything (backward compatible).
|
||||
#[serde(default)]
|
||||
pub extraction_rules: Option<ExtractionRules>,
|
||||
}
|
||||
|
||||
impl MemDbModel for Account {
|
||||
@@ -139,6 +357,8 @@ impl Account {
|
||||
imap_quota_window: request.imap_quota_window,
|
||||
download_schedule: request.download_schedule,
|
||||
deleting: false,
|
||||
archive_rules: request.archive_rules,
|
||||
extraction_rules: request.extraction_rules,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -362,6 +582,7 @@ impl Account {
|
||||
.map(|account: AccountModel| MinimalAccount {
|
||||
id: account.id,
|
||||
email: account.email,
|
||||
name: account.account_name,
|
||||
})
|
||||
.collect::<Vec<MinimalAccount>>();
|
||||
Ok(result)
|
||||
@@ -477,7 +698,299 @@ impl Account {
|
||||
if request.clear_download_schedule == Some(true) {
|
||||
new.download_schedule = None;
|
||||
}
|
||||
if request.extraction_rules.is_some() {
|
||||
new.extraction_rules = request.extraction_rules;
|
||||
}
|
||||
if request.archive_rules.is_some() {
|
||||
new.archive_rules = request.archive_rules;
|
||||
}
|
||||
new.updated_at = utc_now!();
|
||||
Ok(new)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// ── FilterRule ───────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn filter_rule_include_only() {
|
||||
let r = FilterRule {
|
||||
include: vec![r"@ok\.com$".into()],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(r.matches_regex("bob@ok.com"));
|
||||
assert!(!r.matches_regex("spam@bad.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_rule_exclude_only() {
|
||||
let r = FilterRule {
|
||||
exclude: vec![r"@spam\.com$".into()],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(r.matches_regex("bob@ok.com"));
|
||||
assert!(!r.matches_regex("bot@spam.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_rule_include_then_exclude() {
|
||||
let r = FilterRule {
|
||||
include: vec![r"@company\.com$".into()],
|
||||
exclude: vec![r"noreply@company\.com$".into()],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(r.matches_regex("bob@company.com"));
|
||||
assert!(!r.matches_regex("noreply@company.com"));
|
||||
assert!(!r.matches_regex("spam@other.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_rule_exact_match() {
|
||||
let r = FilterRule {
|
||||
include: vec!["pdf".into(), "docx".into()],
|
||||
exclude: vec!["xlsx".into()],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(r.matches_exact("pdf"));
|
||||
assert!(r.matches_exact("docx"));
|
||||
assert!(r.matches_exact("DOCX")); // case-insensitive
|
||||
assert!(!r.matches_exact("xlsx"));
|
||||
assert!(!r.matches_exact("txt"));
|
||||
}
|
||||
|
||||
// ── ExtractionRules ─────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn extraction_rules_master_switch() {
|
||||
let rules = ExtractionRules {
|
||||
enabled: false,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!rules.should_extract("pdf", None, None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_rules_extension_include() {
|
||||
let rules = ExtractionRules {
|
||||
enabled: true,
|
||||
extensions: FilterRule {
|
||||
include: vec!["pdf".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.should_extract("pdf", None, None, None));
|
||||
assert!(!rules.should_extract("docx", None, None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_rules_extension_exclude() {
|
||||
let rules = ExtractionRules {
|
||||
enabled: true,
|
||||
extensions: FilterRule {
|
||||
exclude: vec!["xlsx".into(), "pptx".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.should_extract("pdf", None, None, None));
|
||||
assert!(!rules.should_extract("xlsx", None, None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_rules_folder_regex() {
|
||||
let rules = ExtractionRules {
|
||||
enabled: true,
|
||||
folders: FilterRule {
|
||||
include: vec![r"^INBOX/Invoices".into(), r"Contracts$".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.should_extract("pdf", Some("INBOX/Invoices"), None, None));
|
||||
assert!(rules.should_extract("pdf", Some("Finance/Contracts"), None, None));
|
||||
assert!(!rules.should_extract("pdf", Some("INBOX/Junk"), None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_rules_attachment_name_regex() {
|
||||
let rules = ExtractionRules {
|
||||
enabled: true,
|
||||
attachment_names: FilterRule {
|
||||
include: vec![r"^invoice-.*\.pdf$".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.should_extract("pdf", None, Some("invoice-2024.pdf"), None));
|
||||
assert!(!rules.should_extract("pdf", None, Some("newsletter.pdf"), None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_rules_sender_regex() {
|
||||
let rules = ExtractionRules {
|
||||
enabled: true,
|
||||
senders: FilterRule {
|
||||
exclude: vec![r"@noreply\.com$".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
// non-excluded sender passes
|
||||
assert!(rules.should_extract("pdf", None, None, Some("bob@ok.com")));
|
||||
// excluded sender blocked
|
||||
assert!(!rules.should_extract("pdf", None, None, Some("bot@noreply.com")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_rules_empty_filters_pass_everything() {
|
||||
let rules = ExtractionRules {
|
||||
enabled: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.should_extract(
|
||||
"anything",
|
||||
Some("any/folder"),
|
||||
Some("any.pdf"),
|
||||
Some("any@x.com")
|
||||
));
|
||||
}
|
||||
|
||||
// ── ArchiveRules ────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn archive_rules_disabled_archives_everything() {
|
||||
let rules = ArchiveRules {
|
||||
enabled: false,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.should_archive(Some("spam@x.com"), Some("BUY NOW"), 999, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_rules_sender_exclude() {
|
||||
let rules = ArchiveRules {
|
||||
enabled: true,
|
||||
senders: FilterRule {
|
||||
exclude: vec![r"@spam\.com$".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!rules.should_archive(Some("bot@spam.com"), None, 100, false));
|
||||
assert!(rules.should_archive(Some("friend@ok.com"), None, 100, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_rules_subject_exclude() {
|
||||
let rules = ArchiveRules {
|
||||
enabled: true,
|
||||
subjects: FilterRule {
|
||||
exclude: vec![r"(?i)unsubscribe|buy now|limited offer".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!rules.should_archive(None, Some("UNSUBSCRIBE NOW"), 100, false));
|
||||
assert!(!rules.should_archive(None, Some("Limited Offer!!"), 100, false));
|
||||
assert!(rules.should_archive(None, Some("Meeting tomorrow"), 100, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_rules_sender_include() {
|
||||
// Only archive emails from specific senders
|
||||
let rules = ArchiveRules {
|
||||
enabled: true,
|
||||
senders: FilterRule {
|
||||
include: vec![r"@partner\.com$".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.should_archive(Some("bob@partner.com"), None, 100, false));
|
||||
assert!(!rules.should_archive(Some("spam@random.com"), None, 100, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_rules_skip_larger_than() {
|
||||
let rules = ArchiveRules {
|
||||
enabled: true,
|
||||
skip_larger_than: Some(50_000_000),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.should_archive(None, None, 1_000_000, false));
|
||||
assert!(!rules.should_archive(None, None, 60_000_000, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_rules_skip_spam_headers() {
|
||||
let rules = ArchiveRules {
|
||||
enabled: true,
|
||||
spam_headers: vec!["X-Spam-Flag".into()],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!rules.should_archive(None, None, 100, true));
|
||||
assert!(rules.should_archive(None, None, 100, false));
|
||||
}
|
||||
|
||||
// ── Validation ──────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn validate_extraction_rules_valid() {
|
||||
let rules = ExtractionRules {
|
||||
folders: FilterRule {
|
||||
include: vec![r"^INBOX/.*".into()],
|
||||
..Default::default()
|
||||
},
|
||||
senders: FilterRule {
|
||||
exclude: vec![r"@spam\.com$".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_extraction_rules_invalid_regex() {
|
||||
let rules = ExtractionRules {
|
||||
folders: FilterRule {
|
||||
include: vec!["***bad[".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_archive_rules_valid() {
|
||||
let rules = ArchiveRules {
|
||||
senders: FilterRule {
|
||||
exclude: vec![r"@spam\.com$".into()],
|
||||
..Default::default()
|
||||
},
|
||||
subjects: FilterRule {
|
||||
include: vec![r"(?i)invoice".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_archive_rules_invalid_regex() {
|
||||
let rules = ArchiveRules {
|
||||
senders: FilterRule {
|
||||
include: vec!["[unclosed".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.validate().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,9 @@
|
||||
use std::str::FromStr;
|
||||
|
||||
use crate::account::entity::ImapConfig;
|
||||
use crate::account::migration::{AccountModel, AccountType, QuotaWindow};
|
||||
use crate::account::migration::{
|
||||
AccountModel, AccountType, ArchiveRules, ExtractionRules, QuotaWindow,
|
||||
};
|
||||
use crate::account::since::{DateSince, RelativeDate};
|
||||
use crate::error::code::ErrorCode;
|
||||
use crate::error::BichonResult;
|
||||
@@ -56,6 +58,12 @@ pub struct AccountCreateRequest {
|
||||
pub imap_quota_window: Option<QuotaWindow>,
|
||||
pub auto_download_new_mailboxes: Option<bool>,
|
||||
pub download_schedule: Option<String>,
|
||||
/// Email archive filtering rules (Pro feature).
|
||||
/// `None` = archive everything (backward compatible).
|
||||
pub archive_rules: Option<ArchiveRules>,
|
||||
/// Attachment text extraction rules (Pro feature).
|
||||
/// `None` = extract everything (backward compatible).
|
||||
pub extraction_rules: Option<ExtractionRules>,
|
||||
}
|
||||
|
||||
impl AccountCreateRequest {
|
||||
@@ -106,6 +114,19 @@ impl AccountCreateRequest {
|
||||
}
|
||||
AccountType::NoSync => {}
|
||||
}
|
||||
if let Some(ref rules) = self.extraction_rules {
|
||||
rules.validate().map_err(|e| {
|
||||
raise_error!(
|
||||
format!("extraction_rules: {}", e),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
}
|
||||
if let Some(ref rules) = self.archive_rules {
|
||||
rules.validate().map_err(|e| {
|
||||
raise_error!(format!("archive_rules: {}", e), ErrorCode::InvalidParameter)
|
||||
})?;
|
||||
}
|
||||
Ok(AccountModel::new(user_id, self)?)
|
||||
}
|
||||
|
||||
@@ -180,6 +201,12 @@ pub struct AccountUpdateRequest {
|
||||
pub auto_download_new_mailboxes: Option<bool>,
|
||||
pub download_schedule: Option<String>,
|
||||
pub clear_download_schedule: Option<bool>,
|
||||
/// Email archive filtering rules (Pro feature).
|
||||
/// `None` = no change. Use `Some(ArchiveRules { .. })` to set.
|
||||
pub archive_rules: Option<ArchiveRules>,
|
||||
/// Attachment text extraction rules (Pro feature).
|
||||
/// `None` = no change. Use `Some(ExtractionRules { .. })` to set.
|
||||
pub extraction_rules: Option<ExtractionRules>,
|
||||
}
|
||||
|
||||
impl AccountUpdateRequest {
|
||||
@@ -235,6 +262,19 @@ impl AccountUpdateRequest {
|
||||
validate_cron_expression(schedule)?;
|
||||
}
|
||||
}
|
||||
if let Some(ref rules) = self.extraction_rules {
|
||||
rules.validate().map_err(|e| {
|
||||
raise_error!(
|
||||
format!("extraction_rules: {}", e),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
}
|
||||
if let Some(ref rules) = self.archive_rules {
|
||||
rules.validate().map_err(|e| {
|
||||
raise_error!(format!("archive_rules: {}", e), ErrorCode::InvalidParameter)
|
||||
})?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -261,6 +301,7 @@ fn validate_cron_expression(expr: &str) -> BichonResult<()> {
|
||||
pub struct MinimalAccount {
|
||||
pub id: u64,
|
||||
pub email: String,
|
||||
pub name: Option<String>,
|
||||
}
|
||||
|
||||
pub fn filter_accessible_accounts<'a>(
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use memdb::{Durability, MemDb};
|
||||
use bichon_memdb::{Durability, MemDb};
|
||||
|
||||
use crate::{
|
||||
database::MemDbModel,
|
||||
|
||||
1023
crates/core/src/cache/imap/download/flow.rs
vendored
1023
crates/core/src/cache/imap/download/flow.rs
vendored
File diff suppressed because it is too large
Load Diff
@@ -17,7 +17,7 @@
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use crate::settings::dir::DATA_DIR_MANAGER;
|
||||
use memdb::{Durability, MemDb};
|
||||
use bichon_memdb::{Durability, MemDb};
|
||||
use std::sync::LazyLock;
|
||||
use std::time::Duration;
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ use crate::common::paginated::Paginated;
|
||||
use crate::error::code::ErrorCode;
|
||||
use crate::error::BichonResult;
|
||||
use crate::raise_error;
|
||||
use memdb::{MemDb, Transaction};
|
||||
use bichon_memdb::{MemDb, Transaction};
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::Serialize;
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use crate::account::migration::AccountModel;
|
||||
use crate::cache::imap::mailbox::MailBox;
|
||||
use crate::common::AddrVec;
|
||||
use crate::envelope::meta::parse_bichon_metadata;
|
||||
@@ -112,6 +113,34 @@ async fn extract_envelope_core(
|
||||
)
|
||||
})?;
|
||||
|
||||
if let Ok(account) = AccountModel::get(account_id) {
|
||||
if let Some(ref rules) = account.archive_rules {
|
||||
let sender = message.from().and_then(|addr| {
|
||||
AddrVec::from(addr).0.into_iter().next().and_then(|a| a.address)
|
||||
});
|
||||
let subject = message.subject().map(|s| s.to_string());
|
||||
|
||||
let is_spam = !rules.spam_headers.is_empty()
|
||||
&& rules.spam_headers.iter().any(|h| {
|
||||
message
|
||||
.header_raw(h.clone())
|
||||
.map(|v| matches!(v.trim().to_lowercase().as_str(), "yes" | "true"))
|
||||
.unwrap_or(false)
|
||||
});
|
||||
|
||||
if !rules.should_archive(sender.as_deref(), subject.as_deref(), size, is_spam) {
|
||||
tracing::debug!(
|
||||
account_id,
|
||||
uid,
|
||||
sender = sender.as_deref().unwrap_or("?"),
|
||||
subject = subject.as_deref().unwrap_or("?"),
|
||||
"Email filtered out by archive rules"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let preview_limit = 100;
|
||||
let text = if let Some(text) = message.body_text(0).map(|cow| cow.into_owned()) {
|
||||
text
|
||||
@@ -173,7 +202,7 @@ async fn extract_envelope_core(
|
||||
.and_then(|add| add.address)
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
let attachment_count = message.attachment_count();
|
||||
let attachments = detach_and_store_attachments(body, &message, &email_content_hash).await;
|
||||
let attachments = detach_and_store_attachments(body, &message, &email_content_hash, account_id, mailbox_id).await;
|
||||
|
||||
let envelope_id = Uuid::new_v4().to_string();
|
||||
let now = utc_now!();
|
||||
@@ -267,6 +296,7 @@ async fn extract_envelope_core(
|
||||
account_email: None,
|
||||
mailbox_name: None,
|
||||
content_hash: email_content_hash.clone(),
|
||||
account_name: None,
|
||||
};
|
||||
// 'attachments' contains both regular and inline attachments
|
||||
let ea = EnvelopeWithAttachments {
|
||||
@@ -361,6 +391,7 @@ pub fn extract_envelope_from_nested_message(
|
||||
regular_attachment_count: Default::default(),
|
||||
tags: Default::default(),
|
||||
account_email: Default::default(),
|
||||
account_name: Default::default(),
|
||||
mailbox_name: Default::default(),
|
||||
content_hash: Default::default(),
|
||||
};
|
||||
@@ -399,7 +430,27 @@ pub async fn detach_and_store_attachments(
|
||||
original_body: &[u8],
|
||||
message: &Message<'_>,
|
||||
eml_content_hash: &str,
|
||||
account_id: u64,
|
||||
mailbox_id: u64,
|
||||
) -> Vec<AttachmentInfo> {
|
||||
let rules = if account_id > 0 {
|
||||
AccountModel::get(account_id)
|
||||
.ok()
|
||||
.and_then(|a| a.extraction_rules)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mailbox_name = match rules.as_ref().map(|r| !r.folders.is_empty()) {
|
||||
Some(true) => MailBox::get(mailbox_id).ok().map(|mb| mb.name),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let sender = message
|
||||
.from()
|
||||
.and_then(|addr| AddrVec::from(addr).0.into_iter().next())
|
||||
.and_then(|add| add.address);
|
||||
|
||||
let mut stripped_eml = original_body.to_vec();
|
||||
let mut attachment_infos = Vec::new();
|
||||
// Step 1: Collect and sort attachment ranges in reverse to maintain offset integrity
|
||||
@@ -468,19 +519,30 @@ pub async fn detach_and_store_attachments(
|
||||
})
|
||||
.unwrap_or_else(|| "application/octet-stream".to_string());
|
||||
let has_cid = att.content_id().is_some();
|
||||
let ext = att
|
||||
.attachment_name()
|
||||
let att_name = att.attachment_name().map(|n| n.to_string());
|
||||
let ext = att_name
|
||||
.as_deref()
|
||||
.and_then(|n| {
|
||||
std::path::Path::new(&n)
|
||||
std::path::Path::new(n)
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|s| s.to_ascii_lowercase())
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let should_extract = rules.as_ref().map_or(true, |r| {
|
||||
r.should_extract(
|
||||
&ext,
|
||||
mailbox_name.as_deref(),
|
||||
att_name.as_deref(),
|
||||
sender.as_deref(),
|
||||
)
|
||||
});
|
||||
|
||||
if !inline || !has_cid {
|
||||
let decoded_len = att.contents().len();
|
||||
if decoded_len <= crate::ext::text_extractor::MAX_EXTRACT_BYTES
|
||||
if should_extract
|
||||
&& decoded_len <= crate::ext::text_extractor::MAX_EXTRACT_BYTES
|
||||
&& crate::ext::text_extractor::should_try_extract(&file_type, &ext)
|
||||
{
|
||||
text_candidates.push(TextCandidate {
|
||||
@@ -731,7 +793,7 @@ async fn recover_message_blob(envelope: &Envelope) -> BichonResult<Bytes> {
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
detach_and_store_attachments(&raw_body, &message, &fetched_hash).await;
|
||||
detach_and_store_attachments(&raw_body, &message, &fetched_hash, envelope.account_id, envelope.mailbox_id).await;
|
||||
|
||||
Ok(Bytes::from(raw_body))
|
||||
}
|
||||
@@ -828,6 +890,8 @@ mod test {
|
||||
truncated,
|
||||
&message,
|
||||
"test_content_hash",
|
||||
0,
|
||||
0,
|
||||
)
|
||||
.await;
|
||||
|
||||
|
||||
@@ -63,8 +63,18 @@ pub const MAX_EXTRACT_BYTES: usize = 10 * 1024 * 1024;
|
||||
pub fn should_try_extract(content_type: &str, ext: &str) -> bool {
|
||||
matches!(
|
||||
ext,
|
||||
"pdf" | "doc" | "docx" | "xls" | "xlsx" | "ppt" | "pptx"
|
||||
| "txt" | "rtf" | "odt" | "ods" | "odp"
|
||||
"pdf"
|
||||
| "doc"
|
||||
| "docx"
|
||||
| "xls"
|
||||
| "xlsx"
|
||||
| "ppt"
|
||||
| "pptx"
|
||||
| "txt"
|
||||
| "rtf"
|
||||
| "odt"
|
||||
| "ods"
|
||||
| "odp"
|
||||
) || content_type.starts_with("text/")
|
||||
}
|
||||
|
||||
|
||||
@@ -34,6 +34,25 @@ use std::ops::DerefMut;
|
||||
use tokio::io::BufWriter;
|
||||
use tracing::debug;
|
||||
|
||||
/// Classify an `io::Error` (from TLS stream I/O) for IMAP connection errors.
|
||||
/// `UnexpectedEof` is treated as a network error because many servers skip
|
||||
/// the TLS `close_notify` alert, causing rustls to emit this error when the
|
||||
/// TCP connection is dropped normally.
|
||||
fn classify_io_error(e: &std::io::Error) -> ErrorCode {
|
||||
use std::io::ErrorKind;
|
||||
matches!(
|
||||
e.kind(),
|
||||
ErrorKind::BrokenPipe
|
||||
| ErrorKind::ConnectionReset
|
||||
| ErrorKind::ConnectionAborted
|
||||
| ErrorKind::TimedOut
|
||||
| ErrorKind::UnexpectedEof
|
||||
| ErrorKind::NotConnected
|
||||
)
|
||||
.then_some(ErrorCode::NetworkError)
|
||||
.unwrap_or(ErrorCode::ImapCommandFailed)
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct Client {
|
||||
inner: ImapClient<Box<dyn SessionStream>>,
|
||||
@@ -141,7 +160,7 @@ impl Client {
|
||||
let _greeting = client
|
||||
.read_response()
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))?
|
||||
.ok_or_else(|| {
|
||||
raise_error!(
|
||||
"Failed to read IMAP greeting — this usually indicates an incorrect encryption setting (SSL vs. STARTTLS). Your current setting is SSL.".into(),
|
||||
@@ -171,7 +190,7 @@ impl Client {
|
||||
let _greeting = client
|
||||
.read_response()
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))?
|
||||
.ok_or_else(|| {
|
||||
raise_error!(
|
||||
"failed to read greeting".into(),
|
||||
@@ -202,7 +221,7 @@ impl Client {
|
||||
let _greeting = client
|
||||
.read_response()
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))?
|
||||
.ok_or_else(|| {
|
||||
raise_error!(
|
||||
"Failed to read IMAP greeting — this usually indicates an incorrect encryption setting (SSL vs. STARTTLS). Your current setting is STARTTLS.".into(),
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::{error::BichonResult, imap::manager::ImapConnectionManager};
|
||||
use async_imap::types::Name;
|
||||
use async_imap::Session;
|
||||
use futures::TryStreamExt;
|
||||
use std::collections::HashSet;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::info;
|
||||
|
||||
@@ -260,7 +260,9 @@ impl ImapExecutor {
|
||||
let mut count = 0u64;
|
||||
let mut skipped = 0u64;
|
||||
let mut max_uid: Option<u32> = None;
|
||||
let size_limit = account.max_email_size_bytes.unwrap_or(DEFAULT_MAX_EMAIL_SIZE);
|
||||
let size_limit = account
|
||||
.max_email_size_bytes
|
||||
.unwrap_or(DEFAULT_MAX_EMAIL_SIZE);
|
||||
while let Some(fetch) = stream
|
||||
.try_next()
|
||||
.await
|
||||
@@ -363,14 +365,14 @@ impl ImapExecutor {
|
||||
let mut size_stream = session
|
||||
.fetch(sequence_set.as_str(), SIZE_ONLY_FETCH)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
raise_error!(format!("{:#?}", e), classify_imap_error(&e))
|
||||
})?;
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?;
|
||||
|
||||
let mut uids: Vec<u32> = Vec::new();
|
||||
while let Some(fetch) = size_stream.try_next().await.map_err(|e| {
|
||||
raise_error!(format!("{:#?}", e), classify_imap_error(&e))
|
||||
})? {
|
||||
while let Some(fetch) = size_stream
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?
|
||||
{
|
||||
let uid = fetch.uid.unwrap_or(0);
|
||||
let msg_size = fetch.size.unwrap_or(0) as u64;
|
||||
if msg_size == 0 || msg_size <= limit {
|
||||
@@ -437,14 +439,14 @@ impl ImapExecutor {
|
||||
let mut size_stream = session
|
||||
.uid_fetch(uid_set, SIZE_ONLY_FETCH)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
raise_error!(format!("{:#?}", e), classify_imap_error(&e))
|
||||
})?;
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?;
|
||||
|
||||
let mut uids: Vec<u32> = Vec::new();
|
||||
while let Some(fetch) = size_stream.try_next().await.map_err(|e| {
|
||||
raise_error!(format!("{:#?}", e), classify_imap_error(&e))
|
||||
})? {
|
||||
while let Some(fetch) = size_stream
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?
|
||||
{
|
||||
let uid = fetch.uid.unwrap_or(0);
|
||||
let msg_size = fetch.size.unwrap_or(0) as u64;
|
||||
if msg_size == 0 || msg_size <= limit {
|
||||
@@ -550,6 +552,37 @@ impl ImapExecutor {
|
||||
) -> BichonResult<Session<Box<dyn SessionStream>>> {
|
||||
ImapConnectionManager::build(account_id).await
|
||||
}
|
||||
|
||||
/// Fetch UID → Message-ID mapping without downloading bodies.
|
||||
/// `uid_set` is an IMAP sequence-set string (e.g. "1:100" or "1,3,5").
|
||||
pub async fn fetch_uid_metadata(
|
||||
session: &mut Session<Box<dyn SessionStream>>,
|
||||
uid_set: &str,
|
||||
token: CancellationToken,
|
||||
) -> BichonResult<HashMap<u32, Option<String>>> {
|
||||
let mut stream = session
|
||||
.uid_fetch(uid_set, "(UID BODY.PEEK[HEADER])")
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?;
|
||||
|
||||
let mut result = HashMap::new();
|
||||
while let Some(fetch) = stream
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?
|
||||
{
|
||||
if token.is_cancelled() {
|
||||
return Err(raise_error!(
|
||||
"Stream cancelled".into(),
|
||||
ErrorCode::InternalError
|
||||
));
|
||||
}
|
||||
let uid = fetch.uid.unwrap_or(0);
|
||||
let msg_id = fetch.header().and_then(parse_message_id_header);
|
||||
result.insert(uid, msg_id);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
pub const DEFAULT_BATCH_SIZE: u32 = 30;
|
||||
@@ -613,6 +646,27 @@ pub fn generate_uid_sequence_hashset(
|
||||
result
|
||||
}
|
||||
|
||||
fn parse_message_id_header(header_bytes: &[u8]) -> Option<String> {
|
||||
let header = std::str::from_utf8(header_bytes).ok()?;
|
||||
for line in header.lines() {
|
||||
if let Some(value) = line
|
||||
.strip_prefix("Message-ID:")
|
||||
.or_else(|| line.strip_prefix("Message-Id:"))
|
||||
.or_else(|| line.strip_prefix("Message-id:"))
|
||||
{
|
||||
// mail_parser strips angle brackets, so we must do the same
|
||||
// to ensure comparisons against the Tantivy index match.
|
||||
let trimmed = value.trim();
|
||||
let stripped = trimmed.strip_prefix('<').unwrap_or(trimmed);
|
||||
let stripped = stripped.strip_suffix('>').unwrap_or(stripped);
|
||||
if !stripped.is_empty() {
|
||||
return Some(stripped.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
@@ -668,4 +722,80 @@ mod test {
|
||||
assert_eq!(batches[2].0, "5");
|
||||
assert_eq!(batches[2].1, 1);
|
||||
}
|
||||
|
||||
// ── parse_message_id_header ─────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn parse_standard_message_id() {
|
||||
let header = b"Message-ID: <abc123@example.com>\r\n";
|
||||
assert_eq!(
|
||||
parse_message_id_header(header),
|
||||
Some("abc123@example.com".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_message_id_lowercase() {
|
||||
let header = b"Message-Id: <foo@bar.com>\r\n";
|
||||
assert_eq!(
|
||||
parse_message_id_header(header),
|
||||
Some("foo@bar.com".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_message_id_extra_whitespace() {
|
||||
let header = b"Message-ID: <spaces@test.com> \r\n";
|
||||
assert_eq!(
|
||||
parse_message_id_header(header),
|
||||
Some("spaces@test.com".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_empty_message_id_returns_none() {
|
||||
let header = b"Message-ID: <>\r\n";
|
||||
assert_eq!(parse_message_id_header(header), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_missing_header_returns_none() {
|
||||
let header = b"X-Custom: something\r\n";
|
||||
assert_eq!(parse_message_id_header(header), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_empty_body_returns_none() {
|
||||
assert_eq!(parse_message_id_header(b""), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_message_id_in_full_header() {
|
||||
// The Message-ID line is in the middle, not at the start.
|
||||
let header = b"From: sender@example.com\r\n\
|
||||
Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\
|
||||
Subject: test\r\n\
|
||||
Message-ID: <mid@example.com>\r\n\
|
||||
To: recipient@example.com\r\n\r\n";
|
||||
assert_eq!(
|
||||
parse_message_id_header(header),
|
||||
Some("mid@example.com".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_message_id_only_in_full_header() {
|
||||
// Only a few headers, Message-ID is among them.
|
||||
let header = b"From: a@b.com\r\nMessage-ID: <x@y.com>\r\n\r\n";
|
||||
assert_eq!(parse_message_id_header(header), Some("x@y.com".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_message_id_no_brackets_still_works() {
|
||||
let header = b"Message-ID: plain@example.com\r\n";
|
||||
assert_eq!(
|
||||
parse_message_id_header(header),
|
||||
Some("plain@example.com".into())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,16 +95,40 @@ impl ImapConnectionManager {
|
||||
|
||||
pub async fn build(account_id: u64) -> BichonResult<Session<Box<dyn SessionStream>>> {
|
||||
let account = AccountModel::get(account_id)?;
|
||||
let client = match Self::create_client(&account).await {
|
||||
Ok(client) => client,
|
||||
Err(error) => {
|
||||
error!(
|
||||
"Failed to create IMAP {}'s client: {:#?}",
|
||||
&account.email, error
|
||||
);
|
||||
return Err(error);
|
||||
let account_email = account.email.clone();
|
||||
|
||||
let mut client = None;
|
||||
for attempt in 0..3u32 {
|
||||
match Self::create_client(&account).await {
|
||||
Ok(c) => {
|
||||
client = Some(c);
|
||||
break;
|
||||
}
|
||||
Err(error) if error.code() == ErrorCode::NetworkError && attempt < 2 => {
|
||||
warn!(
|
||||
"IMAP connection attempt {}/3 to {} failed (network error), retrying...",
|
||||
attempt + 1,
|
||||
account_email
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||||
continue;
|
||||
}
|
||||
Err(error) => {
|
||||
error!(
|
||||
"Failed to create IMAP {}'s client: {:#?}",
|
||||
account_email, error
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
let client = client.ok_or_else(|| {
|
||||
raise_error!(
|
||||
format!("Failed to create IMAP {}'s client after 3 attempts", account_email),
|
||||
ErrorCode::NetworkError
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut session = match Self::authenticate(client, &account).await {
|
||||
Ok(session) => session,
|
||||
|
||||
538
crates/core/src/imap/mock_server.rs
Normal file
538
crates/core/src/imap/mock_server.rs
Normal file
@@ -0,0 +1,538 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//! A minimal scriptable IMAP server for integration testing.
|
||||
//!
|
||||
//! Each instance listens on a random localhost port and responds to a
|
||||
//! pre-configured script of (expected_command, response) pairs. Commands
|
||||
//! are matched by substring — the first matching pattern wins.
|
||||
//!
|
||||
//! # Example
|
||||
//! ```ignore
|
||||
//! let server = MockImapServer::new()
|
||||
//! .greeting("* OK ready\r\n")
|
||||
//! .respond("LOGIN", "A0 OK logged in\r\n")
|
||||
//! .respond("CAPABILITY", "* CAPABILITY IMAP4rev1\r\nA0 OK done\r\n")
|
||||
//! .respond("STATUS", "* STATUS INBOX (MESSAGES 10 UIDVALIDITY 42)\r\nA0 OK\r\n")
|
||||
//! .respond("LOGOUT", "* BYE\r\nA0 OK\r\n")
|
||||
//! .start()
|
||||
//! .await;
|
||||
//!
|
||||
//! let (host, port) = server.addr();
|
||||
//! // connect to host:port with Encryption::None
|
||||
//! ```
|
||||
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
|
||||
type Response = Vec<u8>;
|
||||
|
||||
pub struct MockImapServer {
|
||||
greeting: Vec<u8>,
|
||||
script: Vec<(String, Response)>,
|
||||
}
|
||||
|
||||
impl MockImapServer {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
greeting: b"* OK Mock IMAP server ready\r\n".to_vec(),
|
||||
script: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the greeting banner sent immediately after connection.
|
||||
pub fn greeting(mut self, banner: impl Into<Vec<u8>>) -> Self {
|
||||
self.greeting = banner.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a script step: when a client command *contains* `pattern` (case-insensitive),
|
||||
/// respond with `response`. Steps are checked in insertion order.
|
||||
pub fn respond(mut self, pattern: impl Into<String>, response: impl Into<Vec<u8>>) -> Self {
|
||||
self.script.push((pattern.into(), response.into()));
|
||||
self
|
||||
}
|
||||
|
||||
/// Start the server on a random port. Returns a handle whose `addr()` gives
|
||||
/// the `(host, port)` to connect to.
|
||||
pub async fn start(self) -> MockImapServerHandle {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
|
||||
let addr = listener.local_addr().expect("local_addr");
|
||||
|
||||
let server = Arc::new(self);
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
match listener.accept().await {
|
||||
Ok((stream, _)) => {
|
||||
let srv = server.clone();
|
||||
tokio::spawn(async move {
|
||||
srv.handle_connection(stream).await;
|
||||
});
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
MockImapServerHandle { addr }
|
||||
}
|
||||
|
||||
async fn handle_connection(&self, mut stream: TcpStream) {
|
||||
let (reader, mut writer) = stream.split();
|
||||
let mut reader = BufReader::new(reader);
|
||||
|
||||
// Send greeting
|
||||
if writer.write_all(&self.greeting).await.is_err() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut line = String::new();
|
||||
loop {
|
||||
line.clear();
|
||||
match reader.read_line(&mut line).await {
|
||||
Ok(0) => break, // EOF
|
||||
Ok(_) => {}
|
||||
Err(_) => break,
|
||||
}
|
||||
|
||||
let tag = extract_tag(&line).unwrap_or("A0");
|
||||
let matched = self.find_match(&line);
|
||||
if let Some(response) = matched {
|
||||
let substituted = substitute_tag(response, tag);
|
||||
if writer.write_all(&substituted).await.is_err() {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
// Default: send tagged OK for commands we don't handle
|
||||
let fallback = format!("{tag} OK done\r\n");
|
||||
if writer.write_all(fallback.as_bytes()).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn find_match(&self, line: &str) -> Option<&[u8]> {
|
||||
let line_lower = line.to_lowercase();
|
||||
for (pattern, response) in &self.script {
|
||||
if line_lower.contains(&pattern.to_lowercase()) {
|
||||
return Some(response);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for MockImapServer {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle to a running mock IMAP server. The server stops when this handle
|
||||
/// is dropped.
|
||||
pub struct MockImapServerHandle {
|
||||
addr: SocketAddr,
|
||||
}
|
||||
|
||||
impl MockImapServerHandle {
|
||||
pub fn host(&self) -> String {
|
||||
self.addr.ip().to_string()
|
||||
}
|
||||
|
||||
pub fn port(&self) -> u16 {
|
||||
self.addr.port()
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_tag(line: &str) -> Option<&str> {
|
||||
line.split_whitespace().next()
|
||||
}
|
||||
|
||||
/// Replace `{TAG}` placeholders in `response` with `tag`.
|
||||
fn substitute_tag(response: &[u8], tag: &str) -> Vec<u8> {
|
||||
let placeholder = b"{TAG}";
|
||||
if response.is_empty() || !contains_slice(response, placeholder) {
|
||||
return response.to_vec();
|
||||
}
|
||||
let tag_bytes = tag.as_bytes();
|
||||
let mut result = Vec::with_capacity(response.len());
|
||||
let mut pos = 0;
|
||||
while let Some(idx) = find_slice(&response[pos..], placeholder) {
|
||||
result.extend_from_slice(&response[pos..pos + idx]);
|
||||
result.extend_from_slice(tag_bytes);
|
||||
pos += idx + placeholder.len();
|
||||
}
|
||||
result.extend_from_slice(&response[pos..]);
|
||||
result
|
||||
}
|
||||
|
||||
fn contains_slice(haystack: &[u8], needle: &[u8]) -> bool {
|
||||
haystack.windows(needle.len()).any(|w| w == needle)
|
||||
}
|
||||
|
||||
fn find_slice(haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
||||
haystack
|
||||
.windows(needle.len())
|
||||
.position(|w| w == needle)
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Pre-built response helpers
|
||||
// ============================================================
|
||||
|
||||
/// Build a tagged OK response.
|
||||
pub fn ok(tag: impl AsRef<str>, msg: impl AsRef<str>) -> Vec<u8> {
|
||||
format!("{} OK {}\r\n", tag.as_ref(), msg.as_ref()).into_bytes()
|
||||
}
|
||||
|
||||
/// Build a STATUS response line.
|
||||
pub fn status_response(
|
||||
mailbox: &str,
|
||||
messages: u32,
|
||||
unseen: u32,
|
||||
uid_next: u32,
|
||||
uid_validity: Option<u32>,
|
||||
) -> Vec<u8> {
|
||||
let uv = uid_validity
|
||||
.map(|v| format!(" UIDVALIDITY {v}"))
|
||||
.unwrap_or_default();
|
||||
let text = format!(
|
||||
"* STATUS \"{mailbox}\" (MESSAGES {messages} UNSEEN {unseen} UIDNEXT {uid_next}{uv})\r\n"
|
||||
);
|
||||
// Clients expect a tagged response after the untagged STATUS line.
|
||||
// We produce a generic OK that works for any tag.
|
||||
let mut out = text.into_bytes();
|
||||
out.extend_from_slice(b"{TAG} OK STATUS completed\r\n");
|
||||
out
|
||||
}
|
||||
|
||||
/// Build an EXAMINE response with mailbox data.
|
||||
pub fn examine_response(
|
||||
_mailbox: &str,
|
||||
exists: u32,
|
||||
uid_validity: u32,
|
||||
uid_next: u32,
|
||||
) -> Vec<u8> {
|
||||
format!(
|
||||
"* FLAGS (\\Seen \\Answered \\Flagged \\Deleted \\Draft)\r\n\
|
||||
* OK [PERMANENTFLAGS ()]\r\n\
|
||||
* {exists} EXISTS\r\n\
|
||||
* 0 RECENT\r\n\
|
||||
* OK [UIDVALIDITY {uid_validity}]\r\n\
|
||||
* OK [UIDNEXT {uid_next}]\r\n\
|
||||
* OK [HIGHESTMODSEQ 1]\r\n\
|
||||
{{TAG}} OK [READ-ONLY] EXAMINE completed\r\n"
|
||||
)
|
||||
.into_bytes()
|
||||
}
|
||||
|
||||
/// Build a UID SEARCH response for the given UID list.
|
||||
pub fn uid_search_response(uids: &[u32]) -> Vec<u8> {
|
||||
let uid_str = uids
|
||||
.iter()
|
||||
.map(|u| u.to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
format!("* SEARCH {uid_str}\r\n{{TAG}} OK SEARCH completed\r\n").into_bytes()
|
||||
}
|
||||
|
||||
/// Build a UID FETCH response returning full headers (for BODY[HEADER]).
|
||||
/// Each entry: (uid, message_id)
|
||||
pub fn uid_fetch_metadata_response(entries: &[(u32, &str)]) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
for (uid, msg_id) in entries {
|
||||
// Build a minimal header that contains the Message-ID line.
|
||||
let header_data = format!(
|
||||
"From: sender@example.com\r\n\
|
||||
To: recipient@example.com\r\n\
|
||||
Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\
|
||||
Subject: test\r\n\
|
||||
Message-ID: {msg_id}\r\n\r\n"
|
||||
);
|
||||
let header_len = header_data.len();
|
||||
let line = format!(
|
||||
"* {uid} FETCH (UID {uid} BODY[HEADER] {{{header_len}}}\r\n\
|
||||
{header_data}\
|
||||
)\r\n",
|
||||
);
|
||||
out.extend_from_slice(line.as_bytes());
|
||||
}
|
||||
out.extend_from_slice(b"{TAG} OK FETCH completed\r\n");
|
||||
out
|
||||
}
|
||||
|
||||
/// Build a UID FETCH RFC822 response with a full email body.
|
||||
pub fn uid_fetch_rfc822_response(uid: u32, eml: &[u8]) -> Vec<u8> {
|
||||
let header = format!(
|
||||
"* {uid} FETCH (UID {uid} RFC822 {{{len}}}\r\n",
|
||||
len = eml.len()
|
||||
);
|
||||
let mut out = header.into_bytes();
|
||||
out.extend_from_slice(eml);
|
||||
out.extend_from_slice(b")\r\n{TAG} OK FETCH completed\r\n");
|
||||
out
|
||||
}
|
||||
|
||||
/// A minimal RFC822 email fixture for testing.
|
||||
pub fn minimal_eml(subject: &str, message_id: &str) -> Vec<u8> {
|
||||
format!(
|
||||
"From: sender@example.com\r\n\
|
||||
To: recipient@example.com\r\n\
|
||||
Subject: {subject}\r\n\
|
||||
Message-ID: <{message_id}>\r\n\
|
||||
Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\
|
||||
MIME-Version: 1.0\r\n\
|
||||
Content-Type: text/plain; charset=utf-8\r\n\
|
||||
\r\n\
|
||||
This is a test email: {subject}.\r\n"
|
||||
)
|
||||
.into_bytes()
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Self-tests for the mock server itself
|
||||
// ============================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
async fn connect_and_read_greeting(host: &str, port: u16) -> String {
|
||||
let mut stream = TcpStream::connect((host, port)).await.unwrap();
|
||||
let (reader, _writer) = stream.split();
|
||||
let mut reader = BufReader::new(reader);
|
||||
let mut line = String::new();
|
||||
reader.read_line(&mut line).await.unwrap();
|
||||
line
|
||||
}
|
||||
|
||||
async fn send_and_recv(host: &str, port: u16, cmd: &str) -> String {
|
||||
let mut stream = TcpStream::connect((host, port)).await.unwrap();
|
||||
let (reader, mut writer) = stream.split();
|
||||
let mut reader = BufReader::new(reader);
|
||||
|
||||
// Read greeting
|
||||
let mut line = String::new();
|
||||
reader.read_line(&mut line).await.unwrap();
|
||||
|
||||
// Send command
|
||||
writer.write_all(cmd.as_bytes()).await.unwrap();
|
||||
writer.write_all(b"\r\n").await.unwrap();
|
||||
|
||||
// Read response (may be multi-line; read until tagged response)
|
||||
let mut out = String::new();
|
||||
loop {
|
||||
line.clear();
|
||||
reader.read_line(&mut line).await.unwrap();
|
||||
out.push_str(&line);
|
||||
if line.starts_with("A0") || line.starts_with("A1") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mock_greeting() {
|
||||
let handle = MockImapServer::new().start().await;
|
||||
let greeting = connect_and_read_greeting(&handle.host(), handle.port()).await;
|
||||
assert!(greeting.starts_with("* OK"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mock_scripted_response() {
|
||||
let handle = MockImapServer::new()
|
||||
.respond(
|
||||
"LOGIN",
|
||||
"A0 OK LOGIN completed\r\n",
|
||||
)
|
||||
.start()
|
||||
.await;
|
||||
|
||||
let resp = send_and_recv(&handle.host(), handle.port(), "A0 LOGIN u p").await;
|
||||
assert!(resp.contains("LOGIN completed"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mock_fallback_on_unmatched() {
|
||||
let handle = MockImapServer::new().start().await;
|
||||
|
||||
// Send a command that has no scripted response
|
||||
let resp = send_and_recv(&handle.host(), handle.port(), "A0 NOOP").await;
|
||||
assert!(resp.contains("OK done"), "unmatched command should get fallback OK");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_status_response_helper() {
|
||||
let resp = status_response("INBOX", 10, 2, 11, Some(42));
|
||||
let text = String::from_utf8(resp).unwrap();
|
||||
assert!(text.contains("MESSAGES 10"));
|
||||
assert!(text.contains("UNSEEN 2"));
|
||||
assert!(text.contains("UIDNEXT 11"));
|
||||
assert!(text.contains("UIDVALIDITY 42"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_status_response_without_uidvalidity() {
|
||||
let resp = status_response("INBOX", 10, 2, 11, None);
|
||||
let text = String::from_utf8(resp).unwrap();
|
||||
assert!(!text.contains("UIDVALIDITY"));
|
||||
assert!(text.contains("MESSAGES 10"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_examine_response() {
|
||||
let resp = examine_response("INBOX", 10, 42, 11);
|
||||
let text = String::from_utf8(resp).unwrap();
|
||||
assert!(text.contains("UIDVALIDITY 42"));
|
||||
assert!(text.contains("10 EXISTS"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_uid_search_response() {
|
||||
let resp = uid_search_response(&[1, 3, 5]);
|
||||
let text = String::from_utf8(resp).unwrap();
|
||||
assert!(text.contains("SEARCH 1 3 5"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_uid_fetch_metadata_response() {
|
||||
let resp = uid_fetch_metadata_response(&[(1, "msg-a@x.com"), (2, "msg-b@x.com")]);
|
||||
let text = String::from_utf8(resp).unwrap();
|
||||
assert!(text.contains("Message-ID: msg-a@x.com"));
|
||||
assert!(text.contains("Message-ID: msg-b@x.com"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_multiple_commands_in_sequence() {
|
||||
let handle = MockImapServer::new()
|
||||
.respond("LOGIN", "A0 OK LOGIN\r\n")
|
||||
.respond("STATUS", status_response("INBOX", 5, 1, 6, Some(99)))
|
||||
.respond("LOGOUT", "* BYE\r\nA0 OK\r\n")
|
||||
.start()
|
||||
.await;
|
||||
|
||||
let mut stream = TcpStream::connect((handle.host(), handle.port()))
|
||||
.await
|
||||
.unwrap();
|
||||
let (reader, mut writer) = stream.split();
|
||||
let mut reader = BufReader::new(reader);
|
||||
|
||||
// Read greeting
|
||||
let mut buf = String::new();
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
|
||||
// LOGIN
|
||||
writer.write_all(b"A0 LOGIN u p\r\n").await.unwrap();
|
||||
buf.clear();
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
assert!(buf.contains("LOGIN"));
|
||||
|
||||
// STATUS
|
||||
writer
|
||||
.write_all(b"A0 STATUS INBOX (MESSAGES UNSEEN UIDNEXT UIDVALIDITY)\r\n")
|
||||
.await
|
||||
.unwrap();
|
||||
buf.clear();
|
||||
// Read multi-line STATUS response (untagged line + tagged OK)
|
||||
loop {
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
if buf.contains("UIDVALIDITY 99") {
|
||||
// Consume the tagged OK line that follows
|
||||
buf.clear();
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// LOGOUT
|
||||
writer.write_all(b"A0 LOGOUT\r\n").await.unwrap();
|
||||
buf.clear();
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
assert!(buf.contains("BYE"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tag_substitution_in_response() {
|
||||
// Use {TAG} placeholder in the response and verify it gets the
|
||||
// client's actual tag ("A5") substituted in.
|
||||
let handle = MockImapServer::new()
|
||||
.respond("LOGIN", "{TAG} OK LOGIN succeeded\r\n")
|
||||
.start()
|
||||
.await;
|
||||
|
||||
let mut stream = TcpStream::connect((handle.host(), handle.port()))
|
||||
.await
|
||||
.unwrap();
|
||||
let (reader, mut writer) = stream.split();
|
||||
let mut reader = BufReader::new(reader);
|
||||
|
||||
// Read greeting
|
||||
let mut buf = String::new();
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
|
||||
// Send LOGIN with non-standard tag
|
||||
writer.write_all(b"A5 LOGIN u p\r\n").await.unwrap();
|
||||
buf.clear();
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
|
||||
assert!(
|
||||
buf.contains("A5 OK LOGIN succeeded"),
|
||||
"expected 'A5 OK LOGIN succeeded', got '{buf}'"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tag_substitution_multiple_placeholders() {
|
||||
let handle = MockImapServer::new()
|
||||
.respond("NOOP", "* 0 RECENT\r\n{TAG} OK NOOP done\r\n")
|
||||
.start()
|
||||
.await;
|
||||
|
||||
let mut stream = TcpStream::connect((handle.host(), handle.port()))
|
||||
.await
|
||||
.unwrap();
|
||||
let (reader, mut writer) = stream.split();
|
||||
let mut reader = BufReader::new(reader);
|
||||
|
||||
// Read greeting
|
||||
let mut buf = String::new();
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
|
||||
// Send with tag "B99"
|
||||
writer.write_all(b"B99 NOOP\r\n").await.unwrap();
|
||||
|
||||
// Read all lines
|
||||
let mut all = String::new();
|
||||
loop {
|
||||
buf.clear();
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
all.push_str(&buf);
|
||||
if buf.starts_with("B99") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert!(all.contains("* 0 RECENT\r\n"));
|
||||
assert!(all.contains("B99 OK NOOP done\r\n"));
|
||||
}
|
||||
}
|
||||
@@ -26,3 +26,5 @@ pub mod session;
|
||||
pub mod stats;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
#[cfg(test)]
|
||||
pub mod mock_server;
|
||||
|
||||
127
crates/core/src/import/history.rs
Normal file
127
crates/core/src/import/history.rs
Normal file
@@ -0,0 +1,127 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
|
||||
use crate::database::MemDbModel;
|
||||
use crate::import::{ImportProgress, ImportStatus};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Maximum number of import history entries to keep per user.
|
||||
pub const MAX_HISTORY_PER_USER: usize = 5;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct ImportHistory {
|
||||
/// Composite key: "{user_id}:{import_id}"
|
||||
pub id: String,
|
||||
pub user_id: u64,
|
||||
pub import_id: String,
|
||||
pub account_id: u64,
|
||||
pub folder: String,
|
||||
pub format: String,
|
||||
pub status: String,
|
||||
pub total: usize,
|
||||
pub success: usize,
|
||||
pub duplicates: usize,
|
||||
pub failed: usize,
|
||||
pub failed_details: Vec<crate::import::FailedItemDetail>,
|
||||
/// Unix timestamp in milliseconds.
|
||||
pub created_at: i64,
|
||||
}
|
||||
|
||||
impl MemDbModel for ImportHistory {
|
||||
fn collection() -> &'static str {
|
||||
"import_history"
|
||||
}
|
||||
fn key(&self) -> String {
|
||||
self.id.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl ImportHistory {
|
||||
pub fn from_progress(
|
||||
user_id: u64,
|
||||
import_id: &str,
|
||||
account_id: u64,
|
||||
folder: &str,
|
||||
progress: &ImportProgress,
|
||||
) -> Self {
|
||||
Self {
|
||||
id: format!("{}:{}", user_id, import_id),
|
||||
user_id,
|
||||
import_id: import_id.to_string(),
|
||||
account_id,
|
||||
folder: folder.to_string(),
|
||||
format: progress.format.clone(),
|
||||
status: match progress.status {
|
||||
ImportStatus::Pending => "pending",
|
||||
ImportStatus::Processing => "processing",
|
||||
ImportStatus::Completed => "completed",
|
||||
ImportStatus::Failed => "failed",
|
||||
}
|
||||
.to_string(),
|
||||
total: progress.total,
|
||||
success: progress.success,
|
||||
duplicates: progress.duplicates,
|
||||
failed: progress.failed,
|
||||
failed_details: progress.failed_details.clone(),
|
||||
created_at: crate::utc_now!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Prune old entries for a user so only the latest `MAX_HISTORY_PER_USER` remain.
|
||||
pub fn prune_user_history(user_id: u64) -> crate::error::BichonResult<()> {
|
||||
use crate::database::manager::DB_MANAGER;
|
||||
use crate::database::batch_delete_impl;
|
||||
use crate::raise_error;
|
||||
use crate::error::code::ErrorCode;
|
||||
let db = DB_MANAGER.db();
|
||||
let coll = db.collection(ImportHistory::collection());
|
||||
let prefix = format!("{}:", user_id);
|
||||
|
||||
let mut entries: Vec<ImportHistory> = coll
|
||||
.scan_prefix(&prefix)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
if entries.len() <= MAX_HISTORY_PER_USER {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Sort by created_at descending (newest first), keep the first N
|
||||
entries.sort_by(|a, b| b.created_at.cmp(&a.created_at));
|
||||
let to_delete: Vec<String> = entries
|
||||
.iter()
|
||||
.skip(MAX_HISTORY_PER_USER)
|
||||
.map(|e| e.id.clone())
|
||||
.collect();
|
||||
|
||||
if !to_delete.is_empty() {
|
||||
batch_delete_impl::<ImportHistory>(db, to_delete)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Save an import history record and prune old entries for the user.
|
||||
pub fn save_import_history(
|
||||
user_id: u64,
|
||||
account_id: u64,
|
||||
folder: &str,
|
||||
progress: &ImportProgress,
|
||||
) {
|
||||
use crate::database::manager::DB_MANAGER;
|
||||
use crate::database::upsert_impl;
|
||||
|
||||
let entry = ImportHistory::from_progress(user_id, &progress.import_id, account_id, folder, progress);
|
||||
let db = DB_MANAGER.db();
|
||||
|
||||
if let Err(e) = upsert_impl::<ImportHistory>(db, entry) {
|
||||
tracing::error!("Failed to save import history: {:?}", e);
|
||||
return;
|
||||
}
|
||||
|
||||
if let Err(e) = prune_user_history(user_id) {
|
||||
tracing::warn!("Failed to prune import history: {:?}", e);
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,16 @@
|
||||
|
||||
|
||||
//use poem_openapi::Object;
|
||||
pub mod history;
|
||||
pub mod reader;
|
||||
pub mod pst;
|
||||
pub use history::ImportHistory;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::Path,
|
||||
sync::RwLock,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
base64_decode_url_safe,
|
||||
@@ -27,15 +36,20 @@ use crate::{
|
||||
cache::imap::mailbox::{Attribute, AttributeEnum, MailBox},
|
||||
envelope::extractor::extract_envelope_from_eml,
|
||||
error::{BichonResult, code::ErrorCode},
|
||||
settings::dir::DATA_DIR_MANAGER,
|
||||
utils::create_hash,
|
||||
},
|
||||
raise_error,
|
||||
};
|
||||
|
||||
/// Skip individual emails larger than this after decoding (100 MB).
|
||||
/// Maximum byte size of an individual email message after splitting (100 MB).
|
||||
const MAX_SINGLE_EML_BYTES: usize = 100 * 1024 * 1024;
|
||||
|
||||
#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||
/// Max file size accepted via the web upload endpoint.
|
||||
pub const MAX_WEB_EML_BYTES: usize = 100 * 1024 * 1024; // 100 MB
|
||||
pub const MAX_WEB_MBOX_BYTES: usize = 1024 * 1024 * 1024; // 1 GB
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct BatchEmlRequest {
|
||||
pub account_id: u64,
|
||||
@@ -46,24 +60,26 @@ pub struct BatchEmlRequest {
|
||||
|
||||
#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct FailedEmlDetail {
|
||||
/// The 0-based index of the failed EML in the request list
|
||||
pub struct FailedItemDetail {
|
||||
/// The index (0-based) of the failed item.
|
||||
pub index: usize,
|
||||
/// The error message that caused the import to fail
|
||||
/// The error message that caused the import to fail.
|
||||
pub error_message: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct BatchEmlResult {
|
||||
/// Total number of emails processed
|
||||
/// Total number of emails processed.
|
||||
pub total: usize,
|
||||
/// Number of emails successfully imported
|
||||
/// Number of emails successfully imported.
|
||||
pub success: usize,
|
||||
/// Number of emails failed to import
|
||||
/// Number of duplicate emails skipped (content hash already existed).
|
||||
pub duplicates: usize,
|
||||
/// Number of emails failed to import.
|
||||
pub failed: usize,
|
||||
/// A list of details for failed imports
|
||||
pub failed_details: Vec<FailedEmlDetail>,
|
||||
/// A list of details for failed imports.
|
||||
pub failed_details: Vec<FailedItemDetail>,
|
||||
}
|
||||
|
||||
pub struct ImportEmls;
|
||||
@@ -116,7 +132,7 @@ impl ImportEmls {
|
||||
|
||||
let account_id = account.id;
|
||||
let mut success_count = 0;
|
||||
let mut failed_details: Vec<FailedEmlDetail> = Vec::new(); // Store failure details
|
||||
let mut failed_details: Vec<FailedItemDetail> = Vec::new(); // Store failure details
|
||||
|
||||
let total = request.emls.len();
|
||||
let mut index: usize = 0;
|
||||
@@ -127,7 +143,7 @@ impl ImportEmls {
|
||||
let error_msg =
|
||||
format!("Failed to decode base64 EML at index {}: {:?}", index, e);
|
||||
tracing::error!("{}", error_msg);
|
||||
failed_details.push(FailedEmlDetail {
|
||||
failed_details.push(FailedItemDetail {
|
||||
index,
|
||||
error_message: error_msg,
|
||||
});
|
||||
@@ -144,7 +160,7 @@ impl ImportEmls {
|
||||
index, size_mb,
|
||||
);
|
||||
tracing::warn!("{}", error_msg);
|
||||
failed_details.push(FailedEmlDetail {
|
||||
failed_details.push(FailedItemDetail {
|
||||
index,
|
||||
error_message: error_msg,
|
||||
});
|
||||
@@ -162,7 +178,7 @@ impl ImportEmls {
|
||||
index, e
|
||||
);
|
||||
tracing::error!("{}", error_msg);
|
||||
failed_details.push(FailedEmlDetail {
|
||||
failed_details.push(FailedItemDetail {
|
||||
index,
|
||||
error_message: error_msg,
|
||||
});
|
||||
@@ -178,8 +194,650 @@ impl ImportEmls {
|
||||
Ok(BatchEmlResult {
|
||||
total,
|
||||
success: success_count,
|
||||
duplicates: 0,
|
||||
failed: failed_count,
|
||||
failed_details, // Return the list of failure details
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── File upload import ──────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))]
|
||||
pub enum ImportStatus {
|
||||
Pending,
|
||||
Processing,
|
||||
Completed,
|
||||
Failed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct ImportProgress {
|
||||
pub import_id: String,
|
||||
pub status: ImportStatus,
|
||||
pub format: String,
|
||||
pub total: usize,
|
||||
pub success: usize,
|
||||
pub duplicates: usize,
|
||||
pub failed: usize,
|
||||
pub failed_details: Vec<FailedItemDetail>,
|
||||
}
|
||||
|
||||
static PROGRESS_STORE: std::sync::LazyLock<RwLock<HashMap<String, ImportProgress>>> =
|
||||
std::sync::LazyLock::new(|| RwLock::new(HashMap::new()));
|
||||
|
||||
pub fn get_import_progress(import_id: &str) -> Option<ImportProgress> {
|
||||
PROGRESS_STORE.read().ok()?.get(import_id).cloned()
|
||||
}
|
||||
|
||||
pub fn update_progress(import_id: &str, progress: ImportProgress) {
|
||||
if let Ok(mut store) = PROGRESS_STORE.write() {
|
||||
store.insert(import_id.to_string(), progress);
|
||||
}
|
||||
}
|
||||
|
||||
/// Check free disk space (in bytes) on the temp directory's filesystem.
|
||||
pub fn check_temp_disk_space() -> BichonResult<u64> {
|
||||
use sysinfo::Disks;
|
||||
let disks = Disks::new_with_refreshed_list();
|
||||
let temp_path = &DATA_DIR_MANAGER.temp_dir;
|
||||
// Use the canonical path so we can match mount points
|
||||
let canonical = std::fs::canonicalize(temp_path).unwrap_or_else(|_| temp_path.clone());
|
||||
for disk in disks.list() {
|
||||
if canonical.starts_with(disk.mount_point()) {
|
||||
return Ok(disk.available_space());
|
||||
}
|
||||
}
|
||||
// Fallback: if we can't find the mount point, report plenty of space
|
||||
Ok(u64::MAX)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum FileFormat {
|
||||
Eml,
|
||||
Mbox,
|
||||
Pst,
|
||||
}
|
||||
|
||||
pub fn detect_format(bytes: &[u8], file_name: &str) -> Option<FileFormat> {
|
||||
// PST files start with OLE2 compound document magic bytes
|
||||
if bytes.len() >= 8 && &bytes[..8] == b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1" {
|
||||
return Some(FileFormat::Pst);
|
||||
}
|
||||
|
||||
// MBOX files start with "From " (note the trailing space after From)
|
||||
if bytes.starts_with(b"From ") {
|
||||
// Double-check: look for a valid date after the first "From " line
|
||||
// MBOX format: "From sender@host DayOfWeek Mon DD HH:MM:SS YYYY"
|
||||
if let Some(first_newline) = bytes.iter().position(|&b| b == b'\n') {
|
||||
let from_line = std::str::from_utf8(&bytes[..first_newline]).unwrap_or("");
|
||||
let parts: Vec<&str> = from_line.split_whitespace().collect();
|
||||
if parts.len() >= 7 {
|
||||
return Some(FileFormat::Mbox);
|
||||
}
|
||||
}
|
||||
}
|
||||
// EML: starts with a header line or "Return-Path:", "Received:", "From:", "Date:", etc.
|
||||
// Or check extension
|
||||
if bytes.starts_with(b"Return-Path:")
|
||||
|| bytes.starts_with(b"Received:")
|
||||
|| bytes.starts_with(b"Date:")
|
||||
|| bytes.starts_with(b"From:")
|
||||
|| bytes.starts_with(b"Subject:")
|
||||
|| bytes.starts_with(b"To:")
|
||||
|| bytes.starts_with(b"Message-ID:")
|
||||
{
|
||||
return Some(FileFormat::Eml);
|
||||
}
|
||||
// Fallback: check file extension
|
||||
let lower = file_name.to_lowercase();
|
||||
if lower.ends_with(".eml") {
|
||||
Some(FileFormat::Eml)
|
||||
} else if lower.ends_with(".mbox") {
|
||||
Some(FileFormat::Mbox)
|
||||
} else if lower.ends_with(".pst") {
|
||||
Some(FileFormat::Pst)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Check whether `bytes` looks like a text file by inspecting the first chunk.
|
||||
/// Returns `true` if it passes, `false` if it appears to be binary (video, executable, etc.).
|
||||
///
|
||||
/// Email files (EML/MBOX) are text-based with printable ASCII, whitespace, and
|
||||
/// optional UTF-8. Binary files like video contain null bytes and high ratios of
|
||||
/// non-printable control characters.
|
||||
pub fn detect_text_file(bytes: &[u8]) -> bool {
|
||||
let check_len = bytes.len().min(8192);
|
||||
if check_len == 0 {
|
||||
return false;
|
||||
}
|
||||
let sample = &bytes[..check_len];
|
||||
|
||||
// Null bytes are a strong binary indicator
|
||||
if sample.contains(&0x00) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut printable = 0usize;
|
||||
let mut total = 0usize;
|
||||
|
||||
let mut i = 0;
|
||||
while i < sample.len() {
|
||||
total += 1;
|
||||
let b = sample[i];
|
||||
|
||||
if b.is_ascii_graphic() || b.is_ascii_whitespace() {
|
||||
// Printable ASCII + whitespace (space, tab, CR, LF)
|
||||
printable += 1;
|
||||
} else if b == 0x1b {
|
||||
// ESC — common in terminal sequences, rare in email
|
||||
// Count as printable to avoid false positives
|
||||
printable += 1;
|
||||
} else if b >= 0x80 {
|
||||
// UTF-8 continuation or multi-byte lead byte — allow.
|
||||
// Check that we have a valid UTF-8 sequence ahead.
|
||||
let seq_len = match b {
|
||||
b if b & 0xE0 == 0xC0 => 2,
|
||||
b if b & 0xF0 == 0xE0 => 3,
|
||||
b if b & 0xF8 == 0xF0 => 4,
|
||||
_ => 0,
|
||||
};
|
||||
if seq_len > 0 && i + seq_len <= sample.len() {
|
||||
let valid = std::str::from_utf8(&sample[i..i + seq_len]).is_ok();
|
||||
if valid {
|
||||
printable += 1;
|
||||
i += 1; // lead byte counted, continuations counted in loop
|
||||
}
|
||||
// if invalid, don't count as printable
|
||||
}
|
||||
// standalone continuation byte — not printable
|
||||
}
|
||||
// Other control characters (0x01-0x1F except whitespace/Esc) are not counted as printable
|
||||
|
||||
i += 1;
|
||||
}
|
||||
|
||||
// Require at least 90% printable characters
|
||||
printable as f64 / total as f64 >= 0.90
|
||||
}
|
||||
|
||||
/// Validate that the target account exists, is enabled, and is NoSync type.
|
||||
fn validate_import_account(account_id: u64) -> BichonResult<AccountModel> {
|
||||
let account = AccountModel::check_account_exists(account_id)?;
|
||||
if !account.enabled {
|
||||
return Err(raise_error!(
|
||||
"The account is disabled.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
if !matches!(account.account_type, AccountType::NoSync) {
|
||||
return Err(raise_error!(
|
||||
"Import is only allowed for NoSync accounts. IMAP accounts sync from the server.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
Ok(account)
|
||||
}
|
||||
|
||||
/// Resolve or create a mailbox/folder for the given account.
|
||||
pub(super) fn resolve_mailbox(account: &AccountModel, folder: &str) -> BichonResult<u64> {
|
||||
match account.account_type {
|
||||
AccountType::IMAP => {
|
||||
// Shouldn't reach here (validated above), but handle gracefully
|
||||
let all_mailboxes = MailBox::list_all(account.id)?;
|
||||
let mailbox = all_mailboxes.into_iter().find(|m| m.name == folder);
|
||||
match mailbox {
|
||||
Some(m) => Ok(m.id),
|
||||
None => Err(raise_error!(
|
||||
format!("Mail folder '{}' not found.", folder).into(),
|
||||
ErrorCode::ResourceNotFound
|
||||
)),
|
||||
}
|
||||
}
|
||||
AccountType::NoSync => {
|
||||
let mailbox = MailBox {
|
||||
id: create_hash(account.id, folder),
|
||||
account_id: account.id,
|
||||
name: folder.to_string(),
|
||||
delimiter: Some("/".to_string()),
|
||||
attributes: vec![Attribute {
|
||||
attr: AttributeEnum::Extension,
|
||||
extension: Some("CreatedByBichon".into()),
|
||||
}],
|
||||
exists: 0,
|
||||
unseen: None,
|
||||
uid_next: None,
|
||||
uid_validity: None,
|
||||
highest_uid: None,
|
||||
};
|
||||
let mailbox_id = mailbox.id;
|
||||
MailBox::batch_upsert(&[mailbox])?;
|
||||
Ok(mailbox_id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve or create a mailbox for a given account_id and folder name.
|
||||
/// Used by PST import to create per-folder mailboxes.
|
||||
pub fn resolve_mailbox_by_account_id(account_id: u64, folder: &str) -> BichonResult<u64> {
|
||||
let account = AccountModel::check_account_exists(account_id)?;
|
||||
resolve_mailbox(&account, folder)
|
||||
}
|
||||
|
||||
/// Process an uploaded file (EML or MBOX) and import into the given account/folder.
|
||||
/// This runs synchronously and should be spawned on a background thread.
|
||||
///
|
||||
/// For MBOX files, the file is memory-mapped via `memmap2` and messages are yielded
|
||||
/// one at a time — the full file is never loaded into RAM. Individual messages
|
||||
/// exceeding `MAX_SINGLE_EML_BYTES` (100 MB) are skipped.
|
||||
pub fn process_uploaded_file(
|
||||
import_id: &str,
|
||||
file_path: &Path,
|
||||
file_name: &str,
|
||||
account_id: u64,
|
||||
folder: &str,
|
||||
user_id: u64,
|
||||
) {
|
||||
let account = match validate_import_account(account_id) {
|
||||
Ok(a) => a,
|
||||
Err(e) => {
|
||||
let progress = ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Failed,
|
||||
format: "unknown".to_string(),
|
||||
total: 0,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![FailedItemDetail {
|
||||
index: 0,
|
||||
error_message: format!("Account validation failed: {:?}", e),
|
||||
}],
|
||||
};
|
||||
update_progress(import_id, progress.clone());
|
||||
history::save_import_history(user_id, account_id, folder, &progress);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mailbox_id = match resolve_mailbox(&account, folder) {
|
||||
Ok(id) => id,
|
||||
Err(e) => {
|
||||
let progress = ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Failed,
|
||||
format: "unknown".to_string(),
|
||||
total: 0,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![FailedItemDetail {
|
||||
index: 0,
|
||||
error_message: format!("Mailbox resolution failed: {:?}", e),
|
||||
}],
|
||||
};
|
||||
update_progress(import_id, progress.clone());
|
||||
history::save_import_history(user_id, account_id, folder, &progress);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Read a small prefix for format detection
|
||||
let format = match detect_format_from_file(file_path, file_name) {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
let progress = ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Failed,
|
||||
format: "unknown".to_string(),
|
||||
total: 0,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![FailedItemDetail {
|
||||
index: 0,
|
||||
error_message: format!("{:?}", e),
|
||||
}],
|
||||
};
|
||||
update_progress(import_id, progress.clone());
|
||||
history::save_import_history(user_id, account_id, folder, &progress);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
match format {
|
||||
FileFormat::Eml => process_eml_file(import_id, file_path, account_id, mailbox_id, user_id, folder),
|
||||
FileFormat::Mbox => process_mbox_file(import_id, file_path, account_id, mailbox_id, user_id, folder),
|
||||
FileFormat::Pst => process_pst_upload(import_id, file_path, account_id, mailbox_id, user_id, folder),
|
||||
}
|
||||
}
|
||||
|
||||
/// Detect format from a file by reading only the first few KB.
|
||||
fn detect_format_from_file(file_path: &Path, file_name: &str) -> BichonResult<FileFormat> {
|
||||
use std::io::Read;
|
||||
let mut file = std::fs::File::open(file_path).map_err(|e| {
|
||||
raise_error!(format!("Failed to open file: {}", e), ErrorCode::InternalError)
|
||||
})?;
|
||||
let mut buf = vec![0u8; 8192];
|
||||
let n = file.read(&mut buf).unwrap_or(0);
|
||||
buf.truncate(n);
|
||||
|
||||
detect_format(&buf, file_name).ok_or_else(|| {
|
||||
raise_error!(
|
||||
"Unknown file format. Supported: .eml, .mbox, .pst".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Process a single EML file. The file is at most `MAX_WEB_EML_BYTES` (100 MB),
|
||||
/// so reading it entirely is safe.
|
||||
fn process_eml_file(
|
||||
import_id: &str,
|
||||
file_path: &Path,
|
||||
account_id: u64,
|
||||
mailbox_id: u64,
|
||||
user_id: u64,
|
||||
folder: &str,
|
||||
) {
|
||||
let file_bytes = match std::fs::read(file_path) {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
fail_progress(import_id, "eml", &format!("Failed to read file: {}", e), user_id, account_id, folder);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let total = 1;
|
||||
update_progress(import_id, ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Processing,
|
||||
format: "eml".to_string(),
|
||||
total,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![],
|
||||
});
|
||||
|
||||
let (success_count, failed_details) = process_single_eml(&file_bytes, 0, account_id, mailbox_id);
|
||||
|
||||
// Clean up
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
|
||||
let final_progress = ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Completed,
|
||||
format: "eml".to_string(),
|
||||
total,
|
||||
success: success_count,
|
||||
duplicates: 0,
|
||||
failed: failed_details.len(),
|
||||
failed_details,
|
||||
};
|
||||
history::save_import_history(user_id, account_id, folder, &final_progress);
|
||||
update_progress(import_id, final_progress);
|
||||
}
|
||||
|
||||
/// Process an MBOX file using memory-mapped I/O. Messages are yielded one at a
|
||||
/// time by `MboxReader` — the full file is never loaded into RAM.
|
||||
fn process_mbox_file(
|
||||
import_id: &str,
|
||||
file_path: &Path,
|
||||
account_id: u64,
|
||||
mailbox_id: u64,
|
||||
user_id: u64,
|
||||
folder: &str,
|
||||
) {
|
||||
let mbox = match reader::MboxFile::from_file(file_path) {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
fail_progress(import_id, "mbox", &format!("Failed to open MBOX file: {}", e), user_id, account_id, folder);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// First pass: count total messages (MboxReader is lazy, so this is O(n) but cheap)
|
||||
let total = mbox.iter().count();
|
||||
|
||||
update_progress(import_id, ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Processing,
|
||||
format: "mbox".to_string(),
|
||||
total,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![],
|
||||
});
|
||||
|
||||
let mut success_count = 0usize;
|
||||
let mut failed_details: Vec<FailedItemDetail> = Vec::new();
|
||||
|
||||
for (index, entry) in mbox.iter().enumerate() {
|
||||
let eml_bytes = entry.data;
|
||||
|
||||
if eml_bytes.len() > MAX_SINGLE_EML_BYTES {
|
||||
let size_mb = eml_bytes.len() as f64 / 1024.0 / 1024.0;
|
||||
failed_details.push(FailedItemDetail {
|
||||
index,
|
||||
error_message: format!(
|
||||
"Email at index {} is {:.1} MB (limit {} MB). Skipping.",
|
||||
index,
|
||||
size_mb,
|
||||
MAX_SINGLE_EML_BYTES / 1024 / 1024
|
||||
),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
match futures::executor::block_on(extract_envelope_from_eml(eml_bytes, account_id, mailbox_id)) {
|
||||
Ok(_) => {
|
||||
success_count += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
failed_details.push(FailedItemDetail {
|
||||
index,
|
||||
error_message: format!("{:?}", e),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
// Update progress every 100 items
|
||||
if index % 100 == 0 || index == total - 1 {
|
||||
update_progress(import_id, ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Processing,
|
||||
format: "mbox".to_string(),
|
||||
total,
|
||||
success: success_count,
|
||||
duplicates: 0,
|
||||
failed: failed_details.len(),
|
||||
failed_details: failed_details.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up temp file (drop the mmap first — MboxFile owns it)
|
||||
drop(mbox);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
|
||||
let final_progress = ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Completed,
|
||||
format: "mbox".to_string(),
|
||||
total,
|
||||
success: success_count,
|
||||
duplicates: 0,
|
||||
failed: failed_details.len(),
|
||||
failed_details,
|
||||
};
|
||||
history::save_import_history(user_id, account_id, folder, &final_progress);
|
||||
update_progress(import_id, final_progress);
|
||||
}
|
||||
|
||||
/// Process a single EML byte slice and return (success_count, failed_details).
|
||||
fn process_single_eml(
|
||||
eml_bytes: &[u8],
|
||||
index: usize,
|
||||
account_id: u64,
|
||||
mailbox_id: u64,
|
||||
) -> (usize, Vec<FailedItemDetail>) {
|
||||
if eml_bytes.len() > MAX_SINGLE_EML_BYTES {
|
||||
let size_mb = eml_bytes.len() as f64 / 1024.0 / 1024.0;
|
||||
return (0, vec![FailedItemDetail {
|
||||
index,
|
||||
error_message: format!(
|
||||
"Email is {:.1} MB (limit {} MB). Skipping.",
|
||||
size_mb,
|
||||
MAX_SINGLE_EML_BYTES / 1024 / 1024
|
||||
),
|
||||
}]);
|
||||
}
|
||||
|
||||
match futures::executor::block_on(extract_envelope_from_eml(eml_bytes, account_id, mailbox_id)) {
|
||||
Ok(_) => (1, vec![]),
|
||||
Err(e) => (0, vec![FailedItemDetail {
|
||||
index,
|
||||
error_message: format!("{:?}", e),
|
||||
}]),
|
||||
}
|
||||
}
|
||||
|
||||
/// Process a PST file uploaded via the web UI.
|
||||
/// Two-pass approach: count messages first, then process with periodic progress updates.
|
||||
fn process_pst_upload(
|
||||
import_id: &str,
|
||||
file_path: &Path,
|
||||
account_id: u64,
|
||||
_mailbox_id: u64, // ignored; PST creates its own mailboxes per folder
|
||||
user_id: u64,
|
||||
folder: &str,
|
||||
) {
|
||||
// Pass 1: count total messages
|
||||
let total = match pst::count_pst_messages(file_path) {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
update_progress(import_id, ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Processing,
|
||||
format: "pst".to_string(),
|
||||
total,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![],
|
||||
});
|
||||
|
||||
// Pass 2: process messages with progress updates
|
||||
let mut success_count: usize = 0;
|
||||
let mut failed_details: Vec<FailedItemDetail> = Vec::new();
|
||||
let mut index: usize = 0;
|
||||
|
||||
let pst_store = match outlook_pst::open_store(file_path) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let ipm_sub_tree = match pst_store.properties().ipm_sub_tree_entry_id() {
|
||||
Ok(id) => id,
|
||||
Err(e) => {
|
||||
fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let ipm_subtree_folder = match pst_store.open_folder(&ipm_sub_tree) {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Progress callback: update progress every 50 messages
|
||||
let import_id = import_id.to_string();
|
||||
let format_str = "pst".to_string();
|
||||
pst::process_folder_with_progress(
|
||||
&ipm_subtree_folder,
|
||||
"", // parent_path starts empty
|
||||
account_id,
|
||||
total, // pass pre-counted total for accurate progress
|
||||
&mut success_count,
|
||||
&mut failed_details,
|
||||
&mut index,
|
||||
&|processed, actual_failed| {
|
||||
update_progress(&import_id, ImportProgress {
|
||||
import_id: import_id.clone(),
|
||||
status: ImportStatus::Processing,
|
||||
format: format_str.clone(),
|
||||
total,
|
||||
success: processed - actual_failed,
|
||||
duplicates: 0,
|
||||
failed: actual_failed,
|
||||
failed_details: vec![],
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
// Clean up temp file
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
|
||||
let final_progress = ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Completed,
|
||||
format: "pst".to_string(),
|
||||
total,
|
||||
success: success_count,
|
||||
duplicates: 0,
|
||||
failed: failed_details.len(),
|
||||
failed_details,
|
||||
};
|
||||
history::save_import_history(user_id, account_id, folder, &final_progress);
|
||||
update_progress(&import_id, final_progress);
|
||||
}
|
||||
|
||||
/// Record a fatal failure and save history.
|
||||
fn fail_progress(
|
||||
import_id: &str,
|
||||
format: &str,
|
||||
message: &str,
|
||||
user_id: u64,
|
||||
account_id: u64,
|
||||
folder: &str,
|
||||
) {
|
||||
let progress = ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Failed,
|
||||
format: format.to_string(),
|
||||
total: 0,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![FailedItemDetail {
|
||||
index: 0,
|
||||
error_message: message.to_string(),
|
||||
}],
|
||||
};
|
||||
update_progress(import_id, progress.clone());
|
||||
history::save_import_history(user_id, account_id, folder, &progress);
|
||||
}
|
||||
|
||||
@@ -16,8 +16,6 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
use compressed_rtf::*;
|
||||
use outlook_pst::ltp::prop_context::PropertyValue;
|
||||
|
||||
pub fn decode_subject(value: &PropertyValue) -> Option<String> {
|
||||
@@ -60,5 +58,5 @@ pub fn decode_html_body(buffer: &[u8], code_page: u16) -> Option<String> {
|
||||
}
|
||||
|
||||
pub fn decode_rtf_compressed(buffer: &[u8]) -> Option<String> {
|
||||
decompress_rtf(buffer).ok()
|
||||
compressed_rtf::decompress_rtf(buffer).ok()
|
||||
}
|
||||
486
crates/core/src/import/pst/mod.rs
Normal file
486
crates/core/src/import/pst/mod.rs
Normal file
@@ -0,0 +1,486 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use crate::base64_encode_url_safe;
|
||||
use crate::envelope::extractor::extract_envelope_from_eml;
|
||||
use chrono::{DateTime, TimeZone, Utc};
|
||||
use mail_send::mail_builder::headers::text::Text;
|
||||
use mail_send::mail_builder::MessageBuilder;
|
||||
use outlook_pst::ltp::prop_context::PropertyValue;
|
||||
use outlook_pst::messaging::attachment::AttachmentProperties;
|
||||
use outlook_pst::messaging::folder::Folder;
|
||||
use outlook_pst::messaging::message::{Message, MessageProperties};
|
||||
use outlook_pst::ndb::node_id::NodeId;
|
||||
use std::rc::Rc;
|
||||
|
||||
mod encoding;
|
||||
|
||||
/// Convert a PST Message into a base64-encoded EML string.
|
||||
pub fn build_eml_base64(message: Rc<dyn Message>) -> Option<String> {
|
||||
let properties = message.properties();
|
||||
|
||||
let mut builder = MessageBuilder::new();
|
||||
if let Some(sub) = extract_subject(properties) {
|
||||
builder = builder.subject(sub);
|
||||
}
|
||||
if let Some(mid) = extract_string_property(properties, 0x1035) {
|
||||
builder = builder.message_id(mid);
|
||||
}
|
||||
if let Some(irt) = extract_string_property(properties, 0x1042) {
|
||||
builder = builder.in_reply_to(irt);
|
||||
}
|
||||
|
||||
if let Some(refs) = extract_string_property(properties, 0x1039) {
|
||||
builder = builder.header("References", Text::new(refs));
|
||||
}
|
||||
|
||||
if let Some(cid_val) = properties.get(0x3013) {
|
||||
if let PropertyValue::Binary(bin) = cid_val {
|
||||
builder = builder.header(
|
||||
"X-Bichon-Conversation-ID",
|
||||
Text::new(hex::encode(bin.buffer())),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let from = extract_string_property(properties, 0x5D01)
|
||||
.or_else(|| extract_string_property(properties, 0x5D02))
|
||||
.or_else(|| extract_string_property(properties, 0x0C1F));
|
||||
|
||||
if let Some(f) = from {
|
||||
builder = builder.from(f);
|
||||
}
|
||||
|
||||
if let Some(filetime) = extract_i64_property(properties, &[0x0039, 0x0E06]) {
|
||||
let dt = filetime_to_datetime(filetime).timestamp();
|
||||
builder = builder.date(dt);
|
||||
}
|
||||
|
||||
let (to, cc, bcc) = extract_recipients_list(&message);
|
||||
if !to.is_empty() {
|
||||
builder = builder.to(to.iter().map(|s| s.as_str()).collect::<Vec<_>>());
|
||||
}
|
||||
if !cc.is_empty() {
|
||||
builder = builder.cc(cc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
|
||||
}
|
||||
if !bcc.is_empty() {
|
||||
builder = builder.bcc(bcc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
if let Some(html) = extract_html(properties) {
|
||||
builder = builder.html_body(html);
|
||||
}
|
||||
|
||||
if let Some(text) = extract_text(properties) {
|
||||
builder = builder.text_body(text);
|
||||
}
|
||||
|
||||
if let Some(attachment_table) = message.attachment_table() {
|
||||
for row in attachment_table.rows_matrix() {
|
||||
let node_id = NodeId::from(u32::from(row.id()));
|
||||
if let Ok(attachment) = message.clone().read_attachment(node_id, None) {
|
||||
let att_props = attachment.properties();
|
||||
let name = extract_attachment_string_property(att_props, 0x3707);
|
||||
let mime = extract_attachment_string_property(att_props, 0x370E)
|
||||
.unwrap_or_else(|| "application/octet-stream".into());
|
||||
let cid = extract_attachment_string_property(att_props, 0x3712);
|
||||
let is_inline = att_props
|
||||
.get(0x3714)
|
||||
.and_then(|val| {
|
||||
if let PropertyValue::Integer32(f) = val {
|
||||
Some(f)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.map(|flag| (flag & 0x4) != 0)
|
||||
.unwrap_or(false);
|
||||
|
||||
if let Some(PropertyValue::Binary(bin)) = att_props.get(0x3701) {
|
||||
let data = bin.buffer().to_vec();
|
||||
let file_name = name.unwrap_or_else(|| "unnamed_attachment".to_string());
|
||||
|
||||
if is_inline && cid.is_some() {
|
||||
let content_id = cid.unwrap();
|
||||
builder = builder.inline(mime, content_id, data);
|
||||
} else {
|
||||
builder = builder.attachment(mime, file_name, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match builder.write_to_vec() {
|
||||
Ok(eml_vec) => Some(base64_encode_url_safe!(eml_vec)),
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to generate EML from PST message: {:?}", e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn filetime_to_datetime(filetime: i64) -> DateTime<Utc> {
|
||||
let unix_secs = (filetime / 10_000_000) - 11_644_473_600;
|
||||
let nsecs = (filetime % 10_000_000) * 100;
|
||||
Utc.timestamp_opt(unix_secs, nsecs as u32).unwrap()
|
||||
}
|
||||
|
||||
fn extract_recipients_list(message: &Rc<dyn Message>) -> (Vec<String>, Vec<String>, Vec<String>) {
|
||||
let mut to = Vec::new();
|
||||
let mut cc = Vec::new();
|
||||
let mut bcc = Vec::new();
|
||||
|
||||
let recipient_table = message.recipient_table();
|
||||
if let Some(recipient_table) = recipient_table {
|
||||
let context = recipient_table.context();
|
||||
for row in recipient_table.rows_matrix() {
|
||||
if let Ok(cols) = row.columns(context) {
|
||||
let mut r_type = 0;
|
||||
let mut email = String::new();
|
||||
|
||||
for (col, val) in context.columns().iter().zip(cols) {
|
||||
let prop_val = val
|
||||
.as_ref()
|
||||
.and_then(|v| recipient_table.read_column(v, col.prop_type()).ok());
|
||||
match col.prop_id() {
|
||||
0x0C15 => {
|
||||
if let Some(PropertyValue::Integer32(t)) = prop_val {
|
||||
r_type = t;
|
||||
}
|
||||
}
|
||||
0x39FE | 0x3003 => {
|
||||
if let Some(s) = prop_val.and_then(|v| extract_string(&v)) {
|
||||
email = s;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if !email.is_empty() {
|
||||
match r_type {
|
||||
1 => to.push(email),
|
||||
2 => cc.push(email),
|
||||
3 => bcc.push(email),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let receiver = extract_string_property(message.properties(), 0x0076);
|
||||
if let Some(receiver) = receiver {
|
||||
to.push(receiver);
|
||||
}
|
||||
}
|
||||
(to, cc, bcc)
|
||||
}
|
||||
|
||||
fn extract_subject(props: &MessageProperties) -> Option<String> {
|
||||
props.get(0x0037).and_then(|val| encoding::decode_subject(val))
|
||||
}
|
||||
|
||||
fn extract_string_property(properties: &MessageProperties, prop_id: u16) -> Option<String> {
|
||||
properties
|
||||
.get(prop_id)
|
||||
.and_then(|value| extract_string(value))
|
||||
}
|
||||
|
||||
fn extract_attachment_string_property(
|
||||
properties: &AttachmentProperties,
|
||||
prop_id: u16,
|
||||
) -> Option<String> {
|
||||
properties
|
||||
.get(prop_id)
|
||||
.and_then(|value| extract_string(value))
|
||||
}
|
||||
|
||||
fn extract_string(value: &PropertyValue) -> Option<String> {
|
||||
match value {
|
||||
PropertyValue::String8(value) => Some(value.to_string()),
|
||||
PropertyValue::Unicode(value) => Some(value.to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_text(properties: &MessageProperties) -> Option<String> {
|
||||
properties.get(0x1000).and_then(extract_string).or_else(|| {
|
||||
properties.get(0x1009).and_then(|value| match value {
|
||||
PropertyValue::Binary(value) => encoding::decode_rtf_compressed(value.buffer()),
|
||||
_ => None,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_html(properties: &MessageProperties) -> Option<String> {
|
||||
properties.get(0x1013).and_then(|value| match value {
|
||||
PropertyValue::Binary(value) => {
|
||||
let code_page = properties
|
||||
.get(0x3FDE)
|
||||
.and_then(|v| {
|
||||
if let PropertyValue::Integer32(cpid) = v {
|
||||
Some(*cpid as u16)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or(65001);
|
||||
encoding::decode_html_body(value.buffer(), code_page)
|
||||
}
|
||||
PropertyValue::String8(value) => Some(value.to_string()),
|
||||
PropertyValue::Unicode(value) => Some(value.to_string()),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_i64_property(properties: &MessageProperties, prop_ids: &[u16]) -> Option<i64> {
|
||||
for &prop_id in prop_ids {
|
||||
if let Some(PropertyValue::Time(value)) = properties.get(prop_id) {
|
||||
return Some(*value);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Open a PST file and count total messages across all folders.
|
||||
/// Called from the web upload flow to get the total before processing.
|
||||
pub fn count_pst_messages(pst_path: &std::path::Path) -> crate::error::BichonResult<usize> {
|
||||
let pst_store = outlook_pst::open_store(pst_path).map_err(|e| {
|
||||
crate::raise_error!(
|
||||
format!("Failed to open PST file: {:?}", e),
|
||||
crate::error::code::ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
|
||||
let ipm_sub_tree = pst_store.properties().ipm_sub_tree_entry_id().map_err(|e| {
|
||||
crate::raise_error!(
|
||||
format!("Could not find IPM_SUBTREE in PST: {:?}", e),
|
||||
crate::error::code::ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
|
||||
let ipm_subtree_folder = pst_store.open_folder(&ipm_sub_tree).map_err(|e| {
|
||||
crate::raise_error!(
|
||||
format!("Failed to open root mailbox folder: {:?}", e),
|
||||
crate::error::code::ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(count_folder_messages(&ipm_subtree_folder))
|
||||
}
|
||||
|
||||
fn count_folder_messages(folder: &Rc<dyn Folder>) -> usize {
|
||||
let mut count = 0usize;
|
||||
|
||||
if let Some(contents_table) = folder.contents_table() {
|
||||
for row in contents_table.rows_matrix() {
|
||||
let store = folder.store().clone();
|
||||
let entry_id = match store
|
||||
.properties()
|
||||
.make_entry_id(NodeId::from(u32::from(row.id())))
|
||||
{
|
||||
Ok(id) => id,
|
||||
Err(_) => continue,
|
||||
};
|
||||
|
||||
if store.open_message(&entry_id, None).is_ok() {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(hierarchy_table) = folder.hierarchy_table() {
|
||||
for row in hierarchy_table.rows_matrix() {
|
||||
let node = NodeId::from(u32::from(row.id()));
|
||||
if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
|
||||
if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
|
||||
count += count_folder_messages(&sub_folder);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
count
|
||||
}
|
||||
|
||||
/// Walk all folders and process messages, calling the progress callback
|
||||
/// every 50 messages. Used by the web upload flow.
|
||||
pub fn process_folder_with_progress<F>(
|
||||
folder: &Rc<dyn Folder>,
|
||||
parent_path: &str,
|
||||
account_id: u64,
|
||||
total: usize,
|
||||
success_count: &mut usize,
|
||||
failed_details: &mut Vec<super::FailedItemDetail>,
|
||||
index: &mut usize,
|
||||
progress_cb: &F,
|
||||
) where
|
||||
F: Fn(usize, usize), // (processed, failed)
|
||||
{
|
||||
process_folder_with_progress_inner(
|
||||
folder,
|
||||
parent_path,
|
||||
account_id,
|
||||
total,
|
||||
success_count,
|
||||
failed_details,
|
||||
index,
|
||||
progress_cb,
|
||||
);
|
||||
}
|
||||
|
||||
fn process_folder_with_progress_inner<F>(
|
||||
folder: &Rc<dyn Folder>,
|
||||
parent_path: &str,
|
||||
account_id: u64,
|
||||
total: usize,
|
||||
success_count: &mut usize,
|
||||
failed_details: &mut Vec<super::FailedItemDetail>,
|
||||
index: &mut usize,
|
||||
progress_cb: &F,
|
||||
) where
|
||||
F: Fn(usize, usize),
|
||||
{
|
||||
let folder_name = folder
|
||||
.properties()
|
||||
.display_name()
|
||||
.unwrap_or_else(|_| "Unknown".to_string());
|
||||
|
||||
let mail_folder = if parent_path.is_empty() {
|
||||
folder_name
|
||||
} else {
|
||||
format!("{}/{}", parent_path, folder_name)
|
||||
};
|
||||
|
||||
tracing::debug!("Processing PST folder: {}", mail_folder);
|
||||
|
||||
let mailbox_id = match super::resolve_mailbox_by_account_id(account_id, &mail_folder) {
|
||||
Ok(id) => id,
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to resolve mailbox '{}': {:?}", mail_folder, e);
|
||||
// Still recurse into sub-folders even if this folder's mailbox creation fails
|
||||
if let Some(hierarchy_table) = folder.hierarchy_table() {
|
||||
for row in hierarchy_table.rows_matrix() {
|
||||
let node = NodeId::from(u32::from(row.id()));
|
||||
if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
|
||||
if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
|
||||
process_folder_with_progress_inner(
|
||||
&sub_folder,
|
||||
&mail_folder,
|
||||
account_id,
|
||||
total,
|
||||
success_count,
|
||||
failed_details,
|
||||
index,
|
||||
progress_cb,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mut batch_size = 0usize;
|
||||
|
||||
if let Some(contents_table) = folder.contents_table() {
|
||||
for row in contents_table.rows_matrix() {
|
||||
let store = folder.store().clone();
|
||||
|
||||
let entry_id = match store
|
||||
.properties()
|
||||
.make_entry_id(NodeId::from(u32::from(row.id())))
|
||||
{
|
||||
Ok(id) => id,
|
||||
Err(e) => {
|
||||
tracing::warn!("Skip PST row {}: {:?}", row.unique(), e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
match store.open_message(&entry_id, None) {
|
||||
Ok(message) => match build_eml_base64(message) {
|
||||
Some(base64_eml) => {
|
||||
let decoded = match crate::base64_decode_url_safe!(base64_eml.as_bytes()) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(e) => {
|
||||
failed_details.push(super::FailedItemDetail {
|
||||
index: *index,
|
||||
error_message: format!(
|
||||
"Failed to decode base64 EML at index {}: {:?}",
|
||||
*index, e
|
||||
),
|
||||
});
|
||||
*index += 1;
|
||||
batch_size += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
match futures::executor::block_on(
|
||||
extract_envelope_from_eml(&decoded, account_id, mailbox_id)
|
||||
) {
|
||||
Ok(_) => {
|
||||
*success_count += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
failed_details.push(super::FailedItemDetail {
|
||||
index: *index,
|
||||
error_message: format!("{:?}", e),
|
||||
});
|
||||
}
|
||||
};
|
||||
*index += 1;
|
||||
batch_size += 1;
|
||||
}
|
||||
None => {}
|
||||
},
|
||||
Err(e) => {
|
||||
tracing::warn!("Open PST message error: {:?}", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Report progress every 50 messages
|
||||
if batch_size % 50 == 0 {
|
||||
progress_cb(*success_count + failed_details.len(), failed_details.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(hierarchy_table) = folder.hierarchy_table() {
|
||||
for row in hierarchy_table.rows_matrix() {
|
||||
let node = NodeId::from(u32::from(row.id()));
|
||||
if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
|
||||
if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
|
||||
process_folder_with_progress_inner(
|
||||
&sub_folder,
|
||||
&mail_folder,
|
||||
account_id,
|
||||
total,
|
||||
success_count,
|
||||
failed_details,
|
||||
index,
|
||||
progress_cb,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,8 @@ use std::fs;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
/// Memory-mapped MBOX file. Messages are yielded one at a time without
|
||||
/// loading the entire file into RAM.
|
||||
pub struct MboxFile {
|
||||
map: Mmap,
|
||||
}
|
||||
@@ -127,10 +129,6 @@ impl<'a> Iterator for MboxReader<'a> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use mail_parser::MessageParser;
|
||||
|
||||
use crate::mbox::gmail::determine_folder;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn collect_entries(data: &[u8]) -> Vec<&[u8]> {
|
||||
@@ -144,6 +142,7 @@ mod tests {
|
||||
let e = collect_entries(data);
|
||||
assert_eq!(e, vec![b"mail1\n", b"mail2\n"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_trailing_newline() {
|
||||
let data = b"From a\nmail1";
|
||||
@@ -204,22 +203,4 @@ mod tests {
|
||||
let e = collect_entries(&data);
|
||||
assert_eq!(e.len(), 1000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test11() {
|
||||
let mbox = MboxFile::from_file(Path::new("e:\\test.mbox")).unwrap();
|
||||
|
||||
for e in mbox.iter() {
|
||||
let body = e.data;
|
||||
|
||||
let message = MessageParser::new().parse(body).unwrap();
|
||||
let labels = message.header("X-Gmail-Labels").unwrap().as_text().unwrap();
|
||||
//println!("offset={} X-Gmail-Labels={:?}", e.offset, labels);
|
||||
println!(
|
||||
"X-Gmail-Labels={:?}, determine_folder={}",
|
||||
labels,
|
||||
determine_folder(labels)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -43,6 +43,10 @@ pub struct EmailSearchFilter {
|
||||
pub to: Option<String>,
|
||||
pub cc: Option<String>,
|
||||
pub bcc: Option<String>,
|
||||
/// Matches if the address appears in `to`, `cc`, or `bcc` (OR semantics).
|
||||
pub any_recipient: Option<String>,
|
||||
/// Matches if the address appears in `from`, `to`, `cc`, or `bcc` (OR semantics).
|
||||
pub any_participant: Option<String>,
|
||||
pub since: Option<i64>,
|
||||
pub before: Option<i64>,
|
||||
/// Lower bound (inclusive) on the IMAP server INTERNALDATE timestamp.
|
||||
|
||||
@@ -404,6 +404,7 @@ impl NewIndexWriter {
|
||||
regular_attachment_count: attachment_docs.len(),
|
||||
tags: None,
|
||||
account_email: None,
|
||||
account_name: None,
|
||||
mailbox_name: None,
|
||||
content_hash: email_content_hash,
|
||||
};
|
||||
|
||||
@@ -20,6 +20,7 @@ use crate::error::code::ErrorCode;
|
||||
use crate::error::BichonResult;
|
||||
use crate::oauth2::{entity::OAuth2, pending::OAuth2PendingEntity, token::OAuth2AccessToken};
|
||||
use crate::settings::proxy::Proxy;
|
||||
use crate::utils::net::parse_proxy_url;
|
||||
use crate::{decrypt, encrypt, raise_error};
|
||||
use oauth2::{
|
||||
basic::BasicClient, AuthUrl, AuthorizationCode, ClientId, ClientSecret, CsrfToken,
|
||||
@@ -265,13 +266,30 @@ impl OAuth2Flow {
|
||||
fn build_http_client(use_proxy: Option<u64>) -> BichonResult<reqwest::Client> {
|
||||
if let Some(proxy_id) = use_proxy {
|
||||
let proxy = Proxy::get(proxy_id)?;
|
||||
// Normalize the URL: reqwest only understands standard format user:pass@host:port.
|
||||
// Our parse_proxy_url handles both standard and non-standard (host:port:user:pass).
|
||||
let proxy_url = match parse_proxy_url(&proxy.url) {
|
||||
Ok(addr) => {
|
||||
if let (Some(user), Some(pass)) = (&addr.username, &addr.password) {
|
||||
format!("socks5://{}:{}@{}:{}", user, pass, addr.host, addr.port)
|
||||
} else if let Some(user) = &addr.username {
|
||||
format!("socks5://{}@{}:{}", user, addr.host, addr.port)
|
||||
} else {
|
||||
format!("socks5://{}:{}", addr.host, addr.port)
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
// Fallback: pass through as-is for backward compatibility
|
||||
proxy.url.clone()
|
||||
}
|
||||
};
|
||||
return oauth2::reqwest::ClientBuilder::new()
|
||||
.redirect(oauth2::reqwest::redirect::Policy::none())
|
||||
.proxy(reqwest::Proxy::all(&proxy.url).map_err(|e| {
|
||||
.proxy(reqwest::Proxy::all(&proxy_url).map_err(|e| {
|
||||
raise_error!(
|
||||
format!(
|
||||
"Failed to configure SOCKS5 proxy ({}): {:#?}. Please check",
|
||||
&proxy.url, e
|
||||
&proxy_url, e
|
||||
),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
|
||||
@@ -308,6 +308,56 @@ pub struct Settings {
|
||||
help = "Enable SMTP authentication requirement"
|
||||
)]
|
||||
pub bichon_smtp_auth_required: bool,
|
||||
|
||||
/// Enable OIDC-based Single Sign-On (Pro/Enterprise feature).
|
||||
#[clap(long, default_value = "false", env, help = "Enable OpenID Connect SSO")]
|
||||
pub bichon_oidc_enabled: bool,
|
||||
|
||||
/// OIDC issuer URL (e.g. https://keycloak.example.com/realms/myorg).
|
||||
#[clap(long, env, help = "OpenID Connect issuer URL")]
|
||||
pub bichon_oidc_issuer_url: Option<String>,
|
||||
|
||||
/// OIDC client ID registered with the IdP.
|
||||
#[clap(long, env, help = "OpenID Connect client ID")]
|
||||
pub bichon_oidc_client_id: Option<String>,
|
||||
|
||||
/// OIDC client secret registered with the IdP.
|
||||
#[clap(long, env, help = "OpenID Connect client secret")]
|
||||
pub bichon_oidc_client_secret: Option<String>,
|
||||
|
||||
/// OIDC redirect URI (must match what's registered with the IdP).
|
||||
#[clap(long, env, help = "OpenID Connect redirect URI")]
|
||||
pub bichon_oidc_redirect_uri: Option<String>,
|
||||
|
||||
/// Maximum HTTP request body size in MB for file uploads (default: 1100 MB).
|
||||
/// Requests exceeding this limit are rejected at the framework level before
|
||||
/// the application reads the body, preventing memory exhaustion attacks.
|
||||
#[clap(
|
||||
long,
|
||||
default_value = "1100",
|
||||
env,
|
||||
help = "Maximum HTTP request body size in MB for file uploads"
|
||||
)]
|
||||
pub bichon_upload_body_limit_mb: u64,
|
||||
|
||||
/// Maximum per-file size in MB for MBOX uploads via the web UI (default: 1024 MB = 1 GB).
|
||||
/// Individual EML files are always capped at 100 MB regardless of this setting.
|
||||
#[clap(
|
||||
long,
|
||||
default_value = "1024",
|
||||
env,
|
||||
help = "Maximum per-file size in MB for MBOX uploads via the web UI"
|
||||
)]
|
||||
pub bichon_web_mbox_upload_limit_mb: u64,
|
||||
|
||||
/// Maximum per-file size in MB for PST uploads via the web UI (default: 2048 MB = 2 GB).
|
||||
#[clap(
|
||||
long,
|
||||
default_value = "2048",
|
||||
env,
|
||||
help = "Maximum per-file size in MB for PST uploads via the web UI"
|
||||
)]
|
||||
pub bichon_web_pst_upload_limit_mb: u64,
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
@@ -317,9 +367,8 @@ impl Settings {
|
||||
// rejects it, fall back to parsing with only the binary name so that
|
||||
// the settings come entirely from environment variables.
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let s = Self::try_parse_from(&args).unwrap_or_else(|_| {
|
||||
Self::parse_from(std::iter::once(args[0].clone()))
|
||||
});
|
||||
let s = Self::try_parse_from(&args)
|
||||
.unwrap_or_else(|_| Self::parse_from(std::iter::once(args[0].clone())));
|
||||
if s.bichon_encrypt_password.is_none() && s.bichon_encrypt_password_file.is_none() {
|
||||
panic!(
|
||||
"One of --bichon_encrypt_password or --bichon_encrypt_password_file has to be set"
|
||||
|
||||
@@ -60,6 +60,17 @@ pub struct SystemConfigurations {
|
||||
pub bichon_smtp_auth_required: bool,
|
||||
pub bichon_smtp_tls_key_path: Option<String>,
|
||||
pub bichon_smtp_tls_cert_path: Option<String>,
|
||||
|
||||
pub bichon_oidc_enabled: bool,
|
||||
pub bichon_oidc_issuer_url: Option<String>,
|
||||
pub bichon_oidc_client_id: Option<String>,
|
||||
pub bichon_oidc_redirect_uri: Option<String>,
|
||||
|
||||
pub bichon_upload_body_limit_mb: u64,
|
||||
|
||||
pub bichon_web_mbox_upload_limit_mb: u64,
|
||||
|
||||
pub bichon_web_pst_upload_limit_mb: u64,
|
||||
}
|
||||
|
||||
impl From<&Settings> for SystemConfigurations {
|
||||
@@ -94,6 +105,13 @@ impl From<&Settings> for SystemConfigurations {
|
||||
bichon_smtp_auth_required: s.bichon_smtp_auth_required,
|
||||
bichon_smtp_tls_key_path: s.bichon_smtp_tls_key_path.clone(),
|
||||
bichon_smtp_tls_cert_path: s.bichon_smtp_tls_cert_path.clone(),
|
||||
bichon_oidc_enabled: s.bichon_oidc_enabled,
|
||||
bichon_oidc_issuer_url: s.bichon_oidc_issuer_url.clone(),
|
||||
bichon_oidc_client_id: s.bichon_oidc_client_id.clone(),
|
||||
bichon_oidc_redirect_uri: s.bichon_oidc_redirect_uri.clone(),
|
||||
bichon_upload_body_limit_mb: s.bichon_upload_body_limit_mb,
|
||||
bichon_web_mbox_upload_limit_mb: s.bichon_web_mbox_upload_limit_mb,
|
||||
bichon_web_pst_upload_limit_mb: s.bichon_web_pst_upload_limit_mb,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ use crate::{
|
||||
},
|
||||
error::{code::ErrorCode, BichonResult},
|
||||
id, raise_error, utc_now,
|
||||
utils::net::parse_proxy_addr,
|
||||
utils::net::parse_proxy_url,
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||
@@ -98,9 +98,9 @@ impl Proxy {
|
||||
insert_impl(DB_MANAGER.db(), self.to_owned())
|
||||
}
|
||||
|
||||
/// Validate that the URL is a valid SOCKS5 proxy URL.
|
||||
/// Validate that the URL is a valid proxy URL.
|
||||
pub fn validate(&self) -> BichonResult<()> {
|
||||
parse_proxy_addr(&self.url)?;
|
||||
parse_proxy_url(&self.url)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -111,7 +111,15 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_valid_proxy_urls() {
|
||||
let urls = vec!["socks5://127.0.0.1:1080", "http://127.0.0.1:8080"];
|
||||
let urls = vec![
|
||||
"socks5://127.0.0.1:1080",
|
||||
"http://127.0.0.1:8080",
|
||||
"socks5://proxy.example.com:1080",
|
||||
"socks5://user:pass@proxy.example.com:1080",
|
||||
"socks5://user@proxy.example.com:1080",
|
||||
// Non-standard format: host:port:user:pass
|
||||
"socks5://server.nodeprovider.com:8080:username123:passwordhere",
|
||||
];
|
||||
|
||||
for url in urls {
|
||||
let proxy = Proxy::new(url.to_string());
|
||||
|
||||
@@ -27,6 +27,7 @@ pub struct Envelope {
|
||||
pub message_id: String,
|
||||
pub account_id: u64,
|
||||
pub account_email: Option<String>,
|
||||
pub account_name: Option<String>,
|
||||
pub mailbox_id: u64,
|
||||
pub mailbox_name: Option<String>,
|
||||
pub uid: u32,
|
||||
|
||||
@@ -172,6 +172,52 @@ impl DedupCache {
|
||||
POPULATE_WINDOW_MS / (24 * 60 * 60 * 1000),
|
||||
);
|
||||
}
|
||||
|
||||
/// Remove all entries for a specific account.
|
||||
pub fn remove_by_account(&self, account_id: u64) {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
let before = entries.len();
|
||||
entries.retain(|(aid, _, _), _| *aid != account_id);
|
||||
let removed = before - entries.len();
|
||||
if removed > 0 {
|
||||
tracing::info!(
|
||||
"DedupCache: removed {} entries for account {}",
|
||||
removed,
|
||||
account_id
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove all entries for a specific mailbox (across all accounts).
|
||||
pub fn remove_by_mailbox(&self, mailbox_id: u64) {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
let before = entries.len();
|
||||
entries.retain(|(_, mid, _), _| *mid != mailbox_id);
|
||||
let removed = before - entries.len();
|
||||
if removed > 0 {
|
||||
tracing::info!(
|
||||
"DedupCache: removed {} entries for mailbox {}",
|
||||
removed,
|
||||
mailbox_id
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove a specific triple (most precise removal).
|
||||
pub fn remove(&self, account_id: u64, mailbox_id: u64, hash: &str) {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
if entries
|
||||
.remove(&(account_id, mailbox_id, hash.to_string()))
|
||||
.is_some()
|
||||
{
|
||||
tracing::debug!(
|
||||
"DedupCache: removed specific entry ({}, {}, {})",
|
||||
account_id,
|
||||
mailbox_id,
|
||||
hash
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||
@@ -366,7 +412,11 @@ mod tests {
|
||||
for segment_reader in searcher.segment_readers() {
|
||||
let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap();
|
||||
let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap();
|
||||
let hash_col = segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap().unwrap();
|
||||
let hash_col = segment_reader
|
||||
.fast_fields()
|
||||
.str(F_CONTENT_HASH)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap();
|
||||
|
||||
let max_doc = segment_reader.max_doc();
|
||||
@@ -381,7 +431,11 @@ mod tests {
|
||||
let account_id = account_col.values.get_val(doc_id);
|
||||
let mailbox_id = mailbox_col.values.get_val(doc_id);
|
||||
|
||||
let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0);
|
||||
let hash_ord = hash_col
|
||||
.ords()
|
||||
.values_for_doc(doc_id as u32)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
let mut hash_buf = String::new();
|
||||
hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap();
|
||||
|
||||
@@ -419,7 +473,11 @@ mod tests {
|
||||
for segment_reader in searcher.segment_readers() {
|
||||
let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap();
|
||||
let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap();
|
||||
let hash_col = segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap().unwrap();
|
||||
let hash_col = segment_reader
|
||||
.fast_fields()
|
||||
.str(F_CONTENT_HASH)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap();
|
||||
|
||||
let max_doc = segment_reader.max_doc();
|
||||
@@ -434,7 +492,11 @@ mod tests {
|
||||
let account_id = account_col.values.get_val(doc_id);
|
||||
let mailbox_id = mailbox_col.values.get_val(doc_id);
|
||||
|
||||
let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0);
|
||||
let hash_ord = hash_col
|
||||
.ords()
|
||||
.values_for_doc(doc_id as u32)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
let mut hash_buf = String::new();
|
||||
hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap();
|
||||
|
||||
@@ -474,7 +536,11 @@ mod tests {
|
||||
for segment_reader in searcher.segment_readers() {
|
||||
let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap();
|
||||
let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap();
|
||||
let hash_col = segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap().unwrap();
|
||||
let hash_col = segment_reader
|
||||
.fast_fields()
|
||||
.str(F_CONTENT_HASH)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap();
|
||||
|
||||
let max_doc = segment_reader.max_doc();
|
||||
@@ -489,7 +555,11 @@ mod tests {
|
||||
let account_id = account_col.values.get_val(doc_id);
|
||||
let mailbox_id = mailbox_col.values.get_val(doc_id);
|
||||
|
||||
let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0);
|
||||
let hash_ord = hash_col
|
||||
.ords()
|
||||
.values_for_doc(doc_id as u32)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
let mut hash_buf = String::new();
|
||||
hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap();
|
||||
|
||||
@@ -501,4 +571,71 @@ mod tests {
|
||||
assert!(cache.contains(1, 10, "hash-keep"));
|
||||
assert!(!cache.contains(1, 10, "hash-delete"));
|
||||
}
|
||||
|
||||
// ── removal methods ─────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn remove_by_account_works() {
|
||||
let cache = DedupCache::new_for_test();
|
||||
|
||||
cache.insert(1, 10, "hash-a1");
|
||||
cache.insert(1, 20, "hash-a2");
|
||||
cache.insert(2, 10, "hash-b1");
|
||||
cache.insert(2, 30, "hash-b2");
|
||||
cache.insert(1, 10, "hash-a3");
|
||||
|
||||
assert_eq!(cache.entries.lock().unwrap().len(), 5);
|
||||
|
||||
cache.remove_by_account(1);
|
||||
|
||||
let entries = cache.entries.lock().unwrap();
|
||||
assert_eq!(entries.len(), 2);
|
||||
assert!(!entries.contains_key(&(1, 10, "hash-a1".to_string())));
|
||||
assert!(!entries.contains_key(&(1, 20, "hash-a2".to_string())));
|
||||
assert!(!entries.contains_key(&(1, 10, "hash-a3".to_string())));
|
||||
|
||||
assert!(entries.contains_key(&(2, 10, "hash-b1".to_string())));
|
||||
assert!(entries.contains_key(&(2, 30, "hash-b2".to_string())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_by_mailbox_works() {
|
||||
let cache = DedupCache::new_for_test();
|
||||
|
||||
cache.insert(1, 10, "hash-1");
|
||||
cache.insert(1, 20, "hash-2");
|
||||
cache.insert(2, 10, "hash-3");
|
||||
cache.insert(3, 20, "hash-4");
|
||||
cache.insert(1, 10, "hash-5");
|
||||
|
||||
cache.remove_by_mailbox(10);
|
||||
|
||||
let entries = cache.entries.lock().unwrap();
|
||||
assert_eq!(entries.len(), 2);
|
||||
|
||||
assert!(entries.contains_key(&(1, 20, "hash-2".to_string())));
|
||||
assert!(entries.contains_key(&(3, 20, "hash-4".to_string())));
|
||||
|
||||
assert!(!entries.contains_key(&(1, 10, "hash-1".to_string())));
|
||||
assert!(!entries.contains_key(&(2, 10, "hash-3".to_string())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_specific_triple_works() {
|
||||
let cache = DedupCache::new_for_test();
|
||||
|
||||
cache.insert(1, 10, "hash-aaa");
|
||||
cache.insert(1, 10, "hash-bbb");
|
||||
cache.insert(2, 20, "hash-aaa");
|
||||
|
||||
assert!(cache.contains(1, 10, "hash-aaa"));
|
||||
assert!(cache.contains(1, 10, "hash-bbb"));
|
||||
assert!(cache.contains(2, 20, "hash-aaa"));
|
||||
|
||||
cache.remove(1, 10, "hash-aaa");
|
||||
|
||||
assert!(!cache.contains(1, 10, "hash-aaa"));
|
||||
assert!(cache.contains(1, 10, "hash-bbb"));
|
||||
assert!(cache.contains(2, 20, "hash-aaa"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,6 +40,7 @@ use crate::{
|
||||
envelope::Envelope,
|
||||
tantivy::{
|
||||
attachment::ATTACHMENT_MANAGER,
|
||||
dedup_cache::DEDUP_CACHE,
|
||||
fatal_commit,
|
||||
fields::{
|
||||
F_ACCOUNT_ID, F_DATE, F_FROM, F_ID, F_INGEST_AT, F_INTERNAL_DATE,
|
||||
@@ -50,8 +51,8 @@ use crate::{
|
||||
tokenizers::EuroTokenizer,
|
||||
},
|
||||
},
|
||||
utils::html::extract_text,
|
||||
utc_now,
|
||||
utils::html::extract_text,
|
||||
};
|
||||
|
||||
use chrono::Utc;
|
||||
@@ -277,6 +278,80 @@ impl IndexManager {
|
||||
Box::new(boolean_query)
|
||||
}
|
||||
|
||||
/// Return all Message-IDs stored in Tantivy for a given mailbox.
|
||||
/// Prefer `mailbox_contains_message_id` for existence checks on large
|
||||
/// mailboxes — this method loads everything into a HashSet.
|
||||
pub fn get_message_ids_for_mailbox(
|
||||
&self,
|
||||
account_id: u64,
|
||||
mailbox_id: u64,
|
||||
) -> BichonResult<HashSet<String>> {
|
||||
let query = self.mailbox_query(account_id, mailbox_id);
|
||||
let fields = SchemaTools::email_fields();
|
||||
let searcher = self.create_searcher()?;
|
||||
|
||||
let docs = searcher
|
||||
.search(&query, &DocSetCollector)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
let mut result = HashSet::new();
|
||||
for doc_address in docs {
|
||||
let doc = searcher
|
||||
.doc::<TantivyDocument>(doc_address)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
if let Some(v) = doc.get_first(fields.f_message_id) {
|
||||
if let Some(s) = v.as_str() {
|
||||
if !s.is_empty() {
|
||||
result.insert(s.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Check whether a specific Message-ID exists in a mailbox.
|
||||
/// Uses a TermQuery — O(1) per call, no allocation proportional to
|
||||
/// mailbox size. Suitable for large mailboxes where
|
||||
/// `get_message_ids_for_mailbox` would allocate too much memory.
|
||||
pub fn mailbox_contains_message_id(
|
||||
&self,
|
||||
account_id: u64,
|
||||
mailbox_id: u64,
|
||||
message_id: &str,
|
||||
) -> BichonResult<bool> {
|
||||
let fields = SchemaTools::email_fields();
|
||||
let query = BooleanQuery::new(vec![
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(fields.f_account_id, account_id),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(fields.f_mailbox_id, mailbox_id),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_text(fields.f_message_id, message_id),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
]);
|
||||
let searcher = self.create_searcher()?;
|
||||
let count = searcher
|
||||
.search(&query, &Count)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
Ok(count > 0)
|
||||
}
|
||||
|
||||
fn envelope_query(&self, account_id: u64, eid: &str) -> Box<dyn Query> {
|
||||
let account_id_query = TermQuery::new(
|
||||
Term::from_field_u64(SchemaTools::email_fields().f_account_id, account_id),
|
||||
@@ -386,6 +461,34 @@ impl IndexManager {
|
||||
}
|
||||
}
|
||||
|
||||
// any_recipient: OR across to, cc, bcc
|
||||
if let Some(ref v) = filter.any_recipient {
|
||||
let mut recipient_queries: Vec<(Occur, Box<dyn Query>)> = Vec::new();
|
||||
for field in [f.f_to_text, f.f_cc_text, f.f_bcc_text] {
|
||||
let query_parser = QueryParser::for_index(&self.index, vec![field]);
|
||||
if let Ok(q) = query_parser.parse_query(v) {
|
||||
recipient_queries.push((Occur::Should, q));
|
||||
}
|
||||
}
|
||||
if !recipient_queries.is_empty() {
|
||||
subqueries.push((Occur::Must, Box::new(BooleanQuery::new(recipient_queries))));
|
||||
}
|
||||
}
|
||||
|
||||
// any_participant: OR across from, to, cc, bcc
|
||||
if let Some(ref v) = filter.any_participant {
|
||||
let mut participant_queries: Vec<(Occur, Box<dyn Query>)> = Vec::new();
|
||||
for field in [f.f_from_text, f.f_to_text, f.f_cc_text, f.f_bcc_text] {
|
||||
let query_parser = QueryParser::for_index(&self.index, vec![field]);
|
||||
if let Ok(q) = query_parser.parse_query(v) {
|
||||
participant_queries.push((Occur::Should, q));
|
||||
}
|
||||
}
|
||||
if !participant_queries.is_empty() {
|
||||
subqueries.push((Occur::Must, Box::new(BooleanQuery::new(participant_queries))));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(has) = filter.has_attachment {
|
||||
let lower: Bound<Term>;
|
||||
let upper: Bound<Term>;
|
||||
@@ -792,6 +895,8 @@ impl IndexManager {
|
||||
attachments_content_hashes,
|
||||
)?;
|
||||
}
|
||||
|
||||
DEDUP_CACHE.remove_by_account(account_id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -815,8 +920,8 @@ impl IndexManager {
|
||||
}
|
||||
|
||||
let mut queries: Vec<Box<dyn Query>> = Vec::with_capacity(mailbox_ids.len());
|
||||
for mailbox_id in mailbox_ids {
|
||||
queries.push(self.mailbox_query(account_id, mailbox_id));
|
||||
for mailbox_id in &mailbox_ids {
|
||||
queries.push(self.mailbox_query(account_id, *mailbox_id));
|
||||
}
|
||||
let mut writer = self.index_writer.lock().await;
|
||||
for query in queries {
|
||||
@@ -835,6 +940,11 @@ impl IndexManager {
|
||||
attachments_content_hashes,
|
||||
)?;
|
||||
}
|
||||
|
||||
for mailbox_id in mailbox_ids {
|
||||
DEDUP_CACHE.remove_by_mailbox(mailbox_id);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -842,6 +952,21 @@ impl IndexManager {
|
||||
&self,
|
||||
query: Box<dyn Query>,
|
||||
) -> BichonResult<(HashSet<String>, HashSet<String>)> {
|
||||
let (eml_with_mailbox, attachments_content_hashes) =
|
||||
self.collect_content_hashes_with_mailbox(query)?;
|
||||
|
||||
let eml_content_hashes = eml_with_mailbox
|
||||
.into_iter()
|
||||
.map(|(hash, _mailbox_id)| hash)
|
||||
.collect();
|
||||
|
||||
Ok((eml_content_hashes, attachments_content_hashes))
|
||||
}
|
||||
|
||||
fn collect_content_hashes_with_mailbox(
|
||||
&self,
|
||||
query: Box<dyn Query>,
|
||||
) -> BichonResult<(HashSet<(String, u64)>, HashSet<String>)> {
|
||||
let mut eml_content_hashes = HashSet::new();
|
||||
let mut attachments_content_hashes = HashSet::new();
|
||||
|
||||
@@ -857,10 +982,14 @@ impl IndexManager {
|
||||
.doc::<TantivyDocument>(doc_address)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
let mailbox_id = doc.get_first(fields.f_mailbox_id).and_then(|v| v.as_u64());
|
||||
|
||||
// Extract content_hash
|
||||
if let Some(content_hash_value) = doc.get_first(fields.f_content_hash) {
|
||||
if let Some(str) = content_hash_value.as_str() {
|
||||
eml_content_hashes.insert(str.to_string());
|
||||
if let (Some(hash_str), Some(mailbox_id)) =
|
||||
(content_hash_value.as_str(), mailbox_id)
|
||||
{
|
||||
eml_content_hashes.insert((hash_str.to_string(), mailbox_id));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -933,7 +1062,7 @@ impl IndexManager {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut eml_content_hashes: HashSet<String> = HashSet::new();
|
||||
let mut eml_content_hash_triples: HashSet<(u64, u64, String)> = HashSet::new();
|
||||
let mut attachments_content_hashes: HashSet<String> = HashSet::new();
|
||||
|
||||
for (account_id, envelope_ids) in &deletes {
|
||||
@@ -944,8 +1073,14 @@ impl IndexManager {
|
||||
|
||||
for eid in unique_ids {
|
||||
let query = self.envelope_query(*account_id, eid);
|
||||
let (eml_hashes, attachment_hashes) = self.collect_content_hashes(query)?;
|
||||
eml_content_hashes.extend(eml_hashes);
|
||||
let (eml_hashes_with_mailbox, attachment_hashes) =
|
||||
self.collect_content_hashes_with_mailbox(query)?;
|
||||
|
||||
eml_content_hash_triples.extend(
|
||||
eml_hashes_with_mailbox
|
||||
.into_iter()
|
||||
.map(|(hash, mailbox_id)| (*account_id, mailbox_id, hash)),
|
||||
);
|
||||
attachments_content_hashes.extend(attachment_hashes);
|
||||
}
|
||||
}
|
||||
@@ -968,7 +1103,12 @@ impl IndexManager {
|
||||
.commit()
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() {
|
||||
if !eml_content_hash_triples.is_empty() || !attachments_content_hashes.is_empty() {
|
||||
let eml_content_hashes: HashSet<String> = eml_content_hash_triples
|
||||
.iter()
|
||||
.map(|(_, _, hash)| hash.clone())
|
||||
.collect();
|
||||
|
||||
self.cleanup_unused_content(
|
||||
&mut writer,
|
||||
eml_content_hashes,
|
||||
@@ -976,6 +1116,10 @@ impl IndexManager {
|
||||
)?;
|
||||
}
|
||||
|
||||
for (aid, mid, hash) in eml_content_hash_triples {
|
||||
DEDUP_CACHE.remove(aid, mid, &hash);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1157,8 +1301,7 @@ impl IndexManager {
|
||||
// f_attachments JSON blob.
|
||||
if let Some(attrs_val) = old_doc.get_first(f.f_attachments) {
|
||||
if let Some(json_str) = attrs_val.as_str() {
|
||||
if let Ok(parsed) =
|
||||
serde_json::from_str::<serde_json::Value>(json_str)
|
||||
if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(json_str)
|
||||
{
|
||||
if let Some(arr) = parsed.as_array() {
|
||||
for att in arr {
|
||||
@@ -1179,14 +1322,8 @@ impl IndexManager {
|
||||
.and_then(|v| v.as_str())
|
||||
.filter(|s| !s.is_empty())
|
||||
{
|
||||
new_doc.add_text(
|
||||
f.f_attachment_name_text,
|
||||
filename,
|
||||
);
|
||||
new_doc.add_text(
|
||||
f.f_attachment_name_exact,
|
||||
filename,
|
||||
);
|
||||
new_doc.add_text(f.f_attachment_name_text, filename);
|
||||
new_doc.add_text(f.f_attachment_name_exact, filename);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1200,22 +1337,18 @@ impl IndexManager {
|
||||
if let Some(content_hash) = hash_val.as_str() {
|
||||
match BLOB_MANAGER.get_email(content_hash) {
|
||||
Ok(Some(eml_bytes)) => {
|
||||
if let Some(message) =
|
||||
MessageParser::new().parse(&eml_bytes)
|
||||
{
|
||||
if let Some(message) = MessageParser::new().parse(&eml_bytes) {
|
||||
let text = message
|
||||
.body_text(0)
|
||||
.map(|cow| cow.into_owned())
|
||||
.or_else(|| {
|
||||
message.body_html(0).map(|cow| {
|
||||
extract_text(cow.into_owned())
|
||||
})
|
||||
message
|
||||
.body_html(0)
|
||||
.map(|cow| extract_text(cow.into_owned()))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let body_text = text
|
||||
.split_whitespace()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
let body_text =
|
||||
text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
if !body_text.is_empty() {
|
||||
new_doc.add_text(f.f_body, &body_text);
|
||||
}
|
||||
@@ -1304,7 +1437,7 @@ impl IndexManager {
|
||||
let mailbox_docs: Vec<DocAddress>;
|
||||
|
||||
match sort_by {
|
||||
SortBy::DATE => {
|
||||
SortBy::DATE => {
|
||||
let date_docs: Vec<(Option<i64>, DocAddress)> = searcher
|
||||
.search(
|
||||
&query,
|
||||
@@ -1801,10 +1934,8 @@ mod tests {
|
||||
if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(json_str) {
|
||||
if let Some(arr) = parsed.as_array() {
|
||||
for att in arr {
|
||||
let is_inline = att
|
||||
.get("inline")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
let is_inline =
|
||||
att.get("inline").and_then(|v| v.as_bool()).unwrap_or(false);
|
||||
let has_cid = att
|
||||
.get("content_id")
|
||||
.and_then(|v| v.as_str())
|
||||
@@ -1841,8 +1972,7 @@ mod tests {
|
||||
.map(|cow| extract_text(cow.into_owned()))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let body_text =
|
||||
text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
let body_text = text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
if !body_text.is_empty() {
|
||||
new_doc.add_text(f.f_body, &body_text);
|
||||
}
|
||||
@@ -1911,8 +2041,7 @@ mod tests {
|
||||
let mut writer2 = index
|
||||
.writer_with_num_threads(1, 15_000_000)
|
||||
.expect("writer2");
|
||||
writer2
|
||||
.delete_term(Term::from_field_text(f.f_id, "test-eid-001"));
|
||||
writer2.delete_term(Term::from_field_text(f.f_id, "test-eid-001"));
|
||||
writer2.add_document(new_doc).unwrap();
|
||||
writer2.commit().unwrap();
|
||||
|
||||
@@ -1922,16 +2051,14 @@ mod tests {
|
||||
let searcher = reader.searcher();
|
||||
|
||||
// Body text (tokenized via "euro")
|
||||
let body_parser =
|
||||
QueryParser::for_index(&index, vec![f.f_body]);
|
||||
let body_parser = QueryParser::for_index(&index, vec![f.f_body]);
|
||||
let body_hits = searcher
|
||||
.search(&body_parser.parse_query("quick brown fox").unwrap(), &Count)
|
||||
.unwrap();
|
||||
assert_eq!(body_hits, 1, "body text should survive tag update");
|
||||
|
||||
// from_text (tokenized via "euro")
|
||||
let from_parser =
|
||||
QueryParser::for_index(&index, vec![f.f_from_text]);
|
||||
let from_parser = QueryParser::for_index(&index, vec![f.f_from_text]);
|
||||
let from_hits = searcher
|
||||
.search(
|
||||
&from_parser.parse_query("alice@example.com").unwrap(),
|
||||
@@ -1943,10 +2070,7 @@ mod tests {
|
||||
// to_text
|
||||
let to_parser = QueryParser::for_index(&index, vec![f.f_to_text]);
|
||||
let to_hits = searcher
|
||||
.search(
|
||||
&to_parser.parse_query("bob@example.com").unwrap(),
|
||||
&Count,
|
||||
)
|
||||
.search(&to_parser.parse_query("bob@example.com").unwrap(), &Count)
|
||||
.unwrap();
|
||||
assert_eq!(to_hits, 1, "to_text should survive tag update");
|
||||
|
||||
@@ -1969,10 +2093,7 @@ mod tests {
|
||||
let tags_hits = searcher
|
||||
.search(
|
||||
&TermQuery::new(
|
||||
Term::from_facet(
|
||||
f.f_tags,
|
||||
&Facet::from_text("/important").unwrap(),
|
||||
),
|
||||
Term::from_facet(f.f_tags, &Facet::from_text("/important").unwrap()),
|
||||
IndexRecordOption::Basic,
|
||||
),
|
||||
&Count,
|
||||
@@ -1984,19 +2105,13 @@ mod tests {
|
||||
let old_tag_hits = searcher
|
||||
.search(
|
||||
&TermQuery::new(
|
||||
Term::from_facet(
|
||||
f.f_tags,
|
||||
&Facet::from_text("/unread").unwrap(),
|
||||
),
|
||||
Term::from_facet(f.f_tags, &Facet::from_text("/unread").unwrap()),
|
||||
IndexRecordOption::Basic,
|
||||
),
|
||||
&Count,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
old_tag_hits, 0,
|
||||
"old tag /unread should have been removed"
|
||||
);
|
||||
assert_eq!(old_tag_hits, 0, "old tag /unread should have been removed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2056,4 +2171,362 @@ mod tests {
|
||||
"body should be absent when EML is missing"
|
||||
);
|
||||
}
|
||||
|
||||
// ── get_message_ids_for_mailbox ─────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn get_message_ids_returns_stored_ids() {
|
||||
let f = SchemaTools::email_fields();
|
||||
let index = Index::create_in_ram(SchemaTools::email_schema());
|
||||
index.tokenizers().register("euro", EuroTokenizer::new());
|
||||
|
||||
// Insert two docs for mailbox 10, one for mailbox 20
|
||||
{
|
||||
let mut writer = index
|
||||
.writer_with_num_threads(1, 15_000_000)
|
||||
.expect("writer");
|
||||
|
||||
let mut doc1 = TantivyDocument::new();
|
||||
doc1.add_u64(f.f_account_id, 1);
|
||||
doc1.add_u64(f.f_mailbox_id, 10);
|
||||
doc1.add_text(f.f_message_id, "<msg-a@test>");
|
||||
doc1.add_text(f.f_id, "id-a");
|
||||
doc1.add_u64(f.f_uid, 1);
|
||||
doc1.add_text(f.f_content_hash, "hash-a");
|
||||
writer.add_document(doc1).unwrap();
|
||||
|
||||
let mut doc2 = TantivyDocument::new();
|
||||
doc2.add_u64(f.f_account_id, 1);
|
||||
doc2.add_u64(f.f_mailbox_id, 10);
|
||||
doc2.add_text(f.f_message_id, "<msg-b@test>");
|
||||
doc2.add_text(f.f_id, "id-b");
|
||||
doc2.add_u64(f.f_uid, 2);
|
||||
doc2.add_text(f.f_content_hash, "hash-b");
|
||||
writer.add_document(doc2).unwrap();
|
||||
|
||||
let mut doc3 = TantivyDocument::new();
|
||||
doc3.add_u64(f.f_account_id, 1);
|
||||
doc3.add_u64(f.f_mailbox_id, 20);
|
||||
doc3.add_text(f.f_message_id, "<msg-c@test>");
|
||||
doc3.add_text(f.f_id, "id-c");
|
||||
doc3.add_u64(f.f_uid, 3);
|
||||
doc3.add_text(f.f_content_hash, "hash-c");
|
||||
writer.add_document(doc3).unwrap();
|
||||
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
let reader = index.reader().unwrap();
|
||||
reader.reload().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
|
||||
// We can't easily call ENVELOPE_MANAGER.get_message_ids_for_mailbox
|
||||
// because it reads from ENVELOPE_MANAGER's own index, not our in-memory one.
|
||||
// Instead, test the query pattern directly.
|
||||
let query: Box<dyn Query> = {
|
||||
let account_query = TermQuery::new(
|
||||
Term::from_field_u64(f.f_account_id, 1),
|
||||
IndexRecordOption::Basic,
|
||||
);
|
||||
let mailbox_query = TermQuery::new(
|
||||
Term::from_field_u64(f.f_mailbox_id, 10),
|
||||
IndexRecordOption::Basic,
|
||||
);
|
||||
Box::new(BooleanQuery::new(vec![
|
||||
(Occur::Must, Box::new(account_query)),
|
||||
(Occur::Must, Box::new(mailbox_query)),
|
||||
]))
|
||||
};
|
||||
|
||||
let docs = searcher
|
||||
.search(&query, &DocSetCollector)
|
||||
.unwrap();
|
||||
|
||||
let mut ids: Vec<String> = Vec::new();
|
||||
for addr in docs {
|
||||
let doc: TantivyDocument = searcher.doc(addr).unwrap();
|
||||
if let Some(v) = doc.get_first(f.f_message_id) {
|
||||
if let Some(s) = v.as_str() {
|
||||
ids.push(s.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
ids.sort();
|
||||
|
||||
assert_eq!(ids, vec!["<msg-a@test>", "<msg-b@test>"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_message_ids_empty_mailbox_returns_empty() {
|
||||
let f = SchemaTools::email_fields();
|
||||
let index = Index::create_in_ram(SchemaTools::email_schema());
|
||||
index.tokenizers().register("euro", EuroTokenizer::new());
|
||||
|
||||
{
|
||||
let mut writer = index
|
||||
.writer_with_num_threads(1, 15_000_000)
|
||||
.expect("writer");
|
||||
|
||||
// Doc for a different mailbox
|
||||
let mut doc = TantivyDocument::new();
|
||||
doc.add_u64(f.f_account_id, 1);
|
||||
doc.add_u64(f.f_mailbox_id, 99);
|
||||
doc.add_text(f.f_message_id, "<other@test>");
|
||||
doc.add_text(f.f_id, "id-other");
|
||||
doc.add_u64(f.f_uid, 1);
|
||||
doc.add_text(f.f_content_hash, "hash-other");
|
||||
writer.add_document(doc).unwrap();
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
let reader = index.reader().unwrap();
|
||||
reader.reload().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
|
||||
let query: Box<dyn Query> = {
|
||||
let account_query = TermQuery::new(
|
||||
Term::from_field_u64(f.f_account_id, 1),
|
||||
IndexRecordOption::Basic,
|
||||
);
|
||||
let mailbox_query = TermQuery::new(
|
||||
Term::from_field_u64(f.f_mailbox_id, 10),
|
||||
IndexRecordOption::Basic,
|
||||
);
|
||||
Box::new(BooleanQuery::new(vec![
|
||||
(Occur::Must, Box::new(account_query)),
|
||||
(Occur::Must, Box::new(mailbox_query)),
|
||||
]))
|
||||
};
|
||||
|
||||
let docs = searcher.search(&query, &DocSetCollector).unwrap();
|
||||
assert!(docs.is_empty());
|
||||
}
|
||||
|
||||
// ── mailbox_contains_message_id ───────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn mailbox_contains_message_id_finds_existing() {
|
||||
let f = SchemaTools::email_fields();
|
||||
let index = Index::create_in_ram(SchemaTools::email_schema());
|
||||
index.tokenizers().register("euro", EuroTokenizer::new());
|
||||
|
||||
{
|
||||
let mut writer = index
|
||||
.writer_with_num_threads(1, 15_000_000)
|
||||
.expect("writer");
|
||||
|
||||
let mut doc = TantivyDocument::new();
|
||||
doc.add_u64(f.f_account_id, 1);
|
||||
doc.add_u64(f.f_mailbox_id, 10);
|
||||
doc.add_text(f.f_message_id, "abc@example.com");
|
||||
doc.add_text(f.f_id, "id-1");
|
||||
doc.add_u64(f.f_uid, 1);
|
||||
doc.add_text(f.f_content_hash, "hash-1");
|
||||
writer.add_document(doc).unwrap();
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
// We test the query pattern directly (can't call ENVELOPE_MANAGER
|
||||
// which uses a different index).
|
||||
let reader = index.reader().unwrap();
|
||||
reader.reload().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
|
||||
let query = BooleanQuery::new(vec![
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_account_id, 1),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_mailbox_id, 10),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_text(f.f_message_id, "abc@example.com"),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
]);
|
||||
|
||||
let count = searcher.search(&query, &Count).unwrap();
|
||||
assert_eq!(count, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mailbox_contains_message_id_returns_zero_for_missing() {
|
||||
let f = SchemaTools::email_fields();
|
||||
let index = Index::create_in_ram(SchemaTools::email_schema());
|
||||
index.tokenizers().register("euro", EuroTokenizer::new());
|
||||
|
||||
{
|
||||
let mut writer = index
|
||||
.writer_with_num_threads(1, 15_000_000)
|
||||
.expect("writer");
|
||||
|
||||
let mut doc = TantivyDocument::new();
|
||||
doc.add_u64(f.f_account_id, 1);
|
||||
doc.add_u64(f.f_mailbox_id, 10);
|
||||
doc.add_text(f.f_message_id, "existing@example.com");
|
||||
doc.add_text(f.f_id, "id-1");
|
||||
doc.add_u64(f.f_uid, 1);
|
||||
doc.add_text(f.f_content_hash, "hash-1");
|
||||
writer.add_document(doc).unwrap();
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
let reader = index.reader().unwrap();
|
||||
reader.reload().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
|
||||
let query = BooleanQuery::new(vec![
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_account_id, 1),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_mailbox_id, 10),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_text(f.f_message_id, "nonexistent@example.com"),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
]);
|
||||
|
||||
let count = searcher.search(&query, &Count).unwrap();
|
||||
assert_eq!(count, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mailbox_contains_message_id_respects_mailbox_boundary() {
|
||||
let f = SchemaTools::email_fields();
|
||||
let index = Index::create_in_ram(SchemaTools::email_schema());
|
||||
index.tokenizers().register("euro", EuroTokenizer::new());
|
||||
|
||||
{
|
||||
let mut writer = index
|
||||
.writer_with_num_threads(1, 15_000_000)
|
||||
.expect("writer");
|
||||
|
||||
// Same Message-ID in mailbox 10
|
||||
let mut doc1 = TantivyDocument::new();
|
||||
doc1.add_u64(f.f_account_id, 1);
|
||||
doc1.add_u64(f.f_mailbox_id, 10);
|
||||
doc1.add_text(f.f_message_id, "shared@example.com");
|
||||
doc1.add_text(f.f_id, "id-1");
|
||||
doc1.add_u64(f.f_uid, 1);
|
||||
doc1.add_text(f.f_content_hash, "hash-1");
|
||||
writer.add_document(doc1).unwrap();
|
||||
|
||||
// Same Message-ID in mailbox 20 (different mailbox)
|
||||
let mut doc2 = TantivyDocument::new();
|
||||
doc2.add_u64(f.f_account_id, 1);
|
||||
doc2.add_u64(f.f_mailbox_id, 20);
|
||||
doc2.add_text(f.f_message_id, "shared@example.com");
|
||||
doc2.add_text(f.f_id, "id-2");
|
||||
doc2.add_u64(f.f_uid, 2);
|
||||
doc2.add_text(f.f_content_hash, "hash-2");
|
||||
writer.add_document(doc2).unwrap();
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
let reader = index.reader().unwrap();
|
||||
reader.reload().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
|
||||
// Query mailbox 10: should find 1
|
||||
let q10 = BooleanQuery::new(vec![
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_account_id, 1),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_mailbox_id, 10),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_text(f.f_message_id, "shared@example.com"),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
]);
|
||||
assert_eq!(searcher.search(&q10, &Count).unwrap(), 1);
|
||||
|
||||
// Query mailbox 20: should find 1
|
||||
let q20 = BooleanQuery::new(vec![
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_account_id, 1),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_mailbox_id, 20),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_text(f.f_message_id, "shared@example.com"),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
]);
|
||||
assert_eq!(searcher.search(&q20, &Count).unwrap(), 1);
|
||||
|
||||
// Query mailbox 99 (no docs): should find 0
|
||||
let q99 = BooleanQuery::new(vec![
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_account_id, 1),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_mailbox_id, 99),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_text(f.f_message_id, "shared@example.com"),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
]);
|
||||
assert_eq!(searcher.search(&q99, &Count).unwrap(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,7 +155,8 @@ impl EnvelopeWithAttachments {
|
||||
id: extract_string_field(doc, fields.f_id, F_ID)?,
|
||||
message_id: extract_string_field(doc, fields.f_message_id, F_MESSAGE_ID)?,
|
||||
account_id,
|
||||
account_email: Some(account.email),
|
||||
account_email: Some(account.email), //https://github.com/rustmailer/bichon/issues/306
|
||||
account_name: account.account_name,
|
||||
mailbox_id,
|
||||
mailbox_name: Some(mailbox.name),
|
||||
uid: extract_u64_field(doc, fields.f_uid, F_UID)? as u32,
|
||||
|
||||
@@ -156,10 +156,16 @@ impl AccessTokenModel {
|
||||
})?;
|
||||
|
||||
if matches!(token_model.token_type, TokenType::WebUI) {
|
||||
let life = utc_now!() - token_model.created_at;
|
||||
let max_life = SETTINGS.bichon_webui_token_expiration_hours * 60 * 60 * 1000;
|
||||
// Use last_access_at if set, otherwise fall back to created_at
|
||||
let last_active = if token_model.last_access_at > 0 {
|
||||
token_model.last_access_at
|
||||
} else {
|
||||
token_model.created_at
|
||||
};
|
||||
let idle = utc_now!() - last_active;
|
||||
let max_life = SETTINGS.bichon_webui_token_expiration_hours as i64 * 60 * 60 * 1000;
|
||||
|
||||
if life > (max_life as i64) {
|
||||
if idle > max_life {
|
||||
return Err(raise_error!(
|
||||
"Permission denied: the WebUI token has expired.".into(),
|
||||
ErrorCode::PermissionDenied
|
||||
@@ -176,13 +182,15 @@ impl AccessTokenModel {
|
||||
));
|
||||
}
|
||||
}
|
||||
update_impl(DB_MANAGER.db(), &token_str, |current: AccessTokenModel| {
|
||||
let mut updated = current.clone();
|
||||
updated.last_access_at = utc_now!();
|
||||
Ok(updated)
|
||||
})?;
|
||||
}
|
||||
|
||||
// Update last_access_at on every successful use for both token types
|
||||
update_impl(DB_MANAGER.db(), &token_str, |current: AccessTokenModel| {
|
||||
let mut updated = current.clone();
|
||||
updated.last_access_at = utc_now!();
|
||||
Ok(updated)
|
||||
})?;
|
||||
|
||||
let user = UserModel::find(token_model.user_id)
|
||||
?
|
||||
.ok_or_else(|| raise_error!("The user associated with this access token does not exist or may have been deleted.".into(), ErrorCode::ResourceNotFound))?;
|
||||
|
||||
@@ -87,6 +87,11 @@ pub struct BichonUserV2 {
|
||||
|
||||
pub theme: Option<String>,
|
||||
pub language: Option<String>,
|
||||
|
||||
/// SSO identity: unique subject ID from the external IdP (e.g. OIDC `sub` claim).
|
||||
pub sso_id: Option<String>,
|
||||
/// SSO provider identifier: `"oidc"` or future `"saml"` / `"ldap"`.
|
||||
pub sso_provider: Option<String>,
|
||||
}
|
||||
|
||||
impl MemDbModel for BichonUserV2 {
|
||||
@@ -192,6 +197,8 @@ impl BichonUserV2 {
|
||||
global_permissions,
|
||||
theme: self.theme,
|
||||
language: self.language,
|
||||
sso_id: self.sso_id,
|
||||
sso_provider: self.sso_provider,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -226,6 +233,8 @@ impl BichonUserV2 {
|
||||
acl: None,
|
||||
theme: None,
|
||||
language: None,
|
||||
sso_id: None,
|
||||
sso_provider: None,
|
||||
};
|
||||
|
||||
// 3. Generate and insert an initial access token for the first-time setup
|
||||
@@ -382,6 +391,8 @@ impl BichonUserV2 {
|
||||
account_access_map: request.account_access_map,
|
||||
theme: request.theme,
|
||||
language: request.language,
|
||||
sso_id: None,
|
||||
sso_provider: None,
|
||||
};
|
||||
|
||||
let user_clone = user.clone();
|
||||
|
||||
@@ -52,4 +52,9 @@ pub struct UserView {
|
||||
pub acl: Option<AccessControl>,
|
||||
pub theme: Option<String>,
|
||||
pub language: Option<String>,
|
||||
|
||||
/// SSO identity: unique subject ID from the external IdP (e.g. OIDC `sub` claim).
|
||||
pub sso_id: Option<String>,
|
||||
/// SSO provider identifier: `"oidc"` or future `"saml"` / `"ldap"`.
|
||||
pub sso_provider: Option<String>,
|
||||
}
|
||||
|
||||
@@ -138,7 +138,7 @@ macro_rules! generate_token {
|
||||
}};
|
||||
}
|
||||
|
||||
pub(crate) fn generate_token_impl(bit_strength: usize) -> String {
|
||||
pub fn generate_token_impl(bit_strength: usize) -> String {
|
||||
let byte_length = (bit_strength + 23) / 24 * 3;
|
||||
let random_bytes: Vec<u8> = (0..byte_length).map(|_| rand::random::<u8>()).collect();
|
||||
let mut encoded = general_purpose::URL_SAFE.encode(&random_bytes);
|
||||
|
||||
@@ -32,6 +32,15 @@ use tracing::error;
|
||||
|
||||
pub(crate) const TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Parsed proxy address components.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ProxyAddr {
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub username: Option<String>,
|
||||
pub password: Option<String>,
|
||||
}
|
||||
|
||||
pub(crate) async fn establish_tcp_connection_with_timeout(
|
||||
address: SocketAddr,
|
||||
use_proxy: Option<u64>,
|
||||
@@ -66,20 +75,27 @@ pub async fn establish_tls_connection(
|
||||
Ok(tls_stream)
|
||||
}
|
||||
|
||||
pub fn parse_proxy_addr(input: &str) -> BichonResult<SocketAddr> {
|
||||
// Normalize and check protocol prefix
|
||||
let (scheme, stripped) = if let Some(rest) = input
|
||||
/// Parse a proxy URL into its components.
|
||||
///
|
||||
/// Supports two formats:
|
||||
/// - **Standard**: `[scheme://][user:pass@]host:port`
|
||||
/// - **Non-standard** (some proxy providers): `[scheme://]host:port:username:password`
|
||||
///
|
||||
/// The distinguishing feature is the `@` sign in the standard format.
|
||||
pub fn parse_proxy_url(input: &str) -> BichonResult<ProxyAddr> {
|
||||
// Normalize and strip scheme prefix
|
||||
let stripped = if let Some(rest) = input
|
||||
.strip_prefix("socks5://")
|
||||
.or_else(|| input.strip_prefix("SOCKS5://"))
|
||||
.or_else(|| input.strip_prefix("Socks5://"))
|
||||
{
|
||||
("socks5", rest)
|
||||
rest
|
||||
} else if let Some(rest) = input
|
||||
.strip_prefix("http://")
|
||||
.or_else(|| input.strip_prefix("HTTP://"))
|
||||
.or_else(|| input.strip_prefix("Http://"))
|
||||
{
|
||||
("http", rest)
|
||||
rest
|
||||
} else {
|
||||
return Err(raise_error!(
|
||||
format!(
|
||||
@@ -90,43 +106,207 @@ pub fn parse_proxy_addr(input: &str) -> BichonResult<SocketAddr> {
|
||||
));
|
||||
};
|
||||
|
||||
// Parse the remaining address
|
||||
let addr = stripped.parse::<SocketAddr>().map_err(|e| {
|
||||
raise_error!(
|
||||
if stripped.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Proxy URL has empty address after scheme.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
|
||||
// Check for standard format: user:pass@host:port
|
||||
if let Some(at_pos) = stripped.rfind('@') {
|
||||
let userinfo = &stripped[..at_pos];
|
||||
let hostport = &stripped[at_pos + 1..];
|
||||
|
||||
let (username, password) = split_userinfo(userinfo)?;
|
||||
let (host, port) = split_hostport(hostport)?;
|
||||
|
||||
return Ok(ProxyAddr {
|
||||
host,
|
||||
port,
|
||||
username,
|
||||
password,
|
||||
});
|
||||
}
|
||||
|
||||
// No '@' — check for non-standard format: host:port:user:pass
|
||||
let parts: Vec<&str> = stripped.rsplitn(4, ':').collect::<Vec<_>>().into_iter().rev().collect::<Vec<_>>();
|
||||
|
||||
match parts.len() {
|
||||
2 => {
|
||||
// host:port, no auth
|
||||
let (host, port) = split_hostport(stripped)?;
|
||||
Ok(ProxyAddr {
|
||||
host,
|
||||
port,
|
||||
username: None,
|
||||
password: None,
|
||||
})
|
||||
}
|
||||
4 => {
|
||||
// Non-standard: host:port:username:password
|
||||
let host = parts[0].to_string();
|
||||
let port = parts[1]
|
||||
.parse::<u16>()
|
||||
.map_err(|_| {
|
||||
raise_error!(
|
||||
format!("Invalid port '{}' in proxy URL.", parts[1]),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
let username = parts[2].to_string();
|
||||
let password = parts[3].to_string();
|
||||
|
||||
if host.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Empty hostname in proxy URL.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
if username.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Empty username in proxy URL.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
if password.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Empty password in proxy URL.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
|
||||
Ok(ProxyAddr {
|
||||
host,
|
||||
port,
|
||||
username: Some(username),
|
||||
password: Some(password),
|
||||
})
|
||||
}
|
||||
_ => Err(raise_error!(
|
||||
format!(
|
||||
"Failed to parse {} proxy address '{}': {}",
|
||||
scheme, stripped, e
|
||||
"Invalid proxy URL format '{}'. Expected '[scheme://][user:pass@]host:port' or 'scheme://host:port:user:pass'.",
|
||||
input
|
||||
),
|
||||
ErrorCode::InvalidParameter
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split "user:pass" into (Some(user), Some(pass)), or "user" into (Some(user), None).
|
||||
fn split_userinfo(userinfo: &str) -> BichonResult<(Option<String>, Option<String>)> {
|
||||
if userinfo.is_empty() {
|
||||
return Ok((None, None));
|
||||
}
|
||||
if let Some(colon_pos) = userinfo.find(':') {
|
||||
let user = &userinfo[..colon_pos];
|
||||
let pass = &userinfo[colon_pos + 1..];
|
||||
if user.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Empty username in proxy URL credentials.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
Ok((Some(user.to_string()), Some(pass.to_string())))
|
||||
} else {
|
||||
Ok((Some(userinfo.to_string()), None))
|
||||
}
|
||||
}
|
||||
|
||||
/// Split "host:port" into (host, port). Handles IPv6 addresses in brackets.
|
||||
fn split_hostport(hostport: &str) -> BichonResult<(String, u16)> {
|
||||
if hostport.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Empty host:port in proxy URL.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
|
||||
// IPv6: [::1]:1080
|
||||
if hostport.starts_with('[') {
|
||||
let close_bracket = hostport.find(']').ok_or_else(|| {
|
||||
raise_error!(
|
||||
format!("Invalid IPv6 address in proxy URL: '{}'.", hostport),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
let host = hostport[1..close_bracket].to_string();
|
||||
let after_bracket = &hostport[close_bracket + 1..];
|
||||
if !after_bracket.starts_with(':') {
|
||||
return Err(raise_error!(
|
||||
format!("Missing port after IPv6 address in proxy URL: '{}'.", hostport),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
let port = after_bracket[1..].parse::<u16>().map_err(|_| {
|
||||
raise_error!(
|
||||
format!("Invalid port in proxy URL: '{}'.", hostport),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
return Ok((host, port));
|
||||
}
|
||||
|
||||
// hostname:port or ip:port — split from right
|
||||
let last_colon = hostport.rfind(':').ok_or_else(|| {
|
||||
raise_error!(
|
||||
format!("Missing port in proxy URL: '{}'.", hostport),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
let host = hostport[..last_colon].to_string();
|
||||
let port = hostport[last_colon + 1..].parse::<u16>().map_err(|_| {
|
||||
raise_error!(
|
||||
format!("Invalid port in proxy URL: '{}'.", hostport),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(addr)
|
||||
if host.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Empty hostname in proxy URL.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
|
||||
Ok((host, port))
|
||||
}
|
||||
|
||||
/// Try to connect via SOCKS5 proxy or TCP with timeout
|
||||
/// Try to connect via SOCKS5 proxy or TCP with timeout.
|
||||
async fn connect_with_optional_proxy(
|
||||
use_proxy: Option<u64>,
|
||||
address: SocketAddr,
|
||||
) -> BichonResult<TcpStream> {
|
||||
// Try if proxy is enabled
|
||||
if let Some(proxy_id) = use_proxy {
|
||||
let proxy = Proxy::get(proxy_id)?;
|
||||
let proxy = parse_proxy_addr(&proxy.url)?;
|
||||
return timeout(TIMEOUT, Socks5Stream::connect(proxy, address))
|
||||
let addr = parse_proxy_url(&proxy.url)?;
|
||||
let proxy_addr = (addr.host.as_str(), addr.port);
|
||||
|
||||
let result = if let (Some(ref user), Some(ref pass)) = (addr.username, addr.password) {
|
||||
timeout(
|
||||
TIMEOUT,
|
||||
Socks5Stream::connect_with_password(proxy_addr, address, user.as_str(), pass.as_str()),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
timeout(TIMEOUT, Socks5Stream::connect(proxy_addr, address)).await
|
||||
};
|
||||
|
||||
return result
|
||||
.map_err(|_| {
|
||||
error!(
|
||||
"SOCKS5 proxy connection to {} via {} timed out after {}s",
|
||||
"SOCKS5 proxy connection to {} via {}:{} timed out after {}s",
|
||||
address,
|
||||
proxy,
|
||||
addr.host,
|
||||
addr.port,
|
||||
TIMEOUT.as_secs()
|
||||
);
|
||||
raise_error!(
|
||||
format!(
|
||||
"SOCKS5 proxy connection to {} via {} timed out after {}s",
|
||||
"SOCKS5 proxy connection to {} via {}:{} timed out after {}s",
|
||||
address,
|
||||
proxy,
|
||||
addr.host,
|
||||
addr.port,
|
||||
TIMEOUT.as_secs()
|
||||
),
|
||||
ErrorCode::ConnectionTimeout
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[package]
|
||||
name = "memdb"
|
||||
name = "bichon-memdb"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use memdb::{DbError, MemDb, Page};
|
||||
use bichon_memdb::{DbError, MemDb, Page};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tempfile::TempDir;
|
||||
|
||||
@@ -458,7 +458,7 @@ fn test_wal_seq_skips_already_snapshotted_entries() {
|
||||
|
||||
// Verify WAL only contains seq=3.
|
||||
let wal_path = dir.path().join("wal.jsonl");
|
||||
let entries = memdb::wal::read_after(&wal_path, 2).unwrap();
|
||||
let entries = bichon_memdb::wal::read_after(&wal_path, 2).unwrap();
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].seq, 3);
|
||||
|
||||
@@ -696,7 +696,7 @@ async fn test_concurrent_writes_wal_seq_monotonic() {
|
||||
|
||||
// Verify WAL seq is strictly monotonic.
|
||||
let wal_path = dir.path().join("wal.jsonl");
|
||||
let entries = memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
assert_eq!(entries.len(), 50);
|
||||
let mut last = 0u64;
|
||||
for e in &entries {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use memdb::{Durability, MemDb, Page};
|
||||
use bichon_memdb::{Durability, MemDb, Page};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -548,7 +548,7 @@ fn stress_wal_seq_monotonic_under_load() {
|
||||
}
|
||||
|
||||
let wal_path = dir.path().join("wal.jsonl");
|
||||
let entries = memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
assert_eq!(entries.len(), n);
|
||||
let mut last = 0u64;
|
||||
for e in &entries {
|
||||
@@ -842,7 +842,7 @@ async fn wal_concurrent_persistent_writes() {
|
||||
|
||||
// Verify strict seq ordering in WAL under concurrent load.
|
||||
let wal_path = db_path.join("wal.jsonl");
|
||||
let entries = memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
assert_eq!(entries.len(), total as usize);
|
||||
let mut last = 0u64;
|
||||
for e in &entries {
|
||||
@@ -1153,7 +1153,7 @@ fn wal_large_transaction_batch() {
|
||||
|
||||
// The entire transaction should be a single WAL entry.
|
||||
let wal_path = dir.path().join("wal.jsonl");
|
||||
let entries = memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].ops.len(), n as usize);
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ timeago.workspace = true
|
||||
chrono.workspace = true
|
||||
tracing.workspace = true
|
||||
tokio.workspace = true
|
||||
futures.workspace = true
|
||||
http.workspace = true
|
||||
urlencoding.workspace = true
|
||||
|
||||
|
||||
@@ -16,14 +16,33 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::common::auth::WrappedContext;
|
||||
use crate::rest::api::ApiTags;
|
||||
use crate::rest::ApiResult;
|
||||
use bichon_core::import::BatchEmlResult;
|
||||
use bichon_core::import::{BatchEmlRequest, ImportEmls};
|
||||
use bichon_core::account::migration::AccountModel;
|
||||
use bichon_core::database::manager::DB_MANAGER;
|
||||
use bichon_core::database::MemDbModel;
|
||||
use bichon_core::import::{
|
||||
check_temp_disk_space, get_import_progress, process_uploaded_file, update_progress,
|
||||
BatchEmlRequest, BatchEmlResult, ImportEmls, ImportHistory, ImportProgress, ImportStatus,
|
||||
MAX_WEB_EML_BYTES,
|
||||
};
|
||||
use bichon_core::import::history::{save_import_history, MAX_HISTORY_PER_USER};
|
||||
use bichon_core::raise_error;
|
||||
use bichon_core::error::code::ErrorCode;
|
||||
use bichon_core::settings::cli::SETTINGS;
|
||||
use bichon_core::settings::dir::DATA_DIR_MANAGER;
|
||||
use bichon_core::users::permissions::Permission;
|
||||
use poem_openapi::payload::Json;
|
||||
use bichon_core::import::detect_text_file;
|
||||
use bichon_core::import::FileFormat;
|
||||
use futures::StreamExt;
|
||||
use poem::Body;
|
||||
use poem_openapi::param::{Path, Query};
|
||||
use poem_openapi::payload::{Json, Binary};
|
||||
use poem_openapi::OpenApi;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
pub struct ImportApi;
|
||||
|
||||
@@ -44,7 +63,362 @@ impl ImportApi {
|
||||
payload: Json<BatchEmlRequest>,
|
||||
context: WrappedContext,
|
||||
) -> ApiResult<Json<BatchEmlResult>> {
|
||||
context.require_permission(Some(payload.0.account_id), Permission::DATA_IMPORT_BATCH)?;
|
||||
Ok(Json(ImportEmls::do_import(payload.0).await?))
|
||||
let account_id = payload.0.account_id;
|
||||
let folder = payload.0.mail_folder.clone();
|
||||
context.require_permission(Some(account_id), Permission::DATA_IMPORT_BATCH)?;
|
||||
let result = ImportEmls::do_import(payload.0).await?;
|
||||
|
||||
// Save import history
|
||||
let progress = ImportProgress {
|
||||
import_id: format!(
|
||||
"batch_{:x}",
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_nanos()
|
||||
),
|
||||
status: if result.failed == 0 {
|
||||
ImportStatus::Completed
|
||||
} else if result.success == 0 {
|
||||
ImportStatus::Failed
|
||||
} else {
|
||||
ImportStatus::Completed
|
||||
},
|
||||
format: "eml".to_string(),
|
||||
total: result.total,
|
||||
success: result.success,
|
||||
duplicates: result.duplicates,
|
||||
failed: result.failed,
|
||||
failed_details: result.failed_details.clone(),
|
||||
};
|
||||
save_import_history(context.user.id, account_id, &folder, &progress);
|
||||
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
||||
/// Upload an EML or MBOX file for import into a NoSync account.
|
||||
///
|
||||
/// The file is sent as the raw request body. Both `account_id` and `mail_folder`
|
||||
/// must be provided as query parameters, along with the original `file_name` for
|
||||
/// extension validation.
|
||||
///
|
||||
/// Returns an `import_id` to poll for progress via `/import-progress/:import_id`.
|
||||
#[oai(path = "/upload-import", method = "post", operation_id = "upload_import")]
|
||||
async fn upload_import(
|
||||
&self,
|
||||
/// Target account ID (must be NoSync type).
|
||||
account_id: Query<u64>,
|
||||
/// Target mail folder name.
|
||||
mail_folder: Query<String>,
|
||||
/// Original file name, used for extension validation (e.g. "export.eml").
|
||||
file_name: Query<String>,
|
||||
/// The raw file bytes (.eml or .mbox).
|
||||
data: Binary<Body>,
|
||||
context: WrappedContext,
|
||||
) -> ApiResult<Json<ImportProgress>> {
|
||||
let account_id = account_id.0;
|
||||
context.require_permission(Some(account_id), Permission::DATA_IMPORT_BATCH)?;
|
||||
|
||||
// Basic account validation (fails fast)
|
||||
AccountModel::check_account_exists(account_id)?;
|
||||
|
||||
let folder = mail_folder.0.trim().to_string();
|
||||
if folder.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"mail_folder is required.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
))?;
|
||||
}
|
||||
|
||||
let file_name = file_name.0.trim().to_string();
|
||||
if file_name.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"file_name is required.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
))?;
|
||||
}
|
||||
|
||||
// Validate file extension
|
||||
let ext_lower = std::path::Path::new(&file_name)
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| e.to_lowercase())
|
||||
.unwrap_or_default();
|
||||
let is_mbox_ext = ext_lower == "mbox";
|
||||
let is_eml_ext = ext_lower == "eml";
|
||||
let is_pst_ext = ext_lower == "pst";
|
||||
if !is_mbox_ext && !is_eml_ext && !is_pst_ext {
|
||||
return Err(raise_error!(
|
||||
format!(
|
||||
"Unsupported file type '.{}'. Only .eml, .mbox and .pst files are allowed.",
|
||||
ext_lower
|
||||
),
|
||||
ErrorCode::InvalidParameter
|
||||
))?;
|
||||
}
|
||||
|
||||
// Check disk space (fail fast before streaming)
|
||||
let max_mbox = SETTINGS.bichon_web_mbox_upload_limit_mb as usize * 1024 * 1024;
|
||||
let max_pst = SETTINGS.bichon_web_pst_upload_limit_mb as usize * 1024 * 1024;
|
||||
let min_required = if is_mbox_ext {
|
||||
max_mbox
|
||||
} else if is_pst_ext {
|
||||
max_pst
|
||||
} else {
|
||||
MAX_WEB_EML_BYTES
|
||||
};
|
||||
let free = check_temp_disk_space()?;
|
||||
if free < min_required as u64 * 2 {
|
||||
let free_gb = free as f64 / 1024.0 / 1024.0 / 1024.0;
|
||||
let need_gb = (min_required as f64 * 2.0) / 1024.0 / 1024.0 / 1024.0;
|
||||
return Err(raise_error!(
|
||||
format!(
|
||||
"Insufficient disk space. Free: {:.1} GB. Need at least {:.1} GB.",
|
||||
free_gb, need_gb
|
||||
),
|
||||
ErrorCode::InvalidParameter
|
||||
))?;
|
||||
}
|
||||
|
||||
// Stream body to temp file, enforcing size limits and validating content
|
||||
let import_id = format!(
|
||||
"imp_{:x}",
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_nanos()
|
||||
);
|
||||
let temp_path = DATA_DIR_MANAGER.temp_dir.join(format!("import_{}.tmp", import_id));
|
||||
|
||||
let (format_detected, file_len) = stream_body_to_temp(
|
||||
data.0,
|
||||
&temp_path,
|
||||
is_mbox_ext,
|
||||
is_pst_ext,
|
||||
).await?;
|
||||
|
||||
let format = format_detected.unwrap_or_else(|| {
|
||||
if is_mbox_ext {
|
||||
FileFormat::Mbox
|
||||
} else if is_pst_ext {
|
||||
FileFormat::Pst
|
||||
} else {
|
||||
FileFormat::Eml
|
||||
}
|
||||
});
|
||||
|
||||
let format_str = match format {
|
||||
FileFormat::Mbox => "mbox".to_string(),
|
||||
FileFormat::Eml => "eml".to_string(),
|
||||
FileFormat::Pst => "pst".to_string(),
|
||||
};
|
||||
|
||||
let max_mbox = SETTINGS.bichon_web_mbox_upload_limit_mb as usize * 1024 * 1024;
|
||||
let max_pst = SETTINGS.bichon_web_pst_upload_limit_mb as usize * 1024 * 1024;
|
||||
let max_size = match format {
|
||||
FileFormat::Mbox => max_mbox,
|
||||
FileFormat::Pst => max_pst,
|
||||
FileFormat::Eml => MAX_WEB_EML_BYTES,
|
||||
};
|
||||
if file_len > max_size {
|
||||
let _ = std::fs::remove_file(&temp_path);
|
||||
let max_mb = max_size as f64 / 1024.0 / 1024.0;
|
||||
let actual_mb = file_len as f64 / 1024.0 / 1024.0;
|
||||
return Err(raise_error!(
|
||||
format!(
|
||||
"File too large ({:.1} MB). Maximum for {} is {:.0} MB. Use the CLI for larger files.",
|
||||
actual_mb, format_str.to_uppercase(), max_mb
|
||||
),
|
||||
ErrorCode::InvalidParameter
|
||||
))?;
|
||||
}
|
||||
|
||||
// Record initial progress
|
||||
let initial = ImportProgress {
|
||||
import_id: import_id.clone(),
|
||||
status: ImportStatus::Pending,
|
||||
format: format_str.clone(),
|
||||
total: 0,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![],
|
||||
};
|
||||
|
||||
// Store initial progress so polling can find it immediately
|
||||
update_progress(&import_id, initial.clone());
|
||||
|
||||
// Spawn background processing
|
||||
let id = import_id.clone();
|
||||
let folder_clone = folder.clone();
|
||||
let user_id = context.user.id;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
process_uploaded_file(&id, &temp_path, &file_name, account_id, &folder_clone, user_id);
|
||||
});
|
||||
|
||||
Ok(Json(initial))
|
||||
}
|
||||
|
||||
/// Poll import progress by import ID.
|
||||
#[oai(
|
||||
path = "/import-progress/:import_id",
|
||||
method = "get",
|
||||
operation_id = "get_import_progress"
|
||||
)]
|
||||
async fn get_import_progress(
|
||||
&self,
|
||||
import_id: Path<String>,
|
||||
context: WrappedContext,
|
||||
) -> ApiResult<Json<ImportProgress>> {
|
||||
let _ = context; // progress queries don't need per-account auth
|
||||
match get_import_progress(&import_id.0) {
|
||||
Some(progress) => Ok(Json(progress)),
|
||||
None => Err(raise_error!(
|
||||
format!("Import {} not found.", import_id.0),
|
||||
ErrorCode::ResourceNotFound
|
||||
))?,
|
||||
}
|
||||
}
|
||||
|
||||
/// Check available disk space on the server's temp directory.
|
||||
#[oai(
|
||||
path = "/check-disk-space",
|
||||
method = "get",
|
||||
operation_id = "check_disk_space"
|
||||
)]
|
||||
async fn check_disk_space(&self, _context: WrappedContext) -> ApiResult<Json<u64>> {
|
||||
let free = check_temp_disk_space()?;
|
||||
Ok(Json(free))
|
||||
}
|
||||
|
||||
/// List import history for the current user (latest first, up to 5 entries).
|
||||
#[oai(
|
||||
path = "/import-history",
|
||||
method = "get",
|
||||
operation_id = "list_import_history"
|
||||
)]
|
||||
async fn list_import_history(
|
||||
&self,
|
||||
context: WrappedContext,
|
||||
) -> ApiResult<Json<Vec<ImportHistory>>> {
|
||||
let prefix = format!("{}:", context.user.id);
|
||||
let coll = DB_MANAGER.db().collection(ImportHistory::collection());
|
||||
let mut entries: Vec<ImportHistory> = coll
|
||||
.scan_prefix(&prefix)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
// Sort by created_at descending (newest first), keep at most N per user
|
||||
entries.sort_by(|a, b| b.created_at.cmp(&a.created_at));
|
||||
entries.truncate(MAX_HISTORY_PER_USER);
|
||||
Ok(Json(entries))
|
||||
}
|
||||
}
|
||||
|
||||
/// Stream a poem `Body` to a temp file while enforcing size limits and
|
||||
/// validating that the content looks like a text-based email file.
|
||||
///
|
||||
/// PST files are binary (OLE2) — text detection is skipped for them.
|
||||
///
|
||||
/// Returns the detected format (if any) and the total bytes written.
|
||||
async fn stream_body_to_temp(
|
||||
body: Body,
|
||||
temp_path: &std::path::Path,
|
||||
is_mbox_ext: bool,
|
||||
is_pst_ext: bool,
|
||||
) -> ApiResult<(Option<FileFormat>, usize)> {
|
||||
let max_mbox = SETTINGS.bichon_web_mbox_upload_limit_mb as usize * 1024 * 1024;
|
||||
let max_pst = SETTINGS.bichon_web_pst_upload_limit_mb as usize * 1024 * 1024;
|
||||
let max_stream = if is_mbox_ext {
|
||||
max_mbox
|
||||
} else if is_pst_ext {
|
||||
max_pst
|
||||
} else {
|
||||
MAX_WEB_EML_BYTES
|
||||
};
|
||||
|
||||
let mut file = tokio::fs::File::create(temp_path).await.map_err(|e| {
|
||||
raise_error!(
|
||||
format!("Failed to create temp file: {}", e),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut body_stream = body.into_bytes_stream();
|
||||
let mut total: usize = 0;
|
||||
let mut first_chunk: Vec<u8> = Vec::new();
|
||||
let mut format_detected: Option<FileFormat> = None;
|
||||
let mut text_checked = false;
|
||||
|
||||
while let Some(chunk_result) = body_stream.next().await {
|
||||
let chunk = chunk_result.map_err(|e| {
|
||||
raise_error!(
|
||||
format!("Failed to read request body: {}", e),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
|
||||
total += chunk.len();
|
||||
|
||||
// Enforce size limit during streaming
|
||||
if total > max_stream {
|
||||
// Clean up partial temp file
|
||||
drop(file);
|
||||
let _ = tokio::fs::remove_file(temp_path).await;
|
||||
let max_mb = max_stream as f64 / 1024.0 / 1024.0;
|
||||
return Err(raise_error!(
|
||||
format!(
|
||||
"Upload exceeds maximum size of {:.0} MB. Use the CLI for larger files.",
|
||||
max_mb
|
||||
),
|
||||
ErrorCode::InvalidParameter
|
||||
))?;
|
||||
}
|
||||
|
||||
// Accumulate first ~8 KB for format & text detection
|
||||
if first_chunk.len() < 8192 {
|
||||
let remaining = 8192 - first_chunk.len();
|
||||
first_chunk.extend_from_slice(&chunk[..chunk.len().min(remaining)]);
|
||||
}
|
||||
|
||||
// Once we have enough data, validate format and text
|
||||
if first_chunk.len() >= 512 && !text_checked {
|
||||
text_checked = true;
|
||||
format_detected = bichon_core::import::detect_format(&first_chunk, "upload");
|
||||
|
||||
// If extension is .eml but content looks like MBOX (or vice versa), that's OK.
|
||||
// PST files are binary — skip text detection.
|
||||
if !is_pst_ext && !detect_text_file(&first_chunk) {
|
||||
drop(file);
|
||||
let _ = tokio::fs::remove_file(temp_path).await;
|
||||
return Err(raise_error!(
|
||||
"The uploaded file appears to be binary (not a valid email file). Only .eml, .mbox and .pst files are accepted.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
))?;
|
||||
}
|
||||
}
|
||||
|
||||
file.write_all(&chunk).await.map_err(|e| {
|
||||
raise_error!(
|
||||
format!("Failed to write temp file: {}", e),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
}
|
||||
|
||||
file.flush().await.map_err(|e| {
|
||||
raise_error!(
|
||||
format!("Failed to flush temp file: {}", e),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
|
||||
// If file is empty, reject
|
||||
if total == 0 {
|
||||
let _ = tokio::fs::remove_file(temp_path).await;
|
||||
return Err(raise_error!(
|
||||
"Empty file is not allowed.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
))?;
|
||||
}
|
||||
|
||||
Ok((format_detected, total))
|
||||
}
|
||||
|
||||
@@ -280,6 +280,33 @@ impl MessageApi {
|
||||
Ok(attachment)
|
||||
}
|
||||
|
||||
/// Returns raw attachment content for in-browser preview with
|
||||
/// `Content-Disposition: inline` and the correct MIME type.
|
||||
#[oai(
|
||||
path = "/preview-attachment/:account_id/:envelope_id",
|
||||
method = "get",
|
||||
operation_id = "preview_attachment"
|
||||
)]
|
||||
async fn preview_attachment(
|
||||
&self,
|
||||
/// The ID of the account.
|
||||
account_id: Path<u64>,
|
||||
/// The ID of the message containing the attachment.
|
||||
envelope_id: Path<String>,
|
||||
/// The content_hash of the attachment to preview.
|
||||
content_hash: Query<String>,
|
||||
context: WrappedContext,
|
||||
) -> ApiResult<Attachment<Body>> {
|
||||
let account_id = account_id.0;
|
||||
let envelope_id = envelope_id.0.trim().to_string();
|
||||
AccountModel::check_account_exists(account_id)?;
|
||||
context.require_permission(Some(account_id), Permission::DATA_READ)?;
|
||||
let content_hash = content_hash.0.trim();
|
||||
let reader = retrieve_attachment_content(account_id, envelope_id, content_hash)?;
|
||||
let body = Body::from_async_read(reader);
|
||||
Ok(Attachment::new(body).attachment_type(AttachmentType::Inline))
|
||||
}
|
||||
|
||||
/// Downloads an attachment from within a nested email (EML file).
|
||||
#[oai(
|
||||
path = "/download-nested-attachment/:account_id/:envelope_id",
|
||||
|
||||
@@ -21,6 +21,7 @@ use std::net::SocketAddr;
|
||||
use std::time::Duration;
|
||||
|
||||
use base64::{prelude::BASE64_STANDARD, Engine as _};
|
||||
use bichon_core::account::migration::AccountType;
|
||||
use bichon_core::cache::imap::mailbox::{Attribute, AttributeEnum};
|
||||
use bichon_core::common::signal::SIGNAL_MANAGER;
|
||||
use bichon_core::envelope::extractor::extract_envelope_from_smtp;
|
||||
@@ -429,8 +430,20 @@ where
|
||||
}
|
||||
|
||||
if is_allowed {
|
||||
session.rcpt_to.push(account);
|
||||
stream.write_all(b"250 OK\r\n").await?;
|
||||
if !matches!(account.account_type, AccountType::NoSync) {
|
||||
tracing::warn!(
|
||||
"SMTP: Rejected journaling attempt to IMAP account <{}>",
|
||||
addr
|
||||
);
|
||||
let err = format!(
|
||||
"550 5.7.1 <{}>: Not a Bichon local account, journaling is not supported\r\n",
|
||||
account.email
|
||||
);
|
||||
stream.write_all(err.as_bytes()).await?;
|
||||
} else {
|
||||
session.rcpt_to.push(account);
|
||||
stream.write_all(b"250 OK\r\n").await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
@@ -615,26 +628,40 @@ async fn parse_email(data: &[u8], session: &Session) -> BichonResult<()> {
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let mailbox = MailBox {
|
||||
id: create_hash(rcpt.id, "INBOX"),
|
||||
account_id: rcpt.id,
|
||||
name: "INBOX".into(),
|
||||
delimiter: Some("/".to_string()),
|
||||
attributes: vec![Attribute {
|
||||
attr: AttributeEnum::Extension,
|
||||
extension: Some("CreatedByBichon".into()),
|
||||
}],
|
||||
exists: 0,
|
||||
unseen: None,
|
||||
uid_next: None,
|
||||
uid_validity: None,
|
||||
highest_uid: None,
|
||||
};
|
||||
let mailbox_id = mailbox.id;
|
||||
let mailbox_id = create_hash(rcpt.id, "INBOX");
|
||||
|
||||
if let Err(e) = MailBox::batch_upsert(&[mailbox]) {
|
||||
tracing::error!("SMTP: Failed to upsert mailbox for {}: {:?}", rcpt.email, e);
|
||||
return Err(e.into());
|
||||
// The INBOX row is owned by the IMAP sync, which maintains `uid_validity`,
|
||||
// `highest_uid` and `uid_next` on it. `batch_upsert` replaces the *whole*
|
||||
// row, so blindly upserting here (with those fields = None) clobbers the
|
||||
// IMAP-maintained state back to None. The next reconcile then sees
|
||||
// `uid_validity` change from Some -> None, treats the mailbox as invalid,
|
||||
// and wipes + rebuilds it — silently losing the local copy of a large
|
||||
// mailbox when that rebuild is interrupted (see #297).
|
||||
//
|
||||
// We only need the row to *exist* so the journaled envelope can attach to
|
||||
// it, so create it only when it is missing and otherwise leave the
|
||||
// IMAP-owned row untouched.
|
||||
if MailBox::find_mailbox(rcpt.id, mailbox_id)?.is_none() {
|
||||
let mailbox = MailBox {
|
||||
id: mailbox_id,
|
||||
account_id: rcpt.id,
|
||||
name: "INBOX".into(),
|
||||
delimiter: Some("/".to_string()),
|
||||
attributes: vec![Attribute {
|
||||
attr: AttributeEnum::Extension,
|
||||
extension: Some("CreatedByBichon".into()),
|
||||
}],
|
||||
exists: 0,
|
||||
unseen: None,
|
||||
uid_next: None,
|
||||
uid_validity: None,
|
||||
highest_uid: None,
|
||||
};
|
||||
|
||||
if let Err(e) = MailBox::batch_upsert(&[mailbox]) {
|
||||
tracing::error!("SMTP: Failed to upsert mailbox for {}: {:?}", rcpt.email, e);
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
|
||||
extract_envelope_from_smtp(data, rcpt.id, mailbox_id)
|
||||
|
||||
@@ -23,6 +23,7 @@ import { PaginatedResponse } from "..";
|
||||
export interface MinimalAccount {
|
||||
id: number;
|
||||
email: string;
|
||||
name?: string;
|
||||
}
|
||||
|
||||
export const minimal_account_list = async () => {
|
||||
|
||||
82
web/src/api/import/api.ts
Normal file
82
web/src/api/import/api.ts
Normal file
@@ -0,0 +1,82 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
|
||||
import axiosInstance from '@/api/axiosInstance';
|
||||
import { list_accounts } from '@/api/account/api';
|
||||
import type { AccountModel } from '@/api/account/api';
|
||||
|
||||
export interface ImportProgress {
|
||||
import_id: string;
|
||||
status: 'Pending' | 'Processing' | 'Completed' | 'Failed';
|
||||
format: string;
|
||||
total: number;
|
||||
success: number;
|
||||
duplicates: number;
|
||||
failed: number;
|
||||
failed_details: { index: number; error_message: string }[];
|
||||
}
|
||||
|
||||
export const upload_import = async (
|
||||
accountId: number,
|
||||
mailFolder: string,
|
||||
fileName: string,
|
||||
file: File,
|
||||
onProgress?: (pct: number) => void
|
||||
): Promise<ImportProgress> => {
|
||||
const response = await axiosInstance.post<ImportProgress>(
|
||||
`api/v1/upload-import`,
|
||||
file,
|
||||
{
|
||||
params: { account_id: accountId, mail_folder: mailFolder, file_name: fileName },
|
||||
headers: { 'Content-Type': 'application/octet-stream' },
|
||||
onUploadProgress: (e) => {
|
||||
if (e.total && onProgress) onProgress(Math.round((e.loaded / e.total) * 100));
|
||||
},
|
||||
}
|
||||
);
|
||||
return response.data;
|
||||
};
|
||||
|
||||
export const get_import_progress = async (importId: string): Promise<ImportProgress> => {
|
||||
const response = await axiosInstance.get<ImportProgress>(
|
||||
`api/v1/import-progress/${importId}`
|
||||
);
|
||||
return response.data;
|
||||
};
|
||||
|
||||
export const check_disk_space = async (): Promise<number> => {
|
||||
const response = await axiosInstance.get<number>('api/v1/check-disk-space');
|
||||
return response.data;
|
||||
};
|
||||
|
||||
export const get_nosync_accounts = async (): Promise<AccountModel[]> => {
|
||||
const data = await list_accounts();
|
||||
return (data.items || []).filter(
|
||||
(a) => a.account_type === 'NoSync' && a.enabled
|
||||
);
|
||||
};
|
||||
|
||||
// ── Import history ────────────────────────────────────────────────
|
||||
|
||||
export interface ImportHistory {
|
||||
id: string;
|
||||
user_id: number;
|
||||
import_id: string;
|
||||
account_id: number;
|
||||
folder: string;
|
||||
format: string;
|
||||
status: 'pending' | 'processing' | 'completed' | 'failed';
|
||||
total: number;
|
||||
success: number;
|
||||
duplicates: number;
|
||||
failed: number;
|
||||
failed_details: { index: number; error_message: string }[];
|
||||
created_at: number;
|
||||
}
|
||||
|
||||
export const list_import_history = async (): Promise<ImportHistory[]> => {
|
||||
const response = await axiosInstance.get<ImportHistory[]>('api/v1/import-history');
|
||||
return response.data;
|
||||
};
|
||||
@@ -32,6 +32,7 @@ export interface EmailEnvelope {
|
||||
account_id: number;
|
||||
mailbox_id: number;
|
||||
account_email: string;
|
||||
account_name?: string;
|
||||
mailbox_name: string;
|
||||
uid: number;
|
||||
subject: string;
|
||||
|
||||
@@ -41,6 +41,18 @@ export const download_attachment = async (accountId: number, id: string, content
|
||||
saveAs(blob, fileName);
|
||||
};
|
||||
|
||||
/** Fetch raw attachment content for in-browser preview (Content-Disposition: inline). */
|
||||
export const preview_attachment = async (accountId: number, id: string, content_hash: string) => {
|
||||
const response = await axiosInstance.get(
|
||||
`api/v1/preview-attachment/${accountId}/${id}`,
|
||||
{
|
||||
params: { content_hash },
|
||||
responseType: 'blob',
|
||||
}
|
||||
);
|
||||
return response.data as Blob;
|
||||
};
|
||||
|
||||
export const download_nested_attachment = async (accountId: number, id: string, content_hash: string, nested_content_hash: string, fileName: string) => {
|
||||
const response = await axiosInstance.get(`api/v1/download-nested-attachment/${accountId}/${id}?content_hash=${content_hash}&nested_content_hash=${nested_content_hash}`, { responseType: 'blob' });
|
||||
const blob = new Blob([response.data]);
|
||||
|
||||
@@ -138,6 +138,10 @@ export type ServerConfigurations = {
|
||||
bichon_smtp_auth_required: boolean
|
||||
bichon_smtp_tls_key_path?: string | null
|
||||
bichon_smtp_tls_cert_path?: string | null
|
||||
|
||||
bichon_upload_body_limit_mb: number
|
||||
bichon_web_mbox_upload_limit_mb: number
|
||||
bichon_web_pst_upload_limit_mb: number
|
||||
}
|
||||
|
||||
export const get_dashboard_stats = async () => {
|
||||
|
||||
@@ -22,7 +22,7 @@ import {
|
||||
IconLayoutDashboard,
|
||||
IconSettings
|
||||
} from '@tabler/icons-react'
|
||||
import { IdCard, Inbox, Paperclip, Search, Users2 } from 'lucide-react'
|
||||
import { IdCard, Inbox, Paperclip, Search, Upload, Users2 } from 'lucide-react'
|
||||
import { type SidebarData } from '../types'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useCurrentUser } from '@/hooks/use-current-user'
|
||||
@@ -57,6 +57,12 @@ export function useSidebarData(): SidebarData {
|
||||
url: '/search',
|
||||
icon: Search,
|
||||
},
|
||||
{
|
||||
title: t('import.title', 'Import'),
|
||||
url: '/import',
|
||||
icon: Upload,
|
||||
visible: require_any_permission(['data:import:batch']),
|
||||
},
|
||||
{
|
||||
title: t('navigation.attachment'),
|
||||
url: '/attachment',
|
||||
|
||||
@@ -30,8 +30,11 @@ DialogOverlay.displayName = DialogPrimitive.Overlay.displayName
|
||||
|
||||
const DialogContent = React.forwardRef<
|
||||
React.ElementRef<typeof DialogPrimitive.Content>,
|
||||
React.ComponentPropsWithoutRef<typeof DialogPrimitive.Content>
|
||||
>(({ className, children, ...props }, ref) => {
|
||||
React.ComponentPropsWithoutRef<typeof DialogPrimitive.Content> & {
|
||||
hideClose?: boolean;
|
||||
hideFullscreen?: boolean;
|
||||
}
|
||||
>(({ className, children, hideClose, hideFullscreen, ...props }, ref) => {
|
||||
const [isFullscreen, setIsFullscreen] = React.useState(false);
|
||||
return <DialogPortal>
|
||||
<DialogOverlay>
|
||||
@@ -45,17 +48,23 @@ const DialogContent = React.forwardRef<
|
||||
{...props}
|
||||
>
|
||||
{children}
|
||||
<div className='absolute right-4 top-4 flex items-center gap-2'>
|
||||
{isFullscreen ? (
|
||||
<Minimize onClick={() => setIsFullscreen(!isFullscreen)} className='rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground h-4 w-4' />
|
||||
) : (
|
||||
<Maximize onClick={() => setIsFullscreen(!isFullscreen)} className='rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground h-4 w-4' />
|
||||
)}
|
||||
<DialogPrimitive.Close className='rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground'>
|
||||
<X className='h-4 w-4' />
|
||||
<span className='sr-only'>Close</span>
|
||||
</DialogPrimitive.Close>
|
||||
</div>
|
||||
{(!hideClose || !hideFullscreen) && (
|
||||
<div className='absolute right-4 top-4 flex items-center gap-2'>
|
||||
{!hideFullscreen && (
|
||||
isFullscreen ? (
|
||||
<Minimize onClick={() => setIsFullscreen(!isFullscreen)} className='rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground h-4 w-4' />
|
||||
) : (
|
||||
<Maximize onClick={() => setIsFullscreen(!isFullscreen)} className='rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground h-4 w-4' />
|
||||
)
|
||||
)}
|
||||
{!hideClose && (
|
||||
<DialogPrimitive.Close className='rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground'>
|
||||
<X className='h-4 w-4' />
|
||||
<span className='sr-only'>Close</span>
|
||||
</DialogPrimitive.Close>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</DialogPrimitive.Content>
|
||||
</DialogOverlay>
|
||||
</DialogPortal>
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import { useState } from 'react'
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import {
|
||||
ColumnDef,
|
||||
ColumnFiltersState,
|
||||
@@ -64,7 +64,19 @@ export function AccountTable({ columns, data }: DataTableProps) {
|
||||
const [rowSelection, setRowSelection] = useState({})
|
||||
const [columnVisibility, setColumnVisibility] = useState<VisibilityState>({})
|
||||
const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([])
|
||||
const [sorting, setSorting] = useState<SortingState>([])
|
||||
const [sorting, setSorting] = useState<SortingState>(() => {
|
||||
const saved = localStorage.getItem('bichon_accounts_sorting');
|
||||
return saved ? JSON.parse(saved) : [];
|
||||
})
|
||||
|
||||
// Persist sorting state to localStorage
|
||||
const prevSortingRef = useRef(sorting);
|
||||
useEffect(() => {
|
||||
if (prevSortingRef.current !== sorting) {
|
||||
localStorage.setItem('bichon_accounts_sorting', JSON.stringify(sorting));
|
||||
prevSortingRef.current = sorting;
|
||||
}
|
||||
}, [sorting]);
|
||||
|
||||
const table = useReactTable({
|
||||
data,
|
||||
|
||||
399
web/src/features/attachment/attachment-preview.tsx
Normal file
399
web/src/features/attachment/attachment-preview.tsx
Normal file
@@ -0,0 +1,399 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import {
|
||||
Download, FileIcon, ZoomIn, ZoomOut, RotateCcw,
|
||||
ChevronLeft, ChevronRight, X,
|
||||
} from 'lucide-react';
|
||||
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Dialog, DialogContent } from '@/components/ui/dialog';
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';
|
||||
import { Separator } from '@/components/ui/separator';
|
||||
import { Skeleton } from '@/components/ui/skeleton';
|
||||
import { toast } from '@/hooks/use-toast';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
|
||||
import { preview_attachment, download_attachment } from '@/api/mailbox/envelope/api';
|
||||
import { getFileConfig } from './mail-message-view';
|
||||
|
||||
const PREVIEWABLE_IMAGE = /^image\/(png|jpeg|gif|webp|svg\+xml)$/;
|
||||
const PREVIEWABLE_TEXT = /^(text\/(plain|csv|html|xml|css|javascript|markdown)|application\/(json|xml|javascript|x-httpd-php|x-sh|x-perl|x-python|x-ruby))$/;
|
||||
|
||||
const EXT_TO_MIME: Record<string, string> = {
|
||||
'.pdf': 'application/pdf',
|
||||
'.png': 'image/png',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.gif': 'image/gif',
|
||||
'.webp': 'image/webp',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.txt': 'text/plain',
|
||||
'.csv': 'text/csv',
|
||||
'.html': 'text/html',
|
||||
'.htm': 'text/html',
|
||||
'.xml': 'application/xml',
|
||||
'.css': 'text/css',
|
||||
'.js': 'application/javascript',
|
||||
'.mjs': 'application/javascript',
|
||||
'.md': 'text/markdown',
|
||||
'.json': 'application/json',
|
||||
'.php': 'application/x-httpd-php',
|
||||
'.sh': 'application/x-sh',
|
||||
'.pl': 'application/x-perl',
|
||||
'.py': 'application/x-python',
|
||||
'.rb': 'application/x-ruby',
|
||||
};
|
||||
|
||||
function getExtensionMime(filename: string): string | null {
|
||||
const dot = filename.lastIndexOf('.');
|
||||
if (dot < 0) return null;
|
||||
const ext = filename.slice(dot).toLowerCase();
|
||||
return EXT_TO_MIME[ext] ?? null;
|
||||
}
|
||||
|
||||
function resolveContentType(contentType: string, fileName: string): string {
|
||||
if (contentType && contentType !== 'application/octet-stream') return contentType;
|
||||
return getExtensionMime(fileName) ?? contentType;
|
||||
}
|
||||
|
||||
export interface PreviewAttachment {
|
||||
content_hash: string;
|
||||
file_type: string;
|
||||
filename: string;
|
||||
}
|
||||
|
||||
interface AttachmentPreviewProps {
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
accountId: number;
|
||||
envelopeId: string;
|
||||
contentHash: string;
|
||||
contentType: string;
|
||||
fileName: string;
|
||||
/** Full attachment list for gallery navigation (optional). */
|
||||
attachments?: PreviewAttachment[];
|
||||
/** Index of the current attachment within `attachments`. */
|
||||
attachmentIndex?: number;
|
||||
}
|
||||
|
||||
function isImagePreview(contentType: string, fileName?: string) {
|
||||
return PREVIEWABLE_IMAGE.test(resolveContentType(contentType, fileName ?? ''));
|
||||
}
|
||||
|
||||
function isPdfPreview(contentType: string, fileName?: string) {
|
||||
return resolveContentType(contentType, fileName ?? '') === 'application/pdf';
|
||||
}
|
||||
|
||||
function isTextPreview(contentType: string, fileName?: string) {
|
||||
return PREVIEWABLE_TEXT.test(resolveContentType(contentType, fileName ?? ''));
|
||||
}
|
||||
|
||||
export default function AttachmentPreview({
|
||||
open,
|
||||
onOpenChange,
|
||||
accountId,
|
||||
envelopeId,
|
||||
contentHash,
|
||||
contentType,
|
||||
fileName,
|
||||
attachments,
|
||||
attachmentIndex,
|
||||
}: AttachmentPreviewProps) {
|
||||
const { t } = useTranslation();
|
||||
const [blobUrl, setBlobUrl] = useState<string | null>(null);
|
||||
const [textContent, setTextContent] = useState<string | null>(null);
|
||||
const [imageZoom, setImageZoom] = useState(1);
|
||||
|
||||
// ── Gallery state ──────────────────────────────────────────────
|
||||
// When attachments list is provided, compute image-only indices for navigation.
|
||||
const imageIndices = useMemo(() => {
|
||||
if (!attachments) return [];
|
||||
return attachments
|
||||
.map((a, i) => (isImagePreview(a.file_type, a.filename) ? i : -1))
|
||||
.filter((i) => i >= 0);
|
||||
}, [attachments]);
|
||||
|
||||
const [currentIndex, setCurrentIndex] = useState(attachmentIndex ?? 0);
|
||||
|
||||
// Reset to the clicked attachment every time the dialog opens.
|
||||
useEffect(() => {
|
||||
if (open) {
|
||||
setCurrentIndex(attachmentIndex ?? 0);
|
||||
}
|
||||
}, [open, attachmentIndex]);
|
||||
|
||||
// Resolve which attachment to display.
|
||||
const resolved = useMemo(() => {
|
||||
if (attachments && currentIndex < attachments.length) {
|
||||
const a = attachments[currentIndex];
|
||||
return {
|
||||
contentHash: a.content_hash,
|
||||
contentType: a.file_type,
|
||||
fileName: a.filename,
|
||||
};
|
||||
}
|
||||
return { contentHash, contentType, fileName };
|
||||
}, [attachments, currentIndex, contentHash, contentType, fileName]);
|
||||
|
||||
// Position within image-only list (for "3 / 12" counter).
|
||||
const imagePos = imageIndices.indexOf(currentIndex); // -1 if not an image
|
||||
const imageTotal = imageIndices.length;
|
||||
|
||||
const goPrev = useCallback(() => {
|
||||
if (imagePos > 0) setCurrentIndex(imageIndices[imagePos - 1]);
|
||||
}, [imagePos, imageIndices]);
|
||||
|
||||
const goNext = useCallback(() => {
|
||||
if (imagePos < imageTotal - 1) setCurrentIndex(imageIndices[imagePos + 1]);
|
||||
}, [imagePos, imageTotal, imageIndices]);
|
||||
|
||||
// Keyboard navigation
|
||||
useEffect(() => {
|
||||
if (!open) return;
|
||||
const handler = (e: KeyboardEvent) => {
|
||||
if (e.key === 'ArrowLeft') goPrev();
|
||||
else if (e.key === 'ArrowRight') goNext();
|
||||
};
|
||||
window.addEventListener('keydown', handler);
|
||||
return () => window.removeEventListener('keydown', handler);
|
||||
}, [open, goPrev, goNext]);
|
||||
|
||||
const effectiveType = resolveContentType(resolved.contentType, resolved.fileName);
|
||||
|
||||
// ── Fetch preview blob ─────────────────────────────────────────
|
||||
const previewMutation = useMutation({
|
||||
mutationFn: () => preview_attachment(accountId, envelopeId, resolved.contentHash),
|
||||
onSuccess: (blob) => {
|
||||
if (isTextPreview(resolved.contentType, resolved.fileName)) {
|
||||
blob.text().then(setTextContent);
|
||||
} else {
|
||||
const typedBlob = new Blob([blob], { type: effectiveType });
|
||||
setBlobUrl(URL.createObjectURL(typedBlob));
|
||||
}
|
||||
},
|
||||
onError: (error: any) => {
|
||||
toast({
|
||||
title: t('attachment_preview.failedToLoad'),
|
||||
description: error.message,
|
||||
variant: 'destructive',
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (open) {
|
||||
setBlobUrl(null);
|
||||
setTextContent(null);
|
||||
setImageZoom(1);
|
||||
previewMutation.mutate();
|
||||
}
|
||||
}, [open, resolved.contentHash]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (blobUrl) URL.revokeObjectURL(blobUrl);
|
||||
};
|
||||
}, [blobUrl]);
|
||||
|
||||
const handleDownload = () => {
|
||||
download_attachment(accountId, envelopeId, resolved.contentHash, resolved.fileName);
|
||||
};
|
||||
|
||||
const { icon } = useMemo(() => getFileConfig(resolved.contentType), [resolved.contentType]);
|
||||
|
||||
const isImage = isImagePreview(resolved.contentType, resolved.fileName);
|
||||
const isPdf = isPdfPreview(resolved.contentType, resolved.fileName);
|
||||
const isText = isTextPreview(resolved.contentType, resolved.fileName);
|
||||
const showArrows = imageTotal > 1 && isImage;
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent
|
||||
className="w-screen h-screen max-w-none rounded-none p-0 gap-0 border-0 bg-slate-700/10"
|
||||
hideClose
|
||||
hideFullscreen
|
||||
onInteractOutside={(e) => {
|
||||
if (isPdf) e.preventDefault();
|
||||
}}
|
||||
>
|
||||
{/* Toolbar — hidden for PDF (browser's native viewer has its own controls) */}
|
||||
{!isPdf && (
|
||||
<div className="absolute top-0 left-0 right-0 z-10 flex items-center justify-between px-4 py-2 bg-gradient-to-b from-black/70 to-transparent text-white">
|
||||
<div className="flex items-center gap-2 min-w-0">
|
||||
{icon}
|
||||
<span className="text-sm font-medium truncate max-w-[400px]">
|
||||
{resolved.fileName}
|
||||
</span>
|
||||
{imagePos >= 0 && imageTotal > 1 && (
|
||||
<span className="text-xs text-white/60 ml-1">
|
||||
{imagePos + 1} / {imageTotal}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center gap-1 pr-12">
|
||||
{isImage && blobUrl && (
|
||||
<>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8 text-white hover:bg-white/20"
|
||||
onClick={() => setImageZoom((z) => Math.min(z + 0.25, 3))}
|
||||
>
|
||||
<ZoomIn className="h-4 w-4" />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>{t('attachment_preview.zoomIn')}</TooltipContent>
|
||||
</Tooltip>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8 text-white hover:bg-white/20"
|
||||
onClick={() => setImageZoom((z) => Math.max(z - 0.25, 0.25))}
|
||||
>
|
||||
<ZoomOut className="h-4 w-4" />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>{t('attachment_preview.zoomOut')}</TooltipContent>
|
||||
</Tooltip>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8 text-white hover:bg-white/20"
|
||||
onClick={() => setImageZoom(1)}
|
||||
>
|
||||
<RotateCcw className="h-4 w-4" />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>{t('attachment_preview.resetZoom')}</TooltipContent>
|
||||
</Tooltip>
|
||||
<Separator orientation="vertical" className="h-5 mx-1 bg-white/20" />
|
||||
</>
|
||||
)}
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8 text-white hover:bg-white/20"
|
||||
onClick={handleDownload}
|
||||
>
|
||||
<Download className="h-4 w-4" />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>{t('attachment.download')}</TooltipContent>
|
||||
</Tooltip>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Close button — positioned below browser PDF toolbar */}
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className={isPdf
|
||||
? 'absolute top-12 right-4 z-50 h-10 w-10 rounded-full text-white bg-black/50 hover:bg-black/70'
|
||||
: 'absolute top-2 right-4 z-50 h-8 w-8 rounded-full text-white hover:bg-white/20'
|
||||
}
|
||||
onClick={() => onOpenChange(false)}
|
||||
>
|
||||
<X className={isPdf ? 'h-5 w-5' : 'h-4 w-4'} />
|
||||
</Button>
|
||||
|
||||
{/* Navigation arrows */}
|
||||
{showArrows && (
|
||||
<>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
disabled={imagePos <= 0}
|
||||
className="absolute left-2 top-1/2 -translate-y-1/2 z-10 h-10 w-10 rounded-full text-white hover:bg-white/20 disabled:opacity-30 disabled:hover:bg-transparent"
|
||||
onClick={goPrev}
|
||||
>
|
||||
<ChevronLeft className="h-6 w-6" />
|
||||
</Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
disabled={imagePos >= imageTotal - 1}
|
||||
className="absolute right-2 top-1/2 -translate-y-1/2 z-10 h-10 w-10 rounded-full text-white hover:bg-white/20 disabled:opacity-30 disabled:hover:bg-transparent"
|
||||
onClick={goNext}
|
||||
>
|
||||
<ChevronRight className="h-6 w-6" />
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Preview body */}
|
||||
<div className="w-full h-full flex items-center justify-center">
|
||||
{previewMutation.isPending ? (
|
||||
<div className="flex flex-col items-center gap-3">
|
||||
<Skeleton className="w-64 h-4 bg-white/10" />
|
||||
<Skeleton className="w-48 h-4 bg-white/10" />
|
||||
<Skeleton className="w-56 h-4 bg-white/10" />
|
||||
</div>
|
||||
) : isImage && blobUrl ? (
|
||||
<div className="w-full h-full overflow-auto flex items-center justify-center">
|
||||
<img
|
||||
src={blobUrl}
|
||||
alt={resolved.fileName}
|
||||
className="max-w-full max-h-full object-contain"
|
||||
style={{
|
||||
transform: `scale(${imageZoom})`,
|
||||
transformOrigin: 'center center',
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
) : isPdf && blobUrl ? (
|
||||
<iframe
|
||||
src={blobUrl}
|
||||
className="w-full h-full border-0"
|
||||
title={resolved.fileName}
|
||||
/>
|
||||
) : isText && textContent !== null ? (
|
||||
<pre className="w-full h-full overflow-auto whitespace-pre-wrap text-sm font-mono p-6 text-white/90">
|
||||
{textContent}
|
||||
</pre>
|
||||
) : !previewMutation.isPending ? (
|
||||
<div className="flex flex-col items-center gap-4 text-white/60">
|
||||
<FileIcon className="h-16 w-16 opacity-30" />
|
||||
<p className="text-sm">{t('attachment_preview.notAvailable')}</p>
|
||||
<p className="text-xs text-center max-w-md">
|
||||
{t('attachment_preview.notAvailableDesc', {
|
||||
type: resolved.contentType || 'unknown',
|
||||
})}
|
||||
</p>
|
||||
<Button variant="outline" size="sm" onClick={handleDownload} className="text-white border-white/20 hover:bg-white/10">
|
||||
<Download className="h-4 w-4 mr-2" />
|
||||
{t('attachment.download')}
|
||||
</Button>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@@ -40,6 +40,7 @@ import { MailThreadDialog } from './thread-dialog';
|
||||
import useMinimalAccountList from '@/hooks/use-minimal-account-list';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { NestedEmailDialog } from './nested-email-dialog';
|
||||
import AttachmentPreview, { type PreviewAttachment } from './attachment-preview';
|
||||
import { EmailEnvelope } from '@/api';
|
||||
|
||||
|
||||
@@ -123,6 +124,7 @@ export function MailMessageView({
|
||||
const [threadOpen, setThreadOpen] = useState(false);
|
||||
const [blockRemote, setBlockRemote] = useState(true);
|
||||
const [hasRemoteContent, setHasRemoteContent] = useState(false);
|
||||
const [previewAttachment, setPreviewAttachment] = useState<{ attachments: PreviewAttachment[]; index: number } | null>(null);
|
||||
|
||||
const toggleBlockRemote = () => {
|
||||
setBlockRemote((prev) => !prev);
|
||||
@@ -328,12 +330,23 @@ export function MailMessageView({
|
||||
{icon}
|
||||
</div>
|
||||
<div className="flex items-center justify-between min-w-0 flex-1 gap-2">
|
||||
<span
|
||||
className="truncate text-xs font-medium text-foreground/90"
|
||||
<button
|
||||
type="button"
|
||||
className="truncate text-xs font-medium text-foreground/90 cursor-pointer hover:text-primary hover:underline transition-colors text-left"
|
||||
title={attachment.filename}
|
||||
onClick={() =>
|
||||
setPreviewAttachment({
|
||||
attachments: nonInline.map((a) => ({
|
||||
content_hash: a.content_hash,
|
||||
file_type: a.file_type,
|
||||
filename: a.filename,
|
||||
})),
|
||||
index: i,
|
||||
})
|
||||
}
|
||||
>
|
||||
{attachment.filename}
|
||||
</span>
|
||||
</button>
|
||||
<span className="flex-shrink-0 text-[9px] font-bold text-muted-foreground/60 bg-muted px-1 py-0.5 rounded uppercase">
|
||||
{attachment.file_type.split('/').pop()}
|
||||
</span>
|
||||
@@ -361,10 +374,10 @@ export function MailMessageView({
|
||||
{formatBytes(attachment.size)}
|
||||
</span>
|
||||
{downloadingAttachmentFileName === attachment.filename ? (
|
||||
<Loader className="w-4 h-4 animate-spin" />
|
||||
<Loader className="w-5 h-5 animate-spin" />
|
||||
) : (
|
||||
<Download
|
||||
className="w-4 h-4 cursor-pointer"
|
||||
className="w-5 h-5 cursor-pointer"
|
||||
onClick={() => {
|
||||
setDownloadingAttachmentFileName(attachment.filename);
|
||||
downloadAttachmentMutation.mutate({ content_hash: attachment.content_hash });
|
||||
@@ -439,6 +452,19 @@ export function MailMessageView({
|
||||
fileName={nestedEmlFile?.filename || ''}
|
||||
content_hash={nestedEmlFile?.content_hash}
|
||||
/>
|
||||
{previewAttachment?.attachments?.[previewAttachment.index] && (
|
||||
<AttachmentPreview
|
||||
open={!!previewAttachment}
|
||||
onOpenChange={(open) => !open && setPreviewAttachment(null)}
|
||||
accountId={envelope.account_id}
|
||||
envelopeId={envelope.id}
|
||||
contentHash={previewAttachment.attachments[previewAttachment.index].content_hash}
|
||||
contentType={previewAttachment.attachments[previewAttachment.index].file_type}
|
||||
fileName={previewAttachment.attachments[previewAttachment.index].filename}
|
||||
attachments={previewAttachment.attachments}
|
||||
attachmentIndex={previewAttachment.index}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
|
||||
|
||||
import { Row } from '@tanstack/react-table'
|
||||
import { useState } from 'react'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import {
|
||||
DropdownMenu,
|
||||
@@ -28,12 +29,13 @@ import {
|
||||
DropdownMenuTrigger,
|
||||
} from '@/components/ui/dropdown-menu'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { Copy, Download, MoreVertical } from 'lucide-react'
|
||||
import { Copy, Download, Eye, MoreVertical } from 'lucide-react'
|
||||
import { AttachmentModel } from '@/api/attachment/api'
|
||||
import { useSearchAttachments } from '@/hooks/use-search-attachments'
|
||||
import { useToast } from '@/hooks/use-toast'
|
||||
import { useMutation } from '@tanstack/react-query'
|
||||
import { download_attachment } from '@/api/mailbox/envelope/api'
|
||||
import AttachmentPreview from '@/features/attachment/attachment-preview'
|
||||
|
||||
interface DataTableRowActionsProps {
|
||||
row: Row<AttachmentModel>
|
||||
@@ -43,6 +45,7 @@ export function DataTableRowActions({ row }: DataTableRowActionsProps) {
|
||||
const { setFilter } = useSearchAttachments();
|
||||
const { t } = useTranslation()
|
||||
const { toast } = useToast();
|
||||
const [previewOpen, setPreviewOpen] = useState(false);
|
||||
|
||||
const downloadMutation = useMutation({
|
||||
mutationFn: (content_hash: string) =>
|
||||
@@ -87,6 +90,18 @@ export function DataTableRowActions({ row }: DataTableRowActionsProps) {
|
||||
</DropdownMenuShortcut>
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuSeparator />
|
||||
<DropdownMenuItem
|
||||
className='text-xs'
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setPreviewOpen(true);
|
||||
}}
|
||||
>
|
||||
{t('attachment.preview')}
|
||||
<DropdownMenuShortcut>
|
||||
<Eye size={16} />
|
||||
</DropdownMenuShortcut>
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem
|
||||
className='text-xs'
|
||||
disabled={downloadMutation.isPending}
|
||||
@@ -105,6 +120,15 @@ export function DataTableRowActions({ row }: DataTableRowActionsProps) {
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
<AttachmentPreview
|
||||
open={previewOpen}
|
||||
onOpenChange={setPreviewOpen}
|
||||
accountId={row.original.account_id}
|
||||
envelopeId={row.original.envelope_id}
|
||||
contentHash={row.original.content_hash}
|
||||
contentType={row.original.content_type}
|
||||
fileName={row.original.name ?? row.original.id}
|
||||
/>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -41,9 +41,10 @@ import { useLocation, useNavigate } from '@tanstack/react-router'
|
||||
import { Button } from '@/components/button'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import i18n from '@/i18n'
|
||||
import { Loader2, LogIn } from 'lucide-react'
|
||||
import { Loader2, LogIn, Shield } from 'lucide-react'
|
||||
import { login } from '@/api/users/api'
|
||||
import { useTheme } from '@/context/theme-context'
|
||||
import { useEdition } from '@/hooks/use-edition'
|
||||
|
||||
type UserAuthFormProps = HTMLAttributes<HTMLDivElement>
|
||||
|
||||
@@ -52,6 +53,7 @@ export function UserAuthForm({ className, ...props }: UserAuthFormProps) {
|
||||
const { setTheme } = useTheme();
|
||||
const navigate = useNavigate()
|
||||
const { t } = useTranslation()
|
||||
const { isPro } = useEdition()
|
||||
|
||||
const { search } = useLocation();
|
||||
const redirect = toSearchParams(search).get('redirect') || '/';
|
||||
@@ -156,6 +158,20 @@ export function UserAuthForm({ className, ...props }: UserAuthFormProps) {
|
||||
{isLoading ? <Loader2 className='animate-spin' /> : <LogIn size={16} className='mr-2' />}
|
||||
{t('auth.login')}
|
||||
</Button>
|
||||
|
||||
{isPro && (
|
||||
<Button
|
||||
variant='outline'
|
||||
className='mt-2'
|
||||
type='button'
|
||||
onClick={() => {
|
||||
window.location.href = '/api/auth/oidc/login'
|
||||
}}
|
||||
>
|
||||
<Shield size={16} className='mr-2' />
|
||||
{t('auth.ssoLogin')}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</form>
|
||||
</Form>
|
||||
|
||||
@@ -25,6 +25,11 @@ import LongText from '@/components/long-text';
|
||||
import { getToken } from '@/stores/authStore';
|
||||
import { useNavigate } from '@tanstack/react-router';
|
||||
import useMinimalAccountList from '@/hooks/use-minimal-account-list';
|
||||
import {
|
||||
Tooltip as TooltipUI,
|
||||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from '@/components/ui/tooltip';
|
||||
|
||||
interface DailyActivity {
|
||||
date: string;
|
||||
@@ -128,6 +133,12 @@ export default function MailArchiveDashboard() {
|
||||
return account ? account.id : null;
|
||||
};
|
||||
|
||||
const getAccountNameByEmail = (email: string): string | null => {
|
||||
if (!minimalList) return null;
|
||||
const account = minimalList.find(a => a.email === email);
|
||||
return account?.name || null;
|
||||
};
|
||||
|
||||
const handleQuickSearch = (filter: Record<string, any>) => {
|
||||
navigate({
|
||||
to: '/search',
|
||||
@@ -519,16 +530,30 @@ export default function MailArchiveDashboard() {
|
||||
<div className="text-xs flex flex-wrap gap-x-1 min-w-0 flex-1">
|
||||
<span className="flex items-center">
|
||||
<LongText className="max-w-[180px] md:max-w-[160px] lg:max-w-[200px] xl:max-w-[220px]">
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
handleQuickSearch({ account_ids: [getAccountIdByEmail(acc.key) || 0] })
|
||||
}}
|
||||
className="hover:text-primary hover:underline transition-colors"
|
||||
>
|
||||
{acc.key}
|
||||
</button>
|
||||
{(() => {
|
||||
const name = getAccountNameByEmail(acc.key);
|
||||
const btn = (
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
handleQuickSearch({ account_ids: [getAccountIdByEmail(acc.key) || 0] })
|
||||
}}
|
||||
className="hover:text-primary hover:underline transition-colors"
|
||||
>
|
||||
{name || acc.key}
|
||||
</button>
|
||||
);
|
||||
if (name) {
|
||||
return (
|
||||
<TooltipUI>
|
||||
<TooltipTrigger asChild>{btn}</TooltipTrigger>
|
||||
<TooltipContent side="top">{acc.key}</TooltipContent>
|
||||
</TooltipUI>
|
||||
);
|
||||
}
|
||||
return btn;
|
||||
})()}
|
||||
</LongText>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
153
web/src/features/import/folder-hint.ts
Normal file
153
web/src/features/import/folder-hint.ts
Normal file
@@ -0,0 +1,153 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
|
||||
/**
|
||||
* Parse raw EML/MBOX headers from the first few KB of a file and return a
|
||||
* suggested folder name, or null if nothing useful was found.
|
||||
*
|
||||
* Mirrors the CLI logic in crates/cli/src/mbox/gmail.rs (determine_folder).
|
||||
*/
|
||||
|
||||
const HEADER_READ_BYTES = 64 * 1024; // read first 64 KB to get headers
|
||||
|
||||
/** RFC 2047 encoded-word prefix. We do a best-effort decode. */
|
||||
function decodeRfc2047(raw: string): string {
|
||||
return raw.replace(/=\?[^?]+\?[BbQq]\?[^?]*\?=/gi, (match) => {
|
||||
try {
|
||||
const parts = match.split('?');
|
||||
const charset = parts[1];
|
||||
const encoding = parts[2].toUpperCase();
|
||||
const encoded = parts[3];
|
||||
let bytes: Uint8Array;
|
||||
if (encoding === 'B') {
|
||||
const bin = atob(encoded);
|
||||
bytes = new Uint8Array(bin.length);
|
||||
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
|
||||
} else {
|
||||
// Q-encoding
|
||||
const hex = encoded.replace(/_/g, ' ').replace(/=([0-9A-Fa-f]{2})/g, (_, h) =>
|
||||
String.fromCharCode(parseInt(h, 16)),
|
||||
);
|
||||
bytes = new TextEncoder().encode(hex);
|
||||
}
|
||||
return new TextDecoder(charset).decode(bytes);
|
||||
} catch {
|
||||
return match;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Extract a single header value from raw email text. Case-insensitive. */
|
||||
function getHeader(raw: string, name: string): string | null {
|
||||
const re = new RegExp(`^${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*:\\s*(.+)$`, 'im');
|
||||
const m = raw.match(re);
|
||||
if (!m) return null;
|
||||
// Unfold continuation lines (leading whitespace)
|
||||
let val = m[1].trim();
|
||||
const startIdx = m.index! + m[0].length;
|
||||
const rest = raw.slice(startIdx);
|
||||
const contRe = /^\s+(.+)$/gm;
|
||||
let cm: RegExpExecArray | null;
|
||||
while ((cm = contRe.exec(rest)) !== null) {
|
||||
val += ' ' + cm[1].trim();
|
||||
}
|
||||
return decodeRfc2047(val);
|
||||
}
|
||||
|
||||
/** Determine folder from X-Gmail-Labels, mirroring the CLI's determine_folder(). */
|
||||
function folderFromGmailLabels(raw: string): string | null {
|
||||
const labelsRaw = getHeader(raw, 'X-Gmail-Labels');
|
||||
if (!labelsRaw) return null;
|
||||
|
||||
const statusBlacklist = new Set(['Opened', 'Unread', 'Archived']);
|
||||
const allLabels = labelsRaw.split(',').map((s) => s.trim()).filter(Boolean);
|
||||
if (allLabels.length === 0) return null;
|
||||
|
||||
const filtered = allLabels.filter((l) => !statusBlacklist.has(l));
|
||||
if (filtered.length === 0) return allLabels[0];
|
||||
if (filtered.length === 1) return filtered[0];
|
||||
|
||||
// Prefer business labels over generic Inbox/Sent
|
||||
const business = filtered.find((l) => l !== 'Inbox' && l !== 'Sent');
|
||||
return business ?? filtered[0];
|
||||
}
|
||||
|
||||
/** Try to read mailbox_name from X-Bichon-Metadata JSON header. */
|
||||
function folderFromBichonMetadata(raw: string): string | null {
|
||||
const metaRaw = getHeader(raw, 'X-Bichon-Metadata');
|
||||
if (!metaRaw) return null;
|
||||
try {
|
||||
const meta = JSON.parse(metaRaw);
|
||||
if (meta?.mailbox_name && typeof meta.mailbox_name === 'string') {
|
||||
return meta.mailbox_name;
|
||||
}
|
||||
} catch {
|
||||
// ignore parse errors
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Derive a folder from the file name (e.g. "Inbox.mbox" → "Inbox"). */
|
||||
function folderFromFileName(fileName: string): string | null {
|
||||
const base = fileName.replace(/\.[^.]+$/, ''); // strip extension
|
||||
if (!base || base === fileName) return null;
|
||||
// Common patterns
|
||||
if (/^[a-zA-Z0-9_/\-.\s]+$/.test(base) && base.length > 0 && base.length < 128) {
|
||||
return base;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export interface FolderHint {
|
||||
/** The suggested folder name. */
|
||||
name: string;
|
||||
/** Where the hint came from. */
|
||||
source: 'gmail-labels' | 'bichon-metadata' | 'filename' | 'mbox-filename' | 'pst-filename';
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the first chunk of a File and return folder hints extracted from headers.
|
||||
* Returns null if no hint could be extracted.
|
||||
*/
|
||||
export async function extractFolderHint(file: File): Promise<FolderHint | null> {
|
||||
const ext = file.name.split('.').pop()?.toLowerCase();
|
||||
const isMbox = ext === 'mbox';
|
||||
const isPst = ext === 'pst';
|
||||
|
||||
// Read first 64 KB — enough for headers of the first message
|
||||
const chunk = new Uint8Array(await file.slice(0, HEADER_READ_BYTES).arrayBuffer());
|
||||
const raw = new TextDecoder('utf-8', { fatal: false }).decode(chunk);
|
||||
|
||||
// MBOX: the first line is "From ...", headers start after the first newline
|
||||
const headers = isMbox
|
||||
? raw.replace(/^From [^\n]*\n/, '') // strip MBOX "From " separator
|
||||
: raw;
|
||||
|
||||
// 1. X-Bichon-Metadata (highest priority, explicit)
|
||||
const bichonFolder = folderFromBichonMetadata(headers);
|
||||
if (bichonFolder) return { name: bichonFolder, source: 'bichon-metadata' };
|
||||
|
||||
// 2. X-Gmail-Labels
|
||||
const gmailFolder = folderFromGmailLabels(headers);
|
||||
if (gmailFolder) return { name: gmailFolder, source: 'gmail-labels' };
|
||||
|
||||
// 3. For MBOX files, use the filename
|
||||
if (isMbox) {
|
||||
const fnFolder = folderFromFileName(file.name);
|
||||
if (fnFolder) return { name: fnFolder, source: 'mbox-filename' };
|
||||
}
|
||||
|
||||
// 4. For EML files, try the filename
|
||||
const fnFolder = folderFromFileName(file.name);
|
||||
if (fnFolder) return { name: fnFolder, source: 'filename' };
|
||||
|
||||
// 5. For PST files, try the filename
|
||||
if (isPst) {
|
||||
const fnFolder = folderFromFileName(file.name);
|
||||
if (fnFolder) return { name: fnFolder, source: 'pst-filename' };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
872
web/src/features/import/index.tsx
Normal file
872
web/src/features/import/index.tsx
Normal file
@@ -0,0 +1,872 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
|
||||
import { useState, useRef, useCallback, useEffect } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { useMutation, useQuery } from '@tanstack/react-query';
|
||||
import {
|
||||
Upload, FileText, X, CheckCircle2, AlertTriangle,
|
||||
Sparkles, PenLine, ListTree, ChevronsUpDown, Check,
|
||||
Clock, ChevronRight,
|
||||
} from 'lucide-react';
|
||||
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { Label } from '@/components/ui/label';
|
||||
import { Progress } from '@/components/ui/progress';
|
||||
import { ScrollArea } from '@/components/ui/scroll-area';
|
||||
import { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { Main } from '@/components/layout/main';
|
||||
import { FixedHeader } from '@/components/layout/fixed-header';
|
||||
import { useToast } from '@/hooks/use-toast';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover';
|
||||
import {
|
||||
Command,
|
||||
CommandEmpty,
|
||||
CommandGroup,
|
||||
CommandInput,
|
||||
CommandItem,
|
||||
CommandList,
|
||||
} from '@/components/ui/command';
|
||||
|
||||
import {
|
||||
upload_import,
|
||||
get_import_progress,
|
||||
get_nosync_accounts,
|
||||
list_import_history,
|
||||
type ImportProgress,
|
||||
type ImportHistory,
|
||||
} from '@/api/import/api';
|
||||
import { get_system_configurations } from '@/api/system/api';
|
||||
import { list_mailboxes } from '@/api/mailbox/api';
|
||||
import { extractFolderHint, type FolderHint } from './folder-hint';
|
||||
|
||||
const MAX_EML = 100 * 1024 * 1024; // 100 MB (hardcoded)
|
||||
const DEFAULT_MAX_MBOX = 1024 * 1024 * 1024; // 1 GB (fallback; actual limit from server settings)
|
||||
const DEFAULT_MAX_PST = 2048 * 1024 * 1024; // 2 GB (fallback; actual limit from server settings)
|
||||
|
||||
// MIME types that are clearly NOT email files — reject these upfront.
|
||||
const BLOCKED_MIME_PREFIXES = [
|
||||
'video/', 'audio/', 'image/', 'font/',
|
||||
'application/zip', 'application/gzip', 'application/x-tar',
|
||||
'application/x-7z', 'application/x-rar',
|
||||
'application/vnd.', 'application/pdf',
|
||||
'application/x-msdownload', 'application/x-executable',
|
||||
];
|
||||
|
||||
function isValidFileType(file: File, ext: string): boolean {
|
||||
// Check MIME type: reject known binary types
|
||||
const mime = file.type.toLowerCase();
|
||||
if (mime) {
|
||||
for (const prefix of BLOCKED_MIME_PREFIXES) {
|
||||
if (mime.startsWith(prefix)) return false;
|
||||
}
|
||||
}
|
||||
// Check extension
|
||||
return ext === 'eml' || ext === 'mbox' || ext === 'pst';
|
||||
}
|
||||
|
||||
type FolderMode = '' | 'header' | 'existing' | 'custom';
|
||||
|
||||
interface QueuedFile {
|
||||
file: File;
|
||||
sizeOk: boolean;
|
||||
typeOk: boolean;
|
||||
}
|
||||
|
||||
function formatSize(bytes: number) {
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
||||
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
||||
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
|
||||
}
|
||||
|
||||
function folderHintLabel(hint: FolderHint): string {
|
||||
switch (hint.source) {
|
||||
case 'gmail-labels': return 'X-Gmail-Labels';
|
||||
case 'bichon-metadata': return 'X-Bichon-Metadata';
|
||||
case 'filename': return 'filename';
|
||||
case 'mbox-filename': return 'mbox filename';
|
||||
case 'pst-filename': return 'PST filename';
|
||||
}
|
||||
}
|
||||
|
||||
export default function ImportPage() {
|
||||
const { t } = useTranslation();
|
||||
const { toast } = useToast();
|
||||
|
||||
const [accountId, setAccountId] = useState<string>('');
|
||||
const [folderMode, setFolderMode] = useState<FolderMode>('');
|
||||
const [folder, setFolder] = useState('INBOX');
|
||||
const [files, setFiles] = useState<QueuedFile[]>([]);
|
||||
const [dragging, setDragging] = useState(false);
|
||||
// const [importId, setImportId] = useState<string | null>(null);
|
||||
const [progress, setProgress] = useState<ImportProgress | null>(null);
|
||||
const [uploadPct, setUploadPct] = useState(0);
|
||||
const [phase, setPhase] = useState<'idle' | 'uploading' | 'processing' | 'done'>('idle');
|
||||
const [folderHint, setFolderHint] = useState<FolderHint | null>(null);
|
||||
const [headerFolder, setHeaderFolder] = useState('INBOX');
|
||||
const [isPstSelected, setIsPstSelected] = useState(false);
|
||||
|
||||
// Combobox state for existing mailbox selection
|
||||
const [mailboxOpen, setMailboxOpen] = useState(false);
|
||||
// Combobox state for account selection
|
||||
const [accountOpen, setAccountOpen] = useState(false);
|
||||
|
||||
const pollRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
|
||||
const { data: accounts = [] } = useQuery({
|
||||
queryKey: ['nosync-accounts'],
|
||||
queryFn: get_nosync_accounts,
|
||||
staleTime: 30_000,
|
||||
});
|
||||
|
||||
const { data: mailboxData } = useQuery({
|
||||
queryKey: ['account-mailboxes', accountId],
|
||||
queryFn: () => list_mailboxes(Number(accountId), false),
|
||||
enabled: !!accountId,
|
||||
staleTime: 60_000,
|
||||
});
|
||||
const mailboxes = mailboxData?.mailboxes ?? [];
|
||||
|
||||
// Fetch system config to get the configured MBOX/PST upload limits.
|
||||
// Falls back to defaults for non-root users or on error.
|
||||
const { data: sysConfig } = useQuery({
|
||||
queryKey: ['system-configurations'],
|
||||
queryFn: get_system_configurations,
|
||||
staleTime: 300_000,
|
||||
retry: false,
|
||||
});
|
||||
const maxMbox = sysConfig
|
||||
? sysConfig.bichon_web_mbox_upload_limit_mb * 1024 * 1024
|
||||
: DEFAULT_MAX_MBOX;
|
||||
const maxPst = sysConfig
|
||||
? sysConfig.bichon_web_pst_upload_limit_mb * 1024 * 1024
|
||||
: DEFAULT_MAX_PST;
|
||||
|
||||
// Import history
|
||||
const { data: history = [], refetch: refetchHistory } = useQuery({
|
||||
queryKey: ['import-history'],
|
||||
queryFn: list_import_history,
|
||||
staleTime: 10_000,
|
||||
});
|
||||
|
||||
// Resolve the effective folder based on current mode
|
||||
const effectiveFolder = (() => {
|
||||
switch (folderMode) {
|
||||
case 'header':
|
||||
return headerFolder;
|
||||
case 'existing':
|
||||
case 'custom':
|
||||
return folder;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
})();
|
||||
|
||||
const startPolling = useCallback((id: string) => {
|
||||
if (pollRef.current) clearInterval(pollRef.current);
|
||||
let retries = 0;
|
||||
pollRef.current = setInterval(async () => {
|
||||
try {
|
||||
const p = await get_import_progress(id);
|
||||
setProgress(p);
|
||||
retries = 0;
|
||||
if (p.status === 'Completed' || p.status === 'Failed') {
|
||||
if (pollRef.current) clearInterval(pollRef.current);
|
||||
setPhase('done');
|
||||
refetchHistory();
|
||||
}
|
||||
} catch {
|
||||
retries++;
|
||||
if (retries > 5) {
|
||||
if (pollRef.current) clearInterval(pollRef.current);
|
||||
setPhase('idle');
|
||||
}
|
||||
}
|
||||
}, 1000);
|
||||
}, [refetchHistory]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => { if (pollRef.current) clearInterval(pollRef.current); };
|
||||
}, []);
|
||||
|
||||
const handleFiles = useCallback(async (newFiles: FileList | File[]) => {
|
||||
const arr = Array.from(newFiles) as File[];
|
||||
const queued: QueuedFile[] = arr.map((f) => {
|
||||
const ext = f.name.split('.').pop()?.toLowerCase() || '';
|
||||
const isMbox = ext === 'mbox';
|
||||
const isPst = ext === 'pst';
|
||||
const max = isMbox ? maxMbox : isPst ? maxPst : MAX_EML;
|
||||
const typeOk = isValidFileType(f, ext);
|
||||
return { file: f, sizeOk: f.size <= max, typeOk };
|
||||
});
|
||||
|
||||
setFiles(queued);
|
||||
setPhase('idle');
|
||||
setProgress(null);
|
||||
//setImportId(null);
|
||||
|
||||
// Extract folder hint from the first valid file.
|
||||
// PST files are binary (OLE2) — headers can't be extracted in-browser.
|
||||
const firstOk = queued.find((q) => q.sizeOk && q.typeOk);
|
||||
if (firstOk) {
|
||||
const ext = firstOk.file.name.split('.').pop()?.toLowerCase() || '';
|
||||
const isPstFile = ext === 'pst';
|
||||
setIsPstSelected(isPstFile);
|
||||
if (isPstFile) {
|
||||
// PST: folder structure is auto-detected, no manual mode needed
|
||||
setFolderHint(null);
|
||||
setHeaderFolder('INBOX');
|
||||
setFolderMode('');
|
||||
} else {
|
||||
// EML/MBOX: default to header auto-detect if no mode selected yet
|
||||
if (!folderMode) {
|
||||
setFolderMode('header');
|
||||
}
|
||||
try {
|
||||
const hint = await extractFolderHint(firstOk.file);
|
||||
if (hint) {
|
||||
setFolderHint(hint);
|
||||
setHeaderFolder(hint.name);
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}, []);
|
||||
|
||||
const removeFile = (idx: number) => {
|
||||
setFiles((prev) => prev.filter((_, i) => i !== idx));
|
||||
if (files.length <= 1) {
|
||||
setFolderHint(null);
|
||||
setHeaderFolder('INBOX');
|
||||
setFolderMode('');
|
||||
setIsPstSelected(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleAccountChange = (v: string) => {
|
||||
setAccountId(v);
|
||||
setFiles([]);
|
||||
setFolderHint(null);
|
||||
setHeaderFolder('INBOX');
|
||||
};
|
||||
|
||||
const handleModeChange = (mode: FolderMode) => {
|
||||
setFolderMode(mode);
|
||||
// When switching to header mode, re-detect from files if available
|
||||
if (mode === 'header' && files.length > 0) {
|
||||
const firstOk = files.find((q) => q.sizeOk && q.typeOk);
|
||||
if (firstOk) {
|
||||
extractFolderHint(firstOk.file).then((hint) => {
|
||||
if (hint) {
|
||||
setFolderHint(hint);
|
||||
setHeaderFolder(hint.name);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const importMutation = useMutation({
|
||||
mutationFn: async () => {
|
||||
if (!accountId || !files.length) return;
|
||||
const file = files[0].file;
|
||||
setPhase('uploading');
|
||||
setUploadPct(0);
|
||||
const result = await upload_import(
|
||||
Number(accountId),
|
||||
effectiveFolder,
|
||||
file.name,
|
||||
file,
|
||||
(pct) => setUploadPct(pct),
|
||||
);
|
||||
//setImportId(result.import_id);
|
||||
setProgress(result);
|
||||
setPhase('processing');
|
||||
startPolling(result.import_id);
|
||||
},
|
||||
onError: (err: any) => {
|
||||
setPhase('idle');
|
||||
toast({
|
||||
title: t('common.failed'),
|
||||
description: err?.response?.data?.message || err.message,
|
||||
variant: 'destructive',
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
const canImport =
|
||||
accountId && effectiveFolder.trim() && files.length > 0 && files.every((f) => f.sizeOk && f.typeOk) && phase === 'idle';
|
||||
|
||||
return (
|
||||
<>
|
||||
<FixedHeader />
|
||||
<Main>
|
||||
<div className="flex-1 space-y-6 p-6 md:p-8 max-w-3xl mx-auto">
|
||||
<div>
|
||||
<h1 className="text-xl font-bold tracking-tight">
|
||||
{t('import.title', 'Import EML / MBOX / PST')}
|
||||
</h1>
|
||||
<p className="text-sm text-muted-foreground mt-1">
|
||||
{t('import.description', 'Import email files into a NoSync account. For larger files, use the CLI.')}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Step 1: Target account */}
|
||||
<Card>
|
||||
<CardHeader className="pb-3">
|
||||
<CardTitle className="text-sm font-medium">
|
||||
{t('import.target', '1. Select target account')}
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<div className="max-w-sm space-y-1.5">
|
||||
<Label className="text-xs">{t('import.account')}</Label>
|
||||
<Popover open={accountOpen} onOpenChange={setAccountOpen}>
|
||||
<PopoverTrigger asChild>
|
||||
<Button
|
||||
variant="outline"
|
||||
role="combobox"
|
||||
className="h-9 justify-between text-xs w-full"
|
||||
>
|
||||
<span className={cn('truncate', !accountId && 'text-muted-foreground')}>
|
||||
{accountId
|
||||
? accounts.find((a) => String(a.id) === accountId)?.account_name
|
||||
|| accounts.find((a) => String(a.id) === accountId)?.email
|
||||
|| accountId
|
||||
: t('import.selectAccount')}
|
||||
</span>
|
||||
<ChevronsUpDown className="ml-2 h-3.5 w-3.5 shrink-0 opacity-50" />
|
||||
</Button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent className="w-[280px] p-0" align="start">
|
||||
<Command>
|
||||
<CommandInput
|
||||
placeholder={t('import.searchAccount', 'Search accounts...')}
|
||||
className="h-9 text-xs"
|
||||
/>
|
||||
<CommandList>
|
||||
<CommandEmpty>
|
||||
{t('import.noAccountFound', 'No account found.')}
|
||||
</CommandEmpty>
|
||||
<CommandGroup>
|
||||
{accounts.map((a) => (
|
||||
<CommandItem
|
||||
key={a.id}
|
||||
value={a.account_name || a.email || String(a.id)}
|
||||
onSelect={() => {
|
||||
handleAccountChange(String(a.id));
|
||||
setAccountOpen(false);
|
||||
}}
|
||||
className='text-xs'
|
||||
>
|
||||
<Check
|
||||
className={cn(
|
||||
'h-4 w-4',
|
||||
accountId === String(a.id) ? 'opacity-100' : 'opacity-0',
|
||||
)}
|
||||
/>
|
||||
{a.account_name || a.email}
|
||||
</CommandItem>
|
||||
))}
|
||||
</CommandGroup>
|
||||
</CommandList>
|
||||
</Command>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Step 2: Folder determination mode */}
|
||||
<Card>
|
||||
<CardHeader className="pb-3">
|
||||
<CardTitle className="text-sm font-medium">
|
||||
{isPstSelected
|
||||
? t('import.folderStructure', '2. Folder structure')
|
||||
: t('import.folderMethod', '2. Choose folder method')}
|
||||
</CardTitle>
|
||||
<CardDescription className="text-xs">
|
||||
{isPstSelected
|
||||
? t('import.pstFolderDesc', 'The PST file contains its own folder structure (e.g. Inbox, Sent Items, etc.). Folders will be automatically created during import.')
|
||||
: files.length === 0
|
||||
? t('import.selectFileFirst', 'Select a file first to determine available options.')
|
||||
: t('import.folderMethodDesc', 'How should the target mail folder be determined?')}
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
{!isPstSelected && (
|
||||
<RadioGroup
|
||||
value={folderMode}
|
||||
onValueChange={(v) => handleModeChange(v as FolderMode)}
|
||||
className="gap-3"
|
||||
>
|
||||
{/* Mode 1: Auto-detect from headers */}
|
||||
<label
|
||||
className={cn(
|
||||
'flex items-start gap-3 rounded-lg border p-3 cursor-pointer transition-colors',
|
||||
folderMode === 'header'
|
||||
? 'border-primary bg-primary/5'
|
||||
: 'border-border hover:bg-muted/50',
|
||||
)}
|
||||
>
|
||||
<RadioGroupItem value="header" id="mode-header" className="mt-0.5" />
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2">
|
||||
<Sparkles className="h-4 w-4 text-primary" />
|
||||
<span className="text-sm font-medium">
|
||||
{t('import.modeHeader', 'Auto-detect from email headers')}
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground mt-1">
|
||||
{t('import.modeHeaderDesc', 'Read X-Gmail-Labels / X-Bichon-Metadata from the uploaded file. Falls back to filename.')}
|
||||
</p>
|
||||
{folderMode === 'header' && (
|
||||
<div className="mt-2 flex items-center gap-2">
|
||||
<Badge variant="secondary" className="text-xs font-normal">
|
||||
{folderHint
|
||||
? t('import.detectedFolder', 'Detected') + ': ' + headerFolder
|
||||
: t('import.noFileYet', 'No file selected yet')}
|
||||
</Badge>
|
||||
{folderHint && (
|
||||
<span className="text-[10px] text-muted-foreground">
|
||||
({t('import.source')}: {folderHintLabel(folderHint)})
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</label>
|
||||
|
||||
{/* Mode 2: Pick from existing mailboxes */}
|
||||
<label
|
||||
className={cn(
|
||||
'flex items-start gap-3 rounded-lg border p-3 cursor-pointer transition-colors',
|
||||
folderMode === 'existing'
|
||||
? 'border-primary bg-primary/5'
|
||||
: 'border-border hover:bg-muted/50',
|
||||
!accountId && 'opacity-50 pointer-events-none',
|
||||
)}
|
||||
>
|
||||
<RadioGroupItem value="existing" id="mode-existing" className="mt-0.5" disabled={!accountId} />
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2">
|
||||
<ListTree className="h-4 w-4 text-primary" />
|
||||
<span className="text-sm font-medium">
|
||||
{t('import.modeExisting', 'Choose from existing mailboxes')}
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground mt-1">
|
||||
{t('import.modeExistingDesc', 'Select one of the mailboxes already present in this account.')}
|
||||
</p>
|
||||
{folderMode === 'existing' && (
|
||||
<div className="mt-2">
|
||||
{mailboxes.length === 0 ? (
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{accountId
|
||||
? t('import.noMailboxes', 'No mailboxes found in this account.')
|
||||
: t('import.selectAccountFirst', 'Select an account first.')}
|
||||
</span>
|
||||
) : (
|
||||
<Popover open={mailboxOpen} onOpenChange={setMailboxOpen}>
|
||||
<PopoverTrigger asChild>
|
||||
<Button
|
||||
variant="outline"
|
||||
role="combobox"
|
||||
className="h-8 justify-between text-xs max-w-xs w-full"
|
||||
>
|
||||
<span className="truncate">
|
||||
{folder || t('import.selectMailbox', 'Select a mailbox...')}
|
||||
</span>
|
||||
<ChevronsUpDown className="ml-2 h-3.5 w-3.5 shrink-0 opacity-50" />
|
||||
</Button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent className="w-[280px] p-0" align="start">
|
||||
<Command>
|
||||
<CommandInput
|
||||
placeholder={t('import.searchMailbox', 'Search mailboxes...')}
|
||||
className="h-9 text-xs"
|
||||
/>
|
||||
<CommandList>
|
||||
<CommandEmpty>
|
||||
{t('import.noMailboxFound', 'No mailbox found.')}
|
||||
</CommandEmpty>
|
||||
<CommandGroup>
|
||||
{mailboxes.map((mb) => (
|
||||
<CommandItem
|
||||
key={mb.id}
|
||||
value={mb.name}
|
||||
onSelect={(value) => {
|
||||
setFolder(value);
|
||||
setMailboxOpen(false);
|
||||
}}
|
||||
className='text-xs'
|
||||
>
|
||||
<Check
|
||||
className={cn(
|
||||
'h-4 w-4',
|
||||
folder === mb.name ? 'opacity-100' : 'opacity-0',
|
||||
)}
|
||||
/>
|
||||
{mb.name}
|
||||
</CommandItem>
|
||||
))}
|
||||
</CommandGroup>
|
||||
</CommandList>
|
||||
</Command>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</label>
|
||||
|
||||
{/* Mode 3: Manual input */}
|
||||
<label
|
||||
className={cn(
|
||||
'flex items-start gap-3 rounded-lg border p-3 cursor-pointer transition-colors',
|
||||
folderMode === 'custom'
|
||||
? 'border-primary bg-primary/5'
|
||||
: 'border-border hover:bg-muted/50',
|
||||
)}
|
||||
>
|
||||
<RadioGroupItem value="custom" id="mode-custom" className="mt-0.5" />
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2">
|
||||
<PenLine className="h-4 w-4 text-primary" />
|
||||
<span className="text-sm font-medium">
|
||||
{t('import.modeCustom', 'Enter a custom folder name')}
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground mt-1">
|
||||
{t('import.modeCustomDesc', 'Manually type the target mail folder name.')}
|
||||
</p>
|
||||
{folderMode === 'custom' && (
|
||||
<div className="mt-2">
|
||||
<Input
|
||||
className="h-8 text-xs max-w-xs"
|
||||
value={folder}
|
||||
onChange={(e) => setFolder(e.target.value)}
|
||||
placeholder="INBOX"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</label>
|
||||
</RadioGroup>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Step 3: File upload */}
|
||||
<Card>
|
||||
<CardHeader className="pb-3">
|
||||
<CardTitle className="text-sm font-medium">
|
||||
{t('import.chooseFiles', '3. Choose files')}
|
||||
</CardTitle>
|
||||
<CardDescription className="text-xs">
|
||||
{t('import.limits', {
|
||||
defaultValue: 'Max: EML 100 MB · MBOX {{maxMbox}} MB · PST {{maxPst}} MB. Larger files → CLI.',
|
||||
maxMbox: (maxMbox / (1024 * 1024)).toFixed(0),
|
||||
maxPst: (maxPst / (1024 * 1024)).toFixed(0)
|
||||
})}
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<div
|
||||
className={cn(
|
||||
'border-2 border-dashed rounded-lg p-8 text-center cursor-pointer transition-colors',
|
||||
dragging ? 'border-primary bg-primary/5' : 'border-muted-foreground/25 hover:border-muted-foreground/50',
|
||||
phase !== 'idle' && 'pointer-events-none opacity-50',
|
||||
)}
|
||||
onDragOver={(e) => { e.preventDefault(); setDragging(true); }}
|
||||
onDragLeave={() => setDragging(false)}
|
||||
onDrop={(e) => { e.preventDefault(); setDragging(false); handleFiles(e.dataTransfer.files); }}
|
||||
onClick={() => {
|
||||
const input = document.createElement('input');
|
||||
input.type = 'file';
|
||||
input.accept = '.eml,.mbox,.pst,message/rfc822,application/mbox,text/plain';
|
||||
input.multiple = true;
|
||||
input.onchange = () => input.files && handleFiles(input.files);
|
||||
input.click();
|
||||
}}
|
||||
>
|
||||
<Upload className="mx-auto h-10 w-10 text-muted-foreground/60 mb-3" />
|
||||
<p className="text-sm font-medium">
|
||||
{t('import.dropHere', 'Drop .eml / .mbox / .pst files here')}
|
||||
</p>
|
||||
<p className="text-xs text-muted-foreground mt-1">
|
||||
{t('import.orClick', 'or click to browse')}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{files.length > 0 && (
|
||||
<div className="mt-4 space-y-2">
|
||||
{files.map((qf, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className={cn(
|
||||
'flex items-center gap-3 px-3 py-2 rounded-md border text-sm',
|
||||
qf.sizeOk && qf.typeOk
|
||||
? 'bg-muted/30 border-border'
|
||||
: 'bg-destructive/5 border-destructive/30 text-destructive',
|
||||
)}
|
||||
>
|
||||
<FileText className="h-4 w-4 shrink-0" />
|
||||
<span className="flex-1 truncate">{qf.file.name}</span>
|
||||
<span className={cn('text-xs shrink-0', qf.sizeOk && qf.typeOk ? 'text-muted-foreground' : 'font-medium')}>
|
||||
{formatSize(qf.file.size)}
|
||||
</span>
|
||||
{!qf.typeOk && (
|
||||
<span className="text-xs font-medium text-destructive shrink-0">Invalid type</span>
|
||||
)}
|
||||
{!qf.sizeOk && qf.typeOk && (
|
||||
<span className="text-xs font-medium text-destructive shrink-0">Too large</span>
|
||||
)}
|
||||
{qf.sizeOk && qf.typeOk ? (
|
||||
<CheckCircle2 className="h-4 w-4 text-green-600 shrink-0" />
|
||||
) : (
|
||||
<AlertTriangle className="h-4 w-4 shrink-0" />
|
||||
)}
|
||||
{phase === 'idle' && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => { e.stopPropagation(); removeFile(i); }}
|
||||
className="p-0.5 hover:bg-muted rounded"
|
||||
>
|
||||
<X className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Step 4: Progress & Results */}
|
||||
{(phase !== 'idle' || progress) && (
|
||||
<Card>
|
||||
<CardHeader className="pb-3">
|
||||
<CardTitle className="text-sm font-medium">
|
||||
{phase === 'uploading' && t('import.uploading', 'Uploading…')}
|
||||
{phase === 'processing' && t('import.processing', 'Processing…')}
|
||||
{phase === 'done' && (progress?.status === 'Completed' ? t('import.completed', 'Import complete') : t('import.failed', 'Import failed'))}
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
{phase === 'uploading' && (
|
||||
<div className="space-y-1.5">
|
||||
<div className="flex justify-between text-xs text-muted-foreground">
|
||||
<span>{t('import.uploadingFile')}</span>
|
||||
<span>{uploadPct}%</span>
|
||||
</div>
|
||||
<Progress value={uploadPct} className="h-2" />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{progress && progress.total > 0 && (
|
||||
<div className="space-y-1.5">
|
||||
<div className="flex justify-between text-xs text-muted-foreground">
|
||||
<span>
|
||||
{t('import.processed', { current: progress.success + progress.failed, total: progress.total })}
|
||||
</span>
|
||||
<span>
|
||||
{progress.total > 0
|
||||
? Math.round(((progress.success + progress.failed) / progress.total) * 100)
|
||||
: 0}%
|
||||
</span>
|
||||
</div>
|
||||
<Progress
|
||||
value={progress.total > 0 ? ((progress.success + progress.failed) / progress.total) * 100 : 0}
|
||||
className="h-2"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{progress && progress.total > 0 && (
|
||||
<div className="flex gap-4 text-xs">
|
||||
<span className="flex items-center gap-1">
|
||||
<CheckCircle2 className="h-3.5 w-3.5 text-green-600" />
|
||||
{t('import.successCount', { count: progress.success })}
|
||||
</span>
|
||||
<span className="flex items-center gap-1">
|
||||
<AlertTriangle className="h-3.5 w-3.5 text-amber-600" />
|
||||
{t('import.failedCount', { count: progress.failed })}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{progress && progress.failed_details.length > 0 && (
|
||||
<details className="text-xs">
|
||||
<summary className="cursor-pointer text-muted-foreground hover:text-foreground">
|
||||
{t('import.failedDetails', 'Failed items')} ({progress.failed_details.length})
|
||||
</summary>
|
||||
<ScrollArea className="h-32 mt-2">
|
||||
<div className="space-y-1">
|
||||
{progress.failed_details.map((d, i) => (
|
||||
<div key={i} className="text-muted-foreground font-mono text-[11px]">
|
||||
#{d.index}: {d.error_message}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</ScrollArea>
|
||||
</details>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{/* Import button */}
|
||||
<div className="flex justify-between items-center">
|
||||
<div className="text-xs text-muted-foreground">
|
||||
{isPstSelected
|
||||
? t('import.pstFolders', 'PST folder structure will be preserved during import')
|
||||
: (<>{t('import.willImportTo', 'Will import to')}: <span className="font-medium text-foreground">{effectiveFolder}</span></>)}
|
||||
</div>
|
||||
<Button
|
||||
onClick={() => importMutation.mutate()}
|
||||
disabled={!canImport || importMutation.isPending}
|
||||
className="gap-2"
|
||||
>
|
||||
{importMutation.isPending ? (
|
||||
<Upload className="h-4 w-4 animate-pulse" />
|
||||
) : (
|
||||
<Upload className="h-4 w-4" />
|
||||
)}
|
||||
{t('import.startImport', 'Import')}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{/* Import history */}
|
||||
{history.length > 0 && (
|
||||
<CollapsibleHistory
|
||||
history={history}
|
||||
t={t}
|
||||
accountLabel={(id: number) =>
|
||||
accounts.find((a) => a.id === id)?.account_name
|
||||
|| accounts.find((a) => a.id === id)?.email
|
||||
|| String(id)
|
||||
}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</Main>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Import history collapsible ──────────────────────────────────────────
|
||||
|
||||
function statusColor(status: string) {
|
||||
switch (status) {
|
||||
case 'completed': return 'text-green-600';
|
||||
case 'failed': return 'text-destructive';
|
||||
case 'processing': return 'text-amber-600';
|
||||
default: return 'text-muted-foreground';
|
||||
}
|
||||
}
|
||||
|
||||
function statusLabel(status: string) {
|
||||
switch (status) {
|
||||
case 'completed': return 'Completed';
|
||||
case 'failed': return 'Failed';
|
||||
case 'processing': return 'Processing';
|
||||
case 'pending': return 'Pending';
|
||||
default: return status;
|
||||
}
|
||||
}
|
||||
|
||||
function timeAgo(ts: number) {
|
||||
const seconds = Math.floor((Date.now() - ts) / 1000);
|
||||
if (seconds < 60) return `${seconds}s ago`;
|
||||
if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago`;
|
||||
if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago`;
|
||||
return new Date(ts).toLocaleDateString();
|
||||
}
|
||||
|
||||
function CollapsibleHistory({
|
||||
history,
|
||||
t,
|
||||
accountLabel,
|
||||
}: {
|
||||
history: ImportHistory[];
|
||||
t: (key: string) => string;
|
||||
accountLabel: (id: number) => string;
|
||||
}) {
|
||||
const [open, setOpen] = useState(false);
|
||||
|
||||
return (
|
||||
<div className="border rounded-lg">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setOpen(!open)}
|
||||
className="w-full flex items-center gap-2 px-4 py-3 text-sm hover:bg-muted/50 transition-colors rounded-lg"
|
||||
>
|
||||
<Clock className="h-4 w-4 text-muted-foreground" />
|
||||
<span className="font-medium">
|
||||
{t('import.importHistory')}
|
||||
</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
({history.length})
|
||||
</span>
|
||||
<ChevronRight
|
||||
className={cn(
|
||||
'h-4 w-4 ml-auto text-muted-foreground transition-transform',
|
||||
open && 'rotate-90',
|
||||
)}
|
||||
/>
|
||||
</button>
|
||||
{open && (
|
||||
<div className="border-t">
|
||||
<div className="divide-y">
|
||||
{history.map((h) => (
|
||||
<div key={h.id} className="px-4 py-3 text-xs space-y-1.5">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className={cn('font-medium', statusColor(h.status))}>
|
||||
{statusLabel(h.status)}
|
||||
</span>
|
||||
<span className="text-muted-foreground">
|
||||
{accountLabel(h.account_id)} / {h.folder}
|
||||
</span>
|
||||
</div>
|
||||
<span className="text-muted-foreground">{timeAgo(h.created_at)}</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-3 text-muted-foreground">
|
||||
<span>{h.format.toUpperCase()}</span>
|
||||
<span className="text-green-600">{h.success} success</span>
|
||||
{h.duplicates > 0 && <span>{h.duplicates} dup</span>}
|
||||
{h.failed > 0 && <span className="text-destructive">{h.failed} failed</span>}
|
||||
<span>{h.total} total</span>
|
||||
</div>
|
||||
{h.failed_details.length > 0 && (
|
||||
<details className="text-[11px]">
|
||||
<summary className="cursor-pointer text-muted-foreground hover:text-foreground">
|
||||
{t('import.failedDetails')} ({h.failed_details.length})
|
||||
</summary>
|
||||
<div className="mt-1 space-y-0.5 max-h-24 overflow-y-auto">
|
||||
{h.failed_details.map((d, i) => (
|
||||
<div key={i} className="text-muted-foreground font-mono">
|
||||
#{d.index}: {d.error_message}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</details>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -31,6 +31,11 @@ import {
|
||||
PopoverContent,
|
||||
PopoverTrigger,
|
||||
} from '@/components/ui/popover'
|
||||
import {
|
||||
Tooltip,
|
||||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from '@/components/ui/tooltip'
|
||||
|
||||
import useMinimalAccountList from '@/hooks/use-minimal-account-list'
|
||||
import { cn } from '@/lib/utils'
|
||||
@@ -83,6 +88,7 @@ export function AccountPopover() {
|
||||
.filter(a =>
|
||||
!q ||
|
||||
a.email.toLowerCase().includes(q) ||
|
||||
a.name?.toLowerCase().includes(q) ||
|
||||
String(a.id).includes(q)
|
||||
)
|
||||
.sort((a, b) => {
|
||||
@@ -184,9 +190,22 @@ export function AccountPopover() {
|
||||
className="flex-1 truncate text-xs cursor-pointer"
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="truncate">
|
||||
{account.email}
|
||||
</span>
|
||||
{account.name ? (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<span className="truncate">
|
||||
{account.name}
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="top">
|
||||
{account.email}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
) : (
|
||||
<span className="truncate">
|
||||
{account.email}
|
||||
</span>
|
||||
)}
|
||||
<span className="text-[10px] text-muted-foreground">
|
||||
#{account.id}
|
||||
</span>
|
||||
|
||||
@@ -30,13 +30,13 @@ import {
|
||||
CommandGroup,
|
||||
CommandInput,
|
||||
CommandItem,
|
||||
CommandList,
|
||||
} from "@/components/ui/command"
|
||||
import { ScrollArea } from "@/components/ui/scroll-area"
|
||||
|
||||
export function MailFilterPopover() {
|
||||
const { t } = useTranslation()
|
||||
const { filter, setFilter } = useSearchContext()
|
||||
const fields = ['from', 'to', 'cc', 'bcc'] as const
|
||||
const fields = ['from', 'to', 'cc', 'bcc', 'any_recipient', 'any_participant'] as const
|
||||
|
||||
const activeCount = fields.filter(k => !!filter[k]).length
|
||||
|
||||
@@ -196,7 +196,7 @@ function ContactSelectorField({
|
||||
value={searchTerm}
|
||||
onValueChange={setSearchTerm}
|
||||
/>
|
||||
<CommandList className="max-h-[360px]">
|
||||
<ScrollArea className="h-[360px]">
|
||||
{isLoading && (
|
||||
<div className="p-4 text-xs text-center opacity-50">{t('search_contacts.loading')}</div>
|
||||
)}
|
||||
@@ -227,7 +227,7 @@ function ContactSelectorField({
|
||||
</div>
|
||||
)}
|
||||
</CommandGroup>
|
||||
</CommandList>
|
||||
</ScrollArea>
|
||||
</Command>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
|
||||
@@ -89,9 +89,9 @@ export function MailListTable({
|
||||
accessorKey: "source",
|
||||
header: t('search.source'),
|
||||
cell: ({ row }) => {
|
||||
const { from, account_email, mailbox_name, account_id, mailbox_id } = row.original;
|
||||
const { from, account_email, account_name, mailbox_name, account_id, mailbox_id } = row.original;
|
||||
const { setFilter } = useSearchMessages();
|
||||
const accountPrefix = account_email.split('@')[0];
|
||||
const accountPrefix = account_name ?? account_email.split('@')[0];//https://github.com/rustmailer/bichon/issues/306
|
||||
|
||||
return (
|
||||
<div className="flex flex-col py-1.5 min-w-0 group">
|
||||
@@ -139,31 +139,62 @@ export function MailListTable({
|
||||
accessorKey: "to",
|
||||
header: t('search.to'),
|
||||
cell: ({ row }) => {
|
||||
const recipients = row.original.to || [];
|
||||
const recipients: string[] = row.original.to || [];
|
||||
const { setFilter } = useSearchMessages();
|
||||
const MAX_VISIBLE = 2;
|
||||
const visible = recipients.slice(0, MAX_VISIBLE);
|
||||
const hidden = recipients.slice(MAX_VISIBLE);
|
||||
|
||||
const handleClick = (e: React.MouseEvent, email: string) => {
|
||||
e.stopPropagation();
|
||||
setFilter((prev: Record<string, any>) => ({ ...prev, to: email }));
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="group relative flex items-center w-full min-w-0 h-full px-2 overflow-hidden">
|
||||
<div className="group relative flex items-center w-full min-w-0 px-2 py-1.5 overflow-hidden">
|
||||
<div className="absolute left-0 top-0 bottom-0 w-[2px] bg-primary opacity-0 group-hover:opacity-100 transition-opacity" />
|
||||
|
||||
<div className="text-xs flex flex-wrap gap-x-1 min-w-0 flex-1">
|
||||
{recipients.map((email, index) => (
|
||||
<div className="text-xs flex flex-wrap items-center gap-x-1 gap-y-0.5 min-w-0 flex-1">
|
||||
{visible.map((email, index) => (
|
||||
<span key={index} className="flex items-center">
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setFilter((prev: Record<string, any>) => ({ ...prev, to: email }));
|
||||
}}
|
||||
className="hover:text-primary hover:underline transition-colors truncate max-w-[150px]"
|
||||
onClick={(e) => handleClick(e, email)}
|
||||
className="hover:text-primary hover:underline transition-colors truncate max-w-[160px]"
|
||||
>
|
||||
{email}
|
||||
</button>
|
||||
{index < recipients.length - 1 && (
|
||||
{index < visible.length - 1 && (
|
||||
<span className="text-muted-foreground ml-0.5">,</span>
|
||||
)}
|
||||
</span>
|
||||
))}
|
||||
|
||||
{hidden.length > 0 && (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<span
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
className="inline-flex items-center justify-center rounded-full bg-muted text-muted-foreground px-1.5 py-0.5 text-[10px] leading-none cursor-default hover:bg-accent hover:text-accent-foreground transition-colors shrink-0"
|
||||
>
|
||||
+{hidden.length}
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent className="max-w-[480px] bg-popover text-popover-foreground border">
|
||||
<div className="flex flex-col gap-1">
|
||||
{hidden.map((email, idx) => (
|
||||
<button
|
||||
key={idx}
|
||||
type="button"
|
||||
onClick={(e) => handleClick(e, email)}
|
||||
className="text-left text-xs px-1 py-0.5 rounded hover:bg-accent hover:text-accent-foreground hover:underline transition-colors"
|
||||
>
|
||||
{email}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -1,278 +0,0 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import { cn, dateFnsLocaleMap, formatBytes } from "@/lib/utils"
|
||||
import { formatDistanceToNow } from "date-fns"
|
||||
import { MailIcon, MoreVertical, Paperclip, TagIcon, Trash2 } from "lucide-react"
|
||||
import { Skeleton } from "@/components/ui/skeleton"
|
||||
import { Checkbox } from "@/components/ui/checkbox"
|
||||
import { EmailEnvelope } from "@/api"
|
||||
import { useSearchContext } from "./context"
|
||||
import { MailBulkActions } from "./bulk-actions"
|
||||
import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from "@/components/ui/dropdown-menu"
|
||||
import { Button } from "@/components/ui/button"
|
||||
import { Badge } from "@/components/ui/badge"
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { enUS } from "date-fns/locale"
|
||||
|
||||
interface MailListProps {
|
||||
items: EmailEnvelope[]
|
||||
isLoading: boolean
|
||||
onEnvelopeChanged: (envelope: EmailEnvelope) => void
|
||||
}
|
||||
|
||||
export function MailList({
|
||||
items,
|
||||
isLoading,
|
||||
onEnvelopeChanged
|
||||
}: MailListProps) {
|
||||
const { t, i18n } = useTranslation()
|
||||
|
||||
const locale = dateFnsLocaleMap[i18n.language.toLowerCase()] ?? enUS;
|
||||
const { setOpen, currentEnvelope, setCurrentEnvelope, selected, setSelected, setToDelete } = useSearchContext()
|
||||
|
||||
const handleToggleAll = () => {
|
||||
const total = Array.from(selected.values())
|
||||
.reduce((sum, set) => sum + set.size, 0);
|
||||
|
||||
if (total === items.length && items.length > 0) {
|
||||
setSelected(new Map());
|
||||
} else {
|
||||
setSelected(prev => {
|
||||
const next = new Map(prev);
|
||||
for (const item of items) {
|
||||
const set = new Set(next.get(item.account_id) || []);
|
||||
set.add(item.id);
|
||||
next.set(item.account_id, set);
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const toggleToDelete = (accountId: number, mailId: string) => {
|
||||
setToDelete(prev => {
|
||||
const next = new Map(prev);
|
||||
const set = new Set(next.get(accountId) || []);
|
||||
|
||||
if (set.has(mailId)) {
|
||||
set.delete(mailId);
|
||||
if (set.size === 0) next.delete(accountId);
|
||||
else next.set(accountId, set);
|
||||
} else {
|
||||
set.add(mailId);
|
||||
next.set(accountId, set);
|
||||
}
|
||||
|
||||
return next;
|
||||
});
|
||||
};
|
||||
|
||||
const toggleSelected = (accountId: number, mailId: string) => {
|
||||
setSelected(prev => {
|
||||
const next = new Map(prev);
|
||||
const set = new Set(next.get(accountId) || []);
|
||||
|
||||
if (set.has(mailId)) {
|
||||
set.delete(mailId);
|
||||
if (set.size === 0) next.delete(accountId);
|
||||
else next.set(accountId, set);
|
||||
} else {
|
||||
set.add(mailId);
|
||||
next.set(accountId, set);
|
||||
}
|
||||
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
const totalSelected = Array.from(selected.values())
|
||||
.reduce((sum, set) => sum + set.size, 0);
|
||||
|
||||
const hasSelected = (accountId: number, mailId: string) => {
|
||||
return selected.get(accountId)?.has(mailId) ?? false;
|
||||
}
|
||||
|
||||
const handleDelete = (envelope: EmailEnvelope) => {
|
||||
setToDelete(new Map());
|
||||
toggleToDelete(envelope.account_id, envelope.id)
|
||||
setOpen("delete")
|
||||
}
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<div className="divide-y divide-border">
|
||||
{Array.from({ length: 8 }).map((_, i) => (
|
||||
<div key={i} className="flex items-center gap-2 px-2 py-1.5">
|
||||
<Skeleton className="h-3 w-3" />
|
||||
<Skeleton className="h-3 w-3 rounded-full" />
|
||||
<Skeleton className="h-3 flex-1" />
|
||||
<Skeleton className="h-2.5 w-16" />
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="divide-y divide-border">
|
||||
{items.length > 0 && (
|
||||
<div className="flex items-center gap-2 px-2 py-1 bg-muted/30">
|
||||
<Checkbox
|
||||
checked={
|
||||
totalSelected === items.length && items.length > 0
|
||||
? true
|
||||
: totalSelected > 0
|
||||
? "indeterminate"
|
||||
: false
|
||||
}
|
||||
onCheckedChange={handleToggleAll}
|
||||
className="h-4 w-4"
|
||||
/>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{totalSelected > 0
|
||||
? `${t('search.bulkActions.selected', { count: totalSelected })}`
|
||||
: t('common.selectAll')}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{items.map((item, index) => {
|
||||
const hasAttachments = item.regular_attachment_count > 0
|
||||
const isSelectedRow = currentEnvelope?.id === item.id
|
||||
const isChecked = hasSelected(item.account_id, item.id)
|
||||
|
||||
return (
|
||||
<div
|
||||
key={index}
|
||||
className={cn(
|
||||
"flex items-center gap-2 px-2 py-1.5 cursor-pointer transition-colors",
|
||||
"hover:bg-accent/50",
|
||||
isSelectedRow && "bg-accent"
|
||||
)}
|
||||
onClick={(e) => {
|
||||
const target = e.target as HTMLElement
|
||||
if (target.closest('input[type="checkbox"], button')) return
|
||||
onEnvelopeChanged(item)
|
||||
}}
|
||||
>
|
||||
<Checkbox
|
||||
checked={isChecked}
|
||||
onCheckedChange={() => toggleSelected(item.account_id, item.id)}
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
className="h-4 w-4 shrink-0"
|
||||
/>
|
||||
|
||||
<MailIcon className="h-3.5 w-3.5 text-muted-foreground shrink-0" />
|
||||
<div className="flex-1 min-w-0 grid grid-cols-1 sm:grid-cols-12 gap-1 sm:gap-0">
|
||||
|
||||
<div className="col-span-1 sm:col-span-8 flex flex-col min-w-0 gap-0.5">
|
||||
<div className="flex items-center gap-1 min-w-0">
|
||||
<p className="text-sm font-medium truncate">{item.from}</p>
|
||||
<h3 className="text-sm text-muted-foreground truncate hidden sm:block">
|
||||
{item.subject}
|
||||
</h3>
|
||||
</div>
|
||||
<div className="flex items-center gap-1.5 text-[10px] text-muted-foreground/60">
|
||||
<span className="truncate">{item.account_email}</span>
|
||||
<span className="scale-75 opacity-50">•</span>
|
||||
<span className="font-medium text-primary/70">{item.mailbox_name}</span>
|
||||
</div>
|
||||
<h3 className="text-sm text-muted-foreground truncate sm:hidden">
|
||||
{item.subject}
|
||||
</h3>
|
||||
|
||||
<div className="flex flex-wrap gap-1 mt-0.25">
|
||||
{item.tags?.map((tag, i) => (
|
||||
<Badge className="px-1 py-0.5 text-[10px] h-auto leading-none" key={i}>{tag}</Badge>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
<div className="col-span-1 sm:col-span-4 flex items-center justify-end gap-1 text-xs text-muted-foreground">
|
||||
|
||||
{hasAttachments && (
|
||||
<div className="flex items-center gap-1">
|
||||
<Paperclip className="h-3 w-3" />
|
||||
<span>{item.regular_attachment_count}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<span className="hidden md:inline">{formatBytes(item.size)}</span>
|
||||
|
||||
<span className={cn(isSelectedRow ? "text-foreground font-medium" : "text-muted-foreground")}>
|
||||
{item.date && formatDistanceToNow(new Date(item.date), { addSuffix: true, locale })}
|
||||
</span>
|
||||
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-6 w-6 p-0 hover:bg-muted rounded-md"
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
>
|
||||
<MoreVertical className="h-3 w-3" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
|
||||
<DropdownMenuContent align="end" className="w-44">
|
||||
<DropdownMenuItem
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
onSelect={(e) => {
|
||||
e.stopPropagation();
|
||||
setCurrentEnvelope(item);
|
||||
setOpen("edit-tags");
|
||||
}}
|
||||
>
|
||||
<TagIcon className="ml-2 h-3.5 w-3.5" />
|
||||
{t('search.editTag')}
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
onSelect={(e) => {
|
||||
e.stopPropagation();
|
||||
setCurrentEnvelope(item);
|
||||
setOpen("restore");
|
||||
}}
|
||||
>
|
||||
<TagIcon className="ml-2 h-3.5 w-3.5" />
|
||||
{t('restore_message.restore_to_imap')}
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem
|
||||
className="text-destructive focus:text-destructive"
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
onSelect={(e) => {
|
||||
e.stopPropagation();
|
||||
handleDelete(item);
|
||||
}}
|
||||
>
|
||||
<Trash2 className="ml-2 h-3.5 w-3.5" />
|
||||
{t('common.delete')}
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
{totalSelected > 0 && <MailBulkActions />}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -40,6 +40,7 @@ import { MailThreadDialog } from './thread-dialog';
|
||||
import useMinimalAccountList from '@/hooks/use-minimal-account-list';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { NestedEmailDialog } from './nested-email-dialog';
|
||||
import AttachmentPreview, { type PreviewAttachment } from '@/features/attachment/attachment-preview';
|
||||
|
||||
|
||||
interface MailMessageViewProps {
|
||||
@@ -131,6 +132,7 @@ export function MailMessageView({
|
||||
const [threadOpen, setThreadOpen] = useState(false);
|
||||
const [blockRemote, setBlockRemote] = useState(true);
|
||||
const [hasRemoteContent, setHasRemoteContent] = useState(false);
|
||||
const [previewAttachment, setPreviewAttachment] = useState<{ attachments: PreviewAttachment[]; index: number } | null>(null);
|
||||
|
||||
const toggleBlockRemote = () => {
|
||||
setBlockRemote((prev) => !prev);
|
||||
@@ -336,12 +338,23 @@ export function MailMessageView({
|
||||
{icon}
|
||||
</div>
|
||||
<div className="flex items-center justify-between min-w-0 flex-1 gap-2">
|
||||
<span
|
||||
className="truncate text-xs font-medium text-foreground/90"
|
||||
<button
|
||||
type="button"
|
||||
className="truncate text-xs font-medium text-foreground/90 cursor-pointer hover:text-primary hover:underline transition-colors text-left"
|
||||
title={attachment.filename}
|
||||
onClick={() =>
|
||||
setPreviewAttachment({
|
||||
attachments: nonInline.map((a) => ({
|
||||
content_hash: a.content_hash,
|
||||
file_type: a.file_type,
|
||||
filename: a.filename,
|
||||
})),
|
||||
index: i,
|
||||
})
|
||||
}
|
||||
>
|
||||
{attachment.filename}
|
||||
</span>
|
||||
</button>
|
||||
<span className="flex-shrink-0 text-[9px] font-bold text-muted-foreground/60 bg-muted px-1 py-0.5 rounded uppercase">
|
||||
{attachment.file_type.split('/').pop()}
|
||||
</span>
|
||||
@@ -369,10 +382,10 @@ export function MailMessageView({
|
||||
{formatBytes(attachment.size)}
|
||||
</span>
|
||||
{downloadingAttachmentFileName === attachment.filename ? (
|
||||
<Loader className="w-4 h-4 animate-spin" />
|
||||
<Loader className="w-5 h-5 animate-spin" />
|
||||
) : (
|
||||
<Download
|
||||
className="w-4 h-4 cursor-pointer"
|
||||
className="w-5 h-5 cursor-pointer"
|
||||
onClick={() => {
|
||||
setDownloadingAttachmentFileName(attachment.filename);
|
||||
downloadAttachmentMutation.mutate({ content_hash: attachment.content_hash });
|
||||
@@ -447,6 +460,19 @@ export function MailMessageView({
|
||||
fileName={nestedEmlFile?.filename || ''}
|
||||
content_hash={nestedEmlFile?.content_hash}
|
||||
/>
|
||||
{previewAttachment?.attachments?.[previewAttachment.index] && (
|
||||
<AttachmentPreview
|
||||
open={!!previewAttachment}
|
||||
onOpenChange={(open) => !open && setPreviewAttachment(null)}
|
||||
accountId={envelope.account_id}
|
||||
envelopeId={envelope.id}
|
||||
contentHash={previewAttachment.attachments[previewAttachment.index].content_hash}
|
||||
contentType={previewAttachment.attachments[previewAttachment.index].file_type}
|
||||
fileName={previewAttachment.attachments[previewAttachment.index].filename}
|
||||
attachments={previewAttachment.attachments}
|
||||
attachmentIndex={previewAttachment.index}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ describe('Proxy Form Schema', () => {
|
||||
if (!result.success) {
|
||||
expect(
|
||||
result.error.issues.some((i) =>
|
||||
i.message?.includes('http:// or socks5://')
|
||||
i.message?.includes('Invalid format')
|
||||
)
|
||||
).toBe(true)
|
||||
}
|
||||
@@ -53,7 +53,7 @@ describe('Proxy Form Schema', () => {
|
||||
if (!result.success) {
|
||||
expect(
|
||||
result.error.issues.some((i) =>
|
||||
i.message?.includes('Invalid URL format')
|
||||
i.message?.includes('Invalid format')
|
||||
)
|
||||
).toBe(true)
|
||||
}
|
||||
@@ -170,4 +170,27 @@ describe('Proxy Form Schema', () => {
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('url field - non-standard format (host:port:user:pass)', () => {
|
||||
it('accepts non-standard format with auth', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'socks5://server.nodeprovider.com:8080:nodeprovider_a1234_alias_com-country-us-region-california-sid-b123123123-filter-medium:passwordhere',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts simple non-standard format', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'socks5://proxy.example.com:1080:myuser:mypassword',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects non-standard format without password', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'socks5://proxy.example.com:1080:myuser',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -46,7 +46,7 @@ export const getColumns = (t: (key: string) => string): ColumnDef<Proxy>[] => [
|
||||
cell: ({ row }) => {
|
||||
return <LongText>{row.original.url}</LongText>
|
||||
},
|
||||
meta: { className: 'w-60' },
|
||||
meta: { className: 'max-w-60' },
|
||||
},
|
||||
{
|
||||
accessorKey: 'created_at',
|
||||
|
||||
@@ -1,63 +1,175 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
// Parse a proxy URL into components. Supports two formats:
|
||||
// Standard: socks5://[user:pass@]host:port
|
||||
// Non-standard: socks5://host:port:user:pass (some proxy providers)
|
||||
function parseProxyUrl(value: string): {
|
||||
scheme: string
|
||||
host: string
|
||||
port: number
|
||||
username?: string
|
||||
password?: string
|
||||
} | null {
|
||||
// Strip scheme
|
||||
let stripped: string
|
||||
let scheme: string
|
||||
const lower = value.toLowerCase()
|
||||
if (lower.startsWith('socks5://')) {
|
||||
scheme = 'socks5'
|
||||
stripped = value.slice('socks5://'.length)
|
||||
} else if (lower.startsWith('http://')) {
|
||||
scheme = 'http'
|
||||
stripped = value.slice('http://'.length)
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
|
||||
if (!stripped) return null
|
||||
|
||||
// Standard format: user:pass@host:port
|
||||
const atIdx = stripped.lastIndexOf('@')
|
||||
if (atIdx >= 0) {
|
||||
const userinfo = stripped.slice(0, atIdx)
|
||||
const hostport = stripped.slice(atIdx + 1)
|
||||
|
||||
// Parse userinfo
|
||||
let username: string | undefined
|
||||
let password: string | undefined
|
||||
if (userinfo) {
|
||||
const colonIdx = userinfo.indexOf(':')
|
||||
if (colonIdx >= 0) {
|
||||
username = userinfo.slice(0, colonIdx)
|
||||
password = userinfo.slice(colonIdx + 1)
|
||||
} else {
|
||||
username = userinfo
|
||||
}
|
||||
}
|
||||
|
||||
// Parse host:port
|
||||
const { host, port } = splitHostPort(hostport)
|
||||
if (!host || !port) return null
|
||||
|
||||
return { scheme, host, port, username, password }
|
||||
}
|
||||
|
||||
// Non-standard format: host:port[:user[:pass]]
|
||||
const parts = stripped.split(':')
|
||||
if (parts.length === 1) {
|
||||
// host only, default port to 1080
|
||||
const host = parts[0]
|
||||
if (!host) return null
|
||||
return { scheme, host, port: 1080 }
|
||||
}
|
||||
if (parts.length === 2) {
|
||||
// host:port, no auth
|
||||
const host = parts[0]
|
||||
const port = parseInt(parts[1], 10)
|
||||
if (!host || isNaN(port)) return null
|
||||
return { scheme, host, port }
|
||||
}
|
||||
if (parts.length >= 4) {
|
||||
// host:port:username:password (and possibly more colons in user/pass)
|
||||
// Last part = password, second-to-last = username, rest = host:port
|
||||
const password = parts[parts.length - 1]
|
||||
const username = parts[parts.length - 2]
|
||||
const hostport = parts.slice(0, parts.length - 2).join(':')
|
||||
const { host, port } = splitHostPort(hostport)
|
||||
if (!host || !port || !username || !password) return null
|
||||
return { scheme, host, port, username, password }
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
function splitHostPort(hostport: string): { host: string; port: number | null } {
|
||||
if (!hostport) return { host: '', port: null }
|
||||
|
||||
// IPv6: [::1]:1080 or [::1]
|
||||
if (hostport.startsWith('[')) {
|
||||
const close = hostport.indexOf(']')
|
||||
if (close < 0) return { host: '', port: null }
|
||||
const host = hostport.slice(1, close)
|
||||
const after = hostport.slice(close + 1)
|
||||
if (!after.startsWith(':')) {
|
||||
// No port specified, default to 1080
|
||||
return { host, port: 1080 }
|
||||
}
|
||||
const port = parseInt(after.slice(1), 10)
|
||||
return { host, port: isNaN(port) ? null : port }
|
||||
}
|
||||
|
||||
const lastColon = hostport.lastIndexOf(':')
|
||||
if (lastColon < 0) {
|
||||
// No port specified, default to 1080
|
||||
return { host: hostport, port: 1080 }
|
||||
}
|
||||
const host = hostport.slice(0, lastColon)
|
||||
const port = parseInt(hostport.slice(lastColon + 1), 10)
|
||||
return { host, port: isNaN(port) ? null : port }
|
||||
}
|
||||
|
||||
export const proxyFormSchema = z.object({
|
||||
url: z
|
||||
.string()
|
||||
.min(1, 'Proxy address cannot be empty')
|
||||
.superRefine((value, ctx) => {
|
||||
if (value.length === 0) {
|
||||
return
|
||||
}
|
||||
if (value.length === 0) return
|
||||
|
||||
let url: URL
|
||||
try {
|
||||
url = new URL(value)
|
||||
} catch (_e) {
|
||||
// Try our custom parser first (handles both standard and non-standard)
|
||||
const parsed = parseProxyUrl(value)
|
||||
|
||||
if (!parsed) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Invalid URL format',
|
||||
message: 'Invalid format. Expected socks5://[user:pass@]host:port or socks5://host:port:user:pass',
|
||||
path: [],
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if (url.protocol !== 'socks5:' && url.protocol !== 'http:') {
|
||||
if (parsed.scheme !== 'socks5' && parsed.scheme !== 'http') {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'URL must start with http:// or socks5://',
|
||||
path: [],
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if (!/^[a-zA-Z0-9\-\.]+$/.test(url.hostname)) {
|
||||
if (!/^[a-zA-Z0-9\-\.]+$/.test(parsed.host)) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Hostname contains invalid characters',
|
||||
path: [],
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
const port = parseInt(url.port || '1080')
|
||||
if (port <= 0 || port > 65535) {
|
||||
if (parsed.port <= 0 || parsed.port > 65535) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Port must be between 1-65535',
|
||||
path: [],
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if (url.username && !url.password) {
|
||||
if (parsed.username && !parsed.password) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Password cannot be empty when username is provided',
|
||||
path: [],
|
||||
})
|
||||
} else if (url.password && url.password.length < 8) {
|
||||
return
|
||||
}
|
||||
|
||||
if (parsed.password && parsed.password.length < 8) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Password must be at least 8 characters',
|
||||
path: [],
|
||||
})
|
||||
return
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
@@ -20,7 +20,7 @@ import { useState, useMemo } from 'react'
|
||||
import { useFieldArray, useForm } from 'react-hook-form'
|
||||
import { zodResolver } from '@hookform/resolvers/zod'
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import { Loader2, Shield, Settings2, UserIcon, Plus, Trash2, Mail } from 'lucide-react'
|
||||
import { Loader2, Shield, Settings2, UserIcon, Plus, Trash2, Mail, Check, ChevronsUpDown } from 'lucide-react'
|
||||
import { AxiosError } from 'axios'
|
||||
|
||||
import { Button } from '@/components/ui/button'
|
||||
@@ -43,11 +43,21 @@ import {
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { Textarea } from '@/components/ui/textarea'
|
||||
import { ScrollArea } from '@/components/ui/scroll-area'
|
||||
import { cn } from '@/lib/utils'
|
||||
import { Checkbox } from '@/components/ui/checkbox'
|
||||
import { Switch } from '@/components/ui/switch'
|
||||
import { Separator } from '@/components/ui/separator'
|
||||
import { toast } from '@/hooks/use-toast'
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"
|
||||
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"
|
||||
import {
|
||||
Command,
|
||||
CommandEmpty,
|
||||
CommandGroup,
|
||||
CommandInput,
|
||||
CommandItem,
|
||||
CommandList,
|
||||
} from "@/components/ui/command"
|
||||
|
||||
import { create_user, update_user, User } from '@/api/users/api'
|
||||
import useMinimalAccountList from '@/hooks/use-minimal-account-list'
|
||||
@@ -65,6 +75,66 @@ interface Props {
|
||||
onOpenChange: (open: boolean) => void
|
||||
}
|
||||
|
||||
function AccountSearchSelect({
|
||||
accounts,
|
||||
value,
|
||||
onChange,
|
||||
placeholder,
|
||||
}: {
|
||||
accounts: { id: number; email: string; name?: string }[]
|
||||
value: number
|
||||
onChange: (v: number) => void
|
||||
placeholder: string
|
||||
}) {
|
||||
const [open, setOpen] = useState(false)
|
||||
const [search, setSearch] = useState('')
|
||||
|
||||
const selectedAccount = accounts.find(a => a.id === value)
|
||||
|
||||
const filtered = useMemo(() => {
|
||||
const q = search.toLowerCase()
|
||||
return accounts
|
||||
.filter(a =>
|
||||
!q || a.email.toLowerCase().includes(q) || a.name?.toLowerCase().includes(q) || String(a.id).includes(q)
|
||||
)
|
||||
.sort((a, b) => a.email.toLowerCase().localeCompare(b.email.toLowerCase()))
|
||||
}, [accounts, search])
|
||||
|
||||
return (
|
||||
<Popover open={open} onOpenChange={(o) => { setOpen(o); if (!o) setSearch('') }}>
|
||||
<PopoverTrigger asChild>
|
||||
<FormControl>
|
||||
<Button variant="outline" role="combobox" className="bg-muted/10 justify-between w-full font-normal">
|
||||
{selectedAccount ? selectedAccount.email : placeholder}
|
||||
<ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
|
||||
</Button>
|
||||
</FormControl>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent className="p-0 w-[--radix-popover-trigger-width]" align="start">
|
||||
<Command shouldFilter={false}>
|
||||
<CommandInput value={search} onValueChange={setSearch} placeholder={placeholder} />
|
||||
<CommandList>
|
||||
<CommandEmpty>No account found</CommandEmpty>
|
||||
<CommandGroup>
|
||||
{filtered.map(acc => (
|
||||
<CommandItem
|
||||
key={acc.id}
|
||||
value={String(acc.id)}
|
||||
onSelect={() => { onChange(acc.id); setOpen(false) }}
|
||||
>
|
||||
<Check className={cn("mr-2 h-4 w-4", acc.id === value ? "opacity-100" : "opacity-0")} />
|
||||
<span className="truncate">{acc.email}</span>
|
||||
{acc.name && <span className="ml-2 text-xs text-muted-foreground">{acc.name}</span>}
|
||||
</CommandItem>
|
||||
))}
|
||||
</CommandGroup>
|
||||
</CommandList>
|
||||
</Command>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
)
|
||||
}
|
||||
|
||||
export function UserActionDialog({ currentRow, open, onOpenChange }: Props) {
|
||||
const { t } = useTranslation()
|
||||
const isEdit = !!currentRow
|
||||
@@ -299,12 +369,12 @@ export function UserActionDialog({ currentRow, open, onOpenChange }: Props) {
|
||||
<div key={item.id} className="flex items-start gap-3 p-3 border rounded-xl bg-card shadow-sm hover:border-primary/30 transition-colors">
|
||||
<FormField control={form.control} name={`account_access_entries.${index}.accountId`} render={({ field }) => (
|
||||
<FormItem className="flex-1">
|
||||
<Select onValueChange={(v) => field.onChange(Number(v))} value={field.value ? String(field.value) : undefined}>
|
||||
<FormControl><SelectTrigger className="bg-muted/10"><SelectValue placeholder={t('users.actions.fields.select_account')} /></SelectTrigger></FormControl>
|
||||
<SelectContent>
|
||||
{allAccounts?.map(acc => <SelectItem key={acc.id} value={String(acc.id)}>{acc.email}</SelectItem>)}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<AccountSearchSelect
|
||||
accounts={allAccounts ?? []}
|
||||
value={field.value}
|
||||
onChange={(v) => field.onChange(v)}
|
||||
placeholder={t('users.actions.fields.select_account')}
|
||||
/>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)} />
|
||||
|
||||
@@ -64,6 +64,9 @@ export function useCurrentUser() {
|
||||
if (accountId !== undefined) {
|
||||
return accountMap.get(accountId)?.has(perm) ?? false
|
||||
}
|
||||
for (const perms of accountMap.values()) {
|
||||
if (perms.has(perm)) return true
|
||||
}
|
||||
return false
|
||||
})
|
||||
}
|
||||
|
||||
28
web/src/hooks/use-edition.ts
Normal file
28
web/src/hooks/use-edition.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
import axiosInstance from '@/api/axiosInstance'
|
||||
import { useQuery } from '@tanstack/react-query'
|
||||
|
||||
export interface EditionInfo {
|
||||
features: string[]
|
||||
edition: 'community' | 'pro' | 'enterprise'
|
||||
version: string
|
||||
}
|
||||
|
||||
async function fetchEdition(): Promise<EditionInfo> {
|
||||
const { data } = await axiosInstance.get<EditionInfo>('api/v1/features')
|
||||
return data
|
||||
}
|
||||
|
||||
export function useEdition() {
|
||||
const { data } = useQuery({
|
||||
queryKey: ['edition'],
|
||||
queryFn: fetchEdition,
|
||||
staleTime: Infinity,
|
||||
retry: 1,
|
||||
})
|
||||
|
||||
return {
|
||||
isPro: data?.edition === 'pro' || data?.edition === 'enterprise',
|
||||
edition: data?.edition ?? 'community',
|
||||
features: data?.features ?? [],
|
||||
} as const
|
||||
}
|
||||
@@ -413,6 +413,14 @@
|
||||
"subject": "الموضوع",
|
||||
"viewEmbeddedEmail": "عرض البريد الإلكتروني المضمن"
|
||||
},
|
||||
"attachment_preview": {
|
||||
"failedToLoad": "فشل تحميل المعاينة",
|
||||
"notAvailable": "المعاينة غير متوفرة",
|
||||
"notAvailableDesc": "لا يمكن معاينة هذا النوع من الملفات.",
|
||||
"resetZoom": "إعادة تعيين التكبير",
|
||||
"zoomIn": "تكبير",
|
||||
"zoomOut": "تصغير"
|
||||
},
|
||||
"auth": {
|
||||
"areYouSureYouWantToLogOut": "هل أنت متأكد أنك تريد تسجيل الخروج؟",
|
||||
"invalidPassword": "كلمة مرور غير صالحة. الرجاء المحاولة مرة أخرى.",
|
||||
@@ -581,6 +589,53 @@
|
||||
"unauthorizedDesc": "يرجى تسجيل الدخول ببيانات الاعتماد المناسبة للوصول إلى هذا المورد.",
|
||||
"unauthorizedTitle": "وصول غير مصرح به"
|
||||
},
|
||||
"import": {
|
||||
"account": "الحساب",
|
||||
"chooseFiles": "3. اختر الملفات",
|
||||
"completed": "اكتمل الاستيراد",
|
||||
"description": "استيراد ملفات البريد إلى حساب محلي (NoSync). للملفات الكبيرة، استخدم CLI.",
|
||||
"detectedFolder": "مكتشف",
|
||||
"detectedFrom": "مكتشف من",
|
||||
"dropHere": "أفلت ملفات .eml / .mbox / .pst هنا",
|
||||
"failed": "فشل الاستيراد",
|
||||
"failedCount": "{{count}} فشل",
|
||||
"failedDetails": "العناصر الفاشلة",
|
||||
"folder": "المجلد",
|
||||
"folderMethod": "2. اختر طريقة تحديد المجلد",
|
||||
"folderMethodDesc": "كيف سيتم تحديد مجلد البريد المستهدف؟",
|
||||
"folderStructure": "2. هيكل المجلدات",
|
||||
"importHistory": "سجل الاستيراد",
|
||||
"limits": "الحد الأقصى: EML 100 م.ب · MBOX {{maxMbox}} م.ب · PST {{maxPst}} م.ب. للملفات الأكبر ← CLI.",
|
||||
"modeCustom": "أدخل اسم مجلد مخصص",
|
||||
"modeCustomDesc": "اكتب اسم مجلد البريد المستهدف يدويًا.",
|
||||
"modeExisting": "اختر من صناديق البريد الحالية",
|
||||
"modeExistingDesc": "حدد أحد صناديق البريد الموجودة بالفعل في هذا الحساب.",
|
||||
"modeHeader": "كشف تلقائي من ترويسات البريد",
|
||||
"modeHeaderDesc": "قراءة X-Gmail-Labels / X-Bichon-Metadata من الملف. يعتمد على اسم الملف كبديل.",
|
||||
"noAccountFound": "لم يتم العثور على حساب.",
|
||||
"noFileYet": "لم يتم اختيار أي ملف بعد",
|
||||
"noMailboxFound": "لم يتم العثور على صندوق بريد.",
|
||||
"noMailboxes": "لم يتم العثور على صناديق بريد في هذا الحساب.",
|
||||
"orClick": "أو انقر للتصفح",
|
||||
"processed": "تم معالجة {{current}} / {{total}}",
|
||||
"processing": "جاري المعالجة…",
|
||||
"pstFolderDesc": "يحتوي ملف PST على هيكل مجلدات خاص به (مثل Inbox و Sent Items وما إلى ذلك). سيتم إنشاء المجلدات تلقائيًا أثناء الاستيراد.",
|
||||
"pstFolders": "سيتم الحفاظ على هيكل مجلدات PST أثناء الاستيراد",
|
||||
"searchAccount": "البحث عن الحسابات...",
|
||||
"searchMailbox": "البحث عن صناديق البريد...",
|
||||
"selectAccount": "اختر حسابًا",
|
||||
"selectAccountFirst": "يرجى اختيار حساب أولاً.",
|
||||
"selectFileFirst": "يرجى اختيار ملف أولاً لتحديد الخيارات المتاحة.",
|
||||
"selectMailbox": "اختر صندوق بريد...",
|
||||
"source": "المصدر",
|
||||
"startImport": "استيراد",
|
||||
"successCount": "تم استيراد {{count}}",
|
||||
"target": "1. اختر الحساب المستهدف",
|
||||
"title": "استيراد",
|
||||
"uploading": "جاري الرفع…",
|
||||
"uploadingFile": "جاري رفع الملف",
|
||||
"willImportTo": "سيتم الاستيراد إلى"
|
||||
},
|
||||
"mail": {
|
||||
"account": "الحساب",
|
||||
"attachments": "المرفقات",
|
||||
@@ -1001,6 +1056,8 @@
|
||||
"advanced": "متقدم",
|
||||
"advancedFilters": "فلاتر متقدمة",
|
||||
"any": "الكل",
|
||||
"any_participant": "أي طرف في المراسلة",
|
||||
"any_recipient": "أي مستلم",
|
||||
"attachmentName": "اسم المرفق",
|
||||
"attachmentsSize": "المرفقات والحجم",
|
||||
"bcc": "نسخة مخفية",
|
||||
|
||||
@@ -413,6 +413,14 @@
|
||||
"subject": "Emne",
|
||||
"viewEmbeddedEmail": "Vis indlejret e-mail"
|
||||
},
|
||||
"attachment_preview": {
|
||||
"failedToLoad": "Kunne ikke indlæse forhåndsvisning",
|
||||
"notAvailable": "Forhåndsvisning ikke tilgængelig",
|
||||
"notAvailableDesc": "Denne filtype kan ikke forhåndsvises.",
|
||||
"resetZoom": "Nulstil zoom",
|
||||
"zoomIn": "Zoom ind",
|
||||
"zoomOut": "Zoom ud"
|
||||
},
|
||||
"auth": {
|
||||
"areYouSureYouWantToLogOut": "Er du sikker på, du vil logge ud?",
|
||||
"invalidPassword": "Ugyldig adgangskode. Prøv venligst igen.",
|
||||
@@ -581,6 +589,53 @@
|
||||
"unauthorizedDesc": "Log venligst ind med passende legitimationsoplysninger for at få adgang til denne ressource.",
|
||||
"unauthorizedTitle": "Uautoriseret adgang"
|
||||
},
|
||||
"import": {
|
||||
"account": "Konto",
|
||||
"chooseFiles": "3. Vælg filer",
|
||||
"completed": "Import fuldført",
|
||||
"description": "Importer e-mailfiler til en lokal konto (NoSync). Brug CLI til større filer.",
|
||||
"detectedFolder": "Registreret",
|
||||
"detectedFrom": "Registreret fra",
|
||||
"dropHere": "Slip .eml / .mbox / .pst-filer her",
|
||||
"failed": "Import mislykkedes",
|
||||
"failedCount": "{{count}} fejlet",
|
||||
"failedDetails": "Fejlede elementer",
|
||||
"folder": "Mappe",
|
||||
"folderMethod": "2. Vælg mappemetode",
|
||||
"folderMethodDesc": "Hvordan skal destinationsmappen bestemmes?",
|
||||
"folderStructure": "2. Mappestruktur",
|
||||
"importHistory": "Importhistorik",
|
||||
"limits": "Maks: EML 100 MB · MBOX {{maxMbox}} MB · PST {{maxPst}} MB. Større filer → CLI.",
|
||||
"modeCustom": "Indtast et brugerdefineret mappenavn",
|
||||
"modeCustomDesc": "Skriv navnet på destinationsmappen manuelt.",
|
||||
"modeExisting": "Vælg fra eksisterende postkasser",
|
||||
"modeExistingDesc": "Vælg en af de postkasser, der allerede findes på denne konto.",
|
||||
"modeHeader": "Registrer automatisk fra e-mailheadere",
|
||||
"modeHeaderDesc": "Læs X-Gmail-Labels / X-Bichon-Metadata fra filen. Falder tilbage til filnavn.",
|
||||
"noAccountFound": "Ingen konto fundet.",
|
||||
"noFileYet": "Ingen fil valgt endnu",
|
||||
"noMailboxFound": "Ingen postkasse fundet.",
|
||||
"noMailboxes": "Ingen postkasser fundet på denne konto.",
|
||||
"orClick": "eller klik for at gennemse",
|
||||
"processed": "{{current}} / {{total}} behandlet",
|
||||
"processing": "Behandler…",
|
||||
"pstFolderDesc": "PST-filen indeholder sin egen mappestruktur (f.eks. Indbakke, Sendt post osv.). Mapper oprettes automatisk under importen.",
|
||||
"pstFolders": "PST-mappestrukturen vil blive bevaret under importen",
|
||||
"searchAccount": "Søg efter konti...",
|
||||
"searchMailbox": "Søg efter postkasser...",
|
||||
"selectAccount": "Vælg en konto",
|
||||
"selectAccountFirst": "Vælg en konto først.",
|
||||
"selectFileFirst": "Vælg en fil først for at bestemme tilgængelige muligheder.",
|
||||
"selectMailbox": "Vælg en postkasse...",
|
||||
"source": "kilde",
|
||||
"startImport": "Importer",
|
||||
"successCount": "{{count}} importeret",
|
||||
"target": "1. Vælg målkonto",
|
||||
"title": "Import",
|
||||
"uploading": "Uploader…",
|
||||
"uploadingFile": "Uploader fil",
|
||||
"willImportTo": "Vil blive importeret til"
|
||||
},
|
||||
"mail": {
|
||||
"account": "Konto",
|
||||
"attachments": "Vedhæftninger",
|
||||
@@ -1001,6 +1056,8 @@
|
||||
"advanced": "Avanceret",
|
||||
"advancedFilters": "Avancerede filtre",
|
||||
"any": "Alle",
|
||||
"any_participant": "Enhver korrespondent",
|
||||
"any_recipient": "Enhver modtager",
|
||||
"attachmentName": "Vedhæftningsfilnavn",
|
||||
"attachmentsSize": "Vedhæftninger & Størrelse",
|
||||
"bcc": "Blindkopi (BCC)",
|
||||
|
||||
@@ -413,6 +413,14 @@
|
||||
"subject": "Betreff",
|
||||
"viewEmbeddedEmail": "Eingebettete E-Mail anzeigen"
|
||||
},
|
||||
"attachment_preview": {
|
||||
"failedToLoad": "Vorschau konnte nicht geladen werden",
|
||||
"notAvailable": "Vorschau nicht verfügbar",
|
||||
"notAvailableDesc": "Vorschau für diesen Dateityp nicht möglich.",
|
||||
"resetZoom": "Zoom zurücksetzen",
|
||||
"zoomIn": "Vergrößern",
|
||||
"zoomOut": "Verkleinern"
|
||||
},
|
||||
"auth": {
|
||||
"areYouSureYouWantToLogOut": "Sind Sie sicher, dass Sie sich abmelden möchten?",
|
||||
"invalidPassword": "Ungültiges Passwort. Bitte versuchen Sie es erneut.",
|
||||
@@ -581,6 +589,53 @@
|
||||
"unauthorizedDesc": "Bitte melden Sie sich mit gültigen Anmeldeinformationen an, um auf diese Ressource zuzugreifen.",
|
||||
"unauthorizedTitle": "Nicht autorisierter Zugriff"
|
||||
},
|
||||
"import": {
|
||||
"account": "Konto",
|
||||
"chooseFiles": "3. Dateien auswählen",
|
||||
"completed": "Import abgeschlossen",
|
||||
"description": "E-Mail-Dateien in ein lokales Konto (NoSync) importieren. Für größere Dateien CLI nutzen.",
|
||||
"detectedFolder": "Erkannt",
|
||||
"detectedFrom": "Erkannt aus",
|
||||
"dropHere": ".eml / .mbox / .pst-Dateien hierher ziehen",
|
||||
"failed": "Import fehlgeschlagen",
|
||||
"failedCount": "{{count}} fehlgeschlagen",
|
||||
"failedDetails": "Fehlgeschlagene Elemente",
|
||||
"folder": "Ordner",
|
||||
"folderMethod": "2. Ordnermethode wählen",
|
||||
"folderMethodDesc": "Wie soll der Zielordner bestimmt werden?",
|
||||
"folderStructure": "2. Ordnerstruktur",
|
||||
"importHistory": "Importverlauf",
|
||||
"limits": "Max: EML 100 MB · MBOX {{maxMbox}} MB · PST {{maxPst}} MB. Größere Dateien → CLI.",
|
||||
"modeCustom": "Benutzerdefinierten Ordnernamen eingeben",
|
||||
"modeCustomDesc": "Geben Sie den Namen des Zielordners manuell ein.",
|
||||
"modeExisting": "Aus bestehenden Postfächern wählen",
|
||||
"modeExistingDesc": "Wählen Sie ein bereits in diesem Konto vorhandenes Postfach aus.",
|
||||
"modeHeader": "Automatisch aus E-Mail-Headern erkennen",
|
||||
"modeHeaderDesc": "Liest X-Gmail-Labels / X-Bichon-Metadata aus der Datei. Fallback auf Dateiname.",
|
||||
"noAccountFound": "Kein Konto gefunden.",
|
||||
"noFileYet": "Noch keine Datei ausgewählt",
|
||||
"noMailboxFound": "Kein Postfach gefunden.",
|
||||
"noMailboxes": "Keine Postfächer in diesem Konto gefunden.",
|
||||
"orClick": "oder zum Durchsuchen klicken",
|
||||
"processed": "{{current}} / {{total}} verarbeitet",
|
||||
"processing": "Verarbeitung…",
|
||||
"pstFolderDesc": "Die PST-Datei enthält eine eigene Ordnerstruktur (z. B. Posteingang, Gesendete Elemente usw.). Ordner werden beim Import automatisch erstellt.",
|
||||
"pstFolders": "Die PST-Ordnerstruktur wird beim Import beibehalten",
|
||||
"searchAccount": "Konten suchen...",
|
||||
"searchMailbox": "Postfächer suchen...",
|
||||
"selectAccount": "Konto auswählen",
|
||||
"selectAccountFirst": "Wählen Sie zuerst ein Konto aus.",
|
||||
"selectFileFirst": "Wählen Sie zuerst eine Datei aus, um die verfügbaren Optionen zu ermitteln.",
|
||||
"selectMailbox": "Postfach auswählen...",
|
||||
"source": "Quelle",
|
||||
"startImport": "Importieren",
|
||||
"successCount": "{{count}} importiert",
|
||||
"target": "1. Zielkonto auswählen",
|
||||
"title": "Import",
|
||||
"uploading": "Hochladen…",
|
||||
"uploadingFile": "Datei wird hochgeladen",
|
||||
"willImportTo": "Wird importiert in"
|
||||
},
|
||||
"mail": {
|
||||
"account": "Konto",
|
||||
"attachments": "Anhänge",
|
||||
@@ -1001,6 +1056,8 @@
|
||||
"advanced": "Erweitert",
|
||||
"advancedFilters": "Erweiterte Filter",
|
||||
"any": "Beliebig",
|
||||
"any_participant": "Beliebiger Beteiligter",
|
||||
"any_recipient": "Beliebiger Empfänger",
|
||||
"attachmentName": "Anhangsname",
|
||||
"attachmentsSize": "Anhänge & Größe",
|
||||
"bcc": "BCC",
|
||||
|
||||
@@ -404,6 +404,7 @@
|
||||
"downloading": "Downloading...",
|
||||
"emailMessageNotFound": "Unable to find the original email. It may have been deleted.",
|
||||
"name": "Filename",
|
||||
"preview": "Preview",
|
||||
"search_input_placeholder": "Search attachments (use \" \" for phrase search)",
|
||||
"sender": "Sender",
|
||||
"sender_with_count": "Sender ({{count}})",
|
||||
@@ -413,6 +414,14 @@
|
||||
"subject": "Subject",
|
||||
"viewEmbeddedEmail": "View embedded email"
|
||||
},
|
||||
"attachment_preview": {
|
||||
"failedToLoad": "Failed to load preview",
|
||||
"notAvailable": "Preview not available",
|
||||
"notAvailableDesc": "Preview is not supported for this file type.",
|
||||
"resetZoom": "Reset zoom",
|
||||
"zoomIn": "Zoom in",
|
||||
"zoomOut": "Zoom out"
|
||||
},
|
||||
"auth": {
|
||||
"areYouSureYouWantToLogOut": "Are you sure you want to log out?",
|
||||
"invalidPassword": "Invalid password. Please try again.",
|
||||
@@ -423,6 +432,7 @@
|
||||
"sessionExpired": "Session expired!",
|
||||
"sessionExpiredDesc": "Your session has ended due to inactivity. Please log in again to continue.",
|
||||
"somethingWentWrong": "Something went wrong",
|
||||
"ssoLogin": "Sign in with SSO",
|
||||
"username": "Username",
|
||||
"welcome": "Welcome to Bichon",
|
||||
"youWillNeedToLogInAgain": "You will need to log in again to access your account."
|
||||
@@ -581,6 +591,53 @@
|
||||
"unauthorizedDesc": "Please log in with the appropriate credentials to access this resource.",
|
||||
"unauthorizedTitle": "Unauthorized Access"
|
||||
},
|
||||
"import": {
|
||||
"account": "Account",
|
||||
"chooseFiles": "3. Choose files",
|
||||
"completed": "Import complete",
|
||||
"description": "Import email files into a local account (NoSync). For larger files, use the CLI.",
|
||||
"detectedFolder": "Detected",
|
||||
"detectedFrom": "Detected from",
|
||||
"dropHere": "Drop .eml / .mbox / .pst files here",
|
||||
"failed": "Import failed",
|
||||
"failedCount": "{{count}} failed",
|
||||
"failedDetails": "Failed items",
|
||||
"folder": "Folder",
|
||||
"folderMethod": "2. Choose folder method",
|
||||
"folderMethodDesc": "How should the target mail folder be determined?",
|
||||
"folderStructure": "2. Folder structure",
|
||||
"importHistory": "Import History",
|
||||
"limits": "Max: EML 100 MB · MBOX {{maxMbox}} MB · PST {{maxPst}} MB. Larger files → CLI.",
|
||||
"modeCustom": "Enter a custom folder name",
|
||||
"modeCustomDesc": "Manually type the target mail folder name.",
|
||||
"modeExisting": "Choose from existing mailboxes",
|
||||
"modeExistingDesc": "Select one of the mailboxes already present in this account.",
|
||||
"modeHeader": "Auto-detect from email headers",
|
||||
"modeHeaderDesc": "Read X-Gmail-Labels / X-Bichon-Metadata from the uploaded file. Falls back to filename.",
|
||||
"noAccountFound": "No account found.",
|
||||
"noFileYet": "No file selected yet",
|
||||
"noMailboxFound": "No mailbox found.",
|
||||
"noMailboxes": "No mailboxes found in this account.",
|
||||
"orClick": "or click to browse",
|
||||
"processed": "{{current}} / {{total}} processed",
|
||||
"processing": "Processing…",
|
||||
"pstFolderDesc": "The PST file contains its own folder structure (e.g. Inbox, Sent Items, etc.). Folders will be automatically created during import.",
|
||||
"pstFolders": "PST folder structure will be preserved during import",
|
||||
"searchAccount": "Search accounts...",
|
||||
"searchMailbox": "Search mailboxes...",
|
||||
"selectAccount": "Select an account",
|
||||
"selectAccountFirst": "Select an account first.",
|
||||
"selectFileFirst": "Select a file first to determine available options.",
|
||||
"selectMailbox": "Select a mailbox...",
|
||||
"source": "source",
|
||||
"startImport": "Import",
|
||||
"successCount": "{{count}} imported",
|
||||
"target": "1. Select target account",
|
||||
"title": "Import",
|
||||
"uploading": "Uploading…",
|
||||
"uploadingFile": "Uploading file",
|
||||
"willImportTo": "Will import to"
|
||||
},
|
||||
"mail": {
|
||||
"account": "Account",
|
||||
"attachments": "Attachments",
|
||||
@@ -1001,6 +1058,8 @@
|
||||
"advanced": "Advanced",
|
||||
"advancedFilters": "Advanced Filters",
|
||||
"any": "Any",
|
||||
"any_participant": "Any correspondent",
|
||||
"any_recipient": "Any recipient",
|
||||
"attachmentName": "Attachment Name",
|
||||
"attachmentsSize": "Attachments & Size",
|
||||
"bcc": "BCC",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user