mirror of
https://github.com/rustmailer/bichon.git
synced 2026-08-31 01:52:30 +00:00
Compare commits
43 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7dee5a7874 | ||
|
|
166ac21549 | ||
|
|
524201c26d | ||
|
|
ef954e255e | ||
|
|
24fcca70a5 | ||
|
|
2da42134d0 | ||
|
|
5fe45795b9 | ||
|
|
4996dac1ea | ||
|
|
42ce36a439 | ||
|
|
40ae2d49d4 | ||
|
|
5034760517 | ||
|
|
875eb2e309 | ||
|
|
f0d0f202ba | ||
|
|
0d37825625 | ||
|
|
cfb8172607 | ||
|
|
8641bb4b56 | ||
|
|
cdf27f2dd4 | ||
|
|
db36272bae | ||
|
|
cad447275f | ||
|
|
8542ba0d28 | ||
|
|
af089958e9 | ||
|
|
bba1ea5cc7 | ||
|
|
9e55026f12 | ||
|
|
b0f229618c | ||
|
|
ff59d47a0d | ||
|
|
b6957c8ceb | ||
|
|
cd98c050b6 | ||
|
|
11e40750fe | ||
|
|
b0374517e8 | ||
|
|
4e12ae5457 | ||
|
|
495c91b7ae | ||
|
|
e442db4a2d | ||
|
|
cc54063669 | ||
|
|
04745458bc | ||
|
|
6fcc9e0e8e | ||
|
|
220aa268c1 | ||
|
|
e3fd9d2f29 | ||
|
|
41c3b84e65 | ||
|
|
89557700ae | ||
|
|
2f5de48c6a | ||
|
|
debb119d3d | ||
|
|
ce3f8944a3 | ||
|
|
6f572b15ae |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -3,4 +3,5 @@
|
||||
.idea
|
||||
config.toml
|
||||
node_modules
|
||||
dedup_report.txt
|
||||
dedup_report.txt
|
||||
crates/*/target
|
||||
608
Cargo.lock
generated
608
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
48
Cargo.toml
48
Cargo.toml
@@ -3,6 +3,7 @@
|
||||
members = [
|
||||
"crates/memdb",
|
||||
"crates/core",
|
||||
"crates/blob",
|
||||
"crates/server",
|
||||
"crates/cli",
|
||||
"crates/admin",
|
||||
@@ -12,14 +13,15 @@ members = [
|
||||
resolver = "2"
|
||||
|
||||
[workspace.package]
|
||||
version = "1.5.1"
|
||||
version = "1.6.2"
|
||||
edition = "2021"
|
||||
|
||||
[workspace.dependencies]
|
||||
chrono = "0.4.44"
|
||||
chrono = "0.4.45"
|
||||
clap = { version = "4.6.1", features = ["derive", "env"] }
|
||||
memdb = { path = "crates/memdb" }
|
||||
itertools = "0.14.0"
|
||||
bichon-memdb = { path = "crates/memdb" }
|
||||
bichon-blob = { path = "crates/blob" }
|
||||
itertools = "0.15.0"
|
||||
ring = { version = "0.17.14", features = ["std"] }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = "1.0.150"
|
||||
@@ -28,7 +30,7 @@ tracing = "0.1.44"
|
||||
tracing-appender = "0.2.3"
|
||||
tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] }
|
||||
base64 = "0.22.1"
|
||||
snafu = "0.9.0"
|
||||
snafu = "0.9.1"
|
||||
reqwest = { version = "0.12.24", default-features = false, features = [
|
||||
"json",
|
||||
"stream",
|
||||
@@ -37,58 +39,58 @@ reqwest = { version = "0.12.24", default-features = false, features = [
|
||||
"blocking",
|
||||
"socks",
|
||||
] }
|
||||
tokio-socks = "0.5.2"
|
||||
http = "1.4.1"
|
||||
regex = "1.12.3"
|
||||
tokio-socks = "0.5.3"
|
||||
http = "1.4.2"
|
||||
regex = "1.12.4"
|
||||
email_address = "0.2.9"
|
||||
futures = "0.3.32"
|
||||
utf7-imap = "0.3.2"
|
||||
mail-parser = { version = '0.11.3', features = ["serde"] }
|
||||
mail-parser = { version = '0.11.4', features = ["serde"] }
|
||||
# mail-send = "0.5.2"
|
||||
tokio-rustls = { version = "0.26.4", default-features = false, features = [
|
||||
"ring",
|
||||
"tls12",
|
||||
] }
|
||||
timeago = "0.6.0"
|
||||
timeago = "0.6.1"
|
||||
oauth2 = { version = "5.0.0", features = ["reqwest-blocking"] }
|
||||
url = { version = "2.5.8", features = ["serde"] }
|
||||
sysinfo = "0.39.2"
|
||||
sysinfo = "0.39.4"
|
||||
num_cpus = "1.17.0"
|
||||
rand = "0.10.1"
|
||||
rand = "0.10.2"
|
||||
encoding_rs = "0.8.35"
|
||||
webpki-roots = "1.0.7"
|
||||
rustls = { version = "0.23.40", default-features = false, features = ["ring"] }
|
||||
rustls-pki-types = "1.14.1"
|
||||
webpki-roots = "1.0.8"
|
||||
rustls = { version = "0.23.41", default-features = false, features = ["ring"] }
|
||||
rustls-pki-types = "1.15.0"
|
||||
tokio-io-timeout = "1.2.1"
|
||||
semver = "1.0.28"
|
||||
governor = "0.10.4"
|
||||
lru = "0.18.0"
|
||||
mime_guess = "2.0.5"
|
||||
hex = "0.4.3"
|
||||
time = { version = "0.3.47", features = [
|
||||
time = { version = "0.3.53", features = [
|
||||
"formatting",
|
||||
"parsing",
|
||||
"local-offset",
|
||||
] }
|
||||
rust-embed = "8.11.0"
|
||||
rust-embed = "8.12.0"
|
||||
murmur3 = "0.5.2"
|
||||
urlencoding = "2.1.3"
|
||||
dashmap = "6.2.1"
|
||||
gethostname = "1.1.0"
|
||||
itoa = "1.0.18"
|
||||
html2text = "0.17.1"
|
||||
bytes = "1.11.1"
|
||||
bytes = "1.12.0"
|
||||
dialoguer = "0.12.0"
|
||||
console = "0.16.3"
|
||||
mail-send = "0.6.0"
|
||||
console = "0.16.4"
|
||||
mail-send = "0.6.1"
|
||||
rcgen = "0.14.8"
|
||||
rustls-pemfile = "2.2.0"
|
||||
blake3 = "1.8.5"
|
||||
uuid = { version = "1.23.1", features = ["v4", "serde"] }
|
||||
fjall = { version = "3.1.4", features = ["lz4", "metrics", "bytes_1"] }
|
||||
uuid = { version = "1.23.4", features = ["v4", "serde"] }
|
||||
fjall = { version = "3.1.6", features = ["lz4", "metrics", "bytes_1"] }
|
||||
tracing-log = "0.2.0"
|
||||
tokio-util = "0.7.18"
|
||||
indicatif = "0.18.4"
|
||||
indicatif = "0.18.6"
|
||||
|
||||
[profile.release]
|
||||
strip = true
|
||||
|
||||
25
README.md
25
README.md
@@ -715,6 +715,31 @@ cargo test
|
||||
|
||||
Feel free to open an [Issue](https://github.com/rustmailer/bichon/issues) or join the [Discord](https://discord.gg/Bq4M2cDmF4) to discuss ideas.
|
||||
|
||||
#### Guidelines
|
||||
|
||||
1. **AI-assisted, not AI-authored.** Use AI to help analyze, debug, or draft code when unsure — but understand and review every change yourself before submitting. Don't submit unreviewed AI-generated content.
|
||||
2. **Keep PRs scoped to one issue.** Don't bundle unrelated changes (CI config, dependency bumps, fixes to other modules) into the same PR. Split them into separate PRs.
|
||||
3. **Frontend/backend changes go together.** If a change affects an API, data structure, or behavior with a frontend consumer, update the frontend in the same PR (or a clearly linked companion PR).
|
||||
4. **Unit tests are required.** New or fixed logic must include tests that reproduce the original issue and verify the fix. PRs without tests won't be merged.
|
||||
5. **Maintain backward compatibility.** Changes to data formats, protocols, configs, or APIs must state whether they're backward compatible. If not, include a migration plan.
|
||||
6. **State the blast radius.** PR descriptions must specify which modules/APIs/data are affected and any downstream impact.
|
||||
|
||||
### Commit Messages
|
||||
|
||||
Format: `<type>(<scope>): <subject>`
|
||||
|
||||
- **type**: `fix`, `feat`, `refactor`, `ci`, `test`, `docs`, `chore`
|
||||
- **scope**: affected module/component (e.g. `rustmailer#286`, `dedup_cache`)
|
||||
- **subject**: imperative, present tense, no period
|
||||
|
||||
Rules:
|
||||
- One logical change per commit — don't mix a fix with CI tweaks or unrelated module changes.
|
||||
- Reference the issue number when applicable (e.g. `fix(#286): ...`).
|
||||
- Body explains *why*, not just *what* — include root cause and how it was verified for non-trivial fixes.
|
||||
- Rebase before submitting — squash WIP/fixup commits into a clean, logical sequence.
|
||||
- No vague messages like `update`, `fix bug`, `wip`.
|
||||
|
||||
|
||||
## Tech Stack
|
||||
|
||||
| Layer | Technology |
|
||||
|
||||
@@ -17,4 +17,4 @@ serde_json.workspace = true
|
||||
|
||||
itertools.workspace = true
|
||||
snafu.workspace = true
|
||||
memdb.workspace = true
|
||||
bichon-memdb.workspace = true
|
||||
@@ -18,9 +18,9 @@ use bichon_core::{
|
||||
token::TokenType,
|
||||
users::{acl::AccessControl, role::RoleType},
|
||||
};
|
||||
use bichon_memdb::{Durability, MemDb};
|
||||
use console::style;
|
||||
use itertools::Itertools;
|
||||
use memdb::{Durability, MemDb};
|
||||
use native_db::*;
|
||||
use native_model::{native_model, Model};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -263,6 +263,8 @@ impl From<AccountV3> for AccountModel {
|
||||
auto_download_new_mailboxes: None,
|
||||
download_schedule: None,
|
||||
deleting: false,
|
||||
archive_rules: None,
|
||||
extraction_rules: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -574,6 +576,8 @@ impl From<BichonUserV2> for bichon_core::users::BichonUserV2 {
|
||||
acl: value.acl,
|
||||
theme: value.theme,
|
||||
language: value.language,
|
||||
sso_id: None,
|
||||
sso_provider: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
1105
crates/blob/Cargo.lock
generated
Normal file
1105
crates/blob/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
24
crates/blob/Cargo.toml
Normal file
24
crates/blob/Cargo.toml
Normal file
@@ -0,0 +1,24 @@
|
||||
[package]
|
||||
name = "bichon-blob"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Embedded KV storage engine for email"
|
||||
|
||||
[dependencies]
|
||||
crc32fast = "1.4"
|
||||
zstd = "0.13"
|
||||
lz4_flex = "0.13.1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
bincode = "1"
|
||||
tracing = "0.1"
|
||||
thiserror = "2"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
rand = "0.10.1"
|
||||
criterion = { version = "0.6", features = ["html_reports"] }
|
||||
|
||||
[[bench]]
|
||||
name = "benchmark"
|
||||
harness = false
|
||||
316
crates/blob/benches/benchmark.rs
Normal file
316
crates/blob/benches/benchmark.rs
Normal file
@@ -0,0 +1,316 @@
|
||||
use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput};
|
||||
use std::time::Duration;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use bichon_blob::{Codec, Config, Engine};
|
||||
|
||||
fn make_key(seed: u64) -> [u8; 32] {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..8].copy_from_slice(&seed.to_le_bytes());
|
||||
key
|
||||
}
|
||||
|
||||
fn make_value(size: usize) -> Vec<u8> {
|
||||
let mut v = Vec::with_capacity(size);
|
||||
// Fill with somewhat realistic text-like data so compression works
|
||||
let pattern = b"The quick brown fox jumps over the lazy dog. ";
|
||||
while v.len() < size {
|
||||
let rem = size - v.len();
|
||||
let n = rem.min(pattern.len());
|
||||
v.extend_from_slice(&pattern[..n]);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
pub fn bench_write_small(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("write");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(1024); // 1 KB
|
||||
let mut counter = 0u64;
|
||||
|
||||
group.bench_function("1KB", |b| {
|
||||
b.iter_batched(
|
||||
|| {
|
||||
counter += 1;
|
||||
(make_key(counter), value.clone())
|
||||
},
|
||||
|(key, val)| {
|
||||
engine
|
||||
.write("bench", key, &val, Codec::Zstd)
|
||||
.unwrap()
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_write_medium(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("write");
|
||||
group.throughput(Throughput::Bytes(64 * 1024));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(64 * 1024); // 64 KB
|
||||
let mut counter = 0u64;
|
||||
|
||||
group.bench_function("64KB", |b| {
|
||||
b.iter_batched(
|
||||
|| {
|
||||
counter += 1;
|
||||
(make_key(counter), value.clone())
|
||||
},
|
||||
|(key, val)| {
|
||||
engine
|
||||
.write("bench", key, &val, Codec::Zstd)
|
||||
.unwrap()
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_write_large(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("write");
|
||||
group.throughput(Throughput::Bytes(1024 * 1024));
|
||||
group.measurement_time(Duration::from_secs(15));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(1024 * 1024); // 1 MB
|
||||
let mut counter = 0u64;
|
||||
|
||||
group.bench_function("1MB", |b| {
|
||||
b.iter_batched(
|
||||
|| {
|
||||
counter += 1;
|
||||
(make_key(counter), value.clone())
|
||||
},
|
||||
|(key, val)| {
|
||||
engine
|
||||
.write("bench", key, &val, Codec::Zstd)
|
||||
.unwrap()
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_read_cache_hit(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("read");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
// Pre-populate: 10 keys, all in same bucket → cache hit after first read
|
||||
let value = make_value(4096);
|
||||
for i in 0..10u64 {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut counter = 0u64;
|
||||
group.bench_function("cache_hit", |b| {
|
||||
b.iter(|| {
|
||||
let key = make_key(counter % 10);
|
||||
counter += 1;
|
||||
std::hint::black_box(engine.read("bench", &key).unwrap());
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_read_cache_miss(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("read");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut config = Config::default();
|
||||
config.lru_bucket_count = 8; // Small cache to force misses
|
||||
let engine = Engine::open(dir.path(), config).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(4096);
|
||||
// Write 1000 keys spread across all 16 buckets — small LRU will thrash
|
||||
for i in 0..1000u64 {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut counter = 0u64;
|
||||
group.bench_function("cache_miss", |b| {
|
||||
b.iter(|| {
|
||||
let key = make_key(counter % 1000);
|
||||
counter += 1;
|
||||
std::hint::black_box(engine.read("bench", &key).unwrap());
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_read_large_value(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("read");
|
||||
group.throughput(Throughput::Bytes(1024 * 1024));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(1024 * 1024); // 1 MB
|
||||
for i in 0..5u64 {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut counter = 0u64;
|
||||
group.bench_function("1MB_cache_hit", |b| {
|
||||
b.iter(|| {
|
||||
let key = make_key(counter % 5);
|
||||
counter += 1;
|
||||
std::hint::black_box(engine.read("bench", &key).unwrap());
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_delete(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("delete");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
group.bench_function("delete", |b| {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(4096);
|
||||
let mut counter = 0u64;
|
||||
|
||||
b.iter_batched(
|
||||
|| {
|
||||
counter += 1;
|
||||
let key = make_key(counter);
|
||||
engine
|
||||
.write("bench", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
key
|
||||
},
|
||||
|key| {
|
||||
engine.delete("bench", &key).unwrap();
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_mixed_workload(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("mixed");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(15));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
// Pre-populate with 500 entries
|
||||
let value = make_value(8192);
|
||||
for i in 0..500u64 {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut counter: u64 = 500;
|
||||
group.bench_function("80w_15r_5d", |b| {
|
||||
b.iter(|| {
|
||||
counter += 1;
|
||||
let op = counter % 100;
|
||||
match op {
|
||||
0..=79 => {
|
||||
// 80% writes
|
||||
let key = make_key(counter);
|
||||
let val = make_value(4096);
|
||||
engine.write("bench", key, &val, Codec::Zstd).unwrap();
|
||||
}
|
||||
80..=94 => {
|
||||
// 15% reads
|
||||
std::hint::black_box(engine.read("bench", &make_key(counter % 500)).unwrap());
|
||||
}
|
||||
_ => {
|
||||
// 5% deletes
|
||||
if counter % 2 == 0 {
|
||||
let key = make_key(counter % 500);
|
||||
let _ = engine.delete("bench", &key);
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_gc(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("gc");
|
||||
group.measurement_time(Duration::from_secs(30));
|
||||
group.sample_size(10);
|
||||
|
||||
group.bench_function("gc_30pct_deleted", |b| {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
// Fill a segment with ~1000 entries, then delete 30%
|
||||
let value = make_value(200_000); // 200KB each → ~1000 entries to fill 256MB
|
||||
let n = 1200u64;
|
||||
for i in 0..n {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
// Delete ~30%
|
||||
for i in (0..n).step_by(3) {
|
||||
engine.delete("bench", &make_key(i)).unwrap();
|
||||
}
|
||||
|
||||
b.iter(|| {
|
||||
engine.gc("bench").unwrap();
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
benches,
|
||||
bench_write_small,
|
||||
bench_write_medium,
|
||||
bench_write_large,
|
||||
bench_read_cache_hit,
|
||||
bench_read_cache_miss,
|
||||
bench_read_large_value,
|
||||
bench_delete,
|
||||
bench_mixed_workload,
|
||||
bench_gc,
|
||||
);
|
||||
criterion_main!(benches);
|
||||
284
crates/blob/src/account.rs
Normal file
284
crates/blob/src/account.rs
Normal file
@@ -0,0 +1,284 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex, RwLock};
|
||||
|
||||
use crate::bucket::{self, BucketFile, IndexRecord};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::file_pool::FilePool;
|
||||
use crate::meta::{AccountMeta, SegmentStats};
|
||||
use crate::segment::{self, SegmentReader, SegmentWriter};
|
||||
use crate::types::Codec;
|
||||
|
||||
// ── AccountHandle ──────────────────────────────────────────────────────────
|
||||
|
||||
pub struct AccountHandle {
|
||||
id: String,
|
||||
dir: PathBuf,
|
||||
inner: RwLock<AccountInner>,
|
||||
pub(crate) write_mutex: Mutex<()>,
|
||||
file_pool: FilePool,
|
||||
}
|
||||
|
||||
impl AccountHandle {
|
||||
pub fn id(&self) -> &str {
|
||||
&self.id
|
||||
}
|
||||
|
||||
pub fn dir(&self) -> &Path {
|
||||
&self.dir
|
||||
}
|
||||
|
||||
/// Open an existing account.
|
||||
pub fn open(store_root: &Path, account_id: &str) -> Result<Arc<Self>> {
|
||||
let dir = store_root.join("accounts").join(account_id);
|
||||
if !dir.exists() {
|
||||
return Err(Error::AccountNotFound(account_id.to_string()));
|
||||
}
|
||||
let inner = AccountInner::open(&dir)?;
|
||||
Ok(Arc::new(Self {
|
||||
id: account_id.to_string(),
|
||||
dir,
|
||||
inner: RwLock::new(inner),
|
||||
write_mutex: Mutex::new(()),
|
||||
file_pool: FilePool::new(8),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Create a new account.
|
||||
pub fn create(store_root: &Path, account_id: &str) -> Result<Arc<Self>> {
|
||||
let dir = store_root.join("accounts").join(account_id);
|
||||
if dir.exists() {
|
||||
return Err(Error::AccountAlreadyExists(account_id.to_string()));
|
||||
}
|
||||
let inner = AccountInner::create(&dir, account_id)?;
|
||||
Ok(Arc::new(Self {
|
||||
id: account_id.to_string(),
|
||||
dir,
|
||||
inner: RwLock::new(inner),
|
||||
write_mutex: Mutex::new(()),
|
||||
file_pool: FilePool::new(8),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Lock the inner state for reading.
|
||||
pub fn read(&self) -> std::sync::RwLockReadGuard<'_, AccountInner> {
|
||||
self.inner.read().unwrap()
|
||||
}
|
||||
|
||||
/// Lock the inner state for writing.
|
||||
pub fn write(&self) -> std::sync::RwLockWriteGuard<'_, AccountInner> {
|
||||
self.inner.write().unwrap()
|
||||
}
|
||||
|
||||
/// Get a cached file handle for a segment.
|
||||
pub fn get_segment_file(&self, seg_id: u32, path: &Path) -> Result<Arc<Mutex<std::fs::File>>> {
|
||||
self.file_pool.get(seg_id, path)
|
||||
}
|
||||
|
||||
/// Invalidate cached file handles for a segment (after GC).
|
||||
pub fn invalidate_file_cache(&self, seg_id: u32) {
|
||||
self.file_pool.invalidate(seg_id);
|
||||
}
|
||||
}
|
||||
|
||||
// ── AccountInner ───────────────────────────────────────────────────────────
|
||||
|
||||
pub struct AccountInner {
|
||||
dir: PathBuf,
|
||||
meta: AccountMeta,
|
||||
active_writer: SegmentWriter,
|
||||
readers: HashMap<u32, SegmentReader>,
|
||||
}
|
||||
|
||||
impl AccountInner {
|
||||
fn open(dir: &Path) -> Result<Self> {
|
||||
let meta = AccountMeta::load(dir)?;
|
||||
|
||||
let seg_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(meta.active_segment_id));
|
||||
let active_writer = if seg_path.exists() {
|
||||
SegmentWriter::open_append(seg_path, meta.active_segment_id)?
|
||||
} else {
|
||||
fs::create_dir_all(dir.join("segments"))?;
|
||||
SegmentWriter::create(seg_path, meta.active_segment_id)?
|
||||
};
|
||||
|
||||
let mut readers = HashMap::new();
|
||||
for (&seg_id, stats) in &meta.segments {
|
||||
if stats.sealed {
|
||||
let seg_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(seg_id));
|
||||
if seg_path.exists() {
|
||||
readers.insert(seg_id, SegmentReader::open(seg_path, seg_id)?);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
dir: dir.to_path_buf(),
|
||||
meta,
|
||||
active_writer,
|
||||
readers,
|
||||
})
|
||||
}
|
||||
|
||||
fn create(dir: &Path, account_id: &str) -> Result<Self> {
|
||||
fs::create_dir_all(dir.join("segments"))?;
|
||||
BucketFile::ensure_dir(dir)?;
|
||||
|
||||
let meta = AccountMeta::new(account_id.to_string(), 1);
|
||||
|
||||
let seg_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(1));
|
||||
let active_writer = SegmentWriter::create(seg_path, 1)?;
|
||||
|
||||
meta.save(dir)?;
|
||||
|
||||
Ok(Self {
|
||||
dir: dir.to_path_buf(),
|
||||
meta,
|
||||
active_writer,
|
||||
readers: HashMap::new(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn meta(&self) -> &AccountMeta {
|
||||
&self.meta
|
||||
}
|
||||
|
||||
/// Mark the segment as indexed up to the given offset and persist meta.
|
||||
pub fn mark_indexed(&mut self, segment_id: u32, indexed_up_to_offset: u64) -> Result<()> {
|
||||
if let Some(stats) = self.meta.segments.get_mut(&segment_id) {
|
||||
if indexed_up_to_offset > stats.indexed_up_to_offset {
|
||||
stats.indexed_up_to_offset = indexed_up_to_offset;
|
||||
}
|
||||
}
|
||||
self.meta.save(&self.dir)
|
||||
}
|
||||
|
||||
/// Append an entry without fsync.
|
||||
pub fn append_entry(
|
||||
&mut self,
|
||||
key: [u8; 32],
|
||||
data: &[u8],
|
||||
flags: u8,
|
||||
codec: Codec,
|
||||
) -> Result<(u32, u64, u32)> {
|
||||
if self.active_writer.is_full() {
|
||||
self.seal_active()?;
|
||||
}
|
||||
|
||||
use crate::segment::Entry;
|
||||
let entry = if flags == 1 {
|
||||
Entry::tombstone(key)
|
||||
} else {
|
||||
Entry::new(key, data, flags, codec)
|
||||
};
|
||||
|
||||
let data_size = entry.data.len() as u32;
|
||||
let segment_id = self.active_writer.id();
|
||||
let offset = self.active_writer.append(&entry)?;
|
||||
|
||||
let stats = self
|
||||
.meta
|
||||
.segments
|
||||
.entry(segment_id)
|
||||
.or_insert_with(|| SegmentStats::new(segment_id));
|
||||
stats.total_bytes += data_size as u64;
|
||||
if flags == 1 {
|
||||
stats.deleted_bytes += entry.raw_size as u64;
|
||||
}
|
||||
stats.recompute_ratio();
|
||||
|
||||
Ok((segment_id, offset, data_size))
|
||||
}
|
||||
|
||||
/// Fsync the active segment and persist meta.
|
||||
pub fn flush_active(&mut self) -> Result<()> {
|
||||
self.active_writer.fsync()?;
|
||||
self.meta.save(&self.dir)
|
||||
}
|
||||
|
||||
/// Write an entry with fsync.
|
||||
pub fn write_entry(
|
||||
&mut self,
|
||||
key: [u8; 32],
|
||||
data: &[u8],
|
||||
flags: u8,
|
||||
codec: Codec,
|
||||
) -> Result<(u32, u64, u32)> {
|
||||
let result = self.append_entry(key, data, flags, codec)?;
|
||||
self.flush_active()?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn seal_active(&mut self) -> Result<()> {
|
||||
let old_id = self.active_writer.id();
|
||||
let old_stats = self
|
||||
.meta
|
||||
.segments
|
||||
.entry(old_id)
|
||||
.or_insert_with(|| SegmentStats::new(old_id));
|
||||
old_stats.sealed = true;
|
||||
|
||||
let seg_path = self
|
||||
.dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(old_id));
|
||||
self.readers
|
||||
.insert(old_id, SegmentReader::open(seg_path, old_id)?);
|
||||
|
||||
let new_id = old_id + 1;
|
||||
self.meta.active_segment_id = new_id;
|
||||
let new_path = self
|
||||
.dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(new_id));
|
||||
self.active_writer = SegmentWriter::create(new_path, new_id)?;
|
||||
self.meta.save(&self.dir)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the on-disk path for a segment.
|
||||
pub fn segment_path(&self, segment_id: u32) -> Result<PathBuf> {
|
||||
let filename = segment::segment_filename(segment_id);
|
||||
let path = self.dir.join("segments").join(&filename);
|
||||
if path.exists() {
|
||||
Ok(path)
|
||||
} else {
|
||||
Err(Error::SegmentNotFound(segment_id))
|
||||
}
|
||||
}
|
||||
|
||||
/// Append index record to the appropriate bucket file.
|
||||
pub fn append_index(&self, record: &IndexRecord) -> Result<()> {
|
||||
let bucket_id = bucket::bucket_id(&record.key);
|
||||
let bf = BucketFile::open(&self.dir, bucket_id);
|
||||
bf.append(record)
|
||||
}
|
||||
|
||||
/// Return list of sealed segment IDs.
|
||||
pub fn sealed_segments(&self) -> Vec<u32> {
|
||||
self.meta
|
||||
.segments
|
||||
.iter()
|
||||
.filter(|(_, s)| s.sealed)
|
||||
.map(|(id, _)| *id)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// All segment IDs (including active).
|
||||
pub fn all_segment_ids(&self) -> Vec<u32> {
|
||||
let mut ids: Vec<u32> = self.meta.segments.keys().copied().collect();
|
||||
if !ids.contains(&self.meta.active_segment_id) {
|
||||
ids.push(self.meta.active_segment_id);
|
||||
}
|
||||
ids.sort_unstable();
|
||||
ids
|
||||
}
|
||||
}
|
||||
329
crates/blob/src/bucket.rs
Normal file
329
crates/blob/src/bucket.rs
Normal file
@@ -0,0 +1,329 @@
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::types::{BUCKET_COUNT, INDEX_RECORD_SIZE};
|
||||
|
||||
/// On-disk format: 52 bytes per record.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct IndexRecord {
|
||||
pub key: [u8; 32],
|
||||
pub segment_id: u32,
|
||||
pub offset: u64,
|
||||
pub data_size: u32,
|
||||
pub flags: u8,
|
||||
}
|
||||
|
||||
impl IndexRecord {
|
||||
pub fn new(key: [u8; 32], segment_id: u32, offset: u64, data_size: u32, flags: u8) -> Self {
|
||||
Self {
|
||||
key,
|
||||
segment_id,
|
||||
offset,
|
||||
data_size,
|
||||
flags,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_tombstone(&self) -> bool {
|
||||
self.flags == 1
|
||||
}
|
||||
|
||||
pub fn encode(&self) -> [u8; INDEX_RECORD_SIZE] {
|
||||
let mut buf = [0u8; INDEX_RECORD_SIZE];
|
||||
buf[0..32].copy_from_slice(&self.key);
|
||||
buf[32..36].copy_from_slice(&self.segment_id.to_le_bytes());
|
||||
buf[36..44].copy_from_slice(&self.offset.to_le_bytes());
|
||||
buf[44..48].copy_from_slice(&self.data_size.to_le_bytes());
|
||||
buf[48] = self.flags;
|
||||
// bytes 49..52 are padding (keep zero)
|
||||
buf
|
||||
}
|
||||
|
||||
pub fn decode(buf: &[u8; INDEX_RECORD_SIZE]) -> Self {
|
||||
let mut key = [0u8; 32];
|
||||
key.copy_from_slice(&buf[0..32]);
|
||||
let segment_id = u32::from_le_bytes(buf[32..36].try_into().unwrap());
|
||||
let offset = u64::from_le_bytes(buf[36..44].try_into().unwrap());
|
||||
let data_size = u32::from_le_bytes(buf[44..48].try_into().unwrap());
|
||||
let flags = buf[48];
|
||||
Self {
|
||||
key,
|
||||
segment_id,
|
||||
offset,
|
||||
data_size,
|
||||
flags,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Represents a loaded and deduplicated bucket in memory.
|
||||
pub struct BucketIndex {
|
||||
pub bucket_id: u16,
|
||||
/// Records sorted by key, deduplicated (one record per key, latest wins).
|
||||
pub records: Vec<IndexRecord>,
|
||||
}
|
||||
|
||||
impl BucketIndex {
|
||||
/// Build from raw records: sort by key, dedup keeping the one with max offset.
|
||||
pub fn from_records(mut records: Vec<IndexRecord>, bucket_id: u16) -> Self {
|
||||
records.sort_by_key(|a| a.key);
|
||||
// Dedup: keep last (max offset) for each key
|
||||
let mut deduped = Vec::with_capacity(records.len());
|
||||
let mut i = 0;
|
||||
while i < records.len() {
|
||||
let mut best = i;
|
||||
let mut j = i + 1;
|
||||
while j < records.len() && records[j].key == records[i].key {
|
||||
if records[j].offset > records[best].offset {
|
||||
best = j;
|
||||
}
|
||||
j += 1;
|
||||
}
|
||||
deduped.push(records[best].clone());
|
||||
i = j;
|
||||
}
|
||||
Self {
|
||||
bucket_id,
|
||||
records: deduped,
|
||||
}
|
||||
}
|
||||
|
||||
/// Binary search for a key. Returns the record if found.
|
||||
pub fn find(&self, key: &[u8; 32]) -> Option<&IndexRecord> {
|
||||
match self.records.binary_search_by(|r| r.key.cmp(key)) {
|
||||
Ok(idx) => Some(&self.records[idx]),
|
||||
Err(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a new record and maintain sorted order.
|
||||
pub fn insert(&mut self, record: IndexRecord) {
|
||||
match self.records.binary_search_by(|r| r.key.cmp(&record.key)) {
|
||||
Ok(idx) => {
|
||||
// Replace if newer (larger offset)
|
||||
if record.offset > self.records[idx].offset {
|
||||
self.records[idx] = record;
|
||||
}
|
||||
}
|
||||
Err(idx) => {
|
||||
self.records.insert(idx, record);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.records.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.records.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// Manages a bucket index file on disk.
|
||||
pub struct BucketFile {
|
||||
path: PathBuf,
|
||||
bucket_id: u16,
|
||||
}
|
||||
|
||||
impl BucketFile {
|
||||
pub fn path_for(account_dir: &Path, bucket_id: u16) -> PathBuf {
|
||||
account_dir.join("buckets").join(format!("{:02x}.idx", bucket_id))
|
||||
}
|
||||
|
||||
pub fn open(account_dir: &Path, bucket_id: u16) -> Self {
|
||||
Self {
|
||||
path: Self::path_for(account_dir, bucket_id),
|
||||
bucket_id,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub fn bucket_id(&self) -> u16 {
|
||||
self.bucket_id
|
||||
}
|
||||
|
||||
/// Ensure the buckets directory exists.
|
||||
pub fn ensure_dir(account_dir: &Path) -> Result<()> {
|
||||
let dir = account_dir.join("buckets");
|
||||
std::fs::create_dir_all(&dir)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append a single record to the bucket file.
|
||||
pub fn append(&self, record: &IndexRecord) -> Result<()> {
|
||||
let mut file = OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&self.path)?;
|
||||
file.write_all(&record.encode())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append multiple records at once.
|
||||
pub fn append_batch(&self, records: &[IndexRecord]) -> Result<()> {
|
||||
if records.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut file = OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&self.path)?;
|
||||
for r in records {
|
||||
file.write_all(&r.encode())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Load all records from the bucket file.
|
||||
/// If the file size is not a multiple of INDEX_RECORD_SIZE (partial write),
|
||||
/// the trailing bytes are silently ignored.
|
||||
pub fn load_all(&self) -> Result<Vec<IndexRecord>> {
|
||||
if !self.path.exists() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let data = std::fs::read(&self.path)?;
|
||||
let remainder = data.len() % INDEX_RECORD_SIZE;
|
||||
let count = data.len() / INDEX_RECORD_SIZE;
|
||||
let mut records = Vec::with_capacity(count);
|
||||
for i in 0..count {
|
||||
let start = i * INDEX_RECORD_SIZE;
|
||||
let end = start + INDEX_RECORD_SIZE;
|
||||
let buf: &[u8; INDEX_RECORD_SIZE] = data[start..end]
|
||||
.try_into()
|
||||
.map_err(|_| crate::error::Error::BucketIndexCorrupt {
|
||||
path: self.path.clone(),
|
||||
reason: format!("unexpected file size {}, not a multiple of {}", data.len(), INDEX_RECORD_SIZE),
|
||||
})?;
|
||||
records.push(IndexRecord::decode(buf));
|
||||
}
|
||||
if remainder > 0 {
|
||||
tracing::warn!(
|
||||
"Bucket file {:?} has {} trailing bytes (expected multiple of {}), ignoring",
|
||||
self.path, remainder, INDEX_RECORD_SIZE
|
||||
);
|
||||
}
|
||||
Ok(records)
|
||||
}
|
||||
|
||||
/// Load all records, sort, and deduplicate into a BucketIndex.
|
||||
pub fn load_index(&self) -> Result<BucketIndex> {
|
||||
let records = self.load_all()?;
|
||||
Ok(BucketIndex::from_records(records, self.bucket_id))
|
||||
}
|
||||
|
||||
/// Rewrite the bucket file with a sorted, deduplicated set of records.
|
||||
/// Uses atomic temp+rename to be safe on NFS.
|
||||
pub fn rewrite(&self, records: &[IndexRecord]) -> Result<()> {
|
||||
let mut buf = Vec::with_capacity(records.len() * INDEX_RECORD_SIZE);
|
||||
for r in records {
|
||||
buf.extend_from_slice(&r.encode());
|
||||
}
|
||||
crate::fs::create_atomic(&self.path, &buf)
|
||||
}
|
||||
|
||||
/// Delete the bucket file.
|
||||
pub fn delete(&self) -> Result<()> {
|
||||
if self.path.exists() {
|
||||
std::fs::remove_file(&self.path)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute bucket_id from a key's first 2 bytes.
|
||||
pub fn bucket_id(key: &[u8; 32]) -> u16 {
|
||||
u16::from_be_bytes([key[0], key[1]]) % BUCKET_COUNT
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_index_record_encode_decode() {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&[1, 2, 3, 4]);
|
||||
let rec = IndexRecord::new(key, 5, 12345, 500, 0);
|
||||
let encoded = rec.encode();
|
||||
let decoded = IndexRecord::decode(&encoded);
|
||||
assert_eq!(rec, decoded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_id_deterministic() {
|
||||
let mut key = [0u8; 32];
|
||||
key[0] = 0x00;
|
||||
key[1] = 0x0F;
|
||||
assert_eq!(bucket_id(&key), 15);
|
||||
key[0] = 0x00;
|
||||
key[1] = 0x10;
|
||||
assert_eq!(bucket_id(&key), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_append_and_load() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let bucket = BucketFile::open(dir.path(), 0);
|
||||
BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
|
||||
let r1 = IndexRecord::new([1u8; 32], 1, 100, 50, 0);
|
||||
let r2 = IndexRecord::new([2u8; 32], 1, 200, 60, 0);
|
||||
|
||||
bucket.append(&r1).unwrap();
|
||||
bucket.append(&r2).unwrap();
|
||||
|
||||
let loaded = bucket.load_all().unwrap();
|
||||
assert_eq!(loaded.len(), 2);
|
||||
assert_eq!(loaded[0].key, [1u8; 32]);
|
||||
assert_eq!(loaded[1].key, [2u8; 32]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_index_dedup() {
|
||||
let recs = vec![
|
||||
IndexRecord::new([1u8; 32], 1, 100, 50, 0),
|
||||
IndexRecord::new([1u8; 32], 2, 200, 50, 0), // newer offset wins
|
||||
IndexRecord::new([2u8; 32], 1, 300, 60, 0),
|
||||
];
|
||||
let idx = BucketIndex::from_records(recs, 0);
|
||||
assert_eq!(idx.len(), 2);
|
||||
let found = idx.find(&[1u8; 32]).unwrap();
|
||||
assert_eq!(found.segment_id, 2);
|
||||
assert_eq!(found.offset, 200);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_index_find_missing() {
|
||||
let recs = vec![IndexRecord::new([1u8; 32], 1, 100, 50, 0)];
|
||||
let idx = BucketIndex::from_records(recs, 0);
|
||||
assert!(idx.find(&[99u8; 32]).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_rewrite() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let bucket = BucketFile::open(dir.path(), 0);
|
||||
BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
|
||||
let r1 = IndexRecord::new([3u8; 32], 1, 300, 70, 0);
|
||||
let r2 = IndexRecord::new([1u8; 32], 1, 100, 50, 0);
|
||||
bucket.append(&r1).unwrap();
|
||||
bucket.append(&r2).unwrap();
|
||||
|
||||
// Rewrite sorted
|
||||
let sorted = vec![r2.clone(), r1.clone()];
|
||||
bucket.rewrite(&sorted).unwrap();
|
||||
|
||||
let loaded = bucket.load_all().unwrap();
|
||||
assert_eq!(loaded.len(), 2);
|
||||
assert_eq!(loaded[0].key, [1u8; 32]);
|
||||
assert_eq!(loaded[1].key, [3u8; 32]);
|
||||
}
|
||||
}
|
||||
219
crates/blob/src/cache.rs
Normal file
219
crates/blob/src/cache.rs
Normal file
@@ -0,0 +1,219 @@
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use crate::bucket::{BucketFile, BucketIndex, IndexRecord};
|
||||
use crate::error::Result;
|
||||
|
||||
type CacheKey = (String, u16);
|
||||
|
||||
/// Thread-safe LRU bucket cache with single-lock interior.
|
||||
/// Eliminates the TOCTOU race in the old two-Mutex design.
|
||||
pub struct BucketCache {
|
||||
inner: Mutex<CacheInner>,
|
||||
}
|
||||
|
||||
struct CacheInner {
|
||||
max_entries: usize,
|
||||
entries: Vec<CacheEntry>,
|
||||
index: HashMap<CacheKey, usize>,
|
||||
}
|
||||
|
||||
struct CacheEntry {
|
||||
key: CacheKey,
|
||||
index: BucketIndex,
|
||||
}
|
||||
|
||||
impl BucketCache {
|
||||
pub fn new(max_entries: usize) -> Self {
|
||||
Self {
|
||||
inner: Mutex::new(CacheInner {
|
||||
max_entries: max_entries.max(1),
|
||||
entries: Vec::new(),
|
||||
index: HashMap::new(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get or load a bucket index. Eliminates TOCTOU via double-checked locking.
|
||||
pub fn get_or_load(
|
||||
&self,
|
||||
account: &str,
|
||||
bucket_id: u16,
|
||||
account_dir: &Path,
|
||||
) -> Result<Vec<IndexRecord>> {
|
||||
let key: CacheKey = (account.to_string(), bucket_id);
|
||||
|
||||
// Check cache
|
||||
{
|
||||
let inner = self.inner.lock().unwrap();
|
||||
if let Some(&pos) = inner.index.get(&key) {
|
||||
return Ok(inner.entries[pos].index.records.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Load from disk
|
||||
let bucket_file = BucketFile::open(account_dir, bucket_id);
|
||||
let bucket_index = bucket_file.load_index()?;
|
||||
let records = bucket_index.records.clone();
|
||||
|
||||
// Insert with double-check (another thread might have beaten us)
|
||||
{
|
||||
let mut inner = self.inner.lock().unwrap();
|
||||
if let Some(&pos) = inner.index.get(&key) {
|
||||
return Ok(inner.entries[pos].index.records.clone());
|
||||
}
|
||||
// Evict if full
|
||||
if inner.entries.len() >= inner.max_entries {
|
||||
if let Some(evicted) = inner.entries.pop() {
|
||||
inner.index.remove(&evicted.key);
|
||||
}
|
||||
}
|
||||
// Insert at front
|
||||
inner.entries.insert(0, CacheEntry {
|
||||
key: key.clone(),
|
||||
index: bucket_index,
|
||||
});
|
||||
// Rebuild index
|
||||
inner.index.clear();
|
||||
for i in 0..inner.entries.len() {
|
||||
let key = inner.entries[i].key.clone();
|
||||
inner.index.insert(key, i);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(records)
|
||||
}
|
||||
|
||||
/// Insert or update a single record in a cached bucket.
|
||||
pub fn update_record(&self, account: &str, bucket_id: u16, record: IndexRecord) {
|
||||
let key: CacheKey = (account.to_string(), bucket_id);
|
||||
let mut inner = self.inner.lock().unwrap();
|
||||
|
||||
if let Some(&pos) = inner.index.get(&key) {
|
||||
inner.entries[pos].index.insert(record);
|
||||
// Move to front
|
||||
let entry = inner.entries.remove(pos);
|
||||
inner.entries.insert(0, entry);
|
||||
// Rebuild index
|
||||
inner.index.clear();
|
||||
for i in 0..inner.entries.len() {
|
||||
let entry_key = inner.entries[i].key.clone();
|
||||
inner.index.insert(entry_key, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Invalidate a cached bucket (after GC rewrites bucket files).
|
||||
pub fn invalidate(&self, account: &str, bucket_id: u16) {
|
||||
let key: CacheKey = (account.to_string(), bucket_id);
|
||||
let mut inner = self.inner.lock().unwrap();
|
||||
|
||||
if let Some(&pos) = inner.index.get(&key) {
|
||||
inner.entries.remove(pos);
|
||||
inner.index.clear();
|
||||
for i in 0..inner.entries.len() {
|
||||
let entry_key = inner.entries[i].key.clone();
|
||||
inner.index.insert(entry_key, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.inner.lock().unwrap().entries.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.inner.lock().unwrap().entries.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::IndexRecord;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_cache_miss_loads_from_disk() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
let bf = BucketFile::open(dir.path(), 0);
|
||||
bf.append(&IndexRecord::new([1u8; 32], 1, 100, 50, 0))
|
||||
.unwrap();
|
||||
|
||||
let cache = BucketCache::new(10);
|
||||
let records = cache
|
||||
.get_or_load("test", 0, dir.path())
|
||||
.unwrap();
|
||||
assert_eq!(records.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_hit() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
let bf = BucketFile::open(dir.path(), 0);
|
||||
bf.append(&IndexRecord::new([2u8; 32], 1, 200, 60, 0))
|
||||
.unwrap();
|
||||
|
||||
let cache = BucketCache::new(10);
|
||||
let _ = cache.get_or_load("test", 0, dir.path()).unwrap();
|
||||
let records = cache
|
||||
.get_or_load("test", 0, dir.path())
|
||||
.unwrap();
|
||||
assert_eq!(records.len(), 1);
|
||||
assert_eq!(cache.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_eviction() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
let cache = BucketCache::new(2);
|
||||
|
||||
for b in 0..4 {
|
||||
let bf = BucketFile::open(dir.path(), b);
|
||||
bf.append(&IndexRecord::new([b as u8; 32], 1, 100, 50, 0))
|
||||
.unwrap();
|
||||
let _ = cache.get_or_load("test", b, dir.path()).unwrap();
|
||||
}
|
||||
|
||||
assert!(cache.len() <= 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_concurrent_get_or_load_no_deadlock() {
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
let bf = BucketFile::open(dir.path(), 0);
|
||||
for i in 0..10u8 {
|
||||
bf.append(&IndexRecord::new([i; 32], 1, i as u64 * 100, 50, 0))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let cache = Arc::new(BucketCache::new(10));
|
||||
let dir_path = dir.path().to_path_buf();
|
||||
|
||||
let mut handles = vec![];
|
||||
for _ in 0..4 {
|
||||
let cache = cache.clone();
|
||||
let dir_path = dir_path.clone();
|
||||
handles.push(thread::spawn(move || {
|
||||
for _ in 0..100 {
|
||||
let records = cache
|
||||
.get_or_load("test", 0, &dir_path)
|
||||
.unwrap();
|
||||
assert_eq!(records.len(), 10);
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
65
crates/blob/src/checksum.rs
Normal file
65
crates/blob/src/checksum.rs
Normal file
@@ -0,0 +1,65 @@
|
||||
use crc32fast::Hasher;
|
||||
|
||||
pub fn crc32(data: &[u8]) -> u32 {
|
||||
let mut h = Hasher::new();
|
||||
h.update(data);
|
||||
h.finalize()
|
||||
}
|
||||
|
||||
pub struct CrcWriter {
|
||||
hasher: Hasher,
|
||||
}
|
||||
|
||||
impl Default for CrcWriter {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl CrcWriter {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
hasher: Hasher::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update(&mut self, data: &[u8]) {
|
||||
self.hasher.update(data);
|
||||
}
|
||||
|
||||
pub fn finalize(self) -> u32 {
|
||||
self.hasher.finalize()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_crc32_deterministic() {
|
||||
let a = crc32(b"hello");
|
||||
let b = crc32(b"hello");
|
||||
assert_eq!(a, b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crc32_different() {
|
||||
let a = crc32(b"hello");
|
||||
let b = crc32(b"world");
|
||||
assert!(a != b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crc_writer_matches_crc32() {
|
||||
let mut w = CrcWriter::new();
|
||||
w.update(b"hello");
|
||||
w.update(b" world");
|
||||
assert_eq!(w.finalize(), crc32(b"hello world"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crc32_empty() {
|
||||
assert_eq!(crc32(b""), 0);
|
||||
}
|
||||
}
|
||||
97
crates/blob/src/compress.rs
Normal file
97
crates/blob/src/compress.rs
Normal file
@@ -0,0 +1,97 @@
|
||||
use crate::types::Codec;
|
||||
|
||||
pub fn compress(data: &[u8], codec: Codec, threshold: usize, level: i32) -> (Vec<u8>, Codec) {
|
||||
if data.len() < threshold {
|
||||
return (data.to_vec(), Codec::None);
|
||||
}
|
||||
let (compressed, actual_codec) = match codec {
|
||||
Codec::Zstd => {
|
||||
match zstd::encode_all(data, level) {
|
||||
Ok(out) => (out, Codec::Zstd),
|
||||
Err(e) => {
|
||||
tracing::warn!("zstd compression failed, storing uncompressed: {}", e);
|
||||
(data.to_vec(), Codec::None)
|
||||
}
|
||||
}
|
||||
}
|
||||
Codec::Lz4 => {
|
||||
let out = lz4_flex::compress(data);
|
||||
(out, Codec::Lz4)
|
||||
}
|
||||
Codec::None => (data.to_vec(), Codec::None),
|
||||
};
|
||||
// If compression made it larger, store uncompressed
|
||||
if compressed.len() >= data.len() {
|
||||
(data.to_vec(), Codec::None)
|
||||
} else {
|
||||
(compressed, actual_codec)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decompress(data: &[u8], codec: Codec, raw_size: usize) -> crate::error::Result<Vec<u8>> {
|
||||
match codec {
|
||||
Codec::None => Ok(data.to_vec()),
|
||||
Codec::Zstd => {
|
||||
zstd::decode_all(data)
|
||||
.map_err(|e| crate::error::Error::Compression(format!("zstd decompress: {}", e)))
|
||||
}
|
||||
Codec::Lz4 => {
|
||||
lz4_flex::decompress(data, raw_size)
|
||||
.map_err(|e| crate::error::Error::Compression(format!("lz4 decompress: {}", e)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_small_data_not_compressed() {
|
||||
let data = b"hi";
|
||||
let (out, codec) = compress(data, Codec::Zstd, 4096, 0);
|
||||
assert_eq!(out, b"hi");
|
||||
assert_eq!(codec, Codec::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_large_data_compressed_zstd() {
|
||||
let data = vec![b'A'; 5000];
|
||||
let (out, codec) = compress(&data, Codec::Zstd, 4096, 0);
|
||||
assert_eq!(codec, Codec::Zstd);
|
||||
assert!(out.len() < data.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_zstd() {
|
||||
let data = vec![b'B'; 10000];
|
||||
let (compressed, codec) = compress(&data, Codec::Zstd, 4096, 0);
|
||||
let decompressed = decompress(&compressed, codec, data.len()).unwrap();
|
||||
assert_eq!(decompressed, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_lz4() {
|
||||
let data = vec![b'C'; 10000];
|
||||
let (compressed, codec) = compress(&data, Codec::Lz4, 4096, 0);
|
||||
let decompressed = decompress(&compressed, codec, data.len()).unwrap();
|
||||
assert_eq!(decompressed, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_none() {
|
||||
let data = vec![b'D'; 100];
|
||||
let (compressed, codec) = compress(&data, Codec::None, 4096, 0);
|
||||
assert_eq!(codec, Codec::None);
|
||||
let decompressed = decompress(&compressed, codec, data.len()).unwrap();
|
||||
assert_eq!(decompressed, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_threshold_zero_always_compresses() {
|
||||
let data = vec![b'E'; 100];
|
||||
let (out, codec) = compress(&data, Codec::Zstd, 0, 0);
|
||||
assert_eq!(codec, Codec::Zstd);
|
||||
assert!(out.len() < data.len());
|
||||
}
|
||||
}
|
||||
398
crates/blob/src/engine.rs
Normal file
398
crates/blob/src/engine.rs
Normal file
@@ -0,0 +1,398 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
use crate::account::AccountHandle;
|
||||
use crate::bucket::{self, IndexRecord};
|
||||
use crate::cache::BucketCache;
|
||||
use crate::compress;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::gc::{self, GcStats};
|
||||
use crate::meta::GlobalMeta;
|
||||
use crate::segment::SegmentReader;
|
||||
use crate::types::{Codec, Config, ENTRY_HEADER_SIZE};
|
||||
|
||||
pub struct Engine {
|
||||
root: PathBuf,
|
||||
config: Config,
|
||||
cache: BucketCache,
|
||||
accounts: RwLock<HashMap<String, Arc<AccountHandle>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AccountStats {
|
||||
pub account_id: String,
|
||||
pub total_keys: u64,
|
||||
pub total_bytes: u64,
|
||||
pub deleted_bytes: u64,
|
||||
pub segment_count: usize,
|
||||
}
|
||||
|
||||
impl Engine {
|
||||
pub fn open(path: &Path, config: Config) -> Result<Self> {
|
||||
config.validate()?;
|
||||
fs::create_dir_all(path)?;
|
||||
fs::create_dir_all(path.join("accounts"))?;
|
||||
|
||||
let mut global = GlobalMeta::load(path)?;
|
||||
global.save(path)?;
|
||||
|
||||
let cache = BucketCache::new(config.lru_bucket_count);
|
||||
|
||||
let accounts_dir = path.join("accounts");
|
||||
let mut accounts = HashMap::new();
|
||||
|
||||
if accounts_dir.exists() {
|
||||
for entry in fs::read_dir(&accounts_dir)? {
|
||||
let entry = entry?;
|
||||
if entry.file_type()?.is_dir() {
|
||||
let account_name = entry.file_name().to_string_lossy().into_owned();
|
||||
|
||||
let _ = crate::recovery::cleanup_temp_files(&entry.path());
|
||||
|
||||
match crate::recovery::recover_account(&entry.path()) {
|
||||
Ok(_meta) => {
|
||||
match AccountHandle::open(path, &account_name) {
|
||||
Ok(handle) => {
|
||||
accounts.insert(account_name, handle);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Failed to open account {}: {}",
|
||||
account_name,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Failed to recover account {}: {}",
|
||||
account_name,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
global.accounts = accounts.keys().cloned().collect();
|
||||
global.save(path)?;
|
||||
|
||||
Ok(Self {
|
||||
root: path.to_path_buf(),
|
||||
config,
|
||||
cache,
|
||||
accounts: RwLock::new(accounts),
|
||||
})
|
||||
}
|
||||
|
||||
// ── Account management ──────────────────────────────────────────────
|
||||
|
||||
pub fn create_account(&self, account_id: &str) -> Result<()> {
|
||||
let mut accounts = self.accounts.write().unwrap();
|
||||
if accounts.contains_key(account_id) {
|
||||
return Err(Error::AccountAlreadyExists(account_id.to_string()));
|
||||
}
|
||||
let handle = AccountHandle::create(&self.root, account_id)?;
|
||||
accounts.insert(account_id.to_string(), handle);
|
||||
|
||||
let mut global = GlobalMeta::load(&self.root)?;
|
||||
global.accounts = accounts.keys().cloned().collect();
|
||||
global.save(&self.root)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn delete_account(&self, account_id: &str) -> Result<()> {
|
||||
let mut accounts = self.accounts.write().unwrap();
|
||||
let handle = accounts
|
||||
.remove(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?;
|
||||
|
||||
let account_dir = handle.dir().to_path_buf();
|
||||
drop(handle);
|
||||
fs::remove_dir_all(&account_dir)?;
|
||||
|
||||
let mut global = GlobalMeta::load(&self.root)?;
|
||||
global.accounts = accounts.keys().cloned().collect();
|
||||
global.save(&self.root)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn list_accounts(&self) -> Vec<String> {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts.keys().cloned().collect()
|
||||
}
|
||||
|
||||
// ── Read / Write / Delete ───────────────────────────────────────────
|
||||
|
||||
pub fn write(
|
||||
&self,
|
||||
account_id: &str,
|
||||
key: [u8; 32],
|
||||
value: &[u8],
|
||||
codec: Codec,
|
||||
) -> Result<()> {
|
||||
if value.len() > crate::types::MAX_VALUE_SIZE {
|
||||
return Err(Error::ValueTooLarge { size: value.len() });
|
||||
}
|
||||
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
let _write_lock = handle.write_mutex.lock().unwrap();
|
||||
let mut inner = handle.write();
|
||||
|
||||
let (data, actual_codec) =
|
||||
compress::compress(value, codec, self.config.compress_threshold, self.config.compression_level);
|
||||
|
||||
let (segment_id, offset, data_size) =
|
||||
inner.write_entry(key, &data, 0, actual_codec)?;
|
||||
|
||||
let record = IndexRecord::new(key, segment_id, offset, data_size, 0);
|
||||
inner.append_index(&record)?;
|
||||
|
||||
let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
inner.mark_indexed(segment_id, entry_end)?;
|
||||
|
||||
let bucket_id = bucket::bucket_id(&key);
|
||||
self.cache.update_record(account_id, bucket_id, record);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn read(&self, account_id: &str, key: &[u8; 32]) -> Result<Option<Vec<u8>>> {
|
||||
let bucket_id = bucket::bucket_id(key);
|
||||
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
let (record, seg_path): (IndexRecord, PathBuf) = {
|
||||
let inner = handle.read();
|
||||
let records = self
|
||||
.cache
|
||||
.get_or_load(account_id, bucket_id, handle.dir())?;
|
||||
match records.binary_search_by(|r| r.key.cmp(key)) {
|
||||
Ok(idx) => {
|
||||
let r = records[idx].clone();
|
||||
if r.is_tombstone() {
|
||||
return Ok(None);
|
||||
}
|
||||
let seg_path = inner.segment_path(r.segment_id)?;
|
||||
(r, seg_path)
|
||||
}
|
||||
Err(_) => return Ok(None),
|
||||
}
|
||||
};
|
||||
|
||||
if !seg_path.exists() {
|
||||
return Err(Error::SegmentNotFound(record.segment_id));
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(seg_path.clone(), record.segment_id)?;
|
||||
let file = handle.get_segment_file(record.segment_id, &seg_path)?;
|
||||
let (entry, _) = reader.read_entry_at_file(record.offset, &file)?;
|
||||
|
||||
let value = compress::decompress(&entry.data, entry.codec, entry.raw_size as usize)?;
|
||||
|
||||
Ok(Some(value))
|
||||
}
|
||||
|
||||
pub fn delete(&self, account_id: &str, key: &[u8; 32]) -> Result<()> {
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
let _write_lock = handle.write_mutex.lock().unwrap();
|
||||
let mut inner = handle.write();
|
||||
|
||||
let (segment_id, offset, data_size) =
|
||||
inner.write_entry(*key, &[], 1, Codec::None)?;
|
||||
|
||||
let record = IndexRecord::new(*key, segment_id, offset, data_size, 1);
|
||||
inner.append_index(&record)?;
|
||||
|
||||
let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
inner.mark_indexed(segment_id, entry_end)?;
|
||||
|
||||
let bucket_id = bucket::bucket_id(key);
|
||||
self.cache.update_record(account_id, bucket_id, record);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── Batch write ─────────────────────────────────────────────────────
|
||||
|
||||
pub fn write_batch(&self, account_id: &str, entries: &[([u8; 32], Vec<u8>, Codec)]) -> Result<()> {
|
||||
if entries.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
let _write_lock = handle.write_mutex.lock().unwrap();
|
||||
let mut inner = handle.write();
|
||||
|
||||
let mut pending: Vec<(IndexRecord, u64)> = Vec::with_capacity(entries.len());
|
||||
for (key, value, codec) in entries {
|
||||
if value.len() > crate::types::MAX_VALUE_SIZE {
|
||||
return Err(Error::ValueTooLarge { size: value.len() });
|
||||
}
|
||||
let (data, actual_codec) =
|
||||
compress::compress(value, *codec, self.config.compress_threshold, self.config.compression_level);
|
||||
|
||||
let (segment_id, offset, data_size) =
|
||||
inner.append_entry(*key, &data, 0, actual_codec)?;
|
||||
|
||||
let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
let record = IndexRecord::new(*key, segment_id, offset, data_size, 0);
|
||||
pending.push((record, entry_end));
|
||||
}
|
||||
|
||||
inner.flush_active()?;
|
||||
|
||||
for (record, entry_end) in &pending {
|
||||
inner.append_index(record)?;
|
||||
inner.mark_indexed(record.segment_id, *entry_end)?;
|
||||
|
||||
let bucket_id = bucket::bucket_id(&record.key);
|
||||
self.cache.update_record(account_id, bucket_id, record.clone());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── GC ──────────────────────────────────────────────────────────────
|
||||
|
||||
pub fn gc(&self, account_id: &str) -> Result<Option<GcStats>> {
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
// Hold write_mutex to prevent concurrent writes from racing
|
||||
// with GC's bucket rebuild phase.
|
||||
let _write_lock = handle.write_mutex.lock().unwrap();
|
||||
|
||||
let result = gc::gc_account(handle.dir(), self.config.gc_deleted_ratio)?;
|
||||
|
||||
// Invalidate FilePool for GC'd segments (they were rewritten via rename)
|
||||
if let Some(ref stats) = result {
|
||||
handle.invalidate_file_cache(stats.segment_id);
|
||||
}
|
||||
|
||||
for bid in 0..crate::types::BUCKET_COUNT {
|
||||
self.cache.invalidate(account_id, bid);
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn compact_buckets(&self, account_id: &str) -> Result<()> {
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
// Hold write_mutex — compact rewrites all bucket files.
|
||||
let _write_lock = handle.write_mutex.lock().unwrap();
|
||||
|
||||
gc::compact_buckets(handle.dir())?;
|
||||
|
||||
for bid in 0..crate::types::BUCKET_COUNT {
|
||||
self.cache.invalidate(account_id, bid);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── Stats / Shutdown ────────────────────────────────────────────────
|
||||
|
||||
pub fn stats(&self, account_id: &str) -> Result<AccountStats> {
|
||||
let handle = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
let inner = handle.read();
|
||||
let meta = inner.meta();
|
||||
let mut total_bytes = 0u64;
|
||||
let mut deleted_bytes = 0u64;
|
||||
|
||||
for seg in meta.segments.values() {
|
||||
total_bytes += seg.total_bytes;
|
||||
deleted_bytes += seg.deleted_bytes;
|
||||
}
|
||||
|
||||
let mut total_keys = 0u64;
|
||||
for bid in 0..crate::types::BUCKET_COUNT {
|
||||
if let Ok(records) =
|
||||
self.cache
|
||||
.get_or_load(account_id, bid, handle.dir())
|
||||
{
|
||||
total_keys += records.iter().filter(|r| !r.is_tombstone()).count() as u64;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(AccountStats {
|
||||
account_id: account_id.to_string(),
|
||||
total_keys,
|
||||
total_bytes,
|
||||
deleted_bytes,
|
||||
segment_count: meta.segments.len(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn shutdown(&self) -> Result<()> {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
for (_, handle) in accounts.iter() {
|
||||
let mut inner = handle.write();
|
||||
inner.flush_active()?;
|
||||
}
|
||||
let global = GlobalMeta::load(&self.root)?;
|
||||
global.save(&self.root)?;
|
||||
tracing::info!("bichon-blob shut down cleanly");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Engine {
|
||||
fn drop(&mut self) {
|
||||
if let Err(e) = self.shutdown() {
|
||||
tracing::error!("bichon-blob shutdown error: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
59
crates/blob/src/error.rs
Normal file
59
crates/blob/src/error.rs
Normal file
@@ -0,0 +1,59 @@
|
||||
use std::{io, path::PathBuf};
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] io::Error),
|
||||
|
||||
#[error("JSON error: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
|
||||
#[error("CRC32 mismatch at {path}:{offset}")]
|
||||
CrcMismatch { path: PathBuf, offset: u64 },
|
||||
|
||||
#[error("Corrupt entry at {path}:{offset}: {reason}")]
|
||||
CorruptEntry {
|
||||
path: PathBuf,
|
||||
offset: u64,
|
||||
reason: String,
|
||||
},
|
||||
|
||||
#[error("Account not found: {0}")]
|
||||
AccountNotFound(String),
|
||||
|
||||
#[error("Account already exists: {0}")]
|
||||
AccountAlreadyExists(String),
|
||||
|
||||
#[error("Segment not found: {0}")]
|
||||
SegmentNotFound(u32),
|
||||
|
||||
#[error("Value too large: {size} bytes (max 100 MB)")]
|
||||
ValueTooLarge { size: usize },
|
||||
|
||||
#[error("Compression error: {0}")]
|
||||
Compression(String),
|
||||
|
||||
#[error("Disk full: {0}")]
|
||||
DiskFull(String),
|
||||
|
||||
#[error("Invalid config: {0}")]
|
||||
InvalidConfig(String),
|
||||
|
||||
#[error("Bucket index corrupt at {path}: {reason}")]
|
||||
BucketIndexCorrupt { path: PathBuf, reason: String },
|
||||
|
||||
#[error("Segment file truncated at {path}: expected {expected}, got {actual}")]
|
||||
SegmentTruncated {
|
||||
path: PathBuf,
|
||||
expected: u64,
|
||||
actual: u64,
|
||||
},
|
||||
|
||||
#[error("Corrupt metadata file: {0}")]
|
||||
CorruptMeta(String),
|
||||
|
||||
#[error("Unsupported metadata version {version} in {path}")]
|
||||
UnsupportedMetaVersion { path: PathBuf, version: u32 },
|
||||
}
|
||||
54
crates/blob/src/file_pool.rs
Normal file
54
crates/blob/src/file_pool.rs
Normal file
@@ -0,0 +1,54 @@
|
||||
use std::collections::VecDeque;
|
||||
use std::fs::File;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::fs as fs_util;
|
||||
|
||||
/// Simple LRU pool of open file handles, keyed by segment_id.
|
||||
/// Uses Arc<Mutex<File>> to allow safe concurrent reads from the same segment.
|
||||
pub struct FilePool {
|
||||
max_entries: usize,
|
||||
entries: Mutex<VecDeque<(u32, Arc<Mutex<File>>)>>,
|
||||
}
|
||||
|
||||
impl FilePool {
|
||||
pub fn new(max_entries: usize) -> Self {
|
||||
Self {
|
||||
max_entries: max_entries.max(1),
|
||||
entries: Mutex::new(VecDeque::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get an open File for the given segment. Reuses cached handle if available.
|
||||
pub fn get(&self, seg_id: u32, path: &Path) -> Result<Arc<Mutex<File>>> {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
|
||||
// Check for existing entry
|
||||
for (i, (id, _)) in entries.iter().enumerate() {
|
||||
if *id == seg_id {
|
||||
let (_, file) = entries.remove(i).unwrap();
|
||||
entries.push_front((seg_id, file.clone()));
|
||||
return Ok(file);
|
||||
}
|
||||
}
|
||||
|
||||
// Open new file
|
||||
let file = Arc::new(Mutex::new(fs_util::open_read(path)?));
|
||||
|
||||
// Evict oldest if full
|
||||
if entries.len() >= self.max_entries {
|
||||
entries.pop_back();
|
||||
}
|
||||
|
||||
entries.push_front((seg_id, file.clone()));
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
/// Remove a cached file handle (e.g. after GC rewrites a segment).
|
||||
pub fn invalidate(&self, seg_id: u32) {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
entries.retain(|(id, _)| *id != seg_id);
|
||||
}
|
||||
}
|
||||
142
crates/blob/src/fs.rs
Normal file
142
crates/blob/src/fs.rs
Normal file
@@ -0,0 +1,142 @@
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, Write};
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::error::Result;
|
||||
|
||||
/// Max retries for transient filesystem errors (NFS ESTALE, CIFS sharing violations, etc.)
|
||||
const MAX_RETRIES: u32 = 5;
|
||||
const RETRY_DELAY: Duration = Duration::from_millis(20);
|
||||
|
||||
/// Check if an I/O error is transient (retryable).
|
||||
fn is_transient(err: &io::Error) -> bool {
|
||||
use std::io::ErrorKind;
|
||||
matches!(
|
||||
err.kind(),
|
||||
ErrorKind::TimedOut
|
||||
| ErrorKind::Interrupted
|
||||
| ErrorKind::WouldBlock
|
||||
| ErrorKind::UnexpectedEof
|
||||
) || err.raw_os_error() == Some(116) // ESTALE on Linux
|
||||
}
|
||||
|
||||
/// Open an existing file for reading, with retry on transient errors (NFS ESTALE etc.).
|
||||
pub fn open_read(path: &Path) -> Result<File> {
|
||||
let mut last_err = None;
|
||||
for attempt in 0..MAX_RETRIES {
|
||||
match File::open(path) {
|
||||
Ok(f) => return Ok(f),
|
||||
Err(e) if is_transient(&e) => {
|
||||
last_err = Some(e);
|
||||
if attempt > 0 {
|
||||
std::thread::sleep(RETRY_DELAY * attempt);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
}
|
||||
Err(crate::error::Error::Io(last_err.unwrap()))
|
||||
}
|
||||
|
||||
/// Open an existing file for writing, with retry on transient errors.
|
||||
pub fn open_write(path: &Path) -> Result<File> {
|
||||
let mut last_err = None;
|
||||
for attempt in 0..MAX_RETRIES {
|
||||
match OpenOptions::new().write(true).open(path) {
|
||||
Ok(f) => return Ok(f),
|
||||
Err(e) if is_transient(&e) => {
|
||||
last_err = Some(e);
|
||||
if attempt > 0 {
|
||||
std::thread::sleep(RETRY_DELAY * attempt);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
}
|
||||
Err(crate::error::Error::Io(last_err.unwrap()))
|
||||
}
|
||||
|
||||
/// Create a new file atomically: write content to a temp file, fsync, then rename.
|
||||
/// Avoids `create_new(true)` which is racy on NFS.
|
||||
pub fn create_atomic(path: &Path, content: &[u8]) -> Result<()> {
|
||||
let tmp = path.with_extension(
|
||||
path.extension()
|
||||
.map(|e| format!("{}.tmp", e.to_string_lossy()))
|
||||
.unwrap_or_else(|| "tmp".to_string()),
|
||||
);
|
||||
|
||||
{
|
||||
let mut f = File::create(&tmp)?;
|
||||
f.write_all(content)?;
|
||||
f.sync_all()?;
|
||||
}
|
||||
|
||||
fs::rename(&tmp, path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Truncate an existing file to the given size, with retry.
|
||||
pub fn truncate(path: &Path, size: u64) -> Result<()> {
|
||||
let f = open_write(path)?;
|
||||
f.set_len(size)?;
|
||||
f.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_open_read_existing() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = dir.path().join("test.txt");
|
||||
std::fs::write(&path, b"hello").unwrap();
|
||||
|
||||
let mut f = open_read(&path).unwrap();
|
||||
let mut s = String::new();
|
||||
std::io::Read::read_to_string(&mut f, &mut s).unwrap();
|
||||
assert_eq!(s, "hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_open_read_missing() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let result = open_read(&dir.path().join("nope.txt"));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_create_atomic_success() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = dir.path().join("data.bin");
|
||||
create_atomic(&path, b"hello world").unwrap();
|
||||
|
||||
let content = std::fs::read(&path).unwrap();
|
||||
assert_eq!(content, b"hello world");
|
||||
// Temp file should not exist
|
||||
assert!(!dir.path().join("data.bin.tmp").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_create_atomic_overwrites() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = dir.path().join("data.bin");
|
||||
create_atomic(&path, b"first").unwrap();
|
||||
create_atomic(&path, b"second").unwrap();
|
||||
assert_eq!(std::fs::read(&path).unwrap(), b"second");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_truncate() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = dir.path().join("trunc.bin");
|
||||
std::fs::write(&path, b"1234567890").unwrap();
|
||||
truncate(&path, 5).unwrap();
|
||||
assert_eq!(std::fs::metadata(&path).unwrap().len(), 5);
|
||||
}
|
||||
}
|
||||
267
crates/blob/src/gc.rs
Normal file
267
crates/blob/src/gc.rs
Normal file
@@ -0,0 +1,267 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::bucket::{self, BucketFile, BucketIndex, IndexRecord};
|
||||
use crate::error::Result;
|
||||
#[cfg(test)]
|
||||
use crate::meta::SegmentStats;
|
||||
use crate::segment::{self, SegmentReader, SegmentWriter};
|
||||
|
||||
/// Result of a GC run.
|
||||
#[derive(Debug)]
|
||||
pub struct GcStats {
|
||||
pub segment_id: u32,
|
||||
pub bytes_before: u64,
|
||||
pub bytes_after: u64,
|
||||
pub entries_kept: usize,
|
||||
pub entries_skipped: usize,
|
||||
}
|
||||
|
||||
/// Run GC on an account: pick the sealed segment with highest deleted_ratio,
|
||||
/// rewrite it without deleted/overwritten entries, then rebuild all bucket files.
|
||||
pub fn gc_account(
|
||||
account_dir: &Path,
|
||||
deleted_ratio_threshold: f64,
|
||||
) -> Result<Option<GcStats>> {
|
||||
let meta = crate::meta::AccountMeta::load(account_dir)?;
|
||||
|
||||
// Find the best candidate
|
||||
let candidate = meta
|
||||
.segments
|
||||
.values()
|
||||
.filter(|s| s.sealed && s.deleted_ratio >= deleted_ratio_threshold)
|
||||
.max_by(|a, b| a.deleted_ratio.partial_cmp(&b.deleted_ratio).unwrap());
|
||||
|
||||
let target = match candidate {
|
||||
Some(s) => s.clone(),
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
let seg_path = account_dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(target.segment_id));
|
||||
let reader = SegmentReader::open(seg_path.clone(), target.segment_id)?;
|
||||
|
||||
// Build a global view: for each key, which entry (segment_id + offset) is the latest?
|
||||
let mut latest_key: HashMap<[u8; 32], (u32, u64)> = HashMap::new();
|
||||
|
||||
for &seg_id in meta.segments.keys() {
|
||||
let rpath = account_dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(seg_id));
|
||||
if !rpath.exists() {
|
||||
continue;
|
||||
}
|
||||
let r = SegmentReader::open(rpath, seg_id)?;
|
||||
let _ = r.scan_entries(0, |entry, offset| {
|
||||
match latest_key.get(&entry.key) {
|
||||
Some((existing_seg, existing_off)) => {
|
||||
if seg_id > *existing_seg
|
||||
|| (seg_id == *existing_seg && offset > *existing_off)
|
||||
{
|
||||
latest_key.insert(entry.key, (seg_id, offset));
|
||||
}
|
||||
}
|
||||
None => {
|
||||
latest_key.insert(entry.key, (seg_id, offset));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
|
||||
// Create temp segment with a unique name
|
||||
let timestamp = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
let temp_name = format!("temp_{:016x}.seg", timestamp);
|
||||
let temp_path = account_dir.join("segments").join(&temp_name);
|
||||
let mut writer = SegmentWriter::create(temp_path.clone(), target.segment_id)?;
|
||||
|
||||
let mut bytes_after: u64 = 0;
|
||||
let mut entries_kept: usize = 0;
|
||||
let mut entries_skipped: usize = 0;
|
||||
|
||||
reader.scan_entries(0, |entry, offset| {
|
||||
// Skip tombstones
|
||||
if entry.is_tombstone() {
|
||||
entries_skipped += 1;
|
||||
return Ok(());
|
||||
}
|
||||
// Skip if this key has a newer entry in another segment
|
||||
if let Some((latest_seg, latest_off)) = latest_key.get(&entry.key) {
|
||||
if *latest_seg != target.segment_id || *latest_off != offset {
|
||||
entries_skipped += 1;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
// Keep this entry
|
||||
writer.append(entry)?;
|
||||
bytes_after += entry.data.len() as u64;
|
||||
entries_kept += 1;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
writer.fsync()?;
|
||||
|
||||
// Atomic rename: replace old segment with new one
|
||||
fs::rename(&temp_path, &seg_path)?;
|
||||
|
||||
// Rebuild all bucket files
|
||||
rebuild_buckets(account_dir, &meta)?;
|
||||
|
||||
// Update meta
|
||||
let mut meta = crate::meta::AccountMeta::load(account_dir)?;
|
||||
if let Some(stats) = meta.segments.get_mut(&target.segment_id) {
|
||||
stats.total_bytes = bytes_after;
|
||||
stats.deleted_bytes = 0;
|
||||
stats.recompute_ratio();
|
||||
}
|
||||
meta.save(account_dir)?;
|
||||
|
||||
Ok(Some(GcStats {
|
||||
segment_id: target.segment_id,
|
||||
bytes_before: target.total_bytes,
|
||||
bytes_after,
|
||||
entries_kept,
|
||||
entries_skipped,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Rebuild all 16 bucket files from scratch by scanning all segments.
|
||||
fn rebuild_buckets(account_dir: &Path, meta: &crate::meta::AccountMeta) -> Result<()> {
|
||||
let mut bucket_records: HashMap<u16, Vec<IndexRecord>> = HashMap::new();
|
||||
for i in 0..crate::types::BUCKET_COUNT {
|
||||
bucket_records.insert(i, Vec::new());
|
||||
}
|
||||
|
||||
for &seg_id in meta.segments.keys() {
|
||||
let seg_path = account_dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(seg_id));
|
||||
if !seg_path.exists() {
|
||||
continue;
|
||||
}
|
||||
let reader = SegmentReader::open(seg_path, seg_id)?;
|
||||
reader.scan_entries(0, |entry, offset| {
|
||||
let bid = bucket::bucket_id(&entry.key);
|
||||
let rec = IndexRecord::new(
|
||||
entry.key,
|
||||
seg_id,
|
||||
offset,
|
||||
entry.data.len() as u32,
|
||||
entry.flags,
|
||||
);
|
||||
bucket_records.entry(bid).or_default().push(rec);
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
|
||||
for (bid, records) in &bucket_records {
|
||||
let index = BucketIndex::from_records(records.clone(), *bid);
|
||||
let bf = BucketFile::open(account_dir, *bid);
|
||||
bf.rewrite(&index.records)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compact bucket files: load, dedup, rewrite.
|
||||
pub fn compact_buckets(account_dir: &Path) -> Result<()> {
|
||||
for bid in 0..crate::types::BUCKET_COUNT {
|
||||
let bf = BucketFile::open(account_dir, bid);
|
||||
if bf.path().exists() {
|
||||
let index = bf.load_index()?;
|
||||
bf.rewrite(&index.records)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::segment::Entry;
|
||||
use crate::types::Codec;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn setup_account(dir: &Path) {
|
||||
fs::create_dir_all(dir.join("segments")).unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir).unwrap();
|
||||
|
||||
let seg_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(1));
|
||||
let mut writer = SegmentWriter::create(seg_path, 1).unwrap();
|
||||
|
||||
// Write 5 entries
|
||||
for i in 0..5u8 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0] = i;
|
||||
let entry = Entry::new(key, &vec![i; 1000], 0, Codec::None);
|
||||
writer.append(&entry).unwrap();
|
||||
}
|
||||
|
||||
// Tombstone entry 2
|
||||
let mut key2 = [0u8; 32];
|
||||
key2[0] = 2;
|
||||
let tomb = Entry::tombstone(key2);
|
||||
writer.append(&tomb).unwrap();
|
||||
|
||||
writer.fsync().unwrap();
|
||||
|
||||
// Save meta
|
||||
let mut meta = crate::meta::AccountMeta::new("test".into(), 2);
|
||||
meta.segments.insert(
|
||||
1,
|
||||
SegmentStats {
|
||||
segment_id: 1,
|
||||
total_bytes: 6000,
|
||||
deleted_bytes: 1000,
|
||||
deleted_ratio: 1000.0 / 6000.0,
|
||||
sealed: true,
|
||||
indexed_up_to_offset: 0,
|
||||
},
|
||||
);
|
||||
// Make segment 2 active so segment 1 is sealed
|
||||
let seg2_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(2));
|
||||
SegmentWriter::create(seg2_path, 2).unwrap();
|
||||
meta.save(dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gc_removes_tombstones() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
setup_account(dir.path());
|
||||
|
||||
let result = gc_account(dir.path(), 0.01).unwrap();
|
||||
assert!(result.is_some());
|
||||
|
||||
// Verify segment 1 no longer has the tombstone'd entry
|
||||
let seg_path = dir
|
||||
.path()
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(1));
|
||||
let reader = SegmentReader::open(seg_path, 1).unwrap();
|
||||
let mut count = 0;
|
||||
reader.scan_entries(0, |entry, _offset| {
|
||||
count += 1;
|
||||
assert!(entry.key[0] != 2);
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
assert_eq!(count, 4); // 5 original - 1 tombstoned
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compact_buckets() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
setup_account(dir.path());
|
||||
compact_buckets(dir.path()).unwrap();
|
||||
// Should not panic
|
||||
}
|
||||
}
|
||||
19
crates/blob/src/lib.rs
Normal file
19
crates/blob/src/lib.rs
Normal file
@@ -0,0 +1,19 @@
|
||||
pub mod account;
|
||||
pub mod bucket;
|
||||
pub mod cache;
|
||||
pub mod checksum;
|
||||
pub mod compress;
|
||||
pub mod engine;
|
||||
pub mod error;
|
||||
pub mod file_pool;
|
||||
pub mod fs;
|
||||
pub mod gc;
|
||||
pub mod meta;
|
||||
pub mod recovery;
|
||||
pub mod segment;
|
||||
pub mod types;
|
||||
|
||||
pub use account::AccountHandle;
|
||||
pub use engine::{AccountStats, Engine};
|
||||
pub use error::{Error, Result};
|
||||
pub use types::{Codec, Config};
|
||||
284
crates/blob/src/meta.rs
Normal file
284
crates/blob/src/meta.rs
Normal file
@@ -0,0 +1,284 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::checksum;
|
||||
use crate::error::Result;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
const META_VERSION: u32 = 1;
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
fn write_bin<T: Serialize>(path: &Path, value: &T) -> Result<()> {
|
||||
let payload = bincode::serialize(value).map_err(|e| {
|
||||
crate::error::Error::CorruptMeta(format!("{}: bincode encode: {}", path.display(), e))
|
||||
})?;
|
||||
let crc = checksum::crc32(&payload);
|
||||
let mut buf = Vec::with_capacity(8 + payload.len());
|
||||
buf.extend_from_slice(&crc.to_le_bytes());
|
||||
buf.extend_from_slice(&META_VERSION.to_le_bytes());
|
||||
buf.extend_from_slice(&payload);
|
||||
|
||||
crate::fs::create_atomic(path, &buf)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_bin<T: for<'de> Deserialize<'de>>(path: &Path) -> Result<T> {
|
||||
let data = std::fs::read(path)?;
|
||||
if data.len() < 8 {
|
||||
return Err(crate::error::Error::CorruptMeta(path.display().to_string()));
|
||||
}
|
||||
let stored_crc = u32::from_le_bytes(data[0..4].try_into().unwrap());
|
||||
let version = u32::from_le_bytes(data[4..8].try_into().unwrap());
|
||||
if version != META_VERSION {
|
||||
return Err(crate::error::Error::UnsupportedMetaVersion {
|
||||
path: path.to_path_buf(),
|
||||
version,
|
||||
});
|
||||
}
|
||||
let computed = checksum::crc32(&data[8..]);
|
||||
if stored_crc != computed {
|
||||
return Err(crate::error::Error::CorruptMeta(path.display().to_string()));
|
||||
}
|
||||
bincode::deserialize(&data[8..]).map_err(|e| {
|
||||
crate::error::Error::CorruptMeta(format!("{}: bincode decode: {}", path.display(), e))
|
||||
})
|
||||
}
|
||||
|
||||
// ── GlobalMeta ─────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GlobalMeta {
|
||||
pub version: u32,
|
||||
pub accounts: Vec<String>,
|
||||
}
|
||||
|
||||
impl Default for GlobalMeta {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
version: META_VERSION,
|
||||
accounts: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl GlobalMeta {
|
||||
pub fn load(store_root: &Path) -> Result<Self> {
|
||||
let bin_path = store_root.join("global_meta.bin");
|
||||
if bin_path.exists() {
|
||||
return read_bin(&bin_path);
|
||||
}
|
||||
// Migration from JSON
|
||||
let json_path = store_root.join("global_meta.json");
|
||||
if json_path.exists() {
|
||||
let data = std::fs::read_to_string(&json_path)?;
|
||||
let mut meta: Self = serde_json::from_str(&data)?;
|
||||
meta.accounts.sort();
|
||||
write_bin(&bin_path, &meta)?;
|
||||
let _ = std::fs::remove_file(&json_path);
|
||||
return Ok(meta);
|
||||
}
|
||||
Ok(Self::default())
|
||||
}
|
||||
|
||||
pub fn save(&self, store_root: &Path) -> Result<()> {
|
||||
let path = store_root.join("global_meta.bin");
|
||||
let mut meta = self.clone();
|
||||
meta.accounts.sort();
|
||||
write_bin(&path, &meta)
|
||||
}
|
||||
}
|
||||
|
||||
// ── SegmentStats ───────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SegmentStats {
|
||||
pub segment_id: u32,
|
||||
pub total_bytes: u64,
|
||||
pub deleted_bytes: u64,
|
||||
pub deleted_ratio: f64,
|
||||
pub sealed: bool,
|
||||
/// Byte offset up to which entries have been indexed in bucket files.
|
||||
/// Recovery starts scanning from here instead of 0.
|
||||
pub indexed_up_to_offset: u64,
|
||||
}
|
||||
|
||||
impl SegmentStats {
|
||||
pub fn new(segment_id: u32) -> Self {
|
||||
Self {
|
||||
segment_id,
|
||||
total_bytes: 0,
|
||||
deleted_bytes: 0,
|
||||
deleted_ratio: 0.0,
|
||||
sealed: false,
|
||||
indexed_up_to_offset: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn recompute_ratio(&mut self) {
|
||||
if self.total_bytes > 0 {
|
||||
self.deleted_ratio = self.deleted_bytes as f64 / self.total_bytes as f64;
|
||||
} else {
|
||||
self.deleted_ratio = 0.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── AccountMeta ────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AccountMeta {
|
||||
pub account_id: String,
|
||||
pub active_segment_id: u32,
|
||||
pub segments: BTreeMap<u32, SegmentStats>,
|
||||
}
|
||||
|
||||
impl AccountMeta {
|
||||
pub fn new(account_id: String, active_segment_id: u32) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
active_segment_id,
|
||||
segments: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load(account_dir: &Path) -> Result<Self> {
|
||||
let bin_path = account_dir.join("meta.bin");
|
||||
if bin_path.exists() {
|
||||
return read_bin(&bin_path);
|
||||
}
|
||||
// Migration from JSON
|
||||
let json_path = account_dir.join("meta.json");
|
||||
if json_path.exists() {
|
||||
let data = std::fs::read_to_string(&json_path)?;
|
||||
let meta: Self = serde_json::from_str(&data)?;
|
||||
write_bin(&bin_path, &meta)?;
|
||||
let _ = std::fs::remove_file(&json_path);
|
||||
return Ok(meta);
|
||||
}
|
||||
Err(crate::error::Error::AccountNotFound(
|
||||
account_dir.to_string_lossy().into(),
|
||||
))
|
||||
}
|
||||
|
||||
pub fn save(&self, account_dir: &Path) -> Result<()> {
|
||||
write_bin(&account_dir.join("meta.bin"), self)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_global_meta_bin_roundtrip() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut meta = GlobalMeta::default();
|
||||
meta.accounts.push("alice".into());
|
||||
meta.save(dir.path()).unwrap();
|
||||
|
||||
let loaded = GlobalMeta::load(dir.path()).unwrap();
|
||||
assert_eq!(loaded.accounts, vec!["alice"]);
|
||||
assert!(!dir.path().join("global_meta.json").exists());
|
||||
assert!(dir.path().join("global_meta.bin").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_global_meta_default_when_missing() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let meta = GlobalMeta::load(dir.path()).unwrap();
|
||||
assert!(meta.accounts.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_json_migration() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
// Write old JSON format
|
||||
let json = r#"{"version":1,"accounts":["bob","alice"]}"#;
|
||||
std::fs::write(dir.path().join("global_meta.json"), json).unwrap();
|
||||
|
||||
let meta = GlobalMeta::load(dir.path()).unwrap();
|
||||
// Should be sorted
|
||||
assert_eq!(meta.accounts, vec!["alice", "bob"]);
|
||||
// JSON should be removed
|
||||
assert!(!dir.path().join("global_meta.json").exists());
|
||||
// BIN should exist
|
||||
assert!(dir.path().join("global_meta.bin").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_account_meta_bin_roundtrip() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut meta = AccountMeta::new("alice".into(), 1);
|
||||
meta.segments.insert(
|
||||
1,
|
||||
SegmentStats {
|
||||
segment_id: 1,
|
||||
total_bytes: 1000,
|
||||
deleted_bytes: 300,
|
||||
deleted_ratio: 0.3,
|
||||
sealed: false,
|
||||
indexed_up_to_offset: 0,
|
||||
},
|
||||
);
|
||||
meta.save(dir.path()).unwrap();
|
||||
|
||||
let loaded = AccountMeta::load(dir.path()).unwrap();
|
||||
assert_eq!(loaded.active_segment_id, 1);
|
||||
assert_eq!(loaded.segments[&1].total_bytes, 1000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_corrupt_bin_detected() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("meta.bin"), vec![0xFFu8; 100]).unwrap();
|
||||
let result = AccountMeta::load(dir.path());
|
||||
assert!(result.is_err());
|
||||
// 0xFFFFFFFF version triggers UnsupportedMetaVersion
|
||||
assert!(matches!(result.unwrap_err(), crate::error::Error::UnsupportedMetaVersion { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crc_corruption_detected() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
// Write a well-formed header (version=1) but with wrong CRC bytes
|
||||
let mut buf = Vec::new();
|
||||
buf.extend_from_slice(&0xDEADBEEFu32.to_le_bytes()); // wrong CRC
|
||||
buf.extend_from_slice(&1u32.to_le_bytes()); // version = 1 (OK)
|
||||
buf.extend_from_slice(b"some payload bytes"); // payload
|
||||
std::fs::write(dir.path().join("meta.bin"), &buf).unwrap();
|
||||
let result = AccountMeta::load(dir.path());
|
||||
assert!(matches!(result.unwrap_err(), crate::error::Error::CorruptMeta(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_account_json_migration() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
// Write old JSON format for AccountMeta
|
||||
let json = r#"{"account_id":"alice","active_segment_id":5,"segments":{}}"#;
|
||||
std::fs::write(dir.path().join("meta.json"), json).unwrap();
|
||||
|
||||
let meta = AccountMeta::load(dir.path()).unwrap();
|
||||
assert_eq!(meta.account_id, "alice");
|
||||
assert_eq!(meta.active_segment_id, 5);
|
||||
// JSON should be removed
|
||||
assert!(!dir.path().join("meta.json").exists());
|
||||
// BIN should exist
|
||||
assert!(dir.path().join("meta.bin").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bin_sorted_keys() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut meta = AccountMeta::new("test".into(), 1);
|
||||
meta.segments.insert(3, SegmentStats::new(3));
|
||||
meta.segments.insert(1, SegmentStats::new(1));
|
||||
meta.segments.insert(2, SegmentStats::new(2));
|
||||
meta.save(dir.path()).unwrap();
|
||||
|
||||
let loaded = AccountMeta::load(dir.path()).unwrap();
|
||||
let keys: Vec<u32> = loaded.segments.keys().copied().collect();
|
||||
assert_eq!(keys, vec![1, 2, 3]);
|
||||
}
|
||||
}
|
||||
202
crates/blob/src/recovery.rs
Normal file
202
crates/blob/src/recovery.rs
Normal file
@@ -0,0 +1,202 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::bucket::{self, BucketFile, IndexRecord};
|
||||
use crate::error::Result;
|
||||
use crate::meta::{AccountMeta, SegmentStats};
|
||||
use crate::segment::{self, SegmentReader};
|
||||
|
||||
/// Recover an account after a crash: scan segments, repair indices, update stats.
|
||||
pub fn recover_account(account_dir: &Path) -> Result<AccountMeta> {
|
||||
let meta_bin = account_dir.join("meta.bin");
|
||||
let meta_json = account_dir.join("meta.json");
|
||||
let meta_exists = meta_bin.exists() || meta_json.exists();
|
||||
let mut meta = if meta_exists {
|
||||
AccountMeta::load(account_dir).unwrap_or_else(|_| {
|
||||
AccountMeta::new(
|
||||
account_dir
|
||||
.file_name()
|
||||
.unwrap_or_default()
|
||||
.to_string_lossy()
|
||||
.into(),
|
||||
1,
|
||||
)
|
||||
})
|
||||
} else {
|
||||
return Ok(AccountMeta::new(
|
||||
account_dir
|
||||
.file_name()
|
||||
.unwrap_or_default()
|
||||
.to_string_lossy()
|
||||
.into(),
|
||||
1,
|
||||
));
|
||||
};
|
||||
|
||||
// Discover all segment files on disk
|
||||
let seg_dir = account_dir.join("segments");
|
||||
if !seg_dir.exists() {
|
||||
fs::create_dir_all(&seg_dir)?;
|
||||
}
|
||||
|
||||
let mut disk_segments: Vec<u32> = Vec::new();
|
||||
if seg_dir.exists() {
|
||||
for entry in fs::read_dir(&seg_dir)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
if name_str.ends_with(".seg") && !name_str.contains("temp_") {
|
||||
if let Some(id_str) = name_str.strip_suffix(".seg") {
|
||||
if let Ok(id) = id_str.parse::<u32>() {
|
||||
disk_segments.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
disk_segments.sort_unstable();
|
||||
|
||||
if disk_segments.is_empty() {
|
||||
meta.active_segment_id = 1;
|
||||
} else {
|
||||
let max_id = *disk_segments.last().unwrap();
|
||||
meta.active_segment_id = max_id;
|
||||
}
|
||||
|
||||
// Ensure buckets directory exists
|
||||
let buckets_dir = account_dir.join("buckets");
|
||||
fs::create_dir_all(&buckets_dir)?;
|
||||
|
||||
// For each segment, scan only the unindexed tail and update stats incrementally
|
||||
for &seg_id in &disk_segments {
|
||||
let seg_path = seg_dir.join(segment::segment_filename(seg_id));
|
||||
let file_size = fs::metadata(&seg_path)?.len();
|
||||
|
||||
// Preserve existing stats; start fresh if this is a newly discovered segment
|
||||
let mut stats = meta.segments.remove(&seg_id).unwrap_or_else(|| SegmentStats::new(seg_id));
|
||||
let is_sealed = seg_id != meta.active_segment_id;
|
||||
stats.sealed = is_sealed;
|
||||
|
||||
// Scan start: from last indexed offset. Clamp defensively.
|
||||
let scan_start = if stats.indexed_up_to_offset <= file_size {
|
||||
stats.indexed_up_to_offset
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
// If fully indexed, skip scanning entirely
|
||||
if scan_start >= file_size {
|
||||
meta.segments.insert(seg_id, stats);
|
||||
continue;
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(seg_path.clone(), seg_id)?;
|
||||
let mut new_records: HashMap<u16, Vec<IndexRecord>> = HashMap::new();
|
||||
|
||||
let truncation_point = reader.scan_entries(scan_start, |entry, offset| {
|
||||
let bid = bucket::bucket_id(&entry.key);
|
||||
let rec = IndexRecord::new(
|
||||
entry.key,
|
||||
seg_id,
|
||||
offset,
|
||||
entry.data.len() as u32,
|
||||
entry.flags,
|
||||
);
|
||||
new_records.entry(bid).or_default().push(rec);
|
||||
|
||||
stats.total_bytes += entry.data.len() as u64;
|
||||
if entry.is_tombstone() {
|
||||
stats.deleted_bytes += entry.raw_size as u64;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
// Merge new records into bucket files (only the newly discovered ones)
|
||||
for (bid, records) in &new_records {
|
||||
let bf = BucketFile::open(account_dir, *bid);
|
||||
bf.append_batch(records)?;
|
||||
}
|
||||
|
||||
// Truncate if tail corruption found
|
||||
if truncation_point < file_size {
|
||||
segment::truncate_segment(&seg_path, truncation_point)?;
|
||||
}
|
||||
|
||||
stats.indexed_up_to_offset = truncation_point;
|
||||
stats.recompute_ratio();
|
||||
meta.segments.insert(seg_id, stats);
|
||||
}
|
||||
|
||||
meta.save(account_dir)?;
|
||||
|
||||
Ok(meta)
|
||||
}
|
||||
|
||||
/// Clean up leftover temp files from interrupted GC.
|
||||
pub fn cleanup_temp_files(account_dir: &Path) -> Result<()> {
|
||||
let seg_dir = account_dir.join("segments");
|
||||
if seg_dir.exists() {
|
||||
for entry in fs::read_dir(&seg_dir)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
if name_str.starts_with("temp_") {
|
||||
let path = entry.path();
|
||||
tracing::warn!("Removing leftover temp file: {:?}", path);
|
||||
fs::remove_file(&path)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Also cleanup temp bucket files
|
||||
let buckets_dir = account_dir.join("buckets");
|
||||
if buckets_dir.exists() {
|
||||
for entry in fs::read_dir(&buckets_dir)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
if name_str.ends_with(".tmp") {
|
||||
let path = entry.path();
|
||||
tracing::warn!("Removing leftover temp bucket file: {:?}", path);
|
||||
fs::remove_file(&path)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_recover_fresh_account() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let account_dir = dir.path().join("test");
|
||||
fs::create_dir_all(&account_dir).unwrap();
|
||||
|
||||
let meta = recover_account(&account_dir).unwrap();
|
||||
assert_eq!(meta.active_segment_id, 1);
|
||||
assert!(meta.segments.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cleanup_temp_files() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let account_dir = dir.path().join("test");
|
||||
fs::create_dir_all(account_dir.join("segments")).unwrap();
|
||||
fs::create_dir_all(account_dir.join("buckets")).unwrap();
|
||||
fs::write(
|
||||
account_dir.join("segments").join("temp_ABC123.seg"),
|
||||
b"garbage",
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(account_dir.join("buckets").join("00.idx.tmp"), b"garbage").unwrap();
|
||||
|
||||
cleanup_temp_files(&account_dir).unwrap();
|
||||
|
||||
assert!(!account_dir.join("segments").join("temp_ABC123.seg").exists());
|
||||
}
|
||||
}
|
||||
539
crates/blob/src/segment.rs
Normal file
539
crates/blob/src/segment.rs
Normal file
@@ -0,0 +1,539 @@
|
||||
use std::fs::{self, File};
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use crate::checksum;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::fs as fs_util;
|
||||
use crate::types::{Codec, ENTRY_HEADER_SIZE, ENTRY_MAGIC, SEGMENT_MAX_SIZE};
|
||||
|
||||
/// In-memory representation of a stored entry.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Entry {
|
||||
pub flags: u8,
|
||||
pub codec: Codec,
|
||||
pub key: [u8; 32],
|
||||
pub raw_size: u32,
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Entry {
|
||||
/// Create a normal data entry.
|
||||
pub fn new(key: [u8; 32], raw_data: &[u8], flags: u8, codec: Codec) -> Self {
|
||||
Self {
|
||||
flags,
|
||||
codec,
|
||||
key,
|
||||
raw_size: raw_data.len() as u32,
|
||||
data: raw_data.to_vec(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a tombstone entry.
|
||||
pub fn tombstone(key: [u8; 32]) -> Self {
|
||||
Self {
|
||||
flags: 1,
|
||||
codec: Codec::None,
|
||||
key,
|
||||
raw_size: 0,
|
||||
data: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_tombstone(&self) -> bool {
|
||||
self.flags == 1
|
||||
}
|
||||
|
||||
/// Total on-disk size: header + data
|
||||
pub fn disk_size(&self) -> usize {
|
||||
ENTRY_HEADER_SIZE + self.data.len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Write entries sequentially to a segment file.
|
||||
pub struct SegmentWriter {
|
||||
file: File,
|
||||
path: PathBuf,
|
||||
id: u32,
|
||||
bytes_written: u64,
|
||||
}
|
||||
|
||||
impl SegmentWriter {
|
||||
pub fn create(path: PathBuf, id: u32) -> Result<Self> {
|
||||
// Use create+truncate instead of create_new to avoid NFS O_EXCL issues.
|
||||
let file = File::create(&path)?;
|
||||
Ok(Self {
|
||||
file,
|
||||
path,
|
||||
id,
|
||||
bytes_written: 0,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn open_append(path: PathBuf, id: u32) -> Result<Self> {
|
||||
let mut file = fs_util::open_write(&path)?;
|
||||
file.seek(SeekFrom::End(0))?;
|
||||
let bytes_written = file.stream_position()?;
|
||||
Ok(Self {
|
||||
file,
|
||||
path,
|
||||
id,
|
||||
bytes_written,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn id(&self) -> u32 {
|
||||
self.id
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub fn bytes_written(&self) -> u64 {
|
||||
self.bytes_written
|
||||
}
|
||||
|
||||
pub fn is_full(&self) -> bool {
|
||||
self.bytes_written >= SEGMENT_MAX_SIZE
|
||||
}
|
||||
|
||||
/// Append an entry. Returns the offset where it was written.
|
||||
pub fn append(&mut self, entry: &Entry) -> Result<u64> {
|
||||
let offset = self.bytes_written;
|
||||
self.write_entry(entry)
|
||||
.map_err(|e| map_io_err(e, &self.path))?;
|
||||
Ok(offset)
|
||||
}
|
||||
|
||||
fn write_entry(&mut self, entry: &Entry) -> Result<()> {
|
||||
let data_size = entry.data.len() as u32;
|
||||
|
||||
// Write magic
|
||||
self.file.write_all(&ENTRY_MAGIC.to_le_bytes())?;
|
||||
|
||||
// CRC32 placeholder: write zeros, remember position
|
||||
let crc_pos = self.file.stream_position()?;
|
||||
self.file.write_all(&0u32.to_le_bytes())?;
|
||||
|
||||
// Write flags, codec, key, raw_size, data_size
|
||||
self.file.write_all(&[entry.flags])?;
|
||||
self.file.write_all(&[entry.codec as u8])?;
|
||||
self.file.write_all(&entry.key)?;
|
||||
self.file.write_all(&entry.raw_size.to_le_bytes())?;
|
||||
self.file.write_all(&data_size.to_le_bytes())?;
|
||||
|
||||
// Write data
|
||||
self.file.write_all(&entry.data)?;
|
||||
|
||||
// Calculate CRC32 over everything after the crc32 field
|
||||
let crc = {
|
||||
let mut hasher = checksum::CrcWriter::new();
|
||||
hasher.update(&[entry.flags]);
|
||||
hasher.update(&[entry.codec as u8]);
|
||||
hasher.update(&entry.key);
|
||||
hasher.update(&entry.raw_size.to_le_bytes());
|
||||
hasher.update(&data_size.to_le_bytes());
|
||||
hasher.update(&entry.data);
|
||||
hasher.finalize()
|
||||
};
|
||||
|
||||
// Seek back and write the real CRC32
|
||||
self.file.seek(SeekFrom::Start(crc_pos))?;
|
||||
self.file.write_all(&crc.to_le_bytes())?;
|
||||
|
||||
// Seek back to end
|
||||
self.file.seek(SeekFrom::End(0))?;
|
||||
|
||||
self.bytes_written += entry.disk_size() as u64;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn fsync(&self) -> Result<()> {
|
||||
self.file.sync_all().map_err(|e| {
|
||||
if e.kind() == std::io::ErrorKind::StorageFull {
|
||||
Error::DiskFull(format!("{}: {}", self.path.display(), e))
|
||||
} else {
|
||||
Error::Io(e)
|
||||
}
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Read entries from a segment file.
|
||||
pub struct SegmentReader {
|
||||
path: PathBuf,
|
||||
id: u32,
|
||||
}
|
||||
|
||||
impl SegmentReader {
|
||||
pub fn open(path: PathBuf, id: u32) -> Result<Self> {
|
||||
Ok(Self { path, id })
|
||||
}
|
||||
|
||||
pub fn id(&self) -> u32 {
|
||||
self.id
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub fn file_size(&self) -> Result<u64> {
|
||||
Ok(fs::metadata(&self.path)?.len())
|
||||
}
|
||||
|
||||
/// Read a single entry at the given offset. Returns the entry and the offset of the next entry.
|
||||
pub fn read_entry_at(&self, offset: u64) -> Result<(Entry, u64)> {
|
||||
let mut file = fs_util::open_read(&self.path)?;
|
||||
file.seek(SeekFrom::Start(offset))?;
|
||||
|
||||
// Read magic
|
||||
let mut magic_buf = [0u8; 4];
|
||||
file.read_exact(&mut magic_buf)?;
|
||||
let magic = u32::from_le_bytes(magic_buf);
|
||||
if magic != ENTRY_MAGIC {
|
||||
return Err(Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
reason: format!("bad magic: 0x{:08X}", magic),
|
||||
});
|
||||
}
|
||||
|
||||
// Read CRC32
|
||||
let mut crc_buf = [0u8; 4];
|
||||
file.read_exact(&mut crc_buf)?;
|
||||
let stored_crc = u32::from_le_bytes(crc_buf);
|
||||
|
||||
// Read flags, codec
|
||||
let mut flags_buf = [0u8; 1];
|
||||
file.read_exact(&mut flags_buf)?;
|
||||
let flags = flags_buf[0];
|
||||
|
||||
let mut codec_buf = [0u8; 1];
|
||||
file.read_exact(&mut codec_buf)?;
|
||||
let codec = Codec::from_u8(codec_buf[0]).ok_or_else(|| Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
reason: format!("unknown codec: {}", codec_buf[0]),
|
||||
})?;
|
||||
|
||||
// Read key, raw_size, data_size
|
||||
let mut key = [0u8; 32];
|
||||
file.read_exact(&mut key)?;
|
||||
|
||||
let mut raw_size_buf = [0u8; 4];
|
||||
file.read_exact(&mut raw_size_buf)?;
|
||||
let raw_size = u32::from_le_bytes(raw_size_buf);
|
||||
|
||||
let mut data_size_buf = [0u8; 4];
|
||||
file.read_exact(&mut data_size_buf)?;
|
||||
let data_size = u32::from_le_bytes(data_size_buf);
|
||||
|
||||
// Read data
|
||||
let mut data = vec![0u8; data_size as usize];
|
||||
file.read_exact(&mut data)?;
|
||||
|
||||
// Verify CRC32 (over everything after the crc32 field)
|
||||
let computed_crc = {
|
||||
let mut hasher = checksum::CrcWriter::new();
|
||||
hasher.update(&[flags]);
|
||||
hasher.update(&[codec as u8]);
|
||||
hasher.update(&key);
|
||||
hasher.update(&raw_size.to_le_bytes());
|
||||
hasher.update(&data_size.to_le_bytes());
|
||||
hasher.update(&data);
|
||||
hasher.finalize()
|
||||
};
|
||||
|
||||
if stored_crc != computed_crc {
|
||||
return Err(Error::CrcMismatch {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
});
|
||||
}
|
||||
|
||||
let next_offset = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
|
||||
Ok((
|
||||
Entry {
|
||||
flags,
|
||||
codec,
|
||||
key,
|
||||
raw_size,
|
||||
data,
|
||||
},
|
||||
next_offset,
|
||||
))
|
||||
}
|
||||
|
||||
/// Read a single entry at the given offset using a pre-opened File (via Mutex).
|
||||
/// This avoids the per-read File::open cost for hot segments.
|
||||
pub fn read_entry_at_file(&self, offset: u64, file: &Mutex<File>) -> Result<(Entry, u64)> {
|
||||
|
||||
let mut file = file.lock().unwrap();
|
||||
|
||||
file.seek(SeekFrom::Start(offset))?;
|
||||
|
||||
// Read magic
|
||||
let mut magic_buf = [0u8; 4];
|
||||
file.read_exact(&mut magic_buf)?;
|
||||
let magic = u32::from_le_bytes(magic_buf);
|
||||
if magic != ENTRY_MAGIC {
|
||||
return Err(Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
reason: format!("bad magic: 0x{:08X}", magic),
|
||||
});
|
||||
}
|
||||
|
||||
// Read CRC32
|
||||
let mut crc_buf = [0u8; 4];
|
||||
file.read_exact(&mut crc_buf)?;
|
||||
let stored_crc = u32::from_le_bytes(crc_buf);
|
||||
|
||||
// Read flags, codec
|
||||
let mut flags_buf = [0u8; 1];
|
||||
file.read_exact(&mut flags_buf)?;
|
||||
let flags = flags_buf[0];
|
||||
|
||||
let mut codec_buf = [0u8; 1];
|
||||
file.read_exact(&mut codec_buf)?;
|
||||
let codec = Codec::from_u8(codec_buf[0]).ok_or_else(|| Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
reason: format!("unknown codec: {}", codec_buf[0]),
|
||||
})?;
|
||||
|
||||
// Read key, raw_size, data_size
|
||||
let mut key = [0u8; 32];
|
||||
file.read_exact(&mut key)?;
|
||||
|
||||
let mut raw_size_buf = [0u8; 4];
|
||||
file.read_exact(&mut raw_size_buf)?;
|
||||
let raw_size = u32::from_le_bytes(raw_size_buf);
|
||||
|
||||
let mut data_size_buf = [0u8; 4];
|
||||
file.read_exact(&mut data_size_buf)?;
|
||||
let data_size = u32::from_le_bytes(data_size_buf);
|
||||
|
||||
// Read data
|
||||
let mut data = vec![0u8; data_size as usize];
|
||||
file.read_exact(&mut data)?;
|
||||
|
||||
// Verify CRC32
|
||||
let computed_crc = {
|
||||
let mut hasher = crate::checksum::CrcWriter::new();
|
||||
hasher.update(&[flags]);
|
||||
hasher.update(&[codec as u8]);
|
||||
hasher.update(&key);
|
||||
hasher.update(&raw_size.to_le_bytes());
|
||||
hasher.update(&data_size.to_le_bytes());
|
||||
hasher.update(&data);
|
||||
hasher.finalize()
|
||||
};
|
||||
|
||||
if stored_crc != computed_crc {
|
||||
return Err(Error::CrcMismatch {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
});
|
||||
}
|
||||
|
||||
let next_offset = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
|
||||
Ok((
|
||||
Entry {
|
||||
flags,
|
||||
codec,
|
||||
key,
|
||||
raw_size,
|
||||
data,
|
||||
},
|
||||
next_offset,
|
||||
))
|
||||
}
|
||||
|
||||
/// Read data portion of an entry (for pread-style reads when you already know offset + data_size).
|
||||
pub fn read_data(&self, offset: u64, data_size: u32) -> Result<Vec<u8>> {
|
||||
let mut file = fs_util::open_read(&self.path)?;
|
||||
// Skip magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4) = 50 bytes
|
||||
let data_start = offset + ENTRY_HEADER_SIZE as u64;
|
||||
file.seek(SeekFrom::Start(data_start))?;
|
||||
let mut buf = vec![0u8; data_size as usize];
|
||||
file.read_exact(&mut buf)?;
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
/// Read the full entry header + data for verification (used by recovery and GC).
|
||||
pub fn read_full_entry(&self, offset: u64, data_size: u32) -> Result<Vec<u8>> {
|
||||
let mut file = fs_util::open_read(&self.path)?;
|
||||
file.seek(SeekFrom::Start(offset))?;
|
||||
let total = ENTRY_HEADER_SIZE + data_size as usize;
|
||||
let mut buf = vec![0u8; total];
|
||||
file.read_exact(&mut buf)?;
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
/// Iterate over all valid entries in the segment, calling f for each.
|
||||
/// Stops when hitting a corrupt/incomplete entry at the tail.
|
||||
pub fn scan_entries<F>(&self, start_offset: u64, mut f: F) -> Result<u64>
|
||||
where
|
||||
F: FnMut(&Entry, u64) -> Result<()>,
|
||||
{
|
||||
let file_size = self.file_size()?;
|
||||
let mut offset = start_offset;
|
||||
|
||||
while offset + ENTRY_HEADER_SIZE as u64 <= file_size {
|
||||
match self.read_entry_at(offset) {
|
||||
Ok((entry, next)) => {
|
||||
f(&entry, offset)?;
|
||||
offset = next;
|
||||
}
|
||||
Err(Error::CrcMismatch { .. }) | Err(Error::CorruptEntry { .. }) => {
|
||||
// If near end of file (within one max entry), truncate
|
||||
if file_size - offset < ENTRY_HEADER_SIZE as u64 + 100 * 1024 * 1024 {
|
||||
// Likely a partial write at tail, stop here
|
||||
break;
|
||||
} else {
|
||||
return Err(Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
reason: "mid-file corruption detected".into(),
|
||||
});
|
||||
}
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(offset) // return the truncation point
|
||||
}
|
||||
}
|
||||
|
||||
/// Truncate a segment file to the given size.
|
||||
pub fn truncate_segment(path: &Path, size: u64) -> Result<()> {
|
||||
fs_util::truncate(path, size)
|
||||
}
|
||||
|
||||
/// Map an Error, converting Io(StorageFull) to DiskFull with path context.
|
||||
fn map_io_err(e: Error, path: &Path) -> Error {
|
||||
match e {
|
||||
Error::Io(io) if io.kind() == std::io::ErrorKind::StorageFull => {
|
||||
Error::DiskFull(format!("{}: {}", path.display(), io))
|
||||
}
|
||||
_ => e,
|
||||
}
|
||||
}
|
||||
|
||||
/// Segment file name from id: "00000001.seg"
|
||||
pub fn segment_filename(id: u32) -> String {
|
||||
format!("{:08}.seg", id)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn temp_segment_path(dir: &TempDir, id: u32) -> PathBuf {
|
||||
dir.path().join(segment_filename(id))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_write_and_read_entry() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = temp_segment_path(&dir, 1);
|
||||
let key = [0xAAu8; 32];
|
||||
let data = b"hello world".to_vec();
|
||||
|
||||
let entry = Entry::new(key, &data, 0, Codec::None);
|
||||
{
|
||||
let mut writer = SegmentWriter::create(path.clone(), 1).unwrap();
|
||||
writer.append(&entry).unwrap();
|
||||
writer.fsync().unwrap();
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(path, 1).unwrap();
|
||||
let (read_entry, next) = reader.read_entry_at(0).unwrap();
|
||||
|
||||
assert_eq!(read_entry.key, key);
|
||||
assert_eq!(read_entry.data, data);
|
||||
assert_eq!(read_entry.flags, 0);
|
||||
assert_eq!(read_entry.raw_size, 11);
|
||||
assert!(next > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tombstone_entry() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = temp_segment_path(&dir, 1);
|
||||
let key = [0xBBu8; 32];
|
||||
|
||||
let entry = Entry::tombstone(key);
|
||||
{
|
||||
let mut writer = SegmentWriter::create(path.clone(), 1).unwrap();
|
||||
writer.append(&entry).unwrap();
|
||||
writer.fsync().unwrap();
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(path, 1).unwrap();
|
||||
let (read_entry, _) = reader.read_entry_at(0).unwrap();
|
||||
|
||||
assert!(read_entry.is_tombstone());
|
||||
assert_eq!(read_entry.data.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_entries() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = temp_segment_path(&dir, 1);
|
||||
|
||||
let entries: Vec<_> = (0..10)
|
||||
.map(|i| {
|
||||
let mut key = [0u8; 32];
|
||||
key[0] = i;
|
||||
Entry::new(key, &vec![i; 100], 0, Codec::None)
|
||||
})
|
||||
.collect();
|
||||
|
||||
{
|
||||
let mut writer = SegmentWriter::create(path.clone(), 1).unwrap();
|
||||
for e in &entries {
|
||||
writer.append(e).unwrap();
|
||||
}
|
||||
writer.fsync().unwrap();
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(path, 1).unwrap();
|
||||
let mut offset = 0u64;
|
||||
for (i, expected) in entries.iter().enumerate() {
|
||||
let (entry, next) = reader.read_entry_at(offset).unwrap();
|
||||
assert_eq!(entry.key[0], i as u8);
|
||||
assert_eq!(entry.data, expected.data);
|
||||
offset = next;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bad_magic_detected() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = temp_segment_path(&dir, 1);
|
||||
// Write garbage
|
||||
std::fs::write(&path, vec![0xFFu8; 100]).unwrap();
|
||||
|
||||
let reader = SegmentReader::open(path, 1).unwrap();
|
||||
let result = reader.read_entry_at(0);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_full() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = temp_segment_path(&dir, 1);
|
||||
let writer = SegmentWriter::create(path, 1).unwrap();
|
||||
assert!(!writer.is_full());
|
||||
}
|
||||
}
|
||||
88
crates/blob/src/types.rs
Normal file
88
crates/blob/src/types.rs
Normal file
@@ -0,0 +1,88 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Magic number for entry identification
|
||||
pub const ENTRY_MAGIC: u32 = 0xB3DB_0001;
|
||||
|
||||
/// Fixed header size: magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4)
|
||||
pub const ENTRY_HEADER_SIZE: usize = 50;
|
||||
|
||||
/// Index record size: key(32) + segment_id(4) + offset(8) + data_size(4) + flags(1) + _pad(3)
|
||||
pub const INDEX_RECORD_SIZE: usize = 52;
|
||||
|
||||
/// Maximum segment size (256 MB)
|
||||
pub const SEGMENT_MAX_SIZE: u64 = 256 * 1024 * 1024;
|
||||
|
||||
/// Number of hash buckets per account
|
||||
pub const BUCKET_COUNT: u16 = 16;
|
||||
|
||||
/// Maximum value size (100 MB)
|
||||
pub const MAX_VALUE_SIZE: usize = 100 * 1024 * 1024;
|
||||
|
||||
/// Default compression threshold (4 KB)
|
||||
pub const DEFAULT_COMPRESS_THRESHOLD: usize = 4096;
|
||||
|
||||
/// Default LRU bucket cache size
|
||||
pub const DEFAULT_LRU_BUCKET_COUNT: usize = 256;
|
||||
|
||||
/// Default GC deleted ratio threshold
|
||||
pub const DEFAULT_GC_DELETED_RATIO: f64 = 0.30;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum Codec {
|
||||
None = 0,
|
||||
Zstd = 1,
|
||||
Lz4 = 2,
|
||||
}
|
||||
|
||||
impl Codec {
|
||||
pub fn from_u8(v: u8) -> Option<Self> {
|
||||
match v {
|
||||
0 => Some(Codec::None),
|
||||
1 => Some(Codec::Zstd),
|
||||
2 => Some(Codec::Lz4),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Config {
|
||||
pub compress_threshold: usize,
|
||||
pub default_codec: Codec,
|
||||
pub compression_level: i32,
|
||||
pub lru_bucket_count: usize,
|
||||
pub gc_deleted_ratio: f64,
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
compress_threshold: DEFAULT_COMPRESS_THRESHOLD,
|
||||
default_codec: Codec::Zstd,
|
||||
compression_level: 0,
|
||||
lru_bucket_count: DEFAULT_LRU_BUCKET_COUNT,
|
||||
gc_deleted_ratio: DEFAULT_GC_DELETED_RATIO,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn validate(&self) -> crate::error::Result<()> {
|
||||
if self.lru_bucket_count == 0 {
|
||||
return Err(crate::error::Error::InvalidConfig(
|
||||
"lru_bucket_count must be > 0".into(),
|
||||
));
|
||||
}
|
||||
if self.gc_deleted_ratio <= 0.0 || self.gc_deleted_ratio >= 1.0 {
|
||||
return Err(crate::error::Error::InvalidConfig(
|
||||
"gc_deleted_ratio must be in (0.0, 1.0)".into(),
|
||||
));
|
||||
}
|
||||
if self.compression_level < 0 {
|
||||
return Err(crate::error::Error::InvalidConfig(
|
||||
"compression_level must be >= 0".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
498
crates/blob/tests/acid_test.rs
Normal file
498
crates/blob/tests/acid_test.rs
Normal file
@@ -0,0 +1,498 @@
|
||||
/// Crash-consistency and ACID property tests for bichon-blob.
|
||||
///
|
||||
/// Since we can't kill the process mid-write in an inline test, we simulate crashes
|
||||
/// by dropping the Engine without calling any cleanup (close/drop is the "crash"),
|
||||
/// then re-opening and verifying recovery produced consistent state.
|
||||
///
|
||||
/// For true power-loss simulation, each test writes data, drops the engine abruptly,
|
||||
/// then reopens and verifies: no corruption, no lost committed data, no partial writes.
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::thread;
|
||||
|
||||
use bichon_blob::{Codec, Config, Engine};
|
||||
use tempfile::TempDir;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn make_key(seed: u64) -> [u8; 32] {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..8].copy_from_slice(&seed.to_le_bytes());
|
||||
key
|
||||
}
|
||||
|
||||
fn make_value(size: usize) -> Vec<u8> {
|
||||
let pattern = b"The quick brown fox jumps over the lazy dog. ";
|
||||
let mut v = Vec::with_capacity(size);
|
||||
while v.len() < size {
|
||||
let rem = size - v.len();
|
||||
let n = rem.min(pattern.len());
|
||||
v.extend_from_slice(&pattern[..n]);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Durability: committed data survives crash
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_durability_single_write_survives_crash() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let key = make_key(42);
|
||||
let value = make_value(8192);
|
||||
|
||||
// Write
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
} // <-- Engine dropped = simulated crash
|
||||
|
||||
// Recover
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_durability_many_writes_survive_crash() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let n = 500;
|
||||
let value = make_value(2048);
|
||||
let mut keys = Vec::new();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
for i in 0..n {
|
||||
let key = make_key(i as u64);
|
||||
keys.push(key);
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
} // crash
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for (i, key) in keys.iter().enumerate() {
|
||||
let result = engine.read("alice", key).unwrap();
|
||||
assert_eq!(result, Some(value.clone()), "missing key at index {}", i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_durability_delete_survives_crash() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let key = make_key(99);
|
||||
let value = make_value(4096);
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
} // crash after write
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.delete("alice", &key).unwrap();
|
||||
} // crash after delete
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, None, "delete should persist across crash");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Atomicity: no partial writes visible after crash
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_atomicity_no_partial_entries_after_crash() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
// Write enough entries to fill part of a segment, then crash
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
let value = make_value(50_000); // big enough to notice
|
||||
for i in 0..200u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
} // crash
|
||||
|
||||
// Recovery should clean up any partial tail entries and all committed
|
||||
// entries should be readable
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let value = make_value(50_000);
|
||||
for i in 0..200u64 {
|
||||
let result = engine.read("alice", &make_key(i)).unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
Some(value.clone()),
|
||||
"committed key {} should be intact",
|
||||
i
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_atomicity_crash_during_segment_roll() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let big_value = make_value(2 * 1024 * 1024); // 2 MB each entry
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
// Write enough to cross at least one segment boundary (256 MB)
|
||||
for i in 0..140u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &big_value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
} // crash mid-way or after multiple segments
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
// All committed writes (that returned Ok) must be readable
|
||||
for i in 0..140u64 {
|
||||
let result = engine.read("alice", &make_key(i)).unwrap();
|
||||
assert!(
|
||||
result.is_some(),
|
||||
"key {} should exist after segment roll recovery",
|
||||
i
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 3. Consistency: CRC detects corruption, no silent data loss
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_consistency_crc_detects_corruption() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let key = make_key(77);
|
||||
let value = make_value(8192);
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Corrupt the segment file by flipping a byte
|
||||
let seg_path = find_first_segment(dir.path(), "alice");
|
||||
let mut data = fs::read(&seg_path).unwrap();
|
||||
// Flip a byte in the data portion, not the header
|
||||
let flip_pos = data.len() - 100;
|
||||
data[flip_pos] ^= 0xFF;
|
||||
fs::write(&seg_path, &data).unwrap();
|
||||
|
||||
// Reading should detect CRC mismatch
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.read("alice", &key);
|
||||
// Either error or None is acceptable — never silently wrong data
|
||||
match result {
|
||||
Err(_) => {} // CRC mismatch detected — good
|
||||
Ok(None) => {} // index may point to truncated/removed data
|
||||
Ok(Some(v)) => {
|
||||
if v == value {
|
||||
panic!("CRC corruption was NOT detected — silent data corruption!");
|
||||
}
|
||||
// If value differs, index pointed elsewhere after recovery
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_consistency_corrupt_magic_truncated_on_recovery() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
for i in 0..10u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &make_value(4096), Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
// Append garbage to the segment file (simulating partial write from crash)
|
||||
let seg_path = find_first_segment(dir.path(), "alice");
|
||||
let mut data = fs::read(&seg_path).unwrap();
|
||||
let orig_len = data.len();
|
||||
// Append garbage that doesn't start with the magic number
|
||||
data.extend_from_slice(&[0xFF; 200]);
|
||||
fs::write(&seg_path, &data).unwrap();
|
||||
|
||||
// Recovery should truncate the garbage
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
// Verify committed data is still intact
|
||||
for i in 0..10u64 {
|
||||
let result = engine.read("alice", &make_key(i)).unwrap();
|
||||
assert!(result.is_some(), "committed key {} should survive tail truncation", i);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify file was actually truncated
|
||||
let truncated_len = fs::metadata(&seg_path).unwrap().len();
|
||||
assert!(truncated_len <= orig_len as u64, "garbage should have been truncated");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 4. Isolation: concurrent reader sees consistent snapshot
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_isolation_reader_sees_snapshot_not_partial_write() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
// Pre-populate a known key
|
||||
let original_value = make_value(4096);
|
||||
let key = make_key(100);
|
||||
engine
|
||||
.write("alice", key, &original_value, Codec::Zstd)
|
||||
.unwrap();
|
||||
|
||||
let running = Arc::new(AtomicBool::new(true));
|
||||
let writer_done = Arc::new(AtomicBool::new(false));
|
||||
|
||||
// Spawn a writer that continuously overwrites the same key
|
||||
let writer_engine = engine.clone();
|
||||
let writer_running = running.clone();
|
||||
let writer_done_flag = writer_done.clone();
|
||||
let writer_key = key;
|
||||
|
||||
let writer = thread::spawn(move || {
|
||||
for i in 0..1000u64 {
|
||||
if !writer_running.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
let val = make_value(4096 + (i as usize % 100));
|
||||
writer_engine
|
||||
.write("alice", writer_key, &val, Codec::Zstd)
|
||||
.unwrap();
|
||||
thread::yield_now();
|
||||
}
|
||||
writer_done_flag.store(true, Ordering::SeqCst);
|
||||
});
|
||||
|
||||
// Concurrent reader: reads should never panic or hang
|
||||
let reader_engine = engine.clone();
|
||||
let reader_running = running.clone();
|
||||
let reader = thread::spawn(move || {
|
||||
let mut reads = 0;
|
||||
while reads < 500 {
|
||||
if !reader_running.load(Ordering::Relaxed) && reads > 0 {
|
||||
break;
|
||||
}
|
||||
let result = reader_engine.read("alice", &key);
|
||||
match result {
|
||||
Ok(Some(_)) | Ok(None) => {} // OK
|
||||
Err(e) => {
|
||||
// Accept transient errors but report them
|
||||
eprintln!("reader saw error: {:?}", e);
|
||||
}
|
||||
}
|
||||
reads += 1;
|
||||
thread::yield_now();
|
||||
}
|
||||
});
|
||||
|
||||
reader.join().unwrap();
|
||||
running.store(false, Ordering::SeqCst);
|
||||
writer.join().unwrap();
|
||||
|
||||
// Final read should see the last committed value (not partial)
|
||||
let final_result = engine.read("alice", &key).unwrap();
|
||||
assert!(final_result.is_some(), "final read should find a value");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Crash during GC: old data intact, no corruption
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_crash_during_gc_leaves_data_intact() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let value = make_value(500_000); // 500 KB each
|
||||
// Write enough entries and delete some to create GC candidate
|
||||
for i in 0..500u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
// Delete ~40%
|
||||
for i in (0..500u64).step_by(5) {
|
||||
engine.delete("alice", &make_key(i)).unwrap();
|
||||
}
|
||||
// Single GC run (may or may not trigger)
|
||||
let _ = engine.gc("alice");
|
||||
} // crash after GC
|
||||
|
||||
// All non-deleted entries must still be readable
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let value = make_value(500_000);
|
||||
for i in 0..500u64 {
|
||||
let key = make_key(i);
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
if i % 5 == 0 {
|
||||
// Deleted keys
|
||||
assert_eq!(result, None, "key {} should be deleted", i);
|
||||
} else {
|
||||
assert_eq!(
|
||||
result,
|
||||
Some(value.clone()),
|
||||
"key {} should survive GC+crash",
|
||||
i
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Multiple crash-reopen cycles (torture test)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_multiple_crash_reopen_cycles() {
|
||||
use std::collections::HashSet;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let value = make_value(4096);
|
||||
let mut alive: HashSet<u64> = HashSet::new();
|
||||
|
||||
// Populate and crash
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
for i in 0..50u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
alive.insert(i);
|
||||
}
|
||||
}
|
||||
|
||||
// Reopen, verify all exist, write more, crash
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for &k in &alive {
|
||||
assert!(engine.read("alice", &make_key(k)).unwrap().is_some());
|
||||
}
|
||||
for i in 100..150u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
alive.insert(i);
|
||||
}
|
||||
}
|
||||
|
||||
// Reopen, verify all exist, delete some, crash
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for &k in &alive {
|
||||
assert!(engine.read("alice", &make_key(k)).unwrap().is_some());
|
||||
}
|
||||
for i in 0..10u64 {
|
||||
engine.delete("alice", &make_key(i)).unwrap();
|
||||
alive.remove(&i);
|
||||
}
|
||||
}
|
||||
|
||||
// Final reopen: survivors exist, deleted gone
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for &k in &alive {
|
||||
assert!(engine.read("alice", &make_key(k)).unwrap().is_some(),
|
||||
"key {} should exist", k);
|
||||
}
|
||||
for i in 0..10u64 {
|
||||
assert_eq!(engine.read("alice", &make_key(i)).unwrap(), None,
|
||||
"key {} should be deleted", i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Account-level isolation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_account_isolation_crash_one_account_does_not_affect_others() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine.create_account("bob").unwrap();
|
||||
|
||||
engine
|
||||
.write("alice", make_key(1), &make_value(4096), Codec::Zstd)
|
||||
.unwrap();
|
||||
engine
|
||||
.write("bob", make_key(1), &make_value(8192), Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Delete alice's account dir partially to simulate corruption
|
||||
// Then verify bob is intact
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
// Bob should be fine
|
||||
let result = engine.read("bob", &make_key(1)).unwrap();
|
||||
assert!(result.is_some(), "bob should be unaffected");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn find_first_segment(store_root: &Path, account: &str) -> std::path::PathBuf {
|
||||
let seg_dir = store_root.join("accounts").join(account).join("segments");
|
||||
for entry in fs::read_dir(&seg_dir).unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
if name.ends_with(".seg") && !name.contains("temp_") {
|
||||
return entry.path();
|
||||
}
|
||||
}
|
||||
panic!("no segment found in {:?}", seg_dir);
|
||||
}
|
||||
497
crates/blob/tests/integration_test.rs
Normal file
497
crates/blob/tests/integration_test.rs
Normal file
@@ -0,0 +1,497 @@
|
||||
use bichon_blob::{Codec, Config, Engine};
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_create_and_list_accounts() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
|
||||
engine.create_account("alice").unwrap();
|
||||
engine.create_account("bob").unwrap();
|
||||
|
||||
let accounts = engine.list_accounts();
|
||||
assert!(accounts.contains(&"alice".to_string()));
|
||||
assert!(accounts.contains(&"bob".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_write_and_read() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xAA; 32];
|
||||
let value = b"Hello, this is a test email!".to_vec();
|
||||
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_read_missing_key() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xFF; 32];
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xBB; 32];
|
||||
let value = b"Some email content".to_vec();
|
||||
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
engine.delete("alice", &key).unwrap();
|
||||
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_account() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine.delete_account("alice").unwrap();
|
||||
|
||||
let accounts = engine.list_accounts();
|
||||
assert!(!accounts.contains(&"alice".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_small_value_not_compressed() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xCC; 32];
|
||||
let value = b"hi"; // Smaller than 4KB threshold
|
||||
|
||||
engine
|
||||
.write("alice", key, value, Codec::Zstd)
|
||||
.unwrap();
|
||||
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value.to_vec()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_large_value() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xDD; 32];
|
||||
let value = vec![b'X'; 100_000]; // 100KB
|
||||
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_keys() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let n = 100;
|
||||
for i in 0..n {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let value = format!("email number {}", i).into_bytes();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
for i in 0..n {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(format!("email number {}", i).into_bytes()));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gc() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
// Write many entries
|
||||
let value = vec![b'Y'; 5000];
|
||||
let n = 100;
|
||||
|
||||
for i in 0..n {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
engine
|
||||
.write("alice", key, &value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Delete even-numbered keys
|
||||
for i in (0..n).step_by(2) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
engine.delete("alice", &key).unwrap();
|
||||
}
|
||||
|
||||
// Run GC
|
||||
let _result = engine.gc("alice").unwrap();
|
||||
|
||||
// Verify remaining keys still readable
|
||||
for i in (1..n).step_by(2) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value.clone()));
|
||||
}
|
||||
|
||||
// Deleted keys should not exist
|
||||
for i in (0..n).step_by(2) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reopen_persistence() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let key = [0xEE; 32];
|
||||
let value = b"persistent data".to_vec();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Reopen
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_stats() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
engine
|
||||
.write("alice", [1u8; 32], b"hello", Codec::None)
|
||||
.unwrap();
|
||||
|
||||
let stats = engine.stats("alice").unwrap();
|
||||
assert!(stats.total_bytes > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_batch_write() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let n = 50;
|
||||
let entries: Vec<_> = (0..n)
|
||||
.map(|i: u64| {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..8].copy_from_slice(&i.to_le_bytes());
|
||||
let value = format!("batch email {}", i).into_bytes();
|
||||
(key, value, Codec::Zstd)
|
||||
})
|
||||
.collect();
|
||||
|
||||
engine.write_batch("alice", &entries).unwrap();
|
||||
|
||||
for (key, value, _) in &entries {
|
||||
let result = engine.read("alice", key).unwrap();
|
||||
assert_eq!(result.as_ref(), Some(value));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_batch_write_persistence() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let entries: Vec<_> = (0..30u64)
|
||||
.map(|i| {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..8].copy_from_slice(&i.to_le_bytes());
|
||||
(key, format!("persist {}", i).into_bytes(), Codec::Zstd)
|
||||
})
|
||||
.collect();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine.write_batch("alice", &entries).unwrap();
|
||||
}
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for (key, value, _) in &entries {
|
||||
let result = engine.read("alice", key).unwrap();
|
||||
assert_eq!(result.as_ref(), Some(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_config_rejected() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut config = Config::default();
|
||||
config.lru_bucket_count = 0;
|
||||
assert!(Engine::open(dir.path(), config).is_err());
|
||||
|
||||
let mut config = Config::default();
|
||||
config.gc_deleted_ratio = 1.5;
|
||||
assert!(Engine::open(dir.path(), config).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_concurrent_reads() {
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
// Write some data
|
||||
for i in 0..50u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
engine.write("alice", key, &vec![i as u8; 1024], Codec::None).unwrap();
|
||||
}
|
||||
|
||||
// Spawn 4 threads, each reading a different subset
|
||||
let mut handles = vec![];
|
||||
for t in 0..4 {
|
||||
let engine = engine.clone();
|
||||
handles.push(thread::spawn(move || {
|
||||
for i in (t * 12)..((t + 1) * 12) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let read = engine.read("alice", &key).unwrap();
|
||||
assert!(read.is_some(), "key {} should exist", i);
|
||||
}
|
||||
}));
|
||||
}
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_concurrent_writes_different_accounts() {
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
|
||||
|
||||
for name in &["alice", "bob", "carol"] {
|
||||
engine.create_account(name).unwrap();
|
||||
}
|
||||
|
||||
let mut handles = vec![];
|
||||
for (t, name) in ["alice", "bob", "carol"].iter().enumerate() {
|
||||
let engine = engine.clone();
|
||||
let account_name = name.to_string();
|
||||
handles.push(thread::spawn(move || {
|
||||
for i in 0..20 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&((t * 100 + i) as u32).to_le_bytes());
|
||||
let value = vec![(t * 100 + i) as u8; 512];
|
||||
engine.write(&account_name, key, &value, Codec::None).unwrap();
|
||||
}
|
||||
}));
|
||||
}
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
|
||||
// Verify all writes persisted
|
||||
for (t, name) in ["alice", "bob", "carol"].iter().enumerate() {
|
||||
for i in 0..20 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&((t * 100 + i) as u32).to_le_bytes());
|
||||
let read = engine.read(name, &key).unwrap();
|
||||
assert!(read.is_some(), "account {} key {} should exist", name, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crash_recovery() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let dir_path = dir.path().to_path_buf();
|
||||
|
||||
// Phase 1: write data, then drop without shutdown (simulates crash)
|
||||
{
|
||||
let engine = Engine::open(&dir_path, Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
for i in 0..50u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
engine.write("alice", key, &vec![i as u8; 512], Codec::None).unwrap();
|
||||
}
|
||||
// Engine dropped here without calling shutdown()
|
||||
}
|
||||
|
||||
// Phase 2: reopen — recovery should run, data should be intact
|
||||
let engine = Engine::open(&dir_path, Config::default()).unwrap();
|
||||
let stats = engine.stats("alice").unwrap();
|
||||
assert!(stats.total_keys > 0, "recovery should preserve data");
|
||||
|
||||
// Verify reads work
|
||||
for i in 0..50u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
let read = engine.read("alice", &key).unwrap();
|
||||
assert!(read.is_some(), "key {} should survive crash recovery", i);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_meta_bin_durability() {
|
||||
// Verify meta.bin has valid CRC and can be read after a write cycle.
|
||||
let dir = TempDir::new().unwrap();
|
||||
let dir_path = dir.path().to_path_buf();
|
||||
|
||||
{
|
||||
let engine = Engine::open(&dir_path, Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0x42u8; 32];
|
||||
engine.write("alice", key, b"durable", Codec::None).unwrap();
|
||||
}
|
||||
// Engine dropped → shutdown() called → meta saved via write_bin (with fsync)
|
||||
|
||||
// Verify meta.bin exists and has valid CRC
|
||||
let meta_path = dir_path
|
||||
.join("accounts")
|
||||
.join("alice")
|
||||
.join("meta.bin");
|
||||
assert!(meta_path.exists(), "meta.bin should exist after clean shutdown");
|
||||
|
||||
let data = std::fs::read(&meta_path).unwrap();
|
||||
assert!(data.len() >= 8, "meta.bin should have at least 8 bytes (crc + version)");
|
||||
|
||||
let stored_crc = u32::from_le_bytes(data[0..4].try_into().unwrap());
|
||||
assert_ne!(stored_crc, 0, "stored CRC should be non-zero");
|
||||
|
||||
// Reopen and verify data is intact
|
||||
let engine = Engine::open(&dir_path, Config::default()).unwrap();
|
||||
let read = engine.read("alice", &[0x42u8; 32]).unwrap();
|
||||
assert_eq!(read, Some(b"durable".to_vec()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gc_concurrent_with_writes() {
|
||||
// GC should not lose entries that are written concurrently.
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::thread;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
// Pre-fill: write enough to trigger eventual GC
|
||||
let big_value = vec![b'X'; 8192];
|
||||
for i in 0..500u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
engine.write("alice", key, &big_value, Codec::None).unwrap();
|
||||
}
|
||||
|
||||
// Delete some to create GC candidates
|
||||
for i in 0..250u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
engine.delete("alice", &key).unwrap();
|
||||
}
|
||||
|
||||
let running = Arc::new(AtomicBool::new(true));
|
||||
let engine_gc = engine.clone();
|
||||
let running_gc = running.clone();
|
||||
|
||||
// Thread 1: run GC in a loop
|
||||
let gc_handle = thread::spawn(move || {
|
||||
while running_gc.load(Ordering::Relaxed) {
|
||||
let _ = engine_gc.gc("alice");
|
||||
thread::sleep(std::time::Duration::from_millis(10));
|
||||
}
|
||||
});
|
||||
|
||||
// Thread 2: keep writing new entries
|
||||
let engine_write = engine.clone();
|
||||
let running_write = running.clone();
|
||||
let write_handle = thread::spawn(move || {
|
||||
let mut counter = 10000u32;
|
||||
while running_write.load(Ordering::Relaxed) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&counter.to_le_bytes());
|
||||
engine_write
|
||||
.write("alice", key, &vec![counter as u8; 256], Codec::None)
|
||||
.unwrap();
|
||||
counter += 1;
|
||||
}
|
||||
counter
|
||||
});
|
||||
|
||||
// Let them race for a bit
|
||||
thread::sleep(std::time::Duration::from_millis(500));
|
||||
running.store(false, Ordering::Relaxed);
|
||||
|
||||
gc_handle.join().unwrap();
|
||||
let final_counter = write_handle.join().unwrap();
|
||||
|
||||
// All written entries must be readable
|
||||
let mut missing = 0;
|
||||
for i in 0..500u32 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
if engine.read("alice", &key).unwrap().is_none() {
|
||||
// Entries 0..250 were deleted, they should be gone
|
||||
if i >= 250 {
|
||||
missing += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert_eq!(missing, 0, "pre-existing entries should survive concurrent GC");
|
||||
|
||||
// Entries written during the race should be readable
|
||||
for i in 10000..final_counter {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
let read = engine.read("alice", &key).unwrap();
|
||||
assert!(read.is_some(), "concurrently written key {} should exist after GC", i);
|
||||
}
|
||||
}
|
||||
@@ -16,12 +16,8 @@ reqwest.workspace = true
|
||||
toml = "0.9.8"
|
||||
memmap2 = "0.9.10"
|
||||
outlook-pst = { git = "https://github.com/rustmailer/outlook-pst-rs.git", branch = "main" }
|
||||
compressed-rtf = "1.0.1"
|
||||
chrono.workspace = true
|
||||
mail-send.workspace = true
|
||||
base64.workspace = true
|
||||
codepage-strings = "1.0.2"
|
||||
hex = "0.4.3"
|
||||
sysinfo.workspace = true
|
||||
indicatif.workspace = true
|
||||
serde_json.workspace = true
|
||||
@@ -21,7 +21,7 @@ use std::path::PathBuf;
|
||||
|
||||
use crate::api::sender::send_batch_request;
|
||||
use crate::mbox::gmail::determine_folder;
|
||||
use crate::mbox::reader::MboxFile;
|
||||
use bichon_core::import::reader::MboxFile;
|
||||
use crate::BichonCliConfig;
|
||||
use bichon_core::base64_encode_url_safe;
|
||||
use bichon_core::envelope::meta::{parse_bichon_metadata, BichonMetadata};
|
||||
@@ -37,7 +37,6 @@ const MAX_EMAIL_BYTES: usize = 100 * 1024 * 1024;
|
||||
const MAX_BUFFER_BYTES: usize = 200 * 1024 * 1024;
|
||||
|
||||
pub mod gmail;
|
||||
pub mod reader;
|
||||
|
||||
pub async fn handle_mbox_single_file_import(
|
||||
config: &BichonCliConfig,
|
||||
|
||||
@@ -16,21 +16,12 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use chrono::{DateTime, TimeZone, Utc};
|
||||
use dialoguer::theme::ColorfulTheme;
|
||||
use dialoguer::Input;
|
||||
use mail_send::mail_builder::headers::text::Text;
|
||||
use mail_send::mail_builder::MessageBuilder;
|
||||
use outlook_pst::ltp::prop_context::PropertyValue;
|
||||
|
||||
use crate::api::sender::send_batch_request;
|
||||
use crate::pst::encoding::decode_subject;
|
||||
use crate::BichonCliConfig;
|
||||
use bichon_core::base64_encode_url_safe;
|
||||
use dialoguer::Confirm;
|
||||
use outlook_pst::messaging::attachment::AttachmentProperties;
|
||||
use bichon_core::import::pst::build_eml_base64;
|
||||
use dialoguer::theme::ColorfulTheme;
|
||||
use dialoguer::{Confirm, Input};
|
||||
use outlook_pst::messaging::folder::Folder;
|
||||
use outlook_pst::messaging::message::{Message, MessageProperties};
|
||||
use outlook_pst::ndb::node_id::NodeId;
|
||||
use reqwest::Client;
|
||||
use std::future::Future;
|
||||
@@ -38,28 +29,6 @@ use std::path::PathBuf;
|
||||
use std::pin::Pin;
|
||||
use std::rc::Rc;
|
||||
|
||||
mod encoding;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct EmailMetadata {
|
||||
pub message_id: Option<String>,
|
||||
pub subject: Option<String>,
|
||||
pub from: Option<String>,
|
||||
pub to: Option<Vec<String>>,
|
||||
pub cc: Option<Vec<String>>,
|
||||
pub bcc: Option<Vec<String>>,
|
||||
pub html: Option<String>,
|
||||
pub text: Option<String>,
|
||||
pub in_reply_to: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct EmailAttachment {
|
||||
pub name: Option<String>,
|
||||
pub mime_type: Option<String>,
|
||||
pub data: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
pub async fn handle_pst_import(config: &BichonCliConfig, account_id: u64, theme: &ColorfulTheme) {
|
||||
let path_str: String = Input::with_theme(theme)
|
||||
.with_prompt("Enter the path to your SINGLE .pst file")
|
||||
@@ -244,167 +213,6 @@ fn process_folder_recursively<'a>(
|
||||
})
|
||||
}
|
||||
|
||||
fn build_eml_base64(message: Rc<dyn Message>) -> Option<String> {
|
||||
let properties = message.properties();
|
||||
|
||||
let mut builder = MessageBuilder::new();
|
||||
if let Some(sub) = extract_subject(properties) {
|
||||
builder = builder.subject(sub);
|
||||
}
|
||||
if let Some(mid) = extract_string_property(properties, 0x1035) {
|
||||
builder = builder.message_id(mid);
|
||||
}
|
||||
if let Some(irt) = extract_string_property(properties, 0x1042) {
|
||||
builder = builder.in_reply_to(irt);
|
||||
}
|
||||
|
||||
if let Some(refs) = extract_string_property(properties, 0x1039) {
|
||||
builder = builder.header("References", Text::new(refs));
|
||||
}
|
||||
|
||||
if let Some(cid_val) = properties.get(0x3013) {
|
||||
if let PropertyValue::Binary(bin) = cid_val {
|
||||
builder = builder.header(
|
||||
"X-Bichon-Conversation-ID",
|
||||
Text::new(hex::encode(bin.buffer())),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let from = extract_string_property(properties, 0x5D01)
|
||||
.or_else(|| extract_string_property(properties, 0x5D02))
|
||||
.or_else(|| extract_string_property(properties, 0x0C1F));
|
||||
|
||||
if let Some(f) = from {
|
||||
builder = builder.from(f);
|
||||
}
|
||||
|
||||
if let Some(filetime) = extract_i64_property(properties, &[0x0039, 0x0E06]) {
|
||||
let dt = filetime_to_datetime(filetime).timestamp();
|
||||
builder = builder.date(dt);
|
||||
}
|
||||
|
||||
let (to, cc, bcc) = extract_recipients_list(&message);
|
||||
if !to.is_empty() {
|
||||
builder = builder.to(to.iter().map(|s| s.as_str()).collect::<Vec<_>>());
|
||||
}
|
||||
if !cc.is_empty() {
|
||||
builder = builder.cc(cc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
|
||||
}
|
||||
if !bcc.is_empty() {
|
||||
builder = builder.bcc(bcc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
if let Some(html) = extract_html(properties) {
|
||||
builder = builder.html_body(html);
|
||||
}
|
||||
|
||||
if let Some(text) = extract_text(properties) {
|
||||
builder = builder.text_body(text);
|
||||
}
|
||||
|
||||
if let Some(attachment_table) = message.attachment_table() {
|
||||
for row in attachment_table.rows_matrix() {
|
||||
let node_id = NodeId::from(u32::from(row.id()));
|
||||
if let Ok(attachment) = message.clone().read_attachment(node_id, None) {
|
||||
let att_props = attachment.properties();
|
||||
let name = extract_attachment_string_property(att_props, 0x3707);
|
||||
let mime = extract_attachment_string_property(att_props, 0x370E)
|
||||
.unwrap_or_else(|| "application/octet-stream".into());
|
||||
let cid = extract_attachment_string_property(att_props, 0x3712);
|
||||
let is_inline = att_props
|
||||
.get(0x3714)
|
||||
.and_then(|val| {
|
||||
if let PropertyValue::Integer32(f) = val {
|
||||
Some(f)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.map(|flag| (flag & 0x4) != 0)
|
||||
.unwrap_or(false);
|
||||
|
||||
if let Some(PropertyValue::Binary(bin)) = att_props.get(0x3701) {
|
||||
let data = bin.buffer().to_vec();
|
||||
let file_name = name.unwrap_or_else(|| "unnamed_attachment".to_string());
|
||||
|
||||
if is_inline && cid.is_some() {
|
||||
let content_id = cid.unwrap();
|
||||
builder = builder.inline(mime, content_id, data);
|
||||
} else {
|
||||
builder = builder.attachment(mime, file_name, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match builder.write_to_vec() {
|
||||
Ok(eml_vec) => Some(base64_encode_url_safe!(eml_vec)),
|
||||
Err(e) => {
|
||||
eprintln!("Failed to generate EML: {:?}", e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn filetime_to_datetime(filetime: i64) -> DateTime<Utc> {
|
||||
let unix_secs = (filetime / 10_000_000) - 11_644_473_600;
|
||||
let nsecs = (filetime % 10_000_000) * 100;
|
||||
Utc.timestamp_opt(unix_secs, nsecs as u32).unwrap()
|
||||
}
|
||||
|
||||
fn extract_recipients_list(message: &Rc<dyn Message>) -> (Vec<String>, Vec<String>, Vec<String>) {
|
||||
let mut to = Vec::new();
|
||||
let mut cc = Vec::new();
|
||||
let mut bcc = Vec::new();
|
||||
|
||||
let recipient_table = message.recipient_table();
|
||||
if let Some(recipient_table) = recipient_table {
|
||||
let context = recipient_table.context();
|
||||
for row in recipient_table.rows_matrix() {
|
||||
if let Ok(cols) = row.columns(context) {
|
||||
let mut r_type = 0;
|
||||
let mut email = String::new();
|
||||
|
||||
for (col, val) in context.columns().iter().zip(cols) {
|
||||
let prop_val = val
|
||||
.as_ref()
|
||||
.and_then(|v| recipient_table.read_column(v, col.prop_type()).ok());
|
||||
match col.prop_id() {
|
||||
0x0C15 => {
|
||||
if let Some(PropertyValue::Integer32(t)) = prop_val {
|
||||
r_type = t;
|
||||
}
|
||||
}
|
||||
0x39FE | 0x3003 => {
|
||||
if let Some(s) = prop_val.and_then(|v| extract_string(&v)) {
|
||||
email = s;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if !email.is_empty() {
|
||||
match r_type {
|
||||
1 => to.push(email),
|
||||
2 => cc.push(email),
|
||||
3 => bcc.push(email),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let receiver = extract_string_property(message.properties(), 0x0076);
|
||||
if let Some(receiver) = receiver {
|
||||
to.push(receiver);
|
||||
}
|
||||
}
|
||||
(to, cc, bcc)
|
||||
}
|
||||
|
||||
async fn send_to_bichon(
|
||||
client: &Client,
|
||||
config: &BichonCliConfig,
|
||||
@@ -414,69 +222,3 @@ async fn send_to_bichon(
|
||||
) {
|
||||
send_batch_request(client, config, account_id, folder_path, emls).await;
|
||||
}
|
||||
|
||||
fn extract_subject(props: &MessageProperties) -> Option<String> {
|
||||
props.get(0x0037).and_then(|val| decode_subject(val))
|
||||
}
|
||||
|
||||
fn extract_string_property(properties: &MessageProperties, prop_id: u16) -> Option<String> {
|
||||
properties
|
||||
.get(prop_id)
|
||||
.and_then(|value| extract_string(value))
|
||||
}
|
||||
|
||||
fn extract_attachment_string_property(
|
||||
properties: &AttachmentProperties,
|
||||
prop_id: u16,
|
||||
) -> Option<String> {
|
||||
properties
|
||||
.get(prop_id)
|
||||
.and_then(|value| extract_string(value))
|
||||
}
|
||||
|
||||
fn extract_string(value: &PropertyValue) -> Option<String> {
|
||||
match value {
|
||||
PropertyValue::String8(value) => Some(value.to_string()),
|
||||
PropertyValue::Unicode(value) => Some(value.to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_text(properties: &MessageProperties) -> Option<String> {
|
||||
properties.get(0x1000).and_then(extract_string).or_else(|| {
|
||||
properties.get(0x1009).and_then(|value| match value {
|
||||
PropertyValue::Binary(value) => encoding::decode_rtf_compressed(value.buffer()),
|
||||
_ => None,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_html(properties: &MessageProperties) -> Option<String> {
|
||||
properties.get(0x1013).and_then(|value| match value {
|
||||
PropertyValue::Binary(value) => {
|
||||
let code_page = properties
|
||||
.get(0x3FDE)
|
||||
.and_then(|v| {
|
||||
if let PropertyValue::Integer32(cpid) = v {
|
||||
Some(*cpid as u16)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or(65001);
|
||||
encoding::decode_html_body(value.buffer(), code_page)
|
||||
}
|
||||
PropertyValue::String8(value) => Some(value.to_string()),
|
||||
PropertyValue::Unicode(value) => Some(value.to_string()),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_i64_property(properties: &MessageProperties, prop_ids: &[u16]) -> Option<i64> {
|
||||
for &prop_id in prop_ids {
|
||||
if let Some(PropertyValue::Time(value)) = properties.get(prop_id) {
|
||||
return Some(*value);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ poem-openapi = { version = "5.1.16", features = [
|
||||
], optional = true }
|
||||
chrono.workspace = true
|
||||
clap.workspace = true
|
||||
memdb.workspace = true
|
||||
bichon-memdb.workspace = true
|
||||
itertools.workspace = true
|
||||
ring.workspace = true
|
||||
serde.workspace = true
|
||||
@@ -69,6 +69,11 @@ tokio-util.workspace = true
|
||||
whichlang = "0.1.1"
|
||||
deunicode = "1.6.2"
|
||||
scopeguard = "1.2.0"
|
||||
cron = "0.15"
|
||||
quick-xml = { version = "0.40.0", features = ["serialize"] }
|
||||
hickory-resolver = "0.26.0-alpha.1"
|
||||
cron = "0.17"
|
||||
quick-xml = { version = "0.41.0", features = ["serialize"] }
|
||||
hickory-resolver = "0.26.1"
|
||||
memmap2 = "0.9.10"
|
||||
outlook-pst = { git = "https://github.com/rustmailer/outlook-pst-rs.git", branch = "main" }
|
||||
compressed-rtf = "1.0.1"
|
||||
codepage-strings = "1.0.2"
|
||||
hex.workspace = true
|
||||
|
||||
@@ -64,6 +64,216 @@ pub enum QuotaWindow {
|
||||
Monthly,
|
||||
}
|
||||
|
||||
/// Include/exclude filter rule.
|
||||
///
|
||||
/// - `include` non-empty: only values matching these patterns pass.
|
||||
/// - `exclude` non-empty: values matching these patterns are rejected.
|
||||
/// - Both empty: all values pass.
|
||||
/// - Both set: include checked first, then exclude.
|
||||
///
|
||||
/// Extension patterns use case-insensitive exact match; all others use regex.
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct FilterRule {
|
||||
#[serde(default)]
|
||||
pub include: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub exclude: Vec<String>,
|
||||
}
|
||||
|
||||
impl FilterRule {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.include.is_empty() && self.exclude.is_empty()
|
||||
}
|
||||
|
||||
fn matches_exact(&self, value: &str) -> bool {
|
||||
if !self.include.is_empty() && !self.include.iter().any(|e| e.eq_ignore_ascii_case(value)) {
|
||||
return false;
|
||||
}
|
||||
if !self.exclude.is_empty() && self.exclude.iter().any(|e| e.eq_ignore_ascii_case(value)) {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn matches_regex(&self, value: &str) -> bool {
|
||||
if !self.include.is_empty() && !matches_any_regex(&self.include, value) {
|
||||
return false;
|
||||
}
|
||||
if !self.exclude.is_empty() && matches_any_regex(&self.exclude, value) {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn validate_regex(&self, field: &str) -> Result<(), String> {
|
||||
validate_patterns(&self.include, &format!("{field}.include"))?;
|
||||
validate_patterns(&self.exclude, &format!("{field}.exclude"))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct ExtractionRules {
|
||||
/// Type 0: Master switch.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Type 1: File extensions (exact match, e.g. `{"include": ["pdf","docx"]}`).
|
||||
#[serde(default)]
|
||||
pub extensions: FilterRule,
|
||||
/// Type 2: Folder patterns (regex, e.g. `{"include": ["^INBOX/Invoices"]}`).
|
||||
#[serde(default)]
|
||||
pub folders: FilterRule,
|
||||
/// Type 3: Attachment filename patterns (regex).
|
||||
#[serde(default)]
|
||||
pub attachment_names: FilterRule,
|
||||
/// Type 4: Sender patterns (regex).
|
||||
#[serde(default)]
|
||||
pub senders: FilterRule,
|
||||
}
|
||||
|
||||
impl ExtractionRules {
|
||||
/// Returns `true` if the attachment should be extracted under these rules.
|
||||
pub fn should_extract(
|
||||
&self,
|
||||
ext: &str,
|
||||
folder: Option<&str>,
|
||||
attachment_name: Option<&str>,
|
||||
sender: Option<&str>,
|
||||
) -> bool {
|
||||
if !self.enabled {
|
||||
return false;
|
||||
}
|
||||
if !self.extensions.matches_exact(ext) {
|
||||
return false;
|
||||
}
|
||||
if !self.folders.is_empty() {
|
||||
if let Some(folder) = folder {
|
||||
if !self.folders.matches_regex(folder) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !self.attachment_names.is_empty() {
|
||||
if let Some(name) = attachment_name {
|
||||
if !self.attachment_names.matches_regex(name) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !self.senders.is_empty() {
|
||||
if let Some(sender) = sender {
|
||||
if !self.senders.matches_regex(sender) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
self.folders.validate_regex("folders")?;
|
||||
self.attachment_names.validate_regex("attachment_names")?;
|
||||
self.senders.validate_regex("senders")?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Archive filtering rules — skip unwanted emails before storage.
|
||||
///
|
||||
/// Rule types:
|
||||
/// 0 — Master switch
|
||||
/// 1 — Sender filter (regex)
|
||||
/// 2 — Subject filter (regex)
|
||||
/// 3 — Skip emails larger than this (bytes)
|
||||
/// 4 — Skip emails with spam headers (X-Spam-Flag, X-Spam)
|
||||
///
|
||||
/// `None` = archive everything (backward compatible).
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct ArchiveRules {
|
||||
/// Type 0: Master switch. `false` = archive everything.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Type 1: Sender filter (regex, include/exclude).
|
||||
#[serde(default)]
|
||||
pub senders: FilterRule,
|
||||
/// Type 2: Subject filter (regex, include/exclude).
|
||||
#[serde(default)]
|
||||
pub subjects: FilterRule,
|
||||
/// Type 3: Skip emails larger than this (bytes). `None` = no size limit.
|
||||
#[serde(default)]
|
||||
pub skip_larger_than: Option<u64>,
|
||||
/// Type 4: Spam header names to check (e.g. `["X-Spam-Flag", "X-Spam"]`).
|
||||
/// When the value is `yes` or `true` (case-insensitive), the email is skipped.
|
||||
/// Empty = don't check. Common headers: `X-Spam-Flag` (SpamAssassin),
|
||||
/// `X-Spam` (rspamd), `X-MS-Exchange-Organization-SCL` (Exchange).
|
||||
#[serde(default)]
|
||||
pub spam_headers: Vec<String>,
|
||||
}
|
||||
|
||||
impl ArchiveRules {
|
||||
/// Returns `true` if the email should be archived under these rules.
|
||||
pub fn should_archive(
|
||||
&self,
|
||||
sender: Option<&str>,
|
||||
subject: Option<&str>,
|
||||
size: u32,
|
||||
is_spam: bool,
|
||||
) -> bool {
|
||||
if !self.enabled {
|
||||
return true;
|
||||
}
|
||||
if !self.senders.is_empty() {
|
||||
if let Some(sender) = sender {
|
||||
if !self.senders.matches_regex(sender) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !self.subjects.is_empty() {
|
||||
if let Some(subject) = subject {
|
||||
if !self.subjects.matches_regex(subject) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(limit) = self.skip_larger_than {
|
||||
if size as u64 > limit {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if !self.spam_headers.is_empty() && is_spam {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Validate all regex patterns are well-formed.
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
self.senders.validate_regex("senders")?;
|
||||
self.subjects.validate_regex("subjects")?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn matches_any_regex(patterns: &[String], value: &str) -> bool {
|
||||
patterns.iter().any(|p| {
|
||||
regex::Regex::new(p)
|
||||
.map(|re| re.is_match(value))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_patterns(patterns: &[String], field_name: &str) -> Result<(), String> {
|
||||
for p in patterns {
|
||||
regex::Regex::new(p)
|
||||
.map_err(|e| format!("{} pattern '{}' is invalid regex: {}", field_name, p, e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct Account {
|
||||
@@ -99,6 +309,14 @@ pub struct Account {
|
||||
pub download_schedule: Option<String>,
|
||||
#[serde(default)]
|
||||
pub deleting: bool,
|
||||
/// Email-level filtering rules (Pro feature).
|
||||
/// `None` = archive everything (backward compatible).
|
||||
#[serde(default)]
|
||||
pub archive_rules: Option<ArchiveRules>,
|
||||
/// Attachment text extraction rules (Pro feature).
|
||||
/// `None` = extract everything (backward compatible).
|
||||
#[serde(default)]
|
||||
pub extraction_rules: Option<ExtractionRules>,
|
||||
}
|
||||
|
||||
impl MemDbModel for Account {
|
||||
@@ -139,6 +357,8 @@ impl Account {
|
||||
imap_quota_window: request.imap_quota_window,
|
||||
download_schedule: request.download_schedule,
|
||||
deleting: false,
|
||||
archive_rules: request.archive_rules,
|
||||
extraction_rules: request.extraction_rules,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -362,6 +582,7 @@ impl Account {
|
||||
.map(|account: AccountModel| MinimalAccount {
|
||||
id: account.id,
|
||||
email: account.email,
|
||||
name: account.account_name,
|
||||
})
|
||||
.collect::<Vec<MinimalAccount>>();
|
||||
Ok(result)
|
||||
@@ -477,7 +698,299 @@ impl Account {
|
||||
if request.clear_download_schedule == Some(true) {
|
||||
new.download_schedule = None;
|
||||
}
|
||||
if request.extraction_rules.is_some() {
|
||||
new.extraction_rules = request.extraction_rules;
|
||||
}
|
||||
if request.archive_rules.is_some() {
|
||||
new.archive_rules = request.archive_rules;
|
||||
}
|
||||
new.updated_at = utc_now!();
|
||||
Ok(new)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// ── FilterRule ───────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn filter_rule_include_only() {
|
||||
let r = FilterRule {
|
||||
include: vec![r"@ok\.com$".into()],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(r.matches_regex("bob@ok.com"));
|
||||
assert!(!r.matches_regex("spam@bad.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_rule_exclude_only() {
|
||||
let r = FilterRule {
|
||||
exclude: vec![r"@spam\.com$".into()],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(r.matches_regex("bob@ok.com"));
|
||||
assert!(!r.matches_regex("bot@spam.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_rule_include_then_exclude() {
|
||||
let r = FilterRule {
|
||||
include: vec![r"@company\.com$".into()],
|
||||
exclude: vec![r"noreply@company\.com$".into()],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(r.matches_regex("bob@company.com"));
|
||||
assert!(!r.matches_regex("noreply@company.com"));
|
||||
assert!(!r.matches_regex("spam@other.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_rule_exact_match() {
|
||||
let r = FilterRule {
|
||||
include: vec!["pdf".into(), "docx".into()],
|
||||
exclude: vec!["xlsx".into()],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(r.matches_exact("pdf"));
|
||||
assert!(r.matches_exact("docx"));
|
||||
assert!(r.matches_exact("DOCX")); // case-insensitive
|
||||
assert!(!r.matches_exact("xlsx"));
|
||||
assert!(!r.matches_exact("txt"));
|
||||
}
|
||||
|
||||
// ── ExtractionRules ─────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn extraction_rules_master_switch() {
|
||||
let rules = ExtractionRules {
|
||||
enabled: false,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!rules.should_extract("pdf", None, None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_rules_extension_include() {
|
||||
let rules = ExtractionRules {
|
||||
enabled: true,
|
||||
extensions: FilterRule {
|
||||
include: vec!["pdf".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.should_extract("pdf", None, None, None));
|
||||
assert!(!rules.should_extract("docx", None, None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_rules_extension_exclude() {
|
||||
let rules = ExtractionRules {
|
||||
enabled: true,
|
||||
extensions: FilterRule {
|
||||
exclude: vec!["xlsx".into(), "pptx".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.should_extract("pdf", None, None, None));
|
||||
assert!(!rules.should_extract("xlsx", None, None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_rules_folder_regex() {
|
||||
let rules = ExtractionRules {
|
||||
enabled: true,
|
||||
folders: FilterRule {
|
||||
include: vec![r"^INBOX/Invoices".into(), r"Contracts$".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.should_extract("pdf", Some("INBOX/Invoices"), None, None));
|
||||
assert!(rules.should_extract("pdf", Some("Finance/Contracts"), None, None));
|
||||
assert!(!rules.should_extract("pdf", Some("INBOX/Junk"), None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_rules_attachment_name_regex() {
|
||||
let rules = ExtractionRules {
|
||||
enabled: true,
|
||||
attachment_names: FilterRule {
|
||||
include: vec![r"^invoice-.*\.pdf$".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.should_extract("pdf", None, Some("invoice-2024.pdf"), None));
|
||||
assert!(!rules.should_extract("pdf", None, Some("newsletter.pdf"), None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_rules_sender_regex() {
|
||||
let rules = ExtractionRules {
|
||||
enabled: true,
|
||||
senders: FilterRule {
|
||||
exclude: vec![r"@noreply\.com$".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
// non-excluded sender passes
|
||||
assert!(rules.should_extract("pdf", None, None, Some("bob@ok.com")));
|
||||
// excluded sender blocked
|
||||
assert!(!rules.should_extract("pdf", None, None, Some("bot@noreply.com")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_rules_empty_filters_pass_everything() {
|
||||
let rules = ExtractionRules {
|
||||
enabled: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.should_extract(
|
||||
"anything",
|
||||
Some("any/folder"),
|
||||
Some("any.pdf"),
|
||||
Some("any@x.com")
|
||||
));
|
||||
}
|
||||
|
||||
// ── ArchiveRules ────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn archive_rules_disabled_archives_everything() {
|
||||
let rules = ArchiveRules {
|
||||
enabled: false,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.should_archive(Some("spam@x.com"), Some("BUY NOW"), 999, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_rules_sender_exclude() {
|
||||
let rules = ArchiveRules {
|
||||
enabled: true,
|
||||
senders: FilterRule {
|
||||
exclude: vec![r"@spam\.com$".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!rules.should_archive(Some("bot@spam.com"), None, 100, false));
|
||||
assert!(rules.should_archive(Some("friend@ok.com"), None, 100, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_rules_subject_exclude() {
|
||||
let rules = ArchiveRules {
|
||||
enabled: true,
|
||||
subjects: FilterRule {
|
||||
exclude: vec![r"(?i)unsubscribe|buy now|limited offer".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!rules.should_archive(None, Some("UNSUBSCRIBE NOW"), 100, false));
|
||||
assert!(!rules.should_archive(None, Some("Limited Offer!!"), 100, false));
|
||||
assert!(rules.should_archive(None, Some("Meeting tomorrow"), 100, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_rules_sender_include() {
|
||||
// Only archive emails from specific senders
|
||||
let rules = ArchiveRules {
|
||||
enabled: true,
|
||||
senders: FilterRule {
|
||||
include: vec![r"@partner\.com$".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.should_archive(Some("bob@partner.com"), None, 100, false));
|
||||
assert!(!rules.should_archive(Some("spam@random.com"), None, 100, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_rules_skip_larger_than() {
|
||||
let rules = ArchiveRules {
|
||||
enabled: true,
|
||||
skip_larger_than: Some(50_000_000),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.should_archive(None, None, 1_000_000, false));
|
||||
assert!(!rules.should_archive(None, None, 60_000_000, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_rules_skip_spam_headers() {
|
||||
let rules = ArchiveRules {
|
||||
enabled: true,
|
||||
spam_headers: vec!["X-Spam-Flag".into()],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!rules.should_archive(None, None, 100, true));
|
||||
assert!(rules.should_archive(None, None, 100, false));
|
||||
}
|
||||
|
||||
// ── Validation ──────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn validate_extraction_rules_valid() {
|
||||
let rules = ExtractionRules {
|
||||
folders: FilterRule {
|
||||
include: vec![r"^INBOX/.*".into()],
|
||||
..Default::default()
|
||||
},
|
||||
senders: FilterRule {
|
||||
exclude: vec![r"@spam\.com$".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_extraction_rules_invalid_regex() {
|
||||
let rules = ExtractionRules {
|
||||
folders: FilterRule {
|
||||
include: vec!["***bad[".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_archive_rules_valid() {
|
||||
let rules = ArchiveRules {
|
||||
senders: FilterRule {
|
||||
exclude: vec![r"@spam\.com$".into()],
|
||||
..Default::default()
|
||||
},
|
||||
subjects: FilterRule {
|
||||
include: vec![r"(?i)invoice".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_archive_rules_invalid_regex() {
|
||||
let rules = ArchiveRules {
|
||||
senders: FilterRule {
|
||||
include: vec!["[unclosed".into()],
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(rules.validate().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,9 @@
|
||||
use std::str::FromStr;
|
||||
|
||||
use crate::account::entity::ImapConfig;
|
||||
use crate::account::migration::{AccountModel, AccountType, QuotaWindow};
|
||||
use crate::account::migration::{
|
||||
AccountModel, AccountType, ArchiveRules, ExtractionRules, QuotaWindow,
|
||||
};
|
||||
use crate::account::since::{DateSince, RelativeDate};
|
||||
use crate::error::code::ErrorCode;
|
||||
use crate::error::BichonResult;
|
||||
@@ -56,6 +58,12 @@ pub struct AccountCreateRequest {
|
||||
pub imap_quota_window: Option<QuotaWindow>,
|
||||
pub auto_download_new_mailboxes: Option<bool>,
|
||||
pub download_schedule: Option<String>,
|
||||
/// Email archive filtering rules (Pro feature).
|
||||
/// `None` = archive everything (backward compatible).
|
||||
pub archive_rules: Option<ArchiveRules>,
|
||||
/// Attachment text extraction rules (Pro feature).
|
||||
/// `None` = extract everything (backward compatible).
|
||||
pub extraction_rules: Option<ExtractionRules>,
|
||||
}
|
||||
|
||||
impl AccountCreateRequest {
|
||||
@@ -106,6 +114,19 @@ impl AccountCreateRequest {
|
||||
}
|
||||
AccountType::NoSync => {}
|
||||
}
|
||||
if let Some(ref rules) = self.extraction_rules {
|
||||
rules.validate().map_err(|e| {
|
||||
raise_error!(
|
||||
format!("extraction_rules: {}", e),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
}
|
||||
if let Some(ref rules) = self.archive_rules {
|
||||
rules.validate().map_err(|e| {
|
||||
raise_error!(format!("archive_rules: {}", e), ErrorCode::InvalidParameter)
|
||||
})?;
|
||||
}
|
||||
Ok(AccountModel::new(user_id, self)?)
|
||||
}
|
||||
|
||||
@@ -180,6 +201,12 @@ pub struct AccountUpdateRequest {
|
||||
pub auto_download_new_mailboxes: Option<bool>,
|
||||
pub download_schedule: Option<String>,
|
||||
pub clear_download_schedule: Option<bool>,
|
||||
/// Email archive filtering rules (Pro feature).
|
||||
/// `None` = no change. Use `Some(ArchiveRules { .. })` to set.
|
||||
pub archive_rules: Option<ArchiveRules>,
|
||||
/// Attachment text extraction rules (Pro feature).
|
||||
/// `None` = no change. Use `Some(ExtractionRules { .. })` to set.
|
||||
pub extraction_rules: Option<ExtractionRules>,
|
||||
}
|
||||
|
||||
impl AccountUpdateRequest {
|
||||
@@ -235,6 +262,19 @@ impl AccountUpdateRequest {
|
||||
validate_cron_expression(schedule)?;
|
||||
}
|
||||
}
|
||||
if let Some(ref rules) = self.extraction_rules {
|
||||
rules.validate().map_err(|e| {
|
||||
raise_error!(
|
||||
format!("extraction_rules: {}", e),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
}
|
||||
if let Some(ref rules) = self.archive_rules {
|
||||
rules.validate().map_err(|e| {
|
||||
raise_error!(format!("archive_rules: {}", e), ErrorCode::InvalidParameter)
|
||||
})?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -261,6 +301,7 @@ fn validate_cron_expression(expr: &str) -> BichonResult<()> {
|
||||
pub struct MinimalAccount {
|
||||
pub id: u64,
|
||||
pub email: String,
|
||||
pub name: Option<String>,
|
||||
}
|
||||
|
||||
pub fn filter_accessible_accounts<'a>(
|
||||
|
||||
@@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize};
|
||||
use crate::{
|
||||
account::{
|
||||
entity::ImapConfig,
|
||||
migration::{AccountModel, AccountType, QuotaWindow},
|
||||
migration::{AccountModel, AccountType, ArchiveRules, QuotaWindow},
|
||||
since::{DateSince, RelativeDate},
|
||||
},
|
||||
users::UserModel,
|
||||
@@ -58,6 +58,7 @@ pub struct AccountResp {
|
||||
pub imap_quota_window: Option<QuotaWindow>,
|
||||
pub auto_download_new_mailboxes: Option<bool>,
|
||||
pub download_schedule: Option<String>,
|
||||
pub archive_rules: Option<ArchiveRules>,
|
||||
pub deleting: bool,
|
||||
}
|
||||
|
||||
@@ -96,6 +97,7 @@ impl AccountResp {
|
||||
imap_quota_window: account.imap_quota_window,
|
||||
auto_download_new_mailboxes: account.auto_download_new_mailboxes,
|
||||
download_schedule: account.download_schedule,
|
||||
archive_rules: account.archive_rules,
|
||||
deleting: account.deleting,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use memdb::{Durability, MemDb};
|
||||
use bichon_memdb::{Durability, MemDb};
|
||||
|
||||
use crate::{
|
||||
database::MemDbModel,
|
||||
|
||||
@@ -16,9 +16,8 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use hickory_resolver::name_server::TokioConnectionProvider;
|
||||
use hickory_resolver::net::runtime::TokioRuntimeProvider;
|
||||
use hickory_resolver::proto::rr::RData;
|
||||
use hickory_resolver::proto::rr::RecordType;
|
||||
use hickory_resolver::TokioResolver;
|
||||
use quick_xml::de::from_str;
|
||||
use reqwest::Client;
|
||||
@@ -108,28 +107,29 @@ async fn fetch_xml(client: &Client, url: &str) -> Option<MailConfig> {
|
||||
}
|
||||
|
||||
async fn lookup_srv(domain: &str) -> Option<MailConfig> {
|
||||
let resolver = TokioResolver::builder(TokioConnectionProvider::default())
|
||||
let resolver = TokioResolver::builder(TokioRuntimeProvider::default())
|
||||
.ok()?
|
||||
.build();
|
||||
.build()
|
||||
.ok()?;
|
||||
|
||||
let imap_srv = format!("_imaps._tcp.{}.", domain);
|
||||
let imap_lookup = resolver.lookup(imap_srv, RecordType::SRV).await.ok()?;
|
||||
let imap_record = imap_lookup.iter().next()?;
|
||||
let (imap_host, imap_port) = match imap_record {
|
||||
let imap_lookup = resolver.srv_lookup(imap_srv).await.ok()?;
|
||||
let imap_record = imap_lookup.answers().first()?;
|
||||
let (imap_host, imap_port) = match &imap_record.data {
|
||||
RData::SRV(srv) => {
|
||||
let host = srv.target().to_string().trim_end_matches('.').to_string();
|
||||
(host, srv.port())
|
||||
let host = srv.target.to_string().trim_end_matches('.').to_string();
|
||||
(host, srv.port)
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let smtp_srv = format!("_submission._tcp.{}.", domain);
|
||||
let smtp_lookup = resolver.lookup(smtp_srv, RecordType::SRV).await.ok()?;
|
||||
let smtp_record = smtp_lookup.iter().next()?;
|
||||
let (smtp_host, smtp_port) = match smtp_record {
|
||||
let smtp_lookup = resolver.srv_lookup(smtp_srv).await.ok()?;
|
||||
let smtp_record = smtp_lookup.answers().first()?;
|
||||
let (smtp_host, smtp_port) = match &smtp_record.data {
|
||||
RData::SRV(srv) => {
|
||||
let host = srv.target().to_string().trim_end_matches('.').to_string();
|
||||
(host, srv.port())
|
||||
let host = srv.target.to_string().trim_end_matches('.').to_string();
|
||||
(host, srv.port)
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
@@ -177,13 +177,19 @@ pub async fn fetch(domain: &str) -> BichonResult<MailConfig> {
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
// ── ISP autoconfig (HTTPS, then HTTP) ──────────────────────────
|
||||
if let Some(config) =
|
||||
fetch_xml(&client, &format!("https://autoconfig.{domain}/mail/config-v1.1.xml")).await
|
||||
if let Some(config) = fetch_xml(
|
||||
&client,
|
||||
&format!("https://autoconfig.{domain}/mail/config-v1.1.xml"),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Ok(config);
|
||||
}
|
||||
if let Some(config) =
|
||||
fetch_xml(&client, &format!("http://autoconfig.{domain}/mail/config-v1.1.xml")).await
|
||||
if let Some(config) = fetch_xml(
|
||||
&client,
|
||||
&format!("http://autoconfig.{domain}/mail/config-v1.1.xml"),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Ok(config);
|
||||
}
|
||||
@@ -212,8 +218,11 @@ pub async fn fetch(domain: &str) -> BichonResult<MailConfig> {
|
||||
}
|
||||
|
||||
// ── Thunderbird central ISPDB ──────────────────────────────────
|
||||
if let Some(config) =
|
||||
fetch_xml(&client, &format!("https://autoconfig.thunderbird.net/v1.1/{domain}")).await
|
||||
if let Some(config) = fetch_xml(
|
||||
&client,
|
||||
&format!("https://autoconfig.thunderbird.net/v1.1/{domain}"),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Ok(config);
|
||||
}
|
||||
@@ -245,20 +254,29 @@ async fn fetch_for_mx(client: &Client, domain: &str) -> Option<MailConfig> {
|
||||
}
|
||||
|
||||
// Try ISPDB for the MX domain
|
||||
if let Some(config) =
|
||||
fetch_xml(client, &format!("https://autoconfig.thunderbird.net/v1.1/{mx_domain}")).await
|
||||
if let Some(config) = fetch_xml(
|
||||
client,
|
||||
&format!("https://autoconfig.thunderbird.net/v1.1/{mx_domain}"),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Some(config);
|
||||
}
|
||||
|
||||
// Try ISP autoconfig for the MX domain (HTTPS then HTTP)
|
||||
if let Some(config) =
|
||||
fetch_xml(client, &format!("https://autoconfig.{mx_domain}/mail/config-v1.1.xml")).await
|
||||
if let Some(config) = fetch_xml(
|
||||
client,
|
||||
&format!("https://autoconfig.{mx_domain}/mail/config-v1.1.xml"),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Some(config);
|
||||
}
|
||||
if let Some(config) =
|
||||
fetch_xml(client, &format!("http://autoconfig.{mx_domain}/mail/config-v1.1.xml")).await
|
||||
if let Some(config) = fetch_xml(
|
||||
client,
|
||||
&format!("http://autoconfig.{mx_domain}/mail/config-v1.1.xml"),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Some(config);
|
||||
}
|
||||
@@ -268,12 +286,16 @@ async fn fetch_for_mx(client: &Client, domain: &str) -> Option<MailConfig> {
|
||||
|
||||
/// DNS MX lookup → extract the second-level domain of the first MX hostname.
|
||||
async fn lookup_mx_domain(domain: &str) -> Option<String> {
|
||||
let resolver = TokioResolver::builder(TokioConnectionProvider::default())
|
||||
let resolver = TokioResolver::builder(TokioRuntimeProvider::default())
|
||||
.ok()?
|
||||
.build();
|
||||
.build()
|
||||
.ok()?;
|
||||
let lookup = resolver.mx_lookup(domain).await.ok()?;
|
||||
let record = lookup.iter().next()?;
|
||||
let mx_host = record.to_string().trim_end_matches('.').to_string();
|
||||
let record = lookup.answers().first()?;
|
||||
let mx_host = match &record.data {
|
||||
RData::MX(mx) => mx.exchange.to_string().trim_end_matches('.').to_string(),
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
// Extract a reasonable base domain from the MX hostname.
|
||||
// E.g., "aspmx.l.google.com" → "google.com"
|
||||
@@ -336,4 +358,95 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_autoconfig_xml() {
|
||||
let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<clientConfig version="1.1">
|
||||
<emailProvider id="example.com">
|
||||
<domain>example.com</domain>
|
||||
<incomingServer type="imap">
|
||||
<hostname>imap.example.com</hostname>
|
||||
<port>993</port>
|
||||
<socketType>SSL</socketType>
|
||||
<username>%EMAILADDRESS%</username>
|
||||
<authentication>password-cleartext</authentication>
|
||||
</incomingServer>
|
||||
<outgoingServer type="smtp">
|
||||
<hostname>smtp.example.com</hostname>
|
||||
<port>587</port>
|
||||
<socketType>STARTTLS</socketType>
|
||||
<username>%EMAILADDRESS%</username>
|
||||
</outgoingServer>
|
||||
</emailProvider>
|
||||
</clientConfig>"#;
|
||||
|
||||
let config = parse_autoconfig_xml(xml).expect("should parse valid XML");
|
||||
assert_eq!(config.incoming.len(), 1);
|
||||
assert_eq!(config.incoming[0].protocol, "imap");
|
||||
assert_eq!(config.incoming[0].hostname, "imap.example.com");
|
||||
assert_eq!(config.incoming[0].port, 993);
|
||||
assert_eq!(config.incoming[0].socket_type, "SSL");
|
||||
assert_eq!(config.incoming[0].username, "%EMAILADDRESS%");
|
||||
assert_eq!(config.incoming[0].authentication, "password-cleartext");
|
||||
|
||||
assert_eq!(config.outgoing.len(), 1);
|
||||
assert_eq!(config.outgoing[0].protocol, "smtp");
|
||||
assert_eq!(config.outgoing[0].hostname, "smtp.example.com");
|
||||
assert_eq!(config.outgoing[0].port, 587);
|
||||
assert_eq!(config.outgoing[0].socket_type, "STARTTLS");
|
||||
assert_eq!(config.outgoing[0].username, "%EMAILADDRESS%");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_autoconfig_xml_invalid() {
|
||||
assert!(parse_autoconfig_xml("not xml").is_none());
|
||||
assert!(parse_autoconfig_xml("<clientConfig></clientConfig>").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_base_domain() {
|
||||
assert_eq!(
|
||||
extract_base_domain("aspmx.l.google.com"),
|
||||
Some("google.com".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
extract_base_domain("company.mail.protection.outlook.com"),
|
||||
Some("outlook.com".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
extract_base_domain("mx.example.com"),
|
||||
Some("example.com".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
extract_base_domain("example.com"),
|
||||
Some("example.com".to_string())
|
||||
);
|
||||
assert_eq!(extract_base_domain("localhost"), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_lookup_srv_gmail() {
|
||||
// Gmail should have SRV records for IMAPS and SMTP submission
|
||||
let config = lookup_srv("gmail.com").await;
|
||||
assert!(config.is_some(), "Gmail should have SRV records");
|
||||
let config = config.unwrap();
|
||||
assert_eq!(config.incoming.len(), 1);
|
||||
assert_eq!(config.incoming[0].protocol, "imap");
|
||||
assert_eq!(config.incoming[0].socket_type, "SSL");
|
||||
assert!(!config.incoming[0].hostname.is_empty());
|
||||
assert!(config.incoming[0].port > 0);
|
||||
assert_eq!(config.outgoing.len(), 1);
|
||||
assert_eq!(config.outgoing[0].protocol, "smtp");
|
||||
assert_eq!(config.outgoing[0].socket_type, "STARTTLS");
|
||||
assert!(!config.outgoing[0].hostname.is_empty());
|
||||
assert!(config.outgoing[0].port > 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_lookup_srv_nonexistent() {
|
||||
// A domain without SRV records should return None
|
||||
let config = lookup_srv("this-domain-definitely-does-not-exist-12345.com").await;
|
||||
assert!(config.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
1023
crates/core/src/cache/imap/download/flow.rs
vendored
1023
crates/core/src/cache/imap/download/flow.rs
vendored
File diff suppressed because it is too large
Load Diff
@@ -17,7 +17,7 @@
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use crate::settings::dir::DATA_DIR_MANAGER;
|
||||
use memdb::{Durability, MemDb};
|
||||
use bichon_memdb::{Durability, MemDb};
|
||||
use std::sync::LazyLock;
|
||||
use std::time::Duration;
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ use crate::common::paginated::Paginated;
|
||||
use crate::error::code::ErrorCode;
|
||||
use crate::error::BichonResult;
|
||||
use crate::raise_error;
|
||||
use memdb::{MemDb, Transaction};
|
||||
use bichon_memdb::{MemDb, Transaction};
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::Serialize;
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use crate::account::migration::AccountModel;
|
||||
use crate::cache::imap::mailbox::MailBox;
|
||||
use crate::common::AddrVec;
|
||||
use crate::envelope::meta::parse_bichon_metadata;
|
||||
@@ -112,6 +113,34 @@ async fn extract_envelope_core(
|
||||
)
|
||||
})?;
|
||||
|
||||
if let Ok(account) = AccountModel::get(account_id) {
|
||||
if let Some(ref rules) = account.archive_rules {
|
||||
let sender = message.from().and_then(|addr| {
|
||||
AddrVec::from(addr).0.into_iter().next().and_then(|a| a.address)
|
||||
});
|
||||
let subject = message.subject().map(|s| s.to_string());
|
||||
|
||||
let is_spam = !rules.spam_headers.is_empty()
|
||||
&& rules.spam_headers.iter().any(|h| {
|
||||
message
|
||||
.header_raw(h.clone())
|
||||
.map(|v| matches!(v.trim().to_lowercase().as_str(), "yes" | "true"))
|
||||
.unwrap_or(false)
|
||||
});
|
||||
|
||||
if !rules.should_archive(sender.as_deref(), subject.as_deref(), size, is_spam) {
|
||||
tracing::debug!(
|
||||
account_id,
|
||||
uid,
|
||||
sender = sender.as_deref().unwrap_or("?"),
|
||||
subject = subject.as_deref().unwrap_or("?"),
|
||||
"Email filtered out by archive rules"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let preview_limit = 100;
|
||||
let text = if let Some(text) = message.body_text(0).map(|cow| cow.into_owned()) {
|
||||
text
|
||||
@@ -173,7 +202,7 @@ async fn extract_envelope_core(
|
||||
.and_then(|add| add.address)
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
let attachment_count = message.attachment_count();
|
||||
let attachments = detach_and_store_attachments(body, &message, &email_content_hash).await;
|
||||
let attachments = detach_and_store_attachments(body, &message, &email_content_hash, account_id, mailbox_id).await;
|
||||
|
||||
let envelope_id = Uuid::new_v4().to_string();
|
||||
let now = utc_now!();
|
||||
@@ -267,6 +296,7 @@ async fn extract_envelope_core(
|
||||
account_email: None,
|
||||
mailbox_name: None,
|
||||
content_hash: email_content_hash.clone(),
|
||||
account_name: None,
|
||||
};
|
||||
// 'attachments' contains both regular and inline attachments
|
||||
let ea = EnvelopeWithAttachments {
|
||||
@@ -361,6 +391,7 @@ pub fn extract_envelope_from_nested_message(
|
||||
regular_attachment_count: Default::default(),
|
||||
tags: Default::default(),
|
||||
account_email: Default::default(),
|
||||
account_name: Default::default(),
|
||||
mailbox_name: Default::default(),
|
||||
content_hash: Default::default(),
|
||||
};
|
||||
@@ -399,7 +430,27 @@ pub async fn detach_and_store_attachments(
|
||||
original_body: &[u8],
|
||||
message: &Message<'_>,
|
||||
eml_content_hash: &str,
|
||||
account_id: u64,
|
||||
mailbox_id: u64,
|
||||
) -> Vec<AttachmentInfo> {
|
||||
let rules = if account_id > 0 {
|
||||
AccountModel::get(account_id)
|
||||
.ok()
|
||||
.and_then(|a| a.extraction_rules)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mailbox_name = match rules.as_ref().map(|r| !r.folders.is_empty()) {
|
||||
Some(true) => MailBox::get(mailbox_id).ok().map(|mb| mb.name),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let sender = message
|
||||
.from()
|
||||
.and_then(|addr| AddrVec::from(addr).0.into_iter().next())
|
||||
.and_then(|add| add.address);
|
||||
|
||||
let mut stripped_eml = original_body.to_vec();
|
||||
let mut attachment_infos = Vec::new();
|
||||
// Step 1: Collect and sort attachment ranges in reverse to maintain offset integrity
|
||||
@@ -468,19 +519,30 @@ pub async fn detach_and_store_attachments(
|
||||
})
|
||||
.unwrap_or_else(|| "application/octet-stream".to_string());
|
||||
let has_cid = att.content_id().is_some();
|
||||
let ext = att
|
||||
.attachment_name()
|
||||
let att_name = att.attachment_name().map(|n| n.to_string());
|
||||
let ext = att_name
|
||||
.as_deref()
|
||||
.and_then(|n| {
|
||||
std::path::Path::new(&n)
|
||||
std::path::Path::new(n)
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|s| s.to_ascii_lowercase())
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let should_extract = rules.as_ref().map_or(true, |r| {
|
||||
r.should_extract(
|
||||
&ext,
|
||||
mailbox_name.as_deref(),
|
||||
att_name.as_deref(),
|
||||
sender.as_deref(),
|
||||
)
|
||||
});
|
||||
|
||||
if !inline || !has_cid {
|
||||
let decoded_len = att.contents().len();
|
||||
if decoded_len <= crate::ext::text_extractor::MAX_EXTRACT_BYTES
|
||||
if should_extract
|
||||
&& decoded_len <= crate::ext::text_extractor::MAX_EXTRACT_BYTES
|
||||
&& crate::ext::text_extractor::should_try_extract(&file_type, &ext)
|
||||
{
|
||||
text_candidates.push(TextCandidate {
|
||||
@@ -731,7 +793,7 @@ async fn recover_message_blob(envelope: &Envelope) -> BichonResult<Bytes> {
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
detach_and_store_attachments(&raw_body, &message, &fetched_hash).await;
|
||||
detach_and_store_attachments(&raw_body, &message, &fetched_hash, envelope.account_id, envelope.mailbox_id).await;
|
||||
|
||||
Ok(Bytes::from(raw_body))
|
||||
}
|
||||
@@ -828,6 +890,8 @@ mod test {
|
||||
truncated,
|
||||
&message,
|
||||
"test_content_hash",
|
||||
0,
|
||||
0,
|
||||
)
|
||||
.await;
|
||||
|
||||
|
||||
@@ -63,8 +63,18 @@ pub const MAX_EXTRACT_BYTES: usize = 10 * 1024 * 1024;
|
||||
pub fn should_try_extract(content_type: &str, ext: &str) -> bool {
|
||||
matches!(
|
||||
ext,
|
||||
"pdf" | "doc" | "docx" | "xls" | "xlsx" | "ppt" | "pptx"
|
||||
| "txt" | "rtf" | "odt" | "ods" | "odp"
|
||||
"pdf"
|
||||
| "doc"
|
||||
| "docx"
|
||||
| "xls"
|
||||
| "xlsx"
|
||||
| "ppt"
|
||||
| "pptx"
|
||||
| "txt"
|
||||
| "rtf"
|
||||
| "odt"
|
||||
| "ods"
|
||||
| "odp"
|
||||
) || content_type.starts_with("text/")
|
||||
}
|
||||
|
||||
|
||||
@@ -34,6 +34,25 @@ use std::ops::DerefMut;
|
||||
use tokio::io::BufWriter;
|
||||
use tracing::debug;
|
||||
|
||||
/// Classify an `io::Error` (from TLS stream I/O) for IMAP connection errors.
|
||||
/// `UnexpectedEof` is treated as a network error because many servers skip
|
||||
/// the TLS `close_notify` alert, causing rustls to emit this error when the
|
||||
/// TCP connection is dropped normally.
|
||||
fn classify_io_error(e: &std::io::Error) -> ErrorCode {
|
||||
use std::io::ErrorKind;
|
||||
matches!(
|
||||
e.kind(),
|
||||
ErrorKind::BrokenPipe
|
||||
| ErrorKind::ConnectionReset
|
||||
| ErrorKind::ConnectionAborted
|
||||
| ErrorKind::TimedOut
|
||||
| ErrorKind::UnexpectedEof
|
||||
| ErrorKind::NotConnected
|
||||
)
|
||||
.then_some(ErrorCode::NetworkError)
|
||||
.unwrap_or(ErrorCode::ImapCommandFailed)
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct Client {
|
||||
inner: ImapClient<Box<dyn SessionStream>>,
|
||||
@@ -141,7 +160,7 @@ impl Client {
|
||||
let _greeting = client
|
||||
.read_response()
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))?
|
||||
.ok_or_else(|| {
|
||||
raise_error!(
|
||||
"Failed to read IMAP greeting — this usually indicates an incorrect encryption setting (SSL vs. STARTTLS). Your current setting is SSL.".into(),
|
||||
@@ -171,7 +190,7 @@ impl Client {
|
||||
let _greeting = client
|
||||
.read_response()
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))?
|
||||
.ok_or_else(|| {
|
||||
raise_error!(
|
||||
"failed to read greeting".into(),
|
||||
@@ -202,7 +221,7 @@ impl Client {
|
||||
let _greeting = client
|
||||
.read_response()
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))?
|
||||
.ok_or_else(|| {
|
||||
raise_error!(
|
||||
"Failed to read IMAP greeting — this usually indicates an incorrect encryption setting (SSL vs. STARTTLS). Your current setting is STARTTLS.".into(),
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::{error::BichonResult, imap::manager::ImapConnectionManager};
|
||||
use async_imap::types::Name;
|
||||
use async_imap::Session;
|
||||
use futures::TryStreamExt;
|
||||
use std::collections::HashSet;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::info;
|
||||
|
||||
@@ -260,7 +260,9 @@ impl ImapExecutor {
|
||||
let mut count = 0u64;
|
||||
let mut skipped = 0u64;
|
||||
let mut max_uid: Option<u32> = None;
|
||||
let size_limit = account.max_email_size_bytes.unwrap_or(DEFAULT_MAX_EMAIL_SIZE);
|
||||
let size_limit = account
|
||||
.max_email_size_bytes
|
||||
.unwrap_or(DEFAULT_MAX_EMAIL_SIZE);
|
||||
while let Some(fetch) = stream
|
||||
.try_next()
|
||||
.await
|
||||
@@ -363,14 +365,14 @@ impl ImapExecutor {
|
||||
let mut size_stream = session
|
||||
.fetch(sequence_set.as_str(), SIZE_ONLY_FETCH)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
raise_error!(format!("{:#?}", e), classify_imap_error(&e))
|
||||
})?;
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?;
|
||||
|
||||
let mut uids: Vec<u32> = Vec::new();
|
||||
while let Some(fetch) = size_stream.try_next().await.map_err(|e| {
|
||||
raise_error!(format!("{:#?}", e), classify_imap_error(&e))
|
||||
})? {
|
||||
while let Some(fetch) = size_stream
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?
|
||||
{
|
||||
let uid = fetch.uid.unwrap_or(0);
|
||||
let msg_size = fetch.size.unwrap_or(0) as u64;
|
||||
if msg_size == 0 || msg_size <= limit {
|
||||
@@ -437,14 +439,14 @@ impl ImapExecutor {
|
||||
let mut size_stream = session
|
||||
.uid_fetch(uid_set, SIZE_ONLY_FETCH)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
raise_error!(format!("{:#?}", e), classify_imap_error(&e))
|
||||
})?;
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?;
|
||||
|
||||
let mut uids: Vec<u32> = Vec::new();
|
||||
while let Some(fetch) = size_stream.try_next().await.map_err(|e| {
|
||||
raise_error!(format!("{:#?}", e), classify_imap_error(&e))
|
||||
})? {
|
||||
while let Some(fetch) = size_stream
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?
|
||||
{
|
||||
let uid = fetch.uid.unwrap_or(0);
|
||||
let msg_size = fetch.size.unwrap_or(0) as u64;
|
||||
if msg_size == 0 || msg_size <= limit {
|
||||
@@ -550,6 +552,37 @@ impl ImapExecutor {
|
||||
) -> BichonResult<Session<Box<dyn SessionStream>>> {
|
||||
ImapConnectionManager::build(account_id).await
|
||||
}
|
||||
|
||||
/// Fetch UID → Message-ID mapping without downloading bodies.
|
||||
/// `uid_set` is an IMAP sequence-set string (e.g. "1:100" or "1,3,5").
|
||||
pub async fn fetch_uid_metadata(
|
||||
session: &mut Session<Box<dyn SessionStream>>,
|
||||
uid_set: &str,
|
||||
token: CancellationToken,
|
||||
) -> BichonResult<HashMap<u32, Option<String>>> {
|
||||
let mut stream = session
|
||||
.uid_fetch(uid_set, "(UID BODY.PEEK[HEADER])")
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?;
|
||||
|
||||
let mut result = HashMap::new();
|
||||
while let Some(fetch) = stream
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?
|
||||
{
|
||||
if token.is_cancelled() {
|
||||
return Err(raise_error!(
|
||||
"Stream cancelled".into(),
|
||||
ErrorCode::InternalError
|
||||
));
|
||||
}
|
||||
let uid = fetch.uid.unwrap_or(0);
|
||||
let msg_id = fetch.header().and_then(parse_message_id_header);
|
||||
result.insert(uid, msg_id);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
pub const DEFAULT_BATCH_SIZE: u32 = 30;
|
||||
@@ -613,6 +646,27 @@ pub fn generate_uid_sequence_hashset(
|
||||
result
|
||||
}
|
||||
|
||||
fn parse_message_id_header(header_bytes: &[u8]) -> Option<String> {
|
||||
let header = std::str::from_utf8(header_bytes).ok()?;
|
||||
for line in header.lines() {
|
||||
if let Some(value) = line
|
||||
.strip_prefix("Message-ID:")
|
||||
.or_else(|| line.strip_prefix("Message-Id:"))
|
||||
.or_else(|| line.strip_prefix("Message-id:"))
|
||||
{
|
||||
// mail_parser strips angle brackets, so we must do the same
|
||||
// to ensure comparisons against the Tantivy index match.
|
||||
let trimmed = value.trim();
|
||||
let stripped = trimmed.strip_prefix('<').unwrap_or(trimmed);
|
||||
let stripped = stripped.strip_suffix('>').unwrap_or(stripped);
|
||||
if !stripped.is_empty() {
|
||||
return Some(stripped.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
@@ -668,4 +722,80 @@ mod test {
|
||||
assert_eq!(batches[2].0, "5");
|
||||
assert_eq!(batches[2].1, 1);
|
||||
}
|
||||
|
||||
// ── parse_message_id_header ─────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn parse_standard_message_id() {
|
||||
let header = b"Message-ID: <abc123@example.com>\r\n";
|
||||
assert_eq!(
|
||||
parse_message_id_header(header),
|
||||
Some("abc123@example.com".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_message_id_lowercase() {
|
||||
let header = b"Message-Id: <foo@bar.com>\r\n";
|
||||
assert_eq!(
|
||||
parse_message_id_header(header),
|
||||
Some("foo@bar.com".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_message_id_extra_whitespace() {
|
||||
let header = b"Message-ID: <spaces@test.com> \r\n";
|
||||
assert_eq!(
|
||||
parse_message_id_header(header),
|
||||
Some("spaces@test.com".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_empty_message_id_returns_none() {
|
||||
let header = b"Message-ID: <>\r\n";
|
||||
assert_eq!(parse_message_id_header(header), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_missing_header_returns_none() {
|
||||
let header = b"X-Custom: something\r\n";
|
||||
assert_eq!(parse_message_id_header(header), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_empty_body_returns_none() {
|
||||
assert_eq!(parse_message_id_header(b""), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_message_id_in_full_header() {
|
||||
// The Message-ID line is in the middle, not at the start.
|
||||
let header = b"From: sender@example.com\r\n\
|
||||
Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\
|
||||
Subject: test\r\n\
|
||||
Message-ID: <mid@example.com>\r\n\
|
||||
To: recipient@example.com\r\n\r\n";
|
||||
assert_eq!(
|
||||
parse_message_id_header(header),
|
||||
Some("mid@example.com".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_message_id_only_in_full_header() {
|
||||
// Only a few headers, Message-ID is among them.
|
||||
let header = b"From: a@b.com\r\nMessage-ID: <x@y.com>\r\n\r\n";
|
||||
assert_eq!(parse_message_id_header(header), Some("x@y.com".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_message_id_no_brackets_still_works() {
|
||||
let header = b"Message-ID: plain@example.com\r\n";
|
||||
assert_eq!(
|
||||
parse_message_id_header(header),
|
||||
Some("plain@example.com".into())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,16 +95,40 @@ impl ImapConnectionManager {
|
||||
|
||||
pub async fn build(account_id: u64) -> BichonResult<Session<Box<dyn SessionStream>>> {
|
||||
let account = AccountModel::get(account_id)?;
|
||||
let client = match Self::create_client(&account).await {
|
||||
Ok(client) => client,
|
||||
Err(error) => {
|
||||
error!(
|
||||
"Failed to create IMAP {}'s client: {:#?}",
|
||||
&account.email, error
|
||||
);
|
||||
return Err(error);
|
||||
let account_email = account.email.clone();
|
||||
|
||||
let mut client = None;
|
||||
for attempt in 0..3u32 {
|
||||
match Self::create_client(&account).await {
|
||||
Ok(c) => {
|
||||
client = Some(c);
|
||||
break;
|
||||
}
|
||||
Err(error) if error.code() == ErrorCode::NetworkError && attempt < 2 => {
|
||||
warn!(
|
||||
"IMAP connection attempt {}/3 to {} failed (network error), retrying...",
|
||||
attempt + 1,
|
||||
account_email
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||||
continue;
|
||||
}
|
||||
Err(error) => {
|
||||
error!(
|
||||
"Failed to create IMAP {}'s client: {:#?}",
|
||||
account_email, error
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
let client = client.ok_or_else(|| {
|
||||
raise_error!(
|
||||
format!("Failed to create IMAP {}'s client after 3 attempts", account_email),
|
||||
ErrorCode::NetworkError
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut session = match Self::authenticate(client, &account).await {
|
||||
Ok(session) => session,
|
||||
|
||||
538
crates/core/src/imap/mock_server.rs
Normal file
538
crates/core/src/imap/mock_server.rs
Normal file
@@ -0,0 +1,538 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//! A minimal scriptable IMAP server for integration testing.
|
||||
//!
|
||||
//! Each instance listens on a random localhost port and responds to a
|
||||
//! pre-configured script of (expected_command, response) pairs. Commands
|
||||
//! are matched by substring — the first matching pattern wins.
|
||||
//!
|
||||
//! # Example
|
||||
//! ```ignore
|
||||
//! let server = MockImapServer::new()
|
||||
//! .greeting("* OK ready\r\n")
|
||||
//! .respond("LOGIN", "A0 OK logged in\r\n")
|
||||
//! .respond("CAPABILITY", "* CAPABILITY IMAP4rev1\r\nA0 OK done\r\n")
|
||||
//! .respond("STATUS", "* STATUS INBOX (MESSAGES 10 UIDVALIDITY 42)\r\nA0 OK\r\n")
|
||||
//! .respond("LOGOUT", "* BYE\r\nA0 OK\r\n")
|
||||
//! .start()
|
||||
//! .await;
|
||||
//!
|
||||
//! let (host, port) = server.addr();
|
||||
//! // connect to host:port with Encryption::None
|
||||
//! ```
|
||||
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
|
||||
type Response = Vec<u8>;
|
||||
|
||||
pub struct MockImapServer {
|
||||
greeting: Vec<u8>,
|
||||
script: Vec<(String, Response)>,
|
||||
}
|
||||
|
||||
impl MockImapServer {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
greeting: b"* OK Mock IMAP server ready\r\n".to_vec(),
|
||||
script: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the greeting banner sent immediately after connection.
|
||||
pub fn greeting(mut self, banner: impl Into<Vec<u8>>) -> Self {
|
||||
self.greeting = banner.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a script step: when a client command *contains* `pattern` (case-insensitive),
|
||||
/// respond with `response`. Steps are checked in insertion order.
|
||||
pub fn respond(mut self, pattern: impl Into<String>, response: impl Into<Vec<u8>>) -> Self {
|
||||
self.script.push((pattern.into(), response.into()));
|
||||
self
|
||||
}
|
||||
|
||||
/// Start the server on a random port. Returns a handle whose `addr()` gives
|
||||
/// the `(host, port)` to connect to.
|
||||
pub async fn start(self) -> MockImapServerHandle {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
|
||||
let addr = listener.local_addr().expect("local_addr");
|
||||
|
||||
let server = Arc::new(self);
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
match listener.accept().await {
|
||||
Ok((stream, _)) => {
|
||||
let srv = server.clone();
|
||||
tokio::spawn(async move {
|
||||
srv.handle_connection(stream).await;
|
||||
});
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
MockImapServerHandle { addr }
|
||||
}
|
||||
|
||||
async fn handle_connection(&self, mut stream: TcpStream) {
|
||||
let (reader, mut writer) = stream.split();
|
||||
let mut reader = BufReader::new(reader);
|
||||
|
||||
// Send greeting
|
||||
if writer.write_all(&self.greeting).await.is_err() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut line = String::new();
|
||||
loop {
|
||||
line.clear();
|
||||
match reader.read_line(&mut line).await {
|
||||
Ok(0) => break, // EOF
|
||||
Ok(_) => {}
|
||||
Err(_) => break,
|
||||
}
|
||||
|
||||
let tag = extract_tag(&line).unwrap_or("A0");
|
||||
let matched = self.find_match(&line);
|
||||
if let Some(response) = matched {
|
||||
let substituted = substitute_tag(response, tag);
|
||||
if writer.write_all(&substituted).await.is_err() {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
// Default: send tagged OK for commands we don't handle
|
||||
let fallback = format!("{tag} OK done\r\n");
|
||||
if writer.write_all(fallback.as_bytes()).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn find_match(&self, line: &str) -> Option<&[u8]> {
|
||||
let line_lower = line.to_lowercase();
|
||||
for (pattern, response) in &self.script {
|
||||
if line_lower.contains(&pattern.to_lowercase()) {
|
||||
return Some(response);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for MockImapServer {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle to a running mock IMAP server. The server stops when this handle
|
||||
/// is dropped.
|
||||
pub struct MockImapServerHandle {
|
||||
addr: SocketAddr,
|
||||
}
|
||||
|
||||
impl MockImapServerHandle {
|
||||
pub fn host(&self) -> String {
|
||||
self.addr.ip().to_string()
|
||||
}
|
||||
|
||||
pub fn port(&self) -> u16 {
|
||||
self.addr.port()
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_tag(line: &str) -> Option<&str> {
|
||||
line.split_whitespace().next()
|
||||
}
|
||||
|
||||
/// Replace `{TAG}` placeholders in `response` with `tag`.
|
||||
fn substitute_tag(response: &[u8], tag: &str) -> Vec<u8> {
|
||||
let placeholder = b"{TAG}";
|
||||
if response.is_empty() || !contains_slice(response, placeholder) {
|
||||
return response.to_vec();
|
||||
}
|
||||
let tag_bytes = tag.as_bytes();
|
||||
let mut result = Vec::with_capacity(response.len());
|
||||
let mut pos = 0;
|
||||
while let Some(idx) = find_slice(&response[pos..], placeholder) {
|
||||
result.extend_from_slice(&response[pos..pos + idx]);
|
||||
result.extend_from_slice(tag_bytes);
|
||||
pos += idx + placeholder.len();
|
||||
}
|
||||
result.extend_from_slice(&response[pos..]);
|
||||
result
|
||||
}
|
||||
|
||||
fn contains_slice(haystack: &[u8], needle: &[u8]) -> bool {
|
||||
haystack.windows(needle.len()).any(|w| w == needle)
|
||||
}
|
||||
|
||||
fn find_slice(haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
||||
haystack
|
||||
.windows(needle.len())
|
||||
.position(|w| w == needle)
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Pre-built response helpers
|
||||
// ============================================================
|
||||
|
||||
/// Build a tagged OK response.
|
||||
pub fn ok(tag: impl AsRef<str>, msg: impl AsRef<str>) -> Vec<u8> {
|
||||
format!("{} OK {}\r\n", tag.as_ref(), msg.as_ref()).into_bytes()
|
||||
}
|
||||
|
||||
/// Build a STATUS response line.
|
||||
pub fn status_response(
|
||||
mailbox: &str,
|
||||
messages: u32,
|
||||
unseen: u32,
|
||||
uid_next: u32,
|
||||
uid_validity: Option<u32>,
|
||||
) -> Vec<u8> {
|
||||
let uv = uid_validity
|
||||
.map(|v| format!(" UIDVALIDITY {v}"))
|
||||
.unwrap_or_default();
|
||||
let text = format!(
|
||||
"* STATUS \"{mailbox}\" (MESSAGES {messages} UNSEEN {unseen} UIDNEXT {uid_next}{uv})\r\n"
|
||||
);
|
||||
// Clients expect a tagged response after the untagged STATUS line.
|
||||
// We produce a generic OK that works for any tag.
|
||||
let mut out = text.into_bytes();
|
||||
out.extend_from_slice(b"{TAG} OK STATUS completed\r\n");
|
||||
out
|
||||
}
|
||||
|
||||
/// Build an EXAMINE response with mailbox data.
|
||||
pub fn examine_response(
|
||||
_mailbox: &str,
|
||||
exists: u32,
|
||||
uid_validity: u32,
|
||||
uid_next: u32,
|
||||
) -> Vec<u8> {
|
||||
format!(
|
||||
"* FLAGS (\\Seen \\Answered \\Flagged \\Deleted \\Draft)\r\n\
|
||||
* OK [PERMANENTFLAGS ()]\r\n\
|
||||
* {exists} EXISTS\r\n\
|
||||
* 0 RECENT\r\n\
|
||||
* OK [UIDVALIDITY {uid_validity}]\r\n\
|
||||
* OK [UIDNEXT {uid_next}]\r\n\
|
||||
* OK [HIGHESTMODSEQ 1]\r\n\
|
||||
{{TAG}} OK [READ-ONLY] EXAMINE completed\r\n"
|
||||
)
|
||||
.into_bytes()
|
||||
}
|
||||
|
||||
/// Build a UID SEARCH response for the given UID list.
|
||||
pub fn uid_search_response(uids: &[u32]) -> Vec<u8> {
|
||||
let uid_str = uids
|
||||
.iter()
|
||||
.map(|u| u.to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
format!("* SEARCH {uid_str}\r\n{{TAG}} OK SEARCH completed\r\n").into_bytes()
|
||||
}
|
||||
|
||||
/// Build a UID FETCH response returning full headers (for BODY[HEADER]).
|
||||
/// Each entry: (uid, message_id)
|
||||
pub fn uid_fetch_metadata_response(entries: &[(u32, &str)]) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
for (uid, msg_id) in entries {
|
||||
// Build a minimal header that contains the Message-ID line.
|
||||
let header_data = format!(
|
||||
"From: sender@example.com\r\n\
|
||||
To: recipient@example.com\r\n\
|
||||
Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\
|
||||
Subject: test\r\n\
|
||||
Message-ID: {msg_id}\r\n\r\n"
|
||||
);
|
||||
let header_len = header_data.len();
|
||||
let line = format!(
|
||||
"* {uid} FETCH (UID {uid} BODY[HEADER] {{{header_len}}}\r\n\
|
||||
{header_data}\
|
||||
)\r\n",
|
||||
);
|
||||
out.extend_from_slice(line.as_bytes());
|
||||
}
|
||||
out.extend_from_slice(b"{TAG} OK FETCH completed\r\n");
|
||||
out
|
||||
}
|
||||
|
||||
/// Build a UID FETCH RFC822 response with a full email body.
|
||||
pub fn uid_fetch_rfc822_response(uid: u32, eml: &[u8]) -> Vec<u8> {
|
||||
let header = format!(
|
||||
"* {uid} FETCH (UID {uid} RFC822 {{{len}}}\r\n",
|
||||
len = eml.len()
|
||||
);
|
||||
let mut out = header.into_bytes();
|
||||
out.extend_from_slice(eml);
|
||||
out.extend_from_slice(b")\r\n{TAG} OK FETCH completed\r\n");
|
||||
out
|
||||
}
|
||||
|
||||
/// A minimal RFC822 email fixture for testing.
|
||||
pub fn minimal_eml(subject: &str, message_id: &str) -> Vec<u8> {
|
||||
format!(
|
||||
"From: sender@example.com\r\n\
|
||||
To: recipient@example.com\r\n\
|
||||
Subject: {subject}\r\n\
|
||||
Message-ID: <{message_id}>\r\n\
|
||||
Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\
|
||||
MIME-Version: 1.0\r\n\
|
||||
Content-Type: text/plain; charset=utf-8\r\n\
|
||||
\r\n\
|
||||
This is a test email: {subject}.\r\n"
|
||||
)
|
||||
.into_bytes()
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Self-tests for the mock server itself
|
||||
// ============================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
async fn connect_and_read_greeting(host: &str, port: u16) -> String {
|
||||
let mut stream = TcpStream::connect((host, port)).await.unwrap();
|
||||
let (reader, _writer) = stream.split();
|
||||
let mut reader = BufReader::new(reader);
|
||||
let mut line = String::new();
|
||||
reader.read_line(&mut line).await.unwrap();
|
||||
line
|
||||
}
|
||||
|
||||
async fn send_and_recv(host: &str, port: u16, cmd: &str) -> String {
|
||||
let mut stream = TcpStream::connect((host, port)).await.unwrap();
|
||||
let (reader, mut writer) = stream.split();
|
||||
let mut reader = BufReader::new(reader);
|
||||
|
||||
// Read greeting
|
||||
let mut line = String::new();
|
||||
reader.read_line(&mut line).await.unwrap();
|
||||
|
||||
// Send command
|
||||
writer.write_all(cmd.as_bytes()).await.unwrap();
|
||||
writer.write_all(b"\r\n").await.unwrap();
|
||||
|
||||
// Read response (may be multi-line; read until tagged response)
|
||||
let mut out = String::new();
|
||||
loop {
|
||||
line.clear();
|
||||
reader.read_line(&mut line).await.unwrap();
|
||||
out.push_str(&line);
|
||||
if line.starts_with("A0") || line.starts_with("A1") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mock_greeting() {
|
||||
let handle = MockImapServer::new().start().await;
|
||||
let greeting = connect_and_read_greeting(&handle.host(), handle.port()).await;
|
||||
assert!(greeting.starts_with("* OK"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mock_scripted_response() {
|
||||
let handle = MockImapServer::new()
|
||||
.respond(
|
||||
"LOGIN",
|
||||
"A0 OK LOGIN completed\r\n",
|
||||
)
|
||||
.start()
|
||||
.await;
|
||||
|
||||
let resp = send_and_recv(&handle.host(), handle.port(), "A0 LOGIN u p").await;
|
||||
assert!(resp.contains("LOGIN completed"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mock_fallback_on_unmatched() {
|
||||
let handle = MockImapServer::new().start().await;
|
||||
|
||||
// Send a command that has no scripted response
|
||||
let resp = send_and_recv(&handle.host(), handle.port(), "A0 NOOP").await;
|
||||
assert!(resp.contains("OK done"), "unmatched command should get fallback OK");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_status_response_helper() {
|
||||
let resp = status_response("INBOX", 10, 2, 11, Some(42));
|
||||
let text = String::from_utf8(resp).unwrap();
|
||||
assert!(text.contains("MESSAGES 10"));
|
||||
assert!(text.contains("UNSEEN 2"));
|
||||
assert!(text.contains("UIDNEXT 11"));
|
||||
assert!(text.contains("UIDVALIDITY 42"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_status_response_without_uidvalidity() {
|
||||
let resp = status_response("INBOX", 10, 2, 11, None);
|
||||
let text = String::from_utf8(resp).unwrap();
|
||||
assert!(!text.contains("UIDVALIDITY"));
|
||||
assert!(text.contains("MESSAGES 10"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_examine_response() {
|
||||
let resp = examine_response("INBOX", 10, 42, 11);
|
||||
let text = String::from_utf8(resp).unwrap();
|
||||
assert!(text.contains("UIDVALIDITY 42"));
|
||||
assert!(text.contains("10 EXISTS"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_uid_search_response() {
|
||||
let resp = uid_search_response(&[1, 3, 5]);
|
||||
let text = String::from_utf8(resp).unwrap();
|
||||
assert!(text.contains("SEARCH 1 3 5"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_uid_fetch_metadata_response() {
|
||||
let resp = uid_fetch_metadata_response(&[(1, "msg-a@x.com"), (2, "msg-b@x.com")]);
|
||||
let text = String::from_utf8(resp).unwrap();
|
||||
assert!(text.contains("Message-ID: msg-a@x.com"));
|
||||
assert!(text.contains("Message-ID: msg-b@x.com"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_multiple_commands_in_sequence() {
|
||||
let handle = MockImapServer::new()
|
||||
.respond("LOGIN", "A0 OK LOGIN\r\n")
|
||||
.respond("STATUS", status_response("INBOX", 5, 1, 6, Some(99)))
|
||||
.respond("LOGOUT", "* BYE\r\nA0 OK\r\n")
|
||||
.start()
|
||||
.await;
|
||||
|
||||
let mut stream = TcpStream::connect((handle.host(), handle.port()))
|
||||
.await
|
||||
.unwrap();
|
||||
let (reader, mut writer) = stream.split();
|
||||
let mut reader = BufReader::new(reader);
|
||||
|
||||
// Read greeting
|
||||
let mut buf = String::new();
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
|
||||
// LOGIN
|
||||
writer.write_all(b"A0 LOGIN u p\r\n").await.unwrap();
|
||||
buf.clear();
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
assert!(buf.contains("LOGIN"));
|
||||
|
||||
// STATUS
|
||||
writer
|
||||
.write_all(b"A0 STATUS INBOX (MESSAGES UNSEEN UIDNEXT UIDVALIDITY)\r\n")
|
||||
.await
|
||||
.unwrap();
|
||||
buf.clear();
|
||||
// Read multi-line STATUS response (untagged line + tagged OK)
|
||||
loop {
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
if buf.contains("UIDVALIDITY 99") {
|
||||
// Consume the tagged OK line that follows
|
||||
buf.clear();
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// LOGOUT
|
||||
writer.write_all(b"A0 LOGOUT\r\n").await.unwrap();
|
||||
buf.clear();
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
assert!(buf.contains("BYE"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tag_substitution_in_response() {
|
||||
// Use {TAG} placeholder in the response and verify it gets the
|
||||
// client's actual tag ("A5") substituted in.
|
||||
let handle = MockImapServer::new()
|
||||
.respond("LOGIN", "{TAG} OK LOGIN succeeded\r\n")
|
||||
.start()
|
||||
.await;
|
||||
|
||||
let mut stream = TcpStream::connect((handle.host(), handle.port()))
|
||||
.await
|
||||
.unwrap();
|
||||
let (reader, mut writer) = stream.split();
|
||||
let mut reader = BufReader::new(reader);
|
||||
|
||||
// Read greeting
|
||||
let mut buf = String::new();
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
|
||||
// Send LOGIN with non-standard tag
|
||||
writer.write_all(b"A5 LOGIN u p\r\n").await.unwrap();
|
||||
buf.clear();
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
|
||||
assert!(
|
||||
buf.contains("A5 OK LOGIN succeeded"),
|
||||
"expected 'A5 OK LOGIN succeeded', got '{buf}'"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tag_substitution_multiple_placeholders() {
|
||||
let handle = MockImapServer::new()
|
||||
.respond("NOOP", "* 0 RECENT\r\n{TAG} OK NOOP done\r\n")
|
||||
.start()
|
||||
.await;
|
||||
|
||||
let mut stream = TcpStream::connect((handle.host(), handle.port()))
|
||||
.await
|
||||
.unwrap();
|
||||
let (reader, mut writer) = stream.split();
|
||||
let mut reader = BufReader::new(reader);
|
||||
|
||||
// Read greeting
|
||||
let mut buf = String::new();
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
|
||||
// Send with tag "B99"
|
||||
writer.write_all(b"B99 NOOP\r\n").await.unwrap();
|
||||
|
||||
// Read all lines
|
||||
let mut all = String::new();
|
||||
loop {
|
||||
buf.clear();
|
||||
reader.read_line(&mut buf).await.unwrap();
|
||||
all.push_str(&buf);
|
||||
if buf.starts_with("B99") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert!(all.contains("* 0 RECENT\r\n"));
|
||||
assert!(all.contains("B99 OK NOOP done\r\n"));
|
||||
}
|
||||
}
|
||||
@@ -26,3 +26,5 @@ pub mod session;
|
||||
pub mod stats;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
#[cfg(test)]
|
||||
pub mod mock_server;
|
||||
|
||||
127
crates/core/src/import/history.rs
Normal file
127
crates/core/src/import/history.rs
Normal file
@@ -0,0 +1,127 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
|
||||
use crate::database::MemDbModel;
|
||||
use crate::import::{ImportProgress, ImportStatus};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Maximum number of import history entries to keep per user.
|
||||
pub const MAX_HISTORY_PER_USER: usize = 5;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct ImportHistory {
|
||||
/// Composite key: "{user_id}:{import_id}"
|
||||
pub id: String,
|
||||
pub user_id: u64,
|
||||
pub import_id: String,
|
||||
pub account_id: u64,
|
||||
pub folder: String,
|
||||
pub format: String,
|
||||
pub status: String,
|
||||
pub total: usize,
|
||||
pub success: usize,
|
||||
pub duplicates: usize,
|
||||
pub failed: usize,
|
||||
pub failed_details: Vec<crate::import::FailedItemDetail>,
|
||||
/// Unix timestamp in milliseconds.
|
||||
pub created_at: i64,
|
||||
}
|
||||
|
||||
impl MemDbModel for ImportHistory {
|
||||
fn collection() -> &'static str {
|
||||
"import_history"
|
||||
}
|
||||
fn key(&self) -> String {
|
||||
self.id.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl ImportHistory {
|
||||
pub fn from_progress(
|
||||
user_id: u64,
|
||||
import_id: &str,
|
||||
account_id: u64,
|
||||
folder: &str,
|
||||
progress: &ImportProgress,
|
||||
) -> Self {
|
||||
Self {
|
||||
id: format!("{}:{}", user_id, import_id),
|
||||
user_id,
|
||||
import_id: import_id.to_string(),
|
||||
account_id,
|
||||
folder: folder.to_string(),
|
||||
format: progress.format.clone(),
|
||||
status: match progress.status {
|
||||
ImportStatus::Pending => "pending",
|
||||
ImportStatus::Processing => "processing",
|
||||
ImportStatus::Completed => "completed",
|
||||
ImportStatus::Failed => "failed",
|
||||
}
|
||||
.to_string(),
|
||||
total: progress.total,
|
||||
success: progress.success,
|
||||
duplicates: progress.duplicates,
|
||||
failed: progress.failed,
|
||||
failed_details: progress.failed_details.clone(),
|
||||
created_at: crate::utc_now!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Prune old entries for a user so only the latest `MAX_HISTORY_PER_USER` remain.
|
||||
pub fn prune_user_history(user_id: u64) -> crate::error::BichonResult<()> {
|
||||
use crate::database::manager::DB_MANAGER;
|
||||
use crate::database::batch_delete_impl;
|
||||
use crate::raise_error;
|
||||
use crate::error::code::ErrorCode;
|
||||
let db = DB_MANAGER.db();
|
||||
let coll = db.collection(ImportHistory::collection());
|
||||
let prefix = format!("{}:", user_id);
|
||||
|
||||
let mut entries: Vec<ImportHistory> = coll
|
||||
.scan_prefix(&prefix)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
if entries.len() <= MAX_HISTORY_PER_USER {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Sort by created_at descending (newest first), keep the first N
|
||||
entries.sort_by(|a, b| b.created_at.cmp(&a.created_at));
|
||||
let to_delete: Vec<String> = entries
|
||||
.iter()
|
||||
.skip(MAX_HISTORY_PER_USER)
|
||||
.map(|e| e.id.clone())
|
||||
.collect();
|
||||
|
||||
if !to_delete.is_empty() {
|
||||
batch_delete_impl::<ImportHistory>(db, to_delete)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Save an import history record and prune old entries for the user.
|
||||
pub fn save_import_history(
|
||||
user_id: u64,
|
||||
account_id: u64,
|
||||
folder: &str,
|
||||
progress: &ImportProgress,
|
||||
) {
|
||||
use crate::database::manager::DB_MANAGER;
|
||||
use crate::database::upsert_impl;
|
||||
|
||||
let entry = ImportHistory::from_progress(user_id, &progress.import_id, account_id, folder, progress);
|
||||
let db = DB_MANAGER.db();
|
||||
|
||||
if let Err(e) = upsert_impl::<ImportHistory>(db, entry) {
|
||||
tracing::error!("Failed to save import history: {:?}", e);
|
||||
return;
|
||||
}
|
||||
|
||||
if let Err(e) = prune_user_history(user_id) {
|
||||
tracing::warn!("Failed to prune import history: {:?}", e);
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,16 @@
|
||||
|
||||
|
||||
//use poem_openapi::Object;
|
||||
pub mod history;
|
||||
pub mod reader;
|
||||
pub mod pst;
|
||||
pub use history::ImportHistory;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::Path,
|
||||
sync::RwLock,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
base64_decode_url_safe,
|
||||
@@ -27,15 +36,20 @@ use crate::{
|
||||
cache::imap::mailbox::{Attribute, AttributeEnum, MailBox},
|
||||
envelope::extractor::extract_envelope_from_eml,
|
||||
error::{BichonResult, code::ErrorCode},
|
||||
settings::dir::DATA_DIR_MANAGER,
|
||||
utils::create_hash,
|
||||
},
|
||||
raise_error,
|
||||
};
|
||||
|
||||
/// Skip individual emails larger than this after decoding (100 MB).
|
||||
/// Maximum byte size of an individual email message after splitting (100 MB).
|
||||
const MAX_SINGLE_EML_BYTES: usize = 100 * 1024 * 1024;
|
||||
|
||||
#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||
/// Max file size accepted via the web upload endpoint.
|
||||
pub const MAX_WEB_EML_BYTES: usize = 100 * 1024 * 1024; // 100 MB
|
||||
pub const MAX_WEB_MBOX_BYTES: usize = 1024 * 1024 * 1024; // 1 GB
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct BatchEmlRequest {
|
||||
pub account_id: u64,
|
||||
@@ -46,24 +60,26 @@ pub struct BatchEmlRequest {
|
||||
|
||||
#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct FailedEmlDetail {
|
||||
/// The 0-based index of the failed EML in the request list
|
||||
pub struct FailedItemDetail {
|
||||
/// The index (0-based) of the failed item.
|
||||
pub index: usize,
|
||||
/// The error message that caused the import to fail
|
||||
/// The error message that caused the import to fail.
|
||||
pub error_message: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct BatchEmlResult {
|
||||
/// Total number of emails processed
|
||||
/// Total number of emails processed.
|
||||
pub total: usize,
|
||||
/// Number of emails successfully imported
|
||||
/// Number of emails successfully imported.
|
||||
pub success: usize,
|
||||
/// Number of emails failed to import
|
||||
/// Number of duplicate emails skipped (content hash already existed).
|
||||
pub duplicates: usize,
|
||||
/// Number of emails failed to import.
|
||||
pub failed: usize,
|
||||
/// A list of details for failed imports
|
||||
pub failed_details: Vec<FailedEmlDetail>,
|
||||
/// A list of details for failed imports.
|
||||
pub failed_details: Vec<FailedItemDetail>,
|
||||
}
|
||||
|
||||
pub struct ImportEmls;
|
||||
@@ -116,7 +132,7 @@ impl ImportEmls {
|
||||
|
||||
let account_id = account.id;
|
||||
let mut success_count = 0;
|
||||
let mut failed_details: Vec<FailedEmlDetail> = Vec::new(); // Store failure details
|
||||
let mut failed_details: Vec<FailedItemDetail> = Vec::new(); // Store failure details
|
||||
|
||||
let total = request.emls.len();
|
||||
let mut index: usize = 0;
|
||||
@@ -127,7 +143,7 @@ impl ImportEmls {
|
||||
let error_msg =
|
||||
format!("Failed to decode base64 EML at index {}: {:?}", index, e);
|
||||
tracing::error!("{}", error_msg);
|
||||
failed_details.push(FailedEmlDetail {
|
||||
failed_details.push(FailedItemDetail {
|
||||
index,
|
||||
error_message: error_msg,
|
||||
});
|
||||
@@ -144,7 +160,7 @@ impl ImportEmls {
|
||||
index, size_mb,
|
||||
);
|
||||
tracing::warn!("{}", error_msg);
|
||||
failed_details.push(FailedEmlDetail {
|
||||
failed_details.push(FailedItemDetail {
|
||||
index,
|
||||
error_message: error_msg,
|
||||
});
|
||||
@@ -162,7 +178,7 @@ impl ImportEmls {
|
||||
index, e
|
||||
);
|
||||
tracing::error!("{}", error_msg);
|
||||
failed_details.push(FailedEmlDetail {
|
||||
failed_details.push(FailedItemDetail {
|
||||
index,
|
||||
error_message: error_msg,
|
||||
});
|
||||
@@ -178,8 +194,650 @@ impl ImportEmls {
|
||||
Ok(BatchEmlResult {
|
||||
total,
|
||||
success: success_count,
|
||||
duplicates: 0,
|
||||
failed: failed_count,
|
||||
failed_details, // Return the list of failure details
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── File upload import ──────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))]
|
||||
pub enum ImportStatus {
|
||||
Pending,
|
||||
Processing,
|
||||
Completed,
|
||||
Failed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct ImportProgress {
|
||||
pub import_id: String,
|
||||
pub status: ImportStatus,
|
||||
pub format: String,
|
||||
pub total: usize,
|
||||
pub success: usize,
|
||||
pub duplicates: usize,
|
||||
pub failed: usize,
|
||||
pub failed_details: Vec<FailedItemDetail>,
|
||||
}
|
||||
|
||||
static PROGRESS_STORE: std::sync::LazyLock<RwLock<HashMap<String, ImportProgress>>> =
|
||||
std::sync::LazyLock::new(|| RwLock::new(HashMap::new()));
|
||||
|
||||
pub fn get_import_progress(import_id: &str) -> Option<ImportProgress> {
|
||||
PROGRESS_STORE.read().ok()?.get(import_id).cloned()
|
||||
}
|
||||
|
||||
pub fn update_progress(import_id: &str, progress: ImportProgress) {
|
||||
if let Ok(mut store) = PROGRESS_STORE.write() {
|
||||
store.insert(import_id.to_string(), progress);
|
||||
}
|
||||
}
|
||||
|
||||
/// Check free disk space (in bytes) on the temp directory's filesystem.
|
||||
pub fn check_temp_disk_space() -> BichonResult<u64> {
|
||||
use sysinfo::Disks;
|
||||
let disks = Disks::new_with_refreshed_list();
|
||||
let temp_path = &DATA_DIR_MANAGER.temp_dir;
|
||||
// Use the canonical path so we can match mount points
|
||||
let canonical = std::fs::canonicalize(temp_path).unwrap_or_else(|_| temp_path.clone());
|
||||
for disk in disks.list() {
|
||||
if canonical.starts_with(disk.mount_point()) {
|
||||
return Ok(disk.available_space());
|
||||
}
|
||||
}
|
||||
// Fallback: if we can't find the mount point, report plenty of space
|
||||
Ok(u64::MAX)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum FileFormat {
|
||||
Eml,
|
||||
Mbox,
|
||||
Pst,
|
||||
}
|
||||
|
||||
pub fn detect_format(bytes: &[u8], file_name: &str) -> Option<FileFormat> {
|
||||
// PST files start with OLE2 compound document magic bytes
|
||||
if bytes.len() >= 8 && &bytes[..8] == b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1" {
|
||||
return Some(FileFormat::Pst);
|
||||
}
|
||||
|
||||
// MBOX files start with "From " (note the trailing space after From)
|
||||
if bytes.starts_with(b"From ") {
|
||||
// Double-check: look for a valid date after the first "From " line
|
||||
// MBOX format: "From sender@host DayOfWeek Mon DD HH:MM:SS YYYY"
|
||||
if let Some(first_newline) = bytes.iter().position(|&b| b == b'\n') {
|
||||
let from_line = std::str::from_utf8(&bytes[..first_newline]).unwrap_or("");
|
||||
let parts: Vec<&str> = from_line.split_whitespace().collect();
|
||||
if parts.len() >= 7 {
|
||||
return Some(FileFormat::Mbox);
|
||||
}
|
||||
}
|
||||
}
|
||||
// EML: starts with a header line or "Return-Path:", "Received:", "From:", "Date:", etc.
|
||||
// Or check extension
|
||||
if bytes.starts_with(b"Return-Path:")
|
||||
|| bytes.starts_with(b"Received:")
|
||||
|| bytes.starts_with(b"Date:")
|
||||
|| bytes.starts_with(b"From:")
|
||||
|| bytes.starts_with(b"Subject:")
|
||||
|| bytes.starts_with(b"To:")
|
||||
|| bytes.starts_with(b"Message-ID:")
|
||||
{
|
||||
return Some(FileFormat::Eml);
|
||||
}
|
||||
// Fallback: check file extension
|
||||
let lower = file_name.to_lowercase();
|
||||
if lower.ends_with(".eml") {
|
||||
Some(FileFormat::Eml)
|
||||
} else if lower.ends_with(".mbox") {
|
||||
Some(FileFormat::Mbox)
|
||||
} else if lower.ends_with(".pst") {
|
||||
Some(FileFormat::Pst)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Check whether `bytes` looks like a text file by inspecting the first chunk.
|
||||
/// Returns `true` if it passes, `false` if it appears to be binary (video, executable, etc.).
|
||||
///
|
||||
/// Email files (EML/MBOX) are text-based with printable ASCII, whitespace, and
|
||||
/// optional UTF-8. Binary files like video contain null bytes and high ratios of
|
||||
/// non-printable control characters.
|
||||
pub fn detect_text_file(bytes: &[u8]) -> bool {
|
||||
let check_len = bytes.len().min(8192);
|
||||
if check_len == 0 {
|
||||
return false;
|
||||
}
|
||||
let sample = &bytes[..check_len];
|
||||
|
||||
// Null bytes are a strong binary indicator
|
||||
if sample.contains(&0x00) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut printable = 0usize;
|
||||
let mut total = 0usize;
|
||||
|
||||
let mut i = 0;
|
||||
while i < sample.len() {
|
||||
total += 1;
|
||||
let b = sample[i];
|
||||
|
||||
if b.is_ascii_graphic() || b.is_ascii_whitespace() {
|
||||
// Printable ASCII + whitespace (space, tab, CR, LF)
|
||||
printable += 1;
|
||||
} else if b == 0x1b {
|
||||
// ESC — common in terminal sequences, rare in email
|
||||
// Count as printable to avoid false positives
|
||||
printable += 1;
|
||||
} else if b >= 0x80 {
|
||||
// UTF-8 continuation or multi-byte lead byte — allow.
|
||||
// Check that we have a valid UTF-8 sequence ahead.
|
||||
let seq_len = match b {
|
||||
b if b & 0xE0 == 0xC0 => 2,
|
||||
b if b & 0xF0 == 0xE0 => 3,
|
||||
b if b & 0xF8 == 0xF0 => 4,
|
||||
_ => 0,
|
||||
};
|
||||
if seq_len > 0 && i + seq_len <= sample.len() {
|
||||
let valid = std::str::from_utf8(&sample[i..i + seq_len]).is_ok();
|
||||
if valid {
|
||||
printable += 1;
|
||||
i += 1; // lead byte counted, continuations counted in loop
|
||||
}
|
||||
// if invalid, don't count as printable
|
||||
}
|
||||
// standalone continuation byte — not printable
|
||||
}
|
||||
// Other control characters (0x01-0x1F except whitespace/Esc) are not counted as printable
|
||||
|
||||
i += 1;
|
||||
}
|
||||
|
||||
// Require at least 90% printable characters
|
||||
printable as f64 / total as f64 >= 0.90
|
||||
}
|
||||
|
||||
/// Validate that the target account exists, is enabled, and is NoSync type.
|
||||
fn validate_import_account(account_id: u64) -> BichonResult<AccountModel> {
|
||||
let account = AccountModel::check_account_exists(account_id)?;
|
||||
if !account.enabled {
|
||||
return Err(raise_error!(
|
||||
"The account is disabled.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
if !matches!(account.account_type, AccountType::NoSync) {
|
||||
return Err(raise_error!(
|
||||
"Import is only allowed for NoSync accounts. IMAP accounts sync from the server.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
Ok(account)
|
||||
}
|
||||
|
||||
/// Resolve or create a mailbox/folder for the given account.
|
||||
pub(super) fn resolve_mailbox(account: &AccountModel, folder: &str) -> BichonResult<u64> {
|
||||
match account.account_type {
|
||||
AccountType::IMAP => {
|
||||
// Shouldn't reach here (validated above), but handle gracefully
|
||||
let all_mailboxes = MailBox::list_all(account.id)?;
|
||||
let mailbox = all_mailboxes.into_iter().find(|m| m.name == folder);
|
||||
match mailbox {
|
||||
Some(m) => Ok(m.id),
|
||||
None => Err(raise_error!(
|
||||
format!("Mail folder '{}' not found.", folder).into(),
|
||||
ErrorCode::ResourceNotFound
|
||||
)),
|
||||
}
|
||||
}
|
||||
AccountType::NoSync => {
|
||||
let mailbox = MailBox {
|
||||
id: create_hash(account.id, folder),
|
||||
account_id: account.id,
|
||||
name: folder.to_string(),
|
||||
delimiter: Some("/".to_string()),
|
||||
attributes: vec![Attribute {
|
||||
attr: AttributeEnum::Extension,
|
||||
extension: Some("CreatedByBichon".into()),
|
||||
}],
|
||||
exists: 0,
|
||||
unseen: None,
|
||||
uid_next: None,
|
||||
uid_validity: None,
|
||||
highest_uid: None,
|
||||
};
|
||||
let mailbox_id = mailbox.id;
|
||||
MailBox::batch_upsert(&[mailbox])?;
|
||||
Ok(mailbox_id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve or create a mailbox for a given account_id and folder name.
|
||||
/// Used by PST import to create per-folder mailboxes.
|
||||
pub fn resolve_mailbox_by_account_id(account_id: u64, folder: &str) -> BichonResult<u64> {
|
||||
let account = AccountModel::check_account_exists(account_id)?;
|
||||
resolve_mailbox(&account, folder)
|
||||
}
|
||||
|
||||
/// Process an uploaded file (EML or MBOX) and import into the given account/folder.
|
||||
/// This runs synchronously and should be spawned on a background thread.
|
||||
///
|
||||
/// For MBOX files, the file is memory-mapped via `memmap2` and messages are yielded
|
||||
/// one at a time — the full file is never loaded into RAM. Individual messages
|
||||
/// exceeding `MAX_SINGLE_EML_BYTES` (100 MB) are skipped.
|
||||
pub fn process_uploaded_file(
|
||||
import_id: &str,
|
||||
file_path: &Path,
|
||||
file_name: &str,
|
||||
account_id: u64,
|
||||
folder: &str,
|
||||
user_id: u64,
|
||||
) {
|
||||
let account = match validate_import_account(account_id) {
|
||||
Ok(a) => a,
|
||||
Err(e) => {
|
||||
let progress = ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Failed,
|
||||
format: "unknown".to_string(),
|
||||
total: 0,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![FailedItemDetail {
|
||||
index: 0,
|
||||
error_message: format!("Account validation failed: {:?}", e),
|
||||
}],
|
||||
};
|
||||
update_progress(import_id, progress.clone());
|
||||
history::save_import_history(user_id, account_id, folder, &progress);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mailbox_id = match resolve_mailbox(&account, folder) {
|
||||
Ok(id) => id,
|
||||
Err(e) => {
|
||||
let progress = ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Failed,
|
||||
format: "unknown".to_string(),
|
||||
total: 0,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![FailedItemDetail {
|
||||
index: 0,
|
||||
error_message: format!("Mailbox resolution failed: {:?}", e),
|
||||
}],
|
||||
};
|
||||
update_progress(import_id, progress.clone());
|
||||
history::save_import_history(user_id, account_id, folder, &progress);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Read a small prefix for format detection
|
||||
let format = match detect_format_from_file(file_path, file_name) {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
let progress = ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Failed,
|
||||
format: "unknown".to_string(),
|
||||
total: 0,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![FailedItemDetail {
|
||||
index: 0,
|
||||
error_message: format!("{:?}", e),
|
||||
}],
|
||||
};
|
||||
update_progress(import_id, progress.clone());
|
||||
history::save_import_history(user_id, account_id, folder, &progress);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
match format {
|
||||
FileFormat::Eml => process_eml_file(import_id, file_path, account_id, mailbox_id, user_id, folder),
|
||||
FileFormat::Mbox => process_mbox_file(import_id, file_path, account_id, mailbox_id, user_id, folder),
|
||||
FileFormat::Pst => process_pst_upload(import_id, file_path, account_id, mailbox_id, user_id, folder),
|
||||
}
|
||||
}
|
||||
|
||||
/// Detect format from a file by reading only the first few KB.
|
||||
fn detect_format_from_file(file_path: &Path, file_name: &str) -> BichonResult<FileFormat> {
|
||||
use std::io::Read;
|
||||
let mut file = std::fs::File::open(file_path).map_err(|e| {
|
||||
raise_error!(format!("Failed to open file: {}", e), ErrorCode::InternalError)
|
||||
})?;
|
||||
let mut buf = vec![0u8; 8192];
|
||||
let n = file.read(&mut buf).unwrap_or(0);
|
||||
buf.truncate(n);
|
||||
|
||||
detect_format(&buf, file_name).ok_or_else(|| {
|
||||
raise_error!(
|
||||
"Unknown file format. Supported: .eml, .mbox, .pst".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Process a single EML file. The file is at most `MAX_WEB_EML_BYTES` (100 MB),
|
||||
/// so reading it entirely is safe.
|
||||
fn process_eml_file(
|
||||
import_id: &str,
|
||||
file_path: &Path,
|
||||
account_id: u64,
|
||||
mailbox_id: u64,
|
||||
user_id: u64,
|
||||
folder: &str,
|
||||
) {
|
||||
let file_bytes = match std::fs::read(file_path) {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
fail_progress(import_id, "eml", &format!("Failed to read file: {}", e), user_id, account_id, folder);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let total = 1;
|
||||
update_progress(import_id, ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Processing,
|
||||
format: "eml".to_string(),
|
||||
total,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![],
|
||||
});
|
||||
|
||||
let (success_count, failed_details) = process_single_eml(&file_bytes, 0, account_id, mailbox_id);
|
||||
|
||||
// Clean up
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
|
||||
let final_progress = ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Completed,
|
||||
format: "eml".to_string(),
|
||||
total,
|
||||
success: success_count,
|
||||
duplicates: 0,
|
||||
failed: failed_details.len(),
|
||||
failed_details,
|
||||
};
|
||||
history::save_import_history(user_id, account_id, folder, &final_progress);
|
||||
update_progress(import_id, final_progress);
|
||||
}
|
||||
|
||||
/// Process an MBOX file using memory-mapped I/O. Messages are yielded one at a
|
||||
/// time by `MboxReader` — the full file is never loaded into RAM.
|
||||
fn process_mbox_file(
|
||||
import_id: &str,
|
||||
file_path: &Path,
|
||||
account_id: u64,
|
||||
mailbox_id: u64,
|
||||
user_id: u64,
|
||||
folder: &str,
|
||||
) {
|
||||
let mbox = match reader::MboxFile::from_file(file_path) {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
fail_progress(import_id, "mbox", &format!("Failed to open MBOX file: {}", e), user_id, account_id, folder);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// First pass: count total messages (MboxReader is lazy, so this is O(n) but cheap)
|
||||
let total = mbox.iter().count();
|
||||
|
||||
update_progress(import_id, ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Processing,
|
||||
format: "mbox".to_string(),
|
||||
total,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![],
|
||||
});
|
||||
|
||||
let mut success_count = 0usize;
|
||||
let mut failed_details: Vec<FailedItemDetail> = Vec::new();
|
||||
|
||||
for (index, entry) in mbox.iter().enumerate() {
|
||||
let eml_bytes = entry.data;
|
||||
|
||||
if eml_bytes.len() > MAX_SINGLE_EML_BYTES {
|
||||
let size_mb = eml_bytes.len() as f64 / 1024.0 / 1024.0;
|
||||
failed_details.push(FailedItemDetail {
|
||||
index,
|
||||
error_message: format!(
|
||||
"Email at index {} is {:.1} MB (limit {} MB). Skipping.",
|
||||
index,
|
||||
size_mb,
|
||||
MAX_SINGLE_EML_BYTES / 1024 / 1024
|
||||
),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
match futures::executor::block_on(extract_envelope_from_eml(eml_bytes, account_id, mailbox_id)) {
|
||||
Ok(_) => {
|
||||
success_count += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
failed_details.push(FailedItemDetail {
|
||||
index,
|
||||
error_message: format!("{:?}", e),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
// Update progress every 100 items
|
||||
if index % 100 == 0 || index == total - 1 {
|
||||
update_progress(import_id, ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Processing,
|
||||
format: "mbox".to_string(),
|
||||
total,
|
||||
success: success_count,
|
||||
duplicates: 0,
|
||||
failed: failed_details.len(),
|
||||
failed_details: failed_details.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up temp file (drop the mmap first — MboxFile owns it)
|
||||
drop(mbox);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
|
||||
let final_progress = ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Completed,
|
||||
format: "mbox".to_string(),
|
||||
total,
|
||||
success: success_count,
|
||||
duplicates: 0,
|
||||
failed: failed_details.len(),
|
||||
failed_details,
|
||||
};
|
||||
history::save_import_history(user_id, account_id, folder, &final_progress);
|
||||
update_progress(import_id, final_progress);
|
||||
}
|
||||
|
||||
/// Process a single EML byte slice and return (success_count, failed_details).
|
||||
fn process_single_eml(
|
||||
eml_bytes: &[u8],
|
||||
index: usize,
|
||||
account_id: u64,
|
||||
mailbox_id: u64,
|
||||
) -> (usize, Vec<FailedItemDetail>) {
|
||||
if eml_bytes.len() > MAX_SINGLE_EML_BYTES {
|
||||
let size_mb = eml_bytes.len() as f64 / 1024.0 / 1024.0;
|
||||
return (0, vec![FailedItemDetail {
|
||||
index,
|
||||
error_message: format!(
|
||||
"Email is {:.1} MB (limit {} MB). Skipping.",
|
||||
size_mb,
|
||||
MAX_SINGLE_EML_BYTES / 1024 / 1024
|
||||
),
|
||||
}]);
|
||||
}
|
||||
|
||||
match futures::executor::block_on(extract_envelope_from_eml(eml_bytes, account_id, mailbox_id)) {
|
||||
Ok(_) => (1, vec![]),
|
||||
Err(e) => (0, vec![FailedItemDetail {
|
||||
index,
|
||||
error_message: format!("{:?}", e),
|
||||
}]),
|
||||
}
|
||||
}
|
||||
|
||||
/// Process a PST file uploaded via the web UI.
|
||||
/// Two-pass approach: count messages first, then process with periodic progress updates.
|
||||
fn process_pst_upload(
|
||||
import_id: &str,
|
||||
file_path: &Path,
|
||||
account_id: u64,
|
||||
_mailbox_id: u64, // ignored; PST creates its own mailboxes per folder
|
||||
user_id: u64,
|
||||
folder: &str,
|
||||
) {
|
||||
// Pass 1: count total messages
|
||||
let total = match pst::count_pst_messages(file_path) {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
update_progress(import_id, ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Processing,
|
||||
format: "pst".to_string(),
|
||||
total,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![],
|
||||
});
|
||||
|
||||
// Pass 2: process messages with progress updates
|
||||
let mut success_count: usize = 0;
|
||||
let mut failed_details: Vec<FailedItemDetail> = Vec::new();
|
||||
let mut index: usize = 0;
|
||||
|
||||
let pst_store = match outlook_pst::open_store(file_path) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let ipm_sub_tree = match pst_store.properties().ipm_sub_tree_entry_id() {
|
||||
Ok(id) => id,
|
||||
Err(e) => {
|
||||
fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let ipm_subtree_folder = match pst_store.open_folder(&ipm_sub_tree) {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Progress callback: update progress every 50 messages
|
||||
let import_id = import_id.to_string();
|
||||
let format_str = "pst".to_string();
|
||||
pst::process_folder_with_progress(
|
||||
&ipm_subtree_folder,
|
||||
"", // parent_path starts empty
|
||||
account_id,
|
||||
total, // pass pre-counted total for accurate progress
|
||||
&mut success_count,
|
||||
&mut failed_details,
|
||||
&mut index,
|
||||
&|processed, actual_failed| {
|
||||
update_progress(&import_id, ImportProgress {
|
||||
import_id: import_id.clone(),
|
||||
status: ImportStatus::Processing,
|
||||
format: format_str.clone(),
|
||||
total,
|
||||
success: processed - actual_failed,
|
||||
duplicates: 0,
|
||||
failed: actual_failed,
|
||||
failed_details: vec![],
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
// Clean up temp file
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
|
||||
let final_progress = ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Completed,
|
||||
format: "pst".to_string(),
|
||||
total,
|
||||
success: success_count,
|
||||
duplicates: 0,
|
||||
failed: failed_details.len(),
|
||||
failed_details,
|
||||
};
|
||||
history::save_import_history(user_id, account_id, folder, &final_progress);
|
||||
update_progress(&import_id, final_progress);
|
||||
}
|
||||
|
||||
/// Record a fatal failure and save history.
|
||||
fn fail_progress(
|
||||
import_id: &str,
|
||||
format: &str,
|
||||
message: &str,
|
||||
user_id: u64,
|
||||
account_id: u64,
|
||||
folder: &str,
|
||||
) {
|
||||
let progress = ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Failed,
|
||||
format: format.to_string(),
|
||||
total: 0,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![FailedItemDetail {
|
||||
index: 0,
|
||||
error_message: message.to_string(),
|
||||
}],
|
||||
};
|
||||
update_progress(import_id, progress.clone());
|
||||
history::save_import_history(user_id, account_id, folder, &progress);
|
||||
}
|
||||
|
||||
@@ -16,8 +16,6 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
use compressed_rtf::*;
|
||||
use outlook_pst::ltp::prop_context::PropertyValue;
|
||||
|
||||
pub fn decode_subject(value: &PropertyValue) -> Option<String> {
|
||||
@@ -60,5 +58,5 @@ pub fn decode_html_body(buffer: &[u8], code_page: u16) -> Option<String> {
|
||||
}
|
||||
|
||||
pub fn decode_rtf_compressed(buffer: &[u8]) -> Option<String> {
|
||||
decompress_rtf(buffer).ok()
|
||||
compressed_rtf::decompress_rtf(buffer).ok()
|
||||
}
|
||||
486
crates/core/src/import/pst/mod.rs
Normal file
486
crates/core/src/import/pst/mod.rs
Normal file
@@ -0,0 +1,486 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use crate::base64_encode_url_safe;
|
||||
use crate::envelope::extractor::extract_envelope_from_eml;
|
||||
use chrono::{DateTime, TimeZone, Utc};
|
||||
use mail_send::mail_builder::headers::text::Text;
|
||||
use mail_send::mail_builder::MessageBuilder;
|
||||
use outlook_pst::ltp::prop_context::PropertyValue;
|
||||
use outlook_pst::messaging::attachment::AttachmentProperties;
|
||||
use outlook_pst::messaging::folder::Folder;
|
||||
use outlook_pst::messaging::message::{Message, MessageProperties};
|
||||
use outlook_pst::ndb::node_id::NodeId;
|
||||
use std::rc::Rc;
|
||||
|
||||
mod encoding;
|
||||
|
||||
/// Convert a PST Message into a base64-encoded EML string.
|
||||
pub fn build_eml_base64(message: Rc<dyn Message>) -> Option<String> {
|
||||
let properties = message.properties();
|
||||
|
||||
let mut builder = MessageBuilder::new();
|
||||
if let Some(sub) = extract_subject(properties) {
|
||||
builder = builder.subject(sub);
|
||||
}
|
||||
if let Some(mid) = extract_string_property(properties, 0x1035) {
|
||||
builder = builder.message_id(mid);
|
||||
}
|
||||
if let Some(irt) = extract_string_property(properties, 0x1042) {
|
||||
builder = builder.in_reply_to(irt);
|
||||
}
|
||||
|
||||
if let Some(refs) = extract_string_property(properties, 0x1039) {
|
||||
builder = builder.header("References", Text::new(refs));
|
||||
}
|
||||
|
||||
if let Some(cid_val) = properties.get(0x3013) {
|
||||
if let PropertyValue::Binary(bin) = cid_val {
|
||||
builder = builder.header(
|
||||
"X-Bichon-Conversation-ID",
|
||||
Text::new(hex::encode(bin.buffer())),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let from = extract_string_property(properties, 0x5D01)
|
||||
.or_else(|| extract_string_property(properties, 0x5D02))
|
||||
.or_else(|| extract_string_property(properties, 0x0C1F));
|
||||
|
||||
if let Some(f) = from {
|
||||
builder = builder.from(f);
|
||||
}
|
||||
|
||||
if let Some(filetime) = extract_i64_property(properties, &[0x0039, 0x0E06]) {
|
||||
let dt = filetime_to_datetime(filetime).timestamp();
|
||||
builder = builder.date(dt);
|
||||
}
|
||||
|
||||
let (to, cc, bcc) = extract_recipients_list(&message);
|
||||
if !to.is_empty() {
|
||||
builder = builder.to(to.iter().map(|s| s.as_str()).collect::<Vec<_>>());
|
||||
}
|
||||
if !cc.is_empty() {
|
||||
builder = builder.cc(cc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
|
||||
}
|
||||
if !bcc.is_empty() {
|
||||
builder = builder.bcc(bcc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
if let Some(html) = extract_html(properties) {
|
||||
builder = builder.html_body(html);
|
||||
}
|
||||
|
||||
if let Some(text) = extract_text(properties) {
|
||||
builder = builder.text_body(text);
|
||||
}
|
||||
|
||||
if let Some(attachment_table) = message.attachment_table() {
|
||||
for row in attachment_table.rows_matrix() {
|
||||
let node_id = NodeId::from(u32::from(row.id()));
|
||||
if let Ok(attachment) = message.clone().read_attachment(node_id, None) {
|
||||
let att_props = attachment.properties();
|
||||
let name = extract_attachment_string_property(att_props, 0x3707);
|
||||
let mime = extract_attachment_string_property(att_props, 0x370E)
|
||||
.unwrap_or_else(|| "application/octet-stream".into());
|
||||
let cid = extract_attachment_string_property(att_props, 0x3712);
|
||||
let is_inline = att_props
|
||||
.get(0x3714)
|
||||
.and_then(|val| {
|
||||
if let PropertyValue::Integer32(f) = val {
|
||||
Some(f)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.map(|flag| (flag & 0x4) != 0)
|
||||
.unwrap_or(false);
|
||||
|
||||
if let Some(PropertyValue::Binary(bin)) = att_props.get(0x3701) {
|
||||
let data = bin.buffer().to_vec();
|
||||
let file_name = name.unwrap_or_else(|| "unnamed_attachment".to_string());
|
||||
|
||||
if is_inline && cid.is_some() {
|
||||
let content_id = cid.unwrap();
|
||||
builder = builder.inline(mime, content_id, data);
|
||||
} else {
|
||||
builder = builder.attachment(mime, file_name, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match builder.write_to_vec() {
|
||||
Ok(eml_vec) => Some(base64_encode_url_safe!(eml_vec)),
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to generate EML from PST message: {:?}", e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn filetime_to_datetime(filetime: i64) -> DateTime<Utc> {
|
||||
let unix_secs = (filetime / 10_000_000) - 11_644_473_600;
|
||||
let nsecs = (filetime % 10_000_000) * 100;
|
||||
Utc.timestamp_opt(unix_secs, nsecs as u32).unwrap()
|
||||
}
|
||||
|
||||
fn extract_recipients_list(message: &Rc<dyn Message>) -> (Vec<String>, Vec<String>, Vec<String>) {
|
||||
let mut to = Vec::new();
|
||||
let mut cc = Vec::new();
|
||||
let mut bcc = Vec::new();
|
||||
|
||||
let recipient_table = message.recipient_table();
|
||||
if let Some(recipient_table) = recipient_table {
|
||||
let context = recipient_table.context();
|
||||
for row in recipient_table.rows_matrix() {
|
||||
if let Ok(cols) = row.columns(context) {
|
||||
let mut r_type = 0;
|
||||
let mut email = String::new();
|
||||
|
||||
for (col, val) in context.columns().iter().zip(cols) {
|
||||
let prop_val = val
|
||||
.as_ref()
|
||||
.and_then(|v| recipient_table.read_column(v, col.prop_type()).ok());
|
||||
match col.prop_id() {
|
||||
0x0C15 => {
|
||||
if let Some(PropertyValue::Integer32(t)) = prop_val {
|
||||
r_type = t;
|
||||
}
|
||||
}
|
||||
0x39FE | 0x3003 => {
|
||||
if let Some(s) = prop_val.and_then(|v| extract_string(&v)) {
|
||||
email = s;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if !email.is_empty() {
|
||||
match r_type {
|
||||
1 => to.push(email),
|
||||
2 => cc.push(email),
|
||||
3 => bcc.push(email),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let receiver = extract_string_property(message.properties(), 0x0076);
|
||||
if let Some(receiver) = receiver {
|
||||
to.push(receiver);
|
||||
}
|
||||
}
|
||||
(to, cc, bcc)
|
||||
}
|
||||
|
||||
fn extract_subject(props: &MessageProperties) -> Option<String> {
|
||||
props.get(0x0037).and_then(|val| encoding::decode_subject(val))
|
||||
}
|
||||
|
||||
fn extract_string_property(properties: &MessageProperties, prop_id: u16) -> Option<String> {
|
||||
properties
|
||||
.get(prop_id)
|
||||
.and_then(|value| extract_string(value))
|
||||
}
|
||||
|
||||
fn extract_attachment_string_property(
|
||||
properties: &AttachmentProperties,
|
||||
prop_id: u16,
|
||||
) -> Option<String> {
|
||||
properties
|
||||
.get(prop_id)
|
||||
.and_then(|value| extract_string(value))
|
||||
}
|
||||
|
||||
fn extract_string(value: &PropertyValue) -> Option<String> {
|
||||
match value {
|
||||
PropertyValue::String8(value) => Some(value.to_string()),
|
||||
PropertyValue::Unicode(value) => Some(value.to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_text(properties: &MessageProperties) -> Option<String> {
|
||||
properties.get(0x1000).and_then(extract_string).or_else(|| {
|
||||
properties.get(0x1009).and_then(|value| match value {
|
||||
PropertyValue::Binary(value) => encoding::decode_rtf_compressed(value.buffer()),
|
||||
_ => None,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_html(properties: &MessageProperties) -> Option<String> {
|
||||
properties.get(0x1013).and_then(|value| match value {
|
||||
PropertyValue::Binary(value) => {
|
||||
let code_page = properties
|
||||
.get(0x3FDE)
|
||||
.and_then(|v| {
|
||||
if let PropertyValue::Integer32(cpid) = v {
|
||||
Some(*cpid as u16)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or(65001);
|
||||
encoding::decode_html_body(value.buffer(), code_page)
|
||||
}
|
||||
PropertyValue::String8(value) => Some(value.to_string()),
|
||||
PropertyValue::Unicode(value) => Some(value.to_string()),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_i64_property(properties: &MessageProperties, prop_ids: &[u16]) -> Option<i64> {
|
||||
for &prop_id in prop_ids {
|
||||
if let Some(PropertyValue::Time(value)) = properties.get(prop_id) {
|
||||
return Some(*value);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Open a PST file and count total messages across all folders.
|
||||
/// Called from the web upload flow to get the total before processing.
|
||||
pub fn count_pst_messages(pst_path: &std::path::Path) -> crate::error::BichonResult<usize> {
|
||||
let pst_store = outlook_pst::open_store(pst_path).map_err(|e| {
|
||||
crate::raise_error!(
|
||||
format!("Failed to open PST file: {:?}", e),
|
||||
crate::error::code::ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
|
||||
let ipm_sub_tree = pst_store.properties().ipm_sub_tree_entry_id().map_err(|e| {
|
||||
crate::raise_error!(
|
||||
format!("Could not find IPM_SUBTREE in PST: {:?}", e),
|
||||
crate::error::code::ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
|
||||
let ipm_subtree_folder = pst_store.open_folder(&ipm_sub_tree).map_err(|e| {
|
||||
crate::raise_error!(
|
||||
format!("Failed to open root mailbox folder: {:?}", e),
|
||||
crate::error::code::ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(count_folder_messages(&ipm_subtree_folder))
|
||||
}
|
||||
|
||||
fn count_folder_messages(folder: &Rc<dyn Folder>) -> usize {
|
||||
let mut count = 0usize;
|
||||
|
||||
if let Some(contents_table) = folder.contents_table() {
|
||||
for row in contents_table.rows_matrix() {
|
||||
let store = folder.store().clone();
|
||||
let entry_id = match store
|
||||
.properties()
|
||||
.make_entry_id(NodeId::from(u32::from(row.id())))
|
||||
{
|
||||
Ok(id) => id,
|
||||
Err(_) => continue,
|
||||
};
|
||||
|
||||
if store.open_message(&entry_id, None).is_ok() {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(hierarchy_table) = folder.hierarchy_table() {
|
||||
for row in hierarchy_table.rows_matrix() {
|
||||
let node = NodeId::from(u32::from(row.id()));
|
||||
if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
|
||||
if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
|
||||
count += count_folder_messages(&sub_folder);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
count
|
||||
}
|
||||
|
||||
/// Walk all folders and process messages, calling the progress callback
|
||||
/// every 50 messages. Used by the web upload flow.
|
||||
pub fn process_folder_with_progress<F>(
|
||||
folder: &Rc<dyn Folder>,
|
||||
parent_path: &str,
|
||||
account_id: u64,
|
||||
total: usize,
|
||||
success_count: &mut usize,
|
||||
failed_details: &mut Vec<super::FailedItemDetail>,
|
||||
index: &mut usize,
|
||||
progress_cb: &F,
|
||||
) where
|
||||
F: Fn(usize, usize), // (processed, failed)
|
||||
{
|
||||
process_folder_with_progress_inner(
|
||||
folder,
|
||||
parent_path,
|
||||
account_id,
|
||||
total,
|
||||
success_count,
|
||||
failed_details,
|
||||
index,
|
||||
progress_cb,
|
||||
);
|
||||
}
|
||||
|
||||
fn process_folder_with_progress_inner<F>(
|
||||
folder: &Rc<dyn Folder>,
|
||||
parent_path: &str,
|
||||
account_id: u64,
|
||||
total: usize,
|
||||
success_count: &mut usize,
|
||||
failed_details: &mut Vec<super::FailedItemDetail>,
|
||||
index: &mut usize,
|
||||
progress_cb: &F,
|
||||
) where
|
||||
F: Fn(usize, usize),
|
||||
{
|
||||
let folder_name = folder
|
||||
.properties()
|
||||
.display_name()
|
||||
.unwrap_or_else(|_| "Unknown".to_string());
|
||||
|
||||
let mail_folder = if parent_path.is_empty() {
|
||||
folder_name
|
||||
} else {
|
||||
format!("{}/{}", parent_path, folder_name)
|
||||
};
|
||||
|
||||
tracing::debug!("Processing PST folder: {}", mail_folder);
|
||||
|
||||
let mailbox_id = match super::resolve_mailbox_by_account_id(account_id, &mail_folder) {
|
||||
Ok(id) => id,
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to resolve mailbox '{}': {:?}", mail_folder, e);
|
||||
// Still recurse into sub-folders even if this folder's mailbox creation fails
|
||||
if let Some(hierarchy_table) = folder.hierarchy_table() {
|
||||
for row in hierarchy_table.rows_matrix() {
|
||||
let node = NodeId::from(u32::from(row.id()));
|
||||
if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
|
||||
if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
|
||||
process_folder_with_progress_inner(
|
||||
&sub_folder,
|
||||
&mail_folder,
|
||||
account_id,
|
||||
total,
|
||||
success_count,
|
||||
failed_details,
|
||||
index,
|
||||
progress_cb,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mut batch_size = 0usize;
|
||||
|
||||
if let Some(contents_table) = folder.contents_table() {
|
||||
for row in contents_table.rows_matrix() {
|
||||
let store = folder.store().clone();
|
||||
|
||||
let entry_id = match store
|
||||
.properties()
|
||||
.make_entry_id(NodeId::from(u32::from(row.id())))
|
||||
{
|
||||
Ok(id) => id,
|
||||
Err(e) => {
|
||||
tracing::warn!("Skip PST row {}: {:?}", row.unique(), e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
match store.open_message(&entry_id, None) {
|
||||
Ok(message) => match build_eml_base64(message) {
|
||||
Some(base64_eml) => {
|
||||
let decoded = match crate::base64_decode_url_safe!(base64_eml.as_bytes()) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(e) => {
|
||||
failed_details.push(super::FailedItemDetail {
|
||||
index: *index,
|
||||
error_message: format!(
|
||||
"Failed to decode base64 EML at index {}: {:?}",
|
||||
*index, e
|
||||
),
|
||||
});
|
||||
*index += 1;
|
||||
batch_size += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
match futures::executor::block_on(
|
||||
extract_envelope_from_eml(&decoded, account_id, mailbox_id)
|
||||
) {
|
||||
Ok(_) => {
|
||||
*success_count += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
failed_details.push(super::FailedItemDetail {
|
||||
index: *index,
|
||||
error_message: format!("{:?}", e),
|
||||
});
|
||||
}
|
||||
};
|
||||
*index += 1;
|
||||
batch_size += 1;
|
||||
}
|
||||
None => {}
|
||||
},
|
||||
Err(e) => {
|
||||
tracing::warn!("Open PST message error: {:?}", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Report progress every 50 messages
|
||||
if batch_size % 50 == 0 {
|
||||
progress_cb(*success_count + failed_details.len(), failed_details.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(hierarchy_table) = folder.hierarchy_table() {
|
||||
for row in hierarchy_table.rows_matrix() {
|
||||
let node = NodeId::from(u32::from(row.id()));
|
||||
if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
|
||||
if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
|
||||
process_folder_with_progress_inner(
|
||||
&sub_folder,
|
||||
&mail_folder,
|
||||
account_id,
|
||||
total,
|
||||
success_count,
|
||||
failed_details,
|
||||
index,
|
||||
progress_cb,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,8 @@ use std::fs;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
/// Memory-mapped MBOX file. Messages are yielded one at a time without
|
||||
/// loading the entire file into RAM.
|
||||
pub struct MboxFile {
|
||||
map: Mmap,
|
||||
}
|
||||
@@ -127,10 +129,6 @@ impl<'a> Iterator for MboxReader<'a> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use mail_parser::MessageParser;
|
||||
|
||||
use crate::mbox::gmail::determine_folder;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn collect_entries(data: &[u8]) -> Vec<&[u8]> {
|
||||
@@ -144,6 +142,7 @@ mod tests {
|
||||
let e = collect_entries(data);
|
||||
assert_eq!(e, vec![b"mail1\n", b"mail2\n"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_trailing_newline() {
|
||||
let data = b"From a\nmail1";
|
||||
@@ -204,22 +203,4 @@ mod tests {
|
||||
let e = collect_entries(&data);
|
||||
assert_eq!(e.len(), 1000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test11() {
|
||||
let mbox = MboxFile::from_file(Path::new("e:\\test.mbox")).unwrap();
|
||||
|
||||
for e in mbox.iter() {
|
||||
let body = e.data;
|
||||
|
||||
let message = MessageParser::new().parse(body).unwrap();
|
||||
let labels = message.header("X-Gmail-Labels").unwrap().as_text().unwrap();
|
||||
//println!("offset={} X-Gmail-Labels={:?}", e.offset, labels);
|
||||
println!(
|
||||
"X-Gmail-Labels={:?}, determine_folder={}",
|
||||
labels,
|
||||
determine_folder(labels)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -43,6 +43,10 @@ pub struct EmailSearchFilter {
|
||||
pub to: Option<String>,
|
||||
pub cc: Option<String>,
|
||||
pub bcc: Option<String>,
|
||||
/// Matches if the address appears in `to`, `cc`, or `bcc` (OR semantics).
|
||||
pub any_recipient: Option<String>,
|
||||
/// Matches if the address appears in `from`, `to`, `cc`, or `bcc` (OR semantics).
|
||||
pub any_participant: Option<String>,
|
||||
pub since: Option<i64>,
|
||||
pub before: Option<i64>,
|
||||
/// Lower bound (inclusive) on the IMAP server INTERNALDATE timestamp.
|
||||
|
||||
@@ -404,6 +404,7 @@ impl NewIndexWriter {
|
||||
regular_attachment_count: attachment_docs.len(),
|
||||
tags: None,
|
||||
account_email: None,
|
||||
account_name: None,
|
||||
mailbox_name: None,
|
||||
content_hash: email_content_hash,
|
||||
};
|
||||
|
||||
@@ -20,6 +20,7 @@ use crate::error::code::ErrorCode;
|
||||
use crate::error::BichonResult;
|
||||
use crate::oauth2::{entity::OAuth2, pending::OAuth2PendingEntity, token::OAuth2AccessToken};
|
||||
use crate::settings::proxy::Proxy;
|
||||
use crate::utils::net::parse_proxy_url;
|
||||
use crate::{decrypt, encrypt, raise_error};
|
||||
use oauth2::{
|
||||
basic::BasicClient, AuthUrl, AuthorizationCode, ClientId, ClientSecret, CsrfToken,
|
||||
@@ -265,14 +266,12 @@ impl OAuth2Flow {
|
||||
fn build_http_client(use_proxy: Option<u64>) -> BichonResult<reqwest::Client> {
|
||||
if let Some(proxy_id) = use_proxy {
|
||||
let proxy = Proxy::get(proxy_id)?;
|
||||
let proxy_url = parse_proxy_url(&proxy.url)?.standard_url();
|
||||
return oauth2::reqwest::ClientBuilder::new()
|
||||
.redirect(oauth2::reqwest::redirect::Policy::none())
|
||||
.proxy(reqwest::Proxy::all(&proxy.url).map_err(|e| {
|
||||
.proxy(reqwest::Proxy::all(&proxy_url).map_err(|_| {
|
||||
raise_error!(
|
||||
format!(
|
||||
"Failed to configure SOCKS5 proxy ({}): {:#?}. Please check",
|
||||
&proxy.url, e
|
||||
),
|
||||
"Failed to configure proxy. Please check the proxy configuration.".into(),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?)
|
||||
|
||||
@@ -308,6 +308,56 @@ pub struct Settings {
|
||||
help = "Enable SMTP authentication requirement"
|
||||
)]
|
||||
pub bichon_smtp_auth_required: bool,
|
||||
|
||||
/// Enable OIDC-based Single Sign-On (Pro/Enterprise feature).
|
||||
#[clap(long, default_value = "false", env, help = "Enable OpenID Connect SSO")]
|
||||
pub bichon_oidc_enabled: bool,
|
||||
|
||||
/// OIDC issuer URL (e.g. https://keycloak.example.com/realms/myorg).
|
||||
#[clap(long, env, help = "OpenID Connect issuer URL")]
|
||||
pub bichon_oidc_issuer_url: Option<String>,
|
||||
|
||||
/// OIDC client ID registered with the IdP.
|
||||
#[clap(long, env, help = "OpenID Connect client ID")]
|
||||
pub bichon_oidc_client_id: Option<String>,
|
||||
|
||||
/// OIDC client secret registered with the IdP.
|
||||
#[clap(long, env, help = "OpenID Connect client secret")]
|
||||
pub bichon_oidc_client_secret: Option<String>,
|
||||
|
||||
/// OIDC redirect URI (must match what's registered with the IdP).
|
||||
#[clap(long, env, help = "OpenID Connect redirect URI")]
|
||||
pub bichon_oidc_redirect_uri: Option<String>,
|
||||
|
||||
/// Maximum HTTP request body size in MB for file uploads (default: 1100 MB).
|
||||
/// Requests exceeding this limit are rejected at the framework level before
|
||||
/// the application reads the body, preventing memory exhaustion attacks.
|
||||
#[clap(
|
||||
long,
|
||||
default_value = "1100",
|
||||
env,
|
||||
help = "Maximum HTTP request body size in MB for file uploads"
|
||||
)]
|
||||
pub bichon_upload_body_limit_mb: u64,
|
||||
|
||||
/// Maximum per-file size in MB for MBOX uploads via the web UI (default: 1024 MB = 1 GB).
|
||||
/// Individual EML files are always capped at 100 MB regardless of this setting.
|
||||
#[clap(
|
||||
long,
|
||||
default_value = "1024",
|
||||
env,
|
||||
help = "Maximum per-file size in MB for MBOX uploads via the web UI"
|
||||
)]
|
||||
pub bichon_web_mbox_upload_limit_mb: u64,
|
||||
|
||||
/// Maximum per-file size in MB for PST uploads via the web UI (default: 2048 MB = 2 GB).
|
||||
#[clap(
|
||||
long,
|
||||
default_value = "2048",
|
||||
env,
|
||||
help = "Maximum per-file size in MB for PST uploads via the web UI"
|
||||
)]
|
||||
pub bichon_web_pst_upload_limit_mb: u64,
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
@@ -317,9 +367,8 @@ impl Settings {
|
||||
// rejects it, fall back to parsing with only the binary name so that
|
||||
// the settings come entirely from environment variables.
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let s = Self::try_parse_from(&args).unwrap_or_else(|_| {
|
||||
Self::parse_from(std::iter::once(args[0].clone()))
|
||||
});
|
||||
let s = Self::try_parse_from(&args)
|
||||
.unwrap_or_else(|_| Self::parse_from(std::iter::once(args[0].clone())));
|
||||
if s.bichon_encrypt_password.is_none() && s.bichon_encrypt_password_file.is_none() {
|
||||
panic!(
|
||||
"One of --bichon_encrypt_password or --bichon_encrypt_password_file has to be set"
|
||||
|
||||
@@ -60,6 +60,17 @@ pub struct SystemConfigurations {
|
||||
pub bichon_smtp_auth_required: bool,
|
||||
pub bichon_smtp_tls_key_path: Option<String>,
|
||||
pub bichon_smtp_tls_cert_path: Option<String>,
|
||||
|
||||
pub bichon_oidc_enabled: bool,
|
||||
pub bichon_oidc_issuer_url: Option<String>,
|
||||
pub bichon_oidc_client_id: Option<String>,
|
||||
pub bichon_oidc_redirect_uri: Option<String>,
|
||||
|
||||
pub bichon_upload_body_limit_mb: u64,
|
||||
|
||||
pub bichon_web_mbox_upload_limit_mb: u64,
|
||||
|
||||
pub bichon_web_pst_upload_limit_mb: u64,
|
||||
}
|
||||
|
||||
impl From<&Settings> for SystemConfigurations {
|
||||
@@ -94,6 +105,13 @@ impl From<&Settings> for SystemConfigurations {
|
||||
bichon_smtp_auth_required: s.bichon_smtp_auth_required,
|
||||
bichon_smtp_tls_key_path: s.bichon_smtp_tls_key_path.clone(),
|
||||
bichon_smtp_tls_cert_path: s.bichon_smtp_tls_cert_path.clone(),
|
||||
bichon_oidc_enabled: s.bichon_oidc_enabled,
|
||||
bichon_oidc_issuer_url: s.bichon_oidc_issuer_url.clone(),
|
||||
bichon_oidc_client_id: s.bichon_oidc_client_id.clone(),
|
||||
bichon_oidc_redirect_uri: s.bichon_oidc_redirect_uri.clone(),
|
||||
bichon_upload_body_limit_mb: s.bichon_upload_body_limit_mb,
|
||||
bichon_web_mbox_upload_limit_mb: s.bichon_web_mbox_upload_limit_mb,
|
||||
bichon_web_pst_upload_limit_mb: s.bichon_web_pst_upload_limit_mb,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
|
||||
//use poem_openapi::Object;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{error::Error, time::Duration};
|
||||
|
||||
use crate::{
|
||||
database::{
|
||||
@@ -26,9 +27,30 @@ use crate::{
|
||||
},
|
||||
error::{code::ErrorCode, BichonResult},
|
||||
id, raise_error, utc_now,
|
||||
utils::net::parse_proxy_addr,
|
||||
utils::net::parse_proxy_url,
|
||||
};
|
||||
|
||||
const PROXY_TEST_TIMEOUT: Duration = Duration::from_secs(8);
|
||||
const GEO_PROVIDERS: &[GeoProvider] = &[
|
||||
GeoProvider {
|
||||
name: "ip-api.com",
|
||||
url: "http://ip-api.com/json/?fields=status,message,query,country,countryCode,regionName,city,isp,timezone,lat,lon",
|
||||
},
|
||||
GeoProvider {
|
||||
name: "ipwho.is",
|
||||
url: "https://ipwho.is/",
|
||||
},
|
||||
GeoProvider {
|
||||
name: "ipapi.co",
|
||||
url: "https://ipapi.co/json/",
|
||||
},
|
||||
];
|
||||
|
||||
struct GeoProvider {
|
||||
name: &'static str,
|
||||
url: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct Proxy {
|
||||
@@ -45,6 +67,16 @@ pub struct Proxy {
|
||||
pub updated_at: i64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct ProxyTestResult {
|
||||
pub ip: Option<String>,
|
||||
pub country: Option<String>,
|
||||
pub region: Option<String>,
|
||||
pub city: Option<String>,
|
||||
pub isp: Option<String>,
|
||||
}
|
||||
|
||||
impl MemDbModel for Proxy {
|
||||
fn collection() -> &'static str {
|
||||
"proxies"
|
||||
@@ -85,9 +117,10 @@ impl Proxy {
|
||||
|
||||
pub fn update(id: u64, url: String) -> BichonResult<()> {
|
||||
update_impl(DB_MANAGER.db(), &id.to_string(), move |current: Proxy| {
|
||||
let mut updated = current.clone();
|
||||
let mut updated = current;
|
||||
updated.url = url;
|
||||
updated.updated_at = utc_now!();
|
||||
updated.validate()?;
|
||||
Ok(updated)
|
||||
})?;
|
||||
Ok(())
|
||||
@@ -98,11 +131,185 @@ impl Proxy {
|
||||
insert_impl(DB_MANAGER.db(), self.to_owned())
|
||||
}
|
||||
|
||||
/// Validate that the URL is a valid SOCKS5 proxy URL.
|
||||
/// Validate that the URL is a valid proxy URL.
|
||||
pub fn validate(&self) -> BichonResult<()> {
|
||||
parse_proxy_addr(&self.url)?;
|
||||
parse_proxy_url(&self.url)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn test_connectivity(&self) -> BichonResult<ProxyTestResult> {
|
||||
test_proxy_url(&self.url).await
|
||||
}
|
||||
|
||||
pub async fn test(id: u64) -> BichonResult<ProxyTestResult> {
|
||||
let proxy = Self::get(id)?;
|
||||
proxy.test_connectivity().await
|
||||
}
|
||||
}
|
||||
|
||||
async fn test_proxy_url(url: &str) -> BichonResult<ProxyTestResult> {
|
||||
let proxy_url = parse_proxy_url(url)?.standard_url();
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(PROXY_TEST_TIMEOUT)
|
||||
.proxy(reqwest::Proxy::all(&proxy_url).map_err(|_| {
|
||||
raise_error!(
|
||||
"Failed to configure proxy. Please check the proxy configuration.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})?)
|
||||
.build()
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
let mut last_error = None;
|
||||
for provider in GEO_PROVIDERS {
|
||||
match test_geo_provider(&client, provider).await {
|
||||
Ok(result) => return Ok(result),
|
||||
Err(err) => last_error = Some(err.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
Err(raise_error!(
|
||||
format!(
|
||||
"Proxy check failed with all geo providers: {}",
|
||||
last_error.unwrap_or_else(|| "unknown error".into())
|
||||
),
|
||||
ErrorCode::NetworkError
|
||||
))
|
||||
}
|
||||
|
||||
async fn test_geo_provider(
|
||||
client: &reqwest::Client,
|
||||
provider: &GeoProvider,
|
||||
) -> BichonResult<ProxyTestResult> {
|
||||
let value = client
|
||||
.get(provider.url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
raise_error!(
|
||||
proxy_request_error_message(
|
||||
&format!("Proxy check request failed via {}", provider.name),
|
||||
&e
|
||||
),
|
||||
ErrorCode::NetworkError
|
||||
)
|
||||
})?
|
||||
.error_for_status()
|
||||
.map_err(|e| {
|
||||
raise_error!(
|
||||
proxy_request_error_message(
|
||||
&format!("Proxy check request failed via {}", provider.name),
|
||||
&e
|
||||
),
|
||||
ErrorCode::NetworkError
|
||||
)
|
||||
})?
|
||||
.json::<serde_json::Value>()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
raise_error!(
|
||||
proxy_request_error_message(
|
||||
&format!("Failed to read proxy check response from {}", provider.name),
|
||||
&e
|
||||
),
|
||||
ErrorCode::NetworkError
|
||||
)
|
||||
})?;
|
||||
|
||||
proxy_test_result_from_value(provider.name, &value)
|
||||
}
|
||||
|
||||
fn proxy_test_result_from_value(
|
||||
provider: &str,
|
||||
value: &serde_json::Value,
|
||||
) -> BichonResult<ProxyTestResult> {
|
||||
if provider == "ip-api.com" && value["status"].as_str() == Some("fail") {
|
||||
return Err(raise_error!(
|
||||
format!(
|
||||
"ip-api.com proxy check failed: {}",
|
||||
value["message"].as_str().unwrap_or("unknown error")
|
||||
),
|
||||
ErrorCode::NetworkError
|
||||
));
|
||||
}
|
||||
if provider == "ipwho.is" && value["success"].as_bool() == Some(false) {
|
||||
return Err(raise_error!(
|
||||
format!(
|
||||
"ipwho.is proxy check failed: {}",
|
||||
value["message"].as_str().unwrap_or("unknown error")
|
||||
),
|
||||
ErrorCode::NetworkError
|
||||
));
|
||||
}
|
||||
if provider == "ipapi.co" && value["error"].as_bool() == Some(true) {
|
||||
return Err(raise_error!(
|
||||
format!(
|
||||
"ipapi.co proxy check failed: {}",
|
||||
value["reason"].as_str().unwrap_or("unknown error")
|
||||
),
|
||||
ErrorCode::NetworkError
|
||||
));
|
||||
}
|
||||
|
||||
let ip_key = if provider == "ip-api.com" {
|
||||
"query"
|
||||
} else {
|
||||
"ip"
|
||||
};
|
||||
let ip = value[ip_key].as_str().ok_or_else(|| {
|
||||
raise_error!(
|
||||
format!("{provider} did not return an IP address"),
|
||||
ErrorCode::NetworkError
|
||||
)
|
||||
})?;
|
||||
let connection = &value["connection"];
|
||||
|
||||
Ok(ProxyTestResult {
|
||||
ip: Some(ip.to_string()),
|
||||
country: value[if provider == "ipapi.co" {
|
||||
"country_name"
|
||||
} else {
|
||||
"country"
|
||||
}]
|
||||
.as_str()
|
||||
.map(str::to_string),
|
||||
region: value[if provider == "ip-api.com" {
|
||||
"regionName"
|
||||
} else {
|
||||
"region"
|
||||
}]
|
||||
.as_str()
|
||||
.map(str::to_string),
|
||||
city: value["city"].as_str().map(str::to_string),
|
||||
isp: if provider == "ipapi.co" {
|
||||
value["org"].as_str().map(str::to_string)
|
||||
} else if provider == "ip-api.com" {
|
||||
value["isp"].as_str().map(str::to_string)
|
||||
} else {
|
||||
connection["isp"].as_str().map(str::to_string)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn proxy_request_error_message(context: &str, err: &reqwest::Error) -> String {
|
||||
let kind = if err.is_timeout() {
|
||||
"timed out"
|
||||
} else if err.is_connect() {
|
||||
"could not connect through the proxy"
|
||||
} else if err.is_status() {
|
||||
"received an error response"
|
||||
} else {
|
||||
"request failed"
|
||||
};
|
||||
let mut message = format!("{context}: {kind}: {err}");
|
||||
let mut source = err.source();
|
||||
|
||||
while let Some(err) = source {
|
||||
message.push_str(&format!(": {err}"));
|
||||
source = err.source();
|
||||
}
|
||||
|
||||
message
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -111,11 +318,54 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_valid_proxy_urls() {
|
||||
let urls = vec!["socks5://127.0.0.1:1080", "http://127.0.0.1:8080"];
|
||||
let urls = vec![
|
||||
"socks5://127.0.0.1:1080",
|
||||
"http://127.0.0.1:8080",
|
||||
"socks5://proxy.example.com:1080",
|
||||
"socks5://user:pass@proxy.example.com:1080",
|
||||
"http://user:pass@proxy.example.com:8080",
|
||||
"socks5://[::1]:1080",
|
||||
"socks5://user:pass@[::1]:1080",
|
||||
// Non-standard format: host:port:user:pass
|
||||
"socks5://server.nodeprovider.com:8080:username123:passwordhere",
|
||||
"http://server.nodeprovider.com:8080:username123:passwordhere",
|
||||
];
|
||||
|
||||
for url in urls {
|
||||
let proxy = Proxy::new(url.to_string());
|
||||
assert!(proxy.validate().is_ok(), "URL should be valid: {}", url);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_proxy_urls() {
|
||||
for url in ["socks5://user@proxy.example.com:1080", "socks5://::1:1080"] {
|
||||
let proxy = Proxy::new(url.to_string());
|
||||
assert!(proxy.validate().is_err(), "URL should be invalid: {}", url);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ipv6_proxy_urls_render_with_brackets() {
|
||||
let addr = parse_proxy_url("socks5://[::1]:1080").unwrap();
|
||||
assert_eq!(addr.standard_url(), "socks5://[::1]:1080");
|
||||
|
||||
let addr = parse_proxy_url("socks5://user:pass@[::1]:1080").unwrap();
|
||||
assert_eq!(addr.standard_url(), "socks5://user:pass@[::1]:1080");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proxy_test_result_rejects_empty_provider_response() {
|
||||
let result = proxy_test_result_from_value("ipwho.is", &serde_json::json!({}));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proxy_test_result_rejects_provider_error_response() {
|
||||
let result = proxy_test_result_from_value(
|
||||
"ipwho.is",
|
||||
&serde_json::json!({ "success": false, "message": "reserved range" }),
|
||||
);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ pub struct Envelope {
|
||||
pub message_id: String,
|
||||
pub account_id: u64,
|
||||
pub account_email: Option<String>,
|
||||
pub account_name: Option<String>,
|
||||
pub mailbox_id: u64,
|
||||
pub mailbox_name: Option<String>,
|
||||
pub uid: u32,
|
||||
|
||||
@@ -172,6 +172,52 @@ impl DedupCache {
|
||||
POPULATE_WINDOW_MS / (24 * 60 * 60 * 1000),
|
||||
);
|
||||
}
|
||||
|
||||
/// Remove all entries for a specific account.
|
||||
pub fn remove_by_account(&self, account_id: u64) {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
let before = entries.len();
|
||||
entries.retain(|(aid, _, _), _| *aid != account_id);
|
||||
let removed = before - entries.len();
|
||||
if removed > 0 {
|
||||
tracing::info!(
|
||||
"DedupCache: removed {} entries for account {}",
|
||||
removed,
|
||||
account_id
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove all entries for a specific mailbox (across all accounts).
|
||||
pub fn remove_by_mailbox(&self, mailbox_id: u64) {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
let before = entries.len();
|
||||
entries.retain(|(_, mid, _), _| *mid != mailbox_id);
|
||||
let removed = before - entries.len();
|
||||
if removed > 0 {
|
||||
tracing::info!(
|
||||
"DedupCache: removed {} entries for mailbox {}",
|
||||
removed,
|
||||
mailbox_id
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove a specific triple (most precise removal).
|
||||
pub fn remove(&self, account_id: u64, mailbox_id: u64, hash: &str) {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
if entries
|
||||
.remove(&(account_id, mailbox_id, hash.to_string()))
|
||||
.is_some()
|
||||
{
|
||||
tracing::debug!(
|
||||
"DedupCache: removed specific entry ({}, {}, {})",
|
||||
account_id,
|
||||
mailbox_id,
|
||||
hash
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────────
|
||||
@@ -366,7 +412,11 @@ mod tests {
|
||||
for segment_reader in searcher.segment_readers() {
|
||||
let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap();
|
||||
let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap();
|
||||
let hash_col = segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap().unwrap();
|
||||
let hash_col = segment_reader
|
||||
.fast_fields()
|
||||
.str(F_CONTENT_HASH)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap();
|
||||
|
||||
let max_doc = segment_reader.max_doc();
|
||||
@@ -381,7 +431,11 @@ mod tests {
|
||||
let account_id = account_col.values.get_val(doc_id);
|
||||
let mailbox_id = mailbox_col.values.get_val(doc_id);
|
||||
|
||||
let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0);
|
||||
let hash_ord = hash_col
|
||||
.ords()
|
||||
.values_for_doc(doc_id as u32)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
let mut hash_buf = String::new();
|
||||
hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap();
|
||||
|
||||
@@ -419,7 +473,11 @@ mod tests {
|
||||
for segment_reader in searcher.segment_readers() {
|
||||
let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap();
|
||||
let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap();
|
||||
let hash_col = segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap().unwrap();
|
||||
let hash_col = segment_reader
|
||||
.fast_fields()
|
||||
.str(F_CONTENT_HASH)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap();
|
||||
|
||||
let max_doc = segment_reader.max_doc();
|
||||
@@ -434,7 +492,11 @@ mod tests {
|
||||
let account_id = account_col.values.get_val(doc_id);
|
||||
let mailbox_id = mailbox_col.values.get_val(doc_id);
|
||||
|
||||
let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0);
|
||||
let hash_ord = hash_col
|
||||
.ords()
|
||||
.values_for_doc(doc_id as u32)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
let mut hash_buf = String::new();
|
||||
hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap();
|
||||
|
||||
@@ -474,7 +536,11 @@ mod tests {
|
||||
for segment_reader in searcher.segment_readers() {
|
||||
let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap();
|
||||
let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap();
|
||||
let hash_col = segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap().unwrap();
|
||||
let hash_col = segment_reader
|
||||
.fast_fields()
|
||||
.str(F_CONTENT_HASH)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap();
|
||||
|
||||
let max_doc = segment_reader.max_doc();
|
||||
@@ -489,7 +555,11 @@ mod tests {
|
||||
let account_id = account_col.values.get_val(doc_id);
|
||||
let mailbox_id = mailbox_col.values.get_val(doc_id);
|
||||
|
||||
let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0);
|
||||
let hash_ord = hash_col
|
||||
.ords()
|
||||
.values_for_doc(doc_id as u32)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
let mut hash_buf = String::new();
|
||||
hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap();
|
||||
|
||||
@@ -501,4 +571,71 @@ mod tests {
|
||||
assert!(cache.contains(1, 10, "hash-keep"));
|
||||
assert!(!cache.contains(1, 10, "hash-delete"));
|
||||
}
|
||||
|
||||
// ── removal methods ─────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn remove_by_account_works() {
|
||||
let cache = DedupCache::new_for_test();
|
||||
|
||||
cache.insert(1, 10, "hash-a1");
|
||||
cache.insert(1, 20, "hash-a2");
|
||||
cache.insert(2, 10, "hash-b1");
|
||||
cache.insert(2, 30, "hash-b2");
|
||||
cache.insert(1, 10, "hash-a3");
|
||||
|
||||
assert_eq!(cache.entries.lock().unwrap().len(), 5);
|
||||
|
||||
cache.remove_by_account(1);
|
||||
|
||||
let entries = cache.entries.lock().unwrap();
|
||||
assert_eq!(entries.len(), 2);
|
||||
assert!(!entries.contains_key(&(1, 10, "hash-a1".to_string())));
|
||||
assert!(!entries.contains_key(&(1, 20, "hash-a2".to_string())));
|
||||
assert!(!entries.contains_key(&(1, 10, "hash-a3".to_string())));
|
||||
|
||||
assert!(entries.contains_key(&(2, 10, "hash-b1".to_string())));
|
||||
assert!(entries.contains_key(&(2, 30, "hash-b2".to_string())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_by_mailbox_works() {
|
||||
let cache = DedupCache::new_for_test();
|
||||
|
||||
cache.insert(1, 10, "hash-1");
|
||||
cache.insert(1, 20, "hash-2");
|
||||
cache.insert(2, 10, "hash-3");
|
||||
cache.insert(3, 20, "hash-4");
|
||||
cache.insert(1, 10, "hash-5");
|
||||
|
||||
cache.remove_by_mailbox(10);
|
||||
|
||||
let entries = cache.entries.lock().unwrap();
|
||||
assert_eq!(entries.len(), 2);
|
||||
|
||||
assert!(entries.contains_key(&(1, 20, "hash-2".to_string())));
|
||||
assert!(entries.contains_key(&(3, 20, "hash-4".to_string())));
|
||||
|
||||
assert!(!entries.contains_key(&(1, 10, "hash-1".to_string())));
|
||||
assert!(!entries.contains_key(&(2, 10, "hash-3".to_string())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_specific_triple_works() {
|
||||
let cache = DedupCache::new_for_test();
|
||||
|
||||
cache.insert(1, 10, "hash-aaa");
|
||||
cache.insert(1, 10, "hash-bbb");
|
||||
cache.insert(2, 20, "hash-aaa");
|
||||
|
||||
assert!(cache.contains(1, 10, "hash-aaa"));
|
||||
assert!(cache.contains(1, 10, "hash-bbb"));
|
||||
assert!(cache.contains(2, 20, "hash-aaa"));
|
||||
|
||||
cache.remove(1, 10, "hash-aaa");
|
||||
|
||||
assert!(!cache.contains(1, 10, "hash-aaa"));
|
||||
assert!(cache.contains(1, 10, "hash-bbb"));
|
||||
assert!(cache.contains(2, 20, "hash-aaa"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,6 +40,7 @@ use crate::{
|
||||
envelope::Envelope,
|
||||
tantivy::{
|
||||
attachment::ATTACHMENT_MANAGER,
|
||||
dedup_cache::DEDUP_CACHE,
|
||||
fatal_commit,
|
||||
fields::{
|
||||
F_ACCOUNT_ID, F_DATE, F_FROM, F_ID, F_INGEST_AT, F_INTERNAL_DATE,
|
||||
@@ -50,8 +51,8 @@ use crate::{
|
||||
tokenizers::EuroTokenizer,
|
||||
},
|
||||
},
|
||||
utils::html::extract_text,
|
||||
utc_now,
|
||||
utils::html::extract_text,
|
||||
};
|
||||
|
||||
use chrono::Utc;
|
||||
@@ -277,6 +278,80 @@ impl IndexManager {
|
||||
Box::new(boolean_query)
|
||||
}
|
||||
|
||||
/// Return all Message-IDs stored in Tantivy for a given mailbox.
|
||||
/// Prefer `mailbox_contains_message_id` for existence checks on large
|
||||
/// mailboxes — this method loads everything into a HashSet.
|
||||
pub fn get_message_ids_for_mailbox(
|
||||
&self,
|
||||
account_id: u64,
|
||||
mailbox_id: u64,
|
||||
) -> BichonResult<HashSet<String>> {
|
||||
let query = self.mailbox_query(account_id, mailbox_id);
|
||||
let fields = SchemaTools::email_fields();
|
||||
let searcher = self.create_searcher()?;
|
||||
|
||||
let docs = searcher
|
||||
.search(&query, &DocSetCollector)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
let mut result = HashSet::new();
|
||||
for doc_address in docs {
|
||||
let doc = searcher
|
||||
.doc::<TantivyDocument>(doc_address)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
if let Some(v) = doc.get_first(fields.f_message_id) {
|
||||
if let Some(s) = v.as_str() {
|
||||
if !s.is_empty() {
|
||||
result.insert(s.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Check whether a specific Message-ID exists in a mailbox.
|
||||
/// Uses a TermQuery — O(1) per call, no allocation proportional to
|
||||
/// mailbox size. Suitable for large mailboxes where
|
||||
/// `get_message_ids_for_mailbox` would allocate too much memory.
|
||||
pub fn mailbox_contains_message_id(
|
||||
&self,
|
||||
account_id: u64,
|
||||
mailbox_id: u64,
|
||||
message_id: &str,
|
||||
) -> BichonResult<bool> {
|
||||
let fields = SchemaTools::email_fields();
|
||||
let query = BooleanQuery::new(vec![
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(fields.f_account_id, account_id),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(fields.f_mailbox_id, mailbox_id),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_text(fields.f_message_id, message_id),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
]);
|
||||
let searcher = self.create_searcher()?;
|
||||
let count = searcher
|
||||
.search(&query, &Count)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
Ok(count > 0)
|
||||
}
|
||||
|
||||
fn envelope_query(&self, account_id: u64, eid: &str) -> Box<dyn Query> {
|
||||
let account_id_query = TermQuery::new(
|
||||
Term::from_field_u64(SchemaTools::email_fields().f_account_id, account_id),
|
||||
@@ -386,6 +461,34 @@ impl IndexManager {
|
||||
}
|
||||
}
|
||||
|
||||
// any_recipient: OR across to, cc, bcc
|
||||
if let Some(ref v) = filter.any_recipient {
|
||||
let mut recipient_queries: Vec<(Occur, Box<dyn Query>)> = Vec::new();
|
||||
for field in [f.f_to_text, f.f_cc_text, f.f_bcc_text] {
|
||||
let query_parser = QueryParser::for_index(&self.index, vec![field]);
|
||||
if let Ok(q) = query_parser.parse_query(v) {
|
||||
recipient_queries.push((Occur::Should, q));
|
||||
}
|
||||
}
|
||||
if !recipient_queries.is_empty() {
|
||||
subqueries.push((Occur::Must, Box::new(BooleanQuery::new(recipient_queries))));
|
||||
}
|
||||
}
|
||||
|
||||
// any_participant: OR across from, to, cc, bcc
|
||||
if let Some(ref v) = filter.any_participant {
|
||||
let mut participant_queries: Vec<(Occur, Box<dyn Query>)> = Vec::new();
|
||||
for field in [f.f_from_text, f.f_to_text, f.f_cc_text, f.f_bcc_text] {
|
||||
let query_parser = QueryParser::for_index(&self.index, vec![field]);
|
||||
if let Ok(q) = query_parser.parse_query(v) {
|
||||
participant_queries.push((Occur::Should, q));
|
||||
}
|
||||
}
|
||||
if !participant_queries.is_empty() {
|
||||
subqueries.push((Occur::Must, Box::new(BooleanQuery::new(participant_queries))));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(has) = filter.has_attachment {
|
||||
let lower: Bound<Term>;
|
||||
let upper: Bound<Term>;
|
||||
@@ -792,6 +895,8 @@ impl IndexManager {
|
||||
attachments_content_hashes,
|
||||
)?;
|
||||
}
|
||||
|
||||
DEDUP_CACHE.remove_by_account(account_id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -815,8 +920,8 @@ impl IndexManager {
|
||||
}
|
||||
|
||||
let mut queries: Vec<Box<dyn Query>> = Vec::with_capacity(mailbox_ids.len());
|
||||
for mailbox_id in mailbox_ids {
|
||||
queries.push(self.mailbox_query(account_id, mailbox_id));
|
||||
for mailbox_id in &mailbox_ids {
|
||||
queries.push(self.mailbox_query(account_id, *mailbox_id));
|
||||
}
|
||||
let mut writer = self.index_writer.lock().await;
|
||||
for query in queries {
|
||||
@@ -835,6 +940,11 @@ impl IndexManager {
|
||||
attachments_content_hashes,
|
||||
)?;
|
||||
}
|
||||
|
||||
for mailbox_id in mailbox_ids {
|
||||
DEDUP_CACHE.remove_by_mailbox(mailbox_id);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -842,6 +952,21 @@ impl IndexManager {
|
||||
&self,
|
||||
query: Box<dyn Query>,
|
||||
) -> BichonResult<(HashSet<String>, HashSet<String>)> {
|
||||
let (eml_with_mailbox, attachments_content_hashes) =
|
||||
self.collect_content_hashes_with_mailbox(query)?;
|
||||
|
||||
let eml_content_hashes = eml_with_mailbox
|
||||
.into_iter()
|
||||
.map(|(hash, _mailbox_id)| hash)
|
||||
.collect();
|
||||
|
||||
Ok((eml_content_hashes, attachments_content_hashes))
|
||||
}
|
||||
|
||||
fn collect_content_hashes_with_mailbox(
|
||||
&self,
|
||||
query: Box<dyn Query>,
|
||||
) -> BichonResult<(HashSet<(String, u64)>, HashSet<String>)> {
|
||||
let mut eml_content_hashes = HashSet::new();
|
||||
let mut attachments_content_hashes = HashSet::new();
|
||||
|
||||
@@ -857,10 +982,14 @@ impl IndexManager {
|
||||
.doc::<TantivyDocument>(doc_address)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
let mailbox_id = doc.get_first(fields.f_mailbox_id).and_then(|v| v.as_u64());
|
||||
|
||||
// Extract content_hash
|
||||
if let Some(content_hash_value) = doc.get_first(fields.f_content_hash) {
|
||||
if let Some(str) = content_hash_value.as_str() {
|
||||
eml_content_hashes.insert(str.to_string());
|
||||
if let (Some(hash_str), Some(mailbox_id)) =
|
||||
(content_hash_value.as_str(), mailbox_id)
|
||||
{
|
||||
eml_content_hashes.insert((hash_str.to_string(), mailbox_id));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -933,7 +1062,7 @@ impl IndexManager {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut eml_content_hashes: HashSet<String> = HashSet::new();
|
||||
let mut eml_content_hash_triples: HashSet<(u64, u64, String)> = HashSet::new();
|
||||
let mut attachments_content_hashes: HashSet<String> = HashSet::new();
|
||||
|
||||
for (account_id, envelope_ids) in &deletes {
|
||||
@@ -944,8 +1073,14 @@ impl IndexManager {
|
||||
|
||||
for eid in unique_ids {
|
||||
let query = self.envelope_query(*account_id, eid);
|
||||
let (eml_hashes, attachment_hashes) = self.collect_content_hashes(query)?;
|
||||
eml_content_hashes.extend(eml_hashes);
|
||||
let (eml_hashes_with_mailbox, attachment_hashes) =
|
||||
self.collect_content_hashes_with_mailbox(query)?;
|
||||
|
||||
eml_content_hash_triples.extend(
|
||||
eml_hashes_with_mailbox
|
||||
.into_iter()
|
||||
.map(|(hash, mailbox_id)| (*account_id, mailbox_id, hash)),
|
||||
);
|
||||
attachments_content_hashes.extend(attachment_hashes);
|
||||
}
|
||||
}
|
||||
@@ -968,7 +1103,12 @@ impl IndexManager {
|
||||
.commit()
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() {
|
||||
if !eml_content_hash_triples.is_empty() || !attachments_content_hashes.is_empty() {
|
||||
let eml_content_hashes: HashSet<String> = eml_content_hash_triples
|
||||
.iter()
|
||||
.map(|(_, _, hash)| hash.clone())
|
||||
.collect();
|
||||
|
||||
self.cleanup_unused_content(
|
||||
&mut writer,
|
||||
eml_content_hashes,
|
||||
@@ -976,6 +1116,10 @@ impl IndexManager {
|
||||
)?;
|
||||
}
|
||||
|
||||
for (aid, mid, hash) in eml_content_hash_triples {
|
||||
DEDUP_CACHE.remove(aid, mid, &hash);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1157,8 +1301,7 @@ impl IndexManager {
|
||||
// f_attachments JSON blob.
|
||||
if let Some(attrs_val) = old_doc.get_first(f.f_attachments) {
|
||||
if let Some(json_str) = attrs_val.as_str() {
|
||||
if let Ok(parsed) =
|
||||
serde_json::from_str::<serde_json::Value>(json_str)
|
||||
if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(json_str)
|
||||
{
|
||||
if let Some(arr) = parsed.as_array() {
|
||||
for att in arr {
|
||||
@@ -1179,14 +1322,8 @@ impl IndexManager {
|
||||
.and_then(|v| v.as_str())
|
||||
.filter(|s| !s.is_empty())
|
||||
{
|
||||
new_doc.add_text(
|
||||
f.f_attachment_name_text,
|
||||
filename,
|
||||
);
|
||||
new_doc.add_text(
|
||||
f.f_attachment_name_exact,
|
||||
filename,
|
||||
);
|
||||
new_doc.add_text(f.f_attachment_name_text, filename);
|
||||
new_doc.add_text(f.f_attachment_name_exact, filename);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1200,22 +1337,18 @@ impl IndexManager {
|
||||
if let Some(content_hash) = hash_val.as_str() {
|
||||
match BLOB_MANAGER.get_email(content_hash) {
|
||||
Ok(Some(eml_bytes)) => {
|
||||
if let Some(message) =
|
||||
MessageParser::new().parse(&eml_bytes)
|
||||
{
|
||||
if let Some(message) = MessageParser::new().parse(&eml_bytes) {
|
||||
let text = message
|
||||
.body_text(0)
|
||||
.map(|cow| cow.into_owned())
|
||||
.or_else(|| {
|
||||
message.body_html(0).map(|cow| {
|
||||
extract_text(cow.into_owned())
|
||||
})
|
||||
message
|
||||
.body_html(0)
|
||||
.map(|cow| extract_text(cow.into_owned()))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let body_text = text
|
||||
.split_whitespace()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
let body_text =
|
||||
text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
if !body_text.is_empty() {
|
||||
new_doc.add_text(f.f_body, &body_text);
|
||||
}
|
||||
@@ -1304,7 +1437,7 @@ impl IndexManager {
|
||||
let mailbox_docs: Vec<DocAddress>;
|
||||
|
||||
match sort_by {
|
||||
SortBy::DATE => {
|
||||
SortBy::DATE => {
|
||||
let date_docs: Vec<(Option<i64>, DocAddress)> = searcher
|
||||
.search(
|
||||
&query,
|
||||
@@ -1801,10 +1934,8 @@ mod tests {
|
||||
if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(json_str) {
|
||||
if let Some(arr) = parsed.as_array() {
|
||||
for att in arr {
|
||||
let is_inline = att
|
||||
.get("inline")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
let is_inline =
|
||||
att.get("inline").and_then(|v| v.as_bool()).unwrap_or(false);
|
||||
let has_cid = att
|
||||
.get("content_id")
|
||||
.and_then(|v| v.as_str())
|
||||
@@ -1841,8 +1972,7 @@ mod tests {
|
||||
.map(|cow| extract_text(cow.into_owned()))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let body_text =
|
||||
text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
let body_text = text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
if !body_text.is_empty() {
|
||||
new_doc.add_text(f.f_body, &body_text);
|
||||
}
|
||||
@@ -1911,8 +2041,7 @@ mod tests {
|
||||
let mut writer2 = index
|
||||
.writer_with_num_threads(1, 15_000_000)
|
||||
.expect("writer2");
|
||||
writer2
|
||||
.delete_term(Term::from_field_text(f.f_id, "test-eid-001"));
|
||||
writer2.delete_term(Term::from_field_text(f.f_id, "test-eid-001"));
|
||||
writer2.add_document(new_doc).unwrap();
|
||||
writer2.commit().unwrap();
|
||||
|
||||
@@ -1922,16 +2051,14 @@ mod tests {
|
||||
let searcher = reader.searcher();
|
||||
|
||||
// Body text (tokenized via "euro")
|
||||
let body_parser =
|
||||
QueryParser::for_index(&index, vec![f.f_body]);
|
||||
let body_parser = QueryParser::for_index(&index, vec![f.f_body]);
|
||||
let body_hits = searcher
|
||||
.search(&body_parser.parse_query("quick brown fox").unwrap(), &Count)
|
||||
.unwrap();
|
||||
assert_eq!(body_hits, 1, "body text should survive tag update");
|
||||
|
||||
// from_text (tokenized via "euro")
|
||||
let from_parser =
|
||||
QueryParser::for_index(&index, vec![f.f_from_text]);
|
||||
let from_parser = QueryParser::for_index(&index, vec![f.f_from_text]);
|
||||
let from_hits = searcher
|
||||
.search(
|
||||
&from_parser.parse_query("alice@example.com").unwrap(),
|
||||
@@ -1943,10 +2070,7 @@ mod tests {
|
||||
// to_text
|
||||
let to_parser = QueryParser::for_index(&index, vec![f.f_to_text]);
|
||||
let to_hits = searcher
|
||||
.search(
|
||||
&to_parser.parse_query("bob@example.com").unwrap(),
|
||||
&Count,
|
||||
)
|
||||
.search(&to_parser.parse_query("bob@example.com").unwrap(), &Count)
|
||||
.unwrap();
|
||||
assert_eq!(to_hits, 1, "to_text should survive tag update");
|
||||
|
||||
@@ -1969,10 +2093,7 @@ mod tests {
|
||||
let tags_hits = searcher
|
||||
.search(
|
||||
&TermQuery::new(
|
||||
Term::from_facet(
|
||||
f.f_tags,
|
||||
&Facet::from_text("/important").unwrap(),
|
||||
),
|
||||
Term::from_facet(f.f_tags, &Facet::from_text("/important").unwrap()),
|
||||
IndexRecordOption::Basic,
|
||||
),
|
||||
&Count,
|
||||
@@ -1984,19 +2105,13 @@ mod tests {
|
||||
let old_tag_hits = searcher
|
||||
.search(
|
||||
&TermQuery::new(
|
||||
Term::from_facet(
|
||||
f.f_tags,
|
||||
&Facet::from_text("/unread").unwrap(),
|
||||
),
|
||||
Term::from_facet(f.f_tags, &Facet::from_text("/unread").unwrap()),
|
||||
IndexRecordOption::Basic,
|
||||
),
|
||||
&Count,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
old_tag_hits, 0,
|
||||
"old tag /unread should have been removed"
|
||||
);
|
||||
assert_eq!(old_tag_hits, 0, "old tag /unread should have been removed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2056,4 +2171,362 @@ mod tests {
|
||||
"body should be absent when EML is missing"
|
||||
);
|
||||
}
|
||||
|
||||
// ── get_message_ids_for_mailbox ─────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn get_message_ids_returns_stored_ids() {
|
||||
let f = SchemaTools::email_fields();
|
||||
let index = Index::create_in_ram(SchemaTools::email_schema());
|
||||
index.tokenizers().register("euro", EuroTokenizer::new());
|
||||
|
||||
// Insert two docs for mailbox 10, one for mailbox 20
|
||||
{
|
||||
let mut writer = index
|
||||
.writer_with_num_threads(1, 15_000_000)
|
||||
.expect("writer");
|
||||
|
||||
let mut doc1 = TantivyDocument::new();
|
||||
doc1.add_u64(f.f_account_id, 1);
|
||||
doc1.add_u64(f.f_mailbox_id, 10);
|
||||
doc1.add_text(f.f_message_id, "<msg-a@test>");
|
||||
doc1.add_text(f.f_id, "id-a");
|
||||
doc1.add_u64(f.f_uid, 1);
|
||||
doc1.add_text(f.f_content_hash, "hash-a");
|
||||
writer.add_document(doc1).unwrap();
|
||||
|
||||
let mut doc2 = TantivyDocument::new();
|
||||
doc2.add_u64(f.f_account_id, 1);
|
||||
doc2.add_u64(f.f_mailbox_id, 10);
|
||||
doc2.add_text(f.f_message_id, "<msg-b@test>");
|
||||
doc2.add_text(f.f_id, "id-b");
|
||||
doc2.add_u64(f.f_uid, 2);
|
||||
doc2.add_text(f.f_content_hash, "hash-b");
|
||||
writer.add_document(doc2).unwrap();
|
||||
|
||||
let mut doc3 = TantivyDocument::new();
|
||||
doc3.add_u64(f.f_account_id, 1);
|
||||
doc3.add_u64(f.f_mailbox_id, 20);
|
||||
doc3.add_text(f.f_message_id, "<msg-c@test>");
|
||||
doc3.add_text(f.f_id, "id-c");
|
||||
doc3.add_u64(f.f_uid, 3);
|
||||
doc3.add_text(f.f_content_hash, "hash-c");
|
||||
writer.add_document(doc3).unwrap();
|
||||
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
let reader = index.reader().unwrap();
|
||||
reader.reload().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
|
||||
// We can't easily call ENVELOPE_MANAGER.get_message_ids_for_mailbox
|
||||
// because it reads from ENVELOPE_MANAGER's own index, not our in-memory one.
|
||||
// Instead, test the query pattern directly.
|
||||
let query: Box<dyn Query> = {
|
||||
let account_query = TermQuery::new(
|
||||
Term::from_field_u64(f.f_account_id, 1),
|
||||
IndexRecordOption::Basic,
|
||||
);
|
||||
let mailbox_query = TermQuery::new(
|
||||
Term::from_field_u64(f.f_mailbox_id, 10),
|
||||
IndexRecordOption::Basic,
|
||||
);
|
||||
Box::new(BooleanQuery::new(vec![
|
||||
(Occur::Must, Box::new(account_query)),
|
||||
(Occur::Must, Box::new(mailbox_query)),
|
||||
]))
|
||||
};
|
||||
|
||||
let docs = searcher
|
||||
.search(&query, &DocSetCollector)
|
||||
.unwrap();
|
||||
|
||||
let mut ids: Vec<String> = Vec::new();
|
||||
for addr in docs {
|
||||
let doc: TantivyDocument = searcher.doc(addr).unwrap();
|
||||
if let Some(v) = doc.get_first(f.f_message_id) {
|
||||
if let Some(s) = v.as_str() {
|
||||
ids.push(s.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
ids.sort();
|
||||
|
||||
assert_eq!(ids, vec!["<msg-a@test>", "<msg-b@test>"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_message_ids_empty_mailbox_returns_empty() {
|
||||
let f = SchemaTools::email_fields();
|
||||
let index = Index::create_in_ram(SchemaTools::email_schema());
|
||||
index.tokenizers().register("euro", EuroTokenizer::new());
|
||||
|
||||
{
|
||||
let mut writer = index
|
||||
.writer_with_num_threads(1, 15_000_000)
|
||||
.expect("writer");
|
||||
|
||||
// Doc for a different mailbox
|
||||
let mut doc = TantivyDocument::new();
|
||||
doc.add_u64(f.f_account_id, 1);
|
||||
doc.add_u64(f.f_mailbox_id, 99);
|
||||
doc.add_text(f.f_message_id, "<other@test>");
|
||||
doc.add_text(f.f_id, "id-other");
|
||||
doc.add_u64(f.f_uid, 1);
|
||||
doc.add_text(f.f_content_hash, "hash-other");
|
||||
writer.add_document(doc).unwrap();
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
let reader = index.reader().unwrap();
|
||||
reader.reload().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
|
||||
let query: Box<dyn Query> = {
|
||||
let account_query = TermQuery::new(
|
||||
Term::from_field_u64(f.f_account_id, 1),
|
||||
IndexRecordOption::Basic,
|
||||
);
|
||||
let mailbox_query = TermQuery::new(
|
||||
Term::from_field_u64(f.f_mailbox_id, 10),
|
||||
IndexRecordOption::Basic,
|
||||
);
|
||||
Box::new(BooleanQuery::new(vec![
|
||||
(Occur::Must, Box::new(account_query)),
|
||||
(Occur::Must, Box::new(mailbox_query)),
|
||||
]))
|
||||
};
|
||||
|
||||
let docs = searcher.search(&query, &DocSetCollector).unwrap();
|
||||
assert!(docs.is_empty());
|
||||
}
|
||||
|
||||
// ── mailbox_contains_message_id ───────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn mailbox_contains_message_id_finds_existing() {
|
||||
let f = SchemaTools::email_fields();
|
||||
let index = Index::create_in_ram(SchemaTools::email_schema());
|
||||
index.tokenizers().register("euro", EuroTokenizer::new());
|
||||
|
||||
{
|
||||
let mut writer = index
|
||||
.writer_with_num_threads(1, 15_000_000)
|
||||
.expect("writer");
|
||||
|
||||
let mut doc = TantivyDocument::new();
|
||||
doc.add_u64(f.f_account_id, 1);
|
||||
doc.add_u64(f.f_mailbox_id, 10);
|
||||
doc.add_text(f.f_message_id, "abc@example.com");
|
||||
doc.add_text(f.f_id, "id-1");
|
||||
doc.add_u64(f.f_uid, 1);
|
||||
doc.add_text(f.f_content_hash, "hash-1");
|
||||
writer.add_document(doc).unwrap();
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
// We test the query pattern directly (can't call ENVELOPE_MANAGER
|
||||
// which uses a different index).
|
||||
let reader = index.reader().unwrap();
|
||||
reader.reload().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
|
||||
let query = BooleanQuery::new(vec![
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_account_id, 1),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_mailbox_id, 10),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_text(f.f_message_id, "abc@example.com"),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
]);
|
||||
|
||||
let count = searcher.search(&query, &Count).unwrap();
|
||||
assert_eq!(count, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mailbox_contains_message_id_returns_zero_for_missing() {
|
||||
let f = SchemaTools::email_fields();
|
||||
let index = Index::create_in_ram(SchemaTools::email_schema());
|
||||
index.tokenizers().register("euro", EuroTokenizer::new());
|
||||
|
||||
{
|
||||
let mut writer = index
|
||||
.writer_with_num_threads(1, 15_000_000)
|
||||
.expect("writer");
|
||||
|
||||
let mut doc = TantivyDocument::new();
|
||||
doc.add_u64(f.f_account_id, 1);
|
||||
doc.add_u64(f.f_mailbox_id, 10);
|
||||
doc.add_text(f.f_message_id, "existing@example.com");
|
||||
doc.add_text(f.f_id, "id-1");
|
||||
doc.add_u64(f.f_uid, 1);
|
||||
doc.add_text(f.f_content_hash, "hash-1");
|
||||
writer.add_document(doc).unwrap();
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
let reader = index.reader().unwrap();
|
||||
reader.reload().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
|
||||
let query = BooleanQuery::new(vec![
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_account_id, 1),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_mailbox_id, 10),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_text(f.f_message_id, "nonexistent@example.com"),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
]);
|
||||
|
||||
let count = searcher.search(&query, &Count).unwrap();
|
||||
assert_eq!(count, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mailbox_contains_message_id_respects_mailbox_boundary() {
|
||||
let f = SchemaTools::email_fields();
|
||||
let index = Index::create_in_ram(SchemaTools::email_schema());
|
||||
index.tokenizers().register("euro", EuroTokenizer::new());
|
||||
|
||||
{
|
||||
let mut writer = index
|
||||
.writer_with_num_threads(1, 15_000_000)
|
||||
.expect("writer");
|
||||
|
||||
// Same Message-ID in mailbox 10
|
||||
let mut doc1 = TantivyDocument::new();
|
||||
doc1.add_u64(f.f_account_id, 1);
|
||||
doc1.add_u64(f.f_mailbox_id, 10);
|
||||
doc1.add_text(f.f_message_id, "shared@example.com");
|
||||
doc1.add_text(f.f_id, "id-1");
|
||||
doc1.add_u64(f.f_uid, 1);
|
||||
doc1.add_text(f.f_content_hash, "hash-1");
|
||||
writer.add_document(doc1).unwrap();
|
||||
|
||||
// Same Message-ID in mailbox 20 (different mailbox)
|
||||
let mut doc2 = TantivyDocument::new();
|
||||
doc2.add_u64(f.f_account_id, 1);
|
||||
doc2.add_u64(f.f_mailbox_id, 20);
|
||||
doc2.add_text(f.f_message_id, "shared@example.com");
|
||||
doc2.add_text(f.f_id, "id-2");
|
||||
doc2.add_u64(f.f_uid, 2);
|
||||
doc2.add_text(f.f_content_hash, "hash-2");
|
||||
writer.add_document(doc2).unwrap();
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
let reader = index.reader().unwrap();
|
||||
reader.reload().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
|
||||
// Query mailbox 10: should find 1
|
||||
let q10 = BooleanQuery::new(vec![
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_account_id, 1),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_mailbox_id, 10),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_text(f.f_message_id, "shared@example.com"),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
]);
|
||||
assert_eq!(searcher.search(&q10, &Count).unwrap(), 1);
|
||||
|
||||
// Query mailbox 20: should find 1
|
||||
let q20 = BooleanQuery::new(vec![
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_account_id, 1),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_mailbox_id, 20),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_text(f.f_message_id, "shared@example.com"),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
]);
|
||||
assert_eq!(searcher.search(&q20, &Count).unwrap(), 1);
|
||||
|
||||
// Query mailbox 99 (no docs): should find 0
|
||||
let q99 = BooleanQuery::new(vec![
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_account_id, 1),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_u64(f.f_mailbox_id, 99),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
(
|
||||
Occur::Must,
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_text(f.f_message_id, "shared@example.com"),
|
||||
IndexRecordOption::Basic,
|
||||
)),
|
||||
),
|
||||
]);
|
||||
assert_eq!(searcher.search(&q99, &Count).unwrap(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,7 +155,8 @@ impl EnvelopeWithAttachments {
|
||||
id: extract_string_field(doc, fields.f_id, F_ID)?,
|
||||
message_id: extract_string_field(doc, fields.f_message_id, F_MESSAGE_ID)?,
|
||||
account_id,
|
||||
account_email: Some(account.email),
|
||||
account_email: Some(account.email), //https://github.com/rustmailer/bichon/issues/306
|
||||
account_name: account.account_name,
|
||||
mailbox_id,
|
||||
mailbox_name: Some(mailbox.name),
|
||||
uid: extract_u64_field(doc, fields.f_uid, F_UID)? as u32,
|
||||
|
||||
@@ -156,10 +156,16 @@ impl AccessTokenModel {
|
||||
})?;
|
||||
|
||||
if matches!(token_model.token_type, TokenType::WebUI) {
|
||||
let life = utc_now!() - token_model.created_at;
|
||||
let max_life = SETTINGS.bichon_webui_token_expiration_hours * 60 * 60 * 1000;
|
||||
// Use last_access_at if set, otherwise fall back to created_at
|
||||
let last_active = if token_model.last_access_at > 0 {
|
||||
token_model.last_access_at
|
||||
} else {
|
||||
token_model.created_at
|
||||
};
|
||||
let idle = utc_now!() - last_active;
|
||||
let max_life = SETTINGS.bichon_webui_token_expiration_hours as i64 * 60 * 60 * 1000;
|
||||
|
||||
if life > (max_life as i64) {
|
||||
if idle > max_life {
|
||||
return Err(raise_error!(
|
||||
"Permission denied: the WebUI token has expired.".into(),
|
||||
ErrorCode::PermissionDenied
|
||||
@@ -176,13 +182,15 @@ impl AccessTokenModel {
|
||||
));
|
||||
}
|
||||
}
|
||||
update_impl(DB_MANAGER.db(), &token_str, |current: AccessTokenModel| {
|
||||
let mut updated = current.clone();
|
||||
updated.last_access_at = utc_now!();
|
||||
Ok(updated)
|
||||
})?;
|
||||
}
|
||||
|
||||
// Update last_access_at on every successful use for both token types
|
||||
update_impl(DB_MANAGER.db(), &token_str, |current: AccessTokenModel| {
|
||||
let mut updated = current.clone();
|
||||
updated.last_access_at = utc_now!();
|
||||
Ok(updated)
|
||||
})?;
|
||||
|
||||
let user = UserModel::find(token_model.user_id)
|
||||
?
|
||||
.ok_or_else(|| raise_error!("The user associated with this access token does not exist or may have been deleted.".into(), ErrorCode::ResourceNotFound))?;
|
||||
|
||||
@@ -87,6 +87,11 @@ pub struct BichonUserV2 {
|
||||
|
||||
pub theme: Option<String>,
|
||||
pub language: Option<String>,
|
||||
|
||||
/// SSO identity: unique subject ID from the external IdP (e.g. OIDC `sub` claim).
|
||||
pub sso_id: Option<String>,
|
||||
/// SSO provider identifier: `"oidc"` or future `"saml"` / `"ldap"`.
|
||||
pub sso_provider: Option<String>,
|
||||
}
|
||||
|
||||
impl MemDbModel for BichonUserV2 {
|
||||
@@ -192,6 +197,8 @@ impl BichonUserV2 {
|
||||
global_permissions,
|
||||
theme: self.theme,
|
||||
language: self.language,
|
||||
sso_id: self.sso_id,
|
||||
sso_provider: self.sso_provider,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -226,6 +233,8 @@ impl BichonUserV2 {
|
||||
acl: None,
|
||||
theme: None,
|
||||
language: None,
|
||||
sso_id: None,
|
||||
sso_provider: None,
|
||||
};
|
||||
|
||||
// 3. Generate and insert an initial access token for the first-time setup
|
||||
@@ -382,6 +391,8 @@ impl BichonUserV2 {
|
||||
account_access_map: request.account_access_map,
|
||||
theme: request.theme,
|
||||
language: request.language,
|
||||
sso_id: None,
|
||||
sso_provider: None,
|
||||
};
|
||||
|
||||
let user_clone = user.clone();
|
||||
|
||||
@@ -52,4 +52,9 @@ pub struct UserView {
|
||||
pub acl: Option<AccessControl>,
|
||||
pub theme: Option<String>,
|
||||
pub language: Option<String>,
|
||||
|
||||
/// SSO identity: unique subject ID from the external IdP (e.g. OIDC `sub` claim).
|
||||
pub sso_id: Option<String>,
|
||||
/// SSO provider identifier: `"oidc"` or future `"saml"` / `"ldap"`.
|
||||
pub sso_provider: Option<String>,
|
||||
}
|
||||
|
||||
@@ -138,7 +138,7 @@ macro_rules! generate_token {
|
||||
}};
|
||||
}
|
||||
|
||||
pub(crate) fn generate_token_impl(bit_strength: usize) -> String {
|
||||
pub fn generate_token_impl(bit_strength: usize) -> String {
|
||||
let byte_length = (bit_strength + 23) / 24 * 3;
|
||||
let random_bytes: Vec<u8> = (0..byte_length).map(|_| rand::random::<u8>()).collect();
|
||||
let mut encoded = general_purpose::URL_SAFE.encode(&random_bytes);
|
||||
|
||||
@@ -21,9 +21,11 @@ use crate::raise_error;
|
||||
use crate::settings::proxy::Proxy;
|
||||
use crate::utils::tls::establish_tls_stream;
|
||||
use crate::{error::BichonResult, imap::session::SessionStream};
|
||||
use base64::{engine::general_purpose, Engine as _};
|
||||
use std::net::SocketAddr;
|
||||
use std::pin::Pin;
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::time::timeout;
|
||||
use tokio_io_timeout::TimeoutStream;
|
||||
@@ -32,6 +34,54 @@ use tracing::error;
|
||||
|
||||
pub(crate) const TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
|
||||
pub enum ProxyScheme {
|
||||
Socks5,
|
||||
Http,
|
||||
}
|
||||
|
||||
impl ProxyScheme {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Socks5 => "socks5",
|
||||
Self::Http => "http",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parsed proxy address components.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ProxyAddr {
|
||||
pub scheme: ProxyScheme,
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub username: Option<String>,
|
||||
pub password: Option<String>,
|
||||
}
|
||||
|
||||
impl ProxyAddr {
|
||||
pub fn standard_url(&self) -> String {
|
||||
let host = if self.host.contains(':') {
|
||||
format!("[{}]", self.host)
|
||||
} else {
|
||||
self.host.clone()
|
||||
};
|
||||
|
||||
if let (Some(user), Some(pass)) = (&self.username, &self.password) {
|
||||
format!(
|
||||
"{}://{}:{}@{}:{}",
|
||||
self.scheme.as_str(),
|
||||
user,
|
||||
pass,
|
||||
host,
|
||||
self.port
|
||||
)
|
||||
} else {
|
||||
format!("{}://{}:{}", self.scheme.as_str(), host, self.port)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn establish_tcp_connection_with_timeout(
|
||||
address: SocketAddr,
|
||||
use_proxy: Option<u64>,
|
||||
@@ -66,74 +116,247 @@ pub async fn establish_tls_connection(
|
||||
Ok(tls_stream)
|
||||
}
|
||||
|
||||
pub fn parse_proxy_addr(input: &str) -> BichonResult<SocketAddr> {
|
||||
// Normalize and check protocol prefix
|
||||
/// Parse a proxy URL into its components.
|
||||
///
|
||||
/// Supports two formats:
|
||||
/// - **Standard**: `[scheme://][user:pass@]host:port`
|
||||
/// - **Non-standard** (some proxy providers): `[scheme://]host:port:username:password`
|
||||
///
|
||||
/// The distinguishing feature is the `@` sign in the standard format.
|
||||
pub fn parse_proxy_url(input: &str) -> BichonResult<ProxyAddr> {
|
||||
// Normalize and strip scheme prefix
|
||||
let (scheme, stripped) = if let Some(rest) = input
|
||||
.strip_prefix("socks5://")
|
||||
.or_else(|| input.strip_prefix("SOCKS5://"))
|
||||
.or_else(|| input.strip_prefix("Socks5://"))
|
||||
{
|
||||
("socks5", rest)
|
||||
(ProxyScheme::Socks5, rest)
|
||||
} else if let Some(rest) = input
|
||||
.strip_prefix("http://")
|
||||
.or_else(|| input.strip_prefix("HTTP://"))
|
||||
.or_else(|| input.strip_prefix("Http://"))
|
||||
{
|
||||
("http", rest)
|
||||
(ProxyScheme::Http, rest)
|
||||
} else {
|
||||
return Err(raise_error!(
|
||||
format!(
|
||||
"Invalid proxy URL: must start with 'http://' or 'socks5://', got '{}'",
|
||||
input
|
||||
),
|
||||
"Invalid proxy URL: must start with 'http://' or 'socks5://'".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
};
|
||||
|
||||
// Parse the remaining address
|
||||
let addr = stripped.parse::<SocketAddr>().map_err(|e| {
|
||||
if stripped.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Proxy URL has empty address after scheme.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
|
||||
// Check for standard format: user:pass@host:port
|
||||
if let Some(at_pos) = stripped.rfind('@') {
|
||||
let userinfo = &stripped[..at_pos];
|
||||
let hostport = &stripped[at_pos + 1..];
|
||||
|
||||
let (username, password) = split_userinfo(userinfo)?;
|
||||
let (host, port) = split_hostport(hostport)?;
|
||||
|
||||
return Ok(ProxyAddr {
|
||||
scheme,
|
||||
host,
|
||||
port,
|
||||
username,
|
||||
password,
|
||||
});
|
||||
}
|
||||
|
||||
// No '@' — check for non-standard format: host:port:user:pass
|
||||
if stripped.starts_with('[') {
|
||||
let (host, port) = split_hostport(stripped)?;
|
||||
return Ok(ProxyAddr {
|
||||
scheme,
|
||||
host,
|
||||
port,
|
||||
username: None,
|
||||
password: None,
|
||||
});
|
||||
}
|
||||
|
||||
let mut parts = stripped.split(':');
|
||||
match (
|
||||
parts.next(),
|
||||
parts.next(),
|
||||
parts.next(),
|
||||
parts.next(),
|
||||
parts.next(),
|
||||
) {
|
||||
(Some(_), Some(_), None, None, None) => {
|
||||
// host:port, no auth
|
||||
let (host, port) = split_hostport(stripped)?;
|
||||
Ok(ProxyAddr {
|
||||
scheme,
|
||||
host,
|
||||
port,
|
||||
username: None,
|
||||
password: None,
|
||||
})
|
||||
}
|
||||
(Some(host), Some(port), Some(username), Some(password), None) => {
|
||||
// Non-standard: host:port:username:password
|
||||
let port = port.parse::<u16>().map_err(|_| {
|
||||
raise_error!(
|
||||
format!("Invalid port '{}' in proxy URL.", port),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
|
||||
if host.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Empty hostname in proxy URL.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
if host.contains(':') || host.contains('[') || host.contains(']') {
|
||||
return Err(raise_error!(
|
||||
"IPv6 proxy hosts are not supported.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
if username.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Empty username in proxy URL.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
if password.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Empty password in proxy URL.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
|
||||
Ok(ProxyAddr {
|
||||
scheme,
|
||||
host: host.to_string(),
|
||||
port,
|
||||
username: Some(username.to_string()),
|
||||
password: Some(password.to_string()),
|
||||
})
|
||||
}
|
||||
_ => Err(raise_error!(
|
||||
"Invalid proxy URL format. Expected '[scheme://][user:pass@]host:port' or 'scheme://host:port:user:pass'.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split "user:pass" into (Some(user), Some(pass)).
|
||||
fn split_userinfo(userinfo: &str) -> BichonResult<(Option<String>, Option<String>)> {
|
||||
if userinfo.is_empty() {
|
||||
return Ok((None, None));
|
||||
}
|
||||
if let Some(colon_pos) = userinfo.find(':') {
|
||||
let user = &userinfo[..colon_pos];
|
||||
let pass = &userinfo[colon_pos + 1..];
|
||||
if user.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Empty username in proxy URL credentials.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
if pass.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Empty password in proxy URL credentials.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
Ok((Some(user.to_string()), Some(pass.to_string())))
|
||||
} else {
|
||||
Err(raise_error!(
|
||||
"Password cannot be empty when username is provided.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Split "host:port" into (host, port). Bracketed IPv6 is accepted.
|
||||
fn split_hostport(hostport: &str) -> BichonResult<(String, u16)> {
|
||||
if hostport.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Empty host:port in proxy URL.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(rest) = hostport.strip_prefix('[') {
|
||||
let Some(close_bracket) = rest.find(']') else {
|
||||
return Err(raise_error!(
|
||||
format!("Invalid IPv6 address in proxy URL: '{}'.", hostport),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
};
|
||||
let host = &rest[..close_bracket];
|
||||
let port_text = rest[close_bracket + 1..].strip_prefix(':').ok_or_else(|| {
|
||||
raise_error!(
|
||||
format!(
|
||||
"Missing port after IPv6 address in proxy URL: '{}'.",
|
||||
hostport
|
||||
),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
let port = port_text.parse::<u16>().map_err(|_| {
|
||||
raise_error!(
|
||||
format!("Invalid port in proxy URL: '{}'.", hostport),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
|
||||
return Ok((host.to_string(), port));
|
||||
}
|
||||
|
||||
// hostname:port or ip:port — split from right
|
||||
let last_colon = hostport.rfind(':').ok_or_else(|| {
|
||||
raise_error!(
|
||||
format!(
|
||||
"Failed to parse {} proxy address '{}': {}",
|
||||
scheme, stripped, e
|
||||
),
|
||||
format!("Missing port in proxy URL: '{}'.", hostport),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
let host = hostport[..last_colon].to_string();
|
||||
let port = hostport[last_colon + 1..].parse::<u16>().map_err(|_| {
|
||||
raise_error!(
|
||||
format!("Invalid port in proxy URL: '{}'.", hostport),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(addr)
|
||||
if host.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Empty hostname in proxy URL.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
if host.contains(':') || host.contains('[') || host.contains(']') {
|
||||
return Err(raise_error!(
|
||||
"IPv6 proxy hosts are not supported.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
|
||||
Ok((host, port))
|
||||
}
|
||||
|
||||
/// Try to connect via SOCKS5 proxy or TCP with timeout
|
||||
/// Try to connect via SOCKS5 proxy or TCP with timeout.
|
||||
async fn connect_with_optional_proxy(
|
||||
use_proxy: Option<u64>,
|
||||
address: SocketAddr,
|
||||
) -> BichonResult<TcpStream> {
|
||||
// Try if proxy is enabled
|
||||
if let Some(proxy_id) = use_proxy {
|
||||
let proxy = Proxy::get(proxy_id)?;
|
||||
let proxy = parse_proxy_addr(&proxy.url)?;
|
||||
return timeout(TIMEOUT, Socks5Stream::connect(proxy, address))
|
||||
.await
|
||||
.map_err(|_| {
|
||||
error!(
|
||||
"SOCKS5 proxy connection to {} via {} timed out after {}s",
|
||||
address,
|
||||
proxy,
|
||||
TIMEOUT.as_secs()
|
||||
);
|
||||
raise_error!(
|
||||
format!(
|
||||
"SOCKS5 proxy connection to {} via {} timed out after {}s",
|
||||
address,
|
||||
proxy,
|
||||
TIMEOUT.as_secs()
|
||||
),
|
||||
ErrorCode::ConnectionTimeout
|
||||
)
|
||||
})?
|
||||
.map(|s| s.into_inner())
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError));
|
||||
let addr = parse_proxy_url(&proxy.url)?;
|
||||
return if addr.scheme == ProxyScheme::Http {
|
||||
connect_via_http_proxy(&addr, address).await
|
||||
} else {
|
||||
connect_via_socks5_proxy(&addr, address).await
|
||||
};
|
||||
}
|
||||
// Fallback to direct TCP connection
|
||||
timeout(TIMEOUT, TcpStream::connect(address))
|
||||
@@ -155,3 +378,135 @@ async fn connect_with_optional_proxy(
|
||||
})?
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError))
|
||||
}
|
||||
|
||||
async fn connect_via_socks5_proxy(
|
||||
addr: &ProxyAddr,
|
||||
address: SocketAddr,
|
||||
) -> BichonResult<TcpStream> {
|
||||
let proxy_addr = (addr.host.as_str(), addr.port);
|
||||
let result = if let (Some(user), Some(pass)) = (&addr.username, &addr.password) {
|
||||
timeout(
|
||||
TIMEOUT,
|
||||
Socks5Stream::connect_with_password(proxy_addr, address, user.as_str(), pass.as_str()),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
timeout(TIMEOUT, Socks5Stream::connect(proxy_addr, address)).await
|
||||
};
|
||||
|
||||
result
|
||||
.map_err(|_| {
|
||||
error!(
|
||||
"SOCKS5 proxy connection to {} via {}:{} timed out after {}s",
|
||||
address,
|
||||
addr.host,
|
||||
addr.port,
|
||||
TIMEOUT.as_secs()
|
||||
);
|
||||
raise_error!(
|
||||
format!(
|
||||
"SOCKS5 proxy connection to {} via {}:{} timed out after {}s",
|
||||
address,
|
||||
addr.host,
|
||||
addr.port,
|
||||
TIMEOUT.as_secs()
|
||||
),
|
||||
ErrorCode::ConnectionTimeout
|
||||
)
|
||||
})?
|
||||
.map(|s| s.into_inner())
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError))
|
||||
}
|
||||
|
||||
async fn connect_via_http_proxy(addr: &ProxyAddr, address: SocketAddr) -> BichonResult<TcpStream> {
|
||||
let mut stream = timeout(TIMEOUT, TcpStream::connect((addr.host.as_str(), addr.port)))
|
||||
.await
|
||||
.map_err(|_| {
|
||||
raise_error!(
|
||||
format!(
|
||||
"HTTP proxy connection to {}:{} timed out after {}s",
|
||||
addr.host,
|
||||
addr.port,
|
||||
TIMEOUT.as_secs()
|
||||
),
|
||||
ErrorCode::ConnectionTimeout
|
||||
)
|
||||
})?
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError))?;
|
||||
|
||||
let mut request = format!(
|
||||
"CONNECT {address} HTTP/1.1\r\nHost: {address}\r\nProxy-Connection: keep-alive\r\n"
|
||||
);
|
||||
if let (Some(user), Some(pass)) = (&addr.username, &addr.password) {
|
||||
let auth = general_purpose::STANDARD.encode(format!("{user}:{pass}"));
|
||||
request.push_str(&format!("Proxy-Authorization: Basic {auth}\r\n"));
|
||||
}
|
||||
request.push_str("\r\n");
|
||||
|
||||
timeout(TIMEOUT, stream.write_all(request.as_bytes()))
|
||||
.await
|
||||
.map_err(|_| {
|
||||
raise_error!(
|
||||
format!(
|
||||
"HTTP proxy CONNECT to {} via {}:{} timed out after {}s",
|
||||
address,
|
||||
addr.host,
|
||||
addr.port,
|
||||
TIMEOUT.as_secs()
|
||||
),
|
||||
ErrorCode::ConnectionTimeout
|
||||
)
|
||||
})?
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError))?;
|
||||
|
||||
let mut response = Vec::new();
|
||||
timeout(TIMEOUT, async {
|
||||
let mut byte = [0u8; 1];
|
||||
while !response.ends_with(b"\r\n\r\n") {
|
||||
stream.read_exact(&mut byte).await?;
|
||||
response.push(byte[0]);
|
||||
if response.len() > 8192 {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"HTTP proxy CONNECT response headers are too large",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok::<(), std::io::Error>(())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| {
|
||||
raise_error!(
|
||||
format!(
|
||||
"HTTP proxy CONNECT response from {}:{} timed out after {}s",
|
||||
addr.host,
|
||||
addr.port,
|
||||
TIMEOUT.as_secs()
|
||||
),
|
||||
ErrorCode::ConnectionTimeout
|
||||
)
|
||||
})?
|
||||
.map_err(|e| {
|
||||
if e.kind() == std::io::ErrorKind::InvalidData {
|
||||
raise_error!(e.to_string(), ErrorCode::NetworkError)
|
||||
} else {
|
||||
raise_error!(format!("{:#?}", e), ErrorCode::NetworkError)
|
||||
}
|
||||
})?;
|
||||
|
||||
let response = String::from_utf8_lossy(&response);
|
||||
if response.starts_with("HTTP/1.1 200") || response.starts_with("HTTP/1.0 200") {
|
||||
Ok(stream)
|
||||
} else {
|
||||
Err(raise_error!(
|
||||
format!(
|
||||
"HTTP proxy CONNECT to {} via {}:{} failed: {}",
|
||||
address,
|
||||
addr.host,
|
||||
addr.port,
|
||||
response.lines().next().unwrap_or("invalid response")
|
||||
),
|
||||
ErrorCode::NetworkError
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[package]
|
||||
name = "memdb"
|
||||
name = "bichon-memdb"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use memdb::{DbError, MemDb, Page};
|
||||
use bichon_memdb::{DbError, MemDb, Page};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tempfile::TempDir;
|
||||
|
||||
@@ -458,7 +458,7 @@ fn test_wal_seq_skips_already_snapshotted_entries() {
|
||||
|
||||
// Verify WAL only contains seq=3.
|
||||
let wal_path = dir.path().join("wal.jsonl");
|
||||
let entries = memdb::wal::read_after(&wal_path, 2).unwrap();
|
||||
let entries = bichon_memdb::wal::read_after(&wal_path, 2).unwrap();
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].seq, 3);
|
||||
|
||||
@@ -696,7 +696,7 @@ async fn test_concurrent_writes_wal_seq_monotonic() {
|
||||
|
||||
// Verify WAL seq is strictly monotonic.
|
||||
let wal_path = dir.path().join("wal.jsonl");
|
||||
let entries = memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
assert_eq!(entries.len(), 50);
|
||||
let mut last = 0u64;
|
||||
for e in &entries {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use memdb::{Durability, MemDb, Page};
|
||||
use bichon_memdb::{Durability, MemDb, Page};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -548,7 +548,7 @@ fn stress_wal_seq_monotonic_under_load() {
|
||||
}
|
||||
|
||||
let wal_path = dir.path().join("wal.jsonl");
|
||||
let entries = memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
assert_eq!(entries.len(), n);
|
||||
let mut last = 0u64;
|
||||
for e in &entries {
|
||||
@@ -842,7 +842,7 @@ async fn wal_concurrent_persistent_writes() {
|
||||
|
||||
// Verify strict seq ordering in WAL under concurrent load.
|
||||
let wal_path = db_path.join("wal.jsonl");
|
||||
let entries = memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
assert_eq!(entries.len(), total as usize);
|
||||
let mut last = 0u64;
|
||||
for e in &entries {
|
||||
@@ -1153,7 +1153,7 @@ fn wal_large_transaction_batch() {
|
||||
|
||||
// The entire transaction should be a single WAL entry.
|
||||
let wal_path = dir.path().join("wal.jsonl");
|
||||
let entries = memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].ops.len(), n as usize);
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ timeago.workspace = true
|
||||
chrono.workspace = true
|
||||
tracing.workspace = true
|
||||
tokio.workspace = true
|
||||
futures.workspace = true
|
||||
http.workspace = true
|
||||
urlencoding.workspace = true
|
||||
|
||||
|
||||
@@ -16,14 +16,33 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::common::auth::WrappedContext;
|
||||
use crate::rest::api::ApiTags;
|
||||
use crate::rest::ApiResult;
|
||||
use bichon_core::import::BatchEmlResult;
|
||||
use bichon_core::import::{BatchEmlRequest, ImportEmls};
|
||||
use bichon_core::account::migration::AccountModel;
|
||||
use bichon_core::database::manager::DB_MANAGER;
|
||||
use bichon_core::database::MemDbModel;
|
||||
use bichon_core::import::{
|
||||
check_temp_disk_space, get_import_progress, process_uploaded_file, update_progress,
|
||||
BatchEmlRequest, BatchEmlResult, ImportEmls, ImportHistory, ImportProgress, ImportStatus,
|
||||
MAX_WEB_EML_BYTES,
|
||||
};
|
||||
use bichon_core::import::history::{save_import_history, MAX_HISTORY_PER_USER};
|
||||
use bichon_core::raise_error;
|
||||
use bichon_core::error::code::ErrorCode;
|
||||
use bichon_core::settings::cli::SETTINGS;
|
||||
use bichon_core::settings::dir::DATA_DIR_MANAGER;
|
||||
use bichon_core::users::permissions::Permission;
|
||||
use poem_openapi::payload::Json;
|
||||
use bichon_core::import::detect_text_file;
|
||||
use bichon_core::import::FileFormat;
|
||||
use futures::StreamExt;
|
||||
use poem::Body;
|
||||
use poem_openapi::param::{Path, Query};
|
||||
use poem_openapi::payload::{Json, Binary};
|
||||
use poem_openapi::OpenApi;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
pub struct ImportApi;
|
||||
|
||||
@@ -44,7 +63,362 @@ impl ImportApi {
|
||||
payload: Json<BatchEmlRequest>,
|
||||
context: WrappedContext,
|
||||
) -> ApiResult<Json<BatchEmlResult>> {
|
||||
context.require_permission(Some(payload.0.account_id), Permission::DATA_IMPORT_BATCH)?;
|
||||
Ok(Json(ImportEmls::do_import(payload.0).await?))
|
||||
let account_id = payload.0.account_id;
|
||||
let folder = payload.0.mail_folder.clone();
|
||||
context.require_permission(Some(account_id), Permission::DATA_IMPORT_BATCH)?;
|
||||
let result = ImportEmls::do_import(payload.0).await?;
|
||||
|
||||
// Save import history
|
||||
let progress = ImportProgress {
|
||||
import_id: format!(
|
||||
"batch_{:x}",
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_nanos()
|
||||
),
|
||||
status: if result.failed == 0 {
|
||||
ImportStatus::Completed
|
||||
} else if result.success == 0 {
|
||||
ImportStatus::Failed
|
||||
} else {
|
||||
ImportStatus::Completed
|
||||
},
|
||||
format: "eml".to_string(),
|
||||
total: result.total,
|
||||
success: result.success,
|
||||
duplicates: result.duplicates,
|
||||
failed: result.failed,
|
||||
failed_details: result.failed_details.clone(),
|
||||
};
|
||||
save_import_history(context.user.id, account_id, &folder, &progress);
|
||||
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
||||
/// Upload an EML or MBOX file for import into a NoSync account.
|
||||
///
|
||||
/// The file is sent as the raw request body. Both `account_id` and `mail_folder`
|
||||
/// must be provided as query parameters, along with the original `file_name` for
|
||||
/// extension validation.
|
||||
///
|
||||
/// Returns an `import_id` to poll for progress via `/import-progress/:import_id`.
|
||||
#[oai(path = "/upload-import", method = "post", operation_id = "upload_import")]
|
||||
async fn upload_import(
|
||||
&self,
|
||||
/// Target account ID (must be NoSync type).
|
||||
account_id: Query<u64>,
|
||||
/// Target mail folder name.
|
||||
mail_folder: Query<String>,
|
||||
/// Original file name, used for extension validation (e.g. "export.eml").
|
||||
file_name: Query<String>,
|
||||
/// The raw file bytes (.eml or .mbox).
|
||||
data: Binary<Body>,
|
||||
context: WrappedContext,
|
||||
) -> ApiResult<Json<ImportProgress>> {
|
||||
let account_id = account_id.0;
|
||||
context.require_permission(Some(account_id), Permission::DATA_IMPORT_BATCH)?;
|
||||
|
||||
// Basic account validation (fails fast)
|
||||
AccountModel::check_account_exists(account_id)?;
|
||||
|
||||
let folder = mail_folder.0.trim().to_string();
|
||||
if folder.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"mail_folder is required.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
))?;
|
||||
}
|
||||
|
||||
let file_name = file_name.0.trim().to_string();
|
||||
if file_name.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"file_name is required.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
))?;
|
||||
}
|
||||
|
||||
// Validate file extension
|
||||
let ext_lower = std::path::Path::new(&file_name)
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| e.to_lowercase())
|
||||
.unwrap_or_default();
|
||||
let is_mbox_ext = ext_lower == "mbox";
|
||||
let is_eml_ext = ext_lower == "eml";
|
||||
let is_pst_ext = ext_lower == "pst";
|
||||
if !is_mbox_ext && !is_eml_ext && !is_pst_ext {
|
||||
return Err(raise_error!(
|
||||
format!(
|
||||
"Unsupported file type '.{}'. Only .eml, .mbox and .pst files are allowed.",
|
||||
ext_lower
|
||||
),
|
||||
ErrorCode::InvalidParameter
|
||||
))?;
|
||||
}
|
||||
|
||||
// Check disk space (fail fast before streaming)
|
||||
let max_mbox = SETTINGS.bichon_web_mbox_upload_limit_mb as usize * 1024 * 1024;
|
||||
let max_pst = SETTINGS.bichon_web_pst_upload_limit_mb as usize * 1024 * 1024;
|
||||
let min_required = if is_mbox_ext {
|
||||
max_mbox
|
||||
} else if is_pst_ext {
|
||||
max_pst
|
||||
} else {
|
||||
MAX_WEB_EML_BYTES
|
||||
};
|
||||
let free = check_temp_disk_space()?;
|
||||
if free < min_required as u64 * 2 {
|
||||
let free_gb = free as f64 / 1024.0 / 1024.0 / 1024.0;
|
||||
let need_gb = (min_required as f64 * 2.0) / 1024.0 / 1024.0 / 1024.0;
|
||||
return Err(raise_error!(
|
||||
format!(
|
||||
"Insufficient disk space. Free: {:.1} GB. Need at least {:.1} GB.",
|
||||
free_gb, need_gb
|
||||
),
|
||||
ErrorCode::InvalidParameter
|
||||
))?;
|
||||
}
|
||||
|
||||
// Stream body to temp file, enforcing size limits and validating content
|
||||
let import_id = format!(
|
||||
"imp_{:x}",
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_nanos()
|
||||
);
|
||||
let temp_path = DATA_DIR_MANAGER.temp_dir.join(format!("import_{}.tmp", import_id));
|
||||
|
||||
let (format_detected, file_len) = stream_body_to_temp(
|
||||
data.0,
|
||||
&temp_path,
|
||||
is_mbox_ext,
|
||||
is_pst_ext,
|
||||
).await?;
|
||||
|
||||
let format = format_detected.unwrap_or_else(|| {
|
||||
if is_mbox_ext {
|
||||
FileFormat::Mbox
|
||||
} else if is_pst_ext {
|
||||
FileFormat::Pst
|
||||
} else {
|
||||
FileFormat::Eml
|
||||
}
|
||||
});
|
||||
|
||||
let format_str = match format {
|
||||
FileFormat::Mbox => "mbox".to_string(),
|
||||
FileFormat::Eml => "eml".to_string(),
|
||||
FileFormat::Pst => "pst".to_string(),
|
||||
};
|
||||
|
||||
let max_mbox = SETTINGS.bichon_web_mbox_upload_limit_mb as usize * 1024 * 1024;
|
||||
let max_pst = SETTINGS.bichon_web_pst_upload_limit_mb as usize * 1024 * 1024;
|
||||
let max_size = match format {
|
||||
FileFormat::Mbox => max_mbox,
|
||||
FileFormat::Pst => max_pst,
|
||||
FileFormat::Eml => MAX_WEB_EML_BYTES,
|
||||
};
|
||||
if file_len > max_size {
|
||||
let _ = std::fs::remove_file(&temp_path);
|
||||
let max_mb = max_size as f64 / 1024.0 / 1024.0;
|
||||
let actual_mb = file_len as f64 / 1024.0 / 1024.0;
|
||||
return Err(raise_error!(
|
||||
format!(
|
||||
"File too large ({:.1} MB). Maximum for {} is {:.0} MB. Use the CLI for larger files.",
|
||||
actual_mb, format_str.to_uppercase(), max_mb
|
||||
),
|
||||
ErrorCode::InvalidParameter
|
||||
))?;
|
||||
}
|
||||
|
||||
// Record initial progress
|
||||
let initial = ImportProgress {
|
||||
import_id: import_id.clone(),
|
||||
status: ImportStatus::Pending,
|
||||
format: format_str.clone(),
|
||||
total: 0,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![],
|
||||
};
|
||||
|
||||
// Store initial progress so polling can find it immediately
|
||||
update_progress(&import_id, initial.clone());
|
||||
|
||||
// Spawn background processing
|
||||
let id = import_id.clone();
|
||||
let folder_clone = folder.clone();
|
||||
let user_id = context.user.id;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
process_uploaded_file(&id, &temp_path, &file_name, account_id, &folder_clone, user_id);
|
||||
});
|
||||
|
||||
Ok(Json(initial))
|
||||
}
|
||||
|
||||
/// Poll import progress by import ID.
|
||||
#[oai(
|
||||
path = "/import-progress/:import_id",
|
||||
method = "get",
|
||||
operation_id = "get_import_progress"
|
||||
)]
|
||||
async fn get_import_progress(
|
||||
&self,
|
||||
import_id: Path<String>,
|
||||
context: WrappedContext,
|
||||
) -> ApiResult<Json<ImportProgress>> {
|
||||
let _ = context; // progress queries don't need per-account auth
|
||||
match get_import_progress(&import_id.0) {
|
||||
Some(progress) => Ok(Json(progress)),
|
||||
None => Err(raise_error!(
|
||||
format!("Import {} not found.", import_id.0),
|
||||
ErrorCode::ResourceNotFound
|
||||
))?,
|
||||
}
|
||||
}
|
||||
|
||||
/// Check available disk space on the server's temp directory.
|
||||
#[oai(
|
||||
path = "/check-disk-space",
|
||||
method = "get",
|
||||
operation_id = "check_disk_space"
|
||||
)]
|
||||
async fn check_disk_space(&self, _context: WrappedContext) -> ApiResult<Json<u64>> {
|
||||
let free = check_temp_disk_space()?;
|
||||
Ok(Json(free))
|
||||
}
|
||||
|
||||
/// List import history for the current user (latest first, up to 5 entries).
|
||||
#[oai(
|
||||
path = "/import-history",
|
||||
method = "get",
|
||||
operation_id = "list_import_history"
|
||||
)]
|
||||
async fn list_import_history(
|
||||
&self,
|
||||
context: WrappedContext,
|
||||
) -> ApiResult<Json<Vec<ImportHistory>>> {
|
||||
let prefix = format!("{}:", context.user.id);
|
||||
let coll = DB_MANAGER.db().collection(ImportHistory::collection());
|
||||
let mut entries: Vec<ImportHistory> = coll
|
||||
.scan_prefix(&prefix)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
// Sort by created_at descending (newest first), keep at most N per user
|
||||
entries.sort_by(|a, b| b.created_at.cmp(&a.created_at));
|
||||
entries.truncate(MAX_HISTORY_PER_USER);
|
||||
Ok(Json(entries))
|
||||
}
|
||||
}
|
||||
|
||||
/// Stream a poem `Body` to a temp file while enforcing size limits and
|
||||
/// validating that the content looks like a text-based email file.
|
||||
///
|
||||
/// PST files are binary (OLE2) — text detection is skipped for them.
|
||||
///
|
||||
/// Returns the detected format (if any) and the total bytes written.
|
||||
async fn stream_body_to_temp(
|
||||
body: Body,
|
||||
temp_path: &std::path::Path,
|
||||
is_mbox_ext: bool,
|
||||
is_pst_ext: bool,
|
||||
) -> ApiResult<(Option<FileFormat>, usize)> {
|
||||
let max_mbox = SETTINGS.bichon_web_mbox_upload_limit_mb as usize * 1024 * 1024;
|
||||
let max_pst = SETTINGS.bichon_web_pst_upload_limit_mb as usize * 1024 * 1024;
|
||||
let max_stream = if is_mbox_ext {
|
||||
max_mbox
|
||||
} else if is_pst_ext {
|
||||
max_pst
|
||||
} else {
|
||||
MAX_WEB_EML_BYTES
|
||||
};
|
||||
|
||||
let mut file = tokio::fs::File::create(temp_path).await.map_err(|e| {
|
||||
raise_error!(
|
||||
format!("Failed to create temp file: {}", e),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut body_stream = body.into_bytes_stream();
|
||||
let mut total: usize = 0;
|
||||
let mut first_chunk: Vec<u8> = Vec::new();
|
||||
let mut format_detected: Option<FileFormat> = None;
|
||||
let mut text_checked = false;
|
||||
|
||||
while let Some(chunk_result) = body_stream.next().await {
|
||||
let chunk = chunk_result.map_err(|e| {
|
||||
raise_error!(
|
||||
format!("Failed to read request body: {}", e),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
|
||||
total += chunk.len();
|
||||
|
||||
// Enforce size limit during streaming
|
||||
if total > max_stream {
|
||||
// Clean up partial temp file
|
||||
drop(file);
|
||||
let _ = tokio::fs::remove_file(temp_path).await;
|
||||
let max_mb = max_stream as f64 / 1024.0 / 1024.0;
|
||||
return Err(raise_error!(
|
||||
format!(
|
||||
"Upload exceeds maximum size of {:.0} MB. Use the CLI for larger files.",
|
||||
max_mb
|
||||
),
|
||||
ErrorCode::InvalidParameter
|
||||
))?;
|
||||
}
|
||||
|
||||
// Accumulate first ~8 KB for format & text detection
|
||||
if first_chunk.len() < 8192 {
|
||||
let remaining = 8192 - first_chunk.len();
|
||||
first_chunk.extend_from_slice(&chunk[..chunk.len().min(remaining)]);
|
||||
}
|
||||
|
||||
// Once we have enough data, validate format and text
|
||||
if first_chunk.len() >= 512 && !text_checked {
|
||||
text_checked = true;
|
||||
format_detected = bichon_core::import::detect_format(&first_chunk, "upload");
|
||||
|
||||
// If extension is .eml but content looks like MBOX (or vice versa), that's OK.
|
||||
// PST files are binary — skip text detection.
|
||||
if !is_pst_ext && !detect_text_file(&first_chunk) {
|
||||
drop(file);
|
||||
let _ = tokio::fs::remove_file(temp_path).await;
|
||||
return Err(raise_error!(
|
||||
"The uploaded file appears to be binary (not a valid email file). Only .eml, .mbox and .pst files are accepted.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
))?;
|
||||
}
|
||||
}
|
||||
|
||||
file.write_all(&chunk).await.map_err(|e| {
|
||||
raise_error!(
|
||||
format!("Failed to write temp file: {}", e),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
}
|
||||
|
||||
file.flush().await.map_err(|e| {
|
||||
raise_error!(
|
||||
format!("Failed to flush temp file: {}", e),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
|
||||
// If file is empty, reject
|
||||
if total == 0 {
|
||||
let _ = tokio::fs::remove_file(temp_path).await;
|
||||
return Err(raise_error!(
|
||||
"Empty file is not allowed.".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
))?;
|
||||
}
|
||||
|
||||
Ok((format_detected, total))
|
||||
}
|
||||
|
||||
@@ -251,7 +251,7 @@ impl MessageApi {
|
||||
Ok(restore_emails(account_id, payload.0.envelope_ids).await?)
|
||||
}
|
||||
|
||||
/// Downloads a specific attachment from an email. Requires `name` query parameter.
|
||||
/// Downloads a specific attachment from an email. Requires `content_hash` query parameter.
|
||||
#[oai(
|
||||
path = "/download-attachment/:account_id/:envelope_id",
|
||||
method = "get",
|
||||
@@ -280,6 +280,33 @@ impl MessageApi {
|
||||
Ok(attachment)
|
||||
}
|
||||
|
||||
/// Returns raw attachment content for in-browser preview with
|
||||
/// `Content-Disposition: inline` and the correct MIME type.
|
||||
#[oai(
|
||||
path = "/preview-attachment/:account_id/:envelope_id",
|
||||
method = "get",
|
||||
operation_id = "preview_attachment"
|
||||
)]
|
||||
async fn preview_attachment(
|
||||
&self,
|
||||
/// The ID of the account.
|
||||
account_id: Path<u64>,
|
||||
/// The ID of the message containing the attachment.
|
||||
envelope_id: Path<String>,
|
||||
/// The content_hash of the attachment to preview.
|
||||
content_hash: Query<String>,
|
||||
context: WrappedContext,
|
||||
) -> ApiResult<Attachment<Body>> {
|
||||
let account_id = account_id.0;
|
||||
let envelope_id = envelope_id.0.trim().to_string();
|
||||
AccountModel::check_account_exists(account_id)?;
|
||||
context.require_permission(Some(account_id), Permission::DATA_READ)?;
|
||||
let content_hash = content_hash.0.trim();
|
||||
let reader = retrieve_attachment_content(account_id, envelope_id, content_hash)?;
|
||||
let body = Body::from_async_read(reader);
|
||||
Ok(Attachment::new(body).attachment_type(AttachmentType::Inline))
|
||||
}
|
||||
|
||||
/// Downloads an attachment from within a nested email (EML file).
|
||||
#[oai(
|
||||
path = "/download-nested-attachment/:account_id/:envelope_id",
|
||||
|
||||
@@ -23,7 +23,7 @@ use bichon_core::dashboard::DashboardStats;
|
||||
use bichon_core::error::code::ErrorCode;
|
||||
use bichon_core::raise_error;
|
||||
use bichon_core::settings::cli::SETTINGS;
|
||||
use bichon_core::settings::proxy::Proxy;
|
||||
use bichon_core::settings::proxy::{Proxy, ProxyTestResult};
|
||||
use bichon_core::settings::SystemConfigurations;
|
||||
use bichon_core::users::permissions::Permission;
|
||||
use bichon_core::version::{fetch_notifications, Notifications};
|
||||
@@ -70,7 +70,7 @@ impl SystemApi {
|
||||
Ok(Json(stats))
|
||||
}
|
||||
|
||||
/// Get the full list of SOCKS5 proxy configurations.
|
||||
/// Get the full list of proxy configurations.
|
||||
#[oai(method = "get", path = "/list-proxy", operation_id = "list_proxy")]
|
||||
async fn list_proxy(&self, _context: WrappedContext) -> ApiResult<Json<Vec<Proxy>>> {
|
||||
//The proxy list is visible to all users.
|
||||
@@ -103,6 +103,17 @@ impl SystemApi {
|
||||
Ok(Json(Proxy::get(id.0)?))
|
||||
}
|
||||
|
||||
/// Test whether a proxy can reach a geo lookup service. Requires root permission.
|
||||
#[oai(path = "/proxy/:id/test", method = "post", operation_id = "test_proxy")]
|
||||
async fn test_proxy(
|
||||
&self,
|
||||
id: Path<u64>,
|
||||
context: WrappedContext,
|
||||
) -> ApiResult<Json<ProxyTestResult>> {
|
||||
context.require_permission(None, Permission::ROOT)?;
|
||||
Ok(Json(Proxy::test(id.0).await?))
|
||||
}
|
||||
|
||||
/// Create a new proxy configuration. Requires root permission.
|
||||
#[oai(path = "/proxy", method = "post", operation_id = "create_proxy")]
|
||||
async fn create_proxy(&self, url: PlainText<String>, context: WrappedContext) -> ApiResult<()> {
|
||||
@@ -122,6 +133,7 @@ impl SystemApi {
|
||||
context.require_permission(None, Permission::ROOT)?;
|
||||
Ok(Proxy::update(id.0, url.0)?)
|
||||
}
|
||||
|
||||
/// Get system configurations.
|
||||
///
|
||||
/// Returns a read-only snapshot of the server configuration
|
||||
|
||||
@@ -21,6 +21,7 @@ use std::net::SocketAddr;
|
||||
use std::time::Duration;
|
||||
|
||||
use base64::{prelude::BASE64_STANDARD, Engine as _};
|
||||
use bichon_core::account::migration::AccountType;
|
||||
use bichon_core::cache::imap::mailbox::{Attribute, AttributeEnum};
|
||||
use bichon_core::common::signal::SIGNAL_MANAGER;
|
||||
use bichon_core::envelope::extractor::extract_envelope_from_smtp;
|
||||
@@ -429,8 +430,20 @@ where
|
||||
}
|
||||
|
||||
if is_allowed {
|
||||
session.rcpt_to.push(account);
|
||||
stream.write_all(b"250 OK\r\n").await?;
|
||||
if !matches!(account.account_type, AccountType::NoSync) {
|
||||
tracing::warn!(
|
||||
"SMTP: Rejected journaling attempt to IMAP account <{}>",
|
||||
addr
|
||||
);
|
||||
let err = format!(
|
||||
"550 5.7.1 <{}>: Not a Bichon local account, journaling is not supported\r\n",
|
||||
account.email
|
||||
);
|
||||
stream.write_all(err.as_bytes()).await?;
|
||||
} else {
|
||||
session.rcpt_to.push(account);
|
||||
stream.write_all(b"250 OK\r\n").await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
@@ -615,26 +628,40 @@ async fn parse_email(data: &[u8], session: &Session) -> BichonResult<()> {
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let mailbox = MailBox {
|
||||
id: create_hash(rcpt.id, "INBOX"),
|
||||
account_id: rcpt.id,
|
||||
name: "INBOX".into(),
|
||||
delimiter: Some("/".to_string()),
|
||||
attributes: vec![Attribute {
|
||||
attr: AttributeEnum::Extension,
|
||||
extension: Some("CreatedByBichon".into()),
|
||||
}],
|
||||
exists: 0,
|
||||
unseen: None,
|
||||
uid_next: None,
|
||||
uid_validity: None,
|
||||
highest_uid: None,
|
||||
};
|
||||
let mailbox_id = mailbox.id;
|
||||
let mailbox_id = create_hash(rcpt.id, "INBOX");
|
||||
|
||||
if let Err(e) = MailBox::batch_upsert(&[mailbox]) {
|
||||
tracing::error!("SMTP: Failed to upsert mailbox for {}: {:?}", rcpt.email, e);
|
||||
return Err(e.into());
|
||||
// The INBOX row is owned by the IMAP sync, which maintains `uid_validity`,
|
||||
// `highest_uid` and `uid_next` on it. `batch_upsert` replaces the *whole*
|
||||
// row, so blindly upserting here (with those fields = None) clobbers the
|
||||
// IMAP-maintained state back to None. The next reconcile then sees
|
||||
// `uid_validity` change from Some -> None, treats the mailbox as invalid,
|
||||
// and wipes + rebuilds it — silently losing the local copy of a large
|
||||
// mailbox when that rebuild is interrupted (see #297).
|
||||
//
|
||||
// We only need the row to *exist* so the journaled envelope can attach to
|
||||
// it, so create it only when it is missing and otherwise leave the
|
||||
// IMAP-owned row untouched.
|
||||
if MailBox::find_mailbox(rcpt.id, mailbox_id)?.is_none() {
|
||||
let mailbox = MailBox {
|
||||
id: mailbox_id,
|
||||
account_id: rcpt.id,
|
||||
name: "INBOX".into(),
|
||||
delimiter: Some("/".to_string()),
|
||||
attributes: vec![Attribute {
|
||||
attr: AttributeEnum::Extension,
|
||||
extension: Some("CreatedByBichon".into()),
|
||||
}],
|
||||
exists: 0,
|
||||
unseen: None,
|
||||
uid_next: None,
|
||||
uid_validity: None,
|
||||
highest_uid: None,
|
||||
};
|
||||
|
||||
if let Err(e) = MailBox::batch_upsert(&[mailbox]) {
|
||||
tracing::error!("SMTP: Failed to upsert mailbox for {}: {:?}", rcpt.email, e);
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
|
||||
extract_envelope_from_smtp(data, rcpt.id, mailbox_id)
|
||||
|
||||
@@ -23,6 +23,7 @@ import { PaginatedResponse } from "..";
|
||||
export interface MinimalAccount {
|
||||
id: number;
|
||||
email: string;
|
||||
name?: string;
|
||||
}
|
||||
|
||||
export const minimal_account_list = async () => {
|
||||
@@ -117,6 +118,20 @@ interface DateSelection {
|
||||
|
||||
|
||||
export type QuotaWindow = 'hourly' | 'daily' | 'weekly' | 'monthly'
|
||||
|
||||
export interface FilterRule {
|
||||
include: string[];
|
||||
exclude: string[];
|
||||
}
|
||||
|
||||
export interface ArchiveRules {
|
||||
enabled: boolean;
|
||||
senders: FilterRule;
|
||||
subjects: FilterRule;
|
||||
skip_larger_than?: number;
|
||||
spam_headers: string[];
|
||||
}
|
||||
|
||||
export interface AccountModel {
|
||||
id: number;
|
||||
account_type: AccountType;
|
||||
@@ -144,6 +159,7 @@ export interface AccountModel {
|
||||
imap_quota_bytes?: number;
|
||||
auto_download_new_mailboxes?: boolean;
|
||||
download_schedule?: string;
|
||||
archive_rules?: ArchiveRules;
|
||||
deleting?: boolean;
|
||||
}
|
||||
|
||||
|
||||
82
web/src/api/import/api.ts
Normal file
82
web/src/api/import/api.ts
Normal file
@@ -0,0 +1,82 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
|
||||
import axiosInstance from '@/api/axiosInstance';
|
||||
import { list_accounts } from '@/api/account/api';
|
||||
import type { AccountModel } from '@/api/account/api';
|
||||
|
||||
export interface ImportProgress {
|
||||
import_id: string;
|
||||
status: 'Pending' | 'Processing' | 'Completed' | 'Failed';
|
||||
format: string;
|
||||
total: number;
|
||||
success: number;
|
||||
duplicates: number;
|
||||
failed: number;
|
||||
failed_details: { index: number; error_message: string }[];
|
||||
}
|
||||
|
||||
export const upload_import = async (
|
||||
accountId: number,
|
||||
mailFolder: string,
|
||||
fileName: string,
|
||||
file: File,
|
||||
onProgress?: (pct: number) => void
|
||||
): Promise<ImportProgress> => {
|
||||
const response = await axiosInstance.post<ImportProgress>(
|
||||
`api/v1/upload-import`,
|
||||
file,
|
||||
{
|
||||
params: { account_id: accountId, mail_folder: mailFolder, file_name: fileName },
|
||||
headers: { 'Content-Type': 'application/octet-stream' },
|
||||
onUploadProgress: (e) => {
|
||||
if (e.total && onProgress) onProgress(Math.round((e.loaded / e.total) * 100));
|
||||
},
|
||||
}
|
||||
);
|
||||
return response.data;
|
||||
};
|
||||
|
||||
export const get_import_progress = async (importId: string): Promise<ImportProgress> => {
|
||||
const response = await axiosInstance.get<ImportProgress>(
|
||||
`api/v1/import-progress/${importId}`
|
||||
);
|
||||
return response.data;
|
||||
};
|
||||
|
||||
export const check_disk_space = async (): Promise<number> => {
|
||||
const response = await axiosInstance.get<number>('api/v1/check-disk-space');
|
||||
return response.data;
|
||||
};
|
||||
|
||||
export const get_nosync_accounts = async (): Promise<AccountModel[]> => {
|
||||
const data = await list_accounts();
|
||||
return (data.items || []).filter(
|
||||
(a) => a.account_type === 'NoSync' && a.enabled
|
||||
);
|
||||
};
|
||||
|
||||
// ── Import history ────────────────────────────────────────────────
|
||||
|
||||
export interface ImportHistory {
|
||||
id: string;
|
||||
user_id: number;
|
||||
import_id: string;
|
||||
account_id: number;
|
||||
folder: string;
|
||||
format: string;
|
||||
status: 'pending' | 'processing' | 'completed' | 'failed';
|
||||
total: number;
|
||||
success: number;
|
||||
duplicates: number;
|
||||
failed: number;
|
||||
failed_details: { index: number; error_message: string }[];
|
||||
created_at: number;
|
||||
}
|
||||
|
||||
export const list_import_history = async (): Promise<ImportHistory[]> => {
|
||||
const response = await axiosInstance.get<ImportHistory[]>('api/v1/import-history');
|
||||
return response.data;
|
||||
};
|
||||
@@ -32,6 +32,7 @@ export interface EmailEnvelope {
|
||||
account_id: number;
|
||||
mailbox_id: number;
|
||||
account_email: string;
|
||||
account_name?: string;
|
||||
mailbox_name: string;
|
||||
uid: number;
|
||||
subject: string;
|
||||
|
||||
@@ -41,6 +41,18 @@ export const download_attachment = async (accountId: number, id: string, content
|
||||
saveAs(blob, fileName);
|
||||
};
|
||||
|
||||
/** Fetch raw attachment content for in-browser preview (Content-Disposition: inline). */
|
||||
export const preview_attachment = async (accountId: number, id: string, content_hash: string) => {
|
||||
const response = await axiosInstance.get(
|
||||
`api/v1/preview-attachment/${accountId}/${id}`,
|
||||
{
|
||||
params: { content_hash },
|
||||
responseType: 'blob',
|
||||
}
|
||||
);
|
||||
return response.data as Blob;
|
||||
};
|
||||
|
||||
export const download_nested_attachment = async (accountId: number, id: string, content_hash: string, nested_content_hash: string, fileName: string) => {
|
||||
const response = await axiosInstance.get(`api/v1/download-nested-attachment/${accountId}/${id}?content_hash=${content_hash}&nested_content_hash=${nested_content_hash}`, { responseType: 'blob' });
|
||||
const blob = new Blob([response.data]);
|
||||
|
||||
@@ -105,6 +105,14 @@ export interface Proxy {
|
||||
updated_at: number;
|
||||
}
|
||||
|
||||
export interface ProxyTestResult {
|
||||
ip?: string | null;
|
||||
country?: string | null;
|
||||
region?: string | null;
|
||||
city?: string | null;
|
||||
isp?: string | null;
|
||||
}
|
||||
|
||||
export type ServerConfigurations = {
|
||||
bichon_log_level: string
|
||||
bichon_http_port: number
|
||||
@@ -138,6 +146,10 @@ export type ServerConfigurations = {
|
||||
bichon_smtp_auth_required: boolean
|
||||
bichon_smtp_tls_key_path?: string | null
|
||||
bichon_smtp_tls_cert_path?: string | null
|
||||
|
||||
bichon_upload_body_limit_mb: number
|
||||
bichon_web_mbox_upload_limit_mb: number
|
||||
bichon_web_pst_upload_limit_mb: number
|
||||
}
|
||||
|
||||
export const get_dashboard_stats = async () => {
|
||||
@@ -164,6 +176,11 @@ export const update_proxy = async (id: number, url: string) => {
|
||||
return response.data;
|
||||
};
|
||||
|
||||
export const test_proxy = async (id: number) => {
|
||||
const response = await axiosInstance.post<ProxyTestResult>(`api/v1/proxy/${id}/test`);
|
||||
return response.data;
|
||||
};
|
||||
|
||||
export const add_proxy = async (url: string) => {
|
||||
const response = await axiosInstance.post(`api/v1/proxy`, url, {
|
||||
headers: {
|
||||
@@ -177,4 +194,4 @@ export const add_proxy = async (url: string) => {
|
||||
export const get_system_configurations = async () => {
|
||||
const response = await axiosInstance.get<ServerConfigurations>(`api/v1/system-configurations`);
|
||||
return response.data;
|
||||
};
|
||||
};
|
||||
|
||||
@@ -22,7 +22,7 @@ import {
|
||||
IconLayoutDashboard,
|
||||
IconSettings
|
||||
} from '@tabler/icons-react'
|
||||
import { IdCard, Inbox, Paperclip, Search, Users2 } from 'lucide-react'
|
||||
import { IdCard, Inbox, Paperclip, Search, Upload, Users2 } from 'lucide-react'
|
||||
import { type SidebarData } from '../types'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useCurrentUser } from '@/hooks/use-current-user'
|
||||
@@ -57,6 +57,12 @@ export function useSidebarData(): SidebarData {
|
||||
url: '/search',
|
||||
icon: Search,
|
||||
},
|
||||
{
|
||||
title: t('import.title', 'Import'),
|
||||
url: '/import',
|
||||
icon: Upload,
|
||||
visible: require_any_permission(['data:import:batch']),
|
||||
},
|
||||
{
|
||||
title: t('navigation.attachment'),
|
||||
url: '/attachment',
|
||||
|
||||
67
web/src/components/ui/breadcrumb.tsx
Normal file
67
web/src/components/ui/breadcrumb.tsx
Normal file
@@ -0,0 +1,67 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import { ChevronRight, Home } from "lucide-react";
|
||||
import { Link } from "@tanstack/react-router";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
export interface BreadcrumbItem {
|
||||
label: string;
|
||||
to?: string;
|
||||
}
|
||||
|
||||
interface BreadcrumbProps {
|
||||
items: BreadcrumbItem[];
|
||||
className?: string;
|
||||
}
|
||||
|
||||
export function Breadcrumb({ items, className }: BreadcrumbProps) {
|
||||
return (
|
||||
<nav
|
||||
aria-label="Breadcrumb"
|
||||
className={cn("flex items-center gap-1.5 text-sm text-muted-foreground", className)}
|
||||
>
|
||||
<Link
|
||||
to="/"
|
||||
className="flex items-center gap-1 hover:text-foreground transition-colors"
|
||||
>
|
||||
<Home className="h-3.5 w-3.5" />
|
||||
</Link>
|
||||
{items.map((item, idx) => {
|
||||
const isLast = idx === items.length - 1;
|
||||
return (
|
||||
<div key={idx} className="flex items-center gap-1.5">
|
||||
<ChevronRight className="h-3.5 w-3.5" />
|
||||
{item.to && !isLast ? (
|
||||
<Link
|
||||
to={item.to}
|
||||
className="hover:text-foreground transition-colors"
|
||||
>
|
||||
{item.label}
|
||||
</Link>
|
||||
) : (
|
||||
<span className={cn(isLast && "text-foreground font-medium")}>
|
||||
{item.label}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</nav>
|
||||
);
|
||||
}
|
||||
@@ -30,8 +30,11 @@ DialogOverlay.displayName = DialogPrimitive.Overlay.displayName
|
||||
|
||||
const DialogContent = React.forwardRef<
|
||||
React.ElementRef<typeof DialogPrimitive.Content>,
|
||||
React.ComponentPropsWithoutRef<typeof DialogPrimitive.Content>
|
||||
>(({ className, children, ...props }, ref) => {
|
||||
React.ComponentPropsWithoutRef<typeof DialogPrimitive.Content> & {
|
||||
hideClose?: boolean;
|
||||
hideFullscreen?: boolean;
|
||||
}
|
||||
>(({ className, children, hideClose, hideFullscreen, ...props }, ref) => {
|
||||
const [isFullscreen, setIsFullscreen] = React.useState(false);
|
||||
return <DialogPortal>
|
||||
<DialogOverlay>
|
||||
@@ -45,17 +48,23 @@ const DialogContent = React.forwardRef<
|
||||
{...props}
|
||||
>
|
||||
{children}
|
||||
<div className='absolute right-4 top-4 flex items-center gap-2'>
|
||||
{isFullscreen ? (
|
||||
<Minimize onClick={() => setIsFullscreen(!isFullscreen)} className='rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground h-4 w-4' />
|
||||
) : (
|
||||
<Maximize onClick={() => setIsFullscreen(!isFullscreen)} className='rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground h-4 w-4' />
|
||||
)}
|
||||
<DialogPrimitive.Close className='rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground'>
|
||||
<X className='h-4 w-4' />
|
||||
<span className='sr-only'>Close</span>
|
||||
</DialogPrimitive.Close>
|
||||
</div>
|
||||
{(!hideClose || !hideFullscreen) && (
|
||||
<div className='absolute right-4 top-4 flex items-center gap-2'>
|
||||
{!hideFullscreen && (
|
||||
isFullscreen ? (
|
||||
<Minimize onClick={() => setIsFullscreen(!isFullscreen)} className='rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground h-4 w-4' />
|
||||
) : (
|
||||
<Maximize onClick={() => setIsFullscreen(!isFullscreen)} className='rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground h-4 w-4' />
|
||||
)
|
||||
)}
|
||||
{!hideClose && (
|
||||
<DialogPrimitive.Close className='rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground'>
|
||||
<X className='h-4 w-4' />
|
||||
<span className='sr-only'>Close</span>
|
||||
</DialogPrimitive.Close>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</DialogPrimitive.Content>
|
||||
</DialogOverlay>
|
||||
</DialogPortal>
|
||||
|
||||
260
web/src/features/accounts/account-new.tsx
Normal file
260
web/src/features/accounts/account-new.tsx
Normal file
@@ -0,0 +1,260 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import { useCallback, useState } from "react";
|
||||
import { useForm, FormProvider } from "react-hook-form";
|
||||
import { zodResolver } from "@hookform/resolvers/zod";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useMutation, useQueryClient } from "@tanstack/react-query";
|
||||
import { useNavigate, Link } from "@tanstack/react-router";
|
||||
import { ArrowLeft, Loader2 } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Form } from "@/components/ui/form";
|
||||
import { useToast } from "@/hooks/use-toast";
|
||||
import { ToastAction } from "@/components/ui/toast";
|
||||
import { Breadcrumb } from "@/components/ui/breadcrumb";
|
||||
import { FixedHeader } from "@/components/layout/fixed-header";
|
||||
import { Main } from "@/components/layout/main";
|
||||
import { TabGeneral } from "./components/tab-general";
|
||||
import { TabServer } from "./components/tab-server";
|
||||
import { TabDownload } from "./components/tab-download";
|
||||
import { TabFilters } from "./components/tab-filters";
|
||||
import { create_account, autoconfig } from "@/api/account/api";
|
||||
import { getAccountSchema, type AccountFormValues } from "./components/schema";
|
||||
import type { AxiosError } from "axios";
|
||||
|
||||
const defaultValues: AccountFormValues = {
|
||||
login_name: undefined,
|
||||
account_name: undefined,
|
||||
email: '',
|
||||
imap: {
|
||||
host: "",
|
||||
port: 993,
|
||||
encryption: 'Ssl',
|
||||
auth: { auth_type: 'Password', password: undefined },
|
||||
use_proxy: undefined,
|
||||
},
|
||||
enabled: true,
|
||||
use_dangerous: false,
|
||||
date_since: undefined,
|
||||
date_before: undefined,
|
||||
download_interval_min: 60,
|
||||
download_batch_size: 30,
|
||||
max_email_size_bytes: 100 * 1024 * 1024,
|
||||
auto_download_new_mailboxes: true,
|
||||
download_schedule: undefined,
|
||||
archive_rules: undefined,
|
||||
};
|
||||
|
||||
function SectionHeader({ title, description }: { title: string; description?: string }) {
|
||||
return (
|
||||
<div className="pb-3">
|
||||
<h3 className="text-base font-semibold">{title}</h3>
|
||||
{description && <p className="text-sm text-muted-foreground mt-0.5">{description}</p>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function AccountNewPage() {
|
||||
const { t } = useTranslation();
|
||||
const navigate = useNavigate();
|
||||
const { toast } = useToast();
|
||||
const queryClient = useQueryClient();
|
||||
const [autoConfigLoading, setAutoConfigLoading] = useState(false);
|
||||
|
||||
const accountSchema = getAccountSchema(false, t);
|
||||
const form = useForm<AccountFormValues>({
|
||||
mode: "onChange",
|
||||
defaultValues,
|
||||
resolver: zodResolver(accountSchema),
|
||||
});
|
||||
|
||||
const createMutation = useMutation({
|
||||
mutationFn: create_account,
|
||||
onSuccess: () => {
|
||||
toast({
|
||||
title: t('accounts.accountCreated'),
|
||||
description: t('accounts.accountCreatedDesc'),
|
||||
action: <ToastAction altText={t('common.close')}>{t('common.close')}</ToastAction>,
|
||||
});
|
||||
queryClient.invalidateQueries({ queryKey: ['account-list'] });
|
||||
navigate({ to: '/accounts' });
|
||||
},
|
||||
onError: (error: AxiosError) => {
|
||||
const errorMessage =
|
||||
(error.response?.data as { message?: string })?.message ||
|
||||
error.message ||
|
||||
t('accounts.creationFailed');
|
||||
toast({
|
||||
variant: "destructive",
|
||||
title: t('accounts.accountCreationFailed'),
|
||||
description: errorMessage as string,
|
||||
action: <ToastAction altText={t('common.tryAgain')}>{t('common.tryAgain')}</ToastAction>,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
const onSubmit = useCallback(
|
||||
(data: AccountFormValues) => {
|
||||
createMutation.mutate({
|
||||
email: data.email,
|
||||
account_name: data.account_name,
|
||||
login_name: data.login_name,
|
||||
imap: {
|
||||
...data.imap,
|
||||
auth: {
|
||||
...data.imap.auth,
|
||||
password: data.imap.auth.auth_type === 'OAuth2' ? undefined : data.imap.auth.password,
|
||||
},
|
||||
},
|
||||
enabled: data.enabled,
|
||||
use_dangerous: data.use_dangerous,
|
||||
date_since: data.date_since,
|
||||
date_before: data.date_before,
|
||||
download_interval_min: data.download_interval_min,
|
||||
download_batch_size: data.download_batch_size,
|
||||
max_email_size_bytes: data.max_email_size_bytes,
|
||||
auto_download_new_mailboxes: data.auto_download_new_mailboxes,
|
||||
download_schedule: data.download_schedule || null,
|
||||
account_type: "IMAP",
|
||||
archive_rules: data.archive_rules || null,
|
||||
});
|
||||
},
|
||||
[createMutation]
|
||||
);
|
||||
|
||||
const handleAutoConfig = async () => {
|
||||
const email = form.getValues('email');
|
||||
if (!email) return;
|
||||
const imap = form.getValues('imap');
|
||||
if (imap.host.trim() !== "" && imap.port > 0) return;
|
||||
|
||||
setAutoConfigLoading(true);
|
||||
try {
|
||||
const result = await autoconfig(email);
|
||||
if (result) {
|
||||
form.setValue('imap.host', result.imap.host);
|
||||
form.setValue('imap.port', result.imap.port);
|
||||
form.setValue('imap.encryption', result.imap.encryption);
|
||||
if (result.oauth2) form.setValue('imap.auth.auth_type', 'OAuth2');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Auto-configuration failed:', error);
|
||||
}
|
||||
setAutoConfigLoading(false);
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<FixedHeader />
|
||||
<Main>
|
||||
<div className="mx-auto w-full max-w-[46rem] px-4 py-6">
|
||||
<div className="mb-6 space-y-3">
|
||||
<Link
|
||||
to="/accounts"
|
||||
className="inline-flex items-center gap-1.5 text-sm text-muted-foreground hover:text-foreground transition-colors"
|
||||
>
|
||||
<ArrowLeft className="h-4 w-4" />
|
||||
{t('accounts.settings.backToAccounts')}
|
||||
</Link>
|
||||
<Breadcrumb items={[
|
||||
{ label: t('accounts.title'), to: '/accounts' },
|
||||
{ label: t('accounts.settings.newAccount') },
|
||||
]} />
|
||||
</div>
|
||||
|
||||
<div className="rounded-lg border shadow-sm bg-card p-6 md:p-8">
|
||||
<div className="mb-6">
|
||||
<h2 className="text-xl font-bold">{t('accounts.addAccount')}</h2>
|
||||
<p className="text-sm text-muted-foreground mt-1">{t('accounts.addNewEmailAccountHere')}</p>
|
||||
</div>
|
||||
|
||||
<FormProvider {...form}>
|
||||
<Form {...form}>
|
||||
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-10">
|
||||
<section>
|
||||
<SectionHeader
|
||||
title={t('accounts.settings.general')}
|
||||
description={t('accounts.settings.generalDesc')}
|
||||
/>
|
||||
<TabGeneral />
|
||||
</section>
|
||||
|
||||
<hr />
|
||||
|
||||
<section>
|
||||
<SectionHeader
|
||||
title={t('accounts.settings.server')}
|
||||
description={t('accounts.settings.serverDesc')}
|
||||
/>
|
||||
<div className="flex items-center gap-2 mb-4">
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
disabled={autoConfigLoading}
|
||||
onClick={handleAutoConfig}
|
||||
>
|
||||
{autoConfigLoading && <Loader2 className="mr-1.5 h-3.5 w-3.5 animate-spin" />}
|
||||
{autoConfigLoading ? t('accounts.autoConfiguring') : t('accounts.autoDiscover')}
|
||||
</Button>
|
||||
</div>
|
||||
<TabServer />
|
||||
</section>
|
||||
|
||||
<hr />
|
||||
|
||||
<section>
|
||||
<SectionHeader
|
||||
title={t('accounts.settings.download')}
|
||||
description={t('accounts.settings.downloadDesc')}
|
||||
/>
|
||||
<TabDownload />
|
||||
</section>
|
||||
|
||||
<hr />
|
||||
|
||||
<section>
|
||||
<SectionHeader
|
||||
title={t('accounts.settings.filters')}
|
||||
description={t('accounts.settings.filtersDesc')}
|
||||
/>
|
||||
<TabFilters />
|
||||
</section>
|
||||
|
||||
<div className="flex items-center justify-between pt-4 border-t">
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
onClick={() => navigate({ to: '/accounts' })}
|
||||
>
|
||||
{t('common.cancel')}
|
||||
</Button>
|
||||
<Button type="submit" size="lg" disabled={createMutation.isPending}>
|
||||
{createMutation.isPending ? t('accounts.creating') : t('accounts.submit')}
|
||||
</Button>
|
||||
</div>
|
||||
</form>
|
||||
</Form>
|
||||
</FormProvider>
|
||||
</div>
|
||||
</div>
|
||||
</Main>
|
||||
</>
|
||||
);
|
||||
}
|
||||
279
web/src/features/accounts/account-settings-page.tsx
Normal file
279
web/src/features/accounts/account-settings-page.tsx
Normal file
@@ -0,0 +1,279 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import { useCallback, useEffect } from "react";
|
||||
import { useForm, FormProvider } from "react-hook-form";
|
||||
import { zodResolver } from "@hookform/resolvers/zod";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useMutation, useQueryClient } from "@tanstack/react-query";
|
||||
import { Link } from "@tanstack/react-router";
|
||||
import { ArrowLeft } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Form } from "@/components/ui/form";
|
||||
import { useToast } from "@/hooks/use-toast";
|
||||
import { ToastAction } from "@/components/ui/toast";
|
||||
import { Breadcrumb } from "@/components/ui/breadcrumb";
|
||||
import { FixedHeader } from "@/components/layout/fixed-header";
|
||||
import { Main } from "@/components/layout/main";
|
||||
import { TabGeneral } from "./components/tab-general";
|
||||
import { TabServer } from "./components/tab-server";
|
||||
import { TabDownload } from "./components/tab-download";
|
||||
import { TabFilters } from "./components/tab-filters";
|
||||
import { update_account, list_accounts, type AccountModel } from "@/api/account/api";
|
||||
import { getAccountSchema, type AccountFormValues } from "./components/schema";
|
||||
import type { AxiosError } from "axios";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
|
||||
const emptyImap = {
|
||||
host: "",
|
||||
port: 0,
|
||||
encryption: "None" as const,
|
||||
auth: { auth_type: "Password" as const, password: undefined },
|
||||
use_proxy: undefined,
|
||||
};
|
||||
|
||||
function mapAccountToFormValues(account: AccountModel): AccountFormValues {
|
||||
const imap = { ...(account.imap ?? emptyImap) };
|
||||
imap.auth = { ...imap.auth, password: undefined };
|
||||
if ((imap as any).use_proxy === null) {
|
||||
(imap as any).use_proxy = undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
account_name: account.account_name ?? undefined,
|
||||
login_name: account.login_name ?? undefined,
|
||||
email: account.email,
|
||||
imap,
|
||||
enabled: account.enabled,
|
||||
use_dangerous: account.use_dangerous,
|
||||
date_since: account.date_since ?? undefined,
|
||||
date_before: account.date_before ?? undefined,
|
||||
download_interval_min: account.download_interval_min ?? 60,
|
||||
download_batch_size: account.download_batch_size ?? 30,
|
||||
max_email_size_bytes: account.max_email_size_bytes ?? 100 * 1024 * 1024,
|
||||
auto_download_new_mailboxes: account.auto_download_new_mailboxes ?? true,
|
||||
download_schedule: account.download_schedule ?? undefined,
|
||||
archive_rules: account.archive_rules ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function SectionHeader({ title, description }: { title: string; description?: string }) {
|
||||
return (
|
||||
<div className="pb-3">
|
||||
<h3 className="text-base font-semibold">{title}</h3>
|
||||
{description && <p className="text-sm text-muted-foreground mt-0.5">{description}</p>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
interface AccountSettingsPageProps {
|
||||
accountId: number;
|
||||
}
|
||||
|
||||
export function AccountSettingsPage({ accountId }: AccountSettingsPageProps) {
|
||||
const { t } = useTranslation();
|
||||
//const navigate = useNavigate();
|
||||
const { toast } = useToast();
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
const { data: accountList } = useQuery({
|
||||
queryKey: ['account-list'],
|
||||
queryFn: list_accounts,
|
||||
});
|
||||
|
||||
const account = accountList?.items?.find((a) => a.id === accountId);
|
||||
|
||||
const accountSchema = getAccountSchema(true, t);
|
||||
const form = useForm<AccountFormValues>({
|
||||
mode: "onChange",
|
||||
defaultValues: account ? mapAccountToFormValues(account) : undefined,
|
||||
resolver: zodResolver(accountSchema),
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (account) {
|
||||
form.reset(mapAccountToFormValues(account));
|
||||
}
|
||||
}, [account?.id]);
|
||||
|
||||
const updateMutation = useMutation({
|
||||
mutationFn: (data: Record<string, any>) => update_account(accountId, data),
|
||||
onSuccess: () => {
|
||||
toast({
|
||||
title: t('accounts.settings.saved'),
|
||||
description: t('accounts.settings.savedDesc'),
|
||||
action: <ToastAction altText={t('common.close')}>{t('common.close')}</ToastAction>,
|
||||
});
|
||||
queryClient.invalidateQueries({ queryKey: ['account-list'] });
|
||||
},
|
||||
onError: (error: AxiosError) => {
|
||||
const errorMessage =
|
||||
(error.response?.data as { message?: string })?.message ||
|
||||
error.message ||
|
||||
t('accounts.updateFailed');
|
||||
toast({
|
||||
variant: "destructive",
|
||||
title: t('accounts.accountUpdateFailed'),
|
||||
description: errorMessage as string,
|
||||
action: <ToastAction altText={t('common.tryAgain')}>{t('common.tryAgain')}</ToastAction>,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
const onSubmit = useCallback(
|
||||
(data: AccountFormValues) => {
|
||||
const payload: Record<string, any> = {
|
||||
email: data.email,
|
||||
account_name: data.account_name,
|
||||
login_name: data.login_name,
|
||||
imap: {
|
||||
...data.imap,
|
||||
auth: {
|
||||
...data.imap.auth,
|
||||
password: data.imap.auth.auth_type === 'OAuth2'
|
||||
? undefined
|
||||
: (data.imap.auth.password ? data.imap.auth.password : undefined),
|
||||
},
|
||||
},
|
||||
enabled: data.enabled,
|
||||
use_dangerous: data.use_dangerous,
|
||||
date_since: data.date_since,
|
||||
date_before: data.date_before,
|
||||
download_interval_min: data.download_interval_min,
|
||||
download_batch_size: data.download_batch_size,
|
||||
max_email_size_bytes: data.max_email_size_bytes,
|
||||
auto_download_new_mailboxes: data.auto_download_new_mailboxes,
|
||||
download_schedule: data.download_schedule || null,
|
||||
archive_rules: data.archive_rules || null,
|
||||
};
|
||||
|
||||
if (!data.date_since && !data.date_before) {
|
||||
payload.clear_date_range = true;
|
||||
}
|
||||
if (!data.download_schedule && account?.download_schedule) {
|
||||
payload.clear_download_schedule = true;
|
||||
}
|
||||
|
||||
updateMutation.mutate(payload);
|
||||
},
|
||||
[updateMutation, account]
|
||||
);
|
||||
|
||||
if (!account) {
|
||||
return (
|
||||
<>
|
||||
<FixedHeader />
|
||||
<Main>
|
||||
<div className="mx-auto w-full max-w-[46rem] px-4 py-12 text-center text-muted-foreground">
|
||||
{t('accounts.settings.loading')}
|
||||
</div>
|
||||
</Main>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<FixedHeader />
|
||||
<Main>
|
||||
<div className="mx-auto w-full max-w-[46rem] px-4 py-6">
|
||||
<div className="mb-6 space-y-3">
|
||||
<Link
|
||||
to="/accounts"
|
||||
className="inline-flex items-center gap-1.5 text-sm text-muted-foreground hover:text-foreground transition-colors"
|
||||
>
|
||||
<ArrowLeft className="h-4 w-4" />
|
||||
{t('accounts.settings.backToAccounts')}
|
||||
</Link>
|
||||
<Breadcrumb items={[
|
||||
{ label: t('accounts.title'), to: '/accounts' },
|
||||
{ label: account.email },
|
||||
{ label: t('accounts.settings.settings') },
|
||||
]} />
|
||||
</div>
|
||||
|
||||
<div className="rounded-lg border shadow-sm bg-card p-6 md:p-8">
|
||||
<div className="mb-6">
|
||||
<h2 className="text-xl font-bold">{account.email}</h2>
|
||||
<p className="text-sm text-muted-foreground mt-1">{t('accounts.updateTheEmailAccountHere')}</p>
|
||||
</div>
|
||||
|
||||
<FormProvider {...form}>
|
||||
<Form {...form}>
|
||||
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-10">
|
||||
<section>
|
||||
<SectionHeader
|
||||
title={t('accounts.settings.general')}
|
||||
description={t('accounts.settings.generalDesc')}
|
||||
/>
|
||||
<TabGeneral isEdit />
|
||||
</section>
|
||||
|
||||
<hr />
|
||||
|
||||
<section>
|
||||
<SectionHeader
|
||||
title={t('accounts.settings.server')}
|
||||
description={t('accounts.settings.serverDesc')}
|
||||
/>
|
||||
<TabServer isEdit />
|
||||
</section>
|
||||
|
||||
<hr />
|
||||
|
||||
<section>
|
||||
<SectionHeader
|
||||
title={t('accounts.settings.download')}
|
||||
description={t('accounts.settings.downloadDesc')}
|
||||
/>
|
||||
<TabDownload />
|
||||
</section>
|
||||
|
||||
<hr />
|
||||
|
||||
<section>
|
||||
<SectionHeader
|
||||
title={t('accounts.settings.filters')}
|
||||
description={t('accounts.settings.filtersDesc')}
|
||||
/>
|
||||
<TabFilters />
|
||||
</section>
|
||||
|
||||
<div className="flex items-center justify-between pt-4 border-t">
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
onClick={() => {
|
||||
if (account) form.reset(mapAccountToFormValues(account));
|
||||
}}
|
||||
>
|
||||
{t('accounts.settings.reset')}
|
||||
</Button>
|
||||
<Button type="submit" size="lg" disabled={updateMutation.isPending}>
|
||||
{updateMutation.isPending ? t('accounts.settings.saving') : t('accounts.saveChanges')}
|
||||
</Button>
|
||||
</div>
|
||||
</form>
|
||||
</Form>
|
||||
</FormProvider>
|
||||
</div>
|
||||
</div>
|
||||
</Main>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,368 +0,0 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import { zodResolver } from '@hookform/resolvers/zod';
|
||||
import * as React from 'react';
|
||||
import { useForm } from 'react-hook-form';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Form } from '@/components/ui/form';
|
||||
import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog';
|
||||
import { ScrollArea } from '@/components/ui/scroll-area';
|
||||
import { useToast } from '@/hooks/use-toast';
|
||||
import Step1 from './step1';
|
||||
import Step2 from './step2';
|
||||
import Step3 from './step3';
|
||||
import Step4 from './step4';
|
||||
import { create_account, autoconfig, update_account, AccountModel, ImapConfig } from '@/api/account/api';
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query';
|
||||
import { ToastAction } from '@/components/ui/toast';
|
||||
import { AxiosError } from 'axios';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { cn } from "@/lib/utils";
|
||||
import { getAccountSchema, type AccountFormValues } from './schema';
|
||||
|
||||
export type Account = AccountFormValues;
|
||||
|
||||
type Step = {
|
||||
id: `step-${number}`;
|
||||
name: string;
|
||||
fields: (keyof Account)[];
|
||||
};
|
||||
|
||||
export type Steps = [...Step[]];
|
||||
|
||||
const getSteps = (t: (key: string) => string): Steps => [
|
||||
{ id: "step-1", name: t('accounts.steps.emailAddress'), fields: ["email", "account_name"] },
|
||||
{ id: "step-2", name: t('accounts.steps.imap'), fields: ["imap", "use_dangerous", "login_name"] },
|
||||
{ id: "step-3", name: t('accounts.steps.syncPreferences'), fields: ["enabled", "date_since", "date_before", "download_interval_min", "download_batch_size", "max_email_size_bytes", "auto_download_new_mailboxes", "download_schedule"] },
|
||||
{ id: "step-4", name: t('accounts.steps.summary'), fields: [] },
|
||||
];
|
||||
|
||||
const LAST_STEP = 4;
|
||||
|
||||
interface Props {
|
||||
currentRow?: AccountModel;
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
}
|
||||
|
||||
const defaultValues: Account = {
|
||||
login_name: undefined,
|
||||
account_name: undefined,
|
||||
email: '',
|
||||
imap: {
|
||||
host: "",
|
||||
port: 993,
|
||||
encryption: 'Ssl',
|
||||
auth: {
|
||||
auth_type: 'Password',
|
||||
password: undefined,
|
||||
},
|
||||
use_proxy: undefined
|
||||
},
|
||||
enabled: true,
|
||||
use_dangerous: false,
|
||||
date_since: undefined,
|
||||
date_before: undefined,
|
||||
download_interval_min: 60,
|
||||
download_batch_size: 30,
|
||||
max_email_size_bytes: 100 * 1024 * 1024,
|
||||
auto_download_new_mailboxes: true,
|
||||
download_schedule: undefined,
|
||||
};
|
||||
|
||||
const emptyImap: ImapConfig = {
|
||||
host: "",
|
||||
port: 0,
|
||||
encryption: "None",
|
||||
auth: { auth_type: "Password", password: undefined },
|
||||
use_proxy: undefined,
|
||||
};
|
||||
|
||||
const mapCurrentRowToFormValues = (currentRow: AccountModel): Account => {
|
||||
const imap = { ...(currentRow.imap ?? emptyImap) };
|
||||
imap.auth = { ...imap.auth, password: undefined };
|
||||
if (imap.use_proxy === null) {
|
||||
imap.use_proxy = undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
account_name: currentRow.account_name ?? undefined,
|
||||
login_name: currentRow.login_name ?? undefined,
|
||||
email: currentRow.email,
|
||||
imap,
|
||||
enabled: currentRow.enabled,
|
||||
use_dangerous: currentRow.use_dangerous,
|
||||
date_since: currentRow.date_since ?? undefined,
|
||||
date_before: currentRow.date_before ?? undefined,
|
||||
download_interval_min: currentRow.download_interval_min ?? 60,
|
||||
download_batch_size: currentRow.download_batch_size ?? 30,
|
||||
max_email_size_bytes: currentRow.max_email_size_bytes ?? 100 * 1024 * 1024,
|
||||
auto_download_new_mailboxes: currentRow.auto_download_new_mailboxes ?? true,
|
||||
download_schedule: currentRow.download_schedule ?? undefined,
|
||||
};
|
||||
};
|
||||
|
||||
export function AccountActionDialog({ currentRow, open, onOpenChange }: Props) {
|
||||
const { t } = useTranslation();
|
||||
const steps = getSteps(t);
|
||||
const isEdit = !!currentRow;
|
||||
const [currentStep, setCurrentStep] = React.useState(1);
|
||||
const { toast } = useToast();
|
||||
const [autoConfigLoading, setAutoConfigLoading] = React.useState(false);
|
||||
|
||||
const accountSchema = getAccountSchema(isEdit, t);
|
||||
const form = useForm<Account>({
|
||||
mode: "onChange",
|
||||
defaultValues: isEdit ? mapCurrentRowToFormValues(currentRow) : defaultValues,
|
||||
resolver: zodResolver(accountSchema),
|
||||
});
|
||||
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
const createMutation = useMutation({
|
||||
mutationFn: create_account,
|
||||
onSuccess: handleSuccess,
|
||||
onError: handleError,
|
||||
});
|
||||
|
||||
const updateMutation = useMutation({
|
||||
mutationFn: (data: Record<string, any>) => update_account(currentRow?.id!, data),
|
||||
onSuccess: handleSuccess,
|
||||
onError: handleError,
|
||||
});
|
||||
|
||||
function handleSuccess() {
|
||||
toast({
|
||||
title: isEdit ? t('accounts.accountUpdated') : t('accounts.accountCreated'),
|
||||
description: isEdit ? t('accounts.accountUpdatedDesc') : t('accounts.accountCreatedDesc'),
|
||||
action: <ToastAction altText={t('common.close')}>{t('common.close')}</ToastAction>,
|
||||
});
|
||||
|
||||
queryClient.invalidateQueries({ queryKey: ['account-list'] });
|
||||
form.reset();
|
||||
onOpenChange(false);
|
||||
}
|
||||
|
||||
function handleError(error: AxiosError) {
|
||||
const errorMessage =
|
||||
(error.response?.data as { message?: string })?.message ||
|
||||
error.message ||
|
||||
(isEdit ? t('accounts.updateFailed') : t('accounts.creationFailed'));
|
||||
|
||||
toast({
|
||||
variant: "destructive",
|
||||
title: isEdit ? t('accounts.accountUpdateFailed') : t('accounts.accountCreationFailed'),
|
||||
description: errorMessage as string,
|
||||
action: <ToastAction altText={t('common.tryAgain')}>{t('common.tryAgain')}</ToastAction>,
|
||||
});
|
||||
console.error(error);
|
||||
}
|
||||
|
||||
const onSubmit = React.useCallback(
|
||||
(data: Account) => {
|
||||
const commonData = {
|
||||
email: data.email,
|
||||
account_name: data.account_name,
|
||||
login_name: data.login_name,
|
||||
imap: {
|
||||
...data.imap,
|
||||
auth: {
|
||||
...data.imap.auth,
|
||||
password: data.imap.auth.auth_type === 'OAuth2'
|
||||
? undefined
|
||||
: (isEdit && !data.imap.auth.password ? undefined : data.imap.auth.password),
|
||||
},
|
||||
},
|
||||
enabled: data.enabled,
|
||||
use_dangerous: data.use_dangerous,
|
||||
date_since: data.date_since,
|
||||
date_before: data.date_before,
|
||||
download_interval_min: data.download_interval_min,
|
||||
download_batch_size: data.download_batch_size,
|
||||
max_email_size_bytes: data.max_email_size_bytes,
|
||||
auto_download_new_mailboxes: data.auto_download_new_mailboxes,
|
||||
download_schedule: data.download_schedule || null,
|
||||
};
|
||||
if (isEdit) {
|
||||
const isAllMode = !data.date_since && !data.date_before;
|
||||
const clear_download_schedule = !data.download_schedule && currentRow?.download_schedule;
|
||||
updateMutation.mutate({
|
||||
...commonData,
|
||||
...(isAllMode ? { clear_date_range: true } : {}),
|
||||
...(clear_download_schedule ? { clear_download_schedule: true } : {})
|
||||
});
|
||||
} else {
|
||||
createMutation.mutate({ ...commonData, account_type: "IMAP" });
|
||||
}
|
||||
},
|
||||
[isEdit, updateMutation, createMutation]
|
||||
);
|
||||
|
||||
const handleNav = async (index: number) => {
|
||||
let isValid = true;
|
||||
let failedStep = currentStep;
|
||||
for (let i = currentStep - 1; i < index - 1 && isValid; i++) {
|
||||
isValid = await form.trigger(steps[i].fields);
|
||||
if (!isValid) failedStep = i;
|
||||
}
|
||||
if (isValid) setCurrentStep(index);
|
||||
else setCurrentStep(failedStep);
|
||||
};
|
||||
|
||||
async function handleContinue() {
|
||||
const isValid = await form.trigger(steps[currentStep - 1].fields);
|
||||
if (!isValid) return;
|
||||
|
||||
if (currentStep === 1) {
|
||||
let allValues = form.getValues();
|
||||
if (allValues.imap.host.trim() !== "" && allValues.imap.port > 0) {
|
||||
handleNav(currentStep + 1);
|
||||
return;
|
||||
}
|
||||
setAutoConfigLoading(true);
|
||||
const email = form.getValues('email');
|
||||
form.setValue('login_name', email);
|
||||
try {
|
||||
const result = await autoconfig(email);
|
||||
if (result) {
|
||||
form.setValue('imap.host', result.imap.host);
|
||||
form.setValue('imap.port', result.imap.port);
|
||||
form.setValue('imap.encryption', result.imap.encryption);
|
||||
if (result.oauth2) form.setValue('imap.auth.auth_type', 'OAuth2');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Auto-configuration failed:', error);
|
||||
}
|
||||
setAutoConfigLoading(false);
|
||||
handleNav(currentStep + 1);
|
||||
} else {
|
||||
handleNav(currentStep + 1);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<Dialog
|
||||
open={open}
|
||||
onOpenChange={(state) => {
|
||||
if (!state) {
|
||||
form.reset();
|
||||
setCurrentStep(1);
|
||||
}
|
||||
onOpenChange(state);
|
||||
}}
|
||||
>
|
||||
<DialogContent className="max-w-[95vw] md:max-w-5xl w-full p-0 overflow-hidden flex flex-col h-[50rem]">
|
||||
<div className="p-6 pb-2 flex-shrink-0">
|
||||
<DialogHeader className="text-left">
|
||||
<DialogTitle>{isEdit ? t('accounts.updateAccount') : t('accounts.addAccount')}</DialogTitle>
|
||||
<DialogDescription>
|
||||
{isEdit ? t('accounts.updateAccount') : t('accounts.addAccount')}
|
||||
{t('accounts.clickSaveWhenDone')}
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col md:flex-row flex-1 min-h-0 overflow-hidden border-y">
|
||||
<div className="md:hidden flex px-6 py-2 space-x-2 overflow-x-auto border-b flex-shrink-0 bg-background/50">
|
||||
{steps.map((step, index) => (
|
||||
<div key={step.id} className="flex flex-col items-center flex-shrink-0 min-w-[70px]">
|
||||
<Button
|
||||
variant={currentStep === index + 1 ? "default" : "secondary"}
|
||||
className="size-8 rounded-full font-bold p-0"
|
||||
disabled={currentStep === index + 1}
|
||||
onClick={() => setCurrentStep(index + 1)}
|
||||
>
|
||||
{index + 1}
|
||||
</Button>
|
||||
<span className="text-[10px] mt-1 text-muted-foreground line-clamp-1">{step.name}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<div className="hidden md:block w-[240px] flex-shrink-0 px-8 py-4 border-r overflow-y-auto">
|
||||
{steps.map((step, index) => (
|
||||
<div className="mb-8 flex items-center" key={step.id}>
|
||||
<Button
|
||||
variant={currentStep === index + 1 ? "default" : "secondary"}
|
||||
className="size-9 rounded-full text-sm font-bold"
|
||||
disabled={currentStep === index + 1}
|
||||
onClick={() => setCurrentStep(index + 1)}
|
||||
>
|
||||
{index + 1}
|
||||
</Button>
|
||||
<div className="flex flex-col items-baseline uppercase ml-4">
|
||||
<span className="text-[10px] text-muted-foreground">{t('accounts.step', { index: index + 1 })}</span>
|
||||
<span className={cn("font-bold text-sm tracking-wider", currentStep === index + 1 ? "text-foreground" : "text-muted-foreground")}>
|
||||
{step.name}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<div className="flex-1 min-h-0 relative">
|
||||
<ScrollArea className="h-full w-full">
|
||||
<div className="p-6 md:p-6 lg:p-8">
|
||||
<Form {...form}>
|
||||
<form id="account-register-form" onSubmit={form.handleSubmit(onSubmit)}>
|
||||
{currentStep === 1 && <Step1 isEdit={isEdit} />}
|
||||
{currentStep === 2 && <Step2 isEdit={isEdit} />}
|
||||
{currentStep === 3 && <Step3 />}
|
||||
{currentStep === 4 && <Step4 />}
|
||||
</form>
|
||||
</Form>
|
||||
</div>
|
||||
</ScrollArea>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<DialogFooter className="p-4 md:p-6 bg-background flex flex-row sm:justify-end gap-2 flex-shrink-0">
|
||||
{currentStep > 1 && (
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
className="flex-1 sm:flex-none"
|
||||
onClick={() => setCurrentStep(currentStep - 1)}
|
||||
>
|
||||
{t('accounts.goBack')}
|
||||
</Button>
|
||||
)}
|
||||
{currentStep < LAST_STEP && (
|
||||
<Button
|
||||
type="button"
|
||||
className="flex-1 sm:flex-none px-8"
|
||||
onClick={handleContinue}
|
||||
>
|
||||
{autoConfigLoading ? t('accounts.autoConfiguring') : t('accounts.continue')}
|
||||
</Button>
|
||||
)}
|
||||
{currentStep === LAST_STEP && (
|
||||
<Button
|
||||
type="submit"
|
||||
form="account-register-form"
|
||||
className="flex-1 sm:flex-none px-10"
|
||||
>
|
||||
{isEdit ? t('accounts.saveChanges') : t('accounts.submit')}
|
||||
</Button>
|
||||
)}
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@@ -1,159 +0,0 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
//
|
||||
|
||||
import * as React from 'react'
|
||||
import { Mail, Database } from 'lucide-react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
import { cn } from '@/lib/utils'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog'
|
||||
import { Label } from '@/components/ui/label'
|
||||
import { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group'
|
||||
import { useAccountContext } from '../context'
|
||||
|
||||
export type AddAccountType = 'IMAP' | 'NoSync'
|
||||
|
||||
interface Props {
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
}
|
||||
|
||||
export function AddAccountDialog({
|
||||
open,
|
||||
onOpenChange
|
||||
}: Props) {
|
||||
const { t } = useTranslation()
|
||||
const { setOpen } = useAccountContext()
|
||||
const [value, setValue] = React.useState<AddAccountType>('IMAP')
|
||||
|
||||
function handleContinue() {
|
||||
if (value === 'IMAP') {
|
||||
setOpen('add-imap')
|
||||
} else {
|
||||
setOpen('add-nosync')
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="sm:max-w-xl">
|
||||
<DialogHeader className="text-left">
|
||||
<DialogTitle>
|
||||
{t('accounts.add')}
|
||||
</DialogTitle>
|
||||
|
||||
<DialogDescription>
|
||||
{t('accounts.selectAccountType')}
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<RadioGroup
|
||||
value={value}
|
||||
onValueChange={(v) => setValue(v as AddAccountType)}
|
||||
className="space-y-4 py-2"
|
||||
>
|
||||
<Label
|
||||
htmlFor="imap-account"
|
||||
className={cn(
|
||||
'flex cursor-pointer items-start gap-4 rounded-2xl border p-5 transition-all',
|
||||
value === 'IMAP'
|
||||
? 'border-primary bg-muted/50'
|
||||
: 'hover:bg-muted/30'
|
||||
)}
|
||||
>
|
||||
<RadioGroupItem
|
||||
value="IMAP"
|
||||
id="imap-account"
|
||||
className="mt-1"
|
||||
/>
|
||||
|
||||
<div className="flex flex-1 gap-4">
|
||||
<div className="rounded-xl border p-2">
|
||||
<Mail className="h-5 w-5" />
|
||||
</div>
|
||||
|
||||
<div className="space-y-1">
|
||||
<div className="font-medium">
|
||||
{t('accounts.imapAccount')}
|
||||
</div>
|
||||
|
||||
<div className="text-sm text-muted-foreground">
|
||||
{t('accounts.imapAccountDescription')}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</Label>
|
||||
|
||||
<Label
|
||||
htmlFor="nosync-account"
|
||||
className={cn(
|
||||
'flex cursor-pointer items-start gap-4 rounded-2xl border p-5 transition-all',
|
||||
value === 'NoSync'
|
||||
? 'border-primary bg-muted/50'
|
||||
: 'hover:bg-muted/30'
|
||||
)}
|
||||
>
|
||||
<RadioGroupItem
|
||||
value="NoSync"
|
||||
id="nosync-account"
|
||||
className="mt-1"
|
||||
/>
|
||||
|
||||
<div className="flex flex-1 gap-4">
|
||||
<div className="rounded-xl border p-2">
|
||||
<Database className="h-5 w-5" />
|
||||
</div>
|
||||
|
||||
<div className="space-y-1">
|
||||
<div className="font-medium">
|
||||
{t('accounts.noSyncAccount')}
|
||||
</div>
|
||||
|
||||
<div className="text-sm text-muted-foreground">
|
||||
{t('accounts.noSyncAccountDescription')}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</Label>
|
||||
</RadioGroup>
|
||||
|
||||
<DialogFooter>
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={() => onOpenChange(false)}
|
||||
>
|
||||
{t('common.cancel')}
|
||||
</Button>
|
||||
|
||||
<Button onClick={handleContinue}>
|
||||
{t('accounts.continue')}
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
)
|
||||
}
|
||||
@@ -30,11 +30,12 @@ import {
|
||||
DropdownMenuTrigger,
|
||||
} from '@/components/ui/dropdown-menu'
|
||||
import { useAccountContext } from '../context'
|
||||
import { Mailbox, MessageSquareMore } from 'lucide-react'
|
||||
import { Mailbox, MessageSquareMore, Settings } from 'lucide-react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useCurrentUser } from '@/hooks/use-current-user'
|
||||
import { AccountModel, cancel_account_download, start_account_download } from '@/api/account/api'
|
||||
import { toast } from '@/hooks/use-toast'
|
||||
import { useNavigate } from '@tanstack/react-router'
|
||||
|
||||
interface DataTableRowActionsProps {
|
||||
row: Row<AccountModel>
|
||||
@@ -43,6 +44,7 @@ interface DataTableRowActionsProps {
|
||||
export function DataTableRowActions({ row }: DataTableRowActionsProps) {
|
||||
const { t } = useTranslation()
|
||||
const { setOpen, setCurrentRow } = useAccountContext()
|
||||
const navigate = useNavigate()
|
||||
|
||||
const account_type = row.original.account_type;
|
||||
const { require_any_permission } = useCurrentUser()
|
||||
@@ -103,11 +105,10 @@ export function DataTableRowActions({ row }: DataTableRowActionsProps) {
|
||||
<DropdownMenuContent align='end' className='w-[220px]'>
|
||||
{hasPermission && <DropdownMenuItem
|
||||
onClick={() => {
|
||||
setCurrentRow(row.original)
|
||||
if (account_type === "IMAP") {
|
||||
setOpen("edit-imap");
|
||||
}
|
||||
if (account_type === "NoSync") {
|
||||
navigate({ to: '/accounts/$id/settings', params: { id: String(row.original.id) } });
|
||||
} else {
|
||||
setCurrentRow(row.original)
|
||||
setOpen("edit-nosync");
|
||||
}
|
||||
}}
|
||||
@@ -117,6 +118,16 @@ export function DataTableRowActions({ row }: DataTableRowActionsProps) {
|
||||
<IconEdit size={16} />
|
||||
</DropdownMenuShortcut>
|
||||
</DropdownMenuItem>}
|
||||
{account_type === "IMAP" && hasPermission && <DropdownMenuItem
|
||||
onClick={() => {
|
||||
navigate({ to: '/accounts/$id/settings', params: { id: String(row.original.id) } });
|
||||
}}
|
||||
>
|
||||
{t('accounts.settings.settings')}
|
||||
<DropdownMenuShortcut>
|
||||
<Settings size={16} />
|
||||
</DropdownMenuShortcut>
|
||||
</DropdownMenuItem>}
|
||||
{account_type === "IMAP" && hasPermission && <DropdownMenuItem
|
||||
onClick={() => {
|
||||
setCurrentRow(row.original)
|
||||
|
||||
90
web/src/features/accounts/components/pattern-input.tsx
Normal file
90
web/src/features/accounts/components/pattern-input.tsx
Normal file
@@ -0,0 +1,90 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import { Input } from "@/components/ui/input";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { X } from "lucide-react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import type { MatchType, PatternEntry } from "@/lib/pattern-utils";
|
||||
import { isValidRegex } from "@/lib/pattern-utils";
|
||||
|
||||
interface PatternInputProps {
|
||||
entry: PatternEntry;
|
||||
onChange: (id: string, entry: Partial<PatternEntry>) => void;
|
||||
onRemove: (id: string) => void;
|
||||
}
|
||||
|
||||
export function PatternInput({ entry, onChange, onRemove }: PatternInputProps) {
|
||||
const { t } = useTranslation();
|
||||
|
||||
const matchTypeLabels: Record<MatchType, string> = {
|
||||
contains: t("accounts.filters.matchType.contains"),
|
||||
starts_with: t("accounts.filters.matchType.startsWith"),
|
||||
ends_with: t("accounts.filters.matchType.endsWith"),
|
||||
is_exactly: t("accounts.filters.matchType.isExactly"),
|
||||
regex: t("accounts.filters.matchType.regex"),
|
||||
};
|
||||
|
||||
const regexValid = entry.matchType === 'regex' ? isValidRegex(entry.value) : true;
|
||||
const showRegexHint = entry.matchType === 'regex' && entry.value && !regexValid;
|
||||
|
||||
return (
|
||||
<div className="flex items-center gap-2 group">
|
||||
<Select
|
||||
value={entry.matchType}
|
||||
onValueChange={(v) => onChange(entry.id, { matchType: v as MatchType })}
|
||||
>
|
||||
<SelectTrigger className="w-[130px] h-9 text-sm">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{Object.entries(matchTypeLabels).map(([value, label]) => (
|
||||
<SelectItem key={value} value={value}>{label}</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<div className="flex-1 relative">
|
||||
<Input
|
||||
className="h-9 text-sm"
|
||||
value={entry.value}
|
||||
onChange={(e) => onChange(entry.id, { value: e.target.value })}
|
||||
/>
|
||||
{showRegexHint && (
|
||||
<span className="text-[10px] text-destructive absolute -bottom-4 left-0">
|
||||
Invalid regex
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8 shrink-0 opacity-50 group-hover:opacity-100 transition-opacity"
|
||||
onClick={() => onRemove(entry.id)}
|
||||
>
|
||||
<X className="h-3.5 w-3.5" />
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -67,6 +67,19 @@ const dateSelectionSchema = (t: (key: string) => string) =>
|
||||
})
|
||||
.optional()
|
||||
|
||||
const filterRuleSchema = z.object({
|
||||
include: z.array(z.string()),
|
||||
exclude: z.array(z.string()),
|
||||
})
|
||||
|
||||
const archiveRulesSchema = z.object({
|
||||
enabled: z.boolean(),
|
||||
senders: filterRuleSchema,
|
||||
subjects: filterRuleSchema,
|
||||
skip_larger_than: z.number().int().positive().optional(),
|
||||
spam_headers: z.array(z.string()),
|
||||
})
|
||||
|
||||
export const getAccountSchema = (isEdit: boolean, t: (key: string) => string) =>
|
||||
z.object({
|
||||
account_name: z.string().optional(),
|
||||
@@ -120,6 +133,7 @@ export const getAccountSchema = (isEdit: boolean, t: (key: string) => string) =>
|
||||
},
|
||||
{ message: t('validation.invalidCronExpression') }
|
||||
),
|
||||
archive_rules: archiveRulesSchema.optional(),
|
||||
})
|
||||
|
||||
export type AccountFormValues = z.infer<
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import { useFormContext } from "react-hook-form";
|
||||
import {
|
||||
FormField,
|
||||
FormItem,
|
||||
FormLabel,
|
||||
FormMessage,
|
||||
FormControl,
|
||||
FormDescription,
|
||||
} from "@/components/ui/form";
|
||||
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Account } from "./action-dialog";
|
||||
import { useTranslation } from "react-i18next";
|
||||
|
||||
interface StepProps {
|
||||
isEdit: boolean;
|
||||
}
|
||||
|
||||
export default function Step1({ isEdit }: StepProps) {
|
||||
const { t } = useTranslation()
|
||||
const { control } = useFormContext<Account>();
|
||||
|
||||
return (
|
||||
<>
|
||||
<h1 className="my-3 md:mt-8">{t('accounts.emailAccountRegistration')}</h1>
|
||||
<p className="mb-5 md:mb-8">
|
||||
{t('accounts.emailAccountRegistrationDesc')}
|
||||
</p>
|
||||
<div className="space-y-8">
|
||||
<FormField
|
||||
control={control}
|
||||
name="email"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel className="flex items-center justify-between">
|
||||
{t('accounts.emailAddress')}:
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder={t('accounts.emailPlaceholder')} disabled={isEdit} {...field} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
{isEdit && (
|
||||
<FormDescription>
|
||||
{t('accounts.emailCannotBeModified')}
|
||||
</FormDescription>
|
||||
)}
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={control}
|
||||
name="account_name"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel className="flex items-center justify-between">
|
||||
{t('accounts.name')}:
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder={t('accounts.namePlaceholder')} {...field} />
|
||||
</FormControl>
|
||||
<FormDescription>{t('accounts.optional')}</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,236 +0,0 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import {
|
||||
FormField,
|
||||
FormItem,
|
||||
FormLabel,
|
||||
FormMessage,
|
||||
FormControl,
|
||||
FormDescription,
|
||||
} from "@/components/ui/form";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { useFormContext, useWatch } from "react-hook-form";
|
||||
import { Account } from "./action-dialog";
|
||||
import { PasswordInput } from "@/components/password-input";
|
||||
import useProxyList from "@/hooks/use-proxy";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
|
||||
interface StepProps {
|
||||
isEdit: boolean;
|
||||
}
|
||||
|
||||
export default function Step2({ isEdit }: StepProps) {
|
||||
const { t } = useTranslation()
|
||||
const { control } = useFormContext<Account>();
|
||||
const { proxyOptions } = useProxyList();
|
||||
|
||||
const imapAuthMethod = useWatch({
|
||||
control,
|
||||
name: "imap.auth.auth_type",
|
||||
});
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="space-y-8">
|
||||
<FormField
|
||||
control={control}
|
||||
name="imap.host"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel className="flex items-center justify-between">
|
||||
{t('accounts.imapHost')}:
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder={t('accounts.imapHostPlaceholder')} {...field} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={control}
|
||||
name="imap.port"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel className="flex items-center justify-between">
|
||||
{t('accounts.imapPort')}:
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<Input type="number" placeholder={t('accounts.imapPortPlaceholder')} {...field} onChange={(e) => field.onChange(parseInt(e.target.value, 10))} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={control}
|
||||
name="imap.encryption"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.imapEncryption')}:</FormLabel>
|
||||
<Select onValueChange={field.onChange} defaultValue={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue placeholder={t('accounts.selectEncryptionMethod')} />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="Ssl">Ssl</SelectItem>
|
||||
<SelectItem value="StartTls">StartTls</SelectItem>
|
||||
<SelectItem value="None">None</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormDescription>
|
||||
{t('accounts.chooseEncryptionMethod')}
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={control}
|
||||
name="use_dangerous"
|
||||
render={({ field }) => (
|
||||
<FormItem className="flex flex-col items-start gap-y-1">
|
||||
<FormLabel>{t('accounts.useDangerous')}:</FormLabel>
|
||||
<FormControl>
|
||||
<Checkbox
|
||||
className="mt-2"
|
||||
checked={field.value}
|
||||
onCheckedChange={field.onChange}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>{t('accounts.useDangerousDescription')}</FormDescription>
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={control}
|
||||
name="login_name"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel className="flex items-center justify-between">
|
||||
{t('accounts.login_name')}:
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder={t('accounts.namePlaceholder')} {...field} disabled={isEdit} />
|
||||
</FormControl>
|
||||
<FormDescription>{t('accounts.nameDescription')}</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={control}
|
||||
name="imap.auth.auth_type"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.imapAuthMethod')}:</FormLabel>
|
||||
<Select onValueChange={field.onChange} defaultValue={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue placeholder={t('accounts.selectAuthMethod')} />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="OAuth2">OAuth2</SelectItem>
|
||||
<SelectItem value="Password">Password</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormDescription>
|
||||
{t('accounts.chooseAuthMethod')}
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
{imapAuthMethod === "Password" && (
|
||||
<FormField
|
||||
control={control}
|
||||
name="imap.auth.password"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel className="flex items-center justify-between">
|
||||
{t('accounts.imapPassword')}:
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<PasswordInput placeholder={isEdit ? t('accounts.leaveEmptyToKeepPassword') : t('accounts.enterPassword')} {...field} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
{isEdit && (
|
||||
<FormDescription>
|
||||
{t('accounts.leaveEmptyToKeepExisting')}
|
||||
</FormDescription>
|
||||
)}
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
<FormField
|
||||
control={control}
|
||||
name='imap.use_proxy'
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel className="flex items-center justify-between">{t('accounts.useProxy')} ({t('accounts.optional')}):</FormLabel>
|
||||
<FormControl>
|
||||
<Select
|
||||
onValueChange={(val) => {
|
||||
field.onChange(val === 'none' ? undefined : Number(val))
|
||||
}}
|
||||
defaultValue={field.value?.toString()}
|
||||
>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue placeholder={t('accounts.selectProxy')} />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem key="none" value="none">
|
||||
{t('accounts.useNoProxy')}
|
||||
</SelectItem>
|
||||
{proxyOptions && proxyOptions.length > 0 && (
|
||||
proxyOptions.map((option) => (
|
||||
<SelectItem key={option.value} value={option.value.toString()}>
|
||||
{option.label}
|
||||
</SelectItem>
|
||||
))
|
||||
)}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormControl>
|
||||
<FormDescription className='flex-1'>
|
||||
{t('accounts.imapProxy')}
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,567 +0,0 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import {
|
||||
FormField,
|
||||
FormItem,
|
||||
FormLabel,
|
||||
FormMessage,
|
||||
FormControl,
|
||||
FormDescription,
|
||||
} from "@/components/ui/form";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { useFormContext } from "react-hook-form";
|
||||
import { Account } from "./action-dialog";
|
||||
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { format } from "date-fns";
|
||||
import { CalendarIcon } from "lucide-react";
|
||||
import { Calendar } from "@/components/ui/calendar";
|
||||
import { cn, dateFnsLocaleMap } from "@/lib/utils";
|
||||
import { useState } from "react";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { enUS } from "date-fns/locale";
|
||||
import i18n from "@/i18n";
|
||||
|
||||
|
||||
type SyncMode = 'all' | 'since_fixed' | 'since_relative' | 'before_relative';
|
||||
type ScheduleMode = 'interval' | 'cron';
|
||||
type CronMode = 'simple' | 'advanced';
|
||||
type CronFrequency = 'daily' | 'weekly' | 'monthly';
|
||||
|
||||
interface CronSimpleState {
|
||||
frequency: CronFrequency;
|
||||
hour: number;
|
||||
minute: number;
|
||||
dayOfWeek: number;
|
||||
dayOfMonth: number;
|
||||
}
|
||||
|
||||
const DEFAULT_CRON_SIMPLE: CronSimpleState = {
|
||||
frequency: 'daily',
|
||||
hour: 0,
|
||||
minute: 0,
|
||||
dayOfWeek: 1,
|
||||
dayOfMonth: 1,
|
||||
};
|
||||
|
||||
function buildCronFromSimple(s: CronSimpleState): string {
|
||||
switch (s.frequency) {
|
||||
case 'daily':
|
||||
return `0 ${s.minute} ${s.hour} * * *`;
|
||||
case 'weekly':
|
||||
return `0 ${s.minute} ${s.hour} * * ${s.dayOfWeek}`;
|
||||
case 'monthly':
|
||||
return `0 ${s.minute} ${s.hour} ${s.dayOfMonth} * *`;
|
||||
}
|
||||
}
|
||||
|
||||
function tryParseCronToSimple(cron: string): CronSimpleState | null {
|
||||
const fields = cron.trim().split(/\s+/);
|
||||
if (fields.length < 6) return null;
|
||||
|
||||
const sec = fields[0];
|
||||
const min = fields[1];
|
||||
const hour = fields[2];
|
||||
const dom = fields[3];
|
||||
const month = fields[4];
|
||||
const dow = fields[5];
|
||||
|
||||
if (sec !== '0') return null;
|
||||
if (month !== '*') return null;
|
||||
|
||||
const minuteVal = parseInt(min, 10);
|
||||
const hourVal = parseInt(hour, 10);
|
||||
if (isNaN(minuteVal) || isNaN(hourVal)) return null;
|
||||
|
||||
if (dom === '*' && dow === '*') {
|
||||
return { frequency: 'daily', hour: hourVal, minute: minuteVal, dayOfWeek: 1, dayOfMonth: 1 };
|
||||
}
|
||||
if (dom === '*') {
|
||||
const dowVal = parseInt(dow, 10);
|
||||
if (!isNaN(dowVal)) {
|
||||
return { frequency: 'weekly', hour: hourVal, minute: minuteVal, dayOfWeek: dowVal, dayOfMonth: 1 };
|
||||
}
|
||||
}
|
||||
if (dow === '*') {
|
||||
const domVal = parseInt(dom, 10);
|
||||
if (!isNaN(domVal)) {
|
||||
return { frequency: 'monthly', hour: hourVal, minute: minuteVal, dayOfWeek: 1, dayOfMonth: domVal };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export default function Step3() {
|
||||
const { t } = useTranslation();
|
||||
const { control, getValues, setValue } = useFormContext<Account>();
|
||||
const current = getValues();
|
||||
|
||||
const [syncMode, setSyncMode] = useState<SyncMode>(() => {
|
||||
if (current.date_before) return 'before_relative';
|
||||
if (current.date_since?.fixed) return 'since_fixed';
|
||||
if (current.date_since?.relative) return 'since_relative';
|
||||
return 'all';
|
||||
});
|
||||
|
||||
const [scheduleMode, setScheduleMode] = useState<ScheduleMode>(() => {
|
||||
if (current.download_schedule) return 'cron';
|
||||
return 'interval';
|
||||
});
|
||||
|
||||
const [cronMode, setCronMode] = useState<CronMode>(() => {
|
||||
if (current.download_schedule && tryParseCronToSimple(current.download_schedule)) {
|
||||
return 'simple';
|
||||
}
|
||||
if (current.download_schedule) return 'advanced';
|
||||
return 'simple';
|
||||
});
|
||||
|
||||
const [cronSimple, setCronSimple] = useState<CronSimpleState>(() => {
|
||||
if (current.download_schedule) {
|
||||
return tryParseCronToSimple(current.download_schedule) ?? DEFAULT_CRON_SIMPLE;
|
||||
}
|
||||
return DEFAULT_CRON_SIMPLE;
|
||||
});
|
||||
|
||||
const updateCronFromSimple = (partial: Partial<CronSimpleState>) => {
|
||||
const next = { ...cronSimple, ...partial };
|
||||
setCronSimple(next);
|
||||
setValue('download_schedule', buildCronFromSimple(next));
|
||||
};
|
||||
|
||||
const handleModeChange = (mode: SyncMode) => {
|
||||
setSyncMode(mode);
|
||||
|
||||
setValue("date_since", undefined);
|
||||
setValue("date_before", undefined);
|
||||
|
||||
if (mode === 'since_fixed') {
|
||||
setValue("date_since.fixed", undefined);
|
||||
} else if (mode === 'since_relative') {
|
||||
setValue("date_since.relative", { value: 1, unit: 'Months' });
|
||||
} else if (mode === 'before_relative') {
|
||||
setValue("date_before", { value: 1, unit: 'Years' });
|
||||
}
|
||||
};
|
||||
|
||||
const handleScheduleModeChange = (mode: ScheduleMode) => {
|
||||
setScheduleMode(mode);
|
||||
if (mode === 'interval') {
|
||||
setValue("download_schedule", undefined);
|
||||
} else {
|
||||
setValue("download_interval_min", 60);
|
||||
if (cronMode === 'simple') {
|
||||
updateCronFromSimple(cronSimple);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-8">
|
||||
<div className="space-y-4">
|
||||
<FormItem>
|
||||
<FormLabel className="text-base font-semibold">{t('accounts.scheduleMode')}</FormLabel>
|
||||
<FormDescription>
|
||||
{t('accounts.scheduleModeDescription')}
|
||||
</FormDescription>
|
||||
<Select value={scheduleMode} onValueChange={(v) => handleScheduleModeChange(v as ScheduleMode)}>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="interval">{t('accounts.scheduleModeInterval')}</SelectItem>
|
||||
<SelectItem value="cron">{t('accounts.scheduleModeCron')}</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-6">
|
||||
{scheduleMode === 'interval' ? (
|
||||
<FormField
|
||||
control={control}
|
||||
name="download_interval_min"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.downloadInterval')}</FormLabel>
|
||||
<FormControl>
|
||||
<Input type="number" {...field} onChange={(e) => field.onChange(parseInt(e.target.value, 10))} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
<FormDescription>
|
||||
{t('accounts.downloadIntervalPlaceholder')}
|
||||
</FormDescription>
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
) : (
|
||||
<div className="space-y-3">
|
||||
<div className="flex items-center gap-2">
|
||||
{/* <FormLabel className="text-sm font-medium">{t('accounts.downloadSchedule')}</FormLabel> */}
|
||||
<div className="flex items-center rounded-md border text-xs">
|
||||
<button
|
||||
type="button"
|
||||
className={`px-2 py-1 rounded-l-md ${cronMode === 'simple' ? 'bg-primary text-primary-foreground' : 'text-muted-foreground hover:text-foreground'}`}
|
||||
onClick={() => setCronMode('simple')}
|
||||
>
|
||||
{t('accounts.cronSimple')}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`px-2 py-1 rounded-r-md ${cronMode === 'advanced' ? 'bg-primary text-primary-foreground' : 'text-muted-foreground hover:text-foreground'}`}
|
||||
onClick={() => setCronMode('advanced')}
|
||||
>
|
||||
{t('accounts.cronAdvanced')}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{cronMode === 'simple' ? (
|
||||
<div className="flex flex-wrap items-end gap-3">
|
||||
<FormItem className="w-[140px]">
|
||||
<FormLabel className="text-xs">{t('accounts.cronFrequency')}</FormLabel>
|
||||
<Select
|
||||
value={cronSimple.frequency}
|
||||
onValueChange={(v) => updateCronFromSimple({ frequency: v as CronFrequency })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="daily">{t('accounts.cronDaily')}</SelectItem>
|
||||
<SelectItem value="weekly">{t('accounts.cronWeekly')}</SelectItem>
|
||||
<SelectItem value="monthly">{t('accounts.cronMonthly')}</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
|
||||
<FormItem className="w-[80px]">
|
||||
<FormLabel className="text-xs">{t('accounts.cronHour')}</FormLabel>
|
||||
<Select
|
||||
value={String(cronSimple.hour)}
|
||||
onValueChange={(v) => updateCronFromSimple({ hour: parseInt(v, 10) })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{Array.from({ length: 24 }, (_, i) => (
|
||||
<SelectItem key={i} value={String(i)}>
|
||||
{String(i).padStart(2, '0')}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
|
||||
<span className="text-muted-foreground pb-2">:</span>
|
||||
|
||||
<FormItem className="w-[80px]">
|
||||
<FormLabel className="text-xs">{t('accounts.cronMinute')}</FormLabel>
|
||||
<Select
|
||||
value={String(cronSimple.minute)}
|
||||
onValueChange={(v) => updateCronFromSimple({ minute: parseInt(v, 10) })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{[0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55].map((m) => (
|
||||
<SelectItem key={m} value={String(m)}>
|
||||
{String(m).padStart(2, '0')}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
|
||||
{cronSimple.frequency === 'weekly' && (
|
||||
<FormItem className="w-[140px]">
|
||||
<FormLabel className="text-xs">{t('accounts.cronDayOfWeek')}</FormLabel>
|
||||
<Select
|
||||
value={String(cronSimple.dayOfWeek)}
|
||||
onValueChange={(v) => updateCronFromSimple({ dayOfWeek: parseInt(v, 10) })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="1">{t('accounts.cronMonday')}</SelectItem>
|
||||
<SelectItem value="2">{t('accounts.cronTuesday')}</SelectItem>
|
||||
<SelectItem value="3">{t('accounts.cronWednesday')}</SelectItem>
|
||||
<SelectItem value="4">{t('accounts.cronThursday')}</SelectItem>
|
||||
<SelectItem value="5">{t('accounts.cronFriday')}</SelectItem>
|
||||
<SelectItem value="6">{t('accounts.cronSaturday')}</SelectItem>
|
||||
<SelectItem value="0">{t('accounts.cronSunday')}</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
)}
|
||||
|
||||
{cronSimple.frequency === 'monthly' && (
|
||||
<FormItem className="w-[90px]">
|
||||
<FormLabel className="text-xs">{t('accounts.cronDayOfMonth')}</FormLabel>
|
||||
<Select
|
||||
value={String(cronSimple.dayOfMonth)}
|
||||
onValueChange={(v) => updateCronFromSimple({ dayOfMonth: parseInt(v, 10) })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent className="max-h-[200px]">
|
||||
{Array.from({ length: 28 }, (_, i) => i + 1).map((d) => (
|
||||
<SelectItem key={d} value={String(d)}>
|
||||
{d}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
)}
|
||||
|
||||
<div className="text-xs text-muted-foreground pb-2 font-mono">
|
||||
= {buildCronFromSimple(cronSimple)}
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<FormField
|
||||
control={control}
|
||||
name="download_schedule"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormControl>
|
||||
<Input
|
||||
{...field}
|
||||
value={field.value ?? ''}
|
||||
placeholder={t('accounts.downloadSchedulePlaceholder')}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
{t('accounts.downloadScheduleDescription')}
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{cronMode === 'simple' && (
|
||||
<FormDescription>{t('accounts.downloadScheduleDescription')}</FormDescription>
|
||||
)}
|
||||
|
||||
<div className="text-xs text-muted-foreground mt-1 flex items-center gap-1">
|
||||
<span>{t('accounts.cronTimezoneNote')}</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<FormField
|
||||
control={control}
|
||||
name="download_batch_size"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.downloadBatchSize')}</FormLabel>
|
||||
<FormControl>
|
||||
<Input type="number" {...field} onChange={(e) => field.onChange(parseInt(e.target.value, 10))} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
<FormDescription>
|
||||
{t('accounts.downloadBatchSizeDescription')}
|
||||
</FormDescription>
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={control}
|
||||
name="max_email_size_bytes"
|
||||
render={({ field }) => {
|
||||
const BYTES_PER_MB = 1024 * 1024;
|
||||
|
||||
return (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.maxEmailSizeBytes')}</FormLabel>
|
||||
<FormControl>
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
type="number"
|
||||
placeholder={t('accounts.maxEmailSizeBytesPlaceholder')}
|
||||
className="flex-1"
|
||||
value={field.value ? field.value / BYTES_PER_MB : ''}
|
||||
onChange={(e) => {
|
||||
const parsed = parseInt(e.target.value, 10);
|
||||
field.onChange(isNaN(parsed) ? parsed : parsed * BYTES_PER_MB);
|
||||
}}
|
||||
/>
|
||||
<span className="text-sm text-muted-foreground whitespace-nowrap">MB</span>
|
||||
</div>
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
<FormDescription>
|
||||
{t('accounts.maxEmailSizeBytesDescription')}
|
||||
</FormDescription>
|
||||
</FormItem>
|
||||
);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<FormField
|
||||
control={control}
|
||||
name="enabled"
|
||||
render={({ field }) => (
|
||||
<FormItem className="flex flex-row items-start space-x-3 space-y-0 rounded-md border p-4 shadow-sm">
|
||||
<FormControl>
|
||||
<Checkbox checked={field.value} onCheckedChange={field.onChange} />
|
||||
</FormControl>
|
||||
<div className="space-y-1 leading-none">
|
||||
<FormLabel>{t('accounts.enabled')}</FormLabel>
|
||||
<FormDescription>{t('accounts.enabledDescription')}</FormDescription>
|
||||
</div>
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<hr className="my-4" />
|
||||
<div className="space-y-4">
|
||||
<FormItem>
|
||||
<FormLabel className="text-base font-semibold">{t('accounts.downloadScope')}</FormLabel>
|
||||
<FormDescription>
|
||||
{t('accounts.downloadScopeDescription')}
|
||||
</FormDescription>
|
||||
<Select value={syncMode} onValueChange={(v) => handleModeChange(v as SyncMode)}>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue placeholder={t('accounts.selectMode')} />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="all">{t('accounts.downloadAll')}</SelectItem>
|
||||
<SelectItem value="since_fixed">{t('accounts.sinceFixed')}</SelectItem>
|
||||
<SelectItem value="since_relative">{t('accounts.sinceRelative')}</SelectItem>
|
||||
<SelectItem value="before_relative">{t('accounts.beforeRelative')}</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
<div className="pl-2 border-l-2 border-primary/20 space-y-4 pt-2">
|
||||
{syncMode === 'since_fixed' && (
|
||||
<FormField
|
||||
control={control}
|
||||
name="date_since.fixed"
|
||||
render={({ field }) => {
|
||||
const currentLang = i18n.language.toLowerCase().replace('_', '-');
|
||||
const dateLocale = dateFnsLocaleMap[currentLang] || enUS;
|
||||
return <FormItem className="flex flex-col">
|
||||
<FormLabel>{t('accounts.selectDate')}</FormLabel>
|
||||
<Popover>
|
||||
<PopoverTrigger asChild>
|
||||
<FormControl>
|
||||
<Button
|
||||
variant="outline"
|
||||
className={cn("w-[440px] pl-3 text-left font-normal", !field.value && "text-muted-foreground")}
|
||||
>
|
||||
{field.value ? format(new Date(field.value), "PPP", { locale: dateLocale }) : <span>{t('accounts.selectDate')}</span>}
|
||||
<CalendarIcon className="ml-auto h-4 w-4 opacity-50" />
|
||||
</Button>
|
||||
</FormControl>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent className="w-auto p-0" align="start">
|
||||
<Calendar
|
||||
mode="single"
|
||||
selected={field.value ? new Date(field.value) : undefined}
|
||||
onSelect={(date) => field.onChange(date?.toLocaleDateString('en-CA'))}
|
||||
disabled={(date) => date > new Date() || date < new Date("1900-01-01")}
|
||||
locale={dateLocale}
|
||||
/>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
<FormMessage />
|
||||
</FormItem>;
|
||||
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
|
||||
{(syncMode === 'since_relative' || syncMode === 'before_relative') && (
|
||||
<div className="flex flex-row items-end gap-4 animate-in fade-in slide-in-from-left-2">
|
||||
<FormField
|
||||
control={control}
|
||||
name={syncMode === 'since_relative' ? "date_since.relative.value" : "date_before.value"}
|
||||
render={({ field }) => (
|
||||
<FormItem className="flex-1 max-w-[150px]">
|
||||
<FormLabel>{t('accounts.duration', 'Duration')}</FormLabel>
|
||||
<FormControl>
|
||||
<Input type="number" {...field} onChange={(e) => field.onChange(parseInt(e.target.value, 10))} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={control}
|
||||
name={syncMode === 'since_relative' ? "date_since.relative.unit" : "date_before.unit"}
|
||||
render={({ field }) => (
|
||||
<FormItem className="w-[180px]">
|
||||
<FormLabel>{t('accounts.unit', 'Unit')}</FormLabel>
|
||||
<Select onValueChange={field.onChange} value={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue placeholder={t('accounts.selectUnit')} />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="Days">{t('accounts.days')}</SelectItem>
|
||||
<SelectItem value="Months">{t('accounts.months')}</SelectItem>
|
||||
<SelectItem value="Years">{t('accounts.years')}</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<hr className="my-4" />
|
||||
|
||||
<FormField
|
||||
control={control}
|
||||
name="auto_download_new_mailboxes"
|
||||
render={({ field }) => (
|
||||
<FormItem className="flex flex-row items-start space-x-3 space-y-0 rounded-md border p-4 shadow-sm">
|
||||
<FormControl>
|
||||
<Checkbox checked={field.value} onCheckedChange={field.onChange} />
|
||||
</FormControl>
|
||||
<div className="space-y-1 leading-none">
|
||||
<FormLabel>{t('accounts.autoDownloadNewMailboxes')}</FormLabel>
|
||||
<FormDescription>{t('accounts.autoDownloadNewMailboxesDescription')}</FormDescription>
|
||||
</div>
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<hr className="my-4" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,188 +0,0 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import { useFormContext } from "react-hook-form";
|
||||
import { Account } from "./action-dialog";
|
||||
import { Accordion, AccordionItem, AccordionTrigger, AccordionContent } from "@/components/ui/accordion";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import useProxyList from "@/hooks/use-proxy";
|
||||
|
||||
export default function Step4() {
|
||||
const { t } = useTranslation();
|
||||
const { getValues } = useFormContext<Account>();
|
||||
const { getUrlById } = useProxyList();
|
||||
const summaryData = getValues();
|
||||
|
||||
|
||||
const sinceText = (() => {
|
||||
if (summaryData.date_since?.fixed) {
|
||||
return summaryData.date_since.fixed;
|
||||
}
|
||||
|
||||
if (summaryData.date_since?.relative?.value) {
|
||||
return `${t('accounts.sinceRelativeValue', {
|
||||
value: summaryData.date_since!.relative!.value,
|
||||
unit: t(`accounts.${summaryData.date_since!.relative!.unit!.toLowerCase()}`)
|
||||
})}`;
|
||||
}
|
||||
|
||||
return t('accounts.syncAll');
|
||||
})();
|
||||
|
||||
const hasSince = !!summaryData.date_since;
|
||||
const hasBefore = !!summaryData.date_before?.value;
|
||||
|
||||
return (
|
||||
<div className="rounded-xl">
|
||||
<Accordion type="multiple" defaultValue={[
|
||||
'email', 'account_name', 'login_name', 'imap', 'date_since',
|
||||
'max_email_size_bytes', 'sync_interval', 'sync_scope',
|
||||
'sync_batch_size', 'download_schedule'
|
||||
]}>
|
||||
<AccordionItem key="email" value="email">
|
||||
<AccordionTrigger className="font-medium capitalize text-gray-600">{t('accounts.email')}:</AccordionTrigger>
|
||||
<AccordionContent>{summaryData.email}</AccordionContent>
|
||||
</AccordionItem>
|
||||
|
||||
<AccordionItem key="account_name" value="account_name">
|
||||
<AccordionTrigger className="font-medium capitalize text-gray-600">{t('accounts.name')}:</AccordionTrigger>
|
||||
<AccordionContent>{summaryData.account_name ?? t('accounts.notAvailable')}</AccordionContent>
|
||||
</AccordionItem>
|
||||
|
||||
<AccordionItem key="login_name" value="login_name">
|
||||
<AccordionTrigger className="font-medium capitalize text-gray-600">{t('accounts.login_name')}:</AccordionTrigger>
|
||||
<AccordionContent>{summaryData.login_name ?? t('accounts.notAvailable')}</AccordionContent>
|
||||
</AccordionItem>
|
||||
|
||||
<AccordionItem key="imap" value="imap">
|
||||
<AccordionTrigger className="font-medium capitalize text-gray-600">{t('accounts.imap')}:</AccordionTrigger>
|
||||
<AccordionContent>
|
||||
<div className="overflow-x-auto">
|
||||
<table className="min-w-full divide-y">
|
||||
<tbody className="divide-y">
|
||||
<tr>
|
||||
<td className="px-6 py-2 whitespace-nowrap text-sm font-medium text-gray-600">{t('accounts.host')}:</td>
|
||||
<td className="px-6 py-2 whitespace-nowrap text-sm">{summaryData.imap.host}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td className="px-6 py-2 whitespace-nowrap text-sm font-medium text-gray-600">{t('accounts.port')}:</td>
|
||||
<td className="px-6 py-2 whitespace-nowrap text-sm">{summaryData.imap.port}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td className="px-6 py-2 whitespace-nowrap text-sm font-medium text-gray-600">{t('accounts.encryption')}:</td>
|
||||
<td className="px-6 py-2 whitespace-nowrap text-sm">{summaryData.imap.encryption}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td className="px-6 py-2 whitespace-nowrap text-sm font-medium text-gray-600">{t('accounts.useDangerous')}:</td>
|
||||
<td className="px-6 py-2 whitespace-nowrap text-sm">{`${summaryData.use_dangerous}`}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td className="px-6 py-2 whitespace-nowrap text-sm font-medium text-gray-600">{t('accounts.authType')}:</td>
|
||||
<td className="px-6 py-2 whitespace-nowrap text-sm">{summaryData.imap.auth.auth_type}</td>
|
||||
</tr>
|
||||
{summaryData.imap.auth.auth_type === 'Password' && (
|
||||
<tr>
|
||||
<td className="px-6 py-2 whitespace-nowrap text-sm font-medium text-gray-600">{t('accounts.password')}:</td>
|
||||
<td className="px-6 py-2 whitespace-nowrap text-sm break-words">{summaryData.imap.auth.password}</td>
|
||||
</tr>
|
||||
)}
|
||||
<tr>
|
||||
<td className="px-6 py-2 whitespace-nowrap text-sm font-medium text-gray-600">{t('accounts.useProxyField')}:</td>
|
||||
<td className="px-6 py-2 whitespace-nowrap text-sm">
|
||||
{(() => {
|
||||
if (!summaryData.imap.use_proxy) {
|
||||
return t('accounts.useNoProxy');
|
||||
}
|
||||
const proxyUrl = getUrlById(summaryData.imap.use_proxy);
|
||||
return proxyUrl || `${t('common.yes')} (${summaryData.imap.use_proxy})`;
|
||||
})()}
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</AccordionContent>
|
||||
</AccordionItem>
|
||||
|
||||
<AccordionItem key="sync_scope" value="sync_scope">
|
||||
<AccordionTrigger className="font-medium capitalize text-gray-600">
|
||||
{t('accounts.downloadScope')}:
|
||||
</AccordionTrigger>
|
||||
|
||||
<AccordionContent className="space-y-3">
|
||||
{hasSince && (
|
||||
<div className="flex flex-col">
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{t('accounts.sinceFixed')}:
|
||||
</span>
|
||||
<span className="text-sm">{sinceText}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{hasBefore && (
|
||||
<div className="flex flex-col border-t pt-2">
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{t('accounts.beforeRelative')}:
|
||||
</span>
|
||||
<span className="text-sm">
|
||||
{t('accounts.beforeRelativeValue', {
|
||||
value: summaryData.date_before!.value,
|
||||
unit: t(`accounts.${summaryData.date_before!.unit!.toLowerCase()}`)
|
||||
})}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!hasSince && !hasBefore && (
|
||||
<span className="text-sm">
|
||||
{t('accounts.downloadAll')}
|
||||
</span>
|
||||
)}
|
||||
</AccordionContent>
|
||||
</AccordionItem>
|
||||
|
||||
|
||||
<AccordionItem key="sync_interval" value="sync_interval">
|
||||
<AccordionTrigger className="font-medium capitalize text-gray-600">{t('accounts.downloadInterval')}:</AccordionTrigger>
|
||||
<AccordionContent>{summaryData.download_interval_min} {t('accounts.minutes')}</AccordionContent>
|
||||
</AccordionItem>
|
||||
|
||||
<AccordionItem key="sync_batch_size" value="sync_batch_size">
|
||||
<AccordionTrigger className="font-medium capitalize text-gray-600">{t('accounts.downloadBatchSize')}:</AccordionTrigger>
|
||||
<AccordionContent>{summaryData.download_batch_size}</AccordionContent>
|
||||
</AccordionItem>
|
||||
|
||||
<AccordionItem key="max_email_size_bytes" value="max_email_size_bytes">
|
||||
<AccordionTrigger className="font-medium capitalize text-gray-600">{t('accounts.maxEmailSizeBytes')}:</AccordionTrigger>
|
||||
<AccordionContent>{summaryData.max_email_size_bytes ? `${(summaryData.max_email_size_bytes / 1024 / 1024).toFixed(0)} MB` : t('accounts.maxEmailSizeBytesUnlimited')}</AccordionContent>
|
||||
</AccordionItem>
|
||||
|
||||
<AccordionItem key="download_schedule" value="download_schedule">
|
||||
<AccordionTrigger className="font-medium capitalize text-gray-600">{t('accounts.downloadSchedule')}:</AccordionTrigger>
|
||||
<AccordionContent>{summaryData.download_schedule || t('accounts.notAvailable')}</AccordionContent>
|
||||
</AccordionItem>
|
||||
|
||||
<AccordionItem key="auto_download_new_mailboxes" value="auto_download_new_mailboxes">
|
||||
<AccordionTrigger className="font-medium capitalize text-gray-600">{t('accounts.autoDownloadNewMailboxes')}:</AccordionTrigger>
|
||||
<AccordionContent>{summaryData.auto_download_new_mailboxes ? t('common.yes') : t('common.no')}</AccordionContent>
|
||||
</AccordionItem>
|
||||
</Accordion>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
564
web/src/features/accounts/components/tab-download.tsx
Normal file
564
web/src/features/accounts/components/tab-download.tsx
Normal file
@@ -0,0 +1,564 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import { useState } from "react";
|
||||
import { useFormContext } from "react-hook-form";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { format } from "date-fns";
|
||||
import { CalendarIcon } from "lucide-react";
|
||||
import { enUS } from "date-fns/locale";
|
||||
import {
|
||||
FormField,
|
||||
FormItem,
|
||||
FormLabel,
|
||||
FormMessage,
|
||||
FormControl,
|
||||
FormDescription,
|
||||
} from "@/components/ui/form";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
|
||||
import { Calendar } from "@/components/ui/calendar";
|
||||
import { cn, dateFnsLocaleMap } from "@/lib/utils";
|
||||
import i18n from "@/i18n";
|
||||
import { AccountFormValues } from "./schema";
|
||||
|
||||
type SyncMode = 'all' | 'since_fixed' | 'since_relative' | 'before_relative';
|
||||
type ScheduleMode = 'interval' | 'cron';
|
||||
type CronMode = 'simple' | 'advanced';
|
||||
type CronFrequency = 'daily' | 'weekly' | 'monthly';
|
||||
|
||||
interface CronSimpleState {
|
||||
frequency: CronFrequency;
|
||||
hour: number;
|
||||
minute: number;
|
||||
dayOfWeek: number;
|
||||
dayOfMonth: number;
|
||||
}
|
||||
|
||||
const DEFAULT_CRON_SIMPLE: CronSimpleState = {
|
||||
frequency: 'daily',
|
||||
hour: 0,
|
||||
minute: 0,
|
||||
dayOfWeek: 1,
|
||||
dayOfMonth: 1,
|
||||
};
|
||||
|
||||
function buildCronFromSimple(s: CronSimpleState): string {
|
||||
switch (s.frequency) {
|
||||
case 'daily':
|
||||
return `0 ${s.minute} ${s.hour} * * *`;
|
||||
case 'weekly':
|
||||
return `0 ${s.minute} ${s.hour} * * ${s.dayOfWeek}`;
|
||||
case 'monthly':
|
||||
return `0 ${s.minute} ${s.hour} ${s.dayOfMonth} * *`;
|
||||
}
|
||||
}
|
||||
|
||||
function tryParseCronToSimple(cron: string): CronSimpleState | null {
|
||||
const fields = cron.trim().split(/\s+/);
|
||||
if (fields.length < 6) return null;
|
||||
const sec = fields[0];
|
||||
const min = fields[1];
|
||||
const hour = fields[2];
|
||||
const dom = fields[3];
|
||||
const month = fields[4];
|
||||
const dow = fields[5];
|
||||
if (sec !== '0') return null;
|
||||
if (month !== '*') return null;
|
||||
const minuteVal = parseInt(min, 10);
|
||||
const hourVal = parseInt(hour, 10);
|
||||
if (isNaN(minuteVal) || isNaN(hourVal)) return null;
|
||||
if (dom === '*' && dow === '*') {
|
||||
return { frequency: 'daily', hour: hourVal, minute: minuteVal, dayOfWeek: 1, dayOfMonth: 1 };
|
||||
}
|
||||
if (dom === '*') {
|
||||
const dowVal = parseInt(dow, 10);
|
||||
if (!isNaN(dowVal)) {
|
||||
return { frequency: 'weekly', hour: hourVal, minute: minuteVal, dayOfWeek: dowVal, dayOfMonth: 1 };
|
||||
}
|
||||
}
|
||||
if (dow === '*') {
|
||||
const domVal = parseInt(dom, 10);
|
||||
if (!isNaN(domVal)) {
|
||||
return { frequency: 'monthly', hour: hourVal, minute: minuteVal, dayOfWeek: 1, dayOfMonth: domVal };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function TabDownload() {
|
||||
const { t } = useTranslation();
|
||||
const { control, getValues, setValue } = useFormContext<AccountFormValues>();
|
||||
const current = getValues();
|
||||
|
||||
const [syncMode, setSyncMode] = useState<SyncMode>(() => {
|
||||
if (current.date_before) return 'before_relative';
|
||||
if (current.date_since?.fixed) return 'since_fixed';
|
||||
if (current.date_since?.relative) return 'since_relative';
|
||||
return 'all';
|
||||
});
|
||||
|
||||
const [scheduleMode, setScheduleMode] = useState<ScheduleMode>(() => {
|
||||
if (current.download_schedule) return 'cron';
|
||||
return 'interval';
|
||||
});
|
||||
|
||||
const [cronMode, setCronMode] = useState<CronMode>(() => {
|
||||
if (current.download_schedule && tryParseCronToSimple(current.download_schedule)) {
|
||||
return 'simple';
|
||||
}
|
||||
if (current.download_schedule) return 'advanced';
|
||||
return 'simple';
|
||||
});
|
||||
|
||||
const [cronSimple, setCronSimple] = useState<CronSimpleState>(() => {
|
||||
if (current.download_schedule) {
|
||||
return tryParseCronToSimple(current.download_schedule) ?? DEFAULT_CRON_SIMPLE;
|
||||
}
|
||||
return DEFAULT_CRON_SIMPLE;
|
||||
});
|
||||
|
||||
const updateCronFromSimple = (partial: Partial<CronSimpleState>) => {
|
||||
const next = { ...cronSimple, ...partial };
|
||||
setCronSimple(next);
|
||||
setValue('download_schedule', buildCronFromSimple(next));
|
||||
};
|
||||
|
||||
const handleModeChange = (mode: SyncMode) => {
|
||||
setSyncMode(mode);
|
||||
setValue("date_since", undefined);
|
||||
setValue("date_before", undefined);
|
||||
if (mode === 'since_fixed') {
|
||||
setValue("date_since.fixed", undefined);
|
||||
} else if (mode === 'since_relative') {
|
||||
setValue("date_since.relative", { value: 1, unit: 'Months' });
|
||||
} else if (mode === 'before_relative') {
|
||||
setValue("date_before", { value: 1, unit: 'Years' });
|
||||
}
|
||||
};
|
||||
|
||||
const handleScheduleModeChange = (mode: ScheduleMode) => {
|
||||
setScheduleMode(mode);
|
||||
if (mode === 'interval') {
|
||||
setValue("download_schedule", undefined);
|
||||
} else {
|
||||
setValue("download_interval_min", 60);
|
||||
if (cronMode === 'simple') {
|
||||
updateCronFromSimple(cronSimple);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const BYTES_PER_MB = 1024 * 1024;
|
||||
|
||||
return (
|
||||
<div className="space-y-8">
|
||||
{/* Schedule */}
|
||||
<div className="space-y-4">
|
||||
<h4 className="text-sm font-semibold uppercase tracking-wider text-muted-foreground">
|
||||
{t('accounts.settings.schedule')}
|
||||
</h4>
|
||||
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.scheduleMode')}</FormLabel>
|
||||
<FormDescription>{t('accounts.scheduleModeDescription')}</FormDescription>
|
||||
<Select value={scheduleMode} onValueChange={(v) => handleScheduleModeChange(v as ScheduleMode)}>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="interval">{t('accounts.scheduleModeInterval')}</SelectItem>
|
||||
<SelectItem value="cron">{t('accounts.scheduleModeCron')}</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
{scheduleMode === 'interval' && (
|
||||
<FormField
|
||||
control={control}
|
||||
name="download_interval_min"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.downloadInterval')}<span className="text-red-500 align-super text-xs">*</span></FormLabel>
|
||||
<FormControl>
|
||||
<Input type="number" {...field} onChange={(e) => field.onChange(parseInt(e.target.value, 10))} />
|
||||
</FormControl>
|
||||
<FormDescription>{t('accounts.downloadIntervalPlaceholder')}</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{scheduleMode === 'cron' && (
|
||||
<div className="space-y-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="flex items-center rounded-md border text-xs">
|
||||
<button
|
||||
type="button"
|
||||
className={`px-2 py-1 rounded-l-md ${cronMode === 'simple' ? 'bg-primary text-primary-foreground' : 'text-muted-foreground hover:text-foreground'}`}
|
||||
onClick={() => setCronMode('simple')}
|
||||
>
|
||||
{t('accounts.cronSimple')}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`px-2 py-1 rounded-r-md ${cronMode === 'advanced' ? 'bg-primary text-primary-foreground' : 'text-muted-foreground hover:text-foreground'}`}
|
||||
onClick={() => setCronMode('advanced')}
|
||||
>
|
||||
{t('accounts.cronAdvanced')}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{cronMode === 'simple' ? (
|
||||
<>
|
||||
<div className="flex items-end gap-2 overflow-x-auto pb-1">
|
||||
<FormItem className="w-[120px] shrink-0">
|
||||
<FormLabel className="text-sm">{t('accounts.cronFrequency')}</FormLabel>
|
||||
<Select
|
||||
value={cronSimple.frequency}
|
||||
onValueChange={(v) => updateCronFromSimple({ frequency: v as CronFrequency })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="daily">{t('accounts.cronDaily')}</SelectItem>
|
||||
<SelectItem value="weekly">{t('accounts.cronWeekly')}</SelectItem>
|
||||
<SelectItem value="monthly">{t('accounts.cronMonthly')}</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
|
||||
<FormItem className="w-[72px] shrink-0">
|
||||
<FormLabel className="text-sm">{t('accounts.cronHour')}</FormLabel>
|
||||
<Select
|
||||
value={String(cronSimple.hour)}
|
||||
onValueChange={(v) => updateCronFromSimple({ hour: parseInt(v, 10) })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{Array.from({ length: 24 }, (_, i) => (
|
||||
<SelectItem key={i} value={String(i)}>
|
||||
{String(i).padStart(2, '0')}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
|
||||
<span className="text-muted-foreground font-medium pb-2 shrink-0">:</span>
|
||||
|
||||
<FormItem className="w-[72px] shrink-0">
|
||||
<FormLabel className="text-sm">{t('accounts.cronMinute')}</FormLabel>
|
||||
<Select
|
||||
value={String(cronSimple.minute)}
|
||||
onValueChange={(v) => updateCronFromSimple({ minute: parseInt(v, 10) })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{[0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55].map((m) => (
|
||||
<SelectItem key={m} value={String(m)}>
|
||||
{String(m).padStart(2, '0')}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
|
||||
{cronSimple.frequency === 'weekly' && (
|
||||
<FormItem className="w-[120px] shrink-0">
|
||||
<FormLabel className="text-sm">{t('accounts.cronDayOfWeek')}</FormLabel>
|
||||
<Select
|
||||
value={String(cronSimple.dayOfWeek)}
|
||||
onValueChange={(v) => updateCronFromSimple({ dayOfWeek: parseInt(v, 10) })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="1">{t('accounts.cronMonday')}</SelectItem>
|
||||
<SelectItem value="2">{t('accounts.cronTuesday')}</SelectItem>
|
||||
<SelectItem value="3">{t('accounts.cronWednesday')}</SelectItem>
|
||||
<SelectItem value="4">{t('accounts.cronThursday')}</SelectItem>
|
||||
<SelectItem value="5">{t('accounts.cronFriday')}</SelectItem>
|
||||
<SelectItem value="6">{t('accounts.cronSaturday')}</SelectItem>
|
||||
<SelectItem value="0">{t('accounts.cronSunday')}</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
)}
|
||||
|
||||
{cronSimple.frequency === 'monthly' && (
|
||||
<FormItem className="w-[80px] shrink-0">
|
||||
<FormLabel className="text-sm">{t('accounts.cronDayOfMonth')}</FormLabel>
|
||||
<Select
|
||||
value={String(cronSimple.dayOfMonth)}
|
||||
onValueChange={(v) => updateCronFromSimple({ dayOfMonth: parseInt(v, 10) })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent className="max-h-[200px]">
|
||||
{Array.from({ length: 31 }, (_, i) => i + 1).map((d) => (
|
||||
<SelectItem key={d} value={String(d)}>{d}</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<p className="text-xs text-muted-foreground">
|
||||
{cronSimple.frequency === 'daily' && t('accounts.cronSummaryDaily', { hour: String(cronSimple.hour).padStart(2, '0'), minute: String(cronSimple.minute).padStart(2, '0') })}
|
||||
{cronSimple.frequency === 'weekly' && (() => {
|
||||
const dayNames = ['cronSunday', 'cronMonday', 'cronTuesday', 'cronWednesday', 'cronThursday', 'cronFriday', 'cronSaturday'];
|
||||
return t('accounts.cronSummaryWeekly', { hour: String(cronSimple.hour).padStart(2, '0'), minute: String(cronSimple.minute).padStart(2, '0'), day: t(`accounts.${dayNames[cronSimple.dayOfWeek]}`) });
|
||||
})()}
|
||||
{cronSimple.frequency === 'monthly' && t('accounts.cronSummaryMonthly', { hour: String(cronSimple.hour).padStart(2, '0'), minute: String(cronSimple.minute).padStart(2, '0'), day: cronSimple.dayOfMonth })}
|
||||
{cronSimple.frequency === 'monthly' && cronSimple.dayOfMonth > 28 && (
|
||||
<span className="block text-xs text-yellow-600 mt-0.5">{t('accounts.cronMonthAlignNote')}</span>
|
||||
)}
|
||||
</p>
|
||||
</>
|
||||
) : (
|
||||
<FormField
|
||||
control={control}
|
||||
name="download_schedule"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormControl>
|
||||
<Input
|
||||
{...field}
|
||||
value={field.value ?? ''}
|
||||
placeholder={t('accounts.downloadSchedulePlaceholder')}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>{t('accounts.downloadScheduleDescription')}</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
|
||||
<div className="text-xs text-muted-foreground mt-1 flex items-center gap-1">
|
||||
<span>{t('accounts.cronTimezoneNote')}</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<hr />
|
||||
|
||||
{/* Batch & Size */}
|
||||
<div className="space-y-4">
|
||||
<h4 className="text-sm font-semibold uppercase tracking-wider text-muted-foreground">
|
||||
{t('accounts.settings.performance')}
|
||||
</h4>
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<FormField
|
||||
control={control}
|
||||
name="download_batch_size"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.downloadBatchSize')}<span className="text-red-500 align-super text-xs">*</span></FormLabel>
|
||||
<FormControl>
|
||||
<Input type="number" {...field} onChange={(e) => field.onChange(parseInt(e.target.value, 10))} />
|
||||
</FormControl>
|
||||
<FormDescription>{t('accounts.downloadBatchSizeDescription')}</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={control}
|
||||
name="max_email_size_bytes"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.maxEmailSizeBytes')}<span className="text-red-500 align-super text-xs">*</span></FormLabel>
|
||||
<FormControl>
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
type="number"
|
||||
placeholder={t('accounts.maxEmailSizeBytesPlaceholder')}
|
||||
className="flex-1"
|
||||
value={field.value ? field.value / BYTES_PER_MB : ''}
|
||||
onChange={(e) => {
|
||||
const parsed = parseInt(e.target.value, 10);
|
||||
field.onChange(isNaN(parsed) ? parsed : parsed * BYTES_PER_MB);
|
||||
}}
|
||||
/>
|
||||
<span className="text-sm text-muted-foreground whitespace-nowrap">MB</span>
|
||||
</div>
|
||||
</FormControl>
|
||||
<FormDescription>{t('accounts.maxEmailSizeBytesDescription')}</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<hr />
|
||||
|
||||
{/* Download Scope */}
|
||||
<div className="space-y-4">
|
||||
<h4 className="text-sm font-semibold uppercase tracking-wider text-muted-foreground">
|
||||
{t('accounts.settings.scope')}
|
||||
</h4>
|
||||
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.downloadScope')}</FormLabel>
|
||||
<FormDescription>{t('accounts.downloadScopeDescription')}</FormDescription>
|
||||
<Select value={syncMode} onValueChange={(v) => handleModeChange(v as SyncMode)}>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="all">{t('accounts.downloadAll')}</SelectItem>
|
||||
<SelectItem value="since_fixed">{t('accounts.sinceFixed')}</SelectItem>
|
||||
<SelectItem value="since_relative">{t('accounts.sinceRelative')}</SelectItem>
|
||||
<SelectItem value="before_relative">{t('accounts.beforeRelative')}</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormDescription className="mt-2">
|
||||
{syncMode === 'all' && t('accounts.downloadAllDesc')}
|
||||
{syncMode === 'since_fixed' && t('accounts.sinceFixedDesc')}
|
||||
{syncMode === 'since_relative' && t('accounts.sinceRelativeDesc')}
|
||||
{syncMode === 'before_relative' && t('accounts.beforeRelativeDesc')}
|
||||
</FormDescription>
|
||||
</FormItem>
|
||||
|
||||
<div className="pl-2 border-l-2 border-primary/20 space-y-4 pt-2">
|
||||
{syncMode === 'since_fixed' && (
|
||||
<FormField
|
||||
control={control}
|
||||
name="date_since.fixed"
|
||||
render={({ field }) => {
|
||||
const currentLang = i18n.language.toLowerCase().replace('_', '-');
|
||||
const dateLocale = dateFnsLocaleMap[currentLang] || enUS;
|
||||
return (
|
||||
<FormItem className="flex flex-col">
|
||||
<FormLabel>{t('accounts.selectDate')}<span className="text-red-500 align-super text-xs">*</span></FormLabel>
|
||||
<Popover>
|
||||
<PopoverTrigger asChild>
|
||||
<FormControl>
|
||||
<Button
|
||||
variant="outline"
|
||||
className={cn("w-[440px] pl-3 text-left font-normal", !field.value && "text-muted-foreground")}
|
||||
>
|
||||
{field.value ? format(new Date(field.value), "PPP", { locale: dateLocale }) : <span>{t('accounts.selectDate')}</span>}
|
||||
<CalendarIcon className="ml-auto h-4 w-4 opacity-50" />
|
||||
</Button>
|
||||
</FormControl>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent className="w-auto p-0" align="start">
|
||||
<Calendar
|
||||
mode="single"
|
||||
selected={field.value ? new Date(field.value) : undefined}
|
||||
onSelect={(date) => field.onChange(date?.toLocaleDateString('en-CA'))}
|
||||
disabled={(date) => date > new Date() || date < new Date("1900-01-01")}
|
||||
locale={dateLocale}
|
||||
/>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
);
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
|
||||
{(syncMode === 'since_relative' || syncMode === 'before_relative') && (
|
||||
<div className="flex flex-row items-end gap-4">
|
||||
<FormField
|
||||
control={control}
|
||||
name={syncMode === 'since_relative' ? "date_since.relative.value" : "date_before.value"}
|
||||
render={({ field }) => (
|
||||
<FormItem className="flex-1 max-w-[150px]">
|
||||
<FormLabel>{t('accounts.duration')}<span className="text-red-500 align-super text-xs">*</span></FormLabel>
|
||||
<FormControl>
|
||||
<Input type="number" {...field} onChange={(e) => field.onChange(parseInt(e.target.value, 10))} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={control}
|
||||
name={syncMode === 'since_relative' ? "date_since.relative.unit" : "date_before.unit"}
|
||||
render={({ field }) => (
|
||||
<FormItem className="w-[180px]">
|
||||
<FormLabel>{t('accounts.unit', 'Unit')}<span className="text-red-500 align-super text-xs">*</span></FormLabel>
|
||||
<Select onValueChange={field.onChange} value={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="Days">{t('accounts.days')}</SelectItem>
|
||||
<SelectItem value="Months">{t('accounts.months')}</SelectItem>
|
||||
<SelectItem value="Years">{t('accounts.years')}</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<hr />
|
||||
|
||||
<FormField
|
||||
control={control}
|
||||
name="auto_download_new_mailboxes"
|
||||
render={({ field }) => (
|
||||
<FormItem className="flex flex-row items-start space-x-3 space-y-0 rounded-md border p-4">
|
||||
<FormControl>
|
||||
<Checkbox checked={field.value} onCheckedChange={field.onChange} />
|
||||
</FormControl>
|
||||
<div className="space-y-1 leading-none">
|
||||
<FormLabel>{t('accounts.autoDownloadNewMailboxes')}</FormLabel>
|
||||
<FormDescription>{t('accounts.autoDownloadNewMailboxesDescription')}</FormDescription>
|
||||
</div>
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
514
web/src/features/accounts/components/tab-filters.tsx
Normal file
514
web/src/features/accounts/components/tab-filters.tsx
Normal file
@@ -0,0 +1,514 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import { useFormContext, useWatch } from "react-hook-form";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import {
|
||||
FormField,
|
||||
FormItem,
|
||||
FormLabel,
|
||||
FormMessage,
|
||||
FormControl,
|
||||
FormDescription,
|
||||
} from "@/components/ui/form";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Plus, HelpCircle } from "lucide-react";
|
||||
import { PatternInput } from "./pattern-input";
|
||||
import type { PatternEntry } from "@/lib/pattern-utils";
|
||||
import { newPatternId, simplePatternToRegex } from "@/lib/pattern-utils";
|
||||
import { useState } from "react";
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
import { AccountFormValues } from "./schema";
|
||||
|
||||
const SUGGESTED_SPAM_HEADERS = [
|
||||
'X-Spam-Flag',
|
||||
'X-Spam',
|
||||
'X-Spam-Status',
|
||||
'X-Barracuda-Spam-Status',
|
||||
'X-Barracuda-Spam-Flag',
|
||||
'X-MS-Exchange-Organization-SCL',
|
||||
];
|
||||
|
||||
function toPatternEntries(patterns: string[]): PatternEntry[] {
|
||||
return patterns.map((p) => ({
|
||||
id: newPatternId(),
|
||||
matchType: 'regex' as const,
|
||||
value: p,
|
||||
}));
|
||||
}
|
||||
|
||||
function patternsToRegexList(entries: PatternEntry[]): string[] {
|
||||
return entries
|
||||
.filter((e) => e.value.trim() !== '')
|
||||
.map((e) => simplePatternToRegex(e.matchType, e.value));
|
||||
}
|
||||
|
||||
export function TabFilters() {
|
||||
const { t } = useTranslation();
|
||||
const { control, setValue } = useFormContext<AccountFormValues>();
|
||||
const archiveRules = useWatch({ control, name: 'archive_rules' });
|
||||
|
||||
const enabled = archiveRules?.enabled ?? false;
|
||||
const sendersInclude = archiveRules?.senders?.include ?? [];
|
||||
const sendersExclude = archiveRules?.senders?.exclude ?? [];
|
||||
const subjectsInclude = archiveRules?.subjects?.include ?? [];
|
||||
const subjectsExclude = archiveRules?.subjects?.exclude ?? [];
|
||||
const spamHeaders = archiveRules?.spam_headers ?? [];
|
||||
|
||||
const [senderIncludeEntries, setSenderIncludeEntries] = useState<PatternEntry[]>(
|
||||
() => toPatternEntries(sendersInclude)
|
||||
);
|
||||
const [senderExcludeEntries, setSenderExcludeEntries] = useState<PatternEntry[]>(
|
||||
() => toPatternEntries(sendersExclude)
|
||||
);
|
||||
const [subjectIncludeEntries, setSubjectIncludeEntries] = useState<PatternEntry[]>(
|
||||
() => toPatternEntries(subjectsInclude)
|
||||
);
|
||||
const [subjectExcludeEntries, setSubjectExcludeEntries] = useState<PatternEntry[]>(
|
||||
() => toPatternEntries(subjectsExclude)
|
||||
);
|
||||
|
||||
// Resync local state when form values change externally (e.g. after form.reset)
|
||||
const [lastSyncKey, setLastSyncKey] = useState<string>('');
|
||||
const syncKey = JSON.stringify({ sendersInclude, sendersExclude, subjectsInclude, subjectsExclude });
|
||||
if (syncKey !== lastSyncKey) {
|
||||
setLastSyncKey(syncKey);
|
||||
setSenderIncludeEntries(toPatternEntries(sendersInclude));
|
||||
setSenderExcludeEntries(toPatternEntries(sendersExclude));
|
||||
setSubjectIncludeEntries(toPatternEntries(subjectsInclude));
|
||||
setSubjectExcludeEntries(toPatternEntries(subjectsExclude));
|
||||
}
|
||||
|
||||
const syncToForm = (
|
||||
includeEntries: PatternEntry[],
|
||||
excludeEntries: PatternEntry[],
|
||||
fieldPrefix: string
|
||||
) => {
|
||||
const include = patternsToRegexList(includeEntries);
|
||||
const exclude = patternsToRegexList(excludeEntries);
|
||||
setValue(`${fieldPrefix}.include` as any, include);
|
||||
setValue(`${fieldPrefix}.exclude` as any, exclude);
|
||||
};
|
||||
|
||||
const addEntry = (
|
||||
side: 'include' | 'exclude',
|
||||
includeEntries: PatternEntry[],
|
||||
excludeEntries: PatternEntry[],
|
||||
setIncludeEntries: React.Dispatch<React.SetStateAction<PatternEntry[]>>,
|
||||
setExcludeEntries: React.Dispatch<React.SetStateAction<PatternEntry[]>>,
|
||||
fieldPrefix: string
|
||||
) => {
|
||||
const newEntry: PatternEntry = { id: newPatternId(), matchType: 'contains', value: '' };
|
||||
const newInclude = side === 'include' ? [...includeEntries, newEntry] : includeEntries;
|
||||
const newExclude = side === 'exclude' ? [...excludeEntries, newEntry] : excludeEntries;
|
||||
setIncludeEntries(newInclude);
|
||||
setExcludeEntries(newExclude);
|
||||
syncToForm(newInclude, newExclude, fieldPrefix);
|
||||
};
|
||||
|
||||
const updateEntry = (
|
||||
id: string,
|
||||
partial: Partial<PatternEntry>,
|
||||
side: 'include' | 'exclude',
|
||||
includeEntries: PatternEntry[],
|
||||
excludeEntries: PatternEntry[],
|
||||
setIncludeEntries: React.Dispatch<React.SetStateAction<PatternEntry[]>>,
|
||||
setExcludeEntries: React.Dispatch<React.SetStateAction<PatternEntry[]>>,
|
||||
fieldPrefix: string
|
||||
) => {
|
||||
if (side === 'include') {
|
||||
const updated = includeEntries.map((e) => (e.id === id ? { ...e, ...partial } : e));
|
||||
setIncludeEntries(updated);
|
||||
syncToForm(updated, excludeEntries, fieldPrefix);
|
||||
} else {
|
||||
const updated = excludeEntries.map((e) => (e.id === id ? { ...e, ...partial } : e));
|
||||
setExcludeEntries(updated);
|
||||
syncToForm(includeEntries, updated, fieldPrefix);
|
||||
}
|
||||
};
|
||||
|
||||
const removeEntry = (
|
||||
id: string,
|
||||
side: 'include' | 'exclude',
|
||||
includeEntries: PatternEntry[],
|
||||
excludeEntries: PatternEntry[],
|
||||
setIncludeEntries: React.Dispatch<React.SetStateAction<PatternEntry[]>>,
|
||||
setExcludeEntries: React.Dispatch<React.SetStateAction<PatternEntry[]>>,
|
||||
fieldPrefix: string
|
||||
) => {
|
||||
if (side === 'include') {
|
||||
const filtered = includeEntries.filter((e) => e.id !== id);
|
||||
setIncludeEntries(filtered);
|
||||
syncToForm(filtered, excludeEntries, fieldPrefix);
|
||||
} else {
|
||||
const filtered = excludeEntries.filter((e) => e.id !== id);
|
||||
setExcludeEntries(filtered);
|
||||
syncToForm(includeEntries, filtered, fieldPrefix);
|
||||
}
|
||||
};
|
||||
|
||||
const handleEnableChange = (checked: boolean) => {
|
||||
if (checked) {
|
||||
setValue('archive_rules', {
|
||||
enabled: true,
|
||||
senders: { include: [], exclude: [] },
|
||||
subjects: { include: [], exclude: [] },
|
||||
skip_larger_than: undefined,
|
||||
spam_headers: [],
|
||||
});
|
||||
} else {
|
||||
setValue('archive_rules', undefined);
|
||||
}
|
||||
};
|
||||
|
||||
const addSpamHeader = (header: string) => {
|
||||
if (!spamHeaders.includes(header)) {
|
||||
setValue('archive_rules.spam_headers', [...spamHeaders, header]);
|
||||
}
|
||||
};
|
||||
|
||||
const removeSpamHeader = (header: string) => {
|
||||
setValue('archive_rules.spam_headers', spamHeaders.filter((h) => h !== header));
|
||||
};
|
||||
|
||||
const [newSpamHeader, setNewSpamHeader] = useState('');
|
||||
|
||||
const handleAddCustomSpamHeader = () => {
|
||||
const trimmed = newSpamHeader.trim();
|
||||
if (trimmed && !spamHeaders.includes(trimmed)) {
|
||||
setValue('archive_rules.spam_headers', [...spamHeaders, trimmed]);
|
||||
setNewSpamHeader('');
|
||||
}
|
||||
};
|
||||
|
||||
const BYTES_PER_MB = 1024 * 1024;
|
||||
|
||||
return (
|
||||
<div className="space-y-8">
|
||||
{/* Master Switch */}
|
||||
<div className="rounded-md border p-5 space-y-2 bg-muted/30">
|
||||
<FormItem className="flex flex-row items-start space-x-3 space-y-0">
|
||||
<FormControl>
|
||||
<Checkbox checked={enabled} onCheckedChange={handleEnableChange} />
|
||||
</FormControl>
|
||||
<div className="space-y-1 leading-none">
|
||||
<FormLabel>{t('accounts.filters.enableFiltering')}</FormLabel>
|
||||
<FormDescription>
|
||||
{t('accounts.filters.enableFilteringDesc')}
|
||||
</FormDescription>
|
||||
</div>
|
||||
</FormItem>
|
||||
</div>
|
||||
|
||||
{enabled && (
|
||||
<>
|
||||
{/* Sender Filters */}
|
||||
<div className="space-y-4 rounded-md border p-5">
|
||||
<div className="flex items-center gap-2">
|
||||
<h4 className="text-sm font-semibold">{t('accounts.filters.senderFilter')}</h4>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<HelpCircle className="h-3.5 w-3.5 text-muted-foreground" />
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
{t('accounts.filters.senderFilterHelp')}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</div>
|
||||
|
||||
<div className="space-y-3">
|
||||
<div>
|
||||
<p className="text-xs font-medium text-muted-foreground mb-2 uppercase tracking-wider">
|
||||
{t('accounts.filters.include')}
|
||||
</p>
|
||||
{senderIncludeEntries.length === 0 ? (
|
||||
<p className="text-xs text-muted-foreground italic">
|
||||
{t('accounts.filters.noIncludePatterns')}
|
||||
</p>
|
||||
) : (
|
||||
<div className="space-y-2">
|
||||
{senderIncludeEntries.map((entry) => (
|
||||
<PatternInput
|
||||
key={entry.id}
|
||||
entry={entry}
|
||||
onChange={(id, partial) =>
|
||||
updateEntry(id, partial, 'include', senderIncludeEntries, senderExcludeEntries, setSenderIncludeEntries, setSenderExcludeEntries, 'archive_rules.senders')
|
||||
}
|
||||
onRemove={(id) =>
|
||||
removeEntry(id, 'include', senderIncludeEntries, senderExcludeEntries, setSenderIncludeEntries, setSenderExcludeEntries, 'archive_rules.senders')
|
||||
}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<Button
|
||||
variant="ghost"
|
||||
type="button"
|
||||
size="sm"
|
||||
className="mt-2 h-8 text-xs"
|
||||
onClick={() => addEntry('include', senderIncludeEntries, senderExcludeEntries, setSenderIncludeEntries, setSenderExcludeEntries, 'archive_rules.senders')}
|
||||
>
|
||||
<Plus className="h-3 w-3 mr-1" />
|
||||
{t('accounts.filters.addPattern')}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<p className="text-xs font-medium text-muted-foreground mb-2 uppercase tracking-wider">
|
||||
{t('accounts.filters.exclude')}
|
||||
</p>
|
||||
{senderExcludeEntries.length === 0 ? (
|
||||
<p className="text-xs text-muted-foreground italic">
|
||||
{t('accounts.filters.noExcludePatterns')}
|
||||
</p>
|
||||
) : (
|
||||
<div className="space-y-2">
|
||||
{senderExcludeEntries.map((entry) => (
|
||||
<PatternInput
|
||||
key={entry.id}
|
||||
entry={entry}
|
||||
onChange={(id, partial) =>
|
||||
updateEntry(id, partial, 'exclude', senderIncludeEntries, senderExcludeEntries, setSenderIncludeEntries, setSenderExcludeEntries, 'archive_rules.senders')
|
||||
}
|
||||
onRemove={(id) =>
|
||||
removeEntry(id, 'exclude', senderIncludeEntries, senderExcludeEntries, setSenderIncludeEntries, setSenderExcludeEntries, 'archive_rules.senders')
|
||||
}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<Button
|
||||
variant="ghost"
|
||||
type="button"
|
||||
size="sm"
|
||||
className="mt-2 h-8 text-xs"
|
||||
onClick={() => addEntry('exclude', senderIncludeEntries, senderExcludeEntries, setSenderIncludeEntries, setSenderExcludeEntries, 'archive_rules.senders')}
|
||||
>
|
||||
<Plus className="h-3 w-3 mr-1" />
|
||||
{t('accounts.filters.addPattern')}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Subject Filters */}
|
||||
<div className="space-y-4 rounded-md border p-5">
|
||||
<div className="flex items-center gap-2">
|
||||
<h4 className="text-sm font-semibold">{t('accounts.filters.subjectFilter')}</h4>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<HelpCircle className="h-3.5 w-3.5 text-muted-foreground" />
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
{t('accounts.filters.subjectFilterHelp')}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</div>
|
||||
<div className="space-y-3">
|
||||
<div>
|
||||
<p className="text-xs font-medium text-muted-foreground mb-2 uppercase tracking-wider">
|
||||
{t('accounts.filters.include')}
|
||||
</p>
|
||||
{subjectIncludeEntries.length === 0 ? (
|
||||
<p className="text-xs text-muted-foreground italic">
|
||||
{t('accounts.filters.noIncludePatterns')}
|
||||
</p>
|
||||
) : (
|
||||
<div className="space-y-2">
|
||||
{subjectIncludeEntries.map((entry) => (
|
||||
<PatternInput
|
||||
key={entry.id}
|
||||
entry={entry}
|
||||
onChange={(id, partial) =>
|
||||
updateEntry(id, partial, 'include', subjectIncludeEntries, subjectExcludeEntries, setSubjectIncludeEntries, setSubjectExcludeEntries, 'archive_rules.subjects')
|
||||
}
|
||||
onRemove={(id) =>
|
||||
removeEntry(id, 'include', subjectIncludeEntries, subjectExcludeEntries, setSubjectIncludeEntries, setSubjectExcludeEntries, 'archive_rules.subjects')
|
||||
}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<Button
|
||||
variant="ghost"
|
||||
type="button"
|
||||
size="sm"
|
||||
className="mt-2 h-8 text-xs"
|
||||
onClick={() => addEntry('include', subjectIncludeEntries, subjectExcludeEntries, setSubjectIncludeEntries, setSubjectExcludeEntries, 'archive_rules.subjects')}
|
||||
>
|
||||
<Plus className="h-3 w-3 mr-1" />
|
||||
{t('accounts.filters.addPattern')}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<p className="text-xs font-medium text-muted-foreground mb-2 uppercase tracking-wider">
|
||||
{t('accounts.filters.exclude')}
|
||||
</p>
|
||||
{subjectExcludeEntries.length === 0 ? (
|
||||
<p className="text-xs text-muted-foreground italic">
|
||||
{t('accounts.filters.noExcludePatterns')}
|
||||
</p>
|
||||
) : (
|
||||
<div className="space-y-2">
|
||||
{subjectExcludeEntries.map((entry) => (
|
||||
<PatternInput
|
||||
key={entry.id}
|
||||
entry={entry}
|
||||
onChange={(id, partial) =>
|
||||
updateEntry(id, partial, 'exclude', subjectIncludeEntries, subjectExcludeEntries, setSubjectIncludeEntries, setSubjectExcludeEntries, 'archive_rules.subjects')
|
||||
}
|
||||
onRemove={(id) =>
|
||||
removeEntry(id, 'exclude', subjectIncludeEntries, subjectExcludeEntries, setSubjectIncludeEntries, setSubjectExcludeEntries, 'archive_rules.subjects')
|
||||
}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<Button
|
||||
variant="ghost"
|
||||
type="button"
|
||||
size="sm"
|
||||
className="mt-2 h-8 text-xs"
|
||||
onClick={() => addEntry('exclude', subjectIncludeEntries, subjectExcludeEntries, setSubjectIncludeEntries, setSubjectExcludeEntries, 'archive_rules.subjects')}
|
||||
>
|
||||
<Plus className="h-3 w-3 mr-1" />
|
||||
{t('accounts.filters.addPattern')}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Size Limit */}
|
||||
<div className="space-y-4 rounded-md border p-5">
|
||||
<div className="flex items-center gap-2">
|
||||
<h4 className="text-sm font-semibold">{t('accounts.filters.sizeLimit')}</h4>
|
||||
</div>
|
||||
<FormField
|
||||
control={control}
|
||||
name="archive_rules.skip_larger_than"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.filters.skipLargerThan')}</FormLabel>
|
||||
<FormControl>
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
type="number"
|
||||
placeholder={t('accounts.filters.noLimit')}
|
||||
className="max-w-[160px]"
|
||||
value={field.value ? field.value / BYTES_PER_MB : ''}
|
||||
onChange={(e) => {
|
||||
const parsed = parseInt(e.target.value, 10);
|
||||
field.onChange(isNaN(parsed) ? undefined : parsed * BYTES_PER_MB);
|
||||
}}
|
||||
/>
|
||||
<span className="text-sm text-muted-foreground">MB</span>
|
||||
</div>
|
||||
</FormControl>
|
||||
<FormDescription>{t('accounts.filters.sizeLimitDesc')}</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Spam Headers */}
|
||||
<div className="space-y-4 rounded-md border p-5">
|
||||
<div className="flex items-center gap-2">
|
||||
<h4 className="text-sm font-semibold">{t('accounts.filters.spamHeaders')}</h4>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<HelpCircle className="h-3.5 w-3.5 text-muted-foreground" />
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
{t('accounts.filters.spamHeadersHelp')}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
{spamHeaders.length > 0 ? (
|
||||
spamHeaders.map((header) => (
|
||||
<div key={header} className="flex items-center gap-2">
|
||||
<div className="flex-1 rounded-md border bg-muted/50 px-3 py-1.5 text-sm font-mono">
|
||||
{header}
|
||||
</div>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-7 w-7 shrink-0"
|
||||
onClick={() => removeSpamHeader(header)}
|
||||
>
|
||||
<span className="text-muted-foreground">✕</span>
|
||||
</Button>
|
||||
</div>
|
||||
))
|
||||
) : (
|
||||
<p className="text-xs text-muted-foreground italic">
|
||||
{t('accounts.filters.noSpamHeaders')}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
className="h-8 text-sm max-w-[220px]"
|
||||
placeholder="X-Spam-Flag"
|
||||
value={newSpamHeader}
|
||||
onChange={(e) => setNewSpamHeader(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') {
|
||||
e.preventDefault();
|
||||
handleAddCustomSpamHeader();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<Button
|
||||
variant="outline"
|
||||
type="button"
|
||||
size="sm"
|
||||
className="h-8 text-xs"
|
||||
onClick={handleAddCustomSpamHeader}
|
||||
>
|
||||
<Plus className="h-3 w-3 mr-1" />
|
||||
{t('accounts.filters.addHeader')}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<p className="text-xs text-muted-foreground mb-2">{t('accounts.filters.suggestions')}</p>
|
||||
<div className="flex flex-wrap gap-1.5">
|
||||
{SUGGESTED_SPAM_HEADERS.filter((h) => !spamHeaders.includes(h)).map((header) => (
|
||||
<button
|
||||
key={header}
|
||||
type="button"
|
||||
className="inline-flex items-center rounded-full border bg-background px-2.5 py-0.5 text-xs font-mono text-muted-foreground hover:text-foreground hover:border-primary/50 transition-colors"
|
||||
onClick={() => addSpamHeader(header)}
|
||||
>
|
||||
+ {header}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
91
web/src/features/accounts/components/tab-general.tsx
Normal file
91
web/src/features/accounts/components/tab-general.tsx
Normal file
@@ -0,0 +1,91 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import { useFormContext } from "react-hook-form";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import {
|
||||
FormField,
|
||||
FormItem,
|
||||
FormLabel,
|
||||
FormMessage,
|
||||
FormControl,
|
||||
FormDescription,
|
||||
} from "@/components/ui/form";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import { AccountFormValues } from "./schema";
|
||||
|
||||
interface TabGeneralProps {
|
||||
isEdit?: boolean;
|
||||
}
|
||||
|
||||
export function TabGeneral({ isEdit }: TabGeneralProps) {
|
||||
const { t } = useTranslation();
|
||||
const { control } = useFormContext<AccountFormValues>();
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<FormField
|
||||
control={control}
|
||||
name="email"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.email')}<span className="text-red-500 align-super text-xs">*</span></FormLabel>
|
||||
<FormControl>
|
||||
<Input {...field} disabled={isEdit} placeholder={t('accounts.emailPlaceholder')} />
|
||||
</FormControl>
|
||||
{isEdit && (
|
||||
<FormDescription>{t('accounts.emailCannotBeModified')}</FormDescription>
|
||||
)}
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={control}
|
||||
name="account_name"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.name')}</FormLabel>
|
||||
<FormControl>
|
||||
<Input {...field} value={field.value ?? ''} placeholder={t('accounts.nameDescription')} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={control}
|
||||
name="enabled"
|
||||
render={({ field }) => (
|
||||
<FormItem className="flex flex-row items-start space-x-3 space-y-0 rounded-md border p-4">
|
||||
<FormControl>
|
||||
<Checkbox checked={field.value} onCheckedChange={field.onChange} />
|
||||
</FormControl>
|
||||
<div className="space-y-1 leading-none">
|
||||
<FormLabel>{t('accounts.enabled')}</FormLabel>
|
||||
<FormDescription>{t('accounts.enabledDescription')}</FormDescription>
|
||||
</div>
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
180
web/src/features/accounts/components/tab-server.tsx
Normal file
180
web/src/features/accounts/components/tab-server.tsx
Normal file
@@ -0,0 +1,180 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import { useFormContext } from "react-hook-form";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import {
|
||||
FormField,
|
||||
FormItem,
|
||||
FormLabel,
|
||||
FormMessage,
|
||||
FormControl,
|
||||
FormDescription,
|
||||
} from "@/components/ui/form";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import { PasswordInput } from "@/components/password-input";
|
||||
import { AccountFormValues } from "./schema";
|
||||
|
||||
interface TabServerProps {
|
||||
isEdit?: boolean;
|
||||
}
|
||||
|
||||
export function TabServer({ isEdit }: TabServerProps) {
|
||||
const { t } = useTranslation();
|
||||
const { control, watch } = useFormContext<AccountFormValues>();
|
||||
const authType = watch('imap.auth.auth_type');
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<FormField
|
||||
control={control}
|
||||
name="imap.host"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.imapHost')}<span className="text-red-500 align-super text-xs">*</span></FormLabel>
|
||||
<FormControl>
|
||||
<Input {...field} placeholder={t('accounts.imapHostPlaceholder')} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<FormField
|
||||
control={control}
|
||||
name="imap.port"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.imapPort')}<span className="text-red-500 align-super text-xs">*</span></FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
type="number"
|
||||
{...field}
|
||||
onChange={(e) => field.onChange(parseInt(e.target.value, 10))}
|
||||
placeholder={t('accounts.imapPortPlaceholder')}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={control}
|
||||
name="imap.encryption"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.imapEncryption')}<span className="text-red-500 align-super text-xs">*</span></FormLabel>
|
||||
<Select onValueChange={field.onChange} value={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="Ssl">SSL/TLS</SelectItem>
|
||||
<SelectItem value="StartTls">StartTLS</SelectItem>
|
||||
<SelectItem value="None">{t('accounts.none')}</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<FormField
|
||||
control={control}
|
||||
name="login_name"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.login_name')}</FormLabel>
|
||||
<FormControl>
|
||||
<Input {...field} value={field.value ?? ''} placeholder={t('accounts.namePlaceholder')} disabled={isEdit} />
|
||||
</FormControl>
|
||||
<FormDescription>{t('accounts.nameDescription')}</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={control}
|
||||
name="imap.auth.auth_type"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.imapAuthMethod')}<span className="text-red-500 align-super text-xs">*</span></FormLabel>
|
||||
<Select onValueChange={field.onChange} value={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
<SelectItem value="Password">{t('accounts.authPassword')}</SelectItem>
|
||||
<SelectItem value="OAuth2">OAuth2</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
{authType === 'Password' && (
|
||||
<FormField
|
||||
control={control}
|
||||
name="imap.auth.password"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.imapPassword')}{!isEdit && <span className="text-red-500 align-super text-xs">*</span>}</FormLabel>
|
||||
<FormControl>
|
||||
<PasswordInput placeholder={isEdit ? t('accounts.leaveEmptyToKeepPassword') : t('accounts.enterPassword')} {...field} />
|
||||
</FormControl>
|
||||
{isEdit && <FormDescription>{t('accounts.leaveEmptyToKeepPassword')}</FormDescription>}
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
|
||||
<FormField
|
||||
control={control}
|
||||
name="use_dangerous"
|
||||
render={({ field }) => (
|
||||
<FormItem className="flex flex-row items-start space-x-3 space-y-0 rounded-md border p-4">
|
||||
<FormControl>
|
||||
<Checkbox checked={field.value} onCheckedChange={field.onChange} />
|
||||
</FormControl>
|
||||
<div className="space-y-1 leading-none">
|
||||
<FormLabel>{t('accounts.useDangerous')}</FormLabel>
|
||||
</div>
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -17,7 +17,7 @@
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import { useState } from 'react'
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import {
|
||||
ColumnDef,
|
||||
ColumnFiltersState,
|
||||
@@ -64,7 +64,19 @@ export function AccountTable({ columns, data }: DataTableProps) {
|
||||
const [rowSelection, setRowSelection] = useState({})
|
||||
const [columnVisibility, setColumnVisibility] = useState<VisibilityState>({})
|
||||
const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([])
|
||||
const [sorting, setSorting] = useState<SortingState>([])
|
||||
const [sorting, setSorting] = useState<SortingState>(() => {
|
||||
const saved = localStorage.getItem('bichon_accounts_sorting');
|
||||
return saved ? JSON.parse(saved) : [];
|
||||
})
|
||||
|
||||
// Persist sorting state to localStorage
|
||||
const prevSortingRef = useRef(sorting);
|
||||
useEffect(() => {
|
||||
if (prevSortingRef.current !== sorting) {
|
||||
localStorage.setItem('bichon_accounts_sorting', JSON.stringify(sorting));
|
||||
prevSortingRef.current = sorting;
|
||||
}
|
||||
}, [sorting]);
|
||||
|
||||
const table = useReactTable({
|
||||
data,
|
||||
|
||||
@@ -21,10 +21,7 @@ import { AccountModel } from '@/api/account/api';
|
||||
import React from 'react'
|
||||
|
||||
export type AccountDialogType =
|
||||
| 'add'
|
||||
| 'add-imap'
|
||||
| 'add-nosync'
|
||||
| 'edit-imap'
|
||||
| 'edit-nosync'
|
||||
| 'delete'
|
||||
| 'detail'
|
||||
|
||||
@@ -19,16 +19,16 @@
|
||||
|
||||
import { useState } from 'react'
|
||||
import useDialogState from '@/hooks/use-dialog-state'
|
||||
import { useNavigate } from '@tanstack/react-router'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Main } from '@/components/layout/main'
|
||||
import { AccountActionDialog } from './components/action-dialog'
|
||||
import { useColumns } from './components/columns'
|
||||
import { AccountDeleteDialog } from './components/delete-dialog'
|
||||
import { AccountTable } from './components/table'
|
||||
import AccountProvider, {
|
||||
type AccountDialogType,
|
||||
} from './context'
|
||||
import { Plus } from 'lucide-react'
|
||||
import { Mail, Database } from 'lucide-react'
|
||||
import Logo from '@/assets/logo.svg'
|
||||
import { AccountDetailDrawer } from './components/account-detail'
|
||||
import { AccountModel, list_accounts } from '@/api/account/api'
|
||||
@@ -42,10 +42,10 @@ import { NoSyncAccountDialog } from './components/nosync-dialog'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { AccountAccessAssignmentDialog } from './components/access-assignment-dialog'
|
||||
import { useCurrentUser } from '@/hooks/use-current-user'
|
||||
import { AddAccountDialog } from './components/add-account-dialog'
|
||||
|
||||
export default function Accounts() {
|
||||
const { t } = useTranslation()
|
||||
const navigate = useNavigate()
|
||||
const columns = useColumns()
|
||||
// Dialog states
|
||||
const [currentRow, setCurrentRow] = useState<AccountModel | null>(null)
|
||||
@@ -77,15 +77,14 @@ export default function Accounts() {
|
||||
</p>
|
||||
</div>
|
||||
{require_any_permission(['system:root', 'account:create']) && <div className="flex gap-2">
|
||||
<div className="flex rounded-md shadow-sm">
|
||||
<Button
|
||||
onClick={() => setOpen("add")}
|
||||
className="border-r-0"
|
||||
>
|
||||
<Plus className="h-4 w-4" />
|
||||
{t('accounts.add')}
|
||||
</Button>
|
||||
</div>
|
||||
<Button onClick={() => navigate({ to: '/accounts/new' })}>
|
||||
<Mail className="mr-1.5 h-4 w-4" />
|
||||
{t('accounts.imapAccount')}
|
||||
</Button>
|
||||
<Button variant="outline" onClick={() => setOpen("add-nosync")}>
|
||||
<Database className="mr-1.5 h-4 w-4" />
|
||||
{t('accounts.noSyncAccount')}
|
||||
</Button>
|
||||
</div>}
|
||||
</div>
|
||||
|
||||
@@ -107,8 +106,13 @@ export default function Accounts() {
|
||||
{t('accounts.noAccountConfigurationsDesc')}
|
||||
</p>
|
||||
<div className="mt-4 flex flex-col items-center gap-3 sm:flex-row sm:flex-wrap sm:justify-center sm:gap-4">
|
||||
<Button variant="default" className="w-64" onClick={() => setOpen("add")}>
|
||||
{t('accounts.add')}
|
||||
<Button variant="default" className="w-64" onClick={() => navigate({ to: '/accounts/new' })}>
|
||||
<Mail className="mr-1.5 h-4 w-4" />
|
||||
{t('accounts.imapAccount')}
|
||||
</Button>
|
||||
<Button variant="outline" className="w-64" onClick={() => setOpen('add-nosync')}>
|
||||
<Database className="mr-1.5 h-4 w-4" />
|
||||
{t('accounts.noSyncAccount')}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
@@ -117,16 +121,6 @@ export default function Accounts() {
|
||||
</div>
|
||||
</div>
|
||||
</Main>
|
||||
<AddAccountDialog
|
||||
key='account-add'
|
||||
open={open === 'add'}
|
||||
onOpenChange={() => setOpen('add')} />
|
||||
|
||||
<AccountActionDialog
|
||||
key='imap-account-add'
|
||||
open={open === 'add-imap'}
|
||||
onOpenChange={() => setOpen('add-imap')}
|
||||
/>
|
||||
|
||||
<NoSyncAccountDialog
|
||||
key='nosync-account-add'
|
||||
@@ -136,17 +130,6 @@ export default function Accounts() {
|
||||
|
||||
{currentRow && (
|
||||
<>
|
||||
<AccountActionDialog
|
||||
key={`imap-account-edit-${currentRow.id}`}
|
||||
open={open === 'edit-imap'}
|
||||
onOpenChange={() => {
|
||||
setOpen('edit-imap')
|
||||
setTimeout(() => {
|
||||
setCurrentRow(null)
|
||||
}, 500)
|
||||
}}
|
||||
currentRow={currentRow}
|
||||
/>
|
||||
<NoSyncAccountDialog
|
||||
key={`nosync-account-edit-${currentRow.id}`}
|
||||
open={open === 'edit-nosync'}
|
||||
|
||||
399
web/src/features/attachment/attachment-preview.tsx
Normal file
399
web/src/features/attachment/attachment-preview.tsx
Normal file
@@ -0,0 +1,399 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import {
|
||||
Download, FileIcon, ZoomIn, ZoomOut, RotateCcw,
|
||||
ChevronLeft, ChevronRight, X,
|
||||
} from 'lucide-react';
|
||||
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Dialog, DialogContent } from '@/components/ui/dialog';
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';
|
||||
import { Separator } from '@/components/ui/separator';
|
||||
import { Skeleton } from '@/components/ui/skeleton';
|
||||
import { toast } from '@/hooks/use-toast';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
|
||||
import { preview_attachment, download_attachment } from '@/api/mailbox/envelope/api';
|
||||
import { getFileConfig } from './mail-message-view';
|
||||
|
||||
const PREVIEWABLE_IMAGE = /^image\/(png|jpeg|gif|webp|svg\+xml)$/;
|
||||
const PREVIEWABLE_TEXT = /^(text\/(plain|csv|html|xml|css|javascript|markdown)|application\/(json|xml|javascript|x-httpd-php|x-sh|x-perl|x-python|x-ruby))$/;
|
||||
|
||||
const EXT_TO_MIME: Record<string, string> = {
|
||||
'.pdf': 'application/pdf',
|
||||
'.png': 'image/png',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.gif': 'image/gif',
|
||||
'.webp': 'image/webp',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.txt': 'text/plain',
|
||||
'.csv': 'text/csv',
|
||||
'.html': 'text/html',
|
||||
'.htm': 'text/html',
|
||||
'.xml': 'application/xml',
|
||||
'.css': 'text/css',
|
||||
'.js': 'application/javascript',
|
||||
'.mjs': 'application/javascript',
|
||||
'.md': 'text/markdown',
|
||||
'.json': 'application/json',
|
||||
'.php': 'application/x-httpd-php',
|
||||
'.sh': 'application/x-sh',
|
||||
'.pl': 'application/x-perl',
|
||||
'.py': 'application/x-python',
|
||||
'.rb': 'application/x-ruby',
|
||||
};
|
||||
|
||||
function getExtensionMime(filename: string): string | null {
|
||||
const dot = filename.lastIndexOf('.');
|
||||
if (dot < 0) return null;
|
||||
const ext = filename.slice(dot).toLowerCase();
|
||||
return EXT_TO_MIME[ext] ?? null;
|
||||
}
|
||||
|
||||
function resolveContentType(contentType: string, fileName: string): string {
|
||||
if (contentType && contentType !== 'application/octet-stream') return contentType;
|
||||
return getExtensionMime(fileName) ?? contentType;
|
||||
}
|
||||
|
||||
export interface PreviewAttachment {
|
||||
content_hash: string;
|
||||
file_type: string;
|
||||
filename: string;
|
||||
}
|
||||
|
||||
interface AttachmentPreviewProps {
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
accountId: number;
|
||||
envelopeId: string;
|
||||
contentHash: string;
|
||||
contentType: string;
|
||||
fileName: string;
|
||||
/** Full attachment list for gallery navigation (optional). */
|
||||
attachments?: PreviewAttachment[];
|
||||
/** Index of the current attachment within `attachments`. */
|
||||
attachmentIndex?: number;
|
||||
}
|
||||
|
||||
function isImagePreview(contentType: string, fileName?: string) {
|
||||
return PREVIEWABLE_IMAGE.test(resolveContentType(contentType, fileName ?? ''));
|
||||
}
|
||||
|
||||
function isPdfPreview(contentType: string, fileName?: string) {
|
||||
return resolveContentType(contentType, fileName ?? '') === 'application/pdf';
|
||||
}
|
||||
|
||||
function isTextPreview(contentType: string, fileName?: string) {
|
||||
return PREVIEWABLE_TEXT.test(resolveContentType(contentType, fileName ?? ''));
|
||||
}
|
||||
|
||||
export default function AttachmentPreview({
|
||||
open,
|
||||
onOpenChange,
|
||||
accountId,
|
||||
envelopeId,
|
||||
contentHash,
|
||||
contentType,
|
||||
fileName,
|
||||
attachments,
|
||||
attachmentIndex,
|
||||
}: AttachmentPreviewProps) {
|
||||
const { t } = useTranslation();
|
||||
const [blobUrl, setBlobUrl] = useState<string | null>(null);
|
||||
const [textContent, setTextContent] = useState<string | null>(null);
|
||||
const [imageZoom, setImageZoom] = useState(1);
|
||||
|
||||
// ── Gallery state ──────────────────────────────────────────────
|
||||
// When attachments list is provided, compute image-only indices for navigation.
|
||||
const imageIndices = useMemo(() => {
|
||||
if (!attachments) return [];
|
||||
return attachments
|
||||
.map((a, i) => (isImagePreview(a.file_type, a.filename) ? i : -1))
|
||||
.filter((i) => i >= 0);
|
||||
}, [attachments]);
|
||||
|
||||
const [currentIndex, setCurrentIndex] = useState(attachmentIndex ?? 0);
|
||||
|
||||
// Reset to the clicked attachment every time the dialog opens.
|
||||
useEffect(() => {
|
||||
if (open) {
|
||||
setCurrentIndex(attachmentIndex ?? 0);
|
||||
}
|
||||
}, [open, attachmentIndex]);
|
||||
|
||||
// Resolve which attachment to display.
|
||||
const resolved = useMemo(() => {
|
||||
if (attachments && currentIndex < attachments.length) {
|
||||
const a = attachments[currentIndex];
|
||||
return {
|
||||
contentHash: a.content_hash,
|
||||
contentType: a.file_type,
|
||||
fileName: a.filename,
|
||||
};
|
||||
}
|
||||
return { contentHash, contentType, fileName };
|
||||
}, [attachments, currentIndex, contentHash, contentType, fileName]);
|
||||
|
||||
// Position within image-only list (for "3 / 12" counter).
|
||||
const imagePos = imageIndices.indexOf(currentIndex); // -1 if not an image
|
||||
const imageTotal = imageIndices.length;
|
||||
|
||||
const goPrev = useCallback(() => {
|
||||
if (imagePos > 0) setCurrentIndex(imageIndices[imagePos - 1]);
|
||||
}, [imagePos, imageIndices]);
|
||||
|
||||
const goNext = useCallback(() => {
|
||||
if (imagePos < imageTotal - 1) setCurrentIndex(imageIndices[imagePos + 1]);
|
||||
}, [imagePos, imageTotal, imageIndices]);
|
||||
|
||||
// Keyboard navigation
|
||||
useEffect(() => {
|
||||
if (!open) return;
|
||||
const handler = (e: KeyboardEvent) => {
|
||||
if (e.key === 'ArrowLeft') goPrev();
|
||||
else if (e.key === 'ArrowRight') goNext();
|
||||
};
|
||||
window.addEventListener('keydown', handler);
|
||||
return () => window.removeEventListener('keydown', handler);
|
||||
}, [open, goPrev, goNext]);
|
||||
|
||||
const effectiveType = resolveContentType(resolved.contentType, resolved.fileName);
|
||||
|
||||
// ── Fetch preview blob ─────────────────────────────────────────
|
||||
const previewMutation = useMutation({
|
||||
mutationFn: () => preview_attachment(accountId, envelopeId, resolved.contentHash),
|
||||
onSuccess: (blob) => {
|
||||
if (isTextPreview(resolved.contentType, resolved.fileName)) {
|
||||
blob.text().then(setTextContent);
|
||||
} else {
|
||||
const typedBlob = new Blob([blob], { type: effectiveType });
|
||||
setBlobUrl(URL.createObjectURL(typedBlob));
|
||||
}
|
||||
},
|
||||
onError: (error: any) => {
|
||||
toast({
|
||||
title: t('attachment_preview.failedToLoad'),
|
||||
description: error.message,
|
||||
variant: 'destructive',
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (open) {
|
||||
setBlobUrl(null);
|
||||
setTextContent(null);
|
||||
setImageZoom(1);
|
||||
previewMutation.mutate();
|
||||
}
|
||||
}, [open, resolved.contentHash]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (blobUrl) URL.revokeObjectURL(blobUrl);
|
||||
};
|
||||
}, [blobUrl]);
|
||||
|
||||
const handleDownload = () => {
|
||||
download_attachment(accountId, envelopeId, resolved.contentHash, resolved.fileName);
|
||||
};
|
||||
|
||||
const { icon } = useMemo(() => getFileConfig(resolved.contentType), [resolved.contentType]);
|
||||
|
||||
const isImage = isImagePreview(resolved.contentType, resolved.fileName);
|
||||
const isPdf = isPdfPreview(resolved.contentType, resolved.fileName);
|
||||
const isText = isTextPreview(resolved.contentType, resolved.fileName);
|
||||
const showArrows = imageTotal > 1 && isImage;
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent
|
||||
className="w-screen h-screen max-w-none rounded-none p-0 gap-0 border-0 bg-slate-700/10"
|
||||
hideClose
|
||||
hideFullscreen
|
||||
onInteractOutside={(e) => {
|
||||
if (isPdf) e.preventDefault();
|
||||
}}
|
||||
>
|
||||
{/* Toolbar — hidden for PDF (browser's native viewer has its own controls) */}
|
||||
{!isPdf && (
|
||||
<div className="absolute top-0 left-0 right-0 z-10 flex items-center justify-between px-4 py-2 bg-gradient-to-b from-black/70 to-transparent text-white">
|
||||
<div className="flex items-center gap-2 min-w-0">
|
||||
{icon}
|
||||
<span className="text-sm font-medium truncate max-w-[400px]">
|
||||
{resolved.fileName}
|
||||
</span>
|
||||
{imagePos >= 0 && imageTotal > 1 && (
|
||||
<span className="text-xs text-white/60 ml-1">
|
||||
{imagePos + 1} / {imageTotal}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center gap-1 pr-12">
|
||||
{isImage && blobUrl && (
|
||||
<>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8 text-white hover:bg-white/20"
|
||||
onClick={() => setImageZoom((z) => Math.min(z + 0.25, 3))}
|
||||
>
|
||||
<ZoomIn className="h-4 w-4" />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>{t('attachment_preview.zoomIn')}</TooltipContent>
|
||||
</Tooltip>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8 text-white hover:bg-white/20"
|
||||
onClick={() => setImageZoom((z) => Math.max(z - 0.25, 0.25))}
|
||||
>
|
||||
<ZoomOut className="h-4 w-4" />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>{t('attachment_preview.zoomOut')}</TooltipContent>
|
||||
</Tooltip>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8 text-white hover:bg-white/20"
|
||||
onClick={() => setImageZoom(1)}
|
||||
>
|
||||
<RotateCcw className="h-4 w-4" />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>{t('attachment_preview.resetZoom')}</TooltipContent>
|
||||
</Tooltip>
|
||||
<Separator orientation="vertical" className="h-5 mx-1 bg-white/20" />
|
||||
</>
|
||||
)}
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8 text-white hover:bg-white/20"
|
||||
onClick={handleDownload}
|
||||
>
|
||||
<Download className="h-4 w-4" />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>{t('attachment.download')}</TooltipContent>
|
||||
</Tooltip>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Close button — positioned below browser PDF toolbar */}
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className={isPdf
|
||||
? 'absolute top-12 right-4 z-50 h-10 w-10 rounded-full text-white bg-black/50 hover:bg-black/70'
|
||||
: 'absolute top-2 right-4 z-50 h-8 w-8 rounded-full text-white hover:bg-white/20'
|
||||
}
|
||||
onClick={() => onOpenChange(false)}
|
||||
>
|
||||
<X className={isPdf ? 'h-5 w-5' : 'h-4 w-4'} />
|
||||
</Button>
|
||||
|
||||
{/* Navigation arrows */}
|
||||
{showArrows && (
|
||||
<>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
disabled={imagePos <= 0}
|
||||
className="absolute left-2 top-1/2 -translate-y-1/2 z-10 h-10 w-10 rounded-full text-white hover:bg-white/20 disabled:opacity-30 disabled:hover:bg-transparent"
|
||||
onClick={goPrev}
|
||||
>
|
||||
<ChevronLeft className="h-6 w-6" />
|
||||
</Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
disabled={imagePos >= imageTotal - 1}
|
||||
className="absolute right-2 top-1/2 -translate-y-1/2 z-10 h-10 w-10 rounded-full text-white hover:bg-white/20 disabled:opacity-30 disabled:hover:bg-transparent"
|
||||
onClick={goNext}
|
||||
>
|
||||
<ChevronRight className="h-6 w-6" />
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Preview body */}
|
||||
<div className="w-full h-full flex items-center justify-center">
|
||||
{previewMutation.isPending ? (
|
||||
<div className="flex flex-col items-center gap-3">
|
||||
<Skeleton className="w-64 h-4 bg-white/10" />
|
||||
<Skeleton className="w-48 h-4 bg-white/10" />
|
||||
<Skeleton className="w-56 h-4 bg-white/10" />
|
||||
</div>
|
||||
) : isImage && blobUrl ? (
|
||||
<div className="w-full h-full overflow-auto flex items-center justify-center">
|
||||
<img
|
||||
src={blobUrl}
|
||||
alt={resolved.fileName}
|
||||
className="max-w-full max-h-full object-contain"
|
||||
style={{
|
||||
transform: `scale(${imageZoom})`,
|
||||
transformOrigin: 'center center',
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
) : isPdf && blobUrl ? (
|
||||
<iframe
|
||||
src={blobUrl}
|
||||
className="w-full h-full border-0"
|
||||
title={resolved.fileName}
|
||||
/>
|
||||
) : isText && textContent !== null ? (
|
||||
<pre className="w-full h-full overflow-auto whitespace-pre-wrap text-sm font-mono p-6 text-white/90">
|
||||
{textContent}
|
||||
</pre>
|
||||
) : !previewMutation.isPending ? (
|
||||
<div className="flex flex-col items-center gap-4 text-white/60">
|
||||
<FileIcon className="h-16 w-16 opacity-30" />
|
||||
<p className="text-sm">{t('attachment_preview.notAvailable')}</p>
|
||||
<p className="text-xs text-center max-w-md">
|
||||
{t('attachment_preview.notAvailableDesc', {
|
||||
type: resolved.contentType || 'unknown',
|
||||
})}
|
||||
</p>
|
||||
<Button variant="outline" size="sm" onClick={handleDownload} className="text-white border-white/20 hover:bg-white/10">
|
||||
<Download className="h-4 w-4 mr-2" />
|
||||
{t('attachment.download')}
|
||||
</Button>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user