mirror of
https://github.com/rustmailer/bichon.git
synced 2026-08-31 01:52:30 +00:00
Compare commits
14 Commits
fix/cli-mb
...
1.1.2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
178b25d27d | ||
|
|
d160ca75f5 | ||
|
|
04136a4ae2 | ||
|
|
1d6f5d9a22 | ||
|
|
105a6d9b15 | ||
|
|
df440c8441 | ||
|
|
4116a59b79 | ||
|
|
609eee1b84 | ||
|
|
79b9f07888 | ||
|
|
ba28369202 | ||
|
|
ff64b66f79 | ||
|
|
a4f8e674c3 | ||
|
|
dde6b990da | ||
|
|
6b1f843bd5 |
10
Cargo.lock
generated
10
Cargo.lock
generated
@@ -293,7 +293,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "bichon-admin"
|
||||
version = "1.0.2"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"bichon-core",
|
||||
"console",
|
||||
@@ -312,7 +312,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bichon-cli"
|
||||
version = "1.0.2"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bichon-core",
|
||||
@@ -338,7 +338,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bichon-core"
|
||||
version = "1.0.2"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"async-imap",
|
||||
"base64 0.22.1",
|
||||
@@ -396,7 +396,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bichon-server"
|
||||
version = "1.0.2"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"bichon-core",
|
||||
"bichon-smtp",
|
||||
@@ -421,7 +421,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bichon-smtp"
|
||||
version = "1.0.2"
|
||||
version = "1.1.2"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bichon-core",
|
||||
|
||||
@@ -11,7 +11,7 @@ members = [
|
||||
resolver = "2"
|
||||
|
||||
[workspace.package]
|
||||
version = "1.0.2"
|
||||
version = "1.1.2"
|
||||
edition = "2021"
|
||||
|
||||
[workspace.dependencies]
|
||||
|
||||
@@ -271,6 +271,10 @@ All settings accept both CLI flags (`--bichon-http-port`) and environment variab
|
||||
> [!TIP]
|
||||
> Place `BICHON_INDEX_DIR` on fast SSD storage for responsive search, and `BICHON_DATA_DIR` on high-capacity HDD for cost-effective blob storage.
|
||||
|
||||
|
||||
> [!IMPORTANT]
|
||||
> Bichon does NOT support writing data directly to a network file system (NFS, CIFS/SMB, etc.). All directories — `BICHON_ROOT_DIR`, `BICHON_DATA_DIR`, and `BICHON_INDEX_DIR` — must reside on a **local file system**; otherwise, data corruption may occur.
|
||||
|
||||
### Performance Tuning
|
||||
|
||||
| Variable | Default | Description |
|
||||
|
||||
@@ -83,16 +83,11 @@ impl DashboardStats {
|
||||
|
||||
stat.email_count = ENVELOPE_MANAGER.total_emails(&authorized_ids)?;
|
||||
stat.attachment_count = ATTACHMENT_MANAGER.total_attachments(&authorized_ids)?;
|
||||
if has_all_accounts {
|
||||
stat.storage_usage_bytes = get_total_size(&DATA_DIR_MANAGER.storage_dir)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
stat.storage_usage_bytes = get_total_size(&DATA_DIR_MANAGER.storage_dir)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
stat.index_usage_bytes = get_total_size(&&DATA_DIR_MANAGER.envelope_dir)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
} else {
|
||||
stat.storage_usage_bytes = 0;
|
||||
stat.index_usage_bytes = 0;
|
||||
}
|
||||
stat.index_usage_bytes = get_total_size(&&DATA_DIR_MANAGER.envelope_dir)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
stat.system_version = bichon_version!().to_string();
|
||||
|
||||
|
||||
@@ -26,6 +26,6 @@ pub async fn delete_messages_impl(request: HashMap<u64, Vec<String>>) -> BichonR
|
||||
.delete_envelopes_multi_account(request.clone())
|
||||
.await?;
|
||||
ATTACHMENT_MANAGER
|
||||
.delete_envelopes_multi_account(request)
|
||||
.delete_attachments_multi_account(request)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -259,6 +259,12 @@ impl NewIndexWriter {
|
||||
.parse(eml_bytes)
|
||||
.ok_or_else(|| raise_error!("failed to parse eml".into(), ErrorCode::InternalError))?;
|
||||
|
||||
if message.parts.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Malformed or completely empty EML (no parts found)".into(),
|
||||
ErrorCode::InternalError
|
||||
));
|
||||
}
|
||||
// ── text / preview ────────────────────────────────────────────────
|
||||
let text = message
|
||||
.body_text(0)
|
||||
@@ -436,7 +442,7 @@ impl NewIndexWriter {
|
||||
.commit()
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
}
|
||||
println!("tantivy commit elasped: {:#?}", start.elapsed());
|
||||
println!("tantivy commit elapsed: {:#?}", start.elapsed());
|
||||
tracing::info!(count = self.pending, "committed tantivy batch");
|
||||
self.pending = 0;
|
||||
Ok(())
|
||||
|
||||
@@ -260,7 +260,7 @@ impl IndexManager {
|
||||
IndexRecordOption::Basic,
|
||||
);
|
||||
let envelope_id_query = TermQuery::new(
|
||||
Term::from_field_text(SchemaTools::attachment_fields().f_id, aid),
|
||||
Term::from_field_text(SchemaTools::attachment_fields().f_envelope_id, aid),
|
||||
IndexRecordOption::Basic,
|
||||
);
|
||||
let boolean_query = BooleanQuery::new(vec![
|
||||
@@ -633,7 +633,7 @@ impl IndexManager {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete_envelopes_multi_account(
|
||||
pub async fn delete_attachments_multi_account(
|
||||
&self,
|
||||
deletes: HashMap<u64, Vec<String>>,
|
||||
) -> BichonResult<()> {
|
||||
|
||||
@@ -158,10 +158,10 @@ fn dedup_account(
|
||||
) -> BichonResult<u64> {
|
||||
let searcher = email_reader.searcher();
|
||||
let fields = SchemaTools::email_fields();
|
||||
eprintln!(
|
||||
"DEBUG dedup_account: entry account={account_id} f_id_field={:?} f_content_hash_field={:?}",
|
||||
fields.f_id, fields.f_content_hash
|
||||
);
|
||||
// eprintln!(
|
||||
// "DEBUG dedup_account: entry account={account_id} f_id_field={:?} f_content_hash_field={:?}",
|
||||
// fields.f_id, fields.f_content_hash
|
||||
// );
|
||||
let mut map: DedupMap = HashMap::new();
|
||||
|
||||
// ── Phase 1: build the dedup map via FAST column scans ──────────────────
|
||||
@@ -204,7 +204,11 @@ fn dedup_account(
|
||||
let ingest_at = ingest_col.values.get_val(doc_id);
|
||||
|
||||
// Read content_hash from the dictionary-encoded string column
|
||||
let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0);
|
||||
let hash_ord = hash_col
|
||||
.ords()
|
||||
.values_for_doc(doc_id as u32)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
let mut hash_buf = String::new();
|
||||
hash_col
|
||||
.ord_to_str(hash_ord, &mut hash_buf)
|
||||
@@ -212,16 +216,20 @@ fn dedup_account(
|
||||
let content_hash = hash_buf;
|
||||
|
||||
// Read f_id from the dictionary-encoded string column
|
||||
let id_ord = id_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0);
|
||||
let id_ord = id_col
|
||||
.ords()
|
||||
.values_for_doc(doc_id as u32)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
let mut id_buf = String::new();
|
||||
id_col
|
||||
.ord_to_str(id_ord, &mut id_buf)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
let email_id = id_buf;
|
||||
|
||||
eprintln!(
|
||||
"DEBUG dedup_account: account={account_id} doc_id={doc_id} mailbox={mailbox_id} hash={content_hash:?} id={email_id:?} ingest_at={ingest_at}"
|
||||
);
|
||||
// eprintln!(
|
||||
// "DEBUG dedup_account: account={account_id} doc_id={doc_id} mailbox={mailbox_id} hash={content_hash:?} id={email_id:?} ingest_at={ingest_at}"
|
||||
// );
|
||||
|
||||
map.entry((mailbox_id, content_hash))
|
||||
.or_default()
|
||||
@@ -247,7 +255,11 @@ fn dedup_account(
|
||||
// uidvalidity, which is required for correct incremental sync.
|
||||
entries.sort_by_key(|e| std::cmp::Reverse(e.ingest_at));
|
||||
|
||||
eprintln!("DEBUG Phase2: key={_key:?} kept={} deleting={}", entries[0].email_id, entries.len() - 1);
|
||||
eprintln!(
|
||||
"DEBUG Phase2: key={_key:?} kept={} deleting={}",
|
||||
entries[0].email_id,
|
||||
entries.len() - 1
|
||||
);
|
||||
// Keep entries[0], soft-delete everything else via term query on f_id
|
||||
for entry in &entries[1..] {
|
||||
eprintln!(
|
||||
@@ -315,13 +327,14 @@ mod tests {
|
||||
|
||||
/// Collect non-deleted f_id values from the email index.
|
||||
fn surviving_email_ids(reader: &IndexReader) -> HashSet<String> {
|
||||
reader
|
||||
.reload()
|
||||
.expect("reader reload failed");
|
||||
reader.reload().expect("reader reload failed");
|
||||
let searcher = reader.searcher();
|
||||
let mut ids = HashSet::new();
|
||||
let segments = searcher.segment_readers();
|
||||
eprintln!("DEBUG surviving_email_ids: segment_count={}", segments.len());
|
||||
eprintln!(
|
||||
"DEBUG surviving_email_ids: segment_count={}",
|
||||
segments.len()
|
||||
);
|
||||
for (seg_idx, seg) in segments.iter().enumerate() {
|
||||
let id_col = seg
|
||||
.fast_fields()
|
||||
@@ -332,7 +345,11 @@ mod tests {
|
||||
eprintln!("DEBUG surviving_email_ids: seg={seg_idx} max_doc={max_doc}");
|
||||
for doc_id in 0..max_doc {
|
||||
let is_del = seg.is_deleted(doc_id);
|
||||
let ord = id_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0);
|
||||
let ord = id_col
|
||||
.ords()
|
||||
.values_for_doc(doc_id as u32)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
let mut buf = String::new();
|
||||
id_col.ord_to_str(ord, &mut buf).unwrap();
|
||||
eprintln!("DEBUG surviving_email_ids: seg={seg_idx} doc_id={doc_id} is_deleted={is_del} ord={ord} buf={buf:?}");
|
||||
@@ -359,7 +376,11 @@ mod tests {
|
||||
if seg.is_deleted(doc_id) {
|
||||
continue;
|
||||
}
|
||||
let ord = id_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0);
|
||||
let ord = id_col
|
||||
.ords()
|
||||
.values_for_doc(doc_id as u32)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
let mut buf = String::new();
|
||||
id_col.ord_to_str(ord, &mut buf).unwrap();
|
||||
ids.insert(buf);
|
||||
@@ -454,15 +475,17 @@ mod tests {
|
||||
|
||||
let email_r = email_idx.reader().unwrap();
|
||||
let survivors = surviving_email_ids(&email_r);
|
||||
let expected: HashSet<String> =
|
||||
expected_emails.iter().map(|s| s.to_string()).collect();
|
||||
let expected: HashSet<String> = expected_emails.iter().map(|s| s.to_string()).collect();
|
||||
assert_eq!(survivors, expected, "[{case}] email survivors mismatch");
|
||||
|
||||
let attach_r = attach_idx.reader().unwrap();
|
||||
let att_survivors = surviving_attachment_ids(&attach_r);
|
||||
let att_expected: HashSet<String> =
|
||||
expected_attachments.iter().map(|s| s.to_string()).collect();
|
||||
assert_eq!(att_survivors, att_expected, "[{case}] attachment survivors mismatch");
|
||||
assert_eq!(
|
||||
att_survivors, att_expected,
|
||||
"[{case}] attachment survivors mismatch"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -494,7 +517,7 @@ mod tests {
|
||||
add_attachment(af, aw, &format!("att-{i}"), &id, 1, 1);
|
||||
}
|
||||
},
|
||||
&["dup-2"], // ingest_at=400, the latest
|
||||
&["dup-2"], // ingest_at=400, the latest
|
||||
&["att-2"],
|
||||
)
|
||||
.await;
|
||||
@@ -591,7 +614,7 @@ mod tests {
|
||||
/// This test is read-only — it does not modify the index.
|
||||
#[test]
|
||||
fn inspect_production_duplicates() {
|
||||
let index_path = r"E:\db\data\bichon-indices\mail_metadata";
|
||||
let index_path = r"E:\bichon-data\bichon-indices\mail_metadata";
|
||||
let report_path = std::path::PathBuf::from(r"E:\bichon\dedup_report.txt");
|
||||
|
||||
let mut report = String::new();
|
||||
@@ -622,26 +645,21 @@ mod tests {
|
||||
let searcher = reader.searcher();
|
||||
|
||||
let mut total_docs = 0u64;
|
||||
let mut groups: std::collections::HashMap<u64, std::collections::HashMap<(u64, String), u64>> =
|
||||
std::collections::HashMap::new();
|
||||
let mut groups: std::collections::HashMap<
|
||||
u64,
|
||||
std::collections::HashMap<(u64, String), u64>,
|
||||
> = std::collections::HashMap::new();
|
||||
|
||||
for segment_reader in searcher.segment_readers() {
|
||||
let account_col = segment_reader
|
||||
.fast_fields()
|
||||
.u64(F_ACCOUNT_ID)
|
||||
.unwrap();
|
||||
let mailbox_col = segment_reader
|
||||
.fast_fields()
|
||||
.u64(F_MAILBOX_ID)
|
||||
.unwrap();
|
||||
let hash_col = match segment_reader
|
||||
.fast_fields()
|
||||
.str(F_CONTENT_HASH)
|
||||
.unwrap()
|
||||
{
|
||||
let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap();
|
||||
let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap();
|
||||
let hash_col = match segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap() {
|
||||
Some(c) => c,
|
||||
None => {
|
||||
let _ = writeln!(report, "Segment has no FAST str column for content_hash, skipping");
|
||||
let _ = writeln!(
|
||||
report,
|
||||
"Segment has no FAST str column for content_hash, skipping"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
@@ -655,7 +673,11 @@ mod tests {
|
||||
let account_id = account_col.values.get_val(doc_id);
|
||||
let mailbox_id = mailbox_col.values.get_val(doc_id);
|
||||
|
||||
let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0);
|
||||
let hash_ord = hash_col
|
||||
.ords()
|
||||
.values_for_doc(doc_id as u32)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
let mut hash_buf = String::new();
|
||||
hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap();
|
||||
let content_hash = hash_buf;
|
||||
|
||||
@@ -26,10 +26,10 @@ use std::sync::LazyLock;
|
||||
|
||||
use crate::error::code::ErrorCode;
|
||||
use crate::error::BichonResult;
|
||||
use crate::settings::cli::SETTINGS;
|
||||
use crate::raise_error;
|
||||
use crate::settings::cli::SETTINGS;
|
||||
|
||||
static ENCRYPT_PASSWORD: LazyLock<String> = LazyLock::new(|| {
|
||||
pub static ENCRYPT_PASSWORD: LazyLock<String> = LazyLock::new(|| {
|
||||
if let Some(file_path) = &SETTINGS.bichon_encrypt_password_file {
|
||||
return fs::read_to_string(file_path)
|
||||
.expect("failed to read the file with the encrypt password")
|
||||
@@ -102,7 +102,10 @@ pub fn internal_encrypt_string(
|
||||
Ok(general_purpose::URL_SAFE.encode(&result))
|
||||
}
|
||||
|
||||
pub fn internal_decrypt_string(password: &str, data: &str) -> Result<String, ring::error::Unspecified> {
|
||||
pub fn internal_decrypt_string(
|
||||
password: &str,
|
||||
data: &str,
|
||||
) -> Result<String, ring::error::Unspecified> {
|
||||
let data = general_purpose::URL_SAFE
|
||||
.decode(data)
|
||||
.map_err(|_| ring::error::Unspecified)?;
|
||||
@@ -146,8 +149,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_wrong_password_fails() {
|
||||
let encrypted =
|
||||
internal_encrypt_string("correct_password", "secret").unwrap();
|
||||
let encrypted = internal_encrypt_string("correct_password", "secret").unwrap();
|
||||
assert!(internal_decrypt_string("wrong_password", &encrypted).is_err());
|
||||
}
|
||||
|
||||
|
||||
@@ -73,8 +73,9 @@ async fn main() -> BichonResult<()> {
|
||||
Ok(false) => {
|
||||
error!("Incompatible data format detected.");
|
||||
error!("Your data was created by an older version of Bichon and must be migrated before use.");
|
||||
error!("Please stop the Bichon v0.3.7 service before migration.");
|
||||
error!("Please run: bichon-admin");
|
||||
error!("Documentation: https://github.com/rustmailer/bichon/wiki/migration");
|
||||
error!("Documentation: https://github.com/rustmailer/bichon/wiki/Bichon-Data-Migration:-v0.3.7-%E2%86%92-v1.0");
|
||||
return Err(raise_error!(
|
||||
"Legacy data layout detected".into(),
|
||||
ErrorCode::InternalError
|
||||
@@ -202,7 +203,10 @@ mod api_tests {
|
||||
.map(|v| v.object().get("name").string())
|
||||
.collect();
|
||||
|
||||
assert!(tag_names.contains(&"AccessToken"), "missing AccessToken tag");
|
||||
assert!(
|
||||
tag_names.contains(&"AccessToken"),
|
||||
"missing AccessToken tag"
|
||||
);
|
||||
assert!(tag_names.contains(&"Attachment"), "missing Attachment tag");
|
||||
assert!(tag_names.contains(&"AutoConfig"), "missing AutoConfig tag");
|
||||
assert!(tag_names.contains(&"Account"), "missing Account tag");
|
||||
|
||||
Reference in New Issue
Block a user