bump to 2.0.0

- Add SyncFull trigger and configurable IMAP socket read timeout
  - Replace streaming UID FETCH with UID SEARCH ALL + batched fetch so per-message progress stays responsive and throttling servers can retry with reconnection
  - Finalize stale Running sessions on startup so interrupted syncs no longer show a phantom "syncing" state
  - Show a live syncing pill on the account row; add elapsed time, current folder, and slow-server warning styling in the dialog

.github/ISSUE_TEMPLATE/bug.md

@@ -8,6 +8,12 @@ assignees: ""
 
 > **Please write and communicate in English.**
 
+
+---
+**Help us build a more stable Bichon!** 🛠️  
+While we look into this bug, consider sharing your usage patterns in our [2026 Roadmap Survey](https://docs.google.com/forms/d/e/1FAIpQLScOlwsiUMfyQPBCLW2MLkygdRmAutEgvXDYPzzvEGPz0HFPXQ/viewform) to help us prioritize stability and features.
+---
+
 ### Version
 Which version are you using?
 

.github/ISSUE_TEMPLATE/feature.md

@@ -8,6 +8,13 @@ assignees: ""
 
 > **Please write and communicate in English.**
 
+---
+### 📢 Shape the Future of Bichon
+**Want your requested feature to be prioritized?** Help us shape the **2026 Roadmap** by filling out our 1-minute survey: 
+👉 **[Bichon User Survey](https://docs.google.com/forms/d/e/1FAIpQLScOlwsiUMfyQPBCLW2MLkygdRmAutEgvXDYPzzvEGPz0HFPXQ/viewform)**
+---
+
+
 ### Description
 What feature would you like to see?
 

.github/workflows/release.yml

@@ -4,8 +4,11 @@ on:
   push:
     tags:
       - '[0-9]+.[0-9]+.[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+-*'
 env:
-  BINARY_NAME: bichon
+  BINARY_NAME: bichon-server
+  BINARY_CLI: bichon-cli
+  BINARY_ADMIN: bichon-admin
 
 permissions:
   contents: write
@@ -80,17 +83,19 @@ jobs:
         if: matrix.target == 'aarch64-unknown-linux-gnu'
         run: |
           cargo install cross --force
-          cross build --release --features vendored-openssl --target=${{ matrix.target }}
+          cross build --release --target=${{ matrix.target }}
 
       - name: Build Rust backend
         if: matrix.target != 'aarch64-unknown-linux-gnu'
         run: |
-          cargo build --release --features vendored-openssl --target=${{ matrix.target }}  
+          cargo build --release --target=${{ matrix.target }}  
 
       - name: Strip binary (Linux and macOS)
         if: matrix.os != 'windows-latest' && matrix.target != 'aarch64-unknown-linux-gnu'
         run: |
           strip target/${{ matrix.target }}/release/${{ env.BINARY_NAME }}
+          strip target/${{ matrix.target }}/release/${{ env.BINARY_CLI }}
+          strip target/${{ matrix.target }}/release/${{ env.BINARY_ADMIN }}
 
       - name: Pack artifact (Linux/macOS)
         if: matrix.os != 'windows-latest'
@@ -99,7 +104,9 @@ jobs:
           mkdir -p release
           BINARY="target/${{ matrix.target }}/release/${{ env.BINARY_NAME }}"
           cp README.md LICENSE release/
-          cp "$BINARY" release/
+          cp target/${{ matrix.target }}/release/${{ env.BINARY_NAME }} release/
+          cp target/${{ matrix.target }}/release/${{ env.BINARY_CLI }} release/
+          cp target/${{ matrix.target }}/release/${{ env.BINARY_ADMIN }} release/
           tar -czvf "${{ env.BINARY_NAME }}-${{ github.ref_name }}-${{ matrix.target }}.tar.gz" -C release .
           mv "${{ env.BINARY_NAME }}-${{ github.ref_name }}-${{ matrix.target }}.tar.gz" release/
 
@@ -115,10 +122,14 @@ jobs:
         shell: pwsh
         run: |
           mkdir -p release
-          $BINARY = "target/${{ matrix.target }}/release/${{ env.BINARY_NAME }}.exe"
-          Copy-Item -Path $BINARY -Destination release/
+
+          Copy-Item "target/${{ matrix.target }}/release/${{ env.BINARY_NAME }}.exe" release/
+          Copy-Item "target/${{ matrix.target }}/release/${{ env.BINARY_CLI }}.exe" release/
+          Copy-Item "target/${{ matrix.target }}/release/${{ env.BINARY_ADMIN }}.exe" release/
+
           Copy-Item -Path README.md -Destination release/
           Copy-Item -Path LICENSE -Destination release/
+          
           Compress-Archive -Path release\* -DestinationPath "release/${{ env.BINARY_NAME }}-${{ github.ref_name }}-${{ matrix.target }}.zip" -Force
 
       - name: Upload build artifact

.gitignore

@@ -1,3 +1,7 @@
 /target
 .vscode
 .idea
+config.toml
+node_modules
+dedup_report.txt
+crates/*/target

CLA.md

@@ -0,0 +1,56 @@
+# Contributor License Agreement
+
+Thank you for contributing to Bichon ("the Project"), maintained by rustmailer ("the Maintainer").
+
+This Agreement clarifies the intellectual property rights granted by You to the Maintainer when You submit a Contribution. By signing this Agreement, You accept and agree to the following terms.
+
+## 1. Definitions
+
+- **"You"** means the individual or legal entity (corporation, organization, etc.) on whose behalf the Contribution is submitted.
+- **"Contribution"** means any original work of authorship, including modifications or additions to existing work, that You intentionally submit to the Project (via Pull Request, patch, email, or any other means).
+
+## 2. Copyright License
+
+You grant the Maintainer a **perpetual, worldwide, non-exclusive, royalty-free, irrevocable** copyright license to:
+
+- Reproduce, modify, adapt, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Your Contribution and any derivative works thereof.
+- **Re-license** Your Contribution under different terms, including but not limited to proprietary or commercial licenses, in any medium now known or later developed.
+
+This license survives termination of this Agreement and continues even if You stop contributing.
+
+## 3. Patent License
+
+You grant the Maintainer a **perpetual, worldwide, non-exclusive, royalty-free, irrevocable** patent license to make, have made, use, sell, offer for sale, import, and otherwise transfer Your Contribution, where such license applies only to patent claims licensable by You that are necessarily infringed by Your Contribution alone or in combination with the Project.
+
+## 4. Moral Rights
+
+To the fullest extent permitted by applicable law, You waive, and agree not to assert, any moral rights (including rights of attribution, integrity, or withdrawal) in Your Contribution against the Maintainer or the Maintainer's licensees. If under Your jurisdiction moral rights cannot be waived, You agree not to enforce them against the Maintainer.
+
+## 5. Employer / Entity Disclaimer
+
+If You are employed by or acting on behalf of an entity, You represent that:
+
+- You have received permission from Your employer or entity to submit the Contribution under the terms of this Agreement.
+- Your employer or entity has waived any rights to the Contribution.
+
+If Your employer or entity has not provided such permission, You must not submit a Contribution.
+
+## 6. Representations
+
+You represent and warrant that:
+
+- You are legally entitled to make the Contribution and grant the licenses under this Agreement.
+- Your Contribution is Your original work, and You have the necessary rights from any third parties whose material is included.
+- You are not aware of any claims, pending litigation, or other issues that would affect the rights granted under this Agreement.
+
+## 7. No Obligation
+
+The Maintainer is under no obligation to accept or use Your Contribution. You are under no obligation to make further Contributions.
+
+## 8. Governing Law
+
+This Agreement is governed by the laws of the State of Delaware, United States of America, without regard to its conflict of law principles. Any dispute arising from this Agreement shall be resolved exclusively in the courts of Delaware.
+
+## 9. Signing
+
+You indicate Your acceptance of this Agreement by signing the [cla-assistant](https://github.com/cla-assistant/cla-assistant) prompt when submitting a Pull Request to the Project. Electronic acceptance constitutes a binding signature under applicable law.

Cargo.toml

@@ -1,49 +1,36 @@
-[package]
-name = "bichon"
-version = "0.2.2"
+[workspace]
+
+members = [
+    "crates/memdb",
+    "crates/core",
+    "crates/blob",
+    "crates/server",
+    "crates/cli",
+    "crates/admin",
+    "crates/smtp",
+]
+
+resolver = "2"
+
+[workspace.package]
+version = "2.0.0"
 edition = "2021"
 
-[[bin]]
-name = "bichon"
-path = "src/main.rs"
-
-
-[features]
-default = []
-vendored-openssl = ["openssl-sys"]
-
-[profile.release]
-strip = true
-lto = true
-opt-level = 3
-codegen-units = 1
-
-[dependencies]
-chrono = "0.4.42"
-clap = { version = "4.5.53", features = ["derive", "env"] }
-mimalloc = "0.1.48"
-native_db = "0.8.2"
-itertools = "0.14.0"
-native_model = "0.4.20"
-poem = { version = "3.1.12", features = ["embed", "compression", "rustls"] }
-poem-derive = "3.1.12"
-poem-openapi = { version = "5.1.16", features = [
-    "openapi-explorer",
-    "rapidoc",
-    "scalar",
-    "redoc",
-    "swagger-ui",
-    "email",
-] }
+[workspace.dependencies]
+chrono = "0.4.45"
+clap = { version = "4.6", features = ["derive", "env"] }
+bichon-memdb = { path = "crates/memdb" }
+bichon-blob = { path = "crates/blob" }
+itertools = "0.15.0"
 ring = { version = "0.17.14", features = ["std"] }
-serde = { version = "1.0.228", features = ["derive"] }
-serde_json = "1.0.148"
-tokio = { version = "1.48.0", features = ["full"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+tokio = { version = "1.53", features = ["full"] }
 tracing = "0.1.44"
 tracing-appender = "0.2.3"
-tracing-subscriber = { version = "0.3.22", features = ["env-filter", "json"] }
-base64 = "0.22.1"
-snafu = "0.8.9"
+tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] }
+base64 = "0.23"
+snafu = "0.9"
 reqwest = { version = "0.12.24", default-features = false, features = [
     "json",
     "stream",
@@ -52,63 +39,61 @@ reqwest = { version = "0.12.24", default-features = false, features = [
     "blocking",
     "socks",
 ] }
-tokio-socks = "0.5.2"
-http = "1.4.0"
-regex = "1.12.2"
+tokio-socks = "0.5.3"
+http = "1.5"
+regex = "1.13"
 email_address = "0.2.9"
-futures = "0.3.31"
+futures = "0.3"
 utf7-imap = "0.3.2"
-imap-proto = "0.16.6"
-mail-parser = { version = '0.11.1', features = ["serde"] }
+mail-parser = { version = '0.11', features = ["serde"] }
 # mail-send = "0.5.2"
 tokio-rustls = { version = "0.26.4", default-features = false, features = [
     "ring",
     "tls12",
 ] }
-timeago = "0.5.0"
-ahash = "0.8.12"
+timeago = "0.6.1"
 oauth2 = { version = "5.0.0", features = ["reqwest-blocking"] }
-url = { version = "2.5.7", features = ["serde"] }
-sysinfo = "0.37.2"
+url = { version = "2.5.8", features = ["serde"] }
+sysinfo = "0.39"
 num_cpus = "1.17.0"
-cacache = { version = "13.1.0", default-features = false, features = [
-    "tokio-runtime",
-    "mmap",
-] }
-rand = "0.9.2"
+rand = "0.10.2"
 encoding_rs = "0.8.35"
-async-imap = { version = "0.11.1", default-features = false, features = [
-    "runtime-tokio",
-    "compress",
-] }
-webpki-roots = "1.0.4"
-rustls = { version = "0.23.35", default-features = false, features = ["ring"] }
-rustls-pki-types = "1.13.2"
+webpki-roots = "1.0"
+rustls = { version = "0.23", default-features = false, features = ["ring"] }
+rustls-pki-types = "1.15"
 tokio-io-timeout = "1.2.1"
-bb8 = "0.9.1"
-semver = "1.0.27"
+semver = "1.0.28"
 governor = "0.10.4"
-lru = "0.16.2"
+lru = "0.18.1"
 mime_guess = "2.0.5"
 hex = "0.4.3"
-time = { version = "0.3.44", features = [
+time = { version = "0.3", features = [
     "formatting",
     "parsing",
     "local-offset",
 ] }
-rust-embed = "8.9.0"
+rust-embed = "8.12"
 murmur3 = "0.5.2"
-autoconfig = "0.4.0"
 urlencoding = "2.1.3"
-dashmap = "6.1.0"
-# Statically links OpenSSL by compiling from source, avoiding system library dependencies
-openssl-sys = { version = "0.9.111", optional = true, features = ["vendored"] }
+dashmap = "6.2.1"
 gethostname = "1.1.0"
-tantivy = { version = "0.25.0", features = ["quickwit", "zstd-compression"] }
-itoa = "1.0.17"
-html2text = "0.16.5"
-bytes = "1.11.0"
-[dev-dependencies]
-#bincode = "1.3.3"
-#secret-lib = "1.0.0"
-tempfile = "3.24.0"
+itoa = "1.0.18"
+html2text = "0.17.1"
+bytes = "1.12"
+dialoguer = "0.12.0"
+console = "0.16.4"
+mail-send = "0.6.1"
+rcgen = "0.14.8"
+rustls-pemfile = "2.2.0"
+blake3 = "1.8.5"
+uuid = { version = "1.24", features = ["v4", "serde"] }
+fjall = { version = "3.1", features = ["lz4", "metrics", "bytes_1"] }
+tracing-log = "0.2.0"
+tokio-util = "0.7"
+indicatif = "0.18.6"
+
+[profile.release]
+strip = true
+lto = true
+opt-level = 3
+codegen-units = 1

build.rs

@@ -1,14 +0,0 @@
-use std::{io::Result, process::Command};
-
-fn main() -> Result<()> {
-    let output = Command::new("git")
-        .args(&["rev-parse", "--short", "HEAD"])
-        .output()
-        .expect("Failed to get git commit hash");
-    let git_hash = String::from_utf8(output.stdout)
-        .expect("Invalid UTF-8")
-        .trim()
-        .to_string();
-    println!("cargo:rustc-env=GIT_HASH={}", git_hash);
-    Ok(())
-}

config.toml

@@ -0,0 +1,2 @@
+base_url = "http://localhost:15630"
+api_token = "eErI7WN3PtKeLwWAbIfSXCP6"

crates/admin/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "bichon-admin"
+version.workspace = true
+edition.workspace = true
+
+
+[dependencies]
+bichon-core = { path = "../core" }
+tokio.workspace = true
+dialoguer.workspace = true
+console.workspace = true
+indicatif.workspace = true
+native_db = "0.8.2"
+native_model = "0.4.20"
+serde.workspace = true
+serde_json.workspace = true
+
+itertools.workspace = true
+snafu.workspace = true
+bichon-memdb.workspace = true
+fjall.workspace = true
+hex.workspace = true
+bichon-blob.workspace = true
+mail-parser.workspace = true
+bytes.workspace = true
+uuid.workspace = true
+tantivy = { version = "0.26.1", features = ["zstd-compression", "quickwit"] }
+chrono.workspace = true
+tracing.workspace = true
+
+[dev-dependencies]
+tempfile = "3"

crates/admin/src/legacy/fields.rs

@@ -66,4 +66,4 @@ pub struct EmlFields {
     pub f_account_id: Field,
     pub f_mailbox_id: Field,
     pub f_eml: Field,
-}
+}

crates/admin/src/legacy/mod.rs

@@ -0,0 +1,2 @@
+pub mod fields;
+pub mod schema;

crates/admin/src/legacy/schema.rs

@@ -0,0 +1,114 @@
+use tantivy::schema::{FacetOptions, Field, Schema, FAST, INDEXED, STORED, STRING, TEXT};
+
+use crate::legacy::fields::{EmlFields, EnvelopeFields, *};
+
+pub struct SchemaTools;
+
+impl SchemaTools {
+    pub fn envelope_schema() -> Schema {
+        EnvelopeSchema::build().0
+    }
+    pub fn eml_schema() -> Schema {
+        EmlSchema::build().0
+    }
+
+    pub fn envelope_fields() -> EnvelopeFields {
+        EnvelopeSchema::fields()
+    }
+    pub fn eml_fields() -> EmlFields {
+        EmlSchema::fields()
+    }
+
+    pub fn envelope_default_fields() -> Vec<Field> {
+        let f = Self::envelope_fields();
+        vec![f.f_subject, f.f_text, f.f_attachments]
+    }
+}
+
+// ─── Schema builders ──────────────────────────────────────────────────────────
+
+struct EnvelopeSchema;
+
+impl EnvelopeSchema {
+    fn build() -> (Schema, EnvelopeFields) {
+        let mut b = Schema::builder();
+
+        let f_id = b.add_u64_field(F_ID, INDEXED | STORED | FAST);
+        let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
+        let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
+        let f_uid = b.add_u64_field(F_UID, INDEXED | STORED | FAST);
+        let f_thread_id = b.add_u64_field(F_THREAD_ID, INDEXED | STORED | FAST);
+
+        let f_subject = b.add_text_field(F_SUBJECT, TEXT | STORED);
+        let f_text = b.add_text_field(F_TEXT, TEXT | STORED);
+        let f_attachments = b.add_text_field(F_ATTACHMENTS, TEXT | STORED);
+
+        let f_from = b.add_text_field(F_FROM, STRING | STORED | FAST);
+        let f_to = b.add_text_field(F_TO, STRING | STORED);
+        let f_cc = b.add_text_field(F_CC, STRING | STORED);
+        let f_bcc = b.add_text_field(F_BCC, STRING | STORED);
+
+        let f_message_id = b.add_text_field(F_MESSAGE_ID, STRING | STORED);
+
+        let f_date = b.add_i64_field(F_DATE, STORED | FAST);
+        let f_internal_date = b.add_i64_field(F_INTERNAL_DATE, STORED | FAST);
+
+        let f_size = b.add_u64_field(F_SIZE, STORED | FAST);
+        let f_has_attachment = b.add_bool_field(F_HAS_ATTACHMENT, INDEXED | STORED | FAST);
+
+        let f_tags = b.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
+
+        let fields = EnvelopeFields {
+            f_id,
+            f_account_id,
+            f_mailbox_id,
+            f_uid,
+            f_thread_id,
+            f_subject,
+            f_text,
+            f_attachments,
+            f_from,
+            f_to,
+            f_cc,
+            f_bcc,
+            f_message_id,
+            f_date,
+            f_internal_date,
+            f_size,
+            f_has_attachment,
+            f_tags,
+        };
+
+        (b.build(), fields)
+    }
+
+    fn fields() -> EnvelopeFields {
+        Self::build().1
+    }
+}
+
+struct EmlSchema;
+
+impl EmlSchema {
+    fn build() -> (Schema, EmlFields) {
+        let mut b = Schema::builder();
+
+        let f_id = b.add_u64_field(F_ID, INDEXED | FAST);
+        let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
+        let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
+        let f_eml = b.add_bytes_field(F_EML, STORED);
+
+        let fields = EmlFields {
+            f_id,
+            f_account_id,
+            f_mailbox_id,
+            f_eml,
+        };
+
+        (b.build(), fields)
+    }
+
+    fn fields() -> EmlFields {
+        Self::build().1
+    }
+}

crates/admin/src/main.rs

@@ -0,0 +1,66 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use console::style;
+use dialoguer::{theme::ColorfulTheme, Select};
+
+use crate::{migrate_v037::handle_migration_v037, migrate_v1::handle_migrate_v1, reset::handle_reset_password};
+
+pub mod legacy;
+pub mod meta;
+pub mod migrate_store_v2;
+pub mod migrate_v037;
+pub mod migrate_v1;
+pub mod reset;
+
+
+fn main() {
+    run_interactive();
+}
+
+#[tokio::main]
+async fn run_interactive() {
+    let theme = ColorfulTheme::default();
+    println!(
+        "\n{}\n",
+        style("BICHON ADMINISTRATIVE TOOL").bold().bright().cyan()
+    );
+
+    let main_options = vec![
+        "Reset Admin Password",
+        "Migrate Legacy v0.3.7 Storage to v2.x (bichon-blob)",
+        "Migrate v1.x Storage to v2.x (Fjall → bichon-blob)",
+        "Exit",
+    ];
+
+    let selection = Select::with_theme(&theme)
+        .with_prompt("Select an operation")
+        .default(0)
+        .items(&main_options)
+        .interact()
+        .unwrap();
+
+    match selection {
+        0 => handle_reset_password(&theme),
+        1 => handle_migration_v037(&theme),
+        2 => handle_migrate_v1(&theme),
+        _ => {
+            println!("{}", style("Exiting...").dim());
+        }
+    }
+}

crates/admin/src/meta.rs

@@ -0,0 +1,916 @@
+use std::{
+    collections::{BTreeMap, BTreeSet},
+    path::PathBuf,
+    sync::{Arc, LazyLock},
+};
+
+use bichon_core::{
+    account::{
+        entity::ImapConfig,
+        migration::{AccountModel, AccountType},
+        since::{DateSince, RelativeDate},
+    },
+    autoconfig::entity::MailServerConfig,
+    cache::imap::mailbox::Attribute,
+    database::batch_insert_impl,
+    error::{code::ErrorCode, BichonError, BichonResult},
+    raise_error,
+    token::TokenType,
+    users::{acl::AccessControl, role::RoleType},
+};
+use bichon_memdb::{Durability, MemDb};
+use console::style;
+use itertools::Itertools;
+use native_db::*;
+use native_model::{native_model, Model};
+use serde::{Deserialize, Serialize};
+
+pub const DEFAULT_ADMIN_USER_ID: u64 = 100000000000000;
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)]
+#[native_model(id = 3, version = 1)]
+#[native_db]
+pub struct CachedMailSettings {
+    #[primary_key]
+    pub domain: String,
+    pub config: MailServerConfig,
+    pub created_at: i64,
+}
+
+impl From<CachedMailSettings> for bichon_core::autoconfig::CachedMailSettings {
+    fn from(value: CachedMailSettings) -> Self {
+        Self {
+            domain: value.domain,
+            config: value.config,
+            created_at: value.created_at,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[native_model(id = 4, version = 1)]
+#[native_db(primary_key(pk -> String))]
+pub struct AccountV1 {
+    #[secondary_key(unique)]
+    pub id: u64,
+    pub imap: Option<ImapConfig>,
+    pub enabled: bool,
+    pub email: String,
+    pub name: Option<String>,
+    pub capabilities: Option<Vec<String>>,
+    pub date_since: Option<DateSince>,
+    pub folder_limit: Option<u32>,
+    pub sync_folders: Option<Vec<String>>,
+    pub account_type: AccountType,
+    pub sync_interval_min: Option<i64>,
+    pub known_folders: Option<BTreeSet<String>>,
+    pub created_at: i64,
+    pub updated_at: i64,
+    pub use_proxy: Option<u64>,
+}
+impl AccountV1 {
+    fn pk(&self) -> String {
+        format!("{}_{}", self.created_at, self.id)
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[native_model(id = 4, version = 2, from = AccountV1)]
+#[native_db(primary_key(pk -> String))]
+pub struct AccountV2 {
+    #[secondary_key(unique)]
+    pub id: u64,
+    pub imap: Option<ImapConfig>,
+    pub enabled: bool,
+    pub email: String,
+    pub name: Option<String>,
+    pub capabilities: Option<Vec<String>>,
+    pub date_since: Option<DateSince>,
+    pub folder_limit: Option<u32>,
+    pub sync_folders: Option<Vec<String>>,
+    pub account_type: AccountType,
+    pub sync_interval_min: Option<i64>,
+    pub known_folders: Option<BTreeSet<String>>,
+    pub created_at: i64,
+    pub updated_at: i64,
+    pub use_proxy: Option<u64>,
+    pub use_dangerous: bool,
+    pub pgp_key: Option<String>,
+}
+
+impl AccountV2 {
+    fn pk(&self) -> String {
+        format!("{}_{}", self.created_at, self.id)
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[native_model(id = 4, version = 3, from = AccountV2)]
+#[native_db(primary_key(pk -> String))]
+pub struct AccountV3 {
+    #[secondary_key(unique)]
+    pub id: u64,
+    pub imap: Option<ImapConfig>,
+    pub enabled: bool,
+    pub email: String,
+    pub name: Option<String>,
+    pub capabilities: Option<Vec<String>>,
+    pub date_since: Option<DateSince>,
+    pub date_before: Option<RelativeDate>,
+    pub folder_limit: Option<u32>,
+    pub sync_folders: Option<Vec<String>>,
+    pub account_type: AccountType,
+    pub sync_interval_min: Option<i64>,
+    pub sync_batch_size: Option<u32>,
+    pub known_folders: Option<BTreeSet<String>>,
+    pub created_at: i64,
+    pub updated_at: i64,
+    pub created_by: u64, //user id
+    pub use_proxy: Option<u64>,
+    pub use_dangerous: bool,
+    pub pgp_key: Option<String>,
+}
+
+impl AccountV3 {
+    fn pk(&self) -> String {
+        format!("{}_{}", self.created_at, self.id)
+    }
+}
+
+impl From<AccountV1> for AccountV2 {
+    fn from(value: AccountV1) -> Self {
+        Self {
+            id: value.id,
+            imap: value.imap,
+            enabled: value.enabled,
+            email: value.email,
+            name: value.name,
+            capabilities: value.capabilities,
+            date_since: value.date_since,
+            folder_limit: value.folder_limit,
+            sync_folders: value.sync_folders,
+            account_type: value.account_type,
+            sync_interval_min: value.sync_interval_min,
+            known_folders: value.known_folders,
+            created_at: value.created_at,
+            updated_at: value.updated_at,
+            use_proxy: value.use_proxy,
+            use_dangerous: false,
+            pgp_key: None,
+        }
+    }
+}
+
+impl From<AccountV2> for AccountV1 {
+    fn from(value: AccountV2) -> Self {
+        Self {
+            id: value.id,
+            imap: value.imap,
+            enabled: value.enabled,
+            email: value.email,
+            name: value.name,
+            capabilities: value.capabilities,
+            date_since: value.date_since,
+            folder_limit: value.folder_limit,
+            sync_folders: value.sync_folders,
+            account_type: value.account_type,
+            sync_interval_min: value.sync_interval_min,
+            known_folders: value.known_folders,
+            created_at: value.created_at,
+            updated_at: value.updated_at,
+            use_proxy: value.use_proxy,
+        }
+    }
+}
+
+impl From<AccountV3> for AccountV2 {
+    fn from(value: AccountV3) -> Self {
+        Self {
+            id: value.id,
+            imap: value.imap,
+            enabled: value.enabled,
+            email: value.email,
+            name: value.name,
+            capabilities: value.capabilities,
+            date_since: value.date_since,
+            folder_limit: value.folder_limit,
+            sync_folders: value.sync_folders,
+            account_type: value.account_type,
+            sync_interval_min: value.sync_interval_min,
+            known_folders: value.known_folders,
+            created_at: value.created_at,
+            updated_at: value.updated_at,
+            use_proxy: value.use_proxy,
+            use_dangerous: value.use_dangerous,
+            pgp_key: value.pgp_key,
+        }
+    }
+}
+
+impl From<AccountV2> for AccountV3 {
+    fn from(value: AccountV2) -> Self {
+        Self {
+            id: value.id,
+            imap: value.imap,
+            enabled: value.enabled,
+            email: value.email,
+            name: value.name,
+            capabilities: value.capabilities,
+            date_since: value.date_since,
+            folder_limit: value.folder_limit,
+            sync_folders: value.sync_folders,
+            account_type: value.account_type,
+            sync_interval_min: value.sync_interval_min,
+            known_folders: value.known_folders,
+            created_at: value.created_at,
+            updated_at: value.updated_at,
+            created_by: DEFAULT_ADMIN_USER_ID,
+            use_proxy: value.use_proxy,
+            use_dangerous: value.use_dangerous,
+            pgp_key: value.pgp_key,
+            sync_batch_size: None,
+            date_before: None,
+        }
+    }
+}
+
+impl From<AccountV3> for AccountModel {
+    fn from(value: AccountV3) -> Self {
+        Self {
+            id: value.id,
+            imap: value.imap,
+            enabled: value.enabled,
+            email: value.email,
+            account_name: None,
+            login_name: value.name,
+            capabilities: value.capabilities,
+            date_since: value.date_since,
+            date_before: value.date_before,
+            download_folders: value.sync_folders,
+            account_type: value.account_type,
+            download_interval_min: value.sync_interval_min,
+            download_batch_size: value.sync_batch_size,
+            max_email_size_bytes: None,
+            known_folders: value.known_folders,
+            created_at: value.created_at,
+            updated_at: value.updated_at,
+            created_by: value.created_by,
+            use_dangerous: value.use_dangerous,
+            pgp_key: value.pgp_key,
+            imap_quota_window: None,
+            imap_quota_bytes: None,
+            auto_download_new_mailboxes: None,
+            download_schedule: None,
+            deleting: false,
+            archive_rules: None,
+            extraction_rules: None,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[native_model(id = 5, version = 1)]
+#[native_db(primary_key(pk -> String))]
+pub struct OAuth2 {
+    /// A unique identifier for the OAuth2 configuration.
+    #[secondary_key(unique)]
+    pub id: u64,
+    /// A description of what this configuration is used for.
+    pub description: Option<String>,
+    /// The client ID used for authenticating the application with the OAuth2 provider.
+    pub client_id: String,
+    /// The client secret used in conjunction with the client ID.
+    ///
+    /// Users should provide a plaintext secret.
+    /// The server will encrypt it using AES-256-GCM and securely store it.
+    /// The plaintext secret is never stored, so users must ensure it is valid for OAuth2 authentication.
+    pub client_secret: String,
+    /// The URL to redirect users to for OAuth2 authorization.
+    pub auth_url: String,
+    /// The URL to exchange authorization codes for access tokens.
+    pub token_url: String,
+    /// The URI where the OAuth2 provider will redirect to after authorization.
+    pub redirect_uri: String,
+    /// The scopes of access that are being requested (e.g., email, profile).
+    pub scopes: Option<Vec<String>>,
+    /// Any additional parameters to include in the OAuth2 requests (e.g., access_type, prompt).
+    pub extra_params: Option<BTreeMap<String, String>>,
+    /// Indicates whether this configuration is enabled or disabled.
+    pub enabled: bool,
+    /// route OAuth through proxy (when direct access is blocked)
+    pub use_proxy: Option<u64>,
+    /// The timestamp when the configuration was created, in milliseconds since the Unix epoch.
+    pub created_at: i64,
+    /// The timestamp when the configuration was last updated, in milliseconds since the Unix epoch.
+    pub updated_at: i64,
+}
+
+impl OAuth2 {
+    fn pk(&self) -> String {
+        format!("{}_{}", &self.created_at, &self.id)
+    }
+}
+
+impl From<OAuth2> for bichon_core::oauth2::entity::OAuth2 {
+    fn from(value: OAuth2) -> Self {
+        Self {
+            id: value.id,
+            description: value.description,
+            client_id: value.client_id,
+            client_secret: value.client_secret,
+            auth_url: value.auth_url,
+            token_url: value.token_url,
+            redirect_uri: value.redirect_uri,
+            scopes: value.scopes,
+            extra_params: value.extra_params,
+            enabled: value.enabled,
+            use_proxy: value.use_proxy,
+            created_at: value.created_at,
+            updated_at: value.updated_at,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[native_model(id = 6, version = 1)]
+#[native_db]
+pub struct OAuth2PendingEntity {
+    /// Unique identifier for the OAuth2 request record
+    pub oauth2_id: u64,
+
+    pub account_id: u64,
+    /// CSRF protection state parameter used to verify the integrity of the authorization request
+    #[primary_key]
+    pub state: String,
+
+    /// PKCE code verifier used in the authorization code exchange process to ensure security
+    pub code_verifier: String,
+
+    /// Timestamp when the OAuth2 request was created, used to determine request expiration
+    pub created_at: i64,
+}
+
+impl From<OAuth2PendingEntity> for bichon_core::oauth2::pending::OAuth2PendingEntity {
+    fn from(value: OAuth2PendingEntity) -> Self {
+        Self {
+            oauth2_id: value.oauth2_id,
+            account_id: value.account_id,
+            state: value.state,
+            code_verifier: value.code_verifier,
+            created_at: value.created_at,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[native_model(id = 7, version = 1)]
+#[native_db]
+pub struct OAuth2AccessToken {
+    /// The ID of the account associated with this access token.
+    #[primary_key]
+    pub account_id: u64,
+    /// The id of the OAuth2 configuration associated with this access token.
+    #[secondary_key]
+    pub oauth2_id: u64,
+    /// The OAuth2 access token used to authenticate requests to the provider.
+    pub access_token: Option<String>,
+    /// The OAuth2 refresh token used to obtain new access tokens.
+    pub refresh_token: Option<String>,
+    /// The timestamp when the token record was created, in milliseconds since the Unix epoch.
+    pub created_at: i64,
+    /// The timestamp when the token record was last updated, in milliseconds since the Unix epoch.
+    pub updated_at: i64,
+}
+
+impl From<OAuth2AccessToken> for bichon_core::oauth2::token::OAuth2AccessToken {
+    fn from(value: OAuth2AccessToken) -> Self {
+        Self {
+            account_id: value.account_id,
+            oauth2_id: value.oauth2_id,
+            access_token: value.access_token,
+            refresh_token: value.refresh_token,
+            created_at: value.created_at,
+            updated_at: value.updated_at,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[native_model(id = 8, version = 1)]
+#[native_db]
+pub struct Proxy {
+    /// The unique identifier for this proxy configuration.
+    #[primary_key]
+    pub id: u64,
+
+    /// The proxy URL (e.g., socks5://127.0.0.1:1080) used to route network requests.
+    pub url: String,
+
+    /// The creation timestamp of this record, represented as milliseconds since the Unix epoch.
+    pub created_at: i64,
+
+    /// The last update timestamp of this record, represented as milliseconds since the Unix epoch.
+    pub updated_at: i64,
+}
+
+impl From<Proxy> for bichon_core::settings::proxy::Proxy {
+    fn from(value: Proxy) -> Self {
+        Self {
+            id: value.id,
+            url: value.url,
+            created_at: value.created_at,
+            updated_at: value.updated_at,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[native_model(id = 9, version = 1)]
+#[native_db]
+pub struct UserRole {
+    #[primary_key]
+    pub id: u64,
+    pub name: String,
+    pub description: Option<String>,
+    pub permissions: BTreeSet<String>,
+    pub is_builtin: bool,
+    pub created_at: i64,
+    pub role_type: RoleType,
+    pub updated_at: i64,
+}
+
+impl From<UserRole> for bichon_core::users::role::UserRole {
+    fn from(value: UserRole) -> Self {
+        Self {
+            id: value.id,
+            name: value.name,
+            description: value.description,
+            permissions: value.permissions,
+            is_builtin: value.is_builtin,
+            created_at: value.created_at,
+            role_type: value.role_type,
+            updated_at: value.updated_at,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[native_model(id = 10, version = 1)]
+#[native_db]
+pub struct BichonUser {
+    #[primary_key]
+    pub id: u64,
+    #[secondary_key(unique)]
+    pub username: String,
+    #[secondary_key(unique)]
+    pub email: String,
+
+    pub password: Option<String>,
+
+    /// Scoped Access: Defines per-account permissions.
+    /// Example:
+    /// { account_id: 1, role_id: role_manager_id } -> Manager on Account 1
+    /// { account_id: 2, role_id: role_viewer_id }  -> Viewer on Account 2
+    pub account_access_map: BTreeMap<u64, u64>,
+
+    pub description: Option<String>,
+
+    /// System Roles: Permissions that apply to the whole system
+    /// (e.g., system settings, creating new users).
+    pub global_roles: Vec<u64>,
+
+    pub avatar: Option<String>,
+    pub created_at: i64,
+    pub updated_at: i64,
+    /// Optional access control settings
+    pub acl: Option<AccessControl>,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[native_model(id = 10, version = 2, from = BichonUser)]
+#[native_db]
+pub struct BichonUserV2 {
+    #[primary_key]
+    pub id: u64,
+    #[secondary_key(unique)]
+    pub username: String,
+    #[secondary_key(unique)]
+    pub email: String,
+
+    pub password: Option<String>,
+
+    /// Scoped Access: Defines per-account permissions.
+    /// Example:
+    /// { account_id: 1, role_id: role_manager_id } -> Manager on Account 1
+    /// { account_id: 2, role_id: role_viewer_id }  -> Viewer on Account 2
+    pub account_access_map: BTreeMap<u64, u64>,
+
+    pub description: Option<String>,
+
+    /// System Roles: Permissions that apply to the whole system
+    /// (e.g., system settings, creating new users).
+    pub global_roles: Vec<u64>,
+
+    pub avatar: Option<String>,
+    pub created_at: i64,
+    pub updated_at: i64,
+    /// Optional access control settings
+    pub acl: Option<AccessControl>,
+
+    pub theme: Option<String>,
+    pub language: Option<String>,
+}
+
+impl From<BichonUserV2> for BichonUser {
+    fn from(value: BichonUserV2) -> Self {
+        BichonUser {
+            id: value.id,
+            username: value.username,
+            email: value.email,
+            password: value.password,
+            account_access_map: value.account_access_map,
+            description: value.description,
+            global_roles: value.global_roles,
+            avatar: value.avatar,
+            created_at: value.created_at,
+            updated_at: value.updated_at,
+            acl: value.acl,
+        }
+    }
+}
+
+impl From<BichonUser> for BichonUserV2 {
+    fn from(value: BichonUser) -> Self {
+        BichonUserV2 {
+            id: value.id,
+            username: value.username,
+            email: value.email,
+            password: value.password,
+            account_access_map: value.account_access_map,
+            description: value.description,
+            global_roles: value.global_roles,
+            avatar: value.avatar,
+            created_at: value.created_at,
+            updated_at: value.updated_at,
+            acl: value.acl,
+            theme: None,
+            language: None,
+        }
+    }
+}
+
+impl From<BichonUserV2> for bichon_core::users::BichonUserV2 {
+    fn from(value: BichonUserV2) -> Self {
+        Self {
+            id: value.id,
+            username: value.username,
+            email: value.email,
+            password: value.password,
+            account_access_map: value.account_access_map,
+            description: value.description,
+            global_roles: value.global_roles,
+            avatar: value.avatar,
+            created_at: value.created_at,
+            updated_at: value.updated_at,
+            acl: value.acl,
+            theme: value.theme,
+            language: value.language,
+            sso_id: None,
+            sso_provider: None,
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
+#[native_model(id = 11, version = 1)]
+#[native_db]
+pub struct AccessTokenModel {
+    /// The ID of the user who owns this token
+    #[secondary_key]
+    pub user_id: u64,
+    /// The unique token string used for authentication
+    #[primary_key]
+    pub token: String,
+    /// An optional name of the token.
+    pub name: Option<String>,
+    /// Token type: WebUI or API
+    pub token_type: TokenType,
+    /// The timestamp (in milliseconds since epoch) when the token was created.
+    pub created_at: i64,
+    /// The timestamp (in milliseconds since epoch) when the token was last updated.
+    pub updated_at: i64,
+    /// The timestamp (in milliseconds since epoch) when the token expires.
+    /// None means the token does not expire (this applies only to API tokens).
+    pub expire_at: Option<i64>,
+    /// The timestamp (in milliseconds since epoch) when the token was last used.
+    pub last_access_at: i64,
+}
+
+impl From<AccessTokenModel> for bichon_core::token::AccessTokenModel {
+    fn from(value: AccessTokenModel) -> Self {
+        Self {
+            user_id: value.user_id,
+            token: value.token,
+            name: value.name,
+            token_type: value.token_type,
+            created_at: value.created_at,
+            updated_at: value.updated_at,
+            expire_at: value.expire_at,
+            last_access_at: value.last_access_at,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[native_model(id = 1, version = 1)]
+#[native_db]
+pub struct MailBox {
+    /// The unique identifier for the mailbox
+    #[primary_key]
+    pub id: u64,
+    /// The ID of the account associated with the mailbox
+    #[secondary_key]
+    pub account_id: u64,
+    /// The unique, decoded, human-readable name of the mailbox (e.g., "INBOX", "Sent Items").
+    /// This is the decoded name as presented to users, derived from the IMAP server's mailbox name
+    /// (e.g., after decoding UTF-7 or other encodings per RFC 3501).
+    pub name: String,
+    /// Optional delimiter used to separate mailbox names in a hierarchy (e.g., "/" or ".").
+    /// Used in IMAP to structure nested mailboxes (e.g., "INBOX/Archive").
+    pub delimiter: Option<String>,
+    /// List of attributes associated with the mailbox (e.g., `\NoSelect`, `\Deleted`).
+    /// These indicate special properties, such as whether the mailbox can hold messages.
+    pub attributes: Vec<Attribute>,
+    /// The number of messages that currently exist in the mailbox.
+    pub exists: u32,
+    /// Optional number of unseen messages in the mailbox (i.e., messages without the `\Seen` flag).
+    pub unseen: Option<u32>,
+    /// The next unique identifier (UID) that will be assigned to a new message in the mailbox.
+    /// If `None`, the IMAP server has not provided this information.
+    pub uid_next: Option<u32>,
+    /// The validity identifier for UIDs in this mailbox, used to ensure UID consistency across sessions.
+    /// If `None`, the IMAP server has not provided this information.
+    pub uid_validity: Option<u32>,
+}
+
+impl From<MailBox> for bichon_core::cache::imap::mailbox::MailBox {
+    fn from(value: MailBox) -> Self {
+        Self {
+            id: value.id,
+            account_id: value.account_id,
+            name: value.name,
+            delimiter: value.delimiter,
+            attributes: value.attributes,
+            exists: value.exists,
+            unseen: value.unseen,
+            uid_next: value.uid_next,
+            uid_validity: value.uid_validity,
+            highest_uid: None,
+        }
+    }
+}
+
+pub static META_MODELS: LazyLock<Models> = LazyLock::new(|| {
+    let mut adapter = ModelsAdapter::new();
+    adapter.register_metadata_models();
+    adapter.models
+});
+
+pub static MAILBOX_MODELS: LazyLock<Models> = LazyLock::new(|| {
+    let mut adapter = ModelsAdapter::new();
+    adapter.register_model::<MailBox>();
+    adapter.models
+});
+
+pub struct ModelsAdapter {
+    pub models: Models,
+}
+
+impl ModelsAdapter {
+    pub fn new() -> Self {
+        ModelsAdapter {
+            models: Models::new(),
+        }
+    }
+
+    pub fn register_model<T: ToInput>(&mut self) {
+        self.models.define::<T>().expect("failed to define model ");
+    }
+
+    pub fn register_metadata_models(&mut self) {
+        self.register_model::<CachedMailSettings>();
+        self.register_model::<AccountV1>();
+        self.register_model::<AccountV2>();
+        self.register_model::<AccountV3>();
+        self.register_model::<OAuth2>();
+        self.register_model::<OAuth2PendingEntity>();
+        self.register_model::<OAuth2AccessToken>();
+        self.register_model::<Proxy>();
+        self.register_model::<UserRole>();
+        self.register_model::<BichonUser>();
+        self.register_model::<BichonUserV2>();
+        self.register_model::<AccessTokenModel>();
+    }
+}
+
+fn init_meta_database(root_path: &PathBuf) -> BichonResult<Arc<Database<'static>>> {
+    let mut database = Builder::new()
+        .set_cache_size(134217728)
+        .create(&META_MODELS, root_path.join("meta.db"))
+        .map_err(handle_database_error)?;
+
+    let rw = database
+        .rw_transaction()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+    rw.migrate::<AccountV3>()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+    rw.migrate::<BichonUserV2>()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+    rw.commit()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+    database
+        .compact()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+    Ok(Arc::new(database))
+}
+
+fn init_evenlope_database(root_path: &PathBuf) -> BichonResult<Arc<Database<'static>>> {
+    let mut database = Builder::new()
+        .set_cache_size(1073741824)
+        .create(&MAILBOX_MODELS, root_path.join("mailbox.db"))
+        .map_err(handle_database_error)?;
+
+    let rw = database
+        .rw_transaction()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+    rw.commit()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+    database
+        .compact()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+    Ok(Arc::new(database))
+}
+
+fn handle_database_error(error: native_db::db_type::Error) -> BichonError {
+    raise_error!(
+        format!("Failed to create database: {:?}", error),
+        ErrorCode::InternalError
+    )
+}
+
+pub fn list_all_impl<T: ToInput + Clone + Send + 'static>(
+    database: &Arc<Database<'static>>,
+) -> BichonResult<Vec<T>> {
+    let r_transaction = database
+        .r_transaction()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+    let entities: Vec<T> = r_transaction
+        .scan()
+        .primary()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
+        .all()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
+        .try_collect()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+    Ok(entities)
+}
+
+pub fn migrate_metadata(root_path: &PathBuf) -> Result<(), Box<dyn std::error::Error>> {
+    // Pre-flight: verify old metadata databases exist
+    let meta_db_path = root_path.join("meta.db");
+    if !meta_db_path.exists() {
+        return Err(format!(
+            "Legacy metadata database not found at '{}'. \
+             Make sure the root directory points to a valid v0.3.7 installation.",
+            meta_db_path.display()
+        )
+        .into());
+    }
+    let mailbox_db_path = root_path.join("mailbox.db");
+    if !mailbox_db_path.exists() {
+        return Err(format!(
+            "Legacy mailbox database not found at '{}'. \
+             Make sure the root directory points to a valid v0.3.7 installation.",
+            mailbox_db_path.display()
+        )
+        .into());
+    }
+
+    // Initialize legacy database connections
+    let meta_db = init_meta_database(root_path)
+        .map_err(|e| format!("Failed to initialize legacy metadata database: {}", e))?;
+    let envelope_db = init_evenlope_database(root_path)
+        .map_err(|e| format!("Failed to initialize legacy envelope database: {}", e))?;
+
+    // Prepare new database directory
+    let db_path = root_path.join("memdb");
+    if !db_path.exists() {
+        std::fs::create_dir_all(&db_path)?;
+    }
+
+    // Open new database (disable full durability for faster bulk writes)
+    let db = MemDb::open_with(&db_path, Durability::Off)
+        .map_err(|e| format!("Failed to open new memdb database: {}", e))?;
+
+    println!(
+        "{}",
+        style("Step 1: Migrating Metadata Entities...")
+            .bold()
+            .cyan()
+    );
+
+    // Migration helper macro to reduce boilerplate
+    macro_rules! migrate_collection {
+        ($name:expr, $old_type:ty, $new_type:ty, $source_db:expr) => {
+            print!("  > {:<25} ", $name);
+            let items = list_all_impl::<$old_type>($source_db)?;
+            let count = items.len();
+            let converted: Vec<$new_type> = items.into_iter().map(|a| a.into()).collect();
+            batch_insert_impl(&db, converted)?;
+            println!("{} ({} items)", style("done").green(), count);
+        };
+    }
+
+    // --- Migrate each entity type ---
+
+    migrate_collection!(
+        "Mail Settings",
+        CachedMailSettings,
+        bichon_core::autoconfig::CachedMailSettings,
+        &meta_db
+    );
+
+    migrate_collection!("Accounts", AccountV3, AccountModel, &meta_db);
+
+    migrate_collection!(
+        "OAuth2 Entities",
+        OAuth2,
+        bichon_core::oauth2::entity::OAuth2,
+        &meta_db
+    );
+
+    migrate_collection!(
+        "OAuth2 Pending",
+        OAuth2PendingEntity,
+        bichon_core::oauth2::pending::OAuth2PendingEntity,
+        &meta_db
+    );
+
+    migrate_collection!(
+        "OAuth2 Access Tokens",
+        OAuth2AccessToken,
+        bichon_core::oauth2::token::OAuth2AccessToken,
+        &meta_db
+    );
+
+    migrate_collection!(
+        "Proxy Settings",
+        Proxy,
+        bichon_core::settings::proxy::Proxy,
+        &meta_db
+    );
+
+    migrate_collection!(
+        "User Roles",
+        UserRole,
+        bichon_core::users::role::UserRole,
+        &meta_db
+    );
+
+    migrate_collection!(
+        "Users",
+        BichonUserV2,
+        bichon_core::users::BichonUserV2,
+        &meta_db
+    );
+
+    migrate_collection!(
+        "Access Tokens",
+        AccessTokenModel,
+        bichon_core::token::AccessTokenModel,
+        &meta_db
+    );
+
+    // Mailboxes (from envelope_db)
+    migrate_collection!(
+        "Mailboxes",
+        MailBox,
+        bichon_core::cache::imap::mailbox::MailBox,
+        &envelope_db
+    );
+
+    // Persist and finish
+    db.snapshot()
+        .map_err(|e| format!("Snapshot save failed: {}", e))?;
+    println!(
+        "{}",
+        style("Metadata migration completed successfully.")
+            .green()
+            .bold()
+    );
+
+    Ok(())
+}

crates/admin/src/migrate_store_v2.rs

@@ -0,0 +1,569 @@
+use std::{path::PathBuf, time::Instant};
+
+use bytes::Bytes;
+use mail_parser::MimeHeaders;
+
+use bichon_core::{
+    envelope::extractor::extract_references, message::content::AttachmentInfo,
+    store::tantivy::tokenizers::EuroTokenizer, utils::compute_content_hash,
+};
+
+use bichon_blob::{Codec, Config, Engine};
+use mail_parser::MessageParser;
+use tantivy::{indexer::NoMergePolicy, Index, IndexWriter, TantivyDocument};
+use uuid::Uuid;
+
+use bichon_core::{
+    common::AddrVec,
+    envelope::extractor::{compute_thread_id, generate_message_id},
+    error::{code::ErrorCode, BichonResult},
+    raise_error,
+    store::envelope::Envelope,
+    store::tantivy::{
+        model::{AttachmentModel, EnvelopeWithAttachments},
+        schema::SchemaTools,
+    },
+    utc_now,
+};
+
+pub struct LegacyDirs {
+    pub envelope_dir: PathBuf,
+    pub eml_dir: PathBuf,
+}
+
+pub struct NewDirs {
+    pub envelope_dir: PathBuf,
+    pub attachment_dir: PathBuf,
+    pub storage_dir: PathBuf,
+}
+
+impl LegacyDirs {
+    pub fn new(index: PathBuf, data: PathBuf) -> Self {
+        Self {
+            envelope_dir: index,
+            eml_dir: data,
+        }
+    }
+}
+
+impl NewDirs {
+    pub fn new(index: PathBuf, data: PathBuf) -> Self {
+        Self {
+            envelope_dir: index.join("mail_metadata"),
+            attachment_dir: index.join("attachment_metadata"),
+            storage_dir: data,
+        }
+    }
+}
+
+pub struct DetachOutput {
+    pub infos: Vec<AttachmentInfo>,
+    pub blobs: Vec<(String, Bytes)>,
+}
+
+fn hex_to_raw_key(hex: &str) -> BichonResult<[u8; 32]> {
+    let mut key = [0u8; 32];
+    hex::decode_to_slice(hex, &mut key).map_err(|e| {
+        raise_error!(
+            format!("invalid content hash: {e:#?}"),
+            ErrorCode::InternalError
+        )
+    })?;
+    Ok(key)
+}
+
+pub fn detach_attachments_standalone(
+    original_body: &[u8],
+    message: &mail_parser::Message<'_>,
+) -> (Vec<u8>, DetachOutput) {
+    let mut stripped_eml = original_body.to_vec();
+    let mut infos = Vec::new();
+    let mut blobs = Vec::new();
+
+    let mut ranges: Vec<_> = message
+        .attachments()
+        .map(|att| {
+            (
+                att.raw_body_offset() as usize,
+                att.raw_end_offset() as usize,
+                att,
+            )
+        })
+        .collect();
+    ranges.sort_by(|a, b| b.0.cmp(&a.0));
+
+    for (raw_start, raw_end, att) in ranges {
+        let content_hash = compute_content_hash(att.contents());
+        let body_len = original_body.len();
+        let raw_start = raw_start.min(body_len);
+        let raw_end = raw_end.min(body_len);
+        let range_valid = raw_start < raw_end;
+
+        if range_valid {
+            blobs.push((
+                content_hash.clone(),
+                Bytes::copy_from_slice(&original_body[raw_start..raw_end]),
+            ));
+        }
+
+        if range_valid {
+            let placeholder = format!("<<BICHON_DETACH_HASH:{}>>", &content_hash);
+            stripped_eml.splice(raw_start..raw_end, placeholder.as_bytes().iter().cloned());
+        }
+
+        infos.push(AttachmentInfo {
+            filename: att.attachment_name().map(|n| n.to_string()),
+            size: att.contents().len(),
+            inline: att
+                .content_disposition()
+                .map(|d| d.is_inline())
+                .unwrap_or_else(|| att.content_id().is_some()),
+            file_type: att
+                .content_type()
+                .map(|ct| {
+                    format!(
+                        "{}/{}",
+                        ct.c_type.as_ref(),
+                        ct.c_subtype.as_deref().unwrap_or("")
+                    )
+                })
+                .unwrap_or_else(|| "application/octet-stream".to_string()),
+            content_id: att.content_id().map(|id| id.to_string()),
+            content_hash,
+            is_message: att.is_message(),
+            extracted_text: None,
+            extracted_page_count: None,
+            extracted_is_ocr: false,
+        });
+    }
+
+    (stripped_eml, DetachOutput { infos, blobs })
+}
+
+pub struct NewIndexWriterV2 {
+    pub envelope_writer: Option<IndexWriter>,
+    pub attachment_writer: Option<IndexWriter>,
+    pub engine: Engine,
+    pending: usize,
+    email_buf: Vec<([u8; 32], Vec<u8>)>,
+    attachment_buf: Vec<([u8; 32], Vec<u8>)>,
+}
+
+impl NewIndexWriterV2 {
+    pub fn open(dirs: NewDirs) -> BichonResult<Self> {
+        // ── envelope index ──────────────────────────────────────────────
+        std::fs::create_dir_all(&dirs.envelope_dir)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        let envelope_index = if dirs
+            .envelope_dir
+            .read_dir()
+            .map(|mut d| d.next().is_none())
+            .unwrap_or(true)
+        {
+            Index::create_in_dir(&dirs.envelope_dir, SchemaTools::email_schema())
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
+        } else {
+            Index::open_in_dir(&dirs.envelope_dir)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
+        };
+
+        envelope_index
+            .tokenizers()
+            .register("euro", EuroTokenizer::new());
+
+        let envelope_writer = envelope_index
+            .writer_with_num_threads(3, 256 * 1024 * 1024)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        envelope_writer.set_merge_policy(Box::new(NoMergePolicy));
+        // ── attachment index ─────────────────────────────────────────────
+        std::fs::create_dir_all(&dirs.attachment_dir)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        let attachment_index = if dirs
+            .attachment_dir
+            .read_dir()
+            .map(|mut d| d.next().is_none())
+            .unwrap_or(true)
+        {
+            Index::create_in_dir(&dirs.attachment_dir, SchemaTools::attachment_schema())
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
+        } else {
+            Index::open_in_dir(&dirs.attachment_dir)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
+        };
+
+        attachment_index
+            .tokenizers()
+            .register("euro", EuroTokenizer::new());
+        let attachment_writer = attachment_index
+            .writer_with_num_threads(3, 256 * 1024 * 1024)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        attachment_writer.set_merge_policy(Box::new(NoMergePolicy));
+
+        // ── blob store (bichon-blob, not fjall) ───────────────────────────
+        std::fs::create_dir_all(&dirs.storage_dir)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        let blob_dir = dirs.storage_dir.join("blobs");
+
+        let mut config = Config::default();
+        config.default_codec = Codec::Zstd;
+        config.compress_threshold = 1024;
+        config.flush_interval_secs = 0;
+        config.gc_interval_secs = 0;
+
+        let engine = Engine::open(&blob_dir, config)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        Ok(Self {
+            envelope_writer: Some(envelope_writer),
+            attachment_writer: Some(attachment_writer),
+            engine,
+            pending: 0,
+            email_buf: Vec::new(),
+            attachment_buf: Vec::new(),
+        })
+    }
+
+    pub fn ingest(
+        &mut self,
+        eml_bytes: &[u8],
+        account_id: u64,
+        mailbox_id: u64,
+        uid: u32,
+        internal_date: i64,
+    ) -> BichonResult<()> {
+        let email_content_hash = compute_content_hash(eml_bytes);
+        let email_raw_key = hex_to_raw_key(&email_content_hash)?;
+
+        let message = MessageParser::new()
+            .parse(eml_bytes)
+            .ok_or_else(|| raise_error!("failed to parse eml".into(), ErrorCode::InternalError))?;
+
+        if message.parts.is_empty() {
+            return Err(raise_error!(
+                "Malformed or completely empty EML (no parts found)".into(),
+                ErrorCode::InternalError
+            ));
+        }
+        // ── text / preview ────────────────────────────────────────────────
+        let text = message
+            .body_text(0)
+            .map(|c| c.into_owned())
+            .or_else(|| {
+                message
+                    .body_html(0)
+                    .map(|html| bichon_core::utils::html::extract_text(html.into_owned()))
+            })
+            .unwrap_or_default();
+        let text = text.split_whitespace().collect::<Vec<_>>().join(" ");
+        let preview = if text.chars().count() > 100 {
+            text.chars().take(100).collect::<String>() + "..."
+        } else {
+            text.clone()
+        };
+
+        // ── headers ───────────────────────────────────────────────────────
+        let message_id = message
+            .message_id()
+            .map(String::from)
+            .unwrap_or_else(generate_message_id);
+
+        let in_reply_to = message.in_reply_to().as_text().map(String::from);
+        let references = extract_references(&message);
+        let thread_id = compute_thread_id(in_reply_to, references, &message_id);
+
+        let subject = message.subject().map(String::from).unwrap_or_default();
+        let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0);
+        let internal_date = if internal_date == 0 {
+            date
+        } else {
+            internal_date
+        };
+
+        let parse_addrs = |addrs: Option<&mail_parser::Address<'_>>| {
+            addrs
+                .map(|addr| {
+                    AddrVec::from(addr)
+                        .0
+                        .into_iter()
+                        .filter_map(|a| a.address)
+                        .collect::<Vec<_>>()
+                })
+                .unwrap_or_default()
+        };
+
+        let from = message
+            .from()
+            .and_then(|addr| AddrVec::from(addr).0.into_iter().next())
+            .and_then(|a| a.address)
+            .unwrap_or_else(|| "unknown".to_string());
+        let to = parse_addrs(message.to());
+        let cc = parse_addrs(message.cc());
+        let bcc = parse_addrs(message.bcc());
+
+        // ── detach attachments → blob ──────────────────────────────────────
+        let (stripped_eml, attachment_output) = detach_attachments_standalone(eml_bytes, &message);
+
+        // Buffer for bulk write — sorted + flushed later.
+        // Key is the raw 32-byte hash (not the hex string).
+        self.email_buf
+            .push((email_raw_key, stripped_eml));
+        for (hash, data) in &attachment_output.blobs {
+            let raw_key = hex_to_raw_key(hash)?;
+            self.attachment_buf.push((raw_key, data.to_vec()));
+        }
+
+        // ── build envelope doc ────────────────────────────────────────────
+        let envelope_id = Uuid::new_v4().to_string();
+        let now = utc_now!();
+
+        let attachment_docs: Vec<TantivyDocument> = attachment_output
+            .infos
+            .iter()
+            .filter(|a| !a.inline || a.content_id.is_none())
+            .map(|a| {
+                AttachmentModel {
+                    id: Uuid::new_v4().to_string(),
+                    envelope_id: envelope_id.clone(),
+                    account_id,
+                    account_email: None,
+                    mailbox_id,
+                    mailbox_name: None,
+                    subject: subject.clone(),
+                    content_hash: a.content_hash.clone(),
+                    from: from.clone(),
+                    date,
+                    ingest_at: now,
+                    size: a.size as u64,
+                    ext: a.get_extension(),
+                    category: a.get_category().to_string(),
+                    content_type: a.file_type.clone(),
+                    shard_id: 0,
+                    text: None,
+                    has_text: false,
+                    is_ocr: false,
+                    page_count: None,
+                    is_indexed: false,
+                    is_message: a.is_message,
+                    name: a.filename.clone(),
+                    tags: None,
+                    auto_tags: None,
+                }
+                .into_document()
+            })
+            .collect();
+
+        let envelope = Envelope {
+            id: envelope_id,
+            message_id,
+            account_id,
+            mailbox_id,
+            uid,
+            subject,
+            preview,
+            from,
+            to,
+            cc,
+            bcc,
+            date,
+            internal_date,
+            ingest_at: now,
+            size: eml_bytes.len() as u32,
+            thread_id,
+            attachment_count: message.attachment_count(),
+            regular_attachment_count: attachment_docs.len(),
+            tags: None,
+            account_email: None,
+            account_name: None,
+            mailbox_name: None,
+            content_hash: email_content_hash,
+        };
+
+        let ea = EnvelopeWithAttachments {
+            envelope,
+            attachments: Some(attachment_output.infos),
+        };
+        let envelope_doc = ea.to_document(&text, 0)?;
+
+        self.envelope_writer
+            .as_mut()
+            .unwrap()
+            .add_document(envelope_doc)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        for doc in attachment_docs {
+            self.attachment_writer
+                .as_mut()
+                .unwrap()
+                .add_document(doc)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        }
+
+        self.pending += 1;
+
+        Ok(())
+    }
+
+    /// Commit pending Tantivy documents (mid-stream) — frees the in-memory
+    /// term dictionary / postings that accumulate in the IndexWriter.
+    fn commit_tantivy(&mut self) -> BichonResult<()> {
+        if self.pending == 0 {
+            return Ok(());
+        }
+        println!("Tantivy committing... this may take 2-3 minutes, please wait.");
+        let start = Instant::now();
+        if let Some(writer) = self.envelope_writer.as_mut() {
+            writer
+                .commit()
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        }
+        if let Some(writer) = self.attachment_writer.as_mut() {
+            writer
+                .commit()
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        }
+        println!("tantivy commit elapsed: {:#?}", start.elapsed());
+        tracing::info!(count = self.pending, "committed tantivy batch");
+        self.pending = 0;
+        Ok(())
+    }
+
+    /// Final commit + segment merge for Tantivy writers (called once at end).
+    pub fn finish_writers(&mut self) -> BichonResult<()> {
+        self.commit_tantivy()?;
+
+        for (name, writer_opt) in [
+            ("envelope", &mut self.envelope_writer),
+            ("attachment", &mut self.attachment_writer),
+        ] {
+            if let Some(writer) = writer_opt.as_mut() {
+                let seg_ids = writer
+                    .index()
+                    .searchable_segment_ids()
+                    .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+                println!("merging {} {} segments...", seg_ids.len(), name);
+                if seg_ids.len() > 1 {
+                    let _ = writer.merge(&seg_ids);
+                }
+            }
+
+            if let Some(writer) = writer_opt.take() {
+                println!("waiting for {} merge to finish...", name);
+                let start = std::time::Instant::now();
+                let _ = writer.wait_merging_threads();
+                println!("{} merge done: {:#?}", name, start.elapsed());
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Write buffered blobs to the bichon-blob engine.
+    /// Also commits the Tantivy writers to bound their in-memory state.
+    pub fn flush_blob_buffers(&mut self) -> BichonResult<()> {
+        self.commit_tantivy()?;
+
+        if !self.email_buf.is_empty() {
+            let mut buf = std::mem::take(&mut self.email_buf);
+            buf.sort_by(|a, b| a.0.cmp(&b.0));
+            buf.dedup_by(|a, b| a.0 == b.0);
+
+            let count = buf.len();
+            let mut skipped = 0usize;
+            let mut batch: Vec<([u8; 32], Vec<u8>, Codec)> = Vec::with_capacity(buf.len());
+            for (key, data) in buf {
+                if data.len() > 100 * 1024 * 1024 {
+                    eprintln!(
+                        "{}",
+                        console::style(format!(
+                            "WARN: skipping oversized email blob key={} ({} bytes)",
+                            hex::encode(key),
+                            data.len()
+                        ))
+                        .yellow()
+                    );
+                    skipped += 1;
+                    continue;
+                }
+                batch.push((key, data, Codec::Zstd));
+            }
+            if !batch.is_empty() {
+                self.engine.put_batch(&batch).map_err(|e| {
+                    raise_error!(
+                        format!("blob engine put_batch error: {e:#?}"),
+                        ErrorCode::InternalError
+                    )
+                })?;
+            }
+            println!("flushed {} email blobs to engine", count - skipped);
+            if skipped > 0 {
+                eprintln!(
+                    "{}",
+                    console::style(format!("skipped {} oversized email blobs", skipped)).yellow()
+                );
+            }
+        }
+
+        if !self.attachment_buf.is_empty() {
+            let mut buf = std::mem::take(&mut self.attachment_buf);
+            buf.sort_by(|a, b| a.0.cmp(&b.0));
+            buf.dedup_by(|a, b| a.0 == b.0);
+
+            let count = buf.len();
+            let mut skipped = 0usize;
+            let mut batch: Vec<([u8; 32], Vec<u8>, Codec)> = Vec::with_capacity(buf.len());
+            for (key, data) in buf {
+                if data.len() > 100 * 1024 * 1024 {
+                    eprintln!(
+                        "{}",
+                        console::style(format!(
+                            "WARN: skipping oversized attachment blob key={} ({} bytes)",
+                            hex::encode(key),
+                            data.len()
+                        ))
+                        .yellow()
+                    );
+                    skipped += 1;
+                    continue;
+                }
+                batch.push((key, data, Codec::Zstd));
+            }
+            if !batch.is_empty() {
+                self.engine.put_batch(&batch).map_err(|e| {
+                    raise_error!(
+                        format!("blob engine put_batch error: {e:#?}"),
+                        ErrorCode::InternalError
+                    )
+                })?;
+            }
+            println!("flushed {} attachment blobs to engine", count - skipped);
+            if skipped > 0 {
+                eprintln!(
+                    "{}",
+                    console::style(format!("skipped {} oversized attachment blobs", skipped))
+                        .yellow()
+                );
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Flush and shutdown the blob engine (called once at the very end).
+    pub fn shutdown_engine(&mut self) -> BichonResult<()> {
+        self.engine.flush().map_err(|e| {
+            raise_error!(
+                format!("engine flush error: {e:#?}"),
+                ErrorCode::InternalError
+            )
+        })?;
+        self.engine.shutdown().map_err(|e| {
+            raise_error!(
+                format!("engine shutdown error: {e:#?}"),
+                ErrorCode::InternalError
+            )
+        })?;
+        Ok(())
+    }
+}

crates/admin/src/migrate_v037.rs

@@ -0,0 +1,691 @@
+use std::{collections::HashMap, path::PathBuf};
+
+use bichon_core::{
+    error::{code::ErrorCode, BichonResult},
+    migrate::{is_tantivy_index_dir, write_storage_version},
+    raise_error,
+};
+use console::style;
+use dialoguer::{theme::ColorfulTheme, Confirm, Input};
+use indicatif::{ProgressBar, ProgressStyle};
+use tantivy::{
+    collector::TopDocs,
+    columnar::Column,
+    query::TermQuery,
+    schema::{IndexRecordOption, Value},
+    DocAddress, Index, TantivyDocument, Term,
+};
+
+use crate::legacy::schema::SchemaTools;
+use crate::migrate_store_v2::{NewDirs, NewIndexWriterV2};
+
+pub struct LegacyDirs {
+    pub envelope_dir: PathBuf,
+    pub eml_dir: PathBuf,
+}
+
+impl LegacyDirs {
+    pub fn new(index: PathBuf, data: PathBuf) -> Self {
+        Self {
+            envelope_dir: index,
+            eml_dir: data,
+        }
+    }
+}
+
+pub fn is_legacy_data_layout_with_paths(
+    envelope_dir: &PathBuf,
+    eml_dir: &PathBuf,
+) -> std::io::Result<bool> {
+    let envelope_result = is_tantivy_index_dir(envelope_dir)?;
+    let eml_result = is_tantivy_index_dir(eml_dir)?;
+    Ok(envelope_result || eml_result)
+}
+
+/// Return the number of segments in the legacy EML Tantivy index.
+pub fn count_eml_segments(legacy: &LegacyDirs) -> BichonResult<usize> {
+    let eml_index = Index::open_in_dir(&legacy.eml_dir)
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+    let reader = eml_index
+        .reader()
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+    let searcher = reader.searcher();
+    Ok(searcher.segment_readers().len())
+}
+
+pub fn handle_migration_v037(theme: &ColorfulTheme) {
+    println!(
+        "\n{}",
+        style("MIGRATION: Bichon v0.3.7 Storage → v2.x (bichon-blob)")
+            .bold()
+            .yellow()
+    );
+
+    println!(
+        "{}",
+        style(
+            "This tool migrates data from the legacy v0.3.7 Tantivy-based storage \
+            architecture directly to the v2.x bichon-blob storage format."
+        )
+        .dim()
+    );
+
+    println!(
+        "{}",
+        style(
+            "Legacy v0.3.7 architecture:\n\
+            • envelope metadata stored in Tantivy\n\
+            • message data stored in Tantivy\n\n\
+                New v2.x architecture:\n\
+            • mail indexes stored in Tantivy\n\
+            • attachment indexes stored in Tantivy\n\
+            • raw message data stored in bichon-blob engine\n\
+            • attachment blobs stored in bichon-blob engine"
+        )
+        .dim()
+    );
+
+    println!(
+        "\n{} {}",
+        style("IMPORTANT:").yellow().bold(),
+        style(
+            "The paths below must exactly match what your old bichon server was configured with."
+        )
+        .yellow()
+    );
+
+    // --- bichon-root-dir ---
+    let root_dir_str: String = Input::with_theme(theme)
+        .with_prompt("Enter --bichon-root-dir (same value used by the old server)")
+        .validate_with(|input: &String| -> Result<(), &str> {
+            let path = PathBuf::from(input);
+            if !path.is_absolute() {
+                return Err("Path must be absolute.");
+            }
+            if !path.exists() {
+                return Err("Directory does not exist.");
+            }
+            Ok(())
+        })
+        .interact_text()
+        .unwrap();
+
+    let root_path = PathBuf::from(&root_dir_str);
+
+    // --- bichon-index-dir ---
+    let default_index = root_path.join("envelope");
+    let default_new_index = root_path.join("bichon-indices");
+    let index_dir_str: String = Input::with_theme(theme)
+        .with_prompt(format!(
+            "Enter --bichon-index-dir (leave blank to use default: {})",
+            style(default_index.display()).cyan()
+        ))
+        .allow_empty(true)
+        .validate_with(|input: &String| -> Result<(), &str> {
+            if input.is_empty() {
+                return Ok(());
+            }
+            let path = PathBuf::from(input);
+            if !path.is_absolute() {
+                return Err("Path must be absolute.");
+            }
+
+            if !path.exists() {
+                return Err("Directory does not exist.");
+            }
+            Ok(())
+        })
+        .interact_text()
+        .unwrap();
+
+    let index_path = if index_dir_str.is_empty() {
+        default_index
+    } else {
+        PathBuf::from(&index_dir_str)
+    };
+
+    let new_index_path = if index_dir_str.is_empty() {
+        default_new_index
+    } else {
+        PathBuf::from(&index_dir_str).join("bichon-indices")
+    };
+
+    // --- bichon-data-dir ---
+    let default_data = root_path.join("eml");
+    let default_new_data = root_path.join("bichon-storage");
+    let data_dir_str: String = Input::with_theme(theme)
+        .with_prompt(format!(
+            "Enter --bichon-data-dir (leave blank to use default: {})",
+            style(default_data.display()).cyan()
+        ))
+        .allow_empty(true)
+        .validate_with(|input: &String| -> Result<(), &str> {
+            if input.is_empty() {
+                return Ok(());
+            }
+            let path = PathBuf::from(input);
+            if !path.is_absolute() {
+                return Err("Path must be absolute.");
+            }
+            if !path.exists() {
+                return Err("Directory does not exist.");
+            }
+            Ok(())
+        })
+        .interact_text()
+        .unwrap();
+
+    let data_path = if data_dir_str.is_empty() {
+        default_data
+    } else {
+        PathBuf::from(&data_dir_str)
+    };
+
+    let new_data_path = if data_dir_str.is_empty() {
+        default_new_data
+    } else {
+        PathBuf::from(&data_dir_str).join("bichon-storage")
+    };
+
+    println!("\n{}", style("Paths to be migrated:").bold());
+    println!("----------------------------------------");
+    println!(
+        "{:<20} : {}",
+        "bichon-root-dir",
+        style(root_path.display()).cyan()
+    );
+    println!(
+        "{:<20} : {}",
+        "bichon-index-dir",
+        style(index_path.display()).cyan()
+    );
+    println!(
+        "{:<20} : {}",
+        "bichon-data-dir",
+        style(data_path.display()).cyan()
+    );
+    println!("----------------------------------------");
+
+    println!(
+        "\n{} Checking legacy v0.3.7 storage layout...",
+        style("⌛").yellow()
+    );
+
+    match is_legacy_data_layout_with_paths(&index_path, &data_path) {
+        Ok(true) => {
+            println!(
+                "{} {}",
+                style("✔").green(),
+                style("Legacy v0.3.7 Tantivy-based storage detected. Migration to v2.x is required.")
+                    .yellow()
+            );
+        }
+        Ok(false) => {
+            println!(
+                "{} {}",
+                style("✔").green(),
+                style("No legacy v0.3.7 storage layout was detected at the specified paths.").green()
+            );
+
+            println!(
+                "{}",
+                style(
+                    "The selected directories may already be using a newer storage architecture."
+                )
+                .dim()
+            );
+
+            return;
+        }
+        Err(e) => {
+            eprintln!(
+                "{} Failed to verify legacy storage layout: {:?}",
+                style("ERROR:").red().bold(),
+                e
+            );
+
+            std::process::exit(1);
+        }
+    }
+
+    println!(
+        "\n{} {}",
+        style("⚠").yellow(),
+        style(
+            "This migration is non-destructive. Existing v0.x storage files will remain unchanged."
+        )
+        .yellow()
+    );
+
+    if !Confirm::with_theme(theme)
+        .with_prompt("Ready to migrate?")
+        .default(true)
+        .interact()
+        .unwrap()
+    {
+        println!("{}", style("Migration cancelled.").dim());
+        return;
+    }
+
+    // Step 1: Migrate metadata (meta.db + mailbox.db → memdb)
+    match crate::meta::migrate_metadata(&root_path) {
+        Ok(()) => {}
+        Err(e) => {
+            eprintln!(
+                "\n{} Metadata migration failed:\n{}",
+                style("✘").red().bold(),
+                style(e).red()
+            );
+            eprintln!(
+                "{}",
+                style("Aborting migration. No changes have been made to Tantivy data.").yellow()
+            );
+            return;
+        }
+    }
+
+    println!(
+        "\n{} {}",
+        style("⌛").yellow(),
+        style("Step 2: Migrating email index and blob data...").cyan()
+    );
+
+    println!(
+        "\n{} {}",
+        style("ℹ").blue(),
+        style("Batch size controls memory usage during migration:").dim()
+    );
+    println!(
+        "  {} 1000  — ~500MB RAM  (slower, low memory)",
+        style("•").dim()
+    );
+    println!("  {} 3000  — ~1GB RAM    (recommended)", style("•").dim());
+    println!(
+        "  {} 5000  — ~2GB RAM    (faster, high memory)",
+        style("•").dim()
+    );
+    println!(
+        "  {} Note: actual memory usage depends on your average email size.",
+        style("•").yellow()
+    );
+    println!(
+        "  {}       If your mailbox contains many large attachments, use a smaller batch size.\n",
+        style(" ").dim()
+    );
+
+    let batch_size: u32 = {
+        let input: String = Input::with_theme(&ColorfulTheme::default())
+            .with_prompt("Enter batch size (affects memory usage, see notes above)")
+            .default("3000".to_string())
+            .validate_with(|s: &String| match s.trim().parse::<usize>() {
+                Ok(n) if n > 0 => Ok(()),
+                _ => Err("Please enter a valid positive number"),
+            })
+            .interact_text()
+            .unwrap_or("3000".to_string());
+        input.trim().parse::<u32>().unwrap_or(3000)
+    };
+
+    println!(
+        "{} Using batch size: {}\n",
+        style("✓").green(),
+        style(batch_size).cyan().bold()
+    );
+
+    let legacy = LegacyDirs::new(index_path.clone(), data_path.clone());
+    let total_segments = match count_eml_segments(&legacy) {
+        Ok(n) => n,
+        Err(e) => {
+            eprintln!(
+                "\n{} Failed to count EML segments:\n{:?}",
+                style("✘").red().bold(),
+                e
+            );
+            return;
+        }
+    };
+
+    if total_segments == 0 {
+        println!(
+            "{} {}",
+            style("✔").green(),
+            style("No EML segments found. Nothing to migrate.").bold()
+        );
+        return;
+    }
+
+    println!(
+        "{} EML segments to migrate: {}",
+        style("⌛").yellow(),
+        style(total_segments).cyan()
+    );
+
+    let pb = ProgressBar::new(total_segments as u64);
+    pb.set_style(
+        ProgressStyle::default_bar()
+            .template(
+                "{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta}) {msg}",
+            )
+            .unwrap()
+            .progress_chars("#>-"),
+    );
+
+    let mut writer = match NewIndexWriterV2::open(NewDirs::new(
+        new_index_path.clone(),
+        new_data_path.clone(),
+    )) {
+        Ok(w) => w,
+        Err(e) => {
+            pb.finish_with_message(format!("{}", style("Migration failed.").red()));
+            eprintln!("\n{} {:?}", style("✘").red().bold(), e);
+            return;
+        }
+    };
+
+    let mut grand_total_migrated: usize = 0;
+    let mut grand_total_skipped: usize = 0;
+
+    for seg_idx in 0..total_segments {
+        let seg_total: std::cell::Cell<usize> = std::cell::Cell::new(0);
+
+        pb.set_message(format!("Segment {}/{}", seg_idx + 1, total_segments));
+        let legacy = LegacyDirs::new(index_path.clone(), data_path.clone());
+        match do_migrate_segment_v2(
+            batch_size,
+            legacy,
+            &mut writer,
+            seg_idx,
+            |msg| {
+                if let Some(data) = msg.strip_prefix("TOTAL:") {
+                    seg_total.set(data.parse().unwrap_or(0));
+                } else if let Some(data) = msg.strip_prefix("PHASE1:") {
+                    let parts: Vec<&str> = data.split('/').collect();
+                    let scanned: usize = parts.get(0).and_then(|s| s.parse().ok()).unwrap_or(0);
+                    let total: usize = parts
+                        .get(1)
+                        .and_then(|s| s.split_once(" skipped:").map(|(n, _)| n))
+                        .and_then(|s| s.parse().ok())
+                        .unwrap_or(0);
+                    let skipped: usize = data
+                        .split_once("skipped:")
+                        .and_then(|(_, s)| s.parse().ok())
+                        .unwrap_or(0);
+                    let pct = if total > 0 {
+                        (scanned * 100) / total
+                    } else {
+                        0
+                    };
+                    pb.set_message(format!(
+                        "Segment {}/{} [scanning {}/{} skipped:{} {}%]",
+                        seg_idx + 1,
+                        total_segments,
+                        scanned,
+                        total,
+                        skipped,
+                        pct,
+                    ));
+                } else if let Some(data) = msg.strip_prefix("PROGRESS:") {
+                    let parts: Vec<&str> = data.split(':').collect();
+                    let migrated: usize = parts.get(0).and_then(|s| s.parse().ok()).unwrap_or(0);
+                    let total = seg_total.get();
+                    let pct = if total > 0 {
+                        (migrated * 100) / total
+                    } else {
+                        0
+                    };
+                    pb.set_message(format!(
+                        "Segment {}/{} [migrating {}/{} {}%]",
+                        seg_idx + 1,
+                        total_segments,
+                        migrated,
+                        total,
+                        pct,
+                    ));
+                } else if let Some(warn) = msg.strip_prefix("WARN:") {
+                    pb.println(format!("{} {}", style("⚠").yellow(), warn));
+                } else if let Some(done_data) = msg.strip_prefix("DONE:") {
+                    let parts: Vec<&str> = done_data.split(':').collect();
+                    let migrated: usize = parts.get(0).and_then(|s| s.parse().ok()).unwrap_or(0);
+                    let skipped: usize = parts.get(1).and_then(|s| s.parse().ok()).unwrap_or(0);
+                    grand_total_migrated += migrated;
+                    grand_total_skipped += skipped;
+                }
+            },
+        ) {
+            Ok(()) => {}
+            Err(e) => {
+                pb.finish_with_message(format!("{}", style("Migration failed.").red()));
+                eprintln!("\n{} {:?}", style("✘").red().bold(), e);
+                return;
+            }
+        }
+
+        pb.set_position((seg_idx + 1) as u64);
+    }
+
+    pb.set_message(style("Finalizing indexes...").dim().to_string());
+    if let Err(e) = writer.finish_writers() {
+        pb.finish_with_message(format!("{}", style("Migration failed.").red()));
+        eprintln!("\n{} {:?}", style("✘").red().bold(), e);
+        return;
+    }
+
+    pb.set_message(style("Shutting down blob engine...").dim().to_string());
+    if let Err(e) = writer.shutdown_engine() {
+        pb.finish_with_message(format!("{}", style("Migration failed.").red()));
+        eprintln!("\n{} {:?}", style("✘").red().bold(), e);
+        return;
+    }
+
+    // Write STORAGE_VERSION = 2 to mark the data as v2.x compatible
+    if let Err(e) = write_storage_version(&root_path, 2) {
+        pb.finish_with_message(format!("{}", style("Migration failed.").red()));
+        eprintln!(
+            "\n{} Failed to write STORAGE_VERSION: {:?}",
+            style("✘").red().bold(),
+            e
+        );
+        return;
+    }
+
+    pb.finish_with_message(format!(
+        "Migration finished. Total: {}, Skipped: {}",
+        grand_total_migrated, grand_total_skipped
+    ));
+
+    println!(
+        "{} {}",
+        style("✔").green(),
+        style("Migration to v2.x completed successfully!").bold()
+    );
+}
+
+/// Migrate all documents from a single EML segment to the v2.x storage layout.
+fn do_migrate_segment_v2<F>(
+    batch_size: u32,
+    legacy: LegacyDirs,
+    writer: &mut NewIndexWriterV2,
+    segment_index: usize,
+    mut on_progress: F,
+) -> BichonResult<()>
+where
+    F: FnMut(&str),
+{
+    // ── open legacy indices ────────────────────────────────────────────
+    let envelope_index = Index::open_in_dir(&legacy.envelope_dir)
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+    let eml_index = Index::open_in_dir(&legacy.eml_dir)
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+    let envelope_reader = envelope_index
+        .reader()
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+    let eml_reader = eml_index
+        .reader()
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+    let envelope_searcher = envelope_reader.searcher();
+    let eml_searcher = eml_reader.searcher();
+
+    let ef = SchemaTools::envelope_fields();
+    let mf = SchemaTools::eml_fields();
+
+    let eml_segments = eml_searcher.segment_readers();
+    let eml_segment = eml_segments.get(segment_index).ok_or_else(|| {
+        raise_error!(
+            format!(
+                "segment index {} out of range ({} segments)",
+                segment_index,
+                eml_segments.len()
+            ),
+            ErrorCode::InternalError
+        )
+    })?;
+
+    let num_docs = eml_segment.num_docs();
+    if num_docs == 0 {
+        on_progress("TOTAL:0");
+        on_progress("DONE:0:0");
+        return Ok(());
+    }
+
+    on_progress(&format!("TOTAL:{}", num_docs));
+
+    let max_doc = eml_segment.max_doc();
+    let ff = eml_segment.fast_fields();
+    let f_id_col: Column<u64> = ff.u64("id").map_err(|e| {
+        raise_error!(
+            format!("failed to open f_id fast field: {e:#?}"),
+            ErrorCode::InternalError
+        )
+    })?;
+
+    // ── Phase 1: build eid → (uid, internal_date) from envelope, then drop it ──
+    let mut envelope_map: HashMap<u64, (u32, i64)> = HashMap::with_capacity(num_docs as usize);
+
+    let mut env_scanned = 0u32;
+    let mut env_skipped = 0u32;
+    for doc_id in 0..max_doc {
+        if eml_segment.is_deleted(doc_id) {
+            continue;
+        }
+        let eid = f_id_col.values.get_val(doc_id);
+
+        let term = Term::from_field_u64(ef.f_id, eid);
+        let query = TermQuery::new(term, IndexRecordOption::Basic);
+        let hits: Vec<(_, DocAddress)> = envelope_searcher
+            .search(&query, &TopDocs::with_limit(1).order_by_score())
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        if let Some((_, addr)) = hits.first() {
+            let env_doc: TantivyDocument = envelope_searcher
+                .doc(*addr)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+            let uid = env_doc
+                .get_first(ef.f_uid)
+                .and_then(|v| v.as_u64())
+                .unwrap_or(0) as u32;
+            let internal_date = env_doc
+                .get_first(ef.f_internal_date)
+                .and_then(|v| v.as_i64())
+                .unwrap_or(0);
+            envelope_map.insert(eid, (uid, internal_date));
+            env_scanned += 1;
+        } else {
+            env_skipped += 1;
+        }
+
+        if env_scanned % 10 == 0 {
+            on_progress(&format!(
+                "PHASE1:{}/{} skipped:{}",
+                env_scanned, max_doc, env_skipped
+            ));
+        }
+    }
+
+    // Free the envelope index before the heavy EML processing.
+    drop(envelope_searcher);
+    drop(envelope_reader);
+    drop(envelope_index);
+
+    // ── Phase 2: process EML docs, streaming one at a time ─────────────
+    let mut total_migrated = 0usize;
+    let mut total_skipped = 0usize;
+
+    let mut chunk_start = 0u32;
+
+    while chunk_start < max_doc {
+        let chunk_end = (chunk_start + batch_size).min(max_doc);
+        let store_reader = eml_segment
+            .get_store_reader(2)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        for doc_id in chunk_start..chunk_end {
+            if eml_segment.is_deleted(doc_id) {
+                continue;
+            }
+
+            let eid = f_id_col.values.get_val(doc_id);
+
+            let (uid, internal_date) = match envelope_map.get(&eid) {
+                Some(v) => *v,
+                None => {
+                    on_progress(&format!("WARN: eid {} envelope not found", eid));
+                    total_skipped += 1;
+                    continue;
+                }
+            };
+
+            let eml_doc: TantivyDocument = store_reader
+                .get(doc_id)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+            let account_id = match eml_doc.get_first(mf.f_account_id).and_then(|v| v.as_u64()) {
+                Some(v) => v,
+                None => {
+                    on_progress(&format!("WARN: eid {} account_id missing", eid));
+                    total_skipped += 1;
+                    continue;
+                }
+            };
+            let mailbox_id = eml_doc
+                .get_first(mf.f_mailbox_id)
+                .and_then(|v| v.as_u64())
+                .unwrap_or(0);
+
+            let eml_bytes = match eml_doc.get_first(mf.f_eml).and_then(|v| v.as_bytes()) {
+                Some(b) => b,
+                None => {
+                    on_progress(&format!("WARN: eid {} eml bytes missing", eid));
+                    total_skipped += 1;
+                    continue;
+                }
+            };
+
+            if let Err(e) = writer.ingest(eml_bytes, account_id, mailbox_id, uid, internal_date) {
+                on_progress(&format!(
+                    "ERROR: Account {} eid {} ingest failed: {}",
+                    account_id, eid, e
+                ));
+                total_skipped += 1;
+                continue;
+            }
+
+            total_migrated += 1;
+
+            if total_migrated % 10 == 0 || total_migrated as u32 == num_docs {
+                on_progress(&format!("PROGRESS:{}:{}", total_migrated, num_docs));
+            }
+        }
+
+        drop(store_reader);
+
+        // Flush blob buffers to bichon-blob engine.
+        writer.flush_blob_buffers()?;
+
+        chunk_start = chunk_end;
+    }
+
+    on_progress(&format!("DONE:{}:{}", total_migrated, total_skipped));
+    Ok(())
+}

crates/admin/src/migrate_v1.rs

@@ -0,0 +1,460 @@
+use std::path::PathBuf;
+
+use bichon_blob::{Codec, Config, Engine};
+use bichon_core::{
+    error::{code::ErrorCode, BichonResult},
+    migrate::write_storage_version,
+    raise_error,
+};
+use console::style;
+use dialoguer::{theme::ColorfulTheme, Input};
+use fjall::{Config as FjallConfig, Database};
+use indicatif::{ProgressBar, ProgressStyle};
+
+fn hex_key_to_raw(hex_bytes: &[u8]) -> BichonResult<[u8; 32]> {
+    let hex_str = std::str::from_utf8(hex_bytes).map_err(|e| {
+        raise_error!(
+            format!("invalid UTF-8 in fjall key: {e:#?}"),
+            ErrorCode::InternalError
+        )
+    })?;
+    let mut raw = [0u8; 32];
+    hex::decode_to_slice(hex_str, &mut raw).map_err(|e| {
+        raise_error!(
+            format!("invalid hex in fjall key '{hex_str}': {e:#?}"),
+            ErrorCode::InternalError
+        )
+    })?;
+    Ok(raw)
+}
+
+fn migrate_keyspace(
+    engine: &Engine,
+    db: &Database,
+    ks_name: &str,
+    label: &str,
+    batch_size: usize,
+) -> BichonResult<u64> {
+
+    let ks = db
+        .keyspace(ks_name, || {
+            panic!("{ks_name} keyspace not found in fjall database")
+        })
+        .map_err(|e| {
+            raise_error!(
+                format!("failed to open fjall keyspace '{ks_name}': {e:#?}"),
+                ErrorCode::InternalError
+            )
+        })?;
+
+    let pb = ProgressBar::new_spinner();
+    pb.set_style(
+        ProgressStyle::with_template("{spinner:.cyan} {msg} [{elapsed_precise}]")
+            .unwrap(),
+    );
+    pb.set_message(format!("Scanning {label} blobs..."));
+
+    let mut count: u64 = 0;
+    let mut batch: Vec<([u8; 32], Vec<u8>, Codec)> = Vec::with_capacity(batch_size);
+
+    for item in ks.iter() {
+        let (key_bytes, value) = item.into_inner().map_err(|e| {
+            raise_error!(
+                format!("fjall iter error in '{ks_name}': {e:#?}"),
+                ErrorCode::InternalError
+            )
+        })?;
+
+        if value.is_empty() {
+            continue;
+        }
+        // MAX_VALUE_SIZE = 100 MB (bichon_blob::types)
+        if value.len() > 100 * 1024 * 1024 {
+            let raw_key = hex_key_to_raw(&key_bytes)?;
+            eprintln!(
+                "{}",
+                console::style(format!(
+                    "WARN: skipping oversized blob key={} ({} bytes)",
+                    hex::encode(raw_key),
+                    value.len()
+                ))
+                .yellow()
+            );
+            continue;
+        }
+        let raw_key = hex_key_to_raw(&key_bytes)?;
+        batch.push((raw_key, value.to_vec(), Codec::Zstd));
+
+        if batch.len() >= batch_size {
+            engine.put_batch(&batch).map_err(|e| {
+                raise_error!(format!("{e:#?}"), ErrorCode::InternalError)
+            })?;
+            count += batch.len() as u64;
+            pb.set_message(format!("{label}: {} blobs migrated...", count));
+            batch.clear();
+        }
+    }
+
+    if !batch.is_empty() {
+        engine.put_batch(&batch).map_err(|e| {
+            raise_error!(format!("{e:#?}"), ErrorCode::InternalError)
+        })?;
+        count += batch.len() as u64;
+    }
+
+    pb.finish_with_message(format!("{label}: {} blobs migrated", count));
+    Ok(count)
+}
+
+pub fn handle_migrate_v1(theme: &ColorfulTheme) {
+    println!(
+        "\n{}",
+        style("MIGRATION: Bichon v1.x Storage → v2.x (Fjall → bichon-blob)")
+            .bold()
+            .yellow()
+    );
+    println!(
+        "{}\n",
+        style("This migrates blob storage from the fjall engine to bichon-blob.\n\
+              Tantivy indexes and metadata (memdb) are NOT affected.")
+            .dim()
+    );
+
+    let root_dir: String = Input::with_theme(theme)
+        .with_prompt("Enter --bichon-root-dir (same value used by the old server)")
+        .validate_with(|input: &String| -> Result<(), &str> {
+            let path = PathBuf::from(input);
+            if !path.is_absolute() {
+                return Err("Path must be absolute.");
+            }
+            if !path.exists() {
+                return Err("Directory does not exist.");
+            }
+            Ok(())
+        })
+        .interact_text()
+        .unwrap();
+    let root_dir = PathBuf::from(root_dir.trim());
+
+    let data_base = {
+        let input: String = Input::with_theme(theme)
+            .with_prompt("Enter --bichon-data-dir (leave blank to use root directory)")
+            .allow_empty(true)
+            .interact_text()
+            .unwrap();
+        if input.trim().is_empty() {
+            root_dir.clone()
+        } else {
+            let path = PathBuf::from(input.trim());
+            if !path.exists() {
+                eprintln!(
+                    "{}",
+                    style(format!("Data directory does not exist: {}", path.display())).red()
+                );
+                return;
+            }
+            path
+        }
+    };
+
+    let fjall_path = data_base.join("bichon-storage");
+    let blob_path = fjall_path.join("blobs");
+
+    if !fjall_path.exists() {
+        println!(
+            "{}",
+            style(format!(
+                "Fjall database not found at '{}'. Is this really a v1.x install?",
+                fjall_path.display()
+            ))
+            .red()
+        );
+        return;
+    }
+
+    if blob_path.exists() {
+        println!(
+            "{}",
+            style(format!(
+                "Target blob directory '{}' already exists.\n\
+                 If you have already migrated, you can remove the old fjall files manually.\n\
+                 Otherwise, delete this directory and re-run the migration.",
+                blob_path.display()
+            ))
+            .yellow()
+        );
+        return;
+    }
+
+    let batch_size: usize = {
+        let input: String = Input::with_theme(theme)
+            .with_prompt("Enter batch size (affects memory usage, higher = faster but uses more RAM)")
+            .default("1000".to_string())
+            .validate_with(|s: &String| match s.trim().parse::<usize>() {
+                Ok(n) if n > 0 => Ok(()),
+                _ => Err("Please enter a valid positive number"),
+            })
+            .interact_text()
+            .unwrap_or("1000".to_string());
+        input.trim().parse::<usize>().unwrap_or(1000)
+    };
+
+    println!(
+        "{} Using batch size: {}\n",
+        style("✓").green(),
+        style(batch_size).cyan().bold()
+    );
+
+    // Open old Fjall database (read-only by nature of the iter API)
+    println!("\n{}", style("Opening fjall database...").dim());
+    let db = match Database::open(FjallConfig::new(&fjall_path)) {
+        Ok(db) => db,
+        Err(e) => {
+            println!(
+                "{}",
+                style(format!("Failed to open fjall database: {e:#?}")).red()
+            );
+            return;
+        }
+    };
+
+    // Open new bichon-blob engine
+    println!("{}", style("Initializing bichon-blob engine...").dim());
+    let mut config = Config::default();
+    config.default_codec = Codec::Zstd;
+    config.compress_threshold = 1024;
+    config.flush_interval_secs = 0;
+    config.gc_interval_secs = 0;
+
+    let engine = match Engine::open(&blob_path, config) {
+        Ok(e) => e,
+        Err(e) => {
+            println!(
+                "{}",
+                style(format!("Failed to open bichon-blob engine: {e:#?}")).red()
+            );
+            return;
+        }
+    };
+
+    // Migrate email blobs
+    let email_count = match migrate_keyspace(&engine, &db, "email", "Email", batch_size) {
+        Ok(n) => n,
+        Err(e) => {
+            println!("{}", style(format!("Email migration failed: {e:#?}")).red());
+            let _ = engine.shutdown();
+            return;
+        }
+    };
+
+    // Migrate attachment blobs
+    let attach_count =
+        match migrate_keyspace(&engine, &db, "attachments", "Attachment", batch_size) {
+            Ok(n) => n,
+            Err(e) => {
+                println!(
+                    "{}",
+                    style(format!("Attachment migration failed: {e:#?}")).red()
+                );
+                let _ = engine.shutdown();
+                return;
+            }
+        };
+
+    // Flush and shutdown
+    println!("\n{}", style("Flushing and shutting down blob engine...").dim());
+    if let Err(e) = engine.flush() {
+        println!("{}", style(format!("flush warning: {e:#?}")).yellow());
+    }
+    if let Err(e) = engine.shutdown() {
+        println!(
+            "{}",
+            style(format!("shutdown error: {e:#?}")).red()
+        );
+        return;
+    }
+
+    // Write STORAGE_VERSION = 2
+    if let Err(e) = write_storage_version(&root_dir, 2) {
+        println!(
+            "{}",
+            style(format!("Failed to write STORAGE_VERSION: {e:#?}")).red()
+        );
+        return;
+    }
+
+    println!(
+        "\n{}",
+        style(format!(
+            "Migration complete!\n  Email blobs: {}\n  Attachment blobs: {}\n  Total: {}\n\n\
+             The old fjall database at '{}' is no longer used.\n\
+             You may delete it to free disk space after verifying everything works.",
+            email_count,
+            attach_count,
+            email_count + attach_count,
+            fjall_path.display()
+        ))
+        .green()
+        .bold()
+    );
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use bichon_core::utils::compute_content_hash;
+
+    // ── hex_key_to_raw ────────────────────────────────────────────────
+
+    #[test]
+    fn hex_key_to_raw_valid() {
+        // "hello" blake3 hex = 64 chars
+        let hash_hex = compute_content_hash(b"hello");
+        assert_eq!(hash_hex.len(), 64);
+
+        let raw = hex_key_to_raw(hash_hex.as_bytes()).unwrap();
+        // Decoding 64 hex chars → 32 bytes
+        assert_eq!(raw.len(), 32);
+        // Round-trip: raw → hex should match original
+        assert_eq!(hex::encode(raw), hash_hex);
+    }
+
+    #[test]
+    fn hex_key_to_raw_invalid_utf8() {
+        // 0xFF is not valid UTF-8
+        let invalid = vec![0xFFu8; 64];
+        let err = hex_key_to_raw(&invalid).unwrap_err();
+        assert!(err.to_string().contains("invalid UTF-8"));
+    }
+
+    #[test]
+    fn hex_key_to_raw_invalid_hex() {
+        // "zz" is valid UTF-8 but not valid hex
+        let invalid = b"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz";
+        let err = hex_key_to_raw(invalid).unwrap_err();
+        assert!(err.to_string().contains("invalid hex"));
+    }
+
+    #[test]
+    fn hex_key_to_raw_wrong_length() {
+        let short = b"abcd";
+        let err = hex_key_to_raw(short).unwrap_err();
+        assert!(err.to_string().contains("invalid hex"));
+    }
+
+    #[test]
+    fn hex_key_to_raw_different_content() {
+        let a = hex_key_to_raw(compute_content_hash(b"a").as_bytes()).unwrap();
+        let b = hex_key_to_raw(compute_content_hash(b"b").as_bytes()).unwrap();
+        assert_ne!(a, b);
+    }
+
+    // ── migrate_keyspace integration ──────────────────────────────────
+
+    #[test]
+    fn migrate_keyspace_end_to_end() {
+        let tmp = tempfile::tempdir().unwrap();
+        let fjall_path = tmp.path().join("fjall");
+        let blob_path = tmp.path().join("blobs");
+
+        use fjall::KeyspaceCreateOptions;
+
+        // --- Setup: create a Fjall database with test blobs ---
+        let fjall_db = Database::open(FjallConfig::new(&fjall_path)).unwrap();
+        let ks = fjall_db
+            .keyspace("test_ks", KeyspaceCreateOptions::default)
+            .unwrap();
+
+        // Insert test blobs using hex string keys (matching v1.x convention)
+        let mut expected: Vec<(String, Vec<u8>)> = Vec::new();
+        for i in 0..10 {
+            let data = format!("blob data {}", i).into_bytes();
+            let hash = compute_content_hash(&data);
+            ks.insert(hash.as_bytes(), data.clone()).unwrap();
+            expected.push((hash, data));
+        }
+
+        // --- Setup: create bichon-blob engine and run migration ---
+        {
+            let mut config = Config::default();
+            config.flush_interval_secs = 0;
+            config.gc_interval_secs = 0;
+            let engine = Engine::open(&blob_path, config).unwrap();
+
+            let count = migrate_keyspace(&engine, &fjall_db, "test_ks", "Test", 100).unwrap();
+            assert_eq!(count, expected.len() as u64);
+
+            engine.flush().unwrap();
+            engine.shutdown().unwrap();
+            // engine dropped here → LOCK released
+        }
+
+        // --- Verify: re-open engine and check all blobs ---
+        let mut config = Config::default();
+        config.flush_interval_secs = 0;
+        config.gc_interval_secs = 0;
+        let engine2 = Engine::open(&blob_path, config).unwrap();
+
+        for (hex_hash, expected_data) in &expected {
+            let mut raw_key = [0u8; 32];
+            hex::decode_to_slice(hex_hash, &mut raw_key).unwrap();
+            let got = engine2.get(&raw_key).unwrap();
+            assert_eq!(
+                got.as_deref(),
+                Some(expected_data.as_slice()),
+                "mismatch for key {}",
+                hex_hash
+            );
+        }
+
+        // Verify non-existent key returns None
+        let fake_hash = compute_content_hash(b"nonexistent");
+        let mut fake_key = [0u8; 32];
+        hex::decode_to_slice(&fake_hash, &mut fake_key).unwrap();
+        assert!(engine2.get(&fake_key).unwrap().is_none());
+
+        engine2.shutdown().unwrap();
+    }
+
+    #[test]
+    fn migrate_empty_keyspace() {
+        let tmp = tempfile::tempdir().unwrap();
+        let fjall_path = tmp.path().join("fjall");
+        let blob_path = tmp.path().join("blobs");
+
+        let fjall_db = Database::open(FjallConfig::new(&fjall_path)).unwrap();
+        let _ks = fjall_db
+            .keyspace("empty_ks", fjall::KeyspaceCreateOptions::default)
+            .unwrap();
+
+        let mut config = Config::default();
+        config.flush_interval_secs = 0;
+        config.gc_interval_secs = 0;
+        let engine = Engine::open(&blob_path, config).unwrap();
+
+        let count = migrate_keyspace(&engine, &fjall_db, "empty_ks", "Empty", 100).unwrap();
+        assert_eq!(count, 0);
+
+        engine.shutdown().unwrap();
+    }
+
+    #[test]
+    fn hex_key_roundtrip_with_real_content_hash() {
+        // Simulate the exact data flow from v1.x to v2.x
+        let eml_data = b"From: sender@example.com\r\nSubject: Test\r\n\r\nHello world";
+        let hash_hex = compute_content_hash(eml_data); // 64-char hex string
+
+        // v1.x: key stored as hash_hex.as_bytes()
+        let fjall_key = hash_hex.as_bytes().to_vec();
+        assert_eq!(fjall_key.len(), 64);
+
+        // Migration: hex decode → raw 32 bytes
+        let raw_key = hex_key_to_raw(&fjall_key).unwrap();
+        assert_eq!(raw_key.len(), 32);
+
+        // v2.x: engine.put(raw_key, data)
+        // Verify round-trip: raw_key → hex → compare
+        let hex_roundtrip = hex::encode(raw_key);
+        assert_eq!(hex_roundtrip, hash_hex);
+    }
+}

crates/admin/src/reset.rs

@@ -0,0 +1,248 @@
+use std::path::{Path, PathBuf};
+
+use bichon_core::{
+    admin::meta::{find_admin, open_database, update_admin_password},
+    utils::encrypt::internal_decrypt_string,
+};
+use console::{style, Emoji};
+use dialoguer::{theme::ColorfulTheme, Confirm, Input, Password, Select};
+
+pub fn handle_reset_password(theme: &ColorfulTheme) {
+    let root_dir_str: String = Input::with_theme(theme)
+        .with_prompt("Enter the absolute path for 'bichon_root_dir'")
+        .validate_with(|input: &String| -> Result<(), &str> {
+            let path = Path::new(input);
+            if !path.is_absolute() {
+                return Err("Path must be absolute.");
+            }
+            if !path.exists() {
+                return Err("Directory does not exist.");
+            }
+            let memdb_dir = path.join("memdb");
+            if !memdb_dir.exists() || !memdb_dir.is_dir() {
+                return Err("Invalid directory: 'memdb' data directory not found.");
+            }
+            Ok(())
+        })
+        .interact_text()
+        .unwrap();
+
+    let root_path = PathBuf::from(&root_dir_str);
+    let database = open_database(&root_path.join("memdb")).unwrap_or_else(|e| {
+        eprintln!(
+            "\n{} Failed to open database.",
+            style("ERROR:").red().bold()
+        );
+        eprintln!("Details: {:?}", e);
+        std::process::exit(1);
+    });
+
+    let admin = find_admin(&database);
+
+    match admin {
+        Ok(Some(user)) => {
+            println!("\n{}", style("Admin user found:").green().bold());
+            println!("----------------------------------------");
+            println!("{:<12} : {}", "Username", style(&user.username).cyan());
+            println!("{:<12} : {}", "Email", style(&user.email).cyan());
+        }
+        Ok(None) => {
+            println!(
+                "\n{}",
+                style("ERROR: No admin user found in the database.")
+                    .red()
+                    .bold()
+            );
+            println!("Please ensure the system has been initialized correctly.");
+            std::process::exit(1);
+        }
+        Err(e) => {
+            eprintln!(
+                "\n{} Failed to query admin user.",
+                style("ERROR:").red().bold()
+            );
+            eprintln!("Details: {:?}", e);
+            std::process::exit(1);
+        }
+    }
+
+    let encryption_key = loop {
+        let auth_methods = vec![
+            "Enter encryption password manually",
+            "Read from password file",
+        ];
+        let method = Select::with_theme(theme)
+            .with_prompt("How would you like to provide the database encryption key?")
+            .items(&auth_methods)
+            .interact()
+            .unwrap();
+
+        let raw_key = if method == 0 {
+            Input::with_theme(theme)
+                .with_prompt("Enter Encryption Password")
+                .interact()
+                .unwrap()
+        } else {
+            let file_path: String = Input::with_theme(theme)
+                .with_prompt("Enter path to encryption password file")
+                .interact_text()
+                .unwrap();
+
+            match std::fs::read_to_string(&file_path) {
+                Ok(content) => content.trim().to_string(),
+                Err(e) => {
+                    println!("{}: {}", style("Failed to read file").red(), e);
+                    continue;
+                }
+            }
+        };
+
+        if raw_key.is_empty() {
+            println!("{}", style("Key cannot be empty.").red());
+            continue;
+        }
+
+        let prompt_message = format!(
+            "Encryption key loaded: [ {} ]\n\n  \
+                {}: This key must match the database encryption key used by the server.\n  \
+                It corresponds to these settings in your service:\n    \
+                - Arguments: {} or {}\n    \
+                - Envs:      {} or {}\n\n  \
+                Do you want to continue?",
+            style(&raw_key).cyan().bold(),
+            style("IMPORTANT").yellow().bold(),
+            style("--bichon_encrypt_password").italic(),
+            style("--bichon_encrypt_password_file").italic(),
+            style("BICHON_ENCRYPT_PASSWORD").green(),
+            style("BICHON_ENCRYPT_PASSWORD_FILE").green()
+        );
+
+        if Confirm::with_theme(theme)
+            .with_prompt(prompt_message)
+            .default(true)
+            .interact()
+            .unwrap()
+        {
+            break raw_key;
+        }
+    };
+
+    let admin = find_admin(&database);
+
+    match admin {
+        Ok(Some(user)) => {
+            println!("----------------------------------------");
+            println!("{:<12} : {}", "Username", style(&user.username).cyan());
+            println!("{:<12} : {}", "Email", style(&user.email).cyan());
+
+            let pwd_display = match &user.password {
+                Some(p) => {
+                    let password = match internal_decrypt_string(&encryption_key, p) {
+                        Ok(p) => p,
+                        Err(e) => {
+                            println!("\n{}", style("ERROR: Decryption Failed").red().bold());
+                            println!(
+                                "{}",
+                                style("The provided encryption key is incorrect or invalid for this database.").yellow()
+                            );
+                            println!(
+                                "{} Please verify your {} or the {} you provided.",
+                                style("➔").cyan(),
+                                style("encryption password").bold(),
+                                style("key file").bold()
+                            );
+                            eprintln!("\nTechnical details: {:?}", e);
+                            std::process::exit(1);
+                        }
+                    };
+                    style(password).yellow().to_string()
+                }
+                None => style("None (No password set)").dim().italic().to_string(),
+            };
+
+            println!("{:<12} : {}", "Password", pwd_display);
+            println!("----------------------------------------");
+
+            if !dialoguer::Confirm::with_theme(theme)
+                .with_prompt(format!(
+                    "Do you want to reset the password for '{}'?",
+                    user.username
+                ))
+                .interact()
+                .unwrap()
+            {
+                println!("Operation cancelled.");
+                return;
+            }
+        }
+        Ok(None) => {
+            println!(
+                "\n{}",
+                style("ERROR: No admin user found in the database.")
+                    .red()
+                    .bold()
+            );
+            println!("Please ensure the system has been initialized correctly.");
+            std::process::exit(1);
+        }
+        Err(e) => {
+            eprintln!(
+                "\n{} Failed to query admin user.",
+                style("ERROR:").red().bold()
+            );
+            eprintln!("Details: {:?}", e);
+            std::process::exit(1);
+        }
+    }
+
+    println!(
+        "\n{}",
+        style("TARGET: Reset password for user 'admin'")
+            .yellow()
+            .bold()
+    );
+
+    let new_login_password = Password::with_theme(theme)
+        .with_prompt("Enter new Admin Login Password")
+        .with_confirmation("Repeat password to confirm", "Passwords do not match!")
+        .interact()
+        .unwrap();
+
+    if !Confirm::with_theme(theme)
+        .with_prompt("Proceed with database update?")
+        .interact()
+        .unwrap()
+    {
+        return;
+    }
+
+    println!("\n{} {}", style("⌛").yellow(), "Updating database...");
+
+    match update_admin_password(&database, new_login_password, &encryption_key) {
+        Ok(_) => {
+            println!(
+                "\n{} {}",
+                Emoji("✨", "*"),
+                style("Success! Admin password has been updated.")
+                    .green()
+                    .bold()
+            );
+            println!(
+                "{}",
+                style("You can now log in with the new password.").dim()
+            );
+        }
+        Err(e) => {
+            println!(
+                "\n{}",
+                style("ERROR: Failed to update database").red().bold()
+            );
+            eprintln!(
+                "{} Could not save the new password to the database.",
+                style("➔").cyan()
+            );
+            eprintln!("\nDetails: {:?}", e);
+            std::process::exit(1);
+        }
+    }
+}

crates/blob/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "bichon-blob"
+version = "0.1.0"
+edition = "2021"
+description = "Embedded KV storage engine for email"
+
+[dependencies]
+crc32fast = "1.4"
+zstd = "0.13"
+lz4_flex = "0.13.1"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+bincode = { package = "bincode_reloaded", version = "3.1.10", default-features = false, features = ["serde", "alloc"] }
+tracing = "0.1"
+thiserror = "2"
+redb = "4.1"
+fs2 = "0.4"
+
+[dev-dependencies]
+tempfile = "3"
+rand = "0.10.2"
+criterion = { version = "0.6", features = ["html_reports"] }
+
+[[bench]]
+name = "benchmark"
+harness = false

crates/blob/README.md

@@ -0,0 +1,177 @@
+# bichon-blob
+
+Content-addressable blob store for [Bichon](https://github.com/rustmailer/bichon) email archival.
+
+All data is keyed by a 32-byte content hash — identical content is stored only once. The caller (Bichon) strips attachments from emails and stores the "holed" raw email + attachments into this store, tracking reference counts in its own schema.
+
+## On-disk layout
+
+```
+<root>/
+├── meta.bin              # global metadata (bincode + CRC32)
+├── index.redb            # key → (segment_id, offset, size) index (redb B-tree)
+├── segments/
+│   ├── 00000001.seg      # append-only segment files (≤ 1 GB each)
+│   ├── 00000002.seg
+│   └── ...
+```
+
+### Segment entry format (50-byte fixed header + variable data)
+
+```
+magic(4)  crc32(4)  flags(1)  codec(1)  key(32)  raw_size(4)  data_size(4)  data(*)
+```
+
+- `magic`: `0xB3DB_0001` — entry boundary validation
+- `crc32`: covers everything after this field
+- `flags`: `0` = live, `1` = tombstone
+- `codec`: `0` = none, `1` = Zstd, `2` = Lz4
+- `key`: 32-byte content hash (BLAKE3 / SHA-256)
+- `raw_size`: original uncompressed size
+- `data_size`: on-disk data size (after compression)
+
+### Index store
+
+The key → (segment_id, offset, data_size, flags) mapping is stored in a single [redb](https://github.com/cberner/redb) database (`index.redb`). redb provides:
+
+- **B-tree + mmap**: O(log N) point lookups with zero heap allocation — pages are faulted in on demand.
+- **ACID transactions**: every index write is durable and atomic.
+- **Crash recovery**: handled transparently by redb's WAL — no manual reload or rebuild logic.
+- **O(1) startup**: only the B-tree root page is read at open time.
+
+Records are stored as fixed-size 56-byte blobs, each carrying an internal CRC32 checksum.
+
+## Read path
+
+```
+get(key)
+  → index_store.get(key)           # redb B-tree lookup, zero-copy
+  → IndexRecord CRC32 verify       # (segment_id, offset, data_size, flags)
+  → pread entry from segment file
+  → entry CRC32 verify → decompress → return value
+```
+
+The index record and the segment entry carry independent CRC32 checksums. Corruption in one record or entry is contained — it never affects other keys.
+
+## Write path
+
+```
+put(key, value, codec)
+  → compress value (Zstd/Lz4 if ≥ 4 KB, else store raw)
+  → append entry to active segment file
+  → insert IndexRecord into redb (single write txn)
+  → update metadata (indexed_up_to_offset)
+  → if segment ≥ 1 GB → seal it, create new segment
+```
+
+## Delete
+
+Deletes are **tombstones** — an entry with `flags=1` and empty data is appended to the active segment. Before writing the tombstone, the existing index record is consulted to increment `deleted_bytes` on the **original** segment (the one that holds the live data). This drives the GC threshold.
+
+```
+delete(key)
+  → index_store.get(key) → find original (segment_id, data_size)
+  → original_segment.deleted_bytes += data_size
+  → recompute deleted_ratio on original segment
+  → append tombstone entry to active segment
+  → insert tombstone IndexRecord into redb
+  → index_store.get(key) now returns None
+```
+
+## GC
+
+**Segment GC:** two-phase, driven by the `deleted_ratio` tracked per segment.
+
+### Trigger
+
+- **Background**: the `blob-gc` thread wakes up every `gc_interval_secs` (default 300s), checks whether any sealed segment's `deleted_ratio ≥ gc_deleted_ratio` (default 0.30), and runs GC on the worst segment if so.
+- **Manual**: `engine.gc_if_needed()` or `engine.gc()`.
+
+### Phase 1 — scan & compact (read-only, no write lock)
+
+1. Pick the sealed segment with the highest `deleted_ratio`.
+2. Scan only that segment's entries.
+3. For each entry, ask the index: "is this entry still the latest version for its key?"
+   - If the index points to this exact `(segment_id, offset)` → **keep**, write to a temp segment file.
+   - If the index points elsewhere (overwritten by a later segment, or a tombstone) → **skip** (stale).
+4. Fsync the temp file.
+
+Phase 1 holds only the read lock — `put` / `delete` continue uninterrupted.
+
+### Phase 2 — commit & update index (write lock)
+
+1. Atomically rename the temp file over the original segment.
+2. Batch-insert new `IndexRecord`s (now at new offsets) into redb. Old records with the same key are naturally overwritten.
+3. Reset the segment's `deleted_bytes` and `deleted_ratio` to zero.
+4. Persist metadata.
+
+Phase 2 holds the write lock, but is fast — no full segment scan, no full index rebuild.
+
+## Data integrity
+
+Every record on disk is independently checksummed:
+
+| Layer | Format | Protection |
+|---|---|---|
+| Segment entry | 50-byte header + data | CRC32 covers all fields + data |
+| Index record | 56 bytes | CRC32 covers key + segment_id + offset + data_size + flags |
+| Global metadata | bincode blob | CRC32 + version header |
+
+Corruption is **contained** — a bad segment entry or index record produces an error for that key only. Recovery and GC skip corrupt records (with a warning) rather than aborting. The index is backed by redb's B-tree which maintains its own internal integrity.
+
+## Crash recovery
+
+- Temp files from interrupted GC are cleaned up on open.
+- Any segment data beyond `indexed_up_to_offset` is scanned and inserted into the index.
+- Partial writes at the tail of a segment (detected via CRC32 mismatch near EOF) are truncated.
+- redb's WAL ensures the index is always consistent — no manual reload or rebuild needed.
+
+## Background threads
+
+Set `Config.flush_interval_secs` and `Config.gc_interval_secs` to positive values to enable periodic background work:
+
+| Thread | Config | Default | What it does |
+|---|---|---|---|
+| `blob-flush` | `flush_interval_secs` | `0` (off) | Fsync the active segment and save metadata |
+| `blob-gc` | `gc_interval_secs` | `0` (off) | Check deleted-ratio, compact one segment if needed |
+
+The two threads are independent — a long GC run never blocks fsync.
+
+## Config
+
+| Field | Default | Notes |
+|---|---|---|
+| `compress_threshold` | 4096 | Bytes; smaller values stored uncompressed |
+| `default_codec` | Zstd | Also supports Lz4 |
+| `compression_level` | 0 | Zstd compression level |
+| `gc_deleted_ratio` | 0.30 | Trigger GC when a sealed segment exceeds this |
+| `flush_interval_secs` | 0 | 0 = disabled; ≥ 5 for periodic background fsync |
+| `gc_interval_secs` | 0 | 0 = disabled; ≥ 10 for periodic background GC |
+
+## Basic usage
+
+```rust
+use bichon_blob::{Codec, Config, Engine};
+
+let engine = Engine::open(path, Config::default())?;
+
+// Store
+let hash = blake3::hash(b"email body").into();
+engine.put(hash, b"email body", Codec::Zstd)?;
+
+// Retrieve
+let value = engine.get(&hash)?; // Some(Vec<u8>) or None
+
+// Delete (caller must track refcounts)
+engine.delete(&hash)?;
+
+// Batch operations
+engine.put_batch(&[(hash1, data1, Codec::Zstd), (hash2, data2, Codec::Lz4)])?;
+engine.delete_batch(&[hash1, hash2])?;
+
+// GC
+engine.gc_if_needed()?;
+
+// Clean shutdown
+engine.shutdown()?;
+```

crates/blob/benches/benchmark.rs

@@ -0,0 +1,293 @@
+use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput};
+use std::time::Duration;
+use tempfile::TempDir;
+
+use bichon_blob::{Codec, Config, Engine};
+
+fn make_key(seed: u64) -> [u8; 32] {
+    let mut key = [0u8; 32];
+    key[0..8].copy_from_slice(&seed.to_le_bytes());
+    key
+}
+
+fn make_value(size: usize) -> Vec<u8> {
+    let mut v = Vec::with_capacity(size);
+    let pattern = b"The quick brown fox jumps over the lazy dog. ";
+    while v.len() < size {
+        let rem = size - v.len();
+        let n = rem.min(pattern.len());
+        v.extend_from_slice(&pattern[..n]);
+    }
+    v
+}
+
+pub fn bench_write_small(c: &mut Criterion) {
+    let mut group = c.benchmark_group("write");
+    group.throughput(Throughput::Elements(1));
+    group.measurement_time(Duration::from_secs(10));
+
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let value = make_value(1024); // 1 KB
+    let mut counter = 0u64;
+
+    group.bench_function("1KB", |b| {
+        b.iter_batched(
+            || {
+                counter += 1;
+                (make_key(counter), value.clone())
+            },
+            |(key, val)| {
+                engine.put(key, &val, Codec::Zstd).unwrap()
+            },
+            BatchSize::SmallInput,
+        )
+    });
+    group.finish();
+}
+
+pub fn bench_write_medium(c: &mut Criterion) {
+    let mut group = c.benchmark_group("write");
+    group.throughput(Throughput::Bytes(64 * 1024));
+    group.measurement_time(Duration::from_secs(10));
+
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let value = make_value(64 * 1024); // 64 KB
+    let mut counter = 0u64;
+
+    group.bench_function("64KB", |b| {
+        b.iter_batched(
+            || {
+                counter += 1;
+                (make_key(counter), value.clone())
+            },
+            |(key, val)| {
+                engine.put(key, &val, Codec::Zstd).unwrap()
+            },
+            BatchSize::SmallInput,
+        )
+    });
+    group.finish();
+}
+
+pub fn bench_write_large(c: &mut Criterion) {
+    let mut group = c.benchmark_group("write");
+    group.throughput(Throughput::Bytes(1024 * 1024));
+    group.measurement_time(Duration::from_secs(15));
+
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let value = make_value(1024 * 1024); // 1 MB
+    let mut counter = 0u64;
+
+    group.bench_function("1MB", |b| {
+        b.iter_batched(
+            || {
+                counter += 1;
+                (make_key(counter), value.clone())
+            },
+            |(key, val)| {
+                engine.put(key, &val, Codec::Zstd).unwrap()
+            },
+            BatchSize::SmallInput,
+        )
+    });
+    group.finish();
+}
+
+pub fn bench_read_hot(c: &mut Criterion) {
+    let mut group = c.benchmark_group("read");
+    group.throughput(Throughput::Elements(1));
+    group.measurement_time(Duration::from_secs(10));
+
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    // Pre-populate: 10 keys
+    let value = make_value(4096);
+    for i in 0..10u64 {
+        engine
+            .put(make_key(i), &value, Codec::Zstd)
+            .unwrap();
+    }
+
+    let mut counter = 0u64;
+    group.bench_function("hot", |b| {
+        b.iter(|| {
+            let key = make_key(counter % 10);
+            counter += 1;
+            std::hint::black_box(engine.get(&key).unwrap());
+        })
+    });
+    group.finish();
+}
+
+pub fn bench_read_cold(c: &mut Criterion) {
+    let mut group = c.benchmark_group("read");
+    group.throughput(Throughput::Elements(1));
+    group.measurement_time(Duration::from_secs(10));
+
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let value = make_value(4096);
+    // Write 5000 keys across all 256 buckets — mmap page faults will occur
+    for i in 0..5000u64 {
+        engine
+            .put(make_key(i), &value, Codec::Zstd)
+            .unwrap();
+    }
+
+    let mut counter = 0u64;
+    group.bench_function("cold", |b| {
+        b.iter(|| {
+            let key = make_key(counter % 5000);
+            counter += 1;
+            std::hint::black_box(engine.get(&key).unwrap());
+        })
+    });
+    group.finish();
+}
+
+pub fn bench_read_large_value(c: &mut Criterion) {
+    let mut group = c.benchmark_group("read");
+    group.throughput(Throughput::Bytes(1024 * 1024));
+    group.measurement_time(Duration::from_secs(10));
+
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let value = make_value(1024 * 1024); // 1 MB
+    for i in 0..5u64 {
+        engine
+            .put(make_key(i), &value, Codec::Zstd)
+            .unwrap();
+    }
+
+    let mut counter = 0u64;
+    group.bench_function("1MB", |b| {
+        b.iter(|| {
+            let key = make_key(counter % 5);
+            counter += 1;
+            std::hint::black_box(engine.get(&key).unwrap());
+        })
+    });
+    group.finish();
+}
+
+pub fn bench_delete(c: &mut Criterion) {
+    let mut group = c.benchmark_group("delete");
+    group.throughput(Throughput::Elements(1));
+    group.measurement_time(Duration::from_secs(10));
+
+    group.bench_function("delete", |b| {
+        let dir = TempDir::new().unwrap();
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+        let value = make_value(4096);
+        let mut counter = 0u64;
+
+        b.iter_batched(
+            || {
+                counter += 1;
+                let key = make_key(counter);
+                engine.put(key, &value, Codec::Zstd).unwrap();
+                key
+            },
+            |key| {
+                engine.delete(&key).unwrap();
+            },
+            BatchSize::SmallInput,
+        )
+    });
+    group.finish();
+}
+
+pub fn bench_mixed_workload(c: &mut Criterion) {
+    let mut group = c.benchmark_group("mixed");
+    group.throughput(Throughput::Elements(1));
+    group.measurement_time(Duration::from_secs(15));
+
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    // Pre-populate with 500 entries
+    let value = make_value(8192);
+    for i in 0..500u64 {
+        engine
+            .put(make_key(i), &value, Codec::Zstd)
+            .unwrap();
+    }
+
+    let mut counter: u64 = 500;
+    group.bench_function("80w_15r_5d", |b| {
+        b.iter(|| {
+            counter += 1;
+            let op = counter % 100;
+            match op {
+                0..=79 => {
+                    let key = make_key(counter);
+                    let val = make_value(4096);
+                    engine.put(key, &val, Codec::Zstd).unwrap();
+                }
+                80..=94 => {
+                    std::hint::black_box(
+                        engine.get(&make_key(counter % 500)).unwrap(),
+                    );
+                }
+                _ => {
+                    if counter % 2 == 0 {
+                        let key = make_key(counter % 500);
+                        let _ = engine.delete(&key);
+                    }
+                }
+            }
+        })
+    });
+    group.finish();
+}
+
+pub fn bench_gc(c: &mut Criterion) {
+    let mut group = c.benchmark_group("gc");
+    group.measurement_time(Duration::from_secs(30));
+    group.sample_size(10);
+
+    group.bench_function("gc_30pct_deleted", |b| {
+        let dir = TempDir::new().unwrap();
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+        let value = make_value(200_000);
+        let n = 1200u64;
+        for i in 0..n {
+            engine
+                .put(make_key(i), &value, Codec::None)
+                .unwrap();
+        }
+        for i in (0..n).step_by(3) {
+            engine.delete(&make_key(i)).unwrap();
+        }
+
+        b.iter(|| {
+            engine.gc().unwrap();
+        })
+    });
+    group.finish();
+}
+
+criterion_group!(
+    benches,
+    bench_write_small,
+    bench_write_medium,
+    bench_write_large,
+    bench_read_hot,
+    bench_read_cold,
+    bench_read_large_value,
+    bench_delete,
+    bench_mixed_workload,
+    bench_gc,
+);
+criterion_main!(benches);

crates/blob/examples/email_archive.rs

@@ -0,0 +1,125 @@
+/// bichon-blob usage example: email archival with content-addressable storage.
+///
+/// This example simulates a mail archival system where multiple accounts
+/// may receive the same email (e.g. CC'd or forwarded).  The blob store
+/// deduplicates by content hash, and the upper application layer tracks
+/// which accounts reference each hash.
+///
+/// Run: cargo run --example email_archive
+
+use std::collections::HashMap;
+
+use bichon_blob::{Codec, Config, Engine};
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // ── Setup ──────────────────────────────────────────────────────────
+
+    let store_path = std::path::Path::new("target/example_blob_store");
+    let _ = std::fs::remove_dir_all(store_path); // clean up from previous run
+    let engine = Engine::open(store_path, Config::default())?;
+
+    println!("Store opened at: {:?}\n", store_path);
+
+    // ── Simulated upper-layer reference tracker ────────────────────────
+    let mut refs: HashMap<String, Vec<[u8; 32]>> = HashMap::new();
+
+    // ── 1. Store emails ────────────────────────────────────────────────
+
+    // Simulate three emails arriving.  Email #2 is a newsletter that
+    // both alice and bob received — identical content, same hash.
+    let emails = vec![
+        ("alice", "Welcome to Bichon Mail!"),
+        ("alice", "Weekly Newsletter: Rust Edition"),
+        ("bob", "Weekly Newsletter: Rust Edition"), // same content as above
+    ];
+
+    for (account, body) in &emails {
+        let hash = mock_content_hash(body.as_bytes());
+        let account_refs = refs.entry(account.to_string()).or_default();
+
+        // Check if this content already exists in the blob store
+        if engine.exists(&hash)? {
+            println!(
+                "[dedup] {}: hash {:02x?}... already stored, skipping",
+                account,
+                &hash[..4]
+            );
+        } else {
+            engine.put(hash, body.as_bytes(), Codec::Zstd)?;
+            println!(
+                "[store] {}: hash {:02x?}..., {} bytes",
+                account,
+                &hash[..4],
+                body.len()
+            );
+        }
+
+        account_refs.push(hash);
+    }
+
+    println!();
+
+    // ── 2. Read back emails ────────────────────────────────────────────
+
+    let hash = mock_content_hash(b"Weekly Newsletter: Rust Edition");
+    let stored = engine.get(&hash)?;
+    println!(
+        "Read newsletter: {:?}",
+        stored.map(|v| String::from_utf8_lossy(&v).to_string())
+    );
+
+    // ── 3. Batch store attachments ─────────────────────────────────────
+
+    let attachments: Vec<([u8; 32], Vec<u8>, Codec)> = (0..10)
+        .map(|i| {
+            let body = format!("Attachment #{}: {}", i, "X".repeat(5000));
+            let hash = mock_content_hash(body.as_bytes());
+            (hash, body.into_bytes(), Codec::Zstd)
+        })
+        .collect();
+
+    engine.put_batch(&attachments)?;
+    println!("\nBatch-stored {} attachments", attachments.len());
+
+    // ── 4. Stats ───────────────────────────────────────────────────────
+
+    let stats = engine.stats()?;
+    println!(
+        "Stats: {} keys, {} bytes, {} segments",
+        stats.total_keys, stats.total_bytes, stats.segment_count
+    );
+
+    // ── 5. Delete an email (simulating: last reference removed) ─────────
+
+    // In production, before calling engine.delete(), you'd check:
+    //   SELECT COUNT(*) FROM email_refs WHERE content_hash = ? AND account_id != ?
+    // If count == 0, it's safe to delete from blob.
+
+    let welcome_hash = mock_content_hash(b"Welcome to Bichon Mail!");
+    engine.delete(&welcome_hash)?;
+    println!("\nDeleted welcome email, still exists? {}", engine.exists(&welcome_hash)?);
+
+    // ── 6. Shutdown ────────────────────────────────────────────────────
+
+    engine.shutdown()?;
+    println!("\nClean shutdown complete.");
+
+    Ok(())
+}
+
+/// In production, use BLAKE3 or SHA-256.
+/// Here we use a trivial hash for demonstration.
+fn mock_content_hash(data: &[u8]) -> [u8; 32] {
+    use std::collections::hash_map::DefaultHasher;
+    use std::hash::{Hash, Hasher};
+
+    let mut hasher = DefaultHasher::new();
+    data.hash(&mut hasher);
+    let h = hasher.finish();
+
+    let mut key = [0u8; 32];
+    key[0..8].copy_from_slice(&h.to_le_bytes());
+    // Mix in the length so different-sized content gets different hashes
+    key[8..16].copy_from_slice(&(data.len() as u64).to_le_bytes());
+    key
+}

crates/blob/src/bucket.rs

@@ -0,0 +1,257 @@
+use std::path::Path;
+
+use redb::{Database, ReadableTable, TableDefinition};
+use redb::ReadableDatabase;
+
+use crate::error::Result;
+use crate::types::INDEX_RECORD_SIZE;
+
+// ── IndexRecord ──────────────────────────────────────────────────────────────
+
+/// On-disk format: 52 bytes per record + 4 bytes CRC = 56 bytes total.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct IndexRecord {
+    pub key: [u8; 32],
+    pub segment_id: u32,
+    pub offset: u64,
+    pub data_size: u32,
+    pub flags: u8,
+}
+
+impl IndexRecord {
+    pub fn new(key: [u8; 32], segment_id: u32, offset: u64, data_size: u32, flags: u8) -> Self {
+        Self {
+            key,
+            segment_id,
+            offset,
+            data_size,
+            flags,
+        }
+    }
+
+    pub fn is_tombstone(&self) -> bool {
+        self.flags == 1
+    }
+
+    pub fn encode(&self) -> [u8; INDEX_RECORD_SIZE] {
+        let mut buf = [0u8; INDEX_RECORD_SIZE];
+        buf[0..32].copy_from_slice(&self.key);
+        buf[32..36].copy_from_slice(&self.segment_id.to_le_bytes());
+        buf[36..44].copy_from_slice(&self.offset.to_le_bytes());
+        buf[44..48].copy_from_slice(&self.data_size.to_le_bytes());
+        buf[48] = self.flags;
+        let crc = crate::checksum::crc32(&buf[..52]);
+        buf[52..56].copy_from_slice(&crc.to_le_bytes());
+        buf
+    }
+
+    pub fn decode(buf: &[u8; INDEX_RECORD_SIZE]) -> crate::error::Result<Self> {
+        let stored_crc = u32::from_le_bytes(buf[52..56].try_into().unwrap());
+        let computed = crate::checksum::crc32(&buf[..52]);
+        if stored_crc != computed {
+            return Err(crate::error::Error::BucketIndexCorrupt {
+                path: std::path::PathBuf::new(),
+                reason: format!(
+                    "CRC mismatch: stored=0x{:08X} computed=0x{:08X}",
+                    stored_crc, computed
+                ),
+            });
+        }
+        let mut key = [0u8; 32];
+        key.copy_from_slice(&buf[0..32]);
+        let segment_id = u32::from_le_bytes(buf[32..36].try_into().unwrap());
+        let offset = u64::from_le_bytes(buf[36..44].try_into().unwrap());
+        let data_size = u32::from_le_bytes(buf[44..48].try_into().unwrap());
+        let flags = buf[48];
+        Ok(Self {
+            key,
+            segment_id,
+            offset,
+            data_size,
+            flags,
+        })
+    }
+}
+
+// ── redb Value impl for fixed-size record bytes ──────────────────────────────
+
+/// Newtype wrapper so we can implement `redb::Value` for `[u8; INDEX_RECORD_SIZE]`.
+#[derive(Debug, Clone, Copy)]
+struct RecordBytes([u8; INDEX_RECORD_SIZE]);
+
+impl redb::Value for RecordBytes {
+    type SelfType<'a> = RecordBytes;
+    type AsBytes<'a> = [u8; INDEX_RECORD_SIZE];
+
+    fn fixed_width() -> Option<usize> {
+        Some(INDEX_RECORD_SIZE)
+    }
+
+    fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>
+    where
+        Self: 'a
+    {
+        let mut arr = [0u8; INDEX_RECORD_SIZE];
+        arr.copy_from_slice(data);
+        RecordBytes(arr)
+    }
+
+    fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> {
+        value.0
+    }
+
+    fn type_name() -> redb::TypeName {
+        redb::TypeName::new("IndexRecord")
+    }
+}
+
+// ── IndexStore ───────────────────────────────────────────────────────────────
+
+const INDEX_TABLE: TableDefinition<[u8; 32], RecordBytes> = TableDefinition::new("blob_index");
+
+/// Zero-heap key-value index backed by redb.
+///
+/// B-tree + mmap + ACID.  No manual compact / sort / dedup / per-bucket mmap
+/// management.  Startup is O(1) — redb reads only its root page.
+pub struct IndexStore {
+    db: Database,
+}
+
+impl IndexStore {
+    /// Open (or create) the index database at `dir/index.redb`.
+    pub fn open(dir: &Path) -> Result<Self> {
+        let path = dir.join("index.redb");
+        let db = Database::create(&path)
+            .map_err(|e| crate::error::Error::IndexDb(format!("failed to create index: {}", e)))?;
+        // Ensure the table exists so reads on a fresh database don't fail.
+        {
+            let txn = db
+                .begin_write()
+                .map_err(|e| crate::error::Error::IndexDb(format!("init write txn: {}", e)))?;
+            txn.open_table(INDEX_TABLE)
+                .map_err(|e| crate::error::Error::IndexDb(format!("init table: {}", e)))?;
+            txn.commit()
+                .map_err(|e| crate::error::Error::IndexDb(format!("init commit: {}", e)))?;
+        }
+        Ok(Self { db })
+    }
+
+    /// Look up a key. Returns the latest IndexRecord, or None if absent/tombstone.
+    pub fn get(&self, key: &[u8; 32]) -> Result<Option<IndexRecord>> {
+        let txn = self
+            .db
+            .begin_read()
+            .map_err(|e| crate::error::Error::IndexDb(format!("read txn: {}", e)))?;
+        let table = txn
+            .open_table(INDEX_TABLE)
+            .map_err(|e| crate::error::Error::IndexDb(format!("open table: {}", e)))?;
+
+        match table
+            .get(key)
+            .map_err(|e| crate::error::Error::IndexDb(format!("get: {}", e)))?
+        {
+            Some(guard) => {
+                let record = IndexRecord::decode(&guard.value().0)?;
+                Ok(if record.is_tombstone() { None } else { Some(record) })
+            }
+            None => Ok(None),
+        }
+    }
+
+    /// Check whether a key exists (non-tombstone) in the store.
+    pub fn exists(&self, key: &[u8; 32]) -> Result<bool> {
+        self.get(key).map(|r| r.is_some())
+    }
+
+    /// Insert or update a record for a key.  Committed in a single write txn.
+    pub fn insert(&self, record: &IndexRecord) -> Result<()> {
+        let txn = self
+            .db
+            .begin_write()
+            .map_err(|e| crate::error::Error::IndexDb(format!("write txn: {}", e)))?;
+        {
+            let mut table = txn
+                .open_table(INDEX_TABLE)
+                .map_err(|e| crate::error::Error::IndexDb(format!("open table: {}", e)))?;
+            table
+                .insert(&record.key, RecordBytes(record.encode()))
+                .map_err(|e| crate::error::Error::IndexDb(format!("insert: {}", e)))?;
+        }
+        txn.commit()
+            .map_err(|e| crate::error::Error::IndexDb(format!("commit: {}", e)))?;
+        Ok(())
+    }
+
+    /// Batch insert multiple records in a single write transaction.
+    pub fn insert_batch(&self, records: &[IndexRecord]) -> Result<()> {
+        if records.is_empty() {
+            return Ok(());
+        }
+        let txn = self
+            .db
+            .begin_write()
+            .map_err(|e| crate::error::Error::IndexDb(format!("write txn: {}", e)))?;
+        {
+            let mut table = txn
+                .open_table(INDEX_TABLE)
+                .map_err(|e| crate::error::Error::IndexDb(format!("open table: {}", e)))?;
+            for record in records {
+                table
+                    .insert(&record.key, RecordBytes(record.encode()))
+                    .map_err(|e| crate::error::Error::IndexDb(format!("insert: {}", e)))?;
+            }
+        }
+        txn.commit()
+            .map_err(|e| crate::error::Error::IndexDb(format!("commit: {}", e)))?;
+        Ok(())
+    }
+
+    /// Remove keys from the index in a single write transaction.
+    pub fn delete_batch(&self, keys: &[[u8; 32]]) -> Result<()> {
+        if keys.is_empty() {
+            return Ok(());
+        }
+        let txn = self
+            .db
+            .begin_write()
+            .map_err(|e| crate::error::Error::IndexDb(format!("write txn: {}", e)))?;
+        {
+            let mut table = txn
+                .open_table(INDEX_TABLE)
+                .map_err(|e| crate::error::Error::IndexDb(format!("open table: {}", e)))?;
+            for key in keys {
+                table
+                    .remove(key)
+                    .map_err(|e| crate::error::Error::IndexDb(format!("remove: {}", e)))?;
+            }
+        }
+        txn.commit()
+            .map_err(|e| crate::error::Error::IndexDb(format!("commit: {}", e)))?;
+        Ok(())
+    }
+
+    /// Total number of live (non-tombstone) keys.
+    pub fn total_keys(&self) -> Result<usize> {
+        let txn = self
+            .db
+            .begin_read()
+            .map_err(|e| crate::error::Error::IndexDb(format!("read txn: {}", e)))?;
+        let table = txn
+            .open_table(INDEX_TABLE)
+            .map_err(|e| crate::error::Error::IndexDb(format!("open table: {}", e)))?;
+
+        let mut count = 0usize;
+        let iter = table
+            .iter()
+            .map_err(|e| crate::error::Error::IndexDb(format!("iter: {}", e)))?;
+        for item in iter {
+            let (_, guard) =
+                item.map_err(|e| crate::error::Error::IndexDb(format!("iter next: {}", e)))?;
+            let record = IndexRecord::decode(&guard.value().0)?;
+            if !record.is_tombstone() {
+                count += 1;
+            }
+        }
+        Ok(count)
+    }
+}

crates/blob/src/checksum.rs

@@ -0,0 +1,65 @@
+use crc32fast::Hasher;
+
+pub fn crc32(data: &[u8]) -> u32 {
+    let mut h = Hasher::new();
+    h.update(data);
+    h.finalize()
+}
+
+pub struct CrcWriter {
+    hasher: Hasher,
+}
+
+impl Default for CrcWriter {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl CrcWriter {
+    pub fn new() -> Self {
+        Self {
+            hasher: Hasher::new(),
+        }
+    }
+
+    pub fn update(&mut self, data: &[u8]) {
+        self.hasher.update(data);
+    }
+
+    pub fn finalize(self) -> u32 {
+        self.hasher.finalize()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_crc32_deterministic() {
+        let a = crc32(b"hello");
+        let b = crc32(b"hello");
+        assert_eq!(a, b);
+    }
+
+    #[test]
+    fn test_crc32_different() {
+        let a = crc32(b"hello");
+        let b = crc32(b"world");
+        assert!(a != b);
+    }
+
+    #[test]
+    fn test_crc_writer_matches_crc32() {
+        let mut w = CrcWriter::new();
+        w.update(b"hello");
+        w.update(b" world");
+        assert_eq!(w.finalize(), crc32(b"hello world"));
+    }
+
+    #[test]
+    fn test_crc32_empty() {
+        assert_eq!(crc32(b""), 0);
+    }
+}

crates/blob/src/compress.rs

@@ -0,0 +1,97 @@
+use crate::types::Codec;
+
+pub fn compress(data: &[u8], codec: Codec, threshold: usize, level: i32) -> (Vec<u8>, Codec) {
+    if data.len() < threshold {
+        return (data.to_vec(), Codec::None);
+    }
+    let (compressed, actual_codec) = match codec {
+        Codec::Zstd => {
+            match zstd::encode_all(data, level) {
+                Ok(out) => (out, Codec::Zstd),
+                Err(e) => {
+                    tracing::warn!("zstd compression failed, storing uncompressed: {}", e);
+                    (data.to_vec(), Codec::None)
+                }
+            }
+        }
+        Codec::Lz4 => {
+            let out = lz4_flex::compress(data);
+            (out, Codec::Lz4)
+        }
+        Codec::None => (data.to_vec(), Codec::None),
+    };
+    // If compression made it larger, store uncompressed
+    if compressed.len() >= data.len() {
+        (data.to_vec(), Codec::None)
+    } else {
+        (compressed, actual_codec)
+    }
+}
+
+pub fn decompress(data: &[u8], codec: Codec, raw_size: usize) -> crate::error::Result<Vec<u8>> {
+    match codec {
+        Codec::None => Ok(data.to_vec()),
+        Codec::Zstd => {
+            zstd::decode_all(data)
+                .map_err(|e| crate::error::Error::Compression(format!("zstd decompress: {}", e)))
+        }
+        Codec::Lz4 => {
+            lz4_flex::decompress(data, raw_size)
+                .map_err(|e| crate::error::Error::Compression(format!("lz4 decompress: {}", e)))
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_small_data_not_compressed() {
+        let data = b"hi";
+        let (out, codec) = compress(data, Codec::Zstd, 4096, 0);
+        assert_eq!(out, b"hi");
+        assert_eq!(codec, Codec::None);
+    }
+
+    #[test]
+    fn test_large_data_compressed_zstd() {
+        let data = vec![b'A'; 5000];
+        let (out, codec) = compress(&data, Codec::Zstd, 4096, 0);
+        assert_eq!(codec, Codec::Zstd);
+        assert!(out.len() < data.len());
+    }
+
+    #[test]
+    fn test_roundtrip_zstd() {
+        let data = vec![b'B'; 10000];
+        let (compressed, codec) = compress(&data, Codec::Zstd, 4096, 0);
+        let decompressed = decompress(&compressed, codec, data.len()).unwrap();
+        assert_eq!(decompressed, data);
+    }
+
+    #[test]
+    fn test_roundtrip_lz4() {
+        let data = vec![b'C'; 10000];
+        let (compressed, codec) = compress(&data, Codec::Lz4, 4096, 0);
+        let decompressed = decompress(&compressed, codec, data.len()).unwrap();
+        assert_eq!(decompressed, data);
+    }
+
+    #[test]
+    fn test_roundtrip_none() {
+        let data = vec![b'D'; 100];
+        let (compressed, codec) = compress(&data, Codec::None, 4096, 0);
+        assert_eq!(codec, Codec::None);
+        let decompressed = decompress(&compressed, codec, data.len()).unwrap();
+        assert_eq!(decompressed, data);
+    }
+
+    #[test]
+    fn test_threshold_zero_always_compresses() {
+        let data = vec![b'E'; 100];
+        let (out, codec) = compress(&data, Codec::Zstd, 0, 0);
+        assert_eq!(codec, Codec::Zstd);
+        assert!(out.len() < data.len());
+    }
+}

crates/blob/src/engine.rs

@@ -0,0 +1,697 @@
+use std::fs;
+use std::fs::File;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex, RwLock};
+use std::thread::{self, JoinHandle};
+use std::time::Duration;
+
+use fs2::FileExt;
+
+use crate::bucket::{IndexRecord, IndexStore};
+use crate::compress;
+use crate::error::{Error, Result};
+use crate::file_pool::FilePool;
+use crate::gc::{self, GcStats};
+use crate::meta::{GlobalMeta, SegmentStats};
+use crate::segment::{self, SegmentReader, SegmentWriter};
+use crate::types::{Codec, Config, ENTRY_HEADER_SIZE};
+
+/// Global content-addressable blob store.
+///
+/// All data is keyed by a 32-byte content hash.  Identical content is stored
+/// only once.  The caller is responsible for tracking which entities reference
+/// which content hashes — this crate is a pure content-addressable KV engine.
+pub struct Engine {
+    shared: Arc<EngineShared>,
+    flush_handle: Mutex<Option<FlushHandle>>,
+    gc_handle: Mutex<Option<GcHandle>>,
+}
+
+struct EngineShared {
+    config: Config,
+    inner: RwLock<EngineInner>,
+    index_store: IndexStore,
+    write_mutex: Mutex<()>,
+    file_pool: FilePool,
+    #[allow(dead_code)]
+    lock_file: File,
+}
+
+struct FlushHandle {
+    handle: JoinHandle<()>,
+    stop: Arc<AtomicBool>,
+}
+
+struct GcHandle {
+    handle: JoinHandle<()>,
+    stop: Arc<AtomicBool>,
+}
+
+struct EngineInner {
+    root: PathBuf,
+    meta: GlobalMeta,
+    active_writer: SegmentWriter,
+}
+
+#[derive(Debug, Clone)]
+pub struct Stats {
+    pub total_keys: u64,
+    pub total_bytes: u64,
+    pub deleted_bytes: u64,
+    pub segment_count: usize,
+}
+
+impl Engine {
+    pub fn open(path: &Path, config: Config) -> Result<Self> {
+        config.validate()?;
+
+        fs::create_dir_all(path)?;
+        fs::create_dir_all(path.join("segments"))?;
+
+        // Acquire an exclusive file lock so that no two processes can open
+        // the same database directory concurrently.
+        let lock_path = path.join("LOCK");
+        let lock_file = File::create(&lock_path)?;
+        lock_file.try_lock_exclusive().map_err(|_| Error::AlreadyOpen {
+            path: path.display().to_string(),
+        })?;
+
+        let (index_store, index_rebuilt) = match IndexStore::open(path) {
+            Ok(s) => (s, false),
+            Err(e) => {
+                tracing::warn!("Index database unreadable, rebuilding from segments: {}", e);
+                let index_path = path.join("index.redb");
+                let _ = std::fs::remove_file(&index_path);
+                (IndexStore::open(path)?, true)
+            }
+        };
+
+        let mut meta = GlobalMeta::load(path)?;
+
+        // Discover existing segments on disk
+        let seg_dir = path.join("segments");
+        let mut disk_segments: Vec<u32> = Vec::new();
+        if seg_dir.exists() {
+            for entry in fs::read_dir(&seg_dir)? {
+                let entry = entry?;
+                let name = entry.file_name();
+                let name_str = name.to_string_lossy();
+                if name_str.ends_with(".seg") && !name_str.contains("temp_") {
+                    if let Some(id_str) = name_str.strip_suffix(".seg") {
+                        if let Ok(id) = id_str.parse::<u32>() {
+                            disk_segments.push(id);
+                        }
+                    }
+                }
+            }
+        }
+        disk_segments.sort_unstable();
+
+        for &seg_id in &disk_segments {
+            if !meta.segments.contains_key(&seg_id) {
+                meta.segments.insert(seg_id, SegmentStats::new(seg_id));
+            }
+        }
+
+        let max_disk_id = disk_segments.last().copied().unwrap_or(0);
+        if max_disk_id > meta.active_segment_id {
+            meta.active_segment_id = max_disk_id;
+        }
+        if meta.active_segment_id == 0 {
+            meta.active_segment_id = 1;
+        }
+
+        crate::recovery::cleanup_temp_files(path)?;
+        let recovered_records = if index_rebuilt {
+            crate::recovery::rebuild_index(path, &mut meta)?
+        } else {
+            crate::recovery::recover(path, &mut meta)?
+        };
+        index_store.insert_batch(&recovered_records)?;
+
+        let seg_path = path
+            .join("segments")
+            .join(segment::segment_filename(meta.active_segment_id));
+        let active_writer = if seg_path.exists() {
+            SegmentWriter::open_append(seg_path, meta.active_segment_id)?
+        } else {
+            SegmentWriter::create(seg_path, meta.active_segment_id)?
+        };
+
+        meta.save(path)?;
+
+        let shared = Arc::new(EngineShared {
+            config: config.clone(),
+            inner: RwLock::new(EngineInner {
+                root: path.to_path_buf(),
+                meta,
+                active_writer,
+            }),
+            index_store,
+            write_mutex: Mutex::new(()),
+            file_pool: FilePool::new(8),
+            lock_file,
+        });
+
+        let flush_handle = if config.flush_interval_secs > 0 {
+            let shared2 = Arc::clone(&shared);
+            let stop = Arc::new(AtomicBool::new(false));
+            let stop2 = Arc::clone(&stop);
+            let interval = Duration::from_secs(config.flush_interval_secs);
+
+            let handle = thread::Builder::new()
+                .name("blob-flush".into())
+                .spawn(move || {
+                    while !stop2.load(Ordering::Acquire) {
+                        thread::park_timeout(interval);
+                        if stop2.load(Ordering::Acquire) {
+                            break;
+                        }
+                        let _lock = shared2.write_mutex.lock().unwrap();
+                        let mut inner = shared2.inner.write().unwrap();
+                        if let Err(e) = inner.flush_active() {
+                            tracing::error!("background flush failed: {}", e);
+                        }
+                    }
+                })
+                .expect("failed to spawn blob-flush thread");
+
+            Some(FlushHandle { handle, stop })
+        } else {
+            None
+        };
+
+        let gc_handle = if config.gc_interval_secs > 0 {
+            let shared3 = Arc::clone(&shared);
+            let stop = Arc::new(AtomicBool::new(false));
+            let stop2 = Arc::clone(&stop);
+            let interval = Duration::from_secs(config.gc_interval_secs);
+
+            let handle = thread::Builder::new()
+                .name("blob-gc".into())
+                .spawn(move || {
+                    while !stop2.load(Ordering::Acquire) {
+                        thread::park_timeout(interval);
+                        if stop2.load(Ordering::Acquire) {
+                            break;
+                        }
+                        // Seal the active segment first so that the data from
+                        // this GC cycle becomes eligible for compaction.
+                        // Without this, the active segment never seals until
+                        // it reaches SEGMENT_MAX_SIZE (1 GB), and GC would
+                        // have no candidates on small databases.
+                        if let Err(e) = shared3.ensure_sealed() {
+                            tracing::error!("background GC: seal failed: {}", e);
+                        }
+                        match shared3.gc_if_needed() {
+                            Ok(Some(stats)) => {
+                                tracing::info!(
+                                    "GC compacted segment {}: {} → {} bytes (kept {}, skipped {})",
+                                    stats.segment_id,
+                                    stats.bytes_before,
+                                    stats.bytes_after,
+                                    stats.entries_kept,
+                                    stats.entries_skipped,
+                                );
+                            }
+                            Ok(None) => {
+                                tracing::debug!("GC check: no segment exceeds deleted-ratio threshold");
+                            }
+                            Err(e) => {
+                                tracing::error!("background GC failed: {}", e);
+                            }
+                        }
+                    }
+                })
+                .expect("failed to spawn blob-gc thread");
+
+            Some(GcHandle { handle, stop })
+        } else {
+            None
+        };
+
+        Ok(Self {
+            shared,
+            flush_handle: Mutex::new(flush_handle),
+            gc_handle: Mutex::new(gc_handle),
+        })
+    }
+
+    // ── Read / Write / Delete ───────────────────────────────────────────
+
+    pub fn put(&self, key: [u8; 32], value: &[u8], codec: Codec) -> Result<()> {
+        if value.len() > crate::types::MAX_VALUE_SIZE {
+            return Err(Error::ValueTooLarge { size: value.len() });
+        }
+
+        let _write_lock = self.shared.write_mutex.lock().unwrap();
+        let mut inner = self.shared.inner.write().unwrap();
+
+        let (data, actual_codec) = compress::compress(
+            value,
+            codec,
+            self.shared.config.compress_threshold,
+            self.shared.config.compression_level,
+        );
+
+        let original_len = value.len() as u32;
+        let (segment_id, offset, data_size) =
+            inner.append_entry(key, &data, original_len, 0, actual_codec)?;
+
+        let record = IndexRecord::new(key, segment_id, offset, data_size, 0);
+        self.shared.index_store.insert(&record)?;
+
+        let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
+        inner.mark_indexed(segment_id, entry_end)?;
+
+        Ok(())
+    }
+
+    pub fn get(&self, key: &[u8; 32]) -> Result<Option<Vec<u8>>> {
+        let record = match self.shared.index_store.get(key)? {
+            Some(r) => r,
+            None => return Ok(None),
+        };
+
+        let inner = self.shared.inner.read().unwrap();
+        let seg_path = inner.segment_path(record.segment_id)?;
+
+        if !seg_path.exists() {
+            return Err(Error::SegmentNotFound(record.segment_id));
+        }
+
+        let reader = SegmentReader::open(seg_path.clone(), record.segment_id)?;
+        let file = self.shared.file_pool.get(record.segment_id, &seg_path)?;
+        let (entry, _) = reader.read_entry_at_file(record.offset, &file)?;
+
+        let value = compress::decompress(&entry.data, entry.codec, entry.raw_size as usize)?;
+        Ok(Some(value))
+    }
+
+    pub fn delete(&self, key: &[u8; 32]) -> Result<()> {
+        let _write_lock = self.shared.write_mutex.lock().unwrap();
+        let mut inner = self.shared.inner.write().unwrap();
+
+        // Look up the existing record so we can account deleted_bytes on the
+        // segment that holds the original data — this is what drives GC.
+        if let Some(rec) = self.shared.index_store.get(key)? {
+            if !rec.is_tombstone() {
+                if let Some(stats) = inner.meta.segments.get_mut(&rec.segment_id) {
+                    stats.deleted_bytes += rec.data_size as u64;
+                    stats.recompute_ratio();
+                }
+            }
+        }
+
+        let (segment_id, offset, data_size) =
+            inner.append_entry(*key, &[], 0, 1, Codec::None)?;
+
+        let record = IndexRecord::new(*key, segment_id, offset, data_size, 1);
+        self.shared.index_store.insert(&record)?;
+
+        let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
+        inner.mark_indexed(segment_id, entry_end)?;
+
+        Ok(())
+    }
+
+    pub fn exists(&self, key: &[u8; 32]) -> Result<bool> {
+        self.shared.index_store.exists(key)
+    }
+
+    // ── Batch delete ─────────────────────────────────────────────────────
+
+    pub fn delete_batch(&self, keys: &[[u8; 32]]) -> Result<()> {
+        if keys.is_empty() {
+            return Ok(());
+        }
+
+        let _write_lock = self.shared.write_mutex.lock().unwrap();
+        let mut inner = self.shared.inner.write().unwrap();
+
+        let mut records: Vec<IndexRecord> = Vec::with_capacity(keys.len());
+        let mut ends: Vec<(u32, u64)> = Vec::with_capacity(keys.len());
+
+        for key in keys {
+            // Track deleted_bytes for GC threshold on the original segment.
+            if let Some(rec) = self.shared.index_store.get(key)? {
+                if !rec.is_tombstone() {
+                    if let Some(stats) = inner.meta.segments.get_mut(&rec.segment_id) {
+                        stats.deleted_bytes += rec.data_size as u64;
+                        stats.recompute_ratio();
+                    }
+                }
+            }
+
+            let (segment_id, offset, data_size) =
+                inner.append_entry(*key, &[], 0, 1, Codec::None)?;
+
+            let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
+            records.push(IndexRecord::new(*key, segment_id, offset, data_size, 1));
+            ends.push((segment_id, entry_end));
+        }
+
+        inner.flush_active()?;
+
+        self.shared.index_store.insert_batch(&records)?;
+
+        for (segment_id, entry_end) in &ends {
+            inner.mark_indexed(*segment_id, *entry_end)?;
+        }
+
+        Ok(())
+    }
+
+    // ── Batch write ─────────────────────────────────────────────────────
+
+    pub fn put_batch(&self, entries: &[([u8; 32], Vec<u8>, Codec)]) -> Result<()> {
+        if entries.is_empty() {
+            return Ok(());
+        }
+
+        let _write_lock = self.shared.write_mutex.lock().unwrap();
+        let mut inner = self.shared.inner.write().unwrap();
+
+        let mut records: Vec<IndexRecord> = Vec::with_capacity(entries.len());
+        let mut ends: Vec<(u32, u64)> = Vec::with_capacity(entries.len());
+
+        for (key, value, codec) in entries {
+            if value.len() > crate::types::MAX_VALUE_SIZE {
+                return Err(Error::ValueTooLarge { size: value.len() });
+            }
+            let (data, actual_codec) = compress::compress(
+                value,
+                *codec,
+                self.shared.config.compress_threshold,
+                self.shared.config.compression_level,
+            );
+
+            let original_len = value.len() as u32;
+            let (segment_id, offset, data_size) =
+                inner.append_entry(*key, &data, original_len, 0, actual_codec)?;
+
+            let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
+            records.push(IndexRecord::new(*key, segment_id, offset, data_size, 0));
+            ends.push((segment_id, entry_end));
+        }
+
+        inner.flush_active()?;
+
+        self.shared.index_store.insert_batch(&records)?;
+
+        // Deduplicate: keep only the max offset per segment.
+        ends.sort_by(|a, b| a.0.cmp(&b.0).then(b.1.cmp(&a.1)));
+        ends.dedup_by(|a, b| a.0 == b.0);
+        for (segment_id, entry_end) in &ends {
+            inner.mark_indexed(*segment_id, *entry_end)?;
+        }
+
+        Ok(())
+    }
+
+    // ── GC ──────────────────────────────────────────────────────────────
+
+    pub fn gc(&self) -> Result<Option<GcStats>> {
+        self.shared.gc()
+    }
+
+    /// Run GC only if some segment exceeds the configured deleted-ratio threshold.
+    /// Returns `Ok(None)` immediately without acquiring the write lock when no
+    /// segment qualifies.
+    pub fn gc_if_needed(&self) -> Result<Option<GcStats>> {
+        self.shared.gc_if_needed()
+    }
+
+    // ── Flush / Stats / Shutdown ────────────────────────────────────────
+
+    /// Fsync the active segment and save metadata without compacting buckets.
+    /// Lightweight checkpoint suitable for periodic calls from the background
+    /// flush thread or external schedulers.
+    pub fn flush(&self) -> Result<()> {
+        let _write_lock = self.shared.write_mutex.lock().unwrap();
+        let mut inner = self.shared.inner.write().unwrap();
+        inner.flush_active()
+    }
+
+    pub fn stats(&self) -> Result<Stats> {
+        let inner = self.shared.inner.read().unwrap();
+        let meta = &inner.meta;
+
+        let mut total_bytes = 0u64;
+        let mut deleted_bytes = 0u64;
+        for seg in meta.segments.values() {
+            total_bytes += seg.total_bytes;
+            deleted_bytes += seg.deleted_bytes;
+        }
+
+        let total_keys = self.shared.index_store.total_keys()? as u64;
+
+        Ok(Stats {
+            total_keys,
+            total_bytes,
+            deleted_bytes,
+            segment_count: meta.segments.len(),
+        })
+    }
+
+    /// Seal the active segment, forcing it to become a GC candidate.
+    #[doc(hidden)]
+    pub fn seal_active_segment(&self) -> Result<u32> {
+        let _write_lock = self.shared.write_mutex.lock().unwrap();
+        let mut inner = self.shared.inner.write().unwrap();
+        let id = inner.active_writer.id();
+        inner.seal_active()?;
+        Ok(id)
+    }
+
+    pub fn shutdown(&self) -> Result<()> {
+        // Stop background threads first
+        if let Some(fh) = self.flush_handle.lock().unwrap().take() {
+            fh.stop.store(true, Ordering::Release);
+            fh.handle.thread().unpark();
+            let _ = fh.handle.join();
+        }
+        if let Some(gh) = self.gc_handle.lock().unwrap().take() {
+            gh.stop.store(true, Ordering::Release);
+            gh.handle.thread().unpark();
+            let _ = gh.handle.join();
+        }
+
+        let mut inner = self.shared.inner.write().unwrap();
+        inner.flush_active()?;
+
+        inner.meta.save(&inner.root)?;
+        tracing::info!("bichon-blob shut down cleanly");
+        Ok(())
+    }
+}
+
+impl Drop for Engine {
+    fn drop(&mut self) {
+        // Best-effort shutdown that is panic-safe: only signal threads to
+        // stop — don't try to acquire write_mutex or inner.write(), which
+        // would deadlock if we're unwinding from a panic that happened while
+        // one of those locks was held.
+        if let Some(fh) = self.flush_handle.lock().ok().and_then(|mut g| g.take()) {
+            fh.stop.store(true, Ordering::Release);
+            fh.handle.thread().unpark();
+            let _ = fh.handle.join();
+        }
+        if let Some(gh) = self.gc_handle.lock().ok().and_then(|mut g| g.take()) {
+            gh.stop.store(true, Ordering::Release);
+            gh.handle.thread().unpark();
+            let _ = gh.handle.join();
+        }
+    }
+}
+
+// ── EngineShared ────────────────────────────────────────────────────────────
+
+impl EngineShared {
+    /// Seal the active segment if its deleted-ratio exceeds the GC threshold,
+    /// so that the upcoming GC pass can compact it.  Called periodically by
+    /// the background GC thread.
+    fn ensure_sealed(&self) -> Result<()> {
+        let mut inner = self.inner.write().unwrap();
+        let active_id = inner.active_writer.id();
+        if let Some(stats) = inner.meta.segments.get(&active_id) {
+            if stats.deleted_ratio >= self.config.gc_deleted_ratio {
+                inner.seal_active()?;
+            }
+        }
+        Ok(())
+    }
+
+    fn gc_if_needed(&self) -> Result<Option<GcStats>> {
+        let inner = self.inner.read().unwrap();
+        let needs_gc = inner
+            .meta
+            .segments
+            .values()
+            .any(|s| s.sealed && s.deleted_ratio >= self.config.gc_deleted_ratio);
+        drop(inner);
+
+        if needs_gc {
+            self.gc()
+        } else {
+            Ok(None)
+        }
+    }
+
+    fn gc(&self) -> Result<Option<GcStats>> {
+        // Phase 1: scan only the target segment, consult bucket index per entry.
+        // Read-only with respect to Engine state — no write_mutex needed.
+        let prep = {
+            let inner = self.inner.read().unwrap();
+            gc::gc_prepare(
+                &inner.root,
+                &inner.meta,
+                self.config.gc_deleted_ratio,
+                &self.index_store,
+            )?
+        };
+
+        let mut prep = match prep {
+            Some(p) => p,
+            None => return Ok(None),
+        };
+
+        // Phase 2: rename temp file + update bucket index.
+        let _write_lock = self.write_mutex.lock().unwrap();
+
+        let kept_records = std::mem::take(&mut prep.kept_records);
+        let deleted_keys = std::mem::take(&mut prep.deleted_keys);
+        let stats = gc::gc_finish(prep)?;
+        self.file_pool.invalidate(stats.segment_id);
+
+        // Insert new index records with updated offsets for kept entries.
+        if !kept_records.is_empty() {
+            self.index_store.insert_batch(&kept_records)?;
+        }
+
+        // Remove tombstone IndexRecords that pointed to entries in this
+        // compacted segment — they are gone now and would accumulate forever.
+        if !deleted_keys.is_empty() {
+            self.index_store.delete_batch(&deleted_keys)?;
+        }
+
+        {
+            let mut inner = self.inner.write().unwrap();
+
+            if stats.bytes_after == 0 {
+                // Segment was completely emptied — remove it from meta first,
+                // then delete the file.  If we crash between the two steps the
+                // orphaned file is rediscovered on next open and retried.
+                inner.meta.segments.remove(&stats.segment_id);
+                inner.meta.save(&inner.root)?;
+                drop(inner);
+
+                let seg_path = self.inner.read().unwrap()
+                    .root
+                    .join("segments")
+                    .join(segment::segment_filename(stats.segment_id));
+                let _ = fs::remove_file(&seg_path);
+            } else {
+                // Update segment stats: now smaller and clean.
+                if let Some(seg_stats) = inner.meta.segments.get_mut(&stats.segment_id) {
+                    seg_stats.total_bytes = stats.bytes_after;
+                    seg_stats.deleted_bytes = 0;
+                    seg_stats.deleted_ratio = 0.0;
+                    seg_stats.indexed_up_to_offset = stats.bytes_after;
+                }
+                inner.meta.save(&inner.root)?;
+            }
+        }
+
+        Ok(Some(stats))
+    }
+}
+
+// ── EngineInner ───────────────────────────────────────────────────────────
+
+impl EngineInner {
+    fn append_entry(
+        &mut self,
+        key: [u8; 32],
+        data: &[u8],
+        raw_size: u32,
+        flags: u8,
+        codec: Codec,
+    ) -> Result<(u32, u64, u32)> {
+        if self.active_writer.is_full() {
+            self.seal_active()?;
+        }
+
+        use crate::segment::Entry;
+        let entry = if flags == 1 {
+            Entry::tombstone(key)
+        } else {
+            Entry::new(key, data, raw_size, flags, codec)
+        };
+
+        let data_size = entry.data.len() as u32;
+        let segment_id = self.active_writer.id();
+        let offset = self.active_writer.append(&entry)?;
+
+        let stats = self
+            .meta
+            .segments
+            .entry(segment_id)
+            .or_insert_with(|| SegmentStats::new(segment_id));
+        stats.total_bytes += data_size as u64;
+        if flags == 1 {
+            stats.deleted_bytes += entry.raw_size as u64;
+        }
+        stats.recompute_ratio();
+
+        Ok((segment_id, offset, data_size))
+    }
+
+    fn flush_active(&mut self) -> Result<()> {
+        self.active_writer.fsync()?;
+        self.meta.save(&self.root)
+    }
+
+    fn mark_indexed(&mut self, segment_id: u32, offset: u64) -> Result<()> {
+        if let Some(stats) = self.meta.segments.get_mut(&segment_id) {
+            if offset > stats.indexed_up_to_offset {
+                stats.indexed_up_to_offset = offset;
+            }
+        }
+        self.meta.save(&self.root)
+    }
+
+    fn seal_active(&mut self) -> Result<()> {
+        let old_id = self.active_writer.id();
+        let old_stats = self
+            .meta
+            .segments
+            .entry(old_id)
+            .or_insert_with(|| SegmentStats::new(old_id));
+        old_stats.sealed = true;
+
+        let new_id = old_id + 1;
+        self.meta.active_segment_id = new_id;
+        let new_path = self
+            .root
+            .join("segments")
+            .join(segment::segment_filename(new_id));
+        self.active_writer = SegmentWriter::create(new_path, new_id)?;
+        self.meta.save(&self.root)?;
+
+        Ok(())
+    }
+
+    fn segment_path(&self, segment_id: u32) -> Result<PathBuf> {
+        let path = self
+            .root
+            .join("segments")
+            .join(segment::segment_filename(segment_id));
+        if path.exists() {
+            Ok(path)
+        } else {
+            Err(Error::SegmentNotFound(segment_id))
+        }
+    }
+}

crates/blob/src/error.rs

@@ -0,0 +1,59 @@
+use std::{io, path::PathBuf};
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("I/O error: {0}")]
+    Io(#[from] io::Error),
+
+    #[error("JSON error: {0}")]
+    Json(#[from] serde_json::Error),
+
+    #[error("CRC32 mismatch at {path}:{offset}")]
+    CrcMismatch { path: PathBuf, offset: u64 },
+
+    #[error("Corrupt entry at {path}:{offset}: {reason}")]
+    CorruptEntry {
+        path: PathBuf,
+        offset: u64,
+        reason: String,
+    },
+
+    #[error("Segment not found: {0}")]
+    SegmentNotFound(u32),
+
+    #[error("Value too large: {size} bytes (max 100 MB)")]
+    ValueTooLarge { size: usize },
+
+    #[error("Compression error: {0}")]
+    Compression(String),
+
+    #[error("Disk full: {0}")]
+    DiskFull(String),
+
+    #[error("Invalid config: {0}")]
+    InvalidConfig(String),
+
+    #[error("Bucket index corrupt at {path}: {reason}")]
+    BucketIndexCorrupt { path: PathBuf, reason: String },
+
+    #[error("Segment file truncated at {path}: expected {expected}, got {actual}")]
+    SegmentTruncated {
+        path: PathBuf,
+        expected: u64,
+        actual: u64,
+    },
+
+    #[error("Corrupt metadata file: {0}")]
+    CorruptMeta(String),
+
+    #[error("Unsupported metadata version {version} in {path}")]
+    UnsupportedMetaVersion { path: PathBuf, version: u32 },
+
+    #[error("Index database error: {0}")]
+    IndexDb(String),
+
+    #[error("Database is already open by another process at {path}")]
+    AlreadyOpen { path: String },
+}

crates/blob/src/file_pool.rs

@@ -0,0 +1,54 @@
+use std::collections::VecDeque;
+use std::fs::File;
+use std::path::Path;
+use std::sync::{Arc, Mutex};
+
+use crate::error::Result;
+use crate::fs as fs_util;
+
+/// Simple LRU pool of open file handles, keyed by segment_id.
+/// Uses pread-based reads so a single `Arc<File>` supports concurrent access.
+pub struct FilePool {
+    max_entries: usize,
+    entries: Mutex<VecDeque<(u32, Arc<File>)>>,
+}
+
+impl FilePool {
+    pub fn new(max_entries: usize) -> Self {
+        Self {
+            max_entries: max_entries.max(1),
+            entries: Mutex::new(VecDeque::new()),
+        }
+    }
+
+    /// Get an open File for the given segment. Reuses cached handle if available.
+    pub fn get(&self, seg_id: u32, path: &Path) -> Result<Arc<File>> {
+        let mut entries = self.entries.lock().unwrap();
+
+        // Check for existing entry
+        for (i, (id, _)) in entries.iter().enumerate() {
+            if *id == seg_id {
+                let (_, file) = entries.remove(i).unwrap();
+                entries.push_front((seg_id, file.clone()));
+                return Ok(file);
+            }
+        }
+
+        // Open new file
+        let file = Arc::new(fs_util::open_read(path)?);
+
+        // Evict oldest if full
+        if entries.len() >= self.max_entries {
+            entries.pop_back();
+        }
+
+        entries.push_front((seg_id, file.clone()));
+        Ok(file)
+    }
+
+    /// Remove a cached file handle (e.g. after GC rewrites a segment).
+    pub fn invalidate(&self, seg_id: u32) {
+        let mut entries = self.entries.lock().unwrap();
+        entries.retain(|(id, _)| *id != seg_id);
+    }
+}

crates/blob/src/fs.rs

@@ -0,0 +1,167 @@
+use std::fs::{self, File, OpenOptions};
+use std::io::{self, Write};
+use std::path::Path;
+use std::time::Duration;
+
+use crate::error::Result;
+
+/// Max retries for transient filesystem errors (NFS ESTALE, CIFS sharing violations, etc.)
+const MAX_RETRIES: u32 = 5;
+const RETRY_DELAY: Duration = Duration::from_millis(20);
+
+/// Check if an I/O error is transient (retryable).
+fn is_transient(err: &io::Error) -> bool {
+    use std::io::ErrorKind;
+    matches!(
+        err.kind(),
+        ErrorKind::TimedOut
+            | ErrorKind::Interrupted
+            | ErrorKind::WouldBlock
+            | ErrorKind::UnexpectedEof
+    ) || err.raw_os_error() == Some(116) // ESTALE on Linux
+}
+
+/// Open an existing file for reading, with retry on transient errors (NFS ESTALE etc.).
+pub fn open_read(path: &Path) -> Result<File> {
+    let mut last_err = None;
+    for attempt in 0..MAX_RETRIES {
+        match File::open(path) {
+            Ok(f) => return Ok(f),
+            Err(e) if is_transient(&e) => {
+                last_err = Some(e);
+                if attempt > 0 {
+                    std::thread::sleep(RETRY_DELAY * attempt);
+                }
+                continue;
+            }
+            Err(e) => return Err(e.into()),
+        }
+    }
+    Err(crate::error::Error::Io(last_err.unwrap()))
+}
+
+/// Open an existing file for writing, with retry on transient errors.
+pub fn open_write(path: &Path) -> Result<File> {
+    let mut last_err = None;
+    for attempt in 0..MAX_RETRIES {
+        match OpenOptions::new().write(true).open(path) {
+            Ok(f) => return Ok(f),
+            Err(e) if is_transient(&e) => {
+                last_err = Some(e);
+                if attempt > 0 {
+                    std::thread::sleep(RETRY_DELAY * attempt);
+                }
+                continue;
+            }
+            Err(e) => return Err(e.into()),
+        }
+    }
+    Err(crate::error::Error::Io(last_err.unwrap()))
+}
+
+/// Create a new file atomically: write content to a temp file, fsync, then rename.
+/// Avoids `create_new(true)` which is racy on NFS.
+pub fn create_atomic(path: &Path, content: &[u8]) -> Result<()> {
+    let tmp = path.with_extension(
+        path.extension()
+            .map(|e| format!("{}.tmp", e.to_string_lossy()))
+            .unwrap_or_else(|| "tmp".to_string()),
+    );
+
+    {
+        let mut f = File::create(&tmp)?;
+        f.write_all(content)?;
+        f.sync_all()?;
+    }
+
+    fs::rename(&tmp, path)?;
+    Ok(())
+}
+
+/// Truncate an existing file to the given size, with retry.
+pub fn truncate(path: &Path, size: u64) -> Result<()> {
+    let f = open_write(path)?;
+    f.set_len(size)?;
+    f.sync_all()?;
+    Ok(())
+}
+
+/// Positional read: read `buf.len()` bytes at `offset` from `file`.
+/// Uses platform-specific pread so `&File` (shared ref) suffices —
+/// no Mutex needed for concurrent reads.
+pub fn pread_exact(file: &File, offset: u64, buf: &mut [u8]) -> Result<()> {
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::FileExt;
+        file.read_exact_at(buf, offset)?;
+    }
+    #[cfg(windows)]
+    {
+        use std::os::windows::fs::FileExt;
+        file.seek_read(buf, offset)?;
+    }
+    #[cfg(not(any(unix, windows)))]
+    {
+        // Fallback: seek+read (requires &mut, so this is best-effort on exotic platforms)
+        use std::io::{Read, Seek, SeekFrom};
+        let mut tmp = file.try_clone()?;
+        tmp.seek(SeekFrom::Start(offset))?;
+        tmp.read_exact(buf)?;
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::TempDir;
+
+    #[test]
+    fn test_open_read_existing() {
+        let dir = TempDir::new().unwrap();
+        let path = dir.path().join("test.txt");
+        std::fs::write(&path, b"hello").unwrap();
+
+        let mut f = open_read(&path).unwrap();
+        let mut s = String::new();
+        std::io::Read::read_to_string(&mut f, &mut s).unwrap();
+        assert_eq!(s, "hello");
+    }
+
+    #[test]
+    fn test_open_read_missing() {
+        let dir = TempDir::new().unwrap();
+        let result = open_read(&dir.path().join("nope.txt"));
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_create_atomic_success() {
+        let dir = TempDir::new().unwrap();
+        let path = dir.path().join("data.bin");
+        create_atomic(&path, b"hello world").unwrap();
+
+        let content = std::fs::read(&path).unwrap();
+        assert_eq!(content, b"hello world");
+        // Temp file should not exist
+        assert!(!dir.path().join("data.bin.tmp").exists());
+    }
+
+    #[test]
+    fn test_create_atomic_overwrites() {
+        let dir = TempDir::new().unwrap();
+        let path = dir.path().join("data.bin");
+        create_atomic(&path, b"first").unwrap();
+        create_atomic(&path, b"second").unwrap();
+        assert_eq!(std::fs::read(&path).unwrap(), b"second");
+    }
+
+    #[test]
+    fn test_truncate() {
+        let dir = TempDir::new().unwrap();
+        let path = dir.path().join("trunc.bin");
+        std::fs::write(&path, b"1234567890").unwrap();
+        truncate(&path, 5).unwrap();
+        assert_eq!(std::fs::metadata(&path).unwrap().len(), 5);
+    }
+}

crates/blob/src/gc.rs

@@ -0,0 +1,148 @@
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use crate::bucket::{IndexRecord, IndexStore};
+use crate::error::Result;
+use crate::meta::GlobalMeta;
+use crate::segment::{self, SegmentReader, SegmentWriter};
+
+/// Result of a GC run.
+#[derive(Debug)]
+pub struct GcStats {
+    pub segment_id: u32,
+    pub bytes_before: u64,
+    pub bytes_after: u64,
+    pub entries_kept: usize,
+    pub entries_skipped: usize,
+}
+
+/// Prepared GC result — the compacted segment has been written to a temp file
+/// but not yet renamed over the original.  `gc_finish` must be called to commit.
+pub struct GcPrepare {
+    pub segment_id: u32,
+    pub bytes_before: u64,
+    pub bytes_after: u64,
+    pub entries_kept: usize,
+    pub entries_skipped: usize,
+    /// Index records for kept entries with their new offsets in the compacted segment.
+    pub kept_records: Vec<IndexRecord>,
+    /// Keys whose tombstone IndexRecord should be removed from redb after
+    /// this segment is compacted (the tombstone entries they pointed to are gone).
+    pub deleted_keys: Vec<[u8; 32]>,
+    temp_path: PathBuf,
+    seg_path: PathBuf,
+}
+
+/// Phase 1: pick the sealed segment with the highest deleted_ratio, then for
+/// each entry in that segment consult the bucket index to decide whether it is
+/// still the latest version.  Live entries are written to a temp file; stale
+/// entries and tombstones are skipped.  Does NOT rename — the caller should
+/// hold the write lock only during `gc_finish`.
+pub fn gc_prepare(
+    store_root: &Path,
+    meta: &GlobalMeta,
+    deleted_ratio_threshold: f64,
+    index_store: &IndexStore,
+) -> Result<Option<GcPrepare>> {
+    let candidate = meta
+        .segments
+        .values()
+        .filter(|s| s.sealed && s.deleted_ratio >= deleted_ratio_threshold)
+        .max_by(|a, b| a.deleted_ratio.partial_cmp(&b.deleted_ratio).unwrap());
+
+    let target = match candidate {
+        Some(s) => s.clone(),
+        None => return Ok(None),
+    };
+
+    let seg_path = store_root
+        .join("segments")
+        .join(segment::segment_filename(target.segment_id));
+
+    // Guard against stale meta entries: if the segment file was deleted
+    // (e.g. after a prior GC emptied it), skip this candidate.
+    if !seg_path.exists() {
+        return Ok(None);
+    }
+
+    let reader = SegmentReader::open(seg_path.clone(), target.segment_id)?;
+
+    // Create temp segment (not renamed yet)
+    let timestamp = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_nanos();
+    let temp_name = format!("temp_{:016x}.seg", timestamp);
+    let temp_path = store_root.join("segments").join(&temp_name);
+    let mut writer = SegmentWriter::create(temp_path.clone(), target.segment_id)?;
+
+    let mut kept_records: Vec<IndexRecord> = Vec::new();
+    let mut deleted_keys: Vec<[u8; 32]> = Vec::new();
+    let mut bytes_after: u64 = 0;
+    let mut entries_kept: usize = 0;
+    let mut entries_skipped: usize = 0;
+
+    reader.scan_entries(0, |entry, offset| {
+        if entry.is_tombstone() {
+            // If this tombstone is the latest version in the index, the key
+            // must be removed from redb after compaction — otherwise it
+            // accumulates forever.
+            match index_store.get(&entry.key)? {
+                Some(rec)
+                    if rec.segment_id == target.segment_id && rec.offset == offset =>
+                {
+                    deleted_keys.push(entry.key);
+                }
+                _ => {}
+            }
+            entries_skipped += 1;
+            return Ok(());
+        }
+
+        // Ask the bucket index whether this entry is still the latest version.
+        match index_store.get(&entry.key)? {
+            Some(rec) if rec.segment_id == target.segment_id && rec.offset == offset => {
+                let new_offset = writer.append(entry)?;
+                kept_records.push(IndexRecord::new(
+                    entry.key,
+                    target.segment_id,
+                    new_offset,
+                    entry.data.len() as u32,
+                    entry.flags,
+                ));
+                bytes_after += entry.data.len() as u64;
+                entries_kept += 1;
+            }
+            _ => {
+                entries_skipped += 1;
+            }
+        }
+        Ok(())
+    })?;
+
+    writer.fsync()?;
+
+    Ok(Some(GcPrepare {
+        segment_id: target.segment_id,
+        bytes_before: target.total_bytes,
+        bytes_after,
+        entries_kept,
+        entries_skipped,
+        kept_records,
+        deleted_keys,
+        temp_path,
+        seg_path,
+    }))
+}
+
+/// Phase 2: atomically replace the old segment with the compacted one.
+pub fn gc_finish(prep: GcPrepare) -> Result<GcStats> {
+    fs::rename(&prep.temp_path, &prep.seg_path)?;
+    Ok(GcStats {
+        segment_id: prep.segment_id,
+        bytes_before: prep.bytes_before,
+        bytes_after: prep.bytes_after,
+        entries_kept: prep.entries_kept,
+        entries_skipped: prep.entries_skipped,
+    })
+}

crates/blob/src/lib.rs

@@ -0,0 +1,16 @@
+pub mod bucket;
+pub mod checksum;
+pub mod compress;
+pub mod engine;
+pub mod error;
+pub mod file_pool;
+pub mod fs;
+pub mod gc;
+pub mod meta;
+pub mod recovery;
+pub mod segment;
+pub mod types;
+
+pub use engine::{Engine, Stats};
+pub use error::{Error, Result};
+pub use types::{Codec, Config};

crates/blob/src/meta.rs

@@ -0,0 +1,180 @@
+use std::collections::BTreeMap;
+use std::path::Path;
+
+use crate::checksum;
+use crate::error::Result;
+use serde::{Deserialize, Serialize};
+
+const META_VERSION: u32 = 2;
+
+// ── Helpers ────────────────────────────────────────────────────────────────
+
+fn write_bin<T: Serialize>(path: &Path, value: &T) -> Result<()> {
+    let payload =
+        bincode::serde::encode_to_vec(value, bincode::config::standard()).map_err(|e| {
+            crate::error::Error::CorruptMeta(format!("{}: bincode encode: {}", path.display(), e))
+        })?;
+    let crc = checksum::crc32(&payload);
+    let mut buf = Vec::with_capacity(8 + payload.len());
+    buf.extend_from_slice(&crc.to_le_bytes());
+    buf.extend_from_slice(&META_VERSION.to_le_bytes());
+    buf.extend_from_slice(&payload);
+
+    crate::fs::create_atomic(path, &buf)?;
+    Ok(())
+}
+
+fn read_bin<T: for<'de> Deserialize<'de>>(path: &Path) -> Result<T> {
+    let data = std::fs::read(path)?;
+    if data.len() < 8 {
+        return Err(crate::error::Error::CorruptMeta(path.display().to_string()));
+    }
+    let stored_crc = u32::from_le_bytes(data[0..4].try_into().unwrap());
+    let version = u32::from_le_bytes(data[4..8].try_into().unwrap());
+    if version != META_VERSION {
+        return Err(crate::error::Error::UnsupportedMetaVersion {
+            path: path.to_path_buf(),
+            version,
+        });
+    }
+    let computed = checksum::crc32(&data[8..]);
+    if stored_crc != computed {
+        return Err(crate::error::Error::CorruptMeta(path.display().to_string()));
+    }
+    bincode::serde::decode_from_slice(&data[8..], bincode::config::standard())
+        .map(|(v, _)| v)
+        .map_err(|e| {
+            crate::error::Error::CorruptMeta(format!("{}: bincode decode: {}", path.display(), e))
+        })
+}
+
+// ── SegmentStats ───────────────────────────────────────────────────────────
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SegmentStats {
+    pub segment_id: u32,
+    pub total_bytes: u64,
+    pub deleted_bytes: u64,
+    pub deleted_ratio: f64,
+    pub sealed: bool,
+    /// Byte offset up to which entries have been indexed in bucket files.
+    pub indexed_up_to_offset: u64,
+    /// Number of compacted (sorted, deduped) records in each bucket file for this segment.
+    /// Used by BucketStore on recovery to know where the clean portion ends.
+    pub bucket_compacted: u64,
+}
+
+impl SegmentStats {
+    pub fn new(segment_id: u32) -> Self {
+        Self {
+            segment_id,
+            total_bytes: 0,
+            deleted_bytes: 0,
+            deleted_ratio: 0.0,
+            sealed: false,
+            indexed_up_to_offset: 0,
+            bucket_compacted: 0,
+        }
+    }
+
+    pub fn recompute_ratio(&mut self) {
+        if self.total_bytes > 0 {
+            self.deleted_ratio = self.deleted_bytes as f64 / self.total_bytes as f64;
+        } else {
+            self.deleted_ratio = 0.0;
+        }
+    }
+}
+
+// ── GlobalMeta ─────────────────────────────────────────────────────────────
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GlobalMeta {
+    pub version: u32,
+    pub active_segment_id: u32,
+    pub segments: BTreeMap<u32, SegmentStats>,
+}
+
+impl GlobalMeta {
+    pub fn new() -> Self {
+        Self {
+            version: META_VERSION,
+            active_segment_id: 1,
+            segments: BTreeMap::new(),
+        }
+    }
+
+    pub fn load(store_root: &Path) -> Result<Self> {
+        let bin_path = store_root.join("meta.bin");
+        if bin_path.exists() {
+            return read_bin(&bin_path);
+        }
+        Ok(Self::new())
+    }
+
+    pub fn save(&self, store_root: &Path) -> Result<()> {
+        let path = store_root.join("meta.bin");
+        write_bin(&path, self)
+    }
+}
+
+impl Default for GlobalMeta {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::TempDir;
+
+    #[test]
+    fn test_global_meta_roundtrip() {
+        let dir = TempDir::new().unwrap();
+        let mut meta = GlobalMeta::new();
+        meta.segments.insert(
+            1,
+            SegmentStats {
+                segment_id: 1,
+                total_bytes: 1000,
+                deleted_bytes: 300,
+                deleted_ratio: 0.3,
+                sealed: false,
+                indexed_up_to_offset: 500,
+                bucket_compacted: 0,
+            },
+        );
+        meta.save(dir.path()).unwrap();
+
+        let loaded = GlobalMeta::load(dir.path()).unwrap();
+        assert_eq!(loaded.active_segment_id, 1);
+        assert_eq!(loaded.segments[&1].total_bytes, 1000);
+        assert_eq!(loaded.segments[&1].indexed_up_to_offset, 500);
+    }
+
+    #[test]
+    fn test_global_meta_default_when_missing() {
+        let dir = TempDir::new().unwrap();
+        let meta = GlobalMeta::load(dir.path()).unwrap();
+        assert_eq!(meta.active_segment_id, 1);
+        assert!(meta.segments.is_empty());
+    }
+
+    #[test]
+    fn test_corrupt_bin_detected() {
+        let dir = TempDir::new().unwrap();
+        std::fs::write(dir.path().join("meta.bin"), vec![0xFFu8; 100]).unwrap();
+        let result = GlobalMeta::load(dir.path());
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_segment_stats_recompute() {
+        let mut s = SegmentStats::new(1);
+        s.total_bytes = 1000;
+        s.deleted_bytes = 250;
+        s.recompute_ratio();
+        assert!((s.deleted_ratio - 0.25).abs() < 0.001);
+    }
+}

crates/blob/src/recovery.rs

@@ -0,0 +1,144 @@
+use std::fs;
+use std::path::Path;
+
+use crate::bucket::IndexRecord;
+use crate::error::Result;
+use crate::meta::{GlobalMeta, SegmentStats};
+use crate::segment::{self, SegmentReader};
+
+/// Recover after a crash: scan any unindexed portions of segments,
+/// update segment stats, and return newly discovered index records.
+///
+/// The caller is responsible for inserting the returned records into
+/// the index store.
+pub fn recover(store_root: &Path, meta: &mut GlobalMeta) -> Result<Vec<IndexRecord>> {
+    scan_segments(store_root, meta, false)
+}
+
+/// Rebuild the entire index from scratch by scanning all segments from
+/// offset 0.  Used when the index database is corrupted or lost.
+///
+/// Resets all segment stats and returns every entry found on disk.
+/// The caller should replace the index database before calling this.
+pub fn rebuild_index(store_root: &Path, meta: &mut GlobalMeta) -> Result<Vec<IndexRecord>> {
+    // Reset all segment stats — they'll be recomputed during the scan.
+    // Also reset indexed_up_to_offset so we scan from 0.
+    for stats in meta.segments.values_mut() {
+        stats.total_bytes = 0;
+        stats.deleted_bytes = 0;
+        stats.deleted_ratio = 0.0;
+        stats.indexed_up_to_offset = 0;
+    }
+    scan_segments(store_root, meta, true)
+}
+
+/// Common implementation: scan segments and collect index records.
+/// When `full_scan` is true, every segment is scanned from offset 0.
+fn scan_segments(
+    store_root: &Path,
+    meta: &mut GlobalMeta,
+    full_scan: bool,
+) -> Result<Vec<IndexRecord>> {
+    let seg_dir = store_root.join("segments");
+    if !seg_dir.exists() {
+        fs::create_dir_all(&seg_dir)?;
+    }
+
+    // Discover all segment files on disk
+    let mut disk_segments: Vec<u32> = Vec::new();
+    if seg_dir.exists() {
+        for entry in fs::read_dir(&seg_dir)? {
+            let entry = entry?;
+            let name = entry.file_name();
+            let name_str = name.to_string_lossy();
+            if name_str.ends_with(".seg") && !name_str.contains("temp_") {
+                if let Some(id_str) = name_str.strip_suffix(".seg") {
+                    if let Ok(id) = id_str.parse::<u32>() {
+                        disk_segments.push(id);
+                    }
+                }
+            }
+        }
+    }
+    disk_segments.sort_unstable();
+
+    let mut all_records: Vec<IndexRecord> = Vec::new();
+
+    // For each segment, scan unindexed portions
+    for &seg_id in &disk_segments {
+        let seg_path = seg_dir.join(segment::segment_filename(seg_id));
+        let file_size = fs::metadata(&seg_path)?.len();
+
+        let mut stats = meta
+            .segments
+            .remove(&seg_id)
+            .unwrap_or_else(|| SegmentStats::new(seg_id));
+        let is_sealed = seg_id != meta.active_segment_id;
+        stats.sealed = is_sealed;
+
+        // Determine scan range.
+        let scan_start = if full_scan {
+            0
+        } else if stats.indexed_up_to_offset <= file_size {
+            stats.indexed_up_to_offset
+        } else {
+            // Segment was replaced (interrupted GC) — full rescan needed.
+            0
+        };
+
+        if scan_start >= file_size {
+            meta.segments.insert(seg_id, stats);
+            continue;
+        }
+
+        let reader = SegmentReader::open(seg_path.clone(), seg_id)?;
+
+        let truncation_point = reader.scan_entries(scan_start, |entry, offset| {
+            all_records.push(IndexRecord::new(
+                entry.key,
+                seg_id,
+                offset,
+                entry.data.len() as u32,
+                entry.flags,
+            ));
+
+            stats.total_bytes += entry.data.len() as u64;
+            if entry.is_tombstone() {
+                stats.deleted_bytes += entry.raw_size as u64;
+            }
+
+            Ok(())
+        })?;
+
+        // Truncate if tail corruption found
+        if truncation_point < file_size {
+            segment::truncate_segment(&seg_path, truncation_point)?;
+        }
+
+        stats.indexed_up_to_offset = truncation_point;
+        stats.recompute_ratio();
+        meta.segments.insert(seg_id, stats);
+    }
+
+    meta.save(store_root)?;
+
+    Ok(all_records)
+}
+
+/// Clean up leftover temp files from interrupted GC.
+pub fn cleanup_temp_files(store_root: &Path) -> Result<()> {
+    let seg_dir = store_root.join("segments");
+    if seg_dir.exists() {
+        for entry in fs::read_dir(&seg_dir)? {
+            let entry = entry?;
+            let name = entry.file_name();
+            let name_str = name.to_string_lossy();
+            if name_str.starts_with("temp_") {
+                let path = entry.path();
+                tracing::warn!("Removing leftover temp file: {:?}", path);
+                fs::remove_file(&path)?;
+            }
+        }
+    }
+    Ok(())
+}

crates/blob/src/segment.rs

@@ -0,0 +1,551 @@
+use std::fs::{self, File};
+use std::io::{Read, Seek, SeekFrom, Write};
+use std::path::{Path, PathBuf};
+
+use crate::checksum;
+use crate::error::{Error, Result};
+use crate::fs as fs_util;
+use crate::types::{Codec, ENTRY_HEADER_SIZE, ENTRY_MAGIC, SEGMENT_MAX_SIZE};
+
+/// In-memory representation of a stored entry.
+#[derive(Debug, Clone)]
+pub struct Entry {
+    pub flags: u8,
+    pub codec: Codec,
+    pub key: [u8; 32],
+    pub raw_size: u32,
+    pub data: Vec<u8>,
+}
+
+impl Entry {
+    /// Create a normal data entry.
+    /// `raw_size` is the original uncompressed size; `data` is what goes to disk.
+    pub fn new(key: [u8; 32], data: &[u8], raw_size: u32, flags: u8, codec: Codec) -> Self {
+        Self {
+            flags,
+            codec,
+            key,
+            raw_size,
+            data: data.to_vec(),
+        }
+    }
+
+    /// Create a tombstone entry.
+    pub fn tombstone(key: [u8; 32]) -> Self {
+        Self {
+            flags: 1,
+            codec: Codec::None,
+            key,
+            raw_size: 0,
+            data: Vec::new(),
+        }
+    }
+
+    pub fn is_tombstone(&self) -> bool {
+        self.flags == 1
+    }
+
+    /// Total on-disk size: header + data
+    pub fn disk_size(&self) -> usize {
+        ENTRY_HEADER_SIZE + self.data.len()
+    }
+}
+
+/// Write entries sequentially to a segment file.
+pub struct SegmentWriter {
+    file: File,
+    path: PathBuf,
+    id: u32,
+    bytes_written: u64,
+}
+
+impl SegmentWriter {
+    pub fn create(path: PathBuf, id: u32) -> Result<Self> {
+        // Use create+truncate instead of create_new to avoid NFS O_EXCL issues.
+        let file = File::create(&path)?;
+        Ok(Self {
+            file,
+            path,
+            id,
+            bytes_written: 0,
+        })
+    }
+
+    pub fn open_append(path: PathBuf, id: u32) -> Result<Self> {
+        let mut file = fs_util::open_write(&path)?;
+        file.seek(SeekFrom::End(0))?;
+        let bytes_written = file.stream_position()?;
+        Ok(Self {
+            file,
+            path,
+            id,
+            bytes_written,
+        })
+    }
+
+    pub fn id(&self) -> u32 {
+        self.id
+    }
+
+    pub fn path(&self) -> &Path {
+        &self.path
+    }
+
+    pub fn bytes_written(&self) -> u64 {
+        self.bytes_written
+    }
+
+    pub fn is_full(&self) -> bool {
+        self.bytes_written >= SEGMENT_MAX_SIZE
+    }
+
+    /// Append an entry. Returns the offset where it was written.
+    pub fn append(&mut self, entry: &Entry) -> Result<u64> {
+        let offset = self.bytes_written;
+        self.write_entry(entry)
+            .map_err(|e| map_io_err(e, &self.path))?;
+        Ok(offset)
+    }
+
+    fn write_entry(&mut self, entry: &Entry) -> Result<()> {
+        let data_size = entry.data.len() as u32;
+
+        // Write magic
+        self.file.write_all(&ENTRY_MAGIC.to_le_bytes())?;
+
+        // CRC32 placeholder: write zeros, remember position
+        let crc_pos = self.file.stream_position()?;
+        self.file.write_all(&0u32.to_le_bytes())?;
+
+        // Write flags, codec, key, raw_size, data_size
+        self.file.write_all(&[entry.flags])?;
+        self.file.write_all(&[entry.codec as u8])?;
+        self.file.write_all(&entry.key)?;
+        self.file.write_all(&entry.raw_size.to_le_bytes())?;
+        self.file.write_all(&data_size.to_le_bytes())?;
+
+        // Write data
+        self.file.write_all(&entry.data)?;
+
+        // Calculate CRC32 over everything after the crc32 field
+        let crc = {
+            let mut hasher = checksum::CrcWriter::new();
+            hasher.update(&[entry.flags]);
+            hasher.update(&[entry.codec as u8]);
+            hasher.update(&entry.key);
+            hasher.update(&entry.raw_size.to_le_bytes());
+            hasher.update(&data_size.to_le_bytes());
+            hasher.update(&entry.data);
+            hasher.finalize()
+        };
+
+        // Seek back and write the real CRC32
+        self.file.seek(SeekFrom::Start(crc_pos))?;
+        self.file.write_all(&crc.to_le_bytes())?;
+
+        // Seek back to end
+        self.file.seek(SeekFrom::End(0))?;
+
+        self.bytes_written += entry.disk_size() as u64;
+        Ok(())
+    }
+
+    pub fn fsync(&self) -> Result<()> {
+        self.file.sync_all().map_err(|e| {
+            if e.kind() == std::io::ErrorKind::StorageFull {
+                Error::DiskFull(format!("{}: {}", self.path.display(), e))
+            } else {
+                Error::Io(e)
+            }
+        })?;
+        Ok(())
+    }
+}
+
+/// Read entries from a segment file.
+pub struct SegmentReader {
+    path: PathBuf,
+    id: u32,
+}
+
+impl SegmentReader {
+    pub fn open(path: PathBuf, id: u32) -> Result<Self> {
+        Ok(Self { path, id })
+    }
+
+    pub fn id(&self) -> u32 {
+        self.id
+    }
+
+    pub fn path(&self) -> &Path {
+        &self.path
+    }
+
+    pub fn file_size(&self) -> Result<u64> {
+        Ok(fs::metadata(&self.path)?.len())
+    }
+
+    /// Read a single entry at the given offset. Returns the entry and the offset of the next entry.
+    pub fn read_entry_at(&self, offset: u64) -> Result<(Entry, u64)> {
+        let mut file = fs_util::open_read(&self.path)?;
+        file.seek(SeekFrom::Start(offset))?;
+
+        // Read magic
+        let mut magic_buf = [0u8; 4];
+        file.read_exact(&mut magic_buf)?;
+        let magic = u32::from_le_bytes(magic_buf);
+        if magic != ENTRY_MAGIC {
+            return Err(Error::CorruptEntry {
+                path: self.path.clone(),
+                offset,
+                reason: format!("bad magic: 0x{:08X}", magic),
+            });
+        }
+
+        // Read CRC32
+        let mut crc_buf = [0u8; 4];
+        file.read_exact(&mut crc_buf)?;
+        let stored_crc = u32::from_le_bytes(crc_buf);
+
+        // Read flags, codec
+        let mut flags_buf = [0u8; 1];
+        file.read_exact(&mut flags_buf)?;
+        let flags = flags_buf[0];
+
+        let mut codec_buf = [0u8; 1];
+        file.read_exact(&mut codec_buf)?;
+        let codec = Codec::from_u8(codec_buf[0]).ok_or_else(|| Error::CorruptEntry {
+            path: self.path.clone(),
+            offset,
+            reason: format!("unknown codec: {}", codec_buf[0]),
+        })?;
+
+        // Read key, raw_size, data_size
+        let mut key = [0u8; 32];
+        file.read_exact(&mut key)?;
+
+        let mut raw_size_buf = [0u8; 4];
+        file.read_exact(&mut raw_size_buf)?;
+        let raw_size = u32::from_le_bytes(raw_size_buf);
+
+        let mut data_size_buf = [0u8; 4];
+        file.read_exact(&mut data_size_buf)?;
+        let data_size = u32::from_le_bytes(data_size_buf);
+
+        if data_size as usize > crate::types::MAX_VALUE_SIZE {
+            return Err(Error::CorruptEntry {
+                path: self.path.clone(),
+                offset,
+                reason: format!("data_size {} exceeds max {}", data_size, crate::types::MAX_VALUE_SIZE),
+            });
+        }
+
+        // Read data
+        let mut data = vec![0u8; data_size as usize];
+        file.read_exact(&mut data)?;
+
+        // Verify CRC32 (over everything after the crc32 field)
+        let computed_crc = {
+            let mut hasher = checksum::CrcWriter::new();
+            hasher.update(&[flags]);
+            hasher.update(&[codec as u8]);
+            hasher.update(&key);
+            hasher.update(&raw_size.to_le_bytes());
+            hasher.update(&data_size.to_le_bytes());
+            hasher.update(&data);
+            hasher.finalize()
+        };
+
+        if stored_crc != computed_crc {
+            return Err(Error::CrcMismatch {
+                path: self.path.clone(),
+                offset,
+            });
+        }
+
+        let next_offset = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
+
+        Ok((
+            Entry {
+                flags,
+                codec,
+                key,
+                raw_size,
+                data,
+            },
+            next_offset,
+        ))
+    }
+
+    /// Read a single entry at the given offset using a pre-opened File.
+    /// Uses pread so concurrent reads on the same segment don't block each other.
+    pub fn read_entry_at_file(&self, offset: u64, file: &File) -> Result<(Entry, u64)> {
+        // Read header (50 bytes)
+        let mut header = [0u8; ENTRY_HEADER_SIZE];
+        fs_util::pread_exact(file, offset, &mut header)?;
+
+        let mut pos = 0;
+
+        // Magic
+        let magic = u32::from_le_bytes(header[pos..pos+4].try_into().unwrap());
+        pos += 4;
+        if magic != ENTRY_MAGIC {
+            return Err(Error::CorruptEntry {
+                path: self.path.clone(),
+                offset,
+                reason: format!("bad magic: 0x{:08X}", magic),
+            });
+        }
+
+        // CRC32
+        let stored_crc = u32::from_le_bytes(header[pos..pos+4].try_into().unwrap());
+        pos += 4;
+
+        // Flags, codec
+        let flags = header[pos];
+        pos += 1;
+        let codec = Codec::from_u8(header[pos]).ok_or_else(|| Error::CorruptEntry {
+            path: self.path.clone(),
+            offset,
+            reason: format!("unknown codec: {}", header[pos]),
+        })?;
+        pos += 1;
+
+        // Key
+        let mut key = [0u8; 32];
+        key.copy_from_slice(&header[pos..pos+32]);
+        pos += 32;
+
+        // Raw size, data size
+        let raw_size = u32::from_le_bytes(header[pos..pos+4].try_into().unwrap());
+        pos += 4;
+        let data_size = u32::from_le_bytes(header[pos..pos+4].try_into().unwrap());
+
+        // Defense against header corruption: refuse absurdly large allocations
+        if data_size as usize > crate::types::MAX_VALUE_SIZE {
+            return Err(Error::CorruptEntry {
+                path: self.path.clone(),
+                offset,
+                reason: format!("data_size {} exceeds max {}", data_size, crate::types::MAX_VALUE_SIZE),
+            });
+        }
+
+        // Read data
+        let data_offset = offset + ENTRY_HEADER_SIZE as u64;
+        let mut data = vec![0u8; data_size as usize];
+        fs_util::pread_exact(file, data_offset, &mut data)?;
+
+        // Verify CRC32 (over everything after the crc32 field: flags+codec+key+raw_size+data_size+data)
+        let computed_crc = {
+            let mut hasher = crate::checksum::CrcWriter::new();
+            hasher.update(&[flags]);
+            hasher.update(&[codec as u8]);
+            hasher.update(&key);
+            hasher.update(&raw_size.to_le_bytes());
+            hasher.update(&data_size.to_le_bytes());
+            hasher.update(&data);
+            hasher.finalize()
+        };
+
+        if stored_crc != computed_crc {
+            return Err(Error::CrcMismatch {
+                path: self.path.clone(),
+                offset,
+            });
+        }
+
+        let next_offset = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
+
+        Ok((
+            Entry {
+                flags,
+                codec,
+                key,
+                raw_size,
+                data,
+            },
+            next_offset,
+        ))
+    }
+
+    /// Read data portion of an entry (for pread-style reads when you already know offset + data_size).
+    pub fn read_data(&self, offset: u64, data_size: u32) -> Result<Vec<u8>> {
+        let mut file = fs_util::open_read(&self.path)?;
+        // Skip magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4) = 50 bytes
+        let data_start = offset + ENTRY_HEADER_SIZE as u64;
+        file.seek(SeekFrom::Start(data_start))?;
+        let mut buf = vec![0u8; data_size as usize];
+        file.read_exact(&mut buf)?;
+        Ok(buf)
+    }
+
+    /// Read the full entry header + data for verification (used by recovery and GC).
+    pub fn read_full_entry(&self, offset: u64, data_size: u32) -> Result<Vec<u8>> {
+        let mut file = fs_util::open_read(&self.path)?;
+        file.seek(SeekFrom::Start(offset))?;
+        let total = ENTRY_HEADER_SIZE + data_size as usize;
+        let mut buf = vec![0u8; total];
+        file.read_exact(&mut buf)?;
+        Ok(buf)
+    }
+
+    /// Iterate over all valid entries in the segment, calling f for each.
+    /// Stops when hitting a corrupt/incomplete entry at the tail.
+    pub fn scan_entries<F>(&self, start_offset: u64, mut f: F) -> Result<u64>
+    where
+        F: FnMut(&Entry, u64) -> Result<()>,
+    {
+        let file_size = self.file_size()?;
+        let mut offset = start_offset;
+
+        while offset + ENTRY_HEADER_SIZE as u64 <= file_size {
+            match self.read_entry_at(offset) {
+                Ok((entry, next)) => {
+                    f(&entry, offset)?;
+                    offset = next;
+                }
+                Err(Error::CrcMismatch { .. }) | Err(Error::CorruptEntry { .. }) => {
+                    // If near end of file (within one max entry), truncate
+                    if file_size - offset < ENTRY_HEADER_SIZE as u64 + 100 * 1024 * 1024 {
+                        // Likely a partial write at tail, stop here
+                        break;
+                    } else {
+                        return Err(Error::CorruptEntry {
+                            path: self.path.clone(),
+                            offset,
+                            reason: "mid-file corruption detected".into(),
+                        });
+                    }
+                }
+                Err(e) => return Err(e),
+            }
+        }
+
+        Ok(offset) // return the truncation point
+    }
+}
+
+/// Truncate a segment file to the given size.
+pub fn truncate_segment(path: &Path, size: u64) -> Result<()> {
+    fs_util::truncate(path, size)
+}
+
+/// Map an Error, converting Io(StorageFull) to DiskFull with path context.
+fn map_io_err(e: Error, path: &Path) -> Error {
+    match e {
+        Error::Io(io) if io.kind() == std::io::ErrorKind::StorageFull => {
+            Error::DiskFull(format!("{}: {}", path.display(), io))
+        }
+        _ => e,
+    }
+}
+
+/// Segment file name from id: "00000001.seg"
+pub fn segment_filename(id: u32) -> String {
+    format!("{:08}.seg", id)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::TempDir;
+
+    fn temp_segment_path(dir: &TempDir, id: u32) -> PathBuf {
+        dir.path().join(segment_filename(id))
+    }
+
+    #[test]
+    fn test_write_and_read_entry() {
+        let dir = TempDir::new().unwrap();
+        let path = temp_segment_path(&dir, 1);
+        let key = [0xAAu8; 32];
+        let data = b"hello world".to_vec();
+
+        let entry = Entry::new(key, &data, data.len() as u32, 0, Codec::None);
+        {
+            let mut writer = SegmentWriter::create(path.clone(), 1).unwrap();
+            writer.append(&entry).unwrap();
+            writer.fsync().unwrap();
+        }
+
+        let reader = SegmentReader::open(path, 1).unwrap();
+        let (read_entry, next) = reader.read_entry_at(0).unwrap();
+
+        assert_eq!(read_entry.key, key);
+        assert_eq!(read_entry.data, data);
+        assert_eq!(read_entry.flags, 0);
+        assert_eq!(read_entry.raw_size, 11);
+        assert!(next > 0);
+    }
+
+    #[test]
+    fn test_tombstone_entry() {
+        let dir = TempDir::new().unwrap();
+        let path = temp_segment_path(&dir, 1);
+        let key = [0xBBu8; 32];
+
+        let entry = Entry::tombstone(key);
+        {
+            let mut writer = SegmentWriter::create(path.clone(), 1).unwrap();
+            writer.append(&entry).unwrap();
+            writer.fsync().unwrap();
+        }
+
+        let reader = SegmentReader::open(path, 1).unwrap();
+        let (read_entry, _) = reader.read_entry_at(0).unwrap();
+
+        assert!(read_entry.is_tombstone());
+        assert_eq!(read_entry.data.len(), 0);
+    }
+
+    #[test]
+    fn test_multiple_entries() {
+        let dir = TempDir::new().unwrap();
+        let path = temp_segment_path(&dir, 1);
+
+        let entries: Vec<_> = (0..10)
+            .map(|i| {
+                let mut key = [0u8; 32];
+                key[0] = i;
+                Entry::new(key, &vec![i; 100], 100, 0, Codec::None)
+            })
+            .collect();
+
+        {
+            let mut writer = SegmentWriter::create(path.clone(), 1).unwrap();
+            for e in &entries {
+                writer.append(e).unwrap();
+            }
+            writer.fsync().unwrap();
+        }
+
+        let reader = SegmentReader::open(path, 1).unwrap();
+        let mut offset = 0u64;
+        for (i, expected) in entries.iter().enumerate() {
+            let (entry, next) = reader.read_entry_at(offset).unwrap();
+            assert_eq!(entry.key[0], i as u8);
+            assert_eq!(entry.data, expected.data);
+            offset = next;
+        }
+    }
+
+    #[test]
+    fn test_bad_magic_detected() {
+        let dir = TempDir::new().unwrap();
+        let path = temp_segment_path(&dir, 1);
+        // Write garbage
+        std::fs::write(&path, vec![0xFFu8; 100]).unwrap();
+
+        let reader = SegmentReader::open(path, 1).unwrap();
+        let result = reader.read_entry_at(0);
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_is_full() {
+        let dir = TempDir::new().unwrap();
+        let path = temp_segment_path(&dir, 1);
+        let writer = SegmentWriter::create(path, 1).unwrap();
+        assert!(!writer.is_full());
+    }
+}

crates/blob/src/types.rs

@@ -0,0 +1,98 @@
+use serde::{Deserialize, Serialize};
+
+/// Magic number for entry identification
+pub const ENTRY_MAGIC: u32 = 0xB3DB_0001;
+
+/// Fixed header size: magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4)
+pub const ENTRY_HEADER_SIZE: usize = 50;
+
+/// Index record size: key(32) + segment_id(4) + offset(8) + data_size(4) + flags(1) + _pad(3) + crc32(4)
+pub const INDEX_RECORD_SIZE: usize = 56;
+
+/// Maximum segment size (1 GB)
+pub const SEGMENT_MAX_SIZE: u64 = 1024 * 1024 * 1024;
+
+/// Maximum value size (100 MB)
+pub const MAX_VALUE_SIZE: usize = 100 * 1024 * 1024;
+
+/// Default compression threshold (4 KB)
+pub const DEFAULT_COMPRESS_THRESHOLD: usize = 4096;
+
+/// Default GC deleted ratio threshold
+pub const DEFAULT_GC_DELETED_RATIO: f64 = 0.30;
+
+/// Default GC interval in seconds (5 minutes)
+pub const DEFAULT_GC_INTERVAL_SECS: u64 = 300;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+pub enum Codec {
+    None = 0,
+    Zstd = 1,
+    Lz4 = 2,
+}
+
+impl Codec {
+    pub fn from_u8(v: u8) -> Option<Self> {
+        match v {
+            0 => Some(Codec::None),
+            1 => Some(Codec::Zstd),
+            2 => Some(Codec::Lz4),
+            _ => None,
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Config {
+    pub compress_threshold: usize,
+    pub default_codec: Codec,
+    pub compression_level: i32,
+    pub gc_deleted_ratio: f64,
+    /// Interval in seconds for periodic background flush (0 = disabled).
+    /// When set, a background thread fsyncs the active segment and saves
+    /// metadata at this interval, bounding recovery time after a crash.
+    pub flush_interval_secs: u64,
+    /// Interval in seconds for periodic background GC (0 = disabled).
+    /// When set, a background thread checks whether any sealed segment
+    /// exceeds the deleted-ratio threshold and compacts it if needed.
+    pub gc_interval_secs: u64,
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            compress_threshold: DEFAULT_COMPRESS_THRESHOLD,
+            default_codec: Codec::Zstd,
+            compression_level: 0,
+            gc_deleted_ratio: DEFAULT_GC_DELETED_RATIO,
+            flush_interval_secs: 0,
+            gc_interval_secs: 0,
+        }
+    }
+}
+
+impl Config {
+    pub fn validate(&self) -> crate::error::Result<()> {
+        if self.gc_deleted_ratio <= 0.0 || self.gc_deleted_ratio >= 1.0 {
+            return Err(crate::error::Error::InvalidConfig(
+                "gc_deleted_ratio must be in (0.0, 1.0)".into(),
+            ));
+        }
+        if self.compression_level < 0 {
+            return Err(crate::error::Error::InvalidConfig(
+                "compression_level must be >= 0".into(),
+            ));
+        }
+        if self.flush_interval_secs > 0 && self.flush_interval_secs < 5 {
+            return Err(crate::error::Error::InvalidConfig(
+                "flush_interval_secs must be 0 (disabled) or >= 5".into(),
+            ));
+        }
+        if self.gc_interval_secs > 0 && self.gc_interval_secs < 10 {
+            return Err(crate::error::Error::InvalidConfig(
+                "gc_interval_secs must be 0 (disabled) or >= 10".into(),
+            ));
+        }
+        Ok(())
+    }
+}

crates/blob/tests/acid_test.rs

@@ -0,0 +1,514 @@
+/// Crash-consistency and ACID property tests for bichon-blob.
+///
+/// Since we can't kill the process mid-write in an inline test, we simulate crashes
+/// by dropping the Engine without calling any cleanup (close/drop is the "crash"),
+/// then re-opening and verifying recovery produced consistent state.
+
+use std::fs;
+use std::path::Path;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::thread;
+
+use bichon_blob::{Codec, Config, Engine};
+use tempfile::TempDir;
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+fn make_key(seed: u64) -> [u8; 32] {
+    let mut key = [0u8; 32];
+    key[0..8].copy_from_slice(&seed.to_le_bytes());
+    key
+}
+
+fn make_value(size: usize) -> Vec<u8> {
+    let pattern = b"The quick brown fox jumps over the lazy dog. ";
+    let mut v = Vec::with_capacity(size);
+    while v.len() < size {
+        let rem = size - v.len();
+        let n = rem.min(pattern.len());
+        v.extend_from_slice(&pattern[..n]);
+    }
+    v
+}
+
+// ---------------------------------------------------------------------------
+// 1. Durability: committed data survives crash
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_durability_single_write_survives_crash() {
+    let dir = TempDir::new().unwrap();
+    let key = make_key(42);
+    let value = make_value(8192);
+
+    // Write
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        engine.put(key, &value, Codec::Zstd).unwrap();
+    } // <-- Engine dropped = simulated crash
+
+    // Recover
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        let result = engine.get(&key).unwrap();
+        assert_eq!(result, Some(value));
+    }
+}
+
+#[test]
+fn test_durability_many_writes_survive_crash() {
+    let dir = TempDir::new().unwrap();
+    let n = 500;
+    let value = make_value(2048);
+    let mut keys = Vec::new();
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for i in 0..n {
+            let key = make_key(i as u64);
+            keys.push(key);
+            engine.put(key, &value, Codec::Zstd).unwrap();
+        }
+    } // crash
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for (i, key) in keys.iter().enumerate() {
+            let result = engine.get(key).unwrap();
+            assert_eq!(
+                result,
+                Some(value.clone()),
+                "missing key at index {}",
+                i
+            );
+        }
+    }
+}
+
+#[test]
+fn test_durability_delete_survives_crash() {
+    let dir = TempDir::new().unwrap();
+    let key = make_key(99);
+    let value = make_value(4096);
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        engine.put(key, &value, Codec::Zstd).unwrap();
+    } // crash after write
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        engine.delete(&key).unwrap();
+    } // crash after delete
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        let result = engine.get(&key).unwrap();
+        assert_eq!(result, None, "delete should persist across crash");
+    }
+}
+
+// ---------------------------------------------------------------------------
+// 2. Atomicity: no partial writes visible after crash
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_atomicity_no_partial_entries_after_crash() {
+    let dir = TempDir::new().unwrap();
+
+    // Write enough entries to fill part of a segment, then crash
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        let value = make_value(50_000);
+        for i in 0..200u64 {
+            engine
+                .put(make_key(i), &value, Codec::None)
+                .unwrap();
+        }
+    } // crash
+
+    // Recovery should clean up any partial tail entries
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        let value = make_value(50_000);
+        for i in 0..200u64 {
+            let result = engine.get(&make_key(i)).unwrap();
+            assert_eq!(
+                result,
+                Some(value.clone()),
+                "committed key {} should be intact",
+                i
+            );
+        }
+    }
+}
+
+#[test]
+fn test_atomicity_crash_during_segment_roll() {
+    let dir = TempDir::new().unwrap();
+    let big_value = make_value(2 * 1024 * 1024); // 2 MB each entry
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        // Write enough to cross at least one segment boundary (256 MB)
+        for i in 0..140u64 {
+            engine
+                .put(make_key(i), &big_value, Codec::None)
+                .unwrap();
+        }
+    } // crash mid-way or after multiple segments
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for i in 0..140u64 {
+            let result = engine.get(&make_key(i)).unwrap();
+            assert!(
+                result.is_some(),
+                "key {} should exist after segment roll recovery",
+                i
+            );
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// 3. Consistency: CRC detects corruption, no silent data loss
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_consistency_crc_detects_corruption() {
+    let dir = TempDir::new().unwrap();
+    let key = make_key(77);
+    let value = make_value(8192);
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        engine.put(key, &value, Codec::Zstd).unwrap();
+    }
+
+    // Corrupt the segment file by flipping a byte
+    let seg_path = find_first_segment(dir.path());
+    let mut data = fs::read(&seg_path).unwrap();
+    let flip_pos = data.len() - 100;
+    data[flip_pos] ^= 0xFF;
+    fs::write(&seg_path, &data).unwrap();
+
+    // Reading should detect CRC mismatch
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        let result = engine.get(&key);
+        match result {
+            Err(_) => {} // CRC mismatch detected - good
+            Ok(None) => {} // index may point to truncated/removed data
+            Ok(Some(v)) => {
+                if v == value {
+                    panic!("CRC corruption was NOT detected - silent data corruption!");
+                }
+            }
+        }
+    }
+}
+
+#[test]
+fn test_consistency_corrupt_magic_truncated_on_recovery() {
+    let dir = TempDir::new().unwrap();
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for i in 0..10u64 {
+            engine
+                .put(make_key(i), &make_value(4096), Codec::Zstd)
+                .unwrap();
+        }
+    }
+
+    // Append garbage to the segment file (simulating partial write from crash)
+    let seg_path = find_first_segment(dir.path());
+    let mut data = fs::read(&seg_path).unwrap();
+    let orig_len = data.len();
+    data.extend_from_slice(&[0xFF; 200]);
+    fs::write(&seg_path, &data).unwrap();
+
+    // Recovery should truncate the garbage
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for i in 0..10u64 {
+            let result = engine.get(&make_key(i)).unwrap();
+            assert!(
+                result.is_some(),
+                "committed key {} should survive tail truncation",
+                i
+            );
+        }
+    }
+
+    // Verify file was actually truncated
+    let truncated_len = fs::metadata(&seg_path).unwrap().len();
+    assert!(
+        truncated_len <= orig_len as u64,
+        "garbage should have been truncated"
+    );
+}
+
+// ---------------------------------------------------------------------------
+// 4. Isolation: concurrent reader sees consistent snapshot
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_isolation_reader_sees_snapshot_not_partial_write() {
+    let dir = TempDir::new().unwrap();
+    let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
+
+    // Pre-populate a known key
+    let original_value = make_value(4096);
+    let key = make_key(100);
+    engine.put(key, &original_value, Codec::Zstd).unwrap();
+
+    let running = Arc::new(AtomicBool::new(true));
+
+    // Spawn a writer that continuously overwrites the same key
+    let writer_engine = engine.clone();
+    let writer_running = running.clone();
+    let writer_key = key;
+
+    let writer = thread::spawn(move || {
+        for i in 0..1000u64 {
+            if !writer_running.load(Ordering::Relaxed) {
+                break;
+            }
+            let val = make_value(4096 + (i as usize % 100));
+            writer_engine
+                .put(writer_key, &val, Codec::Zstd)
+                .unwrap();
+            thread::yield_now();
+        }
+    });
+
+    // Concurrent reader: reads should never panic or hang
+    let reader_engine = engine.clone();
+    let reader_running = running.clone();
+    let reader = thread::spawn(move || {
+        let mut reads = 0;
+        while reads < 500 {
+            if !reader_running.load(Ordering::Relaxed) && reads > 0 {
+                break;
+            }
+            let result = reader_engine.get(&key);
+            match result {
+                Ok(Some(_)) | Ok(None) => {}
+                Err(e) => {
+                    eprintln!("reader saw error: {:?}", e);
+                }
+            }
+            reads += 1;
+            thread::yield_now();
+        }
+    });
+
+    reader.join().unwrap();
+    running.store(false, Ordering::SeqCst);
+    writer.join().unwrap();
+
+    let final_result = engine.get(&key).unwrap();
+    assert!(final_result.is_some(), "final read should find a value");
+}
+
+// ---------------------------------------------------------------------------
+// 5. Crash during GC: old data intact, no corruption
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_crash_during_gc_leaves_data_intact() {
+    let dir = TempDir::new().unwrap();
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+        let value = make_value(500_000);
+        for i in 0..500u64 {
+            engine
+                .put(make_key(i), &value, Codec::None)
+                .unwrap();
+        }
+        // Delete ~40%
+        for i in (0..500u64).step_by(5) {
+            engine.delete(&make_key(i)).unwrap();
+        }
+        let _ = engine.gc();
+    } // crash after GC
+
+    // All non-deleted entries must still be readable
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        let value = make_value(500_000);
+        for i in 0..500u64 {
+            let key = make_key(i);
+            let result = engine.get(&key).unwrap();
+            if i % 5 == 0 {
+                assert_eq!(result, None, "key {} should be deleted", i);
+            } else {
+                assert_eq!(
+                    result,
+                    Some(value.clone()),
+                    "key {} should survive GC+crash",
+                    i
+                );
+            }
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// 6. Multiple crash-reopen cycles (torture test)
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_multiple_crash_reopen_cycles() {
+    use std::collections::HashSet;
+
+    let dir = TempDir::new().unwrap();
+    let value = make_value(4096);
+    let mut alive: HashSet<u64> = HashSet::new();
+
+    // Populate and crash
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for i in 0..50u64 {
+            engine.put(make_key(i), &value, Codec::Zstd).unwrap();
+            alive.insert(i);
+        }
+    }
+
+    // Reopen, verify all exist, write more, crash
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for &k in &alive {
+            assert!(engine.get(&make_key(k)).unwrap().is_some());
+        }
+        for i in 100..150u64 {
+            engine
+                .put(make_key(i), &value, Codec::Zstd)
+                .unwrap();
+            alive.insert(i);
+        }
+    }
+
+    // Reopen, verify all exist, delete some, crash
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for &k in &alive {
+            assert!(engine.get(&make_key(k)).unwrap().is_some());
+        }
+        for i in 0..10u64 {
+            engine.delete(&make_key(i)).unwrap();
+            alive.remove(&i);
+        }
+    }
+
+    // Final reopen: survivors exist, deleted gone
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for &k in &alive {
+            assert!(
+                engine.get(&make_key(k)).unwrap().is_some(),
+                "key {} should exist",
+                k
+            );
+        }
+        for i in 0..10u64 {
+            assert_eq!(
+                engine.get(&make_key(i)).unwrap(),
+                None,
+                "key {} should be deleted",
+                i
+            );
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// 7. Global dedup: same content stored once
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_global_dedup_after_crash() {
+    let dir = TempDir::new().unwrap();
+    let key = make_key(42);
+    let value = make_value(8192);
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        // Write same key twice (simulating two sources with same content)
+        engine.put(key, &value, Codec::Zstd).unwrap();
+        engine.put(key, &value, Codec::Zstd).unwrap();
+    }
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        let result = engine.get(&key).unwrap();
+        assert_eq!(result, Some(value));
+    }
+}
+
+// ---------------------------------------------------------------------------
+// 8. Recovery + mmap consistency: entries re-indexed by recovery are readable
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_recovery_reloads_bucket_mmaps() {
+    use bichon_blob::meta::GlobalMeta;
+
+    let dir = TempDir::new().unwrap();
+    let value = make_value(8192);
+    let n = 50u64;
+
+    // Phase 1: write data with clean shutdown, then corrupt meta to force
+    // re-indexing on next open (simulates crash where mark_indexed didn't run).
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for i in 0..n {
+            engine.put(make_key(i), &value, Codec::None).unwrap();
+        }
+    } // clean shutdown — everything is indexed and fsynced
+
+    // Corrupt meta: zero out indexed_up_to_offset so recovery re-scans.
+    let mut meta = GlobalMeta::load(dir.path()).expect("failed to load meta");
+    for seg in meta.segments.values_mut() {
+        seg.indexed_up_to_offset = 0;
+    }
+    meta.save(dir.path()).expect("failed to save corrupted meta");
+
+    // Phase 2: reopen — recovery must re-scan the segment and append to bucket
+    // files. After the fix, reload_all() ensures the mmaps include recovered data.
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for i in 0..n {
+            let result = engine.get(&make_key(i)).unwrap();
+            assert_eq!(
+                result,
+                Some(value.clone()),
+                "key {} should be readable after recovery reload",
+                i
+            );
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+fn find_first_segment(store_root: &Path) -> std::path::PathBuf {
+    let seg_dir = store_root.join("segments");
+    for entry in fs::read_dir(&seg_dir).unwrap() {
+        let entry = entry.unwrap();
+        let name = entry.file_name().to_string_lossy().into_owned();
+        if name.ends_with(".seg") && !name.contains("temp_") {
+            return entry.path();
+        }
+    }
+    panic!("no segment found in {:?}", seg_dir);
+}

crates/blob/tests/fuzz_test.rs

@@ -0,0 +1,380 @@
+/// Fuzz-style tests: randomized operation sequences, corruption injection,
+/// and crash-recovery stress testing.
+use bichon_blob::{Codec, Config, Engine};
+use rand::RngExt;
+use std::collections::HashMap;
+use tempfile::TempDir;
+
+/// Number of iterations for each randomized test.
+const FUZZ_OPS: usize = 2000;
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+fn make_key(i: u64) -> [u8; 32] {
+    let mut key = [0u8; 32];
+    key[0..8].copy_from_slice(&i.to_le_bytes());
+    key
+}
+
+fn make_value(rng: &mut impl RngExt) -> Vec<u8> {
+    let size = match rng.random_range(0..100) {
+        0..=4   => rng.random_range(0..64),              // tiny
+        5..=9   => 0,                                  // empty
+        10..=79 => rng.random_range(64..4096),            // small
+        80..=89 => rng.random_range(4096..65536),         // medium
+        90..=94 => rng.random_range(65536..500_000),      // large
+        _       => rng.random_range(500_000..2_000_000),  // xl (near max)
+    };
+    let mut v = vec![0u8; size];
+    rng.fill(&mut v[..]);
+    v
+}
+
+fn random_codec(rng: &mut impl RngExt) -> Codec {
+    match rng.random_range(0..4) {
+        0 => Codec::None,
+        1 => Codec::Zstd,
+        _ => Codec::Lz4,
+    }
+}
+
+// ── Fuzz: random operation sequence ─────────────────────────────────────────
+
+#[test]
+fn fuzz_random_ops() {
+    let mut rng = rand::rng();
+    let dir = TempDir::new().unwrap();
+    let mut config = Config::default();
+    config.flush_interval_secs = 0; // manual flush only
+    config.gc_interval_secs = 0;
+
+    let engine = Engine::open(dir.path(), config).unwrap();
+
+    // Oracle: track expected values in memory
+    let mut oracle: HashMap<[u8; 32], Vec<u8>> = HashMap::new();
+    let mut next_key = 0u64;
+
+    for _ in 0..FUZZ_OPS {
+        match rng.random_range(0..100) {
+            // 45%: put new key
+            0..=44 => {
+                let key = make_key(next_key);
+                next_key += 1;
+                let value = make_value(&mut rng);
+                let codec = random_codec(&mut rng);
+                engine.put(key, &value, codec).unwrap();
+                oracle.insert(key, value);
+            }
+            // 20%: put overwrite existing key
+            45..=64 => {
+                if oracle.is_empty() { continue; }
+                let idx = rng.random_range(0..oracle.len());
+                let key = *oracle.iter().nth(idx).unwrap().0;
+                let value = make_value(&mut rng);
+                let codec = random_codec(&mut rng);
+                engine.put(key, &value, codec).unwrap();
+                oracle.insert(key, value);
+            }
+            // 15%: read and verify
+            65..=79 => {
+                if oracle.is_empty() { continue; }
+                let idx = rng.random_range(0..oracle.len());
+                let key = *oracle.iter().nth(idx).unwrap().0;
+                let expected = oracle.get(&key).unwrap();
+                let got = engine.get(&key).unwrap();
+                assert_eq!(got.as_ref(), Some(expected), "key mismatch on read");
+            }
+            // 10%: delete
+            80..=89 => {
+                if oracle.is_empty() { continue; }
+                let idx = rng.random_range(0..oracle.len());
+                let key = *oracle.iter().nth(idx).unwrap().0;
+                engine.delete(&key).unwrap();
+                oracle.remove(&key);
+                let got = engine.get(&key).unwrap();
+                assert_eq!(got, None, "deleted key should return None");
+            }
+            // 5%: read non-existent key
+            90..=94 => {
+                let key = make_key(next_key + rng.random_range(1000u64..10000));
+                let got = engine.get(&key).unwrap();
+                assert_eq!(got, None, "non-existent key should return None");
+            }
+            // 5%: flush
+            _ => {
+                engine.flush().unwrap();
+            }
+        }
+    }
+
+    // Final verification: all oracle entries must match
+    for (key, expected) in &oracle {
+        let got = engine.get(key).unwrap();
+        assert_eq!(got.as_ref(), Some(expected), "final verification: key mismatch");
+    }
+}
+
+// ── Fuzz: crash + reopen cycle ──────────────────────────────────────────────
+
+#[test]
+fn fuzz_crash_reopen_cycles() {
+    let mut rng = rand::rng();
+    let dir = TempDir::new().unwrap();
+    let dir_path = dir.path().to_path_buf();
+
+    let mut oracle: HashMap<[u8; 32], Vec<u8>> = HashMap::new();
+    let mut next_key = 0u64;
+    let cycles = 20;
+
+    for _cycle in 0..cycles {
+        // Open database
+        let mut config = Config::default();
+        config.flush_interval_secs = 0;
+        config.gc_interval_secs = 0;
+        let engine = Engine::open(&dir_path, config).unwrap();
+
+        // Do some work
+        let ops = rng.random_range(50..200);
+        for _ in 0..ops {
+            match rng.random_range(0..100) {
+                0..=50 => {
+                    let key = make_key(next_key);
+                    next_key += 1;
+                    let value = make_value(&mut rng);
+                    if engine.put(key, &value, Codec::Zstd).is_ok() {
+                        oracle.insert(key, value);
+                    }
+                }
+                51..=65 => {
+                    if oracle.is_empty() { continue; }
+                    let idx = rng.random_range(0..oracle.len());
+                    let key = *oracle.iter().nth(idx).unwrap().0;
+                    engine.delete(&key).unwrap();
+                    oracle.remove(&key);
+                }
+                66..=85 => {
+                    if oracle.is_empty() { continue; }
+                    let idx = rng.random_range(0..oracle.len());
+                    let key = *oracle.iter().nth(idx).unwrap().0;
+                    let expected = oracle.get(&key).unwrap();
+                    if let Ok(Some(got)) = engine.get(&key) {
+                        assert_eq!(&got, expected, "pre-crash read mismatch");
+                    }
+                }
+                _ => {
+                    let _ = engine.flush();
+                }
+            }
+        }
+
+        // Simulate crash: drop without shutdown
+        drop(engine);
+    }
+
+    // Final reopen: all oracle entries must be intact
+    let config = Config::default();
+    let engine = Engine::open(&dir_path, config).unwrap();
+    for (key, expected) in &oracle {
+        let got = engine.get(key).unwrap();
+        assert_eq!(got.as_ref(), Some(expected), "after {} crash cycles", cycles);
+    }
+}
+
+// ── Fuzz: batch operations ──────────────────────────────────────────────────
+
+#[test]
+fn fuzz_batch_ops() {
+    let mut rng = rand::rng();
+    let dir = TempDir::new().unwrap();
+
+    let config = Config::default();
+    let engine = Engine::open(dir.path(), config).unwrap();
+
+    let mut oracle: HashMap<[u8; 32], Vec<u8>> = HashMap::new();
+    let mut next_key = 0u64;
+
+    for _ in 0..200 {
+        match rng.random_range(0..100) {
+            // 50%: batch write
+            0..=49 => {
+                let batch_size = rng.random_range(1..30);
+                let entries: Vec<_> = (0..batch_size)
+                    .map(|_| {
+                        let key = make_key(next_key);
+                        next_key += 1;
+                        let value = make_value(&mut rng);
+                        oracle.insert(key, value.clone());
+                        (key, value, Codec::Zstd)
+                    })
+                    .collect();
+                engine.put_batch(&entries).unwrap();
+            }
+            // 30%: verify random subset
+            50..=79 => {
+                if oracle.is_empty() { continue; }
+                let n = rng.random_range(1..=20.min(oracle.len()));
+                for _ in 0..n {
+                    let idx = rng.random_range(0..oracle.len());
+                    let (key, expected) = oracle.iter().nth(idx).unwrap();
+                    let got = engine.get(key).unwrap();
+                    assert_eq!(got.as_ref(), Some(expected));
+                }
+            }
+            // 20%: batch delete
+            _ => {
+                if oracle.is_empty() { continue; }
+                let n = rng.random_range(1..=20.min(oracle.len()));
+                let keys: Vec<[u8; 32]> = (0..n)
+                    .map(|_| {
+                        let idx = rng.random_range(0..oracle.len());
+                        let key = *oracle.iter().nth(idx).unwrap().0;
+                        oracle.remove(&key);
+                        key
+                    })
+                    .collect();
+                engine.delete_batch(&keys).unwrap();
+            }
+        }
+    }
+
+    for (key, expected) in &oracle {
+        let got = engine.get(key).unwrap();
+        assert_eq!(got.as_ref(), Some(expected));
+    }
+}
+
+// ── Fuzz: GC stress ─────────────────────────────────────────────────────────
+
+#[test]
+fn fuzz_gc_stress() {
+    let mut rng = rand::rng();
+    let dir = TempDir::new().unwrap();
+
+    let mut config = Config::default();
+    config.gc_deleted_ratio = 0.1; // aggressive GC trigger
+    let engine = Engine::open(dir.path(), config).unwrap();
+
+    let mut oracle: HashMap<[u8; 32], Vec<u8>> = HashMap::new();
+    let mut next_key = 0u64;
+
+    for round in 0..10 {
+        // Write a batch of keys
+        let n = rng.random_range(50..150);
+        let mut round_keys: Vec<[u8; 32]> = Vec::new();
+        let value_size = rng.random_range(100..10000);
+        let value: Vec<u8> = (0..value_size).map(|_| rng.random::<u8>()).collect();
+
+        for _ in 0..n {
+            let key = make_key(next_key);
+            next_key += 1;
+            engine.put(key, &value, Codec::None).unwrap();
+            oracle.insert(key, value.clone());
+            round_keys.push(key);
+        }
+
+        // Delete some fraction
+        let delete_frac: f64 = rng.random_range(0.2..0.8);
+        let delete_count = (round_keys.len() as f64 * delete_frac) as usize;
+        for _ in 0..delete_count {
+            let idx = rng.random_range(0..round_keys.len());
+            let key = round_keys.swap_remove(idx);
+            engine.delete(&key).unwrap();
+            oracle.remove(&key);
+        }
+
+        // Seal and run GC
+        engine.seal_active_segment().unwrap();
+        let _ = engine.gc().unwrap();
+
+        // Verify all remaining oracle entries
+        for (key, expected) in &oracle {
+            let got = engine.get(key).unwrap();
+            assert_eq!(
+                got.as_ref(),
+                Some(expected),
+                "GC round {}: key mismatch",
+                round
+            );
+        }
+    }
+}
+
+// ── Fuzz: bitflip corruption resilience ─────────────────────────────────────
+
+#[test]
+fn fuzz_corruption_resilience() {
+    let mut rng = rand::rng();
+    let dir = TempDir::new().unwrap();
+    let dir_path = dir.path().to_path_buf();
+
+    let config = Config::default();
+    let engine = Engine::open(&dir_path, config).unwrap();
+
+    // Write known data
+    let mut good_keys: Vec<[u8; 32]> = Vec::new();
+    let n = 100u64;
+    for i in 0..n {
+        let key = make_key(i);
+        let value = vec![i as u8; 2048];
+        engine.put(key, &value, Codec::None).unwrap();
+        good_keys.push(key);
+    }
+    engine.flush().unwrap();
+
+    // Seal so segment file is durable on disk
+    engine.seal_active_segment().unwrap();
+    engine.shutdown().unwrap();
+    drop(engine);
+
+    // Find segment files and corrupt random bytes
+    let seg_dir = dir_path.join("segments");
+    let mut seg_files: Vec<_> = std::fs::read_dir(&seg_dir)
+        .unwrap()
+        .filter_map(|e| e.ok())
+        .filter(|e| {
+            e.file_name()
+                .to_string_lossy()
+                .ends_with(".seg")
+        })
+        .collect();
+    seg_files.sort_by_key(|e| e.file_name());
+
+    // Corrupt 3 random bytes in the first segment
+    if let Some(seg) = seg_files.first() {
+        let path = seg.path();
+        let mut data = std::fs::read(&path).unwrap();
+        if data.len() > 100 {
+            for _ in 0..3 {
+                let pos = rng.random_range(50..data.len());
+                data[pos] ^= 0xFF; // flip all bits
+            }
+            std::fs::write(&path, &data).unwrap();
+        }
+    }
+
+    // Reopen: recovery must succeed (not panic), even if some keys are lost
+    let config = Config::default();
+    let engine = Engine::open(&dir_path, config).unwrap();
+
+    // At least some uncorrupted keys should still be readable
+    let mut readable = 0;
+    let mut corrupted = 0;
+    for key in &good_keys {
+        match engine.get(key) {
+            Ok(Some(_)) => readable += 1,
+            Ok(None) => { /* key might be lost due to corruption */ }
+            Err(_) => corrupted += 1,
+        }
+    }
+    // The vast majority should still be ok (corruption only hit 3 bytes in one segment)
+    assert!(
+        readable + corrupted > 0,
+        "at least some outcomes should be observable"
+    );
+    assert!(
+        readable > n as usize / 2,
+        "majority of keys should survive localized corruption ({} of {})",
+        readable,
+        n
+    );
+}

crates/blob/tests/integration_test.rs

@@ -0,0 +1,445 @@
+use bichon_blob::{Codec, Config, Engine};
+use tempfile::TempDir;
+
+#[test]
+fn test_write_and_read() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let key = [0xAA; 32];
+    let value = b"Hello, this is a test email!".to_vec();
+
+    engine.put(key, &value, Codec::Zstd).unwrap();
+
+    let result = engine.get(&key).unwrap();
+    assert_eq!(result, Some(value));
+}
+
+#[test]
+fn test_read_missing_key() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let key = [0xFF; 32];
+    let result = engine.get(&key).unwrap();
+    assert_eq!(result, None);
+}
+
+#[test]
+fn test_delete() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let key = [0xBB; 32];
+    let value = b"Some email content".to_vec();
+
+    engine.put(key, &value, Codec::Zstd).unwrap();
+    engine.delete(&key).unwrap();
+
+    let result = engine.get(&key).unwrap();
+    assert_eq!(result, None);
+}
+
+#[test]
+fn test_exists() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let key = [0xCC; 32];
+    assert!(!engine.exists(&key).unwrap());
+
+    engine.put(key, b"data", Codec::None).unwrap();
+    assert!(engine.exists(&key).unwrap());
+
+    engine.delete(&key).unwrap();
+    assert!(!engine.exists(&key).unwrap());
+}
+
+#[test]
+fn test_small_value_not_compressed() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let key = [0xCC; 32];
+    let value = b"hi"; // Smaller than 4KB threshold
+
+    engine.put(key, value, Codec::Zstd).unwrap();
+
+    let result = engine.get(&key).unwrap();
+    assert_eq!(result, Some(value.to_vec()));
+}
+
+#[test]
+fn test_large_value() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let key = [0xDD; 32];
+    let value = vec![b'X'; 100_000]; // 100KB
+
+    engine.put(key, &value, Codec::Zstd).unwrap();
+
+    let result = engine.get(&key).unwrap();
+    assert_eq!(result, Some(value));
+}
+
+#[test]
+fn test_multiple_keys() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let n = 100;
+    for i in 0..n {
+        let mut key = [0u8; 32];
+        key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
+        let value = format!("email number {}", i).into_bytes();
+        engine.put(key, &value, Codec::Zstd).unwrap();
+    }
+
+    for i in 0..n {
+        let mut key = [0u8; 32];
+        key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
+        let result = engine.get(&key).unwrap();
+        assert_eq!(result, Some(format!("email number {}", i).into_bytes()));
+    }
+}
+
+#[test]
+fn test_gc() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    // Write many entries
+    let value = vec![b'Y'; 5000];
+    let n = 100;
+
+    for i in 0..n {
+        let mut key = [0u8; 32];
+        key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
+        engine.put(key, &value, Codec::None).unwrap();
+    }
+
+    // Delete even-numbered keys
+    for i in (0..n).step_by(2) {
+        let mut key = [0u8; 32];
+        key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
+        engine.delete(&key).unwrap();
+    }
+
+    // Run GC
+    let _result = engine.gc().unwrap();
+
+    // Verify remaining keys still readable
+    for i in (1..n).step_by(2) {
+        let mut key = [0u8; 32];
+        key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
+        let result = engine.get(&key).unwrap();
+        assert_eq!(result, Some(value.clone()));
+    }
+
+    // Deleted keys should not exist
+    for i in (0..n).step_by(2) {
+        let mut key = [0u8; 32];
+        key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
+        let result = engine.get(&key).unwrap();
+        assert_eq!(result, None);
+    }
+}
+
+#[test]
+fn test_reopen_persistence() {
+    let dir = TempDir::new().unwrap();
+    let key = [0xEE; 32];
+    let value = b"persistent data".to_vec();
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        engine.put(key, &value, Codec::Zstd).unwrap();
+    }
+
+    // Reopen
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        let result = engine.get(&key).unwrap();
+        assert_eq!(result, Some(value));
+    }
+}
+
+#[test]
+fn test_stats() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    engine.put([1u8; 32], b"hello", Codec::None).unwrap();
+
+    let stats = engine.stats().unwrap();
+    assert!(stats.total_bytes > 0);
+    assert!(stats.total_keys > 0);
+}
+
+#[test]
+fn test_batch_write() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let n = 50;
+    let entries: Vec<_> = (0..n)
+        .map(|i: u64| {
+            let mut key = [0u8; 32];
+            key[0..8].copy_from_slice(&i.to_le_bytes());
+            let value = format!("batch email {}", i).into_bytes();
+            (key, value, Codec::Zstd)
+        })
+        .collect();
+
+    engine.put_batch(&entries).unwrap();
+
+    for (key, value, _) in &entries {
+        let result = engine.get(key).unwrap();
+        assert_eq!(result.as_ref(), Some(value));
+    }
+}
+
+#[test]
+fn test_batch_write_persistence() {
+    let dir = TempDir::new().unwrap();
+    let entries: Vec<_> = (0..30u64)
+        .map(|i| {
+            let mut key = [0u8; 32];
+            key[0..8].copy_from_slice(&i.to_le_bytes());
+            (key, format!("persist {}", i).into_bytes(), Codec::Zstd)
+        })
+        .collect();
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        engine.put_batch(&entries).unwrap();
+    }
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for (key, value, _) in &entries {
+            let result = engine.get(key).unwrap();
+            assert_eq!(result.as_ref(), Some(value));
+        }
+    }
+}
+
+#[test]
+fn test_invalid_config_rejected() {
+    let dir = TempDir::new().unwrap();
+    let mut config = Config::default();
+    config.compression_level = -1;
+    assert!(Engine::open(dir.path(), config).is_err());
+
+    let mut config = Config::default();
+    config.gc_deleted_ratio = 1.5;
+    assert!(Engine::open(dir.path(), config).is_err());
+}
+
+#[test]
+fn test_concurrent_reads() {
+    use std::sync::Arc;
+    use std::thread;
+
+    let dir = TempDir::new().unwrap();
+    let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
+
+    // Write some data
+    for i in 0..50u32 {
+        let mut key = [0u8; 32];
+        key[0..4].copy_from_slice(&i.to_le_bytes());
+        engine
+            .put(key, &vec![i as u8; 1024], Codec::None)
+            .unwrap();
+    }
+
+    // Spawn 4 threads, each reading a different subset
+    let mut handles = vec![];
+    for t in 0..4 {
+        let engine = engine.clone();
+        handles.push(thread::spawn(move || {
+            for i in (t * 12)..((t + 1) * 12) {
+                let mut key = [0u8; 32];
+                key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
+                let read = engine.get(&key).unwrap();
+                assert!(read.is_some(), "key {} should exist", i);
+            }
+        }));
+    }
+    for h in handles {
+        h.join().unwrap();
+    }
+}
+
+#[test]
+fn test_global_dedup() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let key = [0x42; 32];
+    let value = b"same content across what would be accounts".to_vec();
+
+    // Write same key twice (simulating two accounts ingesting the same email)
+    engine.put(key, &value, Codec::Zstd).unwrap();
+    engine.put(key, &value, Codec::Zstd).unwrap();
+
+    // Should still be readable
+    let result = engine.get(&key).unwrap();
+    assert_eq!(result, Some(value));
+
+    // Stats should reflect dedup (not double count)
+    let stats = engine.stats().unwrap();
+    // The key appears once in the bucket store
+    assert!(stats.total_keys > 0);
+}
+
+#[test]
+fn test_crash_recovery() {
+    let dir = TempDir::new().unwrap();
+    let dir_path = dir.path().to_path_buf();
+
+    // Phase 1: write data, then drop without shutdown (simulates crash)
+    {
+        let engine = Engine::open(&dir_path, Config::default()).unwrap();
+
+        for i in 0..50u32 {
+            let mut key = [0u8; 32];
+            key[0..4].copy_from_slice(&i.to_le_bytes());
+            engine
+                .put(key, &vec![i as u8; 512], Codec::None)
+                .unwrap();
+        }
+        // Engine dropped here without calling shutdown()
+    }
+
+    // Phase 2: reopen - recovery should run, data should be intact
+    let engine = Engine::open(&dir_path, Config::default()).unwrap();
+    let stats = engine.stats().unwrap();
+    assert!(stats.total_keys > 0, "recovery should preserve data");
+
+    // Verify reads work
+    for i in 0..50u32 {
+        let mut key = [0u8; 32];
+        key[0..4].copy_from_slice(&i.to_le_bytes());
+        let read = engine.get(&key).unwrap();
+        assert!(read.is_some(), "key {} should survive crash recovery", i);
+    }
+}
+
+#[test]
+fn test_meta_bin_durability() {
+    let dir = TempDir::new().unwrap();
+    let dir_path = dir.path().to_path_buf();
+
+    {
+        let engine = Engine::open(&dir_path, Config::default()).unwrap();
+
+        let key = [0x42u8; 32];
+        engine.put(key, b"durable", Codec::None).unwrap();
+    }
+    // Engine dropped -> shutdown() called -> meta saved
+
+    // Verify meta.bin exists
+    let meta_path = dir_path.join("meta.bin");
+    assert!(meta_path.exists(), "meta.bin should exist after clean shutdown");
+
+    let data = std::fs::read(&meta_path).unwrap();
+    assert!(
+        data.len() >= 8,
+        "meta.bin should have at least 8 bytes"
+    );
+
+    let stored_crc = u32::from_le_bytes(data[0..4].try_into().unwrap());
+    assert_ne!(stored_crc, 0, "stored CRC should be non-zero");
+
+    // Reopen and verify data is intact
+    let engine = Engine::open(&dir_path, Config::default()).unwrap();
+    let read = engine.get(&[0x42u8; 32]).unwrap();
+    assert_eq!(read, Some(b"durable".to_vec()));
+}
+
+#[test]
+fn test_gc_deletes_empty_segment() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    // Write data, seal, then delete everything so the sealed segment
+    // becomes 100% garbage.  GC should remove the segment file entirely.
+    let n = 50u64;
+    for i in 0..n {
+        let mut key = [0u8; 32];
+        key[0..8].copy_from_slice(&i.to_le_bytes());
+        engine.put(key, &vec![b'X'; 4096], Codec::None).unwrap();
+    }
+
+    // Force seal so the segment becomes a GC candidate.
+    let sealed_id = engine.seal_active_segment().unwrap();
+
+    // Delete all keys — the sealed segment is now entirely garbage.
+    let keys: Vec<[u8; 32]> = (0..n)
+        .map(|i| {
+            let mut key = [0u8; 32];
+            key[0..8].copy_from_slice(&i.to_le_bytes());
+            key
+        })
+        .collect();
+    engine.delete_batch(&keys).unwrap();
+
+    // Run GC — this should empty and then delete the sealed segment.
+    let result = engine.gc().unwrap();
+    assert!(result.is_some(), "GC should have found a candidate");
+    let stats = result.unwrap();
+    assert_eq!(stats.segment_id, sealed_id);
+    assert_eq!(stats.bytes_after, 0);
+
+    // Segment file must be deleted.
+    let seg_path = dir
+        .path()
+        .join("segments")
+        .join(format!("{:08}.seg", sealed_id));
+    assert!(
+        !seg_path.exists(),
+        "emptied segment file should have been deleted, but {:?} exists",
+        seg_path
+    );
+
+    // Subsequent reads / writes must still work (no corruption).
+    let new_key = [0x99u8; 32];
+    engine
+        .put(new_key, b"post-gc data", Codec::None)
+        .unwrap();
+    let result = engine.get(&new_key).unwrap();
+    assert_eq!(result, Some(b"post-gc data".to_vec()));
+
+    // Engine stats should be consistent.
+    let stats = engine.stats().unwrap();
+    assert_eq!(stats.total_keys, 1);
+
+    // Shutdown and reopen — persistence must be intact.
+    drop(engine);
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+    let result = engine.get(&new_key).unwrap();
+    assert_eq!(result, Some(b"post-gc data".to_vec()));
+    let result = engine.get(&keys[0]).unwrap();
+    assert_eq!(result, None);
+}
+
+#[test]
+fn test_file_lock_prevents_concurrent_open() {
+    let dir = TempDir::new().unwrap();
+    let _engine1 = Engine::open(dir.path(), Config::default()).unwrap();
+    let result = Engine::open(dir.path(), Config::default());
+    assert!(result.is_err(), "second open on same directory must fail");
+}
+
+#[test]
+fn test_file_lock_released_after_close() {
+    let dir = TempDir::new().unwrap();
+    {
+        let _engine = Engine::open(dir.path(), Config::default()).unwrap();
+    }
+    // Lock should be released after engine is dropped
+    let engine = Engine::open(dir.path(), Config::default());
+    assert!(engine.is_ok(), "reopen after close must succeed");
+}

crates/cli/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "bichon-cli"
+version.workspace = true
+edition.workspace = true
+
+
+[dependencies]
+bichon-core = { path = "../core" }
+tokio.workspace = true
+serde.workspace = true
+clap.workspace = true
+dialoguer.workspace = true
+console.workspace = true
+mail-parser.workspace = true
+reqwest.workspace = true
+toml = "0.9.8"
+memmap2 = "0.9.10"
+outlook-pst = { git = "https://github.com/rustmailer/outlook-pst-rs.git", branch = "main" }
+chrono.workspace = true
+base64.workspace = true
+sysinfo.workspace = true
+indicatif.workspace = true
+serde_json.workspace = true

crates/cli/src/api/download.rs

@@ -0,0 +1,81 @@
+use crate::BichonCliConfig;
+use bichon_core::{base64_encode, envelope::meta::BichonMetadata, store::envelope::Envelope};
+use chrono::{TimeZone, Utc};
+use reqwest::Client;
+use tokio::io::AsyncWriteExt;
+
+pub async fn download_and_export_with_json_header(
+    client: &Client,
+    config: &BichonCliConfig,
+    envelope: Envelope,
+    file: &mut tokio::fs::File,
+) -> bool {
+    let url = format!(
+        "{}/api/v1/download-message/{}/{}",
+        config.base_url, &envelope.account_id, &envelope.id
+    );
+
+    let response = match client
+        .get(&url)
+        .header("Authorization", format!("Bearer {}", config.api_token))
+        .send()
+        .await
+    {
+        Ok(res) => {
+            if !res.status().is_success() {
+                eprintln!(
+                    " ✘ HTTP Error {}: Failed for {}",
+                    res.status(),
+                    &envelope.id
+                );
+                return false;
+            }
+            res
+        }
+        Err(e) => {
+            eprintln!(" ✘ Network error: {} for {}", e, &envelope.id);
+            return false;
+        }
+    };
+
+    let email_bytes = match response.bytes().await {
+        Ok(b) => b,
+        Err(e) => {
+            eprintln!(
+                " ✘ Failed to read response body for {}: {}",
+                &envelope.id, e
+            );
+            return false;
+        }
+    };
+
+    let date_dt = Utc.timestamp_opt(envelope.date / 1000, 0).unwrap();
+    let date_str = date_dt.format("%a %b %e %H:%M:%S %Y").to_string();
+    let from_line = format!("From {} {}\n", envelope.from.clone(), date_str);
+
+    let custom_header = build_metadata_header(BichonMetadata {
+        account_email: envelope.account_email,
+        mailbox_name: envelope.mailbox_name,
+        tags: envelope.tags,
+    });
+
+    let mut final_buffer =
+        Vec::with_capacity(from_line.len() + custom_header.len() + email_bytes.len() + 2);
+    final_buffer.extend_from_slice(from_line.as_bytes());
+    final_buffer.extend_from_slice(custom_header.as_bytes());
+    final_buffer.extend_from_slice(&email_bytes);
+    final_buffer.extend_from_slice(b"\n\n");
+
+    if let Err(e) = file.write_all(&final_buffer).await {
+        eprintln!(" ✘ IO Error: Failed to write to mbox: {}", e);
+        return false;
+    }
+
+    true
+}
+
+fn build_metadata_header(meta: BichonMetadata) -> String {
+    let json_str = serde_json::to_string(&meta).ok().unwrap();
+    let encoded = base64_encode!(json_str);
+    format!("X-Bichon-Metadata: {}\r\n", encoded)
+}

crates/cli/src/api/mod.rs

@@ -0,0 +1,4 @@
+pub mod download;
+pub mod search;
+pub mod sender;
+pub mod stats;

crates/cli/src/api/search.rs

@@ -0,0 +1,58 @@
+use bichon_core::{
+    common::paginated::DataPage,
+    message::search::{EmailSearchFilter, EmailSearchRequest, SortBy},
+    store::envelope::Envelope,
+};
+use reqwest::Client;
+
+use crate::BichonCliConfig;
+
+pub async fn search_messages(
+    client: &Client,
+    config: &BichonCliConfig,
+    account_ids: Option<std::collections::HashSet<u64>>,
+    page: u64,
+    page_size: u64,
+) -> Option<DataPage<Envelope>> {
+    let url = format!("{}/api/v1/search-messages", config.base_url);
+
+    let payload = EmailSearchRequest {
+        filter: EmailSearchFilter {
+            account_ids,
+            ..Default::default()
+        },
+        page,
+        page_size,
+        sort_by: Some(SortBy::DATE),
+        desc: Some(false),
+    };
+
+    match client
+        .post(&url)
+        .header("Authorization", format!("Bearer {}", config.api_token))
+        .json(&payload)
+        .send()
+        .await
+    {
+        Ok(res) if res.status().is_success() => match res.json::<DataPage<Envelope>>().await {
+            Ok(data) => Some(data),
+            Err(e) => {
+                eprintln!(" ✘ Failed to parse search response: {}", e);
+                None
+            }
+        },
+        Ok(res) => {
+            let status = res.status();
+            let error_body = res.text().await.unwrap_or_default();
+            eprintln!(
+                " ✘ Failed to search messages. Status: {}\n  Server error: {}",
+                status, error_body
+            );
+            None
+        }
+        Err(e) => {
+            eprintln!(" ✘ Network error performing search: {}", e);
+            None
+        }
+    }
+}

crates/cli/src/api/sender.rs

@@ -0,0 +1,77 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use console::style;
+use reqwest::Client;
+
+use bichon_core::import::BatchEmlRequest;
+
+use crate::BichonCliConfig;
+
+pub async fn send_batch_request(
+    client: &Client,
+    config: &BichonCliConfig,
+    account_id: u64,
+    folder: &str,
+    emls: Vec<String>,
+) {
+    let url = format!("{}/api/v1/import", config.base_url);
+    let payload = BatchEmlRequest {
+        account_id,
+        mail_folder: folder.to_string(),
+        emls,
+    };
+
+    let count = payload.emls.len();
+
+    match client
+        .post(&url)
+        .header("Authorization", format!("Bearer {}", config.api_token))
+        .json(&payload)
+        .send()
+        .await
+    {
+        Ok(res) if res.status().is_success() => {
+            println!(
+                "  {} Sent {} emails to [{}]",
+                style("✔").green(),
+                count,
+                folder
+            );
+        }
+        Ok(res) => {
+            let status = res.status();
+            let error_body = res.text().await.unwrap_or_default();
+            eprintln!(
+                "  {} Failed to send to [{}]. Status: {}\n  Server error: {}",
+                style("✘").red(),
+                folder,
+                status,
+                error_body
+            );
+        }
+        Err(e) => {
+            eprintln!(
+                "  {} Network error on [{}]: {}",
+                style("✘").red(),
+                folder,
+                e
+            );
+        }
+    }
+}

crates/cli/src/api/stats.rs

@@ -0,0 +1,48 @@
+use bichon_core::account::stats::AccountStats;
+use reqwest::Client;
+
+use crate::BichonCliConfig;
+
+pub async fn fetch_account_stats(
+    client: &Client,
+    config: &BichonCliConfig,
+    account_id: u64,
+) -> Option<AccountStats> {
+    let url = format!("{}/api/v1/accounts/{}/stats", config.base_url, account_id);
+
+    match client
+        .get(&url)
+        .header("Authorization", format!("Bearer {}", config.api_token))
+        .send()
+        .await
+    {
+        Ok(res) if res.status().is_success() => {
+            match res.json::<AccountStats>().await {
+                Ok(stats) => Some(stats),
+                Err(e) => {
+                    eprintln!(" ✘ Failed to parse stats response: {}", e);
+                    None
+                }
+            }
+        }
+        Ok(res) => {
+            let status = res.status();
+            let error_body = res.text().await.unwrap_or_default();
+            eprintln!(
+                " ✘ Failed to fetch stats for account [{}]. Status: {}\n  Server error: {}",
+                account_id,
+                status,
+                error_body
+            );
+            None
+        }
+        Err(e) => {
+            eprintln!(
+                " ✘ Network error fetching stats for [{}]: {}",
+                account_id,
+                e
+            );
+            None
+        }
+    }
+}

crates/cli/src/auth.rs

@@ -0,0 +1,217 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::process;
+
+use console::style;
+use dialoguer::{theme::ColorfulTheme, Select};
+use reqwest::Client;
+
+use bichon_core::{
+    account::payload::MinimalAccount,
+    users::{permissions::Permission, view::UserView},
+};
+
+use crate::BichonCliConfig;
+
+async fn fetch_json<T: serde::de::DeserializeOwned>(
+    client: &Client,
+    url: &str,
+    token: &str,
+    label: &str,
+) -> T {
+    let response = match client
+        .get(url)
+        .header("Authorization", format!("Bearer {}", token))
+        .send()
+        .await
+    {
+        Ok(res) => res,
+        Err(e) => {
+            eprintln!(
+                "\n{} {}",
+                style("✘ Network Error:").red().bold(),
+                "Could not connect to Bichon service."
+            );
+            eprintln!("{} {}", style("Details:").dim(), e);
+            eprintln!(
+                "\n{} Please check if the Base URL is correct and the server is running.",
+                style("Tip:").cyan()
+            );
+            process::exit(1);
+        }
+    };
+
+    let status = response.status();
+    let body = response.text().await.unwrap_or_else(|_| String::new());
+
+    if !status.is_success() {
+        eprintln!(
+            "\n{} Server returned an error (Status: {})",
+            style("✘ API Error:").red().bold(),
+            style(status).yellow()
+        );
+        if status == 401 {
+            eprintln!(
+                "{} Your API Token seems to be invalid or expired.",
+                style("Context:").dim()
+            );
+        } else if status == 404 {
+            eprintln!(
+                "{} The endpoint was not found. Please check your Base URL.",
+                style("Context:").dim()
+            );
+        }
+        eprintln!("{} {}", style("Response:").dim(), body);
+        process::exit(1);
+    }
+
+    if body.is_empty() {
+        eprintln!(
+            "\n{} Server returned an empty response for [{}] (Status: {})",
+            style("✘ Empty Response:").red().bold(),
+            label,
+            status
+        );
+        eprintln!(
+            "{} This may be caused by a reverse proxy or middleware issue.",
+            style("Tip:").cyan()
+        );
+        process::exit(1);
+    }
+
+    match serde_json::from_str::<T>(&body) {
+        Ok(data) => data,
+        Err(e) => {
+            eprintln!(
+                "\n{} Failed to parse response for [{}]: {}",
+                style("✘ Parse Error:").red().bold(),
+                label,
+                e
+            );
+            eprintln!("{} Raw body: {}", style("Debug:").dim(), body);
+            process::exit(1);
+        }
+    }
+}
+
+pub async fn verify_user_and_get_account(
+    config: &BichonCliConfig,
+    theme: &ColorfulTheme,
+    only_nosync: bool,
+) -> MinimalAccount {
+    let client = Client::new();
+
+    let user: UserView = fetch_json(
+        &client,
+        &format!("{}/api/v1/current-user", config.base_url),
+        &config.api_token,
+        "current-user",
+    )
+    .await;
+
+    println!("Welcome, {}!", style(&user.username).cyan());
+
+    let accounts: Vec<MinimalAccount> = fetch_json(
+        &client,
+        &format!(
+            "{}/api/v1/minimal-account-list?only_nosync={only_nosync}",
+            config.base_url
+        ),
+        &config.api_token,
+        "minimal-account-list",
+    )
+    .await;
+
+    if accounts.is_empty() {
+        println!(
+            "\n{}",
+            style("Error: No 'nosync' accounts found.").red().bold()
+        );
+        println!(
+            "{}",
+            style("Mail import is only supported for 'nosync' type accounts.").dim()
+        );
+        println!(
+            "Please create a new {} account in the Bichon web interface first.",
+            style("Nosync").bold().yellow()
+        );
+        process::exit(1);
+    }
+
+    let required_permission = Permission::DATA_IMPORT_BATCH;
+    let mut selectable_accounts = Vec::new();
+    let mut options = Vec::new();
+
+    for acc in accounts {
+        let has_permission = if let Some(perms) = user.account_permissions.get(&acc.id) {
+            perms.iter().any(|p| p == required_permission)
+        } else {
+            user.global_permissions
+                .iter()
+                .any(|p| p == Permission::DATA_MANAGE_ALL || p == Permission::ROOT)
+        };
+
+        let status_prefix = if has_permission {
+            style(" [READY] ").green()
+        } else {
+            style(" [NO PERMISSION] ").red()
+        };
+
+        options.push(format!(
+            "{}{} - {}",
+            status_prefix,
+            style(&acc.email).bold(),
+            style(format!("ID: {}", acc.id)).dim()
+        ));
+
+        selectable_accounts.push((acc, has_permission));
+    }
+
+    let selection = Select::with_theme(theme)
+        .with_prompt("Select the target account for import")
+        .items(&options)
+        .default(0)
+        .max_length(10)
+        .interact()
+        .unwrap();
+
+    let (selected_acc, can_import) = &selectable_accounts[selection];
+
+    if !*can_import {
+        eprintln!(
+            "\n{} You do not have '{}' permission for account {}.",
+            style("✘ Permission Denied:").red().bold(),
+            style(required_permission).yellow(),
+            style(&selected_acc.email).cyan()
+        );
+        eprintln!(
+            "{} Please contact your administrator to upgrade your role for this account.",
+            style("Tip:").dim()
+        );
+        process::exit(1);
+    }
+
+    println!(
+        "{} Targeting account: {}",
+        style("✔").green(),
+        style(&selected_acc.email).cyan().bold()
+    );
+
+    selected_acc.clone()
+}

crates/cli/src/eml/mod.rs

@@ -0,0 +1,150 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+    collections::HashMap,
+    fs,
+    path::{Path, PathBuf},
+};
+
+use console::style;
+use dialoguer::{theme::ColorfulTheme, Input};
+use mail_parser::MessageParser;
+use reqwest::Client;
+
+use bichon_core::base64_encode_url_safe;
+
+use crate::{BichonCliConfig, api::sender::send_batch_request};
+
+pub async fn handle_eml_directory_import(
+    config: &BichonCliConfig,
+    account_id: u64,
+    theme: &ColorfulTheme,
+) {
+    let root_str: String = Input::with_theme(theme)
+        .with_prompt("Enter the ROOT directory to scan for .eml files")
+        .validate_with(|input: &String| {
+            let p = std::path::Path::new(input);
+            if p.exists() && p.is_dir() {
+                Ok(())
+            } else {
+                Err("Directory not found.")
+            }
+        })
+        .interact_text()
+        .unwrap();
+
+    let root_path = std::path::PathBuf::from(root_str);
+    let mut tasks: HashMap<String, Vec<PathBuf>> = HashMap::new();
+    println!(
+        "{}",
+        style("🔍 Scanning recursively using std::fs...").dim()
+    );
+    if let Err(e) = scan_dir(&root_path, &root_path, &mut tasks) {
+        eprintln!("Error scanning directory: {}", e);
+        return;
+    }
+
+    if tasks.is_empty() {
+        println!("{}", style("No .eml files found.").yellow());
+    } else {
+        process_and_upload(config, account_id, tasks).await;
+    }
+}
+
+fn scan_dir(
+    root: &Path,
+    current: &Path,
+    tasks: &mut HashMap<String, Vec<PathBuf>>,
+) -> std::io::Result<()> {
+    if current.is_dir() {
+        for entry in fs::read_dir(current)? {
+            let entry = entry?;
+            let path = entry.path();
+
+            if path.is_dir() {
+                scan_dir(root, &path, tasks)?;
+            } else if path.is_file() {
+                if path.extension().and_then(|s| s.to_str()) == Some("eml") {
+                    let rel_path = path.strip_prefix(root).unwrap_or(Path::new(""));
+                    let mailbox_name = rel_path
+                        .parent()
+                        .map(|p| p.to_string_lossy().replace('\\', "/"))
+                        .unwrap_or_default();
+                    let folder = if mailbox_name.is_empty() {
+                        "Inbox".to_string()
+                    } else {
+                        mailbox_name
+                    };
+                    tasks.entry(folder).or_insert_with(|| Vec::new()).push(path);
+                }
+            }
+        }
+    }
+    Ok(())
+}
+
+async fn process_and_upload(
+    config: &BichonCliConfig,
+    account_id: u64,
+    tasks: HashMap<String, Vec<PathBuf>>,
+) {
+    let client = Client::new();
+    let batch_size = 50;
+
+    for (mailbox, files) in tasks {
+        println!("\n🚀 Processing mailbox: {}", style(&mailbox).cyan().bold());
+
+        let mut current_batch = Vec::new();
+
+        for file_path in files {
+            let body = match fs::read(&file_path) {
+                Ok(b) => b,
+                Err(e) => {
+                    eprintln!(
+                        "  {} Failed to read file {:?}: {}",
+                        style("✘").red(),
+                        file_path,
+                        e
+                    );
+                    continue;
+                }
+            };
+
+            if MessageParser::new().parse(&body).is_some() {
+                let b64_content = base64_encode_url_safe!(&body);
+                current_batch.push(b64_content);
+
+                if current_batch.len() >= batch_size {
+                    let to_send = current_batch;
+                    current_batch = Vec::with_capacity(batch_size);
+                    send_batch_request(&client, config, account_id, &mailbox, to_send).await;
+                }
+            } else {
+                eprintln!(
+                    "  {} Invalid format, skipping: {:?}",
+                    style("⚠").yellow(),
+                    file_path
+                );
+            }
+        }
+        if !current_batch.is_empty() {
+            send_batch_request(&client, config, account_id, &mailbox, current_batch).await;
+        }
+    }
+}

crates/cli/src/export/mod.rs

@@ -0,0 +1,215 @@
+use crate::api::download::download_and_export_with_json_header;
+use crate::api::search::search_messages;
+use crate::api::stats::fetch_account_stats;
+use crate::BichonCliConfig;
+use bichon_core::account::payload::MinimalAccount;
+use console::style;
+use dialoguer::Confirm;
+use dialoguer::{theme::ColorfulTheme, Input};
+use indicatif::{ProgressBar, ProgressStyle};
+use reqwest::Client;
+use std::path::{Path, PathBuf};
+use sysinfo::Disks;
+
+pub async fn handle_account_export(
+    config: &BichonCliConfig,
+    account: MinimalAccount,
+    theme: &ColorfulTheme,
+) {
+    let client = Client::new();
+
+    println!("Fetching account statistics...");
+    let stats = match fetch_account_stats(&client, config, account.id).await {
+        Some(s) => s,
+        None => {
+            eprintln!("{} Failed to fetch account statistics.", style("✘").red());
+            return;
+        }
+    };
+
+    println!("\n--- Account Statistics ---");
+    println!("  Total Emails: {}", style(stats.total_count).cyan());
+    println!(
+        "  Total Size:   {}",
+        style(format_bytes(stats.total_size)).cyan()
+    );
+
+    let path = loop {
+        let input: String = Input::with_theme(theme)
+            .with_prompt("Enter ABSOLUTE directory path for MBOX file")
+            .interact_text()
+            .unwrap();
+
+        let p = PathBuf::from(&input);
+
+        if !p.is_absolute() {
+            eprintln!(
+                " {} {}",
+                style("✘").red(),
+                style("Invalid path: Must be an absolute path.").red()
+            );
+            continue;
+        }
+
+        if !p.exists() {
+            eprintln!(
+                " {} {}",
+                style("✘").red(),
+                style("Invalid path: Directory does not exist.").red()
+            );
+            continue;
+        }
+
+        if !p.is_dir() {
+            eprintln!(
+                " {} {}",
+                style("✘").red(),
+                style("Invalid path: The path provided is not a directory.").red()
+            );
+            continue;
+        }
+        break p;
+    };
+
+    let disks = Disks::new_with_refreshed_list();
+    let disk_result = disks
+        .list()
+        .iter()
+        .find(|d| path.starts_with(d.mount_point()))
+        .ok_or_else(|| "Could not identify the disk for the provided path.");
+
+    match disk_result {
+        Ok(disk) => {
+            let free_space = disk.available_space();
+            let required_space = (stats.total_size as f64 * 1.2) as u64;
+
+            if free_space < required_space {
+                eprintln!(
+                " {} Insufficient disk space (including 10% safety buffer)!\n   Required:  {} (Base: {})\n   Available: {}",
+                style("✘").red(),
+                style(format_bytes(required_space)).yellow(),
+                style(format_bytes(stats.total_size)).yellow(),
+                style(format_bytes(free_space)).yellow()
+            );
+                return;
+            }
+
+            println!(
+                " {} Disk space check passed. (Required: {}, Available: {})",
+                style("✔").green(),
+                style(format_bytes(required_space)).cyan(),
+                style(format_bytes(free_space)).cyan()
+            );
+        }
+        Err(e) => {
+            eprintln!(" {} {}", style("✘").red(), style(e).red());
+            return;
+        }
+    }
+    let mbox_file = get_unique_mbox_path(&path, account.id, &account.email);
+    if Confirm::with_theme(theme)
+        .with_prompt(format!(
+            "Export {} emails to '{}'?",
+            stats.total_count,
+            mbox_file.display()
+        ))
+        .default(true)
+        .interact()
+        .unwrap()
+    {
+        println!(
+            " {} Starting export ({} items per page)...",
+            style("✔").green(),
+            100
+        );
+
+        let pb = ProgressBar::new(stats.total_count as u64);
+        pb.set_style(ProgressStyle::with_template(
+            "{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta}) {msg}"
+        ).unwrap());
+
+        let mut file = match tokio::fs::OpenOptions::new()
+            .append(true)
+            .create(true)
+            .open(&mbox_file)
+            .await
+        {
+            Ok(f) => f,
+            Err(e) => {
+                eprintln!(" ✘ Failed to open file '{}': {}", path.display(), e);
+                return;
+            }
+        };
+
+        let page_size = 100;
+        let mut current_page = 1;
+        let mut total_pages;
+
+        loop {
+            let account_ids = Some(std::collections::HashSet::from([account.id]));
+            if let Some(batch) = search_messages(&client, config, account_ids, current_page, page_size).await {
+                total_pages = batch.total_pages.unwrap();
+
+                pb.set_message(format!("Page {}/{}", current_page, total_pages));
+
+                for envelope in batch.items {
+                    let success =
+                        download_and_export_with_json_header(&client, config, envelope.clone(), &mut file)
+                            .await;
+
+                    if !success {
+                        eprintln!(
+                            " ✘ Failed to export email {}, skipping...",
+                            envelope.id
+                        );
+                        continue;
+                    }
+                    pb.inc(1);
+                }
+                if current_page >= total_pages {
+                    break;
+                }
+                current_page += 1;
+            } else {
+                pb.finish_with_message("Error");
+                eprintln!(
+                    " ✘ Failed to fetch page {}. Aborting process...",
+                    current_page
+                );
+                return;
+            }
+        }
+        pb.finish();
+        println!(" {} Export complete!", style("✔").green());
+    }
+}
+
+fn format_bytes(bytes: u64) -> String {
+    if bytes < 1024 {
+        format!("{:.2} B", bytes)
+    } else if bytes < 1024 * 1024 {
+        format!("{:.2} KB", bytes / 1024)
+    } else if bytes < 1024 * 1024 * 1024 {
+        format!("{:.2} MB", bytes / 1024 / 1024)
+    } else {
+        format!("{:.2} GB", bytes / 1024 / 1024 / 1024)
+    }
+}
+
+fn get_unique_mbox_path(base_dir: &Path, account_id: u64, email: &str) -> PathBuf {
+    let email_part = email.replace(' ', "_");
+
+    let mut base_name = format!("account_{}_{}", account_id, email_part);
+    if base_name.starts_with('.') {
+        base_name = format!("_{}", base_name);
+    }
+
+    let mut final_path = base_dir.join(format!("{}.mbox", base_name));
+
+    let mut counter = 1;
+    while final_path.exists() {
+        final_path = base_dir.join(format!("{}_{}.mbox", base_name, counter));
+        counter += 1;
+    }
+    final_path
+}

crates/cli/src/main.rs

@@ -0,0 +1,172 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use bichon_core::bichon_version;
+use clap::Parser;
+use console::style;
+use dialoguer::{theme::ColorfulTheme, Confirm, Input, Select};
+use serde::{Deserialize, Serialize};
+use std::fs;
+
+use crate::{
+    auth::verify_user_and_get_account, eml::handle_eml_directory_import,
+    export::handle_account_export, mbox::handle_mbox_single_file_import, pst::handle_pst_import,
+    thunderbird::handle_thunderbird_import,
+};
+
+pub mod api;
+pub mod auth;
+pub mod eml;
+pub mod export;
+pub mod mbox;
+pub mod pst;
+pub mod thunderbird;
+
+#[derive(Parser, Debug)]
+#[command(
+    name = "bichon-cli",
+    author = "rustmailer",
+    version = bichon_version!(),
+    about = "A CLI tool to import email data into Bichon service"
+)]
+pub struct BichonCli {
+    /// Path to the configuration file
+    #[arg(
+        short,
+        long,
+        default_value = "config.toml",
+        value_name = "FILE",
+        help = "Sets a custom config file"
+    )]
+    pub config: std::path::PathBuf,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct BichonCliConfig {
+    pub base_url: String,
+    pub api_token: String,
+}
+
+#[tokio::main]
+async fn main() {
+    let cli = BichonCli::parse();
+    let theme = ColorfulTheme::default();
+    let config_path = &cli.config;
+    let mut current_config: Option<BichonCliConfig> = None;
+
+    if config_path.exists() {
+        if let Ok(content) = fs::read_to_string(config_path) {
+            if let Ok(config) = toml::from_str::<BichonCliConfig>(&content) {
+                println!("{}", style("✔ Existing configuration found:").green());
+                println!("  Base URL: {}", style(&config.base_url).yellow());
+                println!(" API Token: {}", style(&config.api_token).yellow());
+
+                // Confirm with user
+                if Confirm::with_theme(&theme)
+                    .with_prompt("Do you want to use this configuration?")
+                    .default(true)
+                    .interact()
+                    .unwrap()
+                {
+                    current_config = Some(config);
+                }
+            }
+        }
+    }
+
+    let final_config = match current_config {
+        Some(conf) => conf,
+        None => {
+            println!("\n{}", style("Please enter Bichon service details:").bold());
+
+            let url: String = Input::with_theme(&theme)
+                .with_prompt("Bichon Base URL")
+                .default("http://localhost:15630".into())
+                .interact_text()
+                .unwrap();
+
+            let token: String = Input::with_theme(&theme)
+                .with_prompt("API Token")
+                .interact_text()
+                .unwrap();
+
+            let conf = BichonCliConfig {
+                base_url: url,
+                api_token: token,
+            };
+
+            // 3. Offer to save the new configuration
+            if Confirm::with_theme(&theme)
+                .with_prompt("Save this configuration for future use?")
+                .default(true)
+                .interact()
+                .unwrap()
+            {
+                let toml_str = toml::to_string(&conf).unwrap();
+                fs::write(config_path, toml_str).expect("Failed to save config file");
+                println!("{}", style("Configuration saved successfully!").green());
+            }
+            conf
+        }
+    };
+
+    let operations = &[
+        "1. Import: Upload email data to Bichon",
+        "2. Export: Download account data as MBOX file",
+    ];
+
+    let op_idx = Select::with_theme(&theme)
+        .with_prompt("Select operation")
+        .items(operations)
+        .default(0)
+        .interact()
+        .unwrap();
+
+    match op_idx {
+        0 => {
+            let target_account = verify_user_and_get_account(&final_config, &theme, true).await;
+
+            let import_modes = &[
+                "1. EML: Scan directory recursively (Maintains folder structure)",
+                "2. MBOX: Single archive file (Stream from one file)",
+                "3. Thunderbird: Import from local profile directory",
+                "4. PST: Outlook Personal Storage (Single .pst file)",
+            ];
+
+            let mode_idx = Select::with_theme(&theme)
+                .with_prompt("Select import method")
+                .items(import_modes)
+                .default(0)
+                .interact()
+                .unwrap();
+
+            match mode_idx {
+                0 => handle_eml_directory_import(&final_config, target_account.id, &theme).await,
+                1 => handle_mbox_single_file_import(&final_config, target_account.id, &theme).await,
+                2 => handle_thunderbird_import(&final_config, target_account.id, &theme).await,
+                3 => handle_pst_import(&final_config, target_account.id, &theme).await,
+                _ => unreachable!(),
+            }
+        }
+        1 => {
+            let target_account = verify_user_and_get_account(&final_config, &theme, false).await;
+            handle_account_export(&final_config, target_account, &theme).await;
+        }
+        _ => unreachable!(),
+    }
+}

crates/cli/src/mbox/gmail.rs

@@ -0,0 +1,134 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+use std::collections::HashSet;
+
+pub fn determine_folder(labels_raw: &str) -> String {
+    let mut status_blacklist = HashSet::new();
+    status_blacklist.insert("Opened");
+    status_blacklist.insert("Unread");
+    status_blacklist.insert("Archived");
+
+    let all_labels: Vec<&str> = labels_raw
+        .split(',')
+        .map(|s| s.trim())
+        .filter(|s| !s.is_empty())
+        .collect();
+
+    if all_labels.is_empty() {
+        return "Unknown".to_string();
+    }
+
+    let filtered: Vec<&str> = all_labels
+        .iter()
+        .filter(|&&l| !status_blacklist.contains(l))
+        .cloned()
+        .collect();
+
+    match filtered.len() {
+        // Case A: If all labels were status labels, fallback to the first original label
+        0 => all_labels[0].to_string(),
+        // Case B: If only one label remains, that's our target destination
+        1 => filtered[0].to_string(),
+        // Case C: Multiple labels remain (e.g., ["Inbox", "medium"])
+        _ => {
+            // Prioritize custom business labels by excluding generic locations like "Inbox" or "Sent"
+            let business_label = filtered.iter().find(|&&l| l != "Inbox" && l != "Sent");
+
+            match business_label {
+                // Return the first non-generic label found
+                Some(label) => label.to_string(),
+                // If only generic labels remain (e.g., ["Sent", "Inbox"]), pick the first available
+                None => filtered[0].to_string(),
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use mail_parser::{HeaderValue, MessageParser};
+
+    use super::*;
+
+    fn parse_x_gmail_labels(raw_message: &[u8]) -> Option<String> {
+        // MessageParser::new() has an empty header_map so the hardcoded match at
+        // parsers/header.rs:76 treats ALL unknown headers as raw (no RFC 2047
+        // decoding).  We need three things to get decoding:
+        // 1. A non-empty header_map (so the else branch runs)
+        // 2. default_header_text() so the fallback fn is parse_unstructured
+        // 3. OR register X-Gmail-Labels explicitly via header_text()
+        let message = MessageParser::new()
+            .with_minimal_headers()
+            .default_header_text()
+            .parse(raw_message)?;
+        let value: &HeaderValue<'_> = message.header("X-Gmail-Labels")?;
+        value.as_text().map(|s| s.to_string())
+    }
+
+    /// Construct a raw MIME message with RFC 2047 encoded X-Gmail-Labels,
+    /// parse it, and verify the header is correctly decoded.
+    fn build_email(x_gmail_labels: &str) -> Vec<u8> {
+        format!(
+            "From: sender@example.com\r\n\
+             To: recipient@example.com\r\n\
+             Subject: Test\r\n\
+             X-Gmail-Labels: {}\r\n\
+             \r\n\
+             Body text here.\r\n",
+            x_gmail_labels
+        )
+        .into_bytes()
+    }
+
+    #[test]
+    fn rfc2047_encoded_labels_are_decoded() {
+        // Exactly the format the user reported: French Gmail labels
+        let raw = build_email("=?UTF-8?Q?Corbeille?=, =?UTF-8?Q?Messages_archiv=C3=A9s?=");
+        let labels = parse_x_gmail_labels(&raw).expect("failed to parse X-Gmail-Labels");
+
+        // mail-parser decodes RFC 2047 header values during initial parsing.
+        // The decoded text should NOT contain raw =?UTF-8?Q?... sequences.
+        assert!(!labels.contains("=?UTF-8"), "labels still encoded: {labels:?}");
+        assert!(labels.contains("Corbeille"), "missing 'Corbeille': {labels:?}");
+        assert!(
+            labels.contains("archivés"),
+            "missing decoded 'archivés': {labels:?}",
+        );
+
+        // Full pipeline: decoded labels → determine_folder
+        let folder = determine_folder(&labels);
+        assert_eq!(folder, "Corbeille");
+    }
+
+    #[test]
+    fn plain_ascii_labels_passthrough() {
+        let raw = build_email("Inbox, Important");
+        let labels = parse_x_gmail_labels(&raw).expect("failed to parse X-Gmail-Labels");
+        assert_eq!(labels, "Inbox, Important");
+        assert_eq!(determine_folder(&labels), "Important");
+    }
+
+    #[test]
+    fn missing_x_gmail_labels_header() {
+        let raw = b"From: sender@example.com\r\nTo: r@example.com\r\n\r\nBody.\r\n";
+        let message = MessageParser::new().parse(raw.as_slice()).unwrap();
+        assert!(message.header("X-Gmail-Labels").is_none());
+    }
+}

crates/cli/src/mbox/mod.rs

@@ -0,0 +1,418 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::HashMap;
+use std::path::PathBuf;
+
+use crate::api::sender::send_batch_request;
+use crate::mbox::gmail::determine_folder;
+use bichon_core::import::reader::MboxFile;
+use crate::BichonCliConfig;
+use bichon_core::base64_encode_url_safe;
+use bichon_core::envelope::meta::{parse_bichon_metadata, BichonMetadata};
+use console::style;
+use dialoguer::{theme::ColorfulTheme, Input};
+use dialoguer::{Confirm, Select};
+use mail_parser::MessageParser;
+use reqwest::Client;
+
+/// Skip emails larger than this with a warning (100 MB).
+const MAX_EMAIL_BYTES: usize = 100 * 1024 * 1024;
+/// Flush a folder buffer when accumulated base64 bytes exceed this (200 MB).
+const MAX_BUFFER_BYTES: usize = 200 * 1024 * 1024;
+
+pub mod gmail;
+
+pub async fn handle_mbox_single_file_import(
+    config: &BichonCliConfig,
+    account_id: u64,
+    theme: &ColorfulTheme,
+) {
+    let path_str: String = Input::with_theme(theme)
+        .with_prompt("Enter the path to your SINGLE .mbox file")
+        .validate_with(|input: &String| {
+            let p = std::path::Path::new(input);
+            if !p.exists() {
+                return Err("The specified path does not exist.");
+            }
+            if !p.is_file() {
+                return Err("MBOX mode requires a SINGLE file, not a directory.");
+            }
+            Ok(())
+        })
+        .interact_text()
+        .unwrap();
+
+    let mbox_path = PathBuf::from(path_str);
+
+    let options = vec![
+        "Use labels from mail headers (X-Gmail-Labels)",
+        "Specify a single target folder for all emails",
+        "Use X-Bichon-Metadata header (Automatic)",
+    ];
+
+    let selection = Select::with_theme(theme)
+        .with_prompt("How should we determine the target folder?")
+        .items(&options)
+        .default(0)
+        .interact()
+        .unwrap();
+
+    let target_folder: Option<String> = match selection {
+        0 => None,
+        1 => {
+            let folder: String = Input::with_theme(theme)
+                .with_prompt("Target folder name")
+                .default("INBOX".into())
+                .interact_text()
+                .unwrap();
+            Some(folder)
+        }
+        2 => None,
+        _ => unreachable!(),
+    };
+
+    if let Some(ref folder) = target_folder {
+        println!(
+            "{}",
+            style(format!("Mode: Fixed folder ({})", folder)).dim()
+        );
+    } else {
+        println!("{}", style("Mode: Dynamic (header-based)").dim());
+    }
+
+    println!(
+        "\n{} Ready to process MBOX file: {}",
+        style("✔").green(),
+        style(mbox_path.display()).cyan()
+    );
+
+    if let Ok(meta) = std::fs::metadata(&mbox_path) {
+        let size_mb = meta.len() as f64 / 1024.0 / 1024.0;
+        println!(
+            "{}",
+            style(format!("Processing file: {:.1} MB", size_mb)).dim()
+        );
+    }
+
+    if Confirm::with_theme(theme)
+        .with_prompt("Start importing?")
+        .default(true)
+        .interact()
+        .unwrap()
+    {
+        run_import(account_id, &mbox_path, config, target_folder).await
+    }
+}
+
+pub async fn run_import(
+    account_id: u64,
+    mbox_path: &PathBuf,
+    config: &BichonCliConfig,
+    target_folder: Option<String>,
+) {
+    let client = Client::new();
+    let mbox = match MboxFile::from_file(mbox_path) {
+        Ok(mbox) => mbox,
+        Err(err) => {
+            println!("Skipping invalid MBOX: {} ({})", mbox_path.display(), err);
+            return;
+        }
+    };
+
+    let mut folder_buffers: HashMap<String, Vec<String>> = HashMap::new();
+    let mut total_buffered_bytes: usize = 0;
+    let batch_limit = 50;
+    let mut skipped_count: u64 = 0;
+
+    println!("Starting import process...");
+
+    for (index, e) in mbox.iter().enumerate() {
+        let msg_num = index + 1;
+        let body = e.data;
+
+        if body.len() > MAX_EMAIL_BYTES {
+            let size_mb = body.len() as f64 / 1024.0 / 1024.0;
+            eprintln!(
+                "{} {}: email #{} is {:.1} MB (limit 100 MB). Skipping...",
+                style("Warning").yellow().bold(),
+                style(format!("oversized")).dim(),
+                msg_num,
+                size_mb,
+            );
+            skipped_count += 1;
+            continue;
+        }
+
+        let message = match MessageParser::new()
+            .with_minimal_headers()
+            .default_header_text()
+            .parse(body)
+        {
+            Some(msg) => msg,
+            None => {
+                eprintln!(
+                    "{} {}: {}",
+                    style("Warning").yellow().bold(),
+                    style(format!("at message #{}", msg_num)).dim(),
+                    "Failed to parse email structure. Skipping..."
+                );
+                skipped_count += 1;
+                continue;
+            }
+        };
+
+        let mut metadata: Option<BichonMetadata> = None;
+        if let Some(meta_header) = message.header_raw("X-Bichon-Metadata") {
+            metadata = parse_bichon_metadata(meta_header);
+        }
+
+        let get_default_folder = || {
+            let labels = message
+                .header("X-Gmail-Labels")
+                .and_then(|h| h.as_text())
+                .map(|s| s.to_string())
+                .unwrap_or_else(|| "INBOX".to_string());
+            determine_folder(&labels)
+        };
+
+        let folder_name = if let Some(ref folder) = target_folder {
+            folder.clone()
+        } else if let Some(ref meta) = metadata {
+            meta.mailbox_name.clone().unwrap_or_else(get_default_folder)
+        } else {
+            get_default_folder()
+        };
+
+        // Drop message before base64-encoding to free MIME parse memory.
+        drop(message);
+
+        let b64_eml = base64_encode_url_safe!(&body);
+        let encoded_len = b64_eml.len();
+
+        let buffer = folder_buffers
+            .entry(folder_name.clone())
+            .or_insert_with(Vec::new);
+        buffer.push(b64_eml);
+        total_buffered_bytes += encoded_len;
+
+        if buffer.len() >= batch_limit || total_buffered_bytes >= MAX_BUFFER_BYTES {
+            let emls_to_send = folder_buffers.remove(&folder_name).unwrap();
+            let freed: usize = emls_to_send.iter().map(|s| s.len()).sum();
+            total_buffered_bytes = total_buffered_bytes.saturating_sub(freed);
+            send_batch_request(&client, config, account_id, &folder_name, emls_to_send).await;
+        }
+    }
+
+    for (folder_name, emls) in folder_buffers {
+        if !emls.is_empty() {
+            send_batch_request(&client, config, account_id, &folder_name, emls).await;
+        }
+    }
+
+    if skipped_count > 0 {
+        println!(
+            "{}",
+            style(format!(
+                "Skipped {} email(s) (oversized or unparseable).",
+                skipped_count
+            ))
+            .yellow()
+            .bold()
+        );
+    }
+
+    println!("{}", style("Import completed successfully!").green().bold());
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::collections::HashMap;
+
+    /// Fake sender: records every flushed batch as (folder_name, email_count, total_bytes).
+    struct FakeSender {
+        batches: Vec<(String, usize, usize)>,
+    }
+
+    impl FakeSender {
+        fn new() -> Self {
+            Self { batches: vec![] }
+        }
+        fn send(&mut self, folder: &str, emls: Vec<String>) {
+            let count = emls.len();
+            let bytes: usize = emls.iter().map(|s| s.len()).sum();
+            self.batches.push((folder.to_string(), count, bytes));
+            // emls is dropped here, simulating real send
+        }
+    }
+
+    fn fake_encode(size: usize) -> String {
+        // base64 expands ~1.33x, so the encoded string is roughly this long.
+        // We just need a predictable byte size, so use a repeated character.
+        "x".repeat(size)
+    }
+
+    #[test]
+    fn flush_on_global_byte_threshold() {
+        let mut buffers: HashMap<String, Vec<String>> = HashMap::new();
+        let mut total_bytes: usize = 0;
+        let batch_limit = 50;
+        let mut sender = FakeSender::new();
+
+        // Simulate 3 emails, each 80 MB encoded, spread across 3 folders.
+        // After each email, global total goes up by 80 MB.
+        // After the 3rd email: 240 MB > 200 MB → flush the folder that got the 3rd email.
+        let emails = vec![
+            ("Inbox", 80_000_000),
+            ("Sent", 80_000_000),
+            ("Archive", 80_000_000),
+        ];
+
+        for (folder, eml_size) in emails {
+            let encoded = fake_encode(eml_size);
+            let len = encoded.len();
+            let buffer = buffers.entry(folder.to_string()).or_insert_with(Vec::new);
+            buffer.push(encoded);
+            total_bytes += len;
+
+            if buffer.len() >= batch_limit || total_bytes >= MAX_BUFFER_BYTES {
+                let sent = buffers.remove(folder).unwrap();
+                let freed: usize = sent.iter().map(|s| s.len()).sum();
+                total_bytes = total_bytes.saturating_sub(freed);
+                sender.send(folder, sent);
+            }
+        }
+
+        // The 3rd email should trigger a global flush of "Archive".
+        assert_eq!(sender.batches.len(), 1);
+        assert_eq!(sender.batches[0].0, "Archive");
+        assert_eq!(sender.batches[0].1, 1);
+        // "Inbox" and "Sent" are still buffered (160 MB total).
+        assert_eq!(buffers.len(), 2);
+        assert!(buffers.contains_key("Inbox"));
+        assert!(buffers.contains_key("Sent"));
+        assert_eq!(total_bytes, 160_000_000);
+    }
+
+    #[test]
+    fn flush_on_count_threshold() {
+        let mut buffers: HashMap<String, Vec<String>> = HashMap::new();
+        let mut total_bytes: usize = 0;
+        let batch_limit = 3;
+        let mut sender = FakeSender::new();
+
+        // 4 small emails all to Inbox, well under byte threshold.
+        for _ in 0..4 {
+            let encoded = fake_encode(100); // tiny
+            let len = encoded.len();
+            let buffer = buffers
+                .entry("Inbox".to_string())
+                .or_insert_with(Vec::new);
+            buffer.push(encoded);
+            total_bytes += len;
+
+            if buffer.len() >= batch_limit || total_bytes >= MAX_BUFFER_BYTES {
+                let sent = buffers.remove("Inbox").unwrap();
+                let freed: usize = sent.iter().map(|s| s.len()).sum();
+                total_bytes = total_bytes.saturating_sub(freed);
+                sender.send("Inbox", sent);
+            }
+        }
+
+        // Count=3 should trigger flush once; the 4th email stays buffered.
+        assert_eq!(sender.batches.len(), 1);
+        assert_eq!(sender.batches[0].1, 3); // 3 emails flushed
+        let remaining = buffers.get("Inbox").unwrap();
+        assert_eq!(remaining.len(), 1); // 1 still buffered
+    }
+
+    #[test]
+    fn global_bytes_exact_boundary() {
+        let mut buffers: HashMap<String, Vec<String>> = HashMap::new();
+        let mut total_bytes: usize = 0;
+        let mut sender = FakeSender::new();
+
+        // Push one email that puts us right at 200 MB.
+        let encoded = fake_encode(MAX_BUFFER_BYTES);
+        let len = encoded.len();
+        buffers
+            .entry("Inbox".to_string())
+            .or_insert_with(Vec::new)
+            .push(encoded);
+        total_bytes += len;
+
+        if total_bytes >= MAX_BUFFER_BYTES {
+            let sent = buffers.remove("Inbox").unwrap();
+            let freed: usize = sent.iter().map(|s| s.len()).sum();
+            total_bytes = total_bytes.saturating_sub(freed);
+            sender.send("Inbox", sent);
+        }
+
+        // Should have flushed on the boundary.
+        assert_eq!(sender.batches.len(), 1);
+        assert_eq!(total_bytes, 0);
+    }
+
+    #[test]
+    fn flush_one_folder_does_not_lose_others() {
+        let mut buffers: HashMap<String, Vec<String>> = HashMap::new();
+        let mut total_bytes: usize = 0;
+        let batch_limit = 50;
+        let mut sender = FakeSender::new();
+
+        // Build up A to 150 MB, B to 100 MB (total 250 MB > 200 MB).
+        // A should trigger flush; B should stay buffered.
+        let folder_a = "A".to_string();
+        let folder_b = "B".to_string();
+
+        // Folder A: 150 MB
+        let encoded = fake_encode(150_000_000);
+        let len = encoded.len();
+        buffers.entry(folder_a.clone()).or_insert_with(Vec::new).push(encoded);
+        total_bytes += len;
+
+        // Folder B: 100 MB → total 250 MB → trigger flush on B
+        let encoded = fake_encode(100_000_000);
+        let len = encoded.len();
+        buffers.entry(folder_b.clone()).or_insert_with(Vec::new).push(encoded);
+        total_bytes += len;
+
+        // Check trigger on B
+        let b_buffer = buffers.get(&folder_b).unwrap();
+        if b_buffer.len() >= batch_limit || total_bytes >= MAX_BUFFER_BYTES {
+            let sent = buffers.remove(&folder_b).unwrap();
+            let freed: usize = sent.iter().map(|s| s.len()).sum();
+            total_bytes = total_bytes.saturating_sub(freed);
+            sender.send(&folder_b, sent);
+        }
+
+        assert_eq!(sender.batches.len(), 1);
+        assert_eq!(sender.batches[0].0, "B"); // B flushed
+        assert!(buffers.contains_key("A")); // A still there
+        assert_eq!(total_bytes, 150_000_000);
+    }
+
+    #[test]
+    fn skip_oversized_email() {
+        assert!(100 <= MAX_EMAIL_BYTES);
+        // Use vec! so the 100 MB array lives on the heap, not the stack.
+        let huge = vec![0u8; MAX_EMAIL_BYTES + 1];
+        assert!(huge.len() > MAX_EMAIL_BYTES);
+    }
+}

crates/cli/src/pst/mod.rs

@@ -0,0 +1,224 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::api::sender::send_batch_request;
+use crate::BichonCliConfig;
+use bichon_core::import::pst::build_eml_base64;
+use dialoguer::theme::ColorfulTheme;
+use dialoguer::{Confirm, Input};
+use outlook_pst::messaging::folder::Folder;
+use outlook_pst::ndb::node_id::NodeId;
+use reqwest::Client;
+use std::future::Future;
+use std::path::PathBuf;
+use std::pin::Pin;
+use std::rc::Rc;
+
+pub async fn handle_pst_import(config: &BichonCliConfig, account_id: u64, theme: &ColorfulTheme) {
+    let path_str: String = Input::with_theme(theme)
+        .with_prompt("Enter the path to your SINGLE .pst file")
+        .validate_with(|input: &String| {
+            let p = std::path::Path::new(input);
+            if !p.exists() {
+                return Err("The specified path does not exist.");
+            }
+
+            if !p.is_file() {
+                return Err("PST mode requires a SINGLE file, not a directory.");
+            }
+            let is_pst = p
+                .extension()
+                .and_then(|ext| ext.to_str())
+                .map(|ext| ext.eq_ignore_ascii_case("pst"))
+                .unwrap_or(false);
+
+            if !is_pst {
+                return Err("The selected file must have a .pst extension.");
+            }
+
+            Ok(())
+        })
+        .interact_text()
+        .unwrap();
+
+    let pst_path = std::path::PathBuf::from(path_str);
+
+    println!(
+        "\n{} Ready to process PST file: {}",
+        console::style("✔").green(),
+        console::style(pst_path.display()).cyan()
+    );
+
+    if let Ok(meta) = std::fs::metadata(&pst_path) {
+        let size_mb = meta.len() as f64 / 1024.0 / 1024.0;
+        println!(
+            "{}",
+            console::style(format!("PST File Size: {:.1} MB", size_mb)).dim()
+        );
+    }
+
+    if Confirm::with_theme(theme)
+        .with_prompt("Start importing emails from this PST?")
+        .default(true)
+        .interact()
+        .unwrap()
+    {
+        parse_pst(pst_path, config, account_id).await;
+    } else {
+        println!("{}", console::style("Operation cancelled by user.").red());
+    }
+}
+
+async fn parse_pst(pst_path: PathBuf, config: &BichonCliConfig, account_id: u64) {
+    let client = Client::new();
+
+    let pst_store = match outlook_pst::open_store(&pst_path) {
+        Ok(store) => store,
+        Err(e) => {
+            println!(
+                "{} Failed to open PST file: {}",
+                console::style("✘").red(),
+                console::style(format!("{:#?}", e)).dim()
+            );
+            return;
+        }
+    };
+
+    let ipm_sub_tree = match pst_store.properties().ipm_sub_tree_entry_id() {
+        Ok(id) => id,
+        Err(e) => {
+            println!(
+                "{} Could not find IPM_SUBTREE (Mailbox Root): {}",
+                console::style("✘").red(),
+                console::style(format!("{:#?}", e)).dim()
+            );
+            return;
+        }
+    };
+
+    let ipm_subtree_folder = match pst_store.open_folder(&ipm_sub_tree) {
+        Ok(folder) => folder,
+        Err(e) => {
+            println!(
+                "{} Failed to open the root mailbox folder: {}",
+                console::style("✘").red(),
+                console::style(format!("{:#?}", e)).dim()
+            );
+            return;
+        }
+    };
+
+    process_folder_recursively(&client, &ipm_subtree_folder, "", config, account_id).await;
+}
+
+fn process_folder_recursively<'a>(
+    client: &'a Client,
+    folder: &'a Rc<dyn Folder>,
+    parent_path: &'a str,
+    config: &'a BichonCliConfig,
+    account_id: u64,
+) -> Pin<Box<dyn Future<Output = ()> + 'a>> {
+    Box::pin(async move {
+        let folder_name = folder
+            .properties()
+            .display_name()
+            .unwrap_or_else(|_| "Unknown".to_string());
+
+        let current_path = if parent_path.is_empty() {
+            folder_name
+        } else {
+            format!("{}/{}", parent_path, folder_name)
+        };
+
+        println!(
+            "{} {}",
+            console::style("📁 Folder:").dim(),
+            console::style(&current_path).cyan()
+        );
+
+        let mut emls_batch = Vec::new();
+
+        if let Some(contents_table) = folder.contents_table() {
+            for row in contents_table.rows_matrix() {
+                let store = folder.store().clone();
+
+                let entry_id = match store
+                    .properties()
+                    .make_entry_id(NodeId::from(u32::from(row.id())))
+                {
+                    Ok(id) => id,
+                    Err(e) => {
+                        eprintln!(
+                            "  {} Skip row {}: {:?}",
+                            console::style("⚠").yellow(),
+                            row.unique(),
+                            e
+                        );
+                        continue;
+                    }
+                };
+
+                match store.open_message(&entry_id, None) {
+                    Ok(message) => match build_eml_base64(message) {
+                        Some(base64_eml) => emls_batch.push(base64_eml),
+                        None => {}
+                    },
+                    Err(e) => eprintln!("  {} Open error: {:?}", console::style("⚠").yellow(), e),
+                }
+
+                if emls_batch.len() >= 50 {
+                    let batch = emls_batch.clone();
+                    emls_batch.clear();
+                    send_to_bichon(client, config, account_id, &current_path, batch).await;
+                }
+            }
+        }
+
+        if !emls_batch.is_empty() {
+            send_to_bichon(client, config, account_id, &current_path, emls_batch).await;
+        }
+
+        if let Some(hierarchy_table) = folder.hierarchy_table() {
+            for row in hierarchy_table.rows_matrix() {
+                let node = NodeId::from(u32::from(row.id()));
+                if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
+                    if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
+                        process_folder_recursively(
+                            client,
+                            &sub_folder,
+                            &current_path,
+                            config,
+                            account_id,
+                        )
+                        .await;
+                    }
+                }
+            }
+        }
+    })
+}
+
+async fn send_to_bichon(
+    client: &Client,
+    config: &BichonCliConfig,
+    account_id: u64,
+    folder_path: &str,
+    emls: Vec<String>,
+) {
+    send_batch_request(client, config, account_id, folder_path, emls).await;
+}

crates/cli/src/thunderbird/mod.rs

@@ -0,0 +1,131 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{collections::HashMap, path::PathBuf};
+
+use crate::{mbox::run_import, BichonCliConfig};
+use console::style;
+use dialoguer::{theme::ColorfulTheme, Confirm, Input};
+
+pub async fn handle_thunderbird_import(
+    config: &BichonCliConfig,
+    account_id: u64,
+    theme: &ColorfulTheme,
+) {
+    let root_str: String = Input::with_theme(theme)
+        .with_prompt("Enter your Thunderbird Mail/ImapMail directory")
+        .validate_with(|input: &String| {
+            let p = std::path::Path::new(input);
+            if p.exists() && p.is_dir() {
+                Ok(())
+            } else {
+                Err("Directory not found.")
+            }
+        })
+        .interact_text()
+        .unwrap();
+
+    let root_path = std::path::PathBuf::from(&root_str);
+    println!("{}", style("🔍 Scanning Thunderbird structure...").dim());
+
+    let mut mbox_tasks: HashMap<String, PathBuf> = HashMap::new();
+
+    fn scan_thunderbird_dir(
+        root: &std::path::Path,
+        current: &std::path::Path,
+        tasks: &mut HashMap<String, PathBuf>,
+    ) {
+        if let Ok(entries) = std::fs::read_dir(current) {
+            for entry in entries.flatten() {
+                let path = entry.path();
+                let file_name = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
+
+                if path.is_dir() {
+                    scan_thunderbird_dir(root, &path, tasks);
+                } else {
+                    let extension = path.extension().and_then(|s| s.to_str()).unwrap_or("");
+                    match extension {
+                        "msf" | "dat" | "html" | "json" | "txt" | "sqlite" => continue,
+                        _ => {}
+                    }
+
+                    if file_name == "filterlog.html" || file_name == "msgFilterRules.dat" {
+                        continue;
+                    }
+
+                    if !extension.is_empty() {
+                        continue;
+                    }
+
+                    if let Ok(rel) = path.strip_prefix(root) {
+                        let mailbox = rel.to_string_lossy().replace(".sbd", "").replace('\\', "/");
+                        tasks.insert(mailbox, path);
+                    }
+                }
+            }
+        }
+    }
+
+    scan_thunderbird_dir(&root_path, &root_path, &mut mbox_tasks);
+    if mbox_tasks.is_empty() {
+        println!(
+            "{}",
+            style("No mailboxes found in the specified directory.").yellow()
+        );
+        return;
+    }
+
+    println!("\n{}", style("🔍 Scanned Mailboxes:").bold().underlined());
+    let mut sorted_keys: Vec<_> = mbox_tasks.keys().collect();
+    sorted_keys.sort();
+
+    for name in &sorted_keys {
+        let path = &mbox_tasks[*name];
+        let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
+        let size_mb = file_size as f64 / 1024.0 / 1024.0;
+
+        println!(
+            "  {} {} ({:.2} MB)",
+            style("•").dim(),
+            style(name).cyan(),
+            size_mb
+        );
+    }
+
+    println!();
+    let prompt = format!("Ready to import {} mailboxes. Proceed?", mbox_tasks.len());
+    if Confirm::with_theme(theme)
+        .with_prompt(prompt)
+        .default(true)
+        .interact()
+        .unwrap()
+    {
+        for (mailbox_name, mbox_file) in mbox_tasks {
+            println!("\n🚀 Importing: {}", style(&mailbox_name).cyan().bold());
+            run_import(account_id, &mbox_file, config, Some(mailbox_name)).await;
+        }
+        println!(
+            "\n{}",
+            style("✨ All mailboxes imported successfully!")
+                .green()
+                .bold()
+        );
+    } else {
+        println!("{}", style("Import cancelled.").yellow());
+    }
+}

crates/core/Cargo.toml

@@ -0,0 +1,79 @@
+[package]
+name = "bichon-core"
+version.workspace = true
+edition.workspace = true
+
+
+[features]
+default = ["web-api"]
+web-api = ["dep:poem-openapi"]
+
+[dependencies]
+poem-openapi = { version = "5.1.16", features = [
+    "openapi-explorer",
+    "rapidoc",
+    "scalar",
+    "redoc",
+    "swagger-ui",
+    "email",
+], optional = true }
+chrono.workspace = true
+clap.workspace = true
+bichon-memdb.workspace = true
+itertools.workspace = true
+ring.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+tokio.workspace = true
+tracing.workspace = true
+tracing-appender.workspace = true
+tracing-subscriber.workspace = true
+base64.workspace = true
+snafu.workspace = true
+reqwest.workspace = true
+tokio-socks.workspace = true
+regex.workspace = true
+email_address.workspace = true
+futures.workspace = true
+utf7-imap.workspace = true
+mail-parser.workspace = true
+tokio-rustls.workspace = true
+oauth2.workspace = true
+sysinfo.workspace = true
+num_cpus.workspace = true
+rand.workspace = true
+encoding_rs.workspace = true
+async-imap = { git = "https://github.com/rustmailer/async-imap.git", branch = "main", default-features = false, features = [
+    "runtime-tokio",
+    "compress",
+] }
+tantivy = { version = "0.26.1", features = ["zstd-compression", "quickwit"] }
+webpki-roots.workspace = true
+rustls.workspace = true
+rustls-pki-types.workspace = true
+tokio-io-timeout.workspace = true
+governor.workspace = true
+lru.workspace = true
+time.workspace = true
+murmur3.workspace = true
+dashmap.workspace = true
+itoa.workspace = true
+html2text.workspace = true
+bytes.workspace = true
+mail-send.workspace = true
+blake3.workspace = true
+uuid.workspace = true
+bichon-blob.workspace = true
+tracing-log.workspace = true
+tokio-util.workspace = true
+whichlang = "0.1.1"
+deunicode = "1.6.2"
+scopeguard = "1.2.0"
+cron = "0.17"
+quick-xml = { version = "0.41.0", features = ["serialize"] }
+hickory-resolver = "0.26.1"
+memmap2 = "0.9.10"
+outlook-pst = { git = "https://github.com/rustmailer/outlook-pst-rs.git", branch = "main" }
+compressed-rtf = "1.0.1"
+codepage-strings = "1.0.2"
+hex.workspace = true

crates/core/src/account/entity.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -16,19 +16,25 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+use crate::{encrypt, error::BichonResult};
 
-use crate::{encrypt, modules::error::BichonResult};
-
-use poem_openapi::{Enum, Object};
+//use poem_openapi::{Enum, Object};
 use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Default, Debug, Eq, PartialEq, Serialize, Deserialize, Object)]
+#[derive(Clone, Default, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct ImapConfig {
     /// IMAP server hostname or IP address
-    #[oai(validator(max_length = 253, pattern = r"^[a-zA-Z0-9\-\.]+$"))]
+    #[cfg_attr(
+        feature = "web-api",
+        oai(validator(max_length = 253, pattern = r"^[a-zA-Z0-9\-\.]+$"))
+    )]
     pub host: String,
     /// IMAP server port number
-    #[oai(validator(minimum(value = "1"), maximum(value = "65535")))]
+    #[cfg_attr(
+        feature = "web-api",
+        oai(validator(minimum(value = "1"), maximum(value = "65535")))
+    )]
     pub port: u16,
     /// Connection encryption method
     pub encryption: Encryption,
@@ -52,8 +58,8 @@ impl ImapConfig {
     }
 }
 
-
-#[derive(Enum, Default, Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[derive(Default, Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))]
 pub enum AuthType {
     /// Standard password authentication (PLAIN/LOGIN)
     #[default]
@@ -62,7 +68,8 @@ pub enum AuthType {
     OAuth2,
 }
 
-#[derive(Object, Default, Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[derive(Default, Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct AuthConfig {
     ///Authentication method to use
     pub auth_type: AuthType,
@@ -70,7 +77,7 @@ pub struct AuthConfig {
     ///
     /// Users should provide a plaintext password (1 to 256 characters).
     /// The server will encrypt the password using AES-256-GCM and securely store it.
-    #[oai(validator(max_length = 256, min_length = 1))]
+    #[cfg_attr(feature = "web-api", oai(validator(max_length = 256, min_length = 1)))]
     pub password: Option<String>,
 }
 
@@ -97,7 +104,8 @@ impl AuthConfig {
     }
 }
 
-#[derive(Clone, Default, Debug, Eq, PartialEq, Serialize, Deserialize, Enum)]
+#[derive(Clone, Default, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))]
 pub enum Encryption {
     /// SSL/TLS encrypted connection
     #[default]

crates/core/src/account/grant.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -16,14 +16,15 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-use poem_openapi::Object;
+//use poem_openapi::Object;
 use serde::{Deserialize, Serialize};
 
 use crate::{
-    modules::{
+    raise_error, utc_now,
+    {
         account::migration::AccountModel,
         common::auth::ClientContext,
-        database::{manager::DB_MANAGER, with_transaction},
+        database::{manager::DB_MANAGER, with_transaction, MemDbModel},
         error::{code::ErrorCode, BichonResult},
         users::{
             permissions::Permission,
@@ -31,10 +32,10 @@ use crate::{
             UserModel,
         },
     },
-    raise_error, utc_now,
 };
 
-#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)]
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct BatchAccountRoleRequest {
     pub account_ids: Vec<u64>,
     pub user_ids: Vec<u64>,
@@ -42,8 +43,8 @@ pub struct BatchAccountRoleRequest {
 }
 
 impl BatchAccountRoleRequest {
-    pub async fn validate_existence(&self) -> BichonResult<()> {
-        let role = UserRole::find(self.role_id).await?.ok_or_else(|| {
+    pub fn validate_existence(&self) -> BichonResult<()> {
+        let role = UserRole::find(self.role_id)?.ok_or_else(|| {
             raise_error!(
                 format!("Role ID {} not found", self.role_id),
                 ErrorCode::ResourceNotFound
@@ -58,7 +59,7 @@ impl BatchAccountRoleRequest {
         }
 
         for id in &self.account_ids {
-            let exists = AccountModel::find(*id).await?; // Assuming an exists helper
+            let exists = AccountModel::find(*id)?; // Assuming an exists helper
             if exists.is_none() {
                 return Err(raise_error!(
                     format!("Account ID {} not found", id),
@@ -68,7 +69,7 @@ impl BatchAccountRoleRequest {
         }
 
         for id in &self.user_ids {
-            let exists = UserModel::find(*id).await?; // Assuming an exists helper
+            let exists = UserModel::find(*id)?; // Assuming an exists helper
             if exists.is_none() {
                 return Err(raise_error!(
                     format!("User ID {} not found", id),
@@ -80,44 +81,38 @@ impl BatchAccountRoleRequest {
         Ok(())
     }
 
-    async fn grant_batch_account_access(
+    fn grant_batch_account_access(
         account_ids: Vec<u64>,
         user_ids: Vec<u64>,
         role_id: u64,
     ) -> BichonResult<()> {
-        with_transaction(DB_MANAGER.meta_db(), move |rw| {
+        with_transaction(DB_MANAGER.db(), move |txn| {
+            let mut txn = txn;
             for &uid in &user_ids {
-                // Fetch the current user record from the database
-                let user = rw
-                    .get()
-                    .primary::<UserModel>(uid)
-                    .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
-                    .ok_or_else(|| {
-                        raise_error!(
-                            format!("User with id={} not found.", uid),
-                            ErrorCode::ResourceNotFound
-                        )
-                    })?;
+                let db = DB_MANAGER.db();
+                let coll = db.collection(UserModel::collection());
+                let key = uid.to_string();
+                let user: UserModel = coll
+                    .get_required(&key)
+                    .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
 
                 let mut updated_user = user.clone();
 
-                // Apply the role to each specified account_id
                 for &aid in &account_ids {
                     updated_user.account_access_map.insert(aid, role_id);
                 }
 
                 updated_user.updated_at = utc_now!();
 
-                // Save the updated user back to the database within the transaction
-                rw.update(user, updated_user)
+                txn = txn
+                    .upsert(UserModel::collection(), key, &updated_user)
                     .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
             }
-            Ok(())
+            Ok(txn)
         })
-        .await
     }
 
-    pub async fn do_assign(self, context: &ClientContext) -> BichonResult<()> {
+    pub fn do_assign(self, context: &ClientContext) -> BichonResult<()> {
         for account_id in &self.account_ids {
             // Get the user's specific access for this account
             let assigned_role_id =
@@ -133,7 +128,7 @@ impl BatchAccountRoleRequest {
                     })?;
 
             // Fetch the role definition from the database
-            let user_scoped_role = UserRole::find(*assigned_role_id).await?.ok_or_else(|| {
+            let user_scoped_role = UserRole::find(*assigned_role_id)?.ok_or_else(|| {
                 raise_error!(
                     "Assigned account role no longer exists".into(),
                     ErrorCode::InternalError
@@ -155,6 +150,6 @@ impl BatchAccountRoleRequest {
             // This is where you'd compare target_role.permissions vs manager's perms
         }
 
-        Self::grant_batch_account_access(self.account_ids, self.user_ids, self.role_id).await
+        Self::grant_batch_account_access(self.account_ids, self.user_ids, self.role_id)
     }
 }

crates/core/src/account/migration.rs

@@ -0,0 +1,990 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use serde::{Deserialize, Serialize};
+use std::collections::BTreeSet;
+use tracing::info;
+
+use crate::{
+    account::{
+        entity::ImapConfig,
+        payload::{AccountCreateRequest, AccountUpdateRequest, MinimalAccount},
+        since::{DateSince, RelativeDate},
+        state::DownloadState,
+    },
+    cache::imap::{mailbox::MailBox, task::SYNC_TASKS},
+    common::paginated::DataPage,
+    context::controller::DOWNLOAD_CONTROLLER,
+    database::{
+        count_impl, delete_impl, find_impl, insert_impl, list_all_impl, manager::DB_MANAGER,
+        paginate_impl, update_impl, MemDbModel,
+    },
+    encrypt,
+    error::{code::ErrorCode, BichonResult},
+    id,
+    oauth2::token::OAuth2AccessToken,
+    raise_error,
+    store::tantivy::{attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER},
+    users::{payload::UserUpdateRequest, role::DEFAULT_ACCOUNT_MANAGER_ROLE_ID, UserModel},
+    utc_now,
+};
+
+pub type AccountModel = Account;
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))]
+pub enum AccountType {
+    #[default]
+    IMAP,
+    NoSync,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))]
+pub enum QuotaWindow {
+    Hourly,
+    #[default]
+    Daily,
+    Weekly,
+    Monthly,
+}
+
+/// Include/exclude filter rule.
+///
+/// - `include` non-empty: only values matching these patterns pass.
+/// - `exclude` non-empty: values matching these patterns are rejected.
+/// - Both empty: all values pass.
+/// - Both set: include checked first, then exclude.
+///
+/// Extension patterns use case-insensitive exact match; all others use regex.
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct FilterRule {
+    #[serde(default)]
+    pub include: Vec<String>,
+    #[serde(default)]
+    pub exclude: Vec<String>,
+}
+
+impl FilterRule {
+    pub fn is_empty(&self) -> bool {
+        self.include.is_empty() && self.exclude.is_empty()
+    }
+
+    fn matches_exact(&self, value: &str) -> bool {
+        if !self.include.is_empty() && !self.include.iter().any(|e| e.eq_ignore_ascii_case(value)) {
+            return false;
+        }
+        if !self.exclude.is_empty() && self.exclude.iter().any(|e| e.eq_ignore_ascii_case(value)) {
+            return false;
+        }
+        true
+    }
+
+    fn matches_regex(&self, value: &str) -> bool {
+        if !self.include.is_empty() && !matches_any_regex(&self.include, value) {
+            return false;
+        }
+        if !self.exclude.is_empty() && matches_any_regex(&self.exclude, value) {
+            return false;
+        }
+        true
+    }
+
+    fn validate_regex(&self, field: &str) -> Result<(), String> {
+        validate_patterns(&self.include, &format!("{field}.include"))?;
+        validate_patterns(&self.exclude, &format!("{field}.exclude"))?;
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct ExtractionRules {
+    /// Type 0: Master switch.
+    #[serde(default)]
+    pub enabled: bool,
+    /// Type 1: File extensions (exact match, e.g. `{"include": ["pdf","docx"]}`).
+    #[serde(default)]
+    pub extensions: FilterRule,
+    /// Type 2: Folder patterns (regex, e.g. `{"include": ["^INBOX/Invoices"]}`).
+    #[serde(default)]
+    pub folders: FilterRule,
+    /// Type 3: Attachment filename patterns (regex).
+    #[serde(default)]
+    pub attachment_names: FilterRule,
+    /// Type 4: Sender patterns (regex).
+    #[serde(default)]
+    pub senders: FilterRule,
+}
+
+impl ExtractionRules {
+    /// Returns `true` if the attachment should be extracted under these rules.
+    pub fn should_extract(
+        &self,
+        ext: &str,
+        folder: Option<&str>,
+        attachment_name: Option<&str>,
+        sender: Option<&str>,
+    ) -> bool {
+        if !self.enabled {
+            return false;
+        }
+        if !self.extensions.matches_exact(ext) {
+            return false;
+        }
+        if !self.folders.is_empty() {
+            if let Some(folder) = folder {
+                if !self.folders.matches_regex(folder) {
+                    return false;
+                }
+            }
+        }
+        if !self.attachment_names.is_empty() {
+            if let Some(name) = attachment_name {
+                if !self.attachment_names.matches_regex(name) {
+                    return false;
+                }
+            }
+        }
+        if !self.senders.is_empty() {
+            if let Some(sender) = sender {
+                if !self.senders.matches_regex(sender) {
+                    return false;
+                }
+            }
+        }
+        true
+    }
+
+    pub fn validate(&self) -> Result<(), String> {
+        self.folders.validate_regex("folders")?;
+        self.attachment_names.validate_regex("attachment_names")?;
+        self.senders.validate_regex("senders")?;
+        Ok(())
+    }
+}
+
+/// Archive filtering rules — skip unwanted emails before storage.
+///
+/// Rule types:
+///   0 — Master switch
+///   1 — Sender filter (regex)
+///   2 — Subject filter (regex)
+///   3 — Skip emails larger than this (bytes)
+///   4 — Skip emails with spam headers (X-Spam-Flag, X-Spam)
+///
+/// `None` = archive everything (backward compatible).
+#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct ArchiveRules {
+    /// Type 0: Master switch. `false` = archive everything.
+    #[serde(default)]
+    pub enabled: bool,
+    /// Type 1: Sender filter (regex, include/exclude).
+    #[serde(default)]
+    pub senders: FilterRule,
+    /// Type 2: Subject filter (regex, include/exclude).
+    #[serde(default)]
+    pub subjects: FilterRule,
+    /// Type 3: Skip emails larger than this (bytes). `None` = no size limit.
+    #[serde(default)]
+    pub skip_larger_than: Option<u64>,
+    /// Type 4: Spam header names to check (e.g. `["X-Spam-Flag", "X-Spam"]`).
+    /// When the value is `yes` or `true` (case-insensitive), the email is skipped.
+    /// Empty = don't check. Common headers: `X-Spam-Flag` (SpamAssassin),
+    /// `X-Spam` (rspamd), `X-MS-Exchange-Organization-SCL` (Exchange).
+    #[serde(default)]
+    pub spam_headers: Vec<String>,
+}
+
+impl ArchiveRules {
+    /// Returns `true` if the email should be archived under these rules.
+    pub fn should_archive(
+        &self,
+        sender: Option<&str>,
+        subject: Option<&str>,
+        size: u32,
+        is_spam: bool,
+    ) -> bool {
+        if !self.enabled {
+            return true;
+        }
+        if !self.senders.is_empty() {
+            if let Some(sender) = sender {
+                if !self.senders.matches_regex(sender) {
+                    return false;
+                }
+            }
+        }
+        if !self.subjects.is_empty() {
+            if let Some(subject) = subject {
+                if !self.subjects.matches_regex(subject) {
+                    return false;
+                }
+            }
+        }
+        if let Some(limit) = self.skip_larger_than {
+            if size as u64 > limit {
+                return false;
+            }
+        }
+        if !self.spam_headers.is_empty() && is_spam {
+            return false;
+        }
+        true
+    }
+
+    /// Validate all regex patterns are well-formed.
+    pub fn validate(&self) -> Result<(), String> {
+        self.senders.validate_regex("senders")?;
+        self.subjects.validate_regex("subjects")?;
+        Ok(())
+    }
+}
+
+fn matches_any_regex(patterns: &[String], value: &str) -> bool {
+    patterns.iter().any(|p| {
+        regex::Regex::new(p)
+            .map(|re| re.is_match(value))
+            .unwrap_or(false)
+    })
+}
+
+fn validate_patterns(patterns: &[String], field_name: &str) -> Result<(), String> {
+    for p in patterns {
+        regex::Regex::new(p)
+            .map_err(|e| format!("{} pattern '{}' is invalid regex: {}", field_name, p, e))?;
+    }
+    Ok(())
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct Account {
+    pub id: u64,
+    pub imap: Option<ImapConfig>,
+    pub enabled: bool,
+    #[cfg_attr(
+        feature = "web-api",
+        oai(validator(custom = "crate::common::validator::EmailValidator"))
+    )]
+    pub email: String,
+    pub account_name: Option<String>,
+    pub login_name: Option<String>,
+    pub capabilities: Option<Vec<String>>,
+    pub date_since: Option<DateSince>,
+    pub date_before: Option<RelativeDate>,
+    pub download_folders: Option<Vec<String>>,
+    pub account_type: AccountType,
+    pub download_interval_min: Option<i64>,
+    pub download_batch_size: Option<u32>,
+    #[serde(default)]
+    pub max_email_size_bytes: Option<u64>,
+    pub known_folders: Option<BTreeSet<String>>,
+    pub created_at: i64,
+    pub updated_at: i64,
+    pub created_by: u64, //user id
+    pub use_dangerous: bool,
+    pub pgp_key: Option<String>,
+    pub imap_quota_bytes: Option<u64>,
+    pub imap_quota_window: Option<QuotaWindow>,
+    pub auto_download_new_mailboxes: Option<bool>,
+    pub download_schedule: Option<String>,
+    #[serde(default)]
+    pub deleting: bool,
+    /// Email-level filtering rules (Pro feature).
+    /// `None` = archive everything (backward compatible).
+    #[serde(default)]
+    pub archive_rules: Option<ArchiveRules>,
+    /// Attachment text extraction rules (Pro feature).
+    /// `None` = extract everything (backward compatible).
+    #[serde(default)]
+    pub extraction_rules: Option<ExtractionRules>,
+}
+
+impl MemDbModel for Account {
+    fn collection() -> &'static str {
+        "accounts"
+    }
+    fn key(&self) -> String {
+        self.id.to_string()
+    }
+}
+
+impl Account {
+    pub fn new(user_id: u64, request: AccountCreateRequest) -> BichonResult<Self> {
+        Ok(Self {
+            id: id!(64),
+            email: request.email,
+            login_name: request.login_name,
+            account_name: request.account_name,
+            imap: request.imap.map(|i| i.try_encrypt_password()).transpose()?,
+            enabled: request.enabled,
+            capabilities: None,
+            date_since: request.date_since,
+            download_folders: None,
+            known_folders: None,
+            account_type: request.account_type,
+            download_interval_min: request.download_interval_min,
+            created_at: utc_now!(),
+            updated_at: utc_now!(),
+            use_dangerous: request.use_dangerous,
+            pgp_key: request.pgp_key,
+            created_by: user_id,
+            download_batch_size: request.download_batch_size,
+            max_email_size_bytes: request.max_email_size_bytes,
+            date_before: request.date_before,
+            auto_download_new_mailboxes: request.auto_download_new_mailboxes,
+            imap_quota_bytes: request.imap_quota_bytes,
+            imap_quota_window: request.imap_quota_window,
+            download_schedule: request.download_schedule,
+            deleting: false,
+            archive_rules: request.archive_rules,
+            extraction_rules: request.extraction_rules,
+        })
+    }
+
+    pub fn check_account_exists(account_id: u64) -> BichonResult<AccountModel> {
+        Self::get(account_id)
+    }
+
+    pub fn get(account_id: u64) -> BichonResult<AccountModel> {
+        let result: AccountModel = Self::find(account_id)?.ok_or_else(|| {
+            raise_error!(
+                format!("Account with ID '{account_id}' not found"),
+                ErrorCode::ResourceNotFound
+            )
+        })?;
+        Ok(result)
+    }
+
+    pub fn find(account_id: u64) -> BichonResult<Option<AccountModel>> {
+        let result = find_impl::<AccountModel>(DB_MANAGER.db(), &account_id.to_string())?;
+        Ok(result)
+    }
+
+    pub async fn create_account(
+        user_id: u64,
+        request: AccountCreateRequest,
+    ) -> BichonResult<AccountModel> {
+        let entity = request.create_entity(user_id)?;
+        let cloned = entity.clone();
+
+        // Insert account into memdb
+        insert_impl(DB_MANAGER.db(), entity)?;
+
+        // Update user's account_access_map
+        let user = UserModel::find(user_id)?.ok_or_else(|| {
+            raise_error!(
+                format!("User with id={} not found.", user_id),
+                ErrorCode::ResourceNotFound
+            )
+        })?;
+
+        let mut updated_map = user.account_access_map.clone();
+        updated_map.insert(cloned.id, DEFAULT_ACCOUNT_MANAGER_ROLE_ID);
+
+        UserModel::update(
+            user_id,
+            UserUpdateRequest {
+                username: None,
+                email: None,
+                password: None,
+                avatar_base64: None,
+                global_roles: None,
+                account_access_map: Some(updated_map),
+                acl: None,
+                description: None,
+                theme: None,
+                language: None,
+            },
+        )?;
+
+        if matches!(cloned.account_type, AccountType::IMAP) {
+            DOWNLOAD_CONTROLLER
+                .trigger_schedule(cloned.id, cloned.email.clone())
+                .await;
+        }
+        Ok(cloned)
+    }
+
+    pub fn update(
+        account_id: u64,
+        request: AccountUpdateRequest,
+        validate: bool,
+    ) -> BichonResult<()> {
+        let account = AccountModel::get(account_id)?;
+        if validate {
+            request.validate_update_request(&account)?;
+        }
+        update_impl(
+            DB_MANAGER.db(),
+            &account_id.to_string(),
+            move |current: Account| Self::apply_update_fields(&current, request),
+        )?;
+
+        Ok(())
+    }
+
+    pub async fn delete(account_id: u64) -> BichonResult<()> {
+        let account = Self::get(account_id)?;
+
+        // Immediately stop scheduling to prevent new downloads
+        if matches!(account.account_type, AccountType::IMAP) {
+            SYNC_TASKS.stop(account.id).await?;
+        }
+
+        // Mark as deleting and disabled so frontend shows status and download tasks skip it
+        update_impl(
+            DB_MANAGER.db(),
+            &account_id.to_string(),
+            move |current: Account| {
+                let mut updated = current.clone();
+                updated.deleting = true;
+                updated.enabled = false;
+                Ok(updated)
+            },
+        )?;
+
+        // Spawn background cleanup — heavy work (Tantivy, attachments) runs off the request path
+        tokio::spawn(async move {
+            if let Err(error) = Self::cleanup_account_resources_sequential(&account).await {
+                tracing::error!(
+                    "[CLEANUP_ACCOUNT_ERROR] Account {}: cleanup failed, reverting deleting flag: {:#?}",
+                    account_id,
+                    error
+                );
+                // Revert deleting flag so the user can retry (only if account record still exists)
+                let _ = update_impl(
+                    DB_MANAGER.db(),
+                    &account_id.to_string(),
+                    move |current: Account| {
+                        let mut updated = current.clone();
+                        updated.deleting = false;
+                        updated.enabled = true;
+                        Ok(updated)
+                    },
+                );
+            }
+        });
+
+        Ok(())
+    }
+
+    fn delete_account(account: &AccountModel) -> BichonResult<()> {
+        delete_impl::<AccountModel>(DB_MANAGER.db(), &account.id.to_string())
+    }
+
+    async fn cleanup_account_resources_sequential(account: &AccountModel) -> BichonResult<()> {
+        // Sync task already stopped in delete() before spawning this background task
+        if matches!(account.account_type, AccountType::IMAP) {
+            DownloadState::delete(account.id)?;
+        }
+        OAuth2AccessToken::try_delete(account.id)?;
+        UserModel::cleanup_account(account.id)?;
+        MailBox::clean(account.id)?;
+        ENVELOPE_MANAGER
+            .delete_account_envelopes(account.id)
+            .await?;
+        ATTACHMENT_MANAGER
+            .delete_account_attachments(account.id)
+            .await?;
+        Self::delete_account(account)?;
+        info!("Sequential cleanup completed for account: {}", account.id);
+        Ok(())
+    }
+
+    pub fn update_download_folders(
+        account_id: u64,
+        download_folders: Vec<String>,
+    ) -> BichonResult<()> {
+        update_impl(
+            DB_MANAGER.db(),
+            &account_id.to_string(),
+            move |current: Account| {
+                let mut updated = current.clone();
+                updated.download_folders = Some(download_folders);
+                Ok(updated)
+            },
+        )?;
+        Ok(())
+    }
+
+    pub fn update_known_folders(
+        account_id: u64,
+        known_folders: BTreeSet<String>,
+    ) -> BichonResult<()> {
+        update_impl(
+            DB_MANAGER.db(),
+            &account_id.to_string(),
+            move |current: Account| {
+                let mut updated = current.clone();
+                updated.known_folders = Some(known_folders);
+                Ok(updated)
+            },
+        )?;
+        Ok(())
+    }
+
+    pub fn update_capabilities(account_id: u64, capabilities: Vec<String>) -> BichonResult<()> {
+        update_impl(
+            DB_MANAGER.db(),
+            &account_id.to_string(),
+            move |current: Account| {
+                let mut updated = current.clone();
+                updated.capabilities = Some(capabilities);
+                Ok(updated)
+            },
+        )?;
+        Ok(())
+    }
+
+    /// Retrieves a list of all `AccountEntity` instances.
+    pub fn list_all() -> BichonResult<Vec<AccountModel>> {
+        list_all_impl::<AccountModel>(DB_MANAGER.db())
+    }
+
+    pub fn find_by_email(email: &str) -> BichonResult<Option<AccountModel>> {
+        let all: Vec<AccountModel> = list_all_impl::<AccountModel>(DB_MANAGER.db())?;
+        let target_email = email.trim().to_lowercase();
+
+        let first_match = all
+            .into_iter()
+            .find(|acc| acc.email.to_lowercase() == target_email);
+
+        Ok(first_match)
+    }
+
+    pub fn minimal_list(only_nosync: bool) -> BichonResult<Vec<MinimalAccount>> {
+        let result = list_all_impl::<AccountModel>(DB_MANAGER.db())?
+            .into_iter()
+            .filter(|account: &AccountModel| {
+                !only_nosync || matches!(account.account_type, AccountType::NoSync)
+            })
+            .map(|account: AccountModel| MinimalAccount {
+                id: account.id,
+                email: account.email,
+                name: account.account_name,
+            })
+            .collect::<Vec<MinimalAccount>>();
+        Ok(result)
+    }
+
+    pub fn count() -> BichonResult<usize> {
+        count_impl::<AccountModel>(DB_MANAGER.db())
+    }
+
+    pub fn paginate_list(
+        page: Option<u64>,
+        page_size: Option<u64>,
+        desc: Option<bool>,
+    ) -> BichonResult<DataPage<AccountModel>> {
+        paginate_impl::<AccountModel>(DB_MANAGER.db(), page, page_size, desc).map(DataPage::from)
+    }
+
+    // This method applies the updates from the request to the old account entity
+    fn apply_update_fields(
+        old: &AccountModel,
+        request: AccountUpdateRequest,
+    ) -> BichonResult<AccountModel> {
+        let mut new = old.clone();
+
+        if let Some(date_since) = request.date_since {
+            new.date_since = Some(date_since);
+            new.date_before = None;
+        }
+
+        if let Some(date_before) = request.date_before {
+            new.date_before = Some(date_before);
+            new.date_since = None;
+        }
+
+        if let Some(clear_date_range) = request.clear_date_range {
+            if clear_date_range {
+                new.date_since = None;
+                new.date_before = None;
+            }
+        }
+
+        if let Some(account_name) = request.account_name {
+            new.account_name = Some(account_name);
+        }
+
+        if matches!(old.account_type, AccountType::IMAP) {
+            if let Some(imap) = &request.imap {
+                if let Some(current_imap) = &mut new.imap {
+                    current_imap.host = imap.host.clone();
+                    current_imap.port = imap.port.clone();
+                    current_imap.encryption = imap.encryption.clone();
+                    current_imap.auth.auth_type = imap.auth.auth_type.clone();
+                    if let Some(password) = &imap.auth.password {
+                        let encrypted_password = encrypt!(password)?;
+                        current_imap.auth.password = Some(encrypted_password);
+                    }
+                    current_imap.use_proxy = imap.use_proxy;
+                }
+            }
+
+            if let Some(folder_names) = request.sync_folders {
+                new.download_folders = Some(folder_names);
+            }
+            if let Some(sync_interval_min) = &request.download_interval_min {
+                new.download_interval_min = Some(*sync_interval_min);
+            }
+
+            if let Some(download_batch_size) = &request.download_batch_size {
+                new.download_batch_size = Some(*download_batch_size);
+            }
+
+            if let Some(max_email_size_bytes) = request.max_email_size_bytes {
+                new.max_email_size_bytes = Some(max_email_size_bytes);
+            }
+        }
+
+        if matches!(old.account_type, AccountType::NoSync) {
+            if let Some(email) = &request.email {
+                new.email = email.clone();
+            }
+        }
+
+        if let Some(enabled) = request.enabled {
+            new.enabled = enabled;
+        }
+
+        if let Some(use_dangerous) = request.use_dangerous {
+            new.use_dangerous = use_dangerous;
+        }
+
+        if let Some(pgp_key) = request.pgp_key {
+            new.pgp_key = Some(pgp_key);
+        }
+
+        if let Some(imap_quota_bytes) = request.imap_quota_bytes {
+            new.imap_quota_bytes = Some(imap_quota_bytes);
+        }
+
+        if let Some(imap_quota_window) = request.imap_quota_window {
+            new.imap_quota_window = Some(imap_quota_window);
+        }
+
+        if let Some(auto_download_new_mailboxes) = request.auto_download_new_mailboxes {
+            new.auto_download_new_mailboxes = Some(auto_download_new_mailboxes);
+        }
+        if let Some(download_schedule) = request.download_schedule {
+            new.download_schedule = Some(download_schedule);
+        }
+        if request.clear_download_schedule == Some(true) {
+            new.download_schedule = None;
+        }
+        if request.extraction_rules.is_some() {
+            new.extraction_rules = request.extraction_rules;
+        }
+        if request.archive_rules.is_some() {
+            new.archive_rules = request.archive_rules;
+        }
+        new.updated_at = utc_now!();
+        Ok(new)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // ── FilterRule ───────────────────────────────────────────────────
+
+    #[test]
+    fn filter_rule_include_only() {
+        let r = FilterRule {
+            include: vec![r"@ok\.com$".into()],
+            ..Default::default()
+        };
+        assert!(r.matches_regex("bob@ok.com"));
+        assert!(!r.matches_regex("spam@bad.com"));
+    }
+
+    #[test]
+    fn filter_rule_exclude_only() {
+        let r = FilterRule {
+            exclude: vec![r"@spam\.com$".into()],
+            ..Default::default()
+        };
+        assert!(r.matches_regex("bob@ok.com"));
+        assert!(!r.matches_regex("bot@spam.com"));
+    }
+
+    #[test]
+    fn filter_rule_include_then_exclude() {
+        let r = FilterRule {
+            include: vec![r"@company\.com$".into()],
+            exclude: vec![r"noreply@company\.com$".into()],
+            ..Default::default()
+        };
+        assert!(r.matches_regex("bob@company.com"));
+        assert!(!r.matches_regex("noreply@company.com"));
+        assert!(!r.matches_regex("spam@other.com"));
+    }
+
+    #[test]
+    fn filter_rule_exact_match() {
+        let r = FilterRule {
+            include: vec!["pdf".into(), "docx".into()],
+            exclude: vec!["xlsx".into()],
+            ..Default::default()
+        };
+        assert!(r.matches_exact("pdf"));
+        assert!(r.matches_exact("docx"));
+        assert!(r.matches_exact("DOCX")); // case-insensitive
+        assert!(!r.matches_exact("xlsx"));
+        assert!(!r.matches_exact("txt"));
+    }
+
+    // ── ExtractionRules ─────────────────────────────────────────────
+
+    #[test]
+    fn extraction_rules_master_switch() {
+        let rules = ExtractionRules {
+            enabled: false,
+            ..Default::default()
+        };
+        assert!(!rules.should_extract("pdf", None, None, None));
+    }
+
+    #[test]
+    fn extraction_rules_extension_include() {
+        let rules = ExtractionRules {
+            enabled: true,
+            extensions: FilterRule {
+                include: vec!["pdf".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.should_extract("pdf", None, None, None));
+        assert!(!rules.should_extract("docx", None, None, None));
+    }
+
+    #[test]
+    fn extraction_rules_extension_exclude() {
+        let rules = ExtractionRules {
+            enabled: true,
+            extensions: FilterRule {
+                exclude: vec!["xlsx".into(), "pptx".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.should_extract("pdf", None, None, None));
+        assert!(!rules.should_extract("xlsx", None, None, None));
+    }
+
+    #[test]
+    fn extraction_rules_folder_regex() {
+        let rules = ExtractionRules {
+            enabled: true,
+            folders: FilterRule {
+                include: vec![r"^INBOX/Invoices".into(), r"Contracts$".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.should_extract("pdf", Some("INBOX/Invoices"), None, None));
+        assert!(rules.should_extract("pdf", Some("Finance/Contracts"), None, None));
+        assert!(!rules.should_extract("pdf", Some("INBOX/Junk"), None, None));
+    }
+
+    #[test]
+    fn extraction_rules_attachment_name_regex() {
+        let rules = ExtractionRules {
+            enabled: true,
+            attachment_names: FilterRule {
+                include: vec![r"^invoice-.*\.pdf$".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.should_extract("pdf", None, Some("invoice-2024.pdf"), None));
+        assert!(!rules.should_extract("pdf", None, Some("newsletter.pdf"), None));
+    }
+
+    #[test]
+    fn extraction_rules_sender_regex() {
+        let rules = ExtractionRules {
+            enabled: true,
+            senders: FilterRule {
+                exclude: vec![r"@noreply\.com$".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        // non-excluded sender passes
+        assert!(rules.should_extract("pdf", None, None, Some("bob@ok.com")));
+        // excluded sender blocked
+        assert!(!rules.should_extract("pdf", None, None, Some("bot@noreply.com")));
+    }
+
+    #[test]
+    fn extraction_rules_empty_filters_pass_everything() {
+        let rules = ExtractionRules {
+            enabled: true,
+            ..Default::default()
+        };
+        assert!(rules.should_extract(
+            "anything",
+            Some("any/folder"),
+            Some("any.pdf"),
+            Some("any@x.com")
+        ));
+    }
+
+    // ── ArchiveRules ────────────────────────────────────────────────
+
+    #[test]
+    fn archive_rules_disabled_archives_everything() {
+        let rules = ArchiveRules {
+            enabled: false,
+            ..Default::default()
+        };
+        assert!(rules.should_archive(Some("spam@x.com"), Some("BUY NOW"), 999, false));
+    }
+
+    #[test]
+    fn archive_rules_sender_exclude() {
+        let rules = ArchiveRules {
+            enabled: true,
+            senders: FilterRule {
+                exclude: vec![r"@spam\.com$".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(!rules.should_archive(Some("bot@spam.com"), None, 100, false));
+        assert!(rules.should_archive(Some("friend@ok.com"), None, 100, false));
+    }
+
+    #[test]
+    fn archive_rules_subject_exclude() {
+        let rules = ArchiveRules {
+            enabled: true,
+            subjects: FilterRule {
+                exclude: vec![r"(?i)unsubscribe|buy now|limited offer".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(!rules.should_archive(None, Some("UNSUBSCRIBE NOW"), 100, false));
+        assert!(!rules.should_archive(None, Some("Limited Offer!!"), 100, false));
+        assert!(rules.should_archive(None, Some("Meeting tomorrow"), 100, false));
+    }
+
+    #[test]
+    fn archive_rules_sender_include() {
+        // Only archive emails from specific senders
+        let rules = ArchiveRules {
+            enabled: true,
+            senders: FilterRule {
+                include: vec![r"@partner\.com$".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.should_archive(Some("bob@partner.com"), None, 100, false));
+        assert!(!rules.should_archive(Some("spam@random.com"), None, 100, false));
+    }
+
+    #[test]
+    fn archive_rules_skip_larger_than() {
+        let rules = ArchiveRules {
+            enabled: true,
+            skip_larger_than: Some(50_000_000),
+            ..Default::default()
+        };
+        assert!(rules.should_archive(None, None, 1_000_000, false));
+        assert!(!rules.should_archive(None, None, 60_000_000, false));
+    }
+
+    #[test]
+    fn archive_rules_skip_spam_headers() {
+        let rules = ArchiveRules {
+            enabled: true,
+            spam_headers: vec!["X-Spam-Flag".into()],
+            ..Default::default()
+        };
+        assert!(!rules.should_archive(None, None, 100, true));
+        assert!(rules.should_archive(None, None, 100, false));
+    }
+
+    // ── Validation ──────────────────────────────────────────────────
+
+    #[test]
+    fn validate_extraction_rules_valid() {
+        let rules = ExtractionRules {
+            folders: FilterRule {
+                include: vec![r"^INBOX/.*".into()],
+                ..Default::default()
+            },
+            senders: FilterRule {
+                exclude: vec![r"@spam\.com$".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.validate().is_ok());
+    }
+
+    #[test]
+    fn validate_extraction_rules_invalid_regex() {
+        let rules = ExtractionRules {
+            folders: FilterRule {
+                include: vec!["***bad[".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.validate().is_err());
+    }
+
+    #[test]
+    fn validate_archive_rules_valid() {
+        let rules = ArchiveRules {
+            senders: FilterRule {
+                exclude: vec![r"@spam\.com$".into()],
+                ..Default::default()
+            },
+            subjects: FilterRule {
+                include: vec![r"(?i)invoice".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.validate().is_ok());
+    }
+
+    #[test]
+    fn validate_archive_rules_invalid_regex() {
+        let rules = ArchiveRules {
+            senders: FilterRule {
+                include: vec!["[unclosed".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.validate().is_err());
+    }
+}

crates/core/src/account/mod.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -16,11 +16,12 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-pub mod dispatcher;
 pub mod entity;
 pub mod grant;
 pub mod migration;
+pub mod old_state;
 pub mod payload;
 pub mod since;
 pub mod state;
+pub mod stats;
 pub mod view;

crates/core/src/account/old_state.rs

@@ -0,0 +1,245 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+pub struct MailboxBatchProgress {
+    pub total_batches: u32,
+    pub current_batch: u32,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+pub struct AccountRunningState {
+    pub account_id: u64,
+    pub last_incremental_sync_start: i64,
+    pub last_incremental_sync_end: Option<i64>,
+    pub errors: Vec<AccountError>,
+    pub is_initial_sync_completed: bool,
+    pub progress: Option<BTreeMap<String, MailboxBatchProgress>>,
+    pub initial_sync_start_time: Option<i64>,
+    pub initial_sync_end_time: Option<i64>,
+    pub initial_sync_failed_time: Option<i64>,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+pub struct AccountError {
+    pub error: String,
+    pub at: i64,
+}
+
+// impl AccountRunningState {
+//     pub async fn add(account_id: u64) -> BichonResult<()> {
+//         let info = AccountRunningState {
+//             account_id,
+//             last_incremental_sync_start: 0,
+//             last_incremental_sync_end: None,
+//             errors: vec![],
+//             is_initial_sync_completed: false,
+//             progress: None,
+//             initial_sync_start_time: Some(utc_now!()),
+//             initial_sync_end_time: None,
+//             initial_sync_failed_time: None,
+//         };
+//         upsert_impl(DB_MANAGER.envelope_db(), info).await
+//     }
+
+//     pub async fn get(account_id: u64) -> BichonResult<Option<AccountRunningState>> {
+//         async_find_impl(DB_MANAGER.envelope_db(), account_id).await
+//     }
+
+//     async fn update_account_running_state(
+//         account_id: u64,
+//         updater: impl FnOnce(&AccountRunningState) -> BichonResult<AccountRunningState> + Send + 'static,
+//     ) -> BichonResult<()> {
+//         if Self::get(account_id).await?.is_some() {
+//             update_impl(
+//                 DB_MANAGER.envelope_db(),
+//                 move |rw| {
+//                     rw.get()
+//                         .primary::<AccountRunningState>(account_id)
+//                         .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
+//                         .ok_or_else(|| {
+//                             raise_error!(
+//                                 format!("Cannot find sync info of account={}", account_id),
+//                                 ErrorCode::ResourceNotFound
+//                             )
+//                         })
+//                 },
+//                 updater,
+//             )
+//             .await?;
+//         }
+//         Ok(())
+//     }
+
+//     pub async fn delete(account_id: u64) -> BichonResult<()> {
+//         if Self::get(account_id).await?.is_none() {
+//             return Ok(());
+//         }
+
+//         delete_impl(DB_MANAGER.envelope_db(), move |rw| {
+//             rw.get()
+//                 .primary::<AccountRunningState>(account_id)
+//                 .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
+//                 .ok_or_else(|| {
+//                     raise_error!(
+//                         format!(
+//                             "AccountRunningState '{}' not found during deletion process.",
+//                             account_id
+//                         ),
+//                         ErrorCode::ResourceNotFound
+//                     )
+//                 })
+//         })
+//         .await
+//     }
+
+//     // pub async fn set_initial_sync_start(account_id: u64) -> BichonResult<()> {
+//     //     Self::update_account_running_state(account_id, move |current| {
+//     //         let mut updated = current.clone();
+//     //         updated.initial_sync_start_time = Some(utc_now!());
+//     //         Ok(updated)
+//     //     })
+//     //     .await
+//     // }
+
+//     pub async fn set_initial_sync_completed(account_id: u64) -> BichonResult<()> {
+//         Self::update_account_running_state(account_id, move |current| {
+//             let mut updated = current.clone();
+//             updated.is_initial_sync_completed = true;
+//             updated.initial_sync_end_time = Some(utc_now!());
+//             Ok(updated)
+//         })
+//         .await
+//     }
+
+//     pub async fn set_initial_sync_failed(account_id: u64) -> BichonResult<()> {
+//         Self::update_account_running_state(account_id, move |current| {
+//             let mut updated = current.clone();
+//             updated.initial_sync_failed_time = Some(utc_now!());
+//             Ok(updated)
+//         })
+//         .await
+//     }
+
+//     pub async fn set_current_sync_batch_number(
+//         account_id: u64,
+//         syncing_folder: String,
+//         batch_number: u32,
+//     ) -> BichonResult<()> {
+//         Self::update_account_running_state(account_id, move |current| {
+//             let mut updated = current.clone();
+//             let mut progress_map = updated.progress.clone().unwrap_or_default();
+//             let entry =
+//                 progress_map
+//                     .entry(syncing_folder.to_string())
+//                     .or_insert(MailboxBatchProgress {
+//                         total_batches: 0,
+//                         current_batch: 0,
+//                     });
+//             entry.current_batch = batch_number;
+//             updated.progress = Some(progress_map);
+//             Ok(updated)
+//         })
+//         .await
+//     }
+
+//     pub async fn set_folder_initial_sync_completed(
+//         account_id: u64,
+//         syncing_folder: String,
+//     ) -> BichonResult<()> {
+//         Self::update_account_running_state(account_id, move |current| {
+//             let mut updated = current.clone();
+//             let mut progress_map = updated.progress.clone().unwrap_or_default();
+//             let entry =
+//                 progress_map
+//                     .entry(syncing_folder.to_string())
+//                     .or_insert(MailboxBatchProgress {
+//                         total_batches: 0,
+//                         current_batch: 0,
+//                     });
+//             entry.current_batch = entry.total_batches;
+//             updated.progress = Some(progress_map);
+//             Ok(updated)
+//         })
+//         .await
+//     }
+
+//     pub async fn set_initial_current_syncing_folder(
+//         account_id: u64,
+//         current_syncing_folder: String,
+//         total_sync_batches: u32,
+//     ) -> BichonResult<()> {
+//         Self::update_account_running_state(account_id, move |current| {
+//             let mut updated = current.clone();
+//             let mut progress_map = updated.progress.clone().unwrap_or_default();
+//             progress_map.insert(
+//                 current_syncing_folder.clone(),
+//                 MailboxBatchProgress {
+//                     total_batches: total_sync_batches,
+//                     current_batch: 0,
+//                 },
+//             );
+//             updated.progress = Some(progress_map);
+//             Ok(updated)
+//         })
+//         .await
+//     }
+
+//     pub async fn set_incremental_sync_start(account_id: u64) -> BichonResult<()> {
+//         Self::update_account_running_state(account_id, move |current| {
+//             let mut updated = current.clone();
+//             updated.last_incremental_sync_start = utc_now!();
+//             updated.last_incremental_sync_end = None;
+//             Ok(updated)
+//         })
+//         .await
+//     }
+
+//     pub async fn set_incremental_sync_end(account_id: u64) -> BichonResult<()> {
+//         Self::update_account_running_state(account_id, move |current| {
+//             let mut updated = current.clone();
+//             updated.last_incremental_sync_end = Some(utc_now!());
+//             Ok(updated)
+//         })
+//         .await
+//     }
+
+//     pub async fn append_error_message(account_id: u64, error: String) -> BichonResult<()> {
+//         Self::update_account_running_state(account_id, move |current| {
+//             let mut updated = current.clone();
+//             updated.append_error_log(error);
+//             Ok(updated)
+//         })
+//         .await
+//     }
+
+//     pub fn append_error_log(&mut self, error: String) {
+//         let new_error = AccountError {
+//             error,
+//             at: utc_now!(),
+//         };
+
+//         self.errors.push(new_error);
+//         if self.errors.len() > ERROR_COUNT_PER_ACCOUNT {
+//             self.errors.remove(0);
+//         }
+//     }
+// }

crates/core/src/account/payload.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -16,34 +16,53 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-use crate::modules::account::entity::ImapConfig;
-use crate::modules::account::migration::{AccountModel, AccountType};
-use crate::modules::account::since::{DateSince, RelativeDate};
-use crate::modules::error::code::ErrorCode;
-use crate::modules::error::BichonResult;
+use std::str::FromStr;
+
+use crate::account::entity::ImapConfig;
+use crate::account::migration::{
+    AccountModel, AccountType, ArchiveRules, ExtractionRules, QuotaWindow,
+};
+use crate::account::since::{DateSince, RelativeDate};
+use crate::error::code::ErrorCode;
+use crate::error::BichonResult;
 use crate::{raise_error, validate_email};
-use poem_openapi::Object;
 use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)]
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct AccountCreateRequest {
-    #[oai(validator(custom = "crate::modules::common::validator::EmailValidator"))]
+    #[cfg_attr(
+        feature = "web-api",
+        oai(validator(custom = "crate::common::validator::EmailValidator"))
+    )]
     pub email: String,
-    pub name: Option<String>,
+    pub login_name: Option<String>,
+    pub account_name: Option<String>,
     pub imap: Option<ImapConfig>,
     pub enabled: bool,
     pub date_since: Option<DateSince>,
     pub date_before: Option<RelativeDate>,
     pub account_type: AccountType,
-    #[oai(validator(minimum(value = "100")))]
-    pub folder_limit: Option<u32>,
-    #[oai(validator(minimum(value = "10"), maximum(value = "480")))]
-    pub sync_interval_min: Option<i64>,
-    #[oai(validator(minimum(value = "30"), maximum(value = "200")))]
-    pub sync_batch_size: Option<u32>,
-    pub use_proxy: Option<u64>,
+    #[cfg_attr(feature = "web-api", oai(validator(minimum(value = "1"))))]
+    pub download_interval_min: Option<i64>,
+    #[cfg_attr(
+        feature = "web-api",
+        oai(validator(minimum(value = "10"), maximum(value = "200")))
+    )]
+    pub download_batch_size: Option<u32>,
+    pub max_email_size_bytes: Option<u64>,
     pub use_dangerous: bool,
     pub pgp_key: Option<String>,
+    pub imap_quota_bytes: Option<u64>,
+    pub imap_quota_window: Option<QuotaWindow>,
+    pub auto_download_new_mailboxes: Option<bool>,
+    pub download_schedule: Option<String>,
+    /// Email archive filtering rules (Pro feature).
+    /// `None` = archive everything (backward compatible).
+    pub archive_rules: Option<ArchiveRules>,
+    /// Attachment text extraction rules (Pro feature).
+    /// `None` = extract everything (backward compatible).
+    pub extraction_rules: Option<ExtractionRules>,
 }
 
 impl AccountCreateRequest {
@@ -56,6 +75,13 @@ impl AccountCreateRequest {
             ));
         }
 
+        if self.imap_quota_bytes.is_some() ^ self.imap_quota_window.is_some() {
+            return Err(raise_error!(
+                "Quota bytes and quota window must be provided together or omitted together".into(),
+                ErrorCode::InvalidParameter
+            ));
+        }
+
         if let Some(date_since) = self.date_since.as_ref() {
             date_since.validate()?;
         }
@@ -75,15 +101,31 @@ impl AccountCreateRequest {
                         ))
                     }
                 }
-                if self.sync_interval_min.is_none() {
+                if self.download_interval_min.is_none() && self.download_schedule.is_none() {
                     return Err(raise_error!(
-                        "`sync_interval_min` is required for IMAP account type".into(),
+                        "`sync_interval_min` or `download_schedule` is required for IMAP account type".into(),
                         ErrorCode::InvalidParameter
                     ));
                 }
+                if let Some(ref schedule) = self.download_schedule {
+                    validate_cron_expression(schedule)?;
+                }
             }
             AccountType::NoSync => {}
         }
+        if let Some(ref rules) = self.extraction_rules {
+            rules.validate().map_err(|e| {
+                raise_error!(
+                    format!("extraction_rules: {}", e),
+                    ErrorCode::InvalidParameter
+                )
+            })?;
+        }
+        if let Some(ref rules) = self.archive_rules {
+            rules.validate().map_err(|e| {
+                raise_error!(format!("archive_rules: {}", e), ErrorCode::InvalidParameter)
+            })?;
+        }
         Ok(AccountModel::new(user_id, self)?)
     }
 
@@ -96,7 +138,8 @@ impl AccountCreateRequest {
     }
 }
 
-#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)]
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct AccountUpdateRequest {
     pub email: Option<String>,
     /// Represents the account activation status.
@@ -105,8 +148,7 @@ pub struct AccountUpdateRequest {
     /// and any attempts to access them should return an error indicating the account
     /// is inactive.
     pub enabled: Option<bool>,
-    /// Display name for the account (optional)
-    pub name: Option<String>,
+    pub account_name: Option<String>,
     /// IMAP server configuration
     pub imap: Option<ImapConfig>,
     /// Controls initial synchronization time range
@@ -122,11 +164,6 @@ pub struct AccountUpdateRequest {
     pub date_since: Option<DateSince>,
     pub date_before: Option<RelativeDate>,
     pub clear_date_range: Option<bool>,
-    /// Max emails to sync for this folder.  
-    /// If not set, sync all emails.  
-    /// otherwise sync up to `n` most recent emails (min 10).
-    #[oai(validator(minimum(value = "100")))]
-    pub folder_limit: Option<u32>,
     /// Configuration for selective folder (mailbox/label) synchronization
     ///
     /// - For IMAP/SMTP accounts:
@@ -141,19 +178,29 @@ pub struct AccountUpdateRequest {
     /// Defaults to standard folders (`INBOX`, `Sent`) if empty.
     /// Modified folders will be automatically synced on the next update.
     pub sync_folders: Option<Vec<String>>,
-    /// Incremental sync interval (seconds)
-    #[oai(validator(minimum(value = "10"), maximum(value = "480")))]
-    pub sync_interval_min: Option<i64>,
-    #[oai(validator(minimum(value = "30"), maximum(value = "200")))]
-    pub sync_batch_size: Option<u32>,
-    /// Optional proxy ID for establishing the connection to external APIs (e.g., Gmail, Outlook).
-    /// - If `None` or not provided, the client will connect directly to the API server.
-    /// - If `Some(proxy_id)`, the client will use the pre-configured proxy with the given ID for API requests.
-    pub use_proxy: Option<u64>,
-
+    /// Incremental download interval (seconds)
+    #[cfg_attr(feature = "web-api", oai(validator(minimum(value = "1"))))]
+    pub download_interval_min: Option<i64>,
+    #[cfg_attr(
+        feature = "web-api",
+        oai(validator(minimum(value = "10"), maximum(value = "200")))
+    )]
+    pub download_batch_size: Option<u32>,
+    pub max_email_size_bytes: Option<u64>,
     pub use_dangerous: Option<bool>,
 
     pub pgp_key: Option<String>,
+    pub imap_quota_bytes: Option<u64>,
+    pub imap_quota_window: Option<QuotaWindow>,
+    pub auto_download_new_mailboxes: Option<bool>,
+    pub download_schedule: Option<String>,
+    pub clear_download_schedule: Option<bool>,
+    /// Email archive filtering rules (Pro feature).
+    /// `None` = no change. Use `Some(ArchiveRules { .. })` to set.
+    pub archive_rules: Option<ArchiveRules>,
+    /// Attachment text extraction rules (Pro feature).
+    /// `None` = no change. Use `Some(ExtractionRules { .. })` to set.
+    pub extraction_rules: Option<ExtractionRules>,
 }
 
 impl AccountUpdateRequest {
@@ -166,6 +213,13 @@ impl AccountUpdateRequest {
             ));
         }
 
+        if self.imap_quota_bytes.is_some() ^ self.imap_quota_window.is_some() {
+            return Err(raise_error!(
+                "Quota bytes and quota window must be provided together or omitted together".into(),
+                ErrorCode::InvalidParameter
+            ));
+        }
+
         if self.clear_date_range == Some(true)
             && (self.date_since.is_some() || self.date_before.is_some())
         {
@@ -192,16 +246,56 @@ impl AccountUpdateRequest {
                 ));
                 }
             }
+            if self.clear_download_schedule == Some(true) && self.download_schedule.is_some() {
+                return Err(raise_error!(
+                    "clear_download_schedule cannot be combined with download_schedule".into(),
+                    ErrorCode::InvalidParameter
+                ));
+            }
+            if let Some(ref schedule) = self.download_schedule {
+                validate_cron_expression(schedule)?;
+            }
+        }
+        if let Some(ref rules) = self.extraction_rules {
+            rules.validate().map_err(|e| {
+                raise_error!(
+                    format!("extraction_rules: {}", e),
+                    ErrorCode::InvalidParameter
+                )
+            })?;
+        }
+        if let Some(ref rules) = self.archive_rules {
+            rules.validate().map_err(|e| {
+                raise_error!(format!("archive_rules: {}", e), ErrorCode::InvalidParameter)
+            })?;
         }
         Ok(())
     }
 }
 
-#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)]
+fn validate_cron_expression(expr: &str) -> BichonResult<()> {
+    if expr.trim().is_empty() {
+        return Err(raise_error!(
+            "download_schedule must not be empty".into(),
+            ErrorCode::InvalidParameter
+        ));
+    }
+    cron::Schedule::from_str(expr).map_err(|e| {
+        raise_error!(
+            format!("Invalid cron expression '{}': {}", expr, e),
+            ErrorCode::InvalidParameter
+        )
+    })?;
+    Ok(())
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 
 pub struct MinimalAccount {
     pub id: u64,
     pub email: String,
+    pub name: Option<String>,
 }
 
 pub fn filter_accessible_accounts<'a>(
@@ -214,3 +308,34 @@ pub fn filter_accessible_accounts<'a>(
         .cloned()
         .collect()
 }
+
+#[cfg(test)]
+mod test {
+    use super::validate_cron_expression;
+
+    #[test]
+    fn valid_cron_expressions() {
+        assert!(validate_cron_expression("0 0 0 * * *").is_ok()); // daily at midnight
+        assert!(validate_cron_expression("0 */5 * * * *").is_ok()); // every 5 minutes
+        assert!(validate_cron_expression("0 0 12 * * 1-5").is_ok()); // weekdays at noon
+        assert!(validate_cron_expression("0 30 4 1 * *").is_ok()); // 1st of month at 04:30
+        assert!(validate_cron_expression("0 0 * * * *").is_ok()); // every hour
+    }
+
+    #[test]
+    fn invalid_cron_expression_too_few_fields() {
+        assert!(validate_cron_expression("0 0 * *").is_err());
+        assert!(validate_cron_expression("* * * * *").is_err()); // 5 fields, needs seconds
+    }
+
+    #[test]
+    fn invalid_cron_expression_empty() {
+        assert!(validate_cron_expression("").is_err());
+        assert!(validate_cron_expression("   ").is_err());
+    }
+
+    #[test]
+    fn invalid_cron_expression_garbage() {
+        assert!(validate_cron_expression("not a cron").is_err());
+    }
+}

crates/core/src/account/since.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -17,14 +17,14 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::{
-    modules::error::{code::ErrorCode, BichonResult},
+    error::{code::ErrorCode, BichonResult},
     raise_error,
 };
 use chrono::{Datelike, Days, Local, Months, NaiveDate, Utc};
-use poem_openapi::{Enum, Object};
 use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)]
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct DateSince {
     /// Absolute date boundary in ISO 8601 format (YYYY-MM-DD)
     ///
@@ -38,7 +38,7 @@ pub struct DateSince {
     ///   "fixed": "2025-05-01"
     /// }
     /// ```
-    #[oai(validator(pattern = r"^\d{4}-\d{2}-\d{2}$"))]
+    #[cfg_attr(feature = "web-api", oai(validator(pattern = r"^\d{4}-\d{2}-\d{2}$")))]
     pub fixed: Option<String>,
     /// Relative time period from current date
     ///
@@ -58,7 +58,8 @@ pub struct DateSince {
     pub relative: Option<RelativeDate>,
 }
 
-#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Enum)]
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))]
 pub enum Unit {
     #[default]
     Days,
@@ -66,12 +67,13 @@ pub enum Unit {
     Years,
 }
 
-#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)]
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct RelativeDate {
     /// The time unit to use for the offset (days, months, or years)
     pub unit: Unit,
     /// The quantity of time units to offset (must be a positive integer)
-    #[oai(validator(minimum(value = "1")))]
+    #[cfg_attr(feature = "web-api", oai(validator(minimum(value = "1"))))]
     pub value: u32,
 }
 
@@ -255,19 +257,47 @@ impl DateSince {
 
 #[cfg(test)]
 mod test {
-    use crate::modules::account::since::{DateSince, RelativeDate, Unit};
+    use crate::account::since::{DateSince, RelativeDate, Unit};
 
     #[test]
-    fn test1() {
+    fn fixed_date_valid() {
         let e = DateSince {
             fixed: Some("2014-09-12".to_string()),
             relative: None,
         };
+        assert!(e.validate().is_ok());
+        assert!(!e.since_date().unwrap().is_empty());
+    }
 
-        e.validate().unwrap();
+    #[test]
+    fn fixed_date_in_future_fails() {
+        let e = DateSince {
+            fixed: Some("2099-01-01".to_string()),
+            relative: None,
+        };
+        assert!(e.validate().is_err());
+    }
 
-        println!("{}", e.since_date().unwrap());
+    #[test]
+    fn fixed_date_before_1970_fails() {
+        let e = DateSince {
+            fixed: Some("1960-01-01".to_string()),
+            relative: None,
+        };
+        assert!(e.validate().is_err());
+    }
 
+    #[test]
+    fn fixed_date_bad_format_fails() {
+        let e = DateSince {
+            fixed: Some("01-01-2020".to_string()),
+            relative: None,
+        };
+        assert!(e.validate().is_err());
+    }
+
+    #[test]
+    fn relative_date_days_valid() {
         let e = DateSince {
             fixed: None,
             relative: Some(RelativeDate {
@@ -275,9 +305,102 @@ mod test {
                 value: 1,
             }),
         };
+        assert!(e.validate().is_ok());
+    }
 
-        e.validate().unwrap();
+    #[test]
+    fn relative_date_months_valid() {
+        let e = DateSince {
+            fixed: None,
+            relative: Some(RelativeDate {
+                unit: Unit::Months,
+                value: 3,
+            }),
+        };
+        assert!(e.validate().is_ok());
+    }
 
-        println!("{}", e.since_date().unwrap());
+    #[test]
+    fn relative_date_years_valid() {
+        let e = DateSince {
+            fixed: None,
+            relative: Some(RelativeDate {
+                unit: Unit::Years,
+                value: 1,
+            }),
+        };
+        assert!(e.validate().is_ok());
+    }
+
+    #[test]
+    fn relative_date_zero_value_fails() {
+        let e = DateSince {
+            fixed: None,
+            relative: Some(RelativeDate {
+                unit: Unit::Days,
+                value: 0,
+            }),
+        };
+        assert!(e.validate().is_err());
+    }
+
+    #[test]
+    fn both_fixed_and_relative_fails() {
+        let e = DateSince {
+            fixed: Some("2014-09-12".to_string()),
+            relative: Some(RelativeDate {
+                unit: Unit::Days,
+                value: 1,
+            }),
+        };
+        assert!(e.validate().is_err());
+    }
+
+    #[test]
+    fn neither_fixed_nor_relative_fails() {
+        let e = DateSince {
+            fixed: None,
+            relative: None,
+        };
+        assert!(e.validate().is_err());
+    }
+
+    // ── Sliding window tests ──────────────────────────────────────
+
+    #[test]
+    fn relative_date_calculate_returns_valid_format() {
+        let r = RelativeDate {
+            unit: Unit::Years,
+            value: 1,
+        };
+        let date_str = r.calculate_date().unwrap();
+        // Expect format like "26-May-2025"
+        assert!(date_str.len() > 5);
+        assert!(date_str.contains('-'));
+    }
+
+    #[test]
+    fn relative_date_one_year_ago_is_before_now() {
+        let r = RelativeDate {
+            unit: Unit::Years,
+            value: 1,
+        };
+        let date_str = r.calculate_date().unwrap();
+        let parsed = chrono::NaiveDate::parse_from_str(&date_str, "%d-%b-%Y").unwrap();
+        let today = chrono::Local::now().date_naive();
+        assert!(parsed < today, "1 year ago ({parsed}) should be before today ({today})");
+    }
+
+    #[test]
+    fn relative_date_one_day_ago_is_yesterday() {
+        let r = RelativeDate {
+            unit: Unit::Days,
+            value: 1,
+        };
+        let date_str = r.calculate_date().unwrap();
+        let parsed = chrono::NaiveDate::parse_from_str(&date_str, "%d-%b-%Y").unwrap();
+        let today = chrono::Local::now().date_naive();
+        let yesterday = today - chrono::Duration::days(1);
+        assert_eq!(parsed, yesterday, "1 day ago should be yesterday");
     }
 }

crates/core/src/account/state.rs

@@ -0,0 +1,569 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{
+    database::{delete_impl, find_impl, manager::DB_MANAGER, update_impl, upsert_impl, MemDbModel},
+    error::BichonResult,
+    utc_now,
+};
+use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))]
+pub enum DownloadStatus {
+    Running,
+    Success,
+    Failed,
+    #[default]
+    Cancelled,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))]
+pub enum TriggerType {
+    Manual,
+    #[default]
+    Scheduled,
+    /// Full re-sync (UID SEARCH ALL) invoked explicitly, e.g. to repair a
+    /// mailbox whose incremental download was interrupted. Semantically a
+    /// manual trigger, tracked distinctly for diagnostics.
+    SyncFull,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))]
+pub enum FolderStatus {
+    #[default]
+    Pending,
+    Downloading,
+    Success,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct FolderProgress {
+    pub folder_name: String,
+    pub planned: u64,
+    pub current: u64,
+    pub status: FolderStatus,
+    pub message: Option<String>,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))]
+pub enum GapFillStatus {
+    #[default]
+    Running,
+    Success,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct GapFillFolderStats {
+    pub downloaded: u64,
+    pub failed: u64,
+    pub candidate_count: u64,
+    /// Live progress hint (e.g. "IMAP server is slow...") shown while the
+    /// folder is being scanned; usually `None` once the folder is done.
+    pub message: Option<String>,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct GapFillRun {
+    pub started_at: i64,
+    pub finished_at: Option<i64>,
+    pub status: GapFillStatus,
+    /// Per-mailbox gap-fill outcome, keyed by mailbox name.
+    pub folders: BTreeMap<String, GapFillFolderStats>,
+    /// Total emails newly downloaded by gap-fill.
+    pub downloaded: u64,
+    /// Total emails that failed to download during gap-fill.
+    pub failed: u64,
+}
+
+/// Independent, repeatable gap-fill history for an account. Gap-fill is a
+/// distinct operation from downloading (it can be run again and again until
+/// `failed == 0`), so its runs are tracked separately from `DownloadState`
+/// instead of being mixed into download sessions.
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct GapFillState {
+    pub account_id: u64,
+    /// The gap-fill run currently in progress, if any.
+    pub active: Option<GapFillRun>,
+    /// Finished runs, most recent last.
+    pub history: Vec<GapFillRun>,
+}
+
+impl MemDbModel for GapFillState {
+    fn collection() -> &'static str {
+        "gap_fill_states"
+    }
+    fn key(&self) -> String {
+        self.account_id.to_string()
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct DownloadSession {
+    pub start_time: i64,
+    pub end_time: Option<i64>,
+    pub status: DownloadStatus,
+    pub message: Option<String>,
+    pub trigger: TriggerType,
+    pub folder_details: BTreeMap<String, FolderProgress>,
+    pub current_folder: Option<String>,
+    pub errors: Vec<AccountError>,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct DownloadState {
+    pub account_id: u64,
+    pub active_session: Option<DownloadSession>,
+    pub history: Vec<DownloadSession>,
+    pub last_trigger_at: i64,
+    pub last_finished_at: Option<i64>,
+}
+
+impl MemDbModel for DownloadState {
+    fn collection() -> &'static str {
+        "download_states"
+    }
+    fn key(&self) -> String {
+        self.account_id.to_string()
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct AccountError {
+    pub error: String,
+    pub at: i64,
+}
+
+impl DownloadState {
+    pub fn empty(account_id: u64) -> Self {
+        DownloadState {
+            account_id,
+            ..Default::default()
+        }
+    }
+
+    pub async fn init(account_id: u64) -> BichonResult<()> {
+        let now = utc_now!();
+        let state = DownloadState {
+            account_id,
+            last_trigger_at: now,
+            active_session: Some(DownloadSession {
+                start_time: now,
+                status: DownloadStatus::Running,
+                trigger: TriggerType::Scheduled,
+                ..Default::default()
+            }),
+            history: Default::default(),
+            last_finished_at: Default::default(),
+        };
+        upsert_impl(DB_MANAGER.db(), state)
+    }
+
+    pub fn get(account_id: u64) -> BichonResult<Option<DownloadState>> {
+        find_impl::<DownloadState>(DB_MANAGER.db(), &account_id.to_string())
+    }
+
+    pub fn start_new_session(account_id: u64, trigger: TriggerType) -> BichonResult<()> {
+        Self::update_state(account_id, move |current| {
+            let mut updated = current.clone();
+            updated.last_trigger_at = utc_now!();
+
+            if let Some(mut old_session) = updated.active_session.take() {
+                if old_session.status == DownloadStatus::Running {
+                    old_session.status = DownloadStatus::Cancelled;
+                    old_session.end_time = Some(utc_now!());
+                    old_session.message = Some("Interrupted by a new download session.".into());
+                }
+                updated.history.push(old_session);
+                if updated.history.len() > 30 {
+                    updated.history.remove(0);
+                }
+            }
+
+            let new_session = DownloadSession {
+                start_time: utc_now!(),
+                status: DownloadStatus::Running,
+                trigger,
+                ..Default::default()
+            };
+
+            updated.active_session = Some(new_session);
+            Ok(updated)
+        })
+    }
+
+    pub fn update_session_status(
+        account_id: u64,
+        status: DownloadStatus,
+        message: Option<String>,
+    ) -> BichonResult<()> {
+        Self::update_state(account_id, move |current| {
+            let mut updated = current.clone();
+            if let Some(mut session) = updated.active_session.take() {
+                session.status = status.clone();
+                if message.is_some() {
+                    session.message = message;
+                }
+                if status == DownloadStatus::Running {
+                    updated.active_session = Some(session);
+                } else {
+                    let now = utc_now!();
+                    session.end_time = Some(now);
+                    updated.last_finished_at = Some(now);
+                    updated.history.push(session);
+                    let to_remove = updated.history.len().saturating_sub(10);
+                    if to_remove > 0 {
+                        updated.history.drain(0..to_remove);
+                    }
+                }
+            }
+            Ok(updated)
+        })
+    }
+
+    /// Moves a stale Running session into history as Cancelled.
+    ///
+    /// A Running `active_session` that survives an Idle decision means the
+    /// previous run was interrupted without a clean shutdown (e.g. process
+    /// killed mid-download). Leaving it in place makes the UI show a phantom
+    /// "syncing" state even though nothing is downloading. Callers invoke this
+    /// only when no download is actually running for the account, so a
+    /// legitimately active session is never touched.
+    ///
+    /// Returns `true` if a stale session was finalized (i.e. the previous sync
+    /// did not finish) and `false` otherwise.
+    pub fn finalize_stale_session(account_id: u64) -> BichonResult<bool> {
+        let stale = Self::get(account_id)?
+            .and_then(|s| s.active_session)
+            .map_or(false, |s| s.status == DownloadStatus::Running);
+        if stale {
+            Self::update_state(account_id, move |current| {
+                let mut updated = current.clone();
+                if updated
+                    .active_session
+                    .as_ref()
+                    .map_or(false, |s| s.status == DownloadStatus::Running)
+                {
+                    if let Some(mut session) = updated.active_session.take() {
+                        session.status = DownloadStatus::Cancelled;
+                        session.end_time = Some(utc_now!());
+                        session.message = Some(
+                            "Previous sync did not finish cleanly; marked as cancelled on startup."
+                                .into(),
+                        );
+                        updated.history.push(session);
+                        updated.last_finished_at = Some(utc_now!());
+                        updated.active_session = None;
+                    }
+                }
+                Ok(updated)
+            })?;
+        }
+        Ok(stale)
+    }
+
+    pub fn update_folder_progress(
+        account_id: u64,
+        folder_name: String,
+        planned: u64,
+        current: u64,
+        status: FolderStatus,
+        message: Option<String>,
+    ) -> BichonResult<()> {
+        Self::update_state(account_id, move |state| {
+            let mut updated = state.clone();
+            if let Some(ref mut session) = updated.active_session {
+                session.current_folder = Some(folder_name.clone());
+
+                let progress =
+                    session
+                        .folder_details
+                        .entry(folder_name.clone())
+                        .or_insert(FolderProgress {
+                            folder_name,
+                            ..Default::default()
+                        });
+
+                progress.planned = planned;
+                progress.current = current;
+                progress.status = status;
+                progress.message = message;
+            }
+            Ok(updated)
+        })
+    }
+
+    /// Touches only `current_folder` without rewriting folder progress. Lets
+    /// long-running IMAP operations (e.g. waiting on a slow server mid-batch)
+    /// keep the UI's "last activity" indicator fresh without spamming writes.
+    pub fn set_current_folder(account_id: u64, folder_name: String) -> BichonResult<()> {
+        Self::update_state(account_id, move |state| {
+            let mut updated = state.clone();
+            if let Some(ref mut session) = updated.active_session {
+                session.current_folder = Some(folder_name);
+            }
+            Ok(updated)
+        })
+    }
+
+    pub fn init_folder_details(account_id: u64, folders: Vec<String>) -> BichonResult<()> {
+        Self::update_state(account_id, move |state| {
+            let mut updated = state.clone();
+            if let Some(ref mut session) = updated.active_session {
+                for name in folders {
+                    session.folder_details.insert(
+                        name.clone(),
+                        FolderProgress {
+                            folder_name: name,
+                            planned: 0,
+                            current: 0,
+                            status: FolderStatus::Pending,
+                            message: None,
+                        },
+                    );
+                }
+            }
+            Ok(updated)
+        })
+    }
+
+    pub fn append_session_error(account_id: u64, error: String) -> BichonResult<()> {
+        Self::update_state(account_id, move |current| {
+            let mut updated = current.clone();
+            let new_error = AccountError {
+                error,
+                at: utc_now!(),
+            };
+            let target = updated
+                .active_session
+                .as_mut()
+                .or_else(|| updated.history.last_mut());
+            if let Some(session) = target {
+                session.errors.push(new_error);
+                let to_remove = session.errors.len().saturating_sub(30);
+                if to_remove > 0 {
+                    session.errors.drain(0..to_remove);
+                }
+            }
+            Ok(updated)
+        })
+    }
+
+    /// Appends/updates a free-form message on the active session without
+    /// changing its status. Used to record the gap-fill summary.
+    pub fn update_session_message(account_id: u64, message: String) -> BichonResult<()> {
+        Self::update_state(account_id, move |current| {
+            let mut updated = current.clone();
+            if let Some(ref mut session) = updated.active_session {
+                session.message = Some(message);
+            }
+            Ok(updated)
+        })
+    }
+
+    fn update_state(
+        account_id: u64,
+        updater: impl FnOnce(DownloadState) -> BichonResult<DownloadState> + Send + 'static,
+    ) -> BichonResult<()> {
+        if Self::get(account_id)?.is_some() {
+            update_impl(DB_MANAGER.db(), &account_id.to_string(), updater)?;
+        }
+        Ok(())
+    }
+
+    pub fn delete(account_id: u64) -> BichonResult<()> {
+        if Self::get(account_id)?.is_none() {
+            return Ok(());
+        }
+
+        delete_impl::<DownloadState>(DB_MANAGER.db(), &account_id.to_string())
+    }
+}
+
+impl GapFillState {
+    pub fn get(account_id: u64) -> BichonResult<Option<GapFillState>> {
+        find_impl::<GapFillState>(DB_MANAGER.db(), &account_id.to_string())
+    }
+
+    /// Starts a new gap-fill run, moving any stale active run into history as
+    /// Cancelled. Creates the state record on first use.
+    pub fn start_run(account_id: u64) -> BichonResult<()> {
+        let now = utc_now!();
+        let run = GapFillRun {
+            started_at: now,
+            status: GapFillStatus::Running,
+            ..Default::default()
+        };
+        if Self::get(account_id)?.is_none() {
+            let state = GapFillState {
+                account_id,
+                active: Some(run),
+                history: Vec::new(),
+            };
+            return upsert_impl(DB_MANAGER.db(), state);
+        }
+        Self::update_state(account_id, move |current| {
+            let mut updated = current.clone();
+            if let Some(mut old) = updated.active.take() {
+                if old.status == GapFillStatus::Running {
+                    old.status = GapFillStatus::Cancelled;
+                    old.finished_at = Some(utc_now!());
+                }
+                updated.history.push(old);
+                let keep = updated.history.len().saturating_sub(10);
+                if keep > 0 {
+                    updated.history.drain(0..keep);
+                }
+            }
+            updated.active = Some(run);
+            Ok(updated)
+        })
+    }
+
+    /// Accumulates a per-folder outcome into the active run.
+    pub fn add_folder_result(
+        account_id: u64,
+        folder_name: String,
+        stats: GapFillFolderStats,
+    ) -> BichonResult<()> {
+        Self::update_state(account_id, move |current| {
+            let mut updated = current.clone();
+            if let Some(ref mut run) = updated.active {
+                run.downloaded += stats.downloaded;
+                run.failed += stats.failed;
+                run.folders.insert(folder_name, stats);
+            }
+            Ok(updated)
+        })
+    }
+    /// Updates the live per-folder progress of the active run (used during a
+    /// gap-fill scan so the UI can show per-folder progress without waiting
+    /// for the folder to finish). `candidate_count` is the planned total,
+    /// `downloaded` the current count, `message` an optional live hint
+    /// (e.g. slow-server notice).
+    pub fn update_folder_progress(
+        account_id: u64,
+        folder_name: String,
+        candidate_count: u64,
+        downloaded: u64,
+        message: Option<String>,
+    ) -> BichonResult<()> {
+        Self::update_state(account_id, move |current| {
+            let mut updated = current.clone();
+            if let Some(ref mut run) = updated.active {
+                let entry = run
+                    .folders
+                    .entry(folder_name.clone())
+                    .or_insert(GapFillFolderStats {
+                        downloaded: 0,
+                        failed: 0,
+                        candidate_count,
+                        message: None,
+                    });
+                entry.candidate_count = candidate_count;
+                entry.downloaded = downloaded;
+                entry.message = message;
+            }
+            Ok(updated)
+        })
+    }
+
+    /// Finalizes the active run: moves it to history with the given status and
+    /// totals. `failed`/`downloaded` are the authoritative accumulated values.
+    pub fn finish_run(
+        account_id: u64,
+        status: GapFillStatus,
+        downloaded: u64,
+        failed: u64,
+    ) -> BichonResult<()> {
+        Self::update_state(account_id, move |current| {
+            let mut updated = current.clone();
+            if let Some(mut run) = updated.active.take() {
+                run.status = status;
+                run.finished_at = Some(utc_now!());
+                run.downloaded = downloaded;
+                run.failed = failed;
+                updated.history.push(run);
+                let keep = updated.history.len().saturating_sub(10);
+                if keep > 0 {
+                    updated.history.drain(0..keep);
+                }
+            }
+            Ok(updated)
+        })
+    }
+
+    /// Moves a stale Running active run into history as Cancelled.
+    ///
+    /// A Running `active` that survives a restart means the previous gap-fill
+    /// run was interrupted without finishing (process killed mid-scan). Leaving
+    /// it in place makes the UI show a phantom "Running" gap-fill. Callers
+    /// invoke this on startup, when no gap-fill is actually running.
+    ///
+    /// Returns `true` if a stale run was finalized.
+    pub fn finalize_stale_run(account_id: u64) -> BichonResult<bool> {
+        let stale = Self::get(account_id)?
+            .and_then(|s| s.active)
+            .map_or(false, |r| r.status == GapFillStatus::Running);
+        if stale {
+            Self::update_state(account_id, move |current| {
+                let mut updated = current.clone();
+                if let Some(mut run) = updated.active.take() {
+                    if run.status == GapFillStatus::Running {
+                        run.status = GapFillStatus::Cancelled;
+                        run.finished_at = Some(utc_now!());
+                        updated.history.push(run);
+                        let keep = updated.history.len().saturating_sub(10);
+                        if keep > 0 {
+                            updated.history.drain(0..keep);
+                        }
+                    } else {
+                        updated.active = Some(run);
+                    }
+                }
+                Ok(updated)
+            })?;
+        }
+        Ok(stale)
+    }
+
+    fn update_state(
+        account_id: u64,
+        updater: impl FnOnce(GapFillState) -> BichonResult<GapFillState> + Send + 'static,
+    ) -> BichonResult<()> {
+        if Self::get(account_id)?.is_some() {
+            update_impl(DB_MANAGER.db(), &account_id.to_string(), updater)?;
+        }
+        Ok(())
+    }
+}

crates/core/src/account/stats.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -16,9 +16,11 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+use serde::{Deserialize, Serialize};
 
-#[tokio::test]
-async fn test() {
-    let config = autoconfig::from_addr("test@gmail.com").await.unwrap();
-    println!("{:#?}", config);
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct AccountStats {
+    pub total_size: u64,
+    pub total_count: u64,
 }

crates/core/src/account/view.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -17,43 +17,48 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::collections::{BTreeSet, HashMap};
-
-use poem_openapi::Object;
 use serde::{Deserialize, Serialize};
 
-use crate::modules::{
+use crate::{
     account::{
         entity::ImapConfig,
-        migration::{AccountModel, AccountType},
+        migration::{AccountModel, AccountType, ArchiveRules, QuotaWindow},
         since::{DateSince, RelativeDate},
     },
     users::UserModel,
 };
 
-#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)]
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct AccountResp {
     pub id: u64,
     pub imap: Option<ImapConfig>,
     pub enabled: bool,
     pub email: String,
-    pub name: Option<String>,
+    pub account_name: Option<String>,
+    pub login_name: Option<String>,
     pub capabilities: Option<Vec<String>>,
     pub date_since: Option<DateSince>,
     pub date_before: Option<RelativeDate>,
-    pub folder_limit: Option<u32>,
-    pub sync_folders: Option<Vec<String>>,
+    pub download_folders: Option<Vec<String>>,
     pub account_type: AccountType,
-    pub sync_interval_min: Option<i64>,
-    pub sync_batch_size: Option<u32>,
+    pub download_interval_min: Option<i64>,
+    pub download_batch_size: Option<u32>,
+    pub max_email_size_bytes: Option<u64>,
     pub known_folders: Option<BTreeSet<String>>,
     pub created_at: i64,
     pub updated_at: i64,
     pub created_by: u64, //user id
     pub created_user_name: String,
     pub created_user_email: String,
-    pub use_proxy: Option<u64>,
     pub use_dangerous: bool,
     pub pgp_key: Option<String>,
+    pub imap_quota_bytes: Option<u64>,
+    pub imap_quota_window: Option<QuotaWindow>,
+    pub auto_download_new_mailboxes: Option<bool>,
+    pub download_schedule: Option<String>,
+    pub archive_rules: Option<ArchiveRules>,
+    pub deleting: bool,
 }
 
 impl AccountResp {
@@ -64,15 +69,16 @@ impl AccountResp {
             imap: account.imap,
             enabled: account.enabled,
             email: account.email,
-            name: account.name,
+            account_name: account.account_name,
+            login_name: account.login_name,
             capabilities: account.capabilities,
             date_since: account.date_since,
             date_before: account.date_before,
-            folder_limit: account.folder_limit,
-            sync_folders: account.sync_folders,
+            download_folders: account.download_folders,
             account_type: account.account_type,
-            sync_interval_min: account.sync_interval_min,
-            sync_batch_size: account.sync_batch_size,
+            download_interval_min: account.download_interval_min,
+            download_batch_size: account.download_batch_size,
+            max_email_size_bytes: account.max_email_size_bytes,
             known_folders: account.known_folders,
             created_at: account.created_at,
             updated_at: account.updated_at,
@@ -83,9 +89,14 @@ impl AccountResp {
             created_user_email: user
                 .map(|u| u.email.clone())
                 .unwrap_or_else(|| "N/A".to_string()),
-            use_proxy: account.use_proxy,
             use_dangerous: account.use_dangerous,
             pgp_key: account.pgp_key,
+            imap_quota_bytes: account.imap_quota_bytes,
+            imap_quota_window: account.imap_quota_window,
+            auto_download_new_mailboxes: account.auto_download_new_mailboxes,
+            download_schedule: account.download_schedule,
+            archive_rules: account.archive_rules,
+            deleting: account.deleting,
         }
     }
 }

crates/core/src/admin/meta.rs

@@ -0,0 +1,68 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::path::Path;
+
+use bichon_memdb::{Durability, MemDb};
+
+use crate::{
+    database::MemDbModel,
+    error::{code::ErrorCode, BichonResult},
+    raise_error,
+    users::{UserModel, DEFAULT_ADMIN_USER_ID},
+    utils::encrypt::internal_encrypt_string,
+};
+
+pub fn open_database(path: impl AsRef<Path>) -> BichonResult<MemDb> {
+    MemDb::open_with(path, Durability::Full).map_err(|e| {
+        raise_error!(
+            format!("Failed to open database: {:?}", e),
+            ErrorCode::InternalError
+        )
+    })
+}
+
+pub fn find_admin(db: &MemDb) -> BichonResult<Option<UserModel>> {
+    let key = DEFAULT_ADMIN_USER_ID.to_string();
+    let coll = db.collection(UserModel::collection());
+    coll.get(&key)
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
+}
+
+pub fn update_admin_password(
+    db: &MemDb,
+    password: String,
+    encrypt_key: &str,
+) -> BichonResult<()> {
+    let key = DEFAULT_ADMIN_USER_ID.to_string();
+    let coll = db.collection(UserModel::collection());
+    let entity: UserModel = coll
+        .get_required(&key)
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+    let mut updated = entity.clone();
+    updated.password = Some(
+        internal_encrypt_string(encrypt_key, &password)
+            .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?,
+    );
+
+    coll.upsert(&key, &updated)
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+    Ok(())
+}

crates/core/src/admin/mod.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -17,4 +17,4 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
-pub mod list;
+pub mod meta;

crates/core/src/autoconfig/client.rs

@@ -0,0 +1,452 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use hickory_resolver::net::runtime::TokioRuntimeProvider;
+use hickory_resolver::proto::rr::RData;
+use hickory_resolver::TokioResolver;
+use quick_xml::de::from_str;
+use reqwest::Client;
+use serde::Deserialize;
+
+use crate::error::code::ErrorCode;
+use crate::error::BichonResult;
+use crate::raise_error;
+
+/// Parsed result from Thunderbird-style autoconfig XML or DNS SRV fallback.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct MailConfig {
+    pub incoming: Vec<IncomingServer>,
+    pub outgoing: Vec<OutgoingServer>,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)]
+pub struct IncomingServer {
+    #[serde(rename = "@type")]
+    pub protocol: String,
+    pub hostname: String,
+    #[serde(default)]
+    pub port: u16,
+    #[serde(rename = "socketType")]
+    pub socket_type: String,
+    pub username: String,
+    /// Authentication method from the XML, e.g. "OAuth2", "password-cleartext",
+    /// "password-encrypted", "GSSAPI", "NTLM".  Absent in DNS SRV fallback.
+    #[serde(default)]
+    pub authentication: String,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)]
+pub struct OutgoingServer {
+    #[serde(rename = "@type")]
+    pub protocol: String,
+    pub hostname: String,
+    #[serde(default)]
+    pub port: u16,
+    #[serde(rename = "socketType")]
+    pub socket_type: String,
+    pub username: String,
+}
+
+// ---------------------------------------------------------------------------
+// Internal XML wrapper structs matching the Thunderbird config-v1.1 schema:
+// <clientConfig> → <emailProvider> → <incomingServer> / <outgoingServer>
+// ---------------------------------------------------------------------------
+
+#[derive(Debug, Deserialize)]
+#[serde(rename = "clientConfig")]
+struct ClientConfig {
+    #[serde(rename = "emailProvider", default)]
+    email_providers: Vec<EmailProvider>,
+}
+
+#[derive(Debug, Deserialize)]
+struct EmailProvider {
+    #[serde(rename = "incomingServer", default)]
+    incoming_servers: Vec<IncomingServer>,
+    #[serde(rename = "outgoingServer", default)]
+    outgoing_servers: Vec<OutgoingServer>,
+}
+
+/// Parse Thunderbird autoconfig XML into a `MailConfig`.
+/// Exposed for unit testing.
+pub(crate) fn parse_autoconfig_xml(xml: &str) -> Option<MailConfig> {
+    let client_config: ClientConfig = from_str(xml).ok()?;
+    let provider = client_config.email_providers.into_iter().next()?;
+    Some(MailConfig {
+        incoming: provider.incoming_servers,
+        outgoing: provider.outgoing_servers,
+    })
+}
+
+// ---------------------------------------------------------------------------
+// Network helpers
+// ---------------------------------------------------------------------------
+
+async fn fetch_xml(client: &Client, url: &str) -> Option<MailConfig> {
+    let resp = client.get(url).send().await.ok()?;
+    if !resp.status().is_success() {
+        return None;
+    }
+    let text = resp.text().await.ok()?;
+    parse_autoconfig_xml(&text)
+}
+
+async fn lookup_srv(domain: &str) -> Option<MailConfig> {
+    let resolver = TokioResolver::builder(TokioRuntimeProvider::default())
+        .ok()?
+        .build()
+        .ok()?;
+
+    let imap_srv = format!("_imaps._tcp.{}.", domain);
+    let imap_lookup = resolver.srv_lookup(imap_srv).await.ok()?;
+    let imap_record = imap_lookup.answers().first()?;
+    let (imap_host, imap_port) = match &imap_record.data {
+        RData::SRV(srv) => {
+            let host = srv.target.to_string().trim_end_matches('.').to_string();
+            (host, srv.port)
+        }
+        _ => return None,
+    };
+
+    let smtp_srv = format!("_submission._tcp.{}.", domain);
+    let smtp_lookup = resolver.srv_lookup(smtp_srv).await.ok()?;
+    let smtp_record = smtp_lookup.answers().first()?;
+    let (smtp_host, smtp_port) = match &smtp_record.data {
+        RData::SRV(srv) => {
+            let host = srv.target.to_string().trim_end_matches('.').to_string();
+            (host, srv.port)
+        }
+        _ => return None,
+    };
+
+    Some(MailConfig {
+        incoming: vec![IncomingServer {
+            protocol: "imap".to_string(),
+            hostname: imap_host,
+            port: imap_port,
+            socket_type: "SSL".to_string(),
+            username: "%EMAILADDRESS%".to_string(),
+            authentication: String::new(),
+        }],
+        outgoing: vec![OutgoingServer {
+            protocol: "smtp".to_string(),
+            hostname: smtp_host,
+            port: smtp_port,
+            socket_type: "STARTTLS".to_string(),
+            username: "%EMAILADDRESS%".to_string(),
+        }],
+    })
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/// Discover mail server configuration for a domain using the Thunderbird
+/// autoconfig protocol (ISPDB), DNS SRV, MX fallback, and finally guessing.
+///
+/// Probe order:
+/// 1. `https://autoconfig.{domain}/mail/config-v1.1.xml`
+/// 2. `http://autoconfig.{domain}/mail/config-v1.1.xml`
+/// 3. `https://{domain}/.well-known/autoconfig/mail/config-v1.1.xml`
+/// 4. `http://{domain}/.well-known/autoconfig/mail/config-v1.1.xml`
+/// 5. DNS SRV records (`_imaps._tcp` / `_submission._tcp`)
+/// 6. Thunderbird central ISPDB (`https://autoconfig.thunderbird.net/v1.1/{domain}`)
+/// 7. MX lookup → ISPDB for MX domain
+/// 8. MX lookup → ISP autoconfig for MX domain
+/// 9. GuessConfig — probe common hostnames + ports
+pub async fn fetch(domain: &str) -> BichonResult<MailConfig> {
+    let client = Client::builder()
+        .timeout(std::time::Duration::from_secs(10))
+        .build()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+    // ── ISP autoconfig (HTTPS, then HTTP) ──────────────────────────
+    if let Some(config) = fetch_xml(
+        &client,
+        &format!("https://autoconfig.{domain}/mail/config-v1.1.xml"),
+    )
+    .await
+    {
+        return Ok(config);
+    }
+    if let Some(config) = fetch_xml(
+        &client,
+        &format!("http://autoconfig.{domain}/mail/config-v1.1.xml"),
+    )
+    .await
+    {
+        return Ok(config);
+    }
+
+    // ── Well-known path (HTTPS, then HTTP) ─────────────────────────
+    if let Some(config) = fetch_xml(
+        &client,
+        &format!("https://{domain}/.well-known/autoconfig/mail/config-v1.1.xml"),
+    )
+    .await
+    {
+        return Ok(config);
+    }
+    if let Some(config) = fetch_xml(
+        &client,
+        &format!("http://{domain}/.well-known/autoconfig/mail/config-v1.1.xml"),
+    )
+    .await
+    {
+        return Ok(config);
+    }
+
+    // ── DNS SRV records ────────────────────────────────────────────
+    if let Some(config) = lookup_srv(domain).await {
+        return Ok(config);
+    }
+
+    // ── Thunderbird central ISPDB ──────────────────────────────────
+    if let Some(config) = fetch_xml(
+        &client,
+        &format!("https://autoconfig.thunderbird.net/v1.1/{domain}"),
+    )
+    .await
+    {
+        return Ok(config);
+    }
+
+    // ── MX fallback ────────────────────────────────────────────────
+    if let Some(config) = fetch_for_mx(&client, domain).await {
+        return Ok(config);
+    }
+
+    // ── GuessConfig ────────────────────────────────────────────────
+    if let Some(config) = crate::autoconfig::guess::guess_config(domain).await {
+        return Ok(config);
+    }
+
+    Err(raise_error!(
+        format!("No autoconfig found for domain: {domain}"),
+        ErrorCode::InternalError
+    ))
+}
+
+/// DNS MX lookup → retry ISPDB and ISP autoconfig for the MX domain.
+///
+/// Many self-hosted domains have their MX pointed at Google, Microsoft, etc.
+/// The MX domain's ISPDB entry covers the original domain.
+async fn fetch_for_mx(client: &Client, domain: &str) -> Option<MailConfig> {
+    let mx_domain = lookup_mx_domain(domain).await?;
+    if mx_domain == domain.to_ascii_lowercase() {
+        return None; // same domain, already tried above
+    }
+
+    // Try ISPDB for the MX domain
+    if let Some(config) = fetch_xml(
+        client,
+        &format!("https://autoconfig.thunderbird.net/v1.1/{mx_domain}"),
+    )
+    .await
+    {
+        return Some(config);
+    }
+
+    // Try ISP autoconfig for the MX domain (HTTPS then HTTP)
+    if let Some(config) = fetch_xml(
+        client,
+        &format!("https://autoconfig.{mx_domain}/mail/config-v1.1.xml"),
+    )
+    .await
+    {
+        return Some(config);
+    }
+    if let Some(config) = fetch_xml(
+        client,
+        &format!("http://autoconfig.{mx_domain}/mail/config-v1.1.xml"),
+    )
+    .await
+    {
+        return Some(config);
+    }
+
+    None
+}
+
+/// DNS MX lookup → extract the second-level domain of the first MX hostname.
+async fn lookup_mx_domain(domain: &str) -> Option<String> {
+    let resolver = TokioResolver::builder(TokioRuntimeProvider::default())
+        .ok()?
+        .build()
+        .ok()?;
+    let lookup = resolver.mx_lookup(domain).await.ok()?;
+    let record = lookup.answers().first()?;
+    let mx_host = match &record.data {
+        RData::MX(mx) => mx.exchange.to_string().trim_end_matches('.').to_string(),
+        _ => return None,
+    };
+
+    // Extract a reasonable base domain from the MX hostname.
+    // E.g., "aspmx.l.google.com" → "google.com"
+    //       "company.mail.protection.outlook.com" → "outlook.com"
+    extract_base_domain(&mx_host)
+}
+
+/// Extract the top two labels from a hostname as a rough base domain.
+fn extract_base_domain(host: &str) -> Option<String> {
+    let parts: Vec<&str> = host.split('.').collect();
+    if parts.len() >= 2 {
+        Some(parts[parts.len() - 2..].join("."))
+    } else {
+        None
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_fetch_valid_domain() {
+        let domains = vec![
+            // North America
+            ("gmail.com", "Google Gmail"),
+            ("outlook.com", "Microsoft Outlook"),
+            ("hotmail.com", "Microsoft Hotmail"),
+            ("yahoo.com", "Yahoo Mail"),
+            ("icloud.com", "Apple iCloud"),
+            ("aol.com", "AOL Mail"),
+            ("protonmail.com", "ProtonMail"),
+            ("zoho.com", "Zoho Mail"),
+            ("fastmail.com", "FastMail"),
+            // Europe
+            ("gmx.de", "GMX Germany"),
+            ("gmx.net", "GMX International"),
+            ("web.de", "Web.de Germany"),
+            ("freenet.de", "Freenet Germany"),
+            ("mail.ru", "Mail.ru Russia"),
+            ("yandex.ru", "Yandex Russia"),
+            ("orange.fr", "Orange France"),
+            ("laposte.net", "La Poste France"),
+            ("libero.it", "Libero Italy"),
+            ("tiscali.it", "Tiscali Italy"),
+            ("telenet.be", "Telenet Belgium"),
+            // Asia Pacific
+            ("qq.com", "Tencent QQ"),
+            ("163.com", "NetEase 163"),
+            ("126.com", "NetEase 126"),
+            ("sina.com", "Sina Mail"),
+            ("naver.com", "Naver Korea"),
+        ];
+
+        for (domain, label) in &domains {
+            let result = fetch(domain).await;
+            match result {
+                Ok(config) => println!("✅ [{label}] {domain}: {config:#?}"),
+                Err(e) => println!("⚠️  [{label}] {domain}: {e:?}"),
+            }
+        }
+    }
+
+    #[test]
+    fn test_parse_autoconfig_xml() {
+        let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
+<clientConfig version="1.1">
+    <emailProvider id="example.com">
+        <domain>example.com</domain>
+        <incomingServer type="imap">
+            <hostname>imap.example.com</hostname>
+            <port>993</port>
+            <socketType>SSL</socketType>
+            <username>%EMAILADDRESS%</username>
+            <authentication>password-cleartext</authentication>
+        </incomingServer>
+        <outgoingServer type="smtp">
+            <hostname>smtp.example.com</hostname>
+            <port>587</port>
+            <socketType>STARTTLS</socketType>
+            <username>%EMAILADDRESS%</username>
+        </outgoingServer>
+    </emailProvider>
+</clientConfig>"#;
+
+        let config = parse_autoconfig_xml(xml).expect("should parse valid XML");
+        assert_eq!(config.incoming.len(), 1);
+        assert_eq!(config.incoming[0].protocol, "imap");
+        assert_eq!(config.incoming[0].hostname, "imap.example.com");
+        assert_eq!(config.incoming[0].port, 993);
+        assert_eq!(config.incoming[0].socket_type, "SSL");
+        assert_eq!(config.incoming[0].username, "%EMAILADDRESS%");
+        assert_eq!(config.incoming[0].authentication, "password-cleartext");
+
+        assert_eq!(config.outgoing.len(), 1);
+        assert_eq!(config.outgoing[0].protocol, "smtp");
+        assert_eq!(config.outgoing[0].hostname, "smtp.example.com");
+        assert_eq!(config.outgoing[0].port, 587);
+        assert_eq!(config.outgoing[0].socket_type, "STARTTLS");
+        assert_eq!(config.outgoing[0].username, "%EMAILADDRESS%");
+    }
+
+    #[test]
+    fn test_parse_autoconfig_xml_invalid() {
+        assert!(parse_autoconfig_xml("not xml").is_none());
+        assert!(parse_autoconfig_xml("<clientConfig></clientConfig>").is_none());
+    }
+
+    #[test]
+    fn test_extract_base_domain() {
+        assert_eq!(
+            extract_base_domain("aspmx.l.google.com"),
+            Some("google.com".to_string())
+        );
+        assert_eq!(
+            extract_base_domain("company.mail.protection.outlook.com"),
+            Some("outlook.com".to_string())
+        );
+        assert_eq!(
+            extract_base_domain("mx.example.com"),
+            Some("example.com".to_string())
+        );
+        assert_eq!(
+            extract_base_domain("example.com"),
+            Some("example.com".to_string())
+        );
+        assert_eq!(extract_base_domain("localhost"), None);
+    }
+
+    #[tokio::test]
+    async fn test_lookup_srv_gmail() {
+        // Gmail should have SRV records for IMAPS and SMTP submission
+        let config = lookup_srv("gmail.com").await;
+        assert!(config.is_some(), "Gmail should have SRV records");
+        let config = config.unwrap();
+        assert_eq!(config.incoming.len(), 1);
+        assert_eq!(config.incoming[0].protocol, "imap");
+        assert_eq!(config.incoming[0].socket_type, "SSL");
+        assert!(!config.incoming[0].hostname.is_empty());
+        assert!(config.incoming[0].port > 0);
+        assert_eq!(config.outgoing.len(), 1);
+        assert_eq!(config.outgoing[0].protocol, "smtp");
+        assert_eq!(config.outgoing[0].socket_type, "STARTTLS");
+        assert!(!config.outgoing[0].hostname.is_empty());
+        assert!(config.outgoing[0].port > 0);
+    }
+
+    #[tokio::test]
+    async fn test_lookup_srv_nonexistent() {
+        // A domain without SRV records should return None
+        let config = lookup_srv("this-domain-definitely-does-not-exist-12345.com").await;
+        assert!(config.is_none());
+    }
+}

crates/core/src/autoconfig/entity.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -16,14 +16,12 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
-use autoconfig::config::OAuth2Config as XOAuth2Config;
-use poem_openapi::Object;
 use serde::{Deserialize, Serialize};
 
-use crate::modules::account::entity::Encryption;
+use crate::account::entity::Encryption;
 
-#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize, Object)]
+#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct ServerConfig {
     /// server hostname or IP address
     pub host: String,
@@ -43,7 +41,8 @@ impl ServerConfig {
     }
 }
 
-#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize, Object)]
+#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct OAuth2Config {
     /// The authorization server's issuer identifier URL
     pub issuer: String,
@@ -54,19 +53,8 @@ pub struct OAuth2Config {
     /// URL of the authorization server's token endpoint
     pub token_url: String,
 }
-
-impl From<&XOAuth2Config> for OAuth2Config {
-    fn from(value: &XOAuth2Config) -> Self {
-        Self {
-            issuer: value.issuer().into(),
-            scope: value.scope().into_iter().map(Into::into).collect(),
-            auth_url: value.auth_url().into(),
-            token_url: value.token_url().into(),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize, Object)]
+#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct MailServerConfig {
     /// IMAP server configuration
     pub imap: ServerConfig,

crates/core/src/autoconfig/guess.rs

@@ -0,0 +1,105 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::account::entity::Encryption;
+use crate::autoconfig::client::{IncomingServer, MailConfig};
+use crate::imap::client::Client;
+use tracing::{debug, info};
+
+/// A single host:port:encryption combination to probe.
+struct Guess {
+    hostname: String,
+    port: u16,
+    encryption: Encryption,
+    socket_type: &'static str,
+}
+
+/// Generate candidates in the same order Thunderbird uses:
+/// 1. imap.{domain}          — most common
+/// 2. mail.{domain}          — fallback
+/// 3. {domain}               — bare domain (rare)
+fn make_guesses(domain: &str) -> Vec<Guess> {
+    let hosts = [
+        format!("imap.{domain}"),
+        format!("mail.{domain}"),
+        domain.to_string(),
+    ];
+
+    let mut guesses = Vec::with_capacity(hosts.len() * 2);
+    for host in &hosts {
+        guesses.push(Guess {
+            hostname: host.clone(),
+            port: 993,
+            encryption: Encryption::Ssl,
+            socket_type: "SSL",
+        });
+        guesses.push(Guess {
+            hostname: host.clone(),
+            port: 143,
+            encryption: Encryption::StartTls,
+            socket_type: "STARTTLS",
+        });
+    }
+    guesses
+}
+
+/// Try to open a connection, read the IMAP banner, and close.
+/// Returns `true` if the server responds with an IMAP greeting.
+async fn probe(hostname: &str, port: u16, encryption: &Encryption) -> bool {
+    match Client::connection(hostname, encryption, port, None, true).await {
+        Ok(_) => {
+            debug!("GuessConfig probe succeeded: {hostname}:{port} ({encryption:?})");
+            true
+        }
+        Err(e) => {
+            debug!("GuessConfig probe failed for {hostname}:{port}: {e:?}");
+            false
+        }
+    }
+}
+
+/// Thunderbird-style guessing: try common hostnames and ports, probing
+/// each with a real TCP connection.
+///
+/// Returns the first working `MailConfig`, or `None` if nothing works.
+pub async fn guess_config(domain: &str) -> Option<MailConfig> {
+    let guesses = make_guesses(domain);
+    info!("GuessConfig: trying {} candidates for {domain}", guesses.len());
+
+    for g in &guesses {
+        if probe(&g.hostname, g.port, &g.encryption).await {
+            info!(
+                "GuessConfig: found working IMAP at {}:{} ({})",
+                g.hostname, g.port, g.socket_type
+            );
+            return Some(MailConfig {
+                incoming: vec![IncomingServer {
+                    protocol: "imap".to_string(),
+                    hostname: g.hostname.clone(),
+                    port: g.port,
+                    socket_type: g.socket_type.to_string(),
+                    username: "%EMAILADDRESS%".to_string(),
+                    authentication: String::new(),
+                }],
+                outgoing: vec![],
+            });
+        }
+    }
+
+    None
+}

crates/core/src/autoconfig/load.rs

@@ -0,0 +1,116 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::account::entity::Encryption;
+use crate::autoconfig::client::{self, MailConfig};
+use crate::autoconfig::entity::{MailServerConfig, ServerConfig};
+use crate::autoconfig::oauth2_providers::lookup_oauth2;
+use crate::autoconfig::CachedMailSettings;
+use crate::error::code::ErrorCode;
+use crate::error::BichonResult;
+use crate::raise_error;
+use email_address::EmailAddress;
+use std::str::FromStr;
+use tracing::error;
+
+/// Map an autoconfig XML `socketType` value to our `Encryption` enum.
+pub(crate) fn socket_type_to_encryption(raw: &str) -> Encryption {
+    match raw.to_ascii_uppercase().as_str() {
+        "SSL" | "TLS" => Encryption::Ssl,
+        "STARTTLS" => Encryption::StartTls,
+        _ => Encryption::None,
+    }
+}
+
+/// Convert the raw `MailConfig` discovered by `client::fetch` into a
+/// `MailServerConfig` suitable for account provisioning.
+pub(crate) fn mail_config_to_server_config(config: &MailConfig) -> Option<MailServerConfig> {
+    let imap = config.incoming.iter().find(|s| {
+        let p = s.protocol.to_ascii_lowercase();
+        p == "imap" || p == "imaps"
+    })?;
+
+    let encryption = socket_type_to_encryption(&imap.socket_type);
+    let port = if imap.port != 0 {
+        imap.port
+    } else {
+        match encryption {
+            Encryption::Ssl => 993,
+            _ => 143,
+        }
+    };
+
+    // Detect OAuth2 support: the XML <authentication> field and a known
+    // hostname → issuer mapping determine whether the provider supports OAuth2.
+    let oauth2 = if imap.authentication.eq_ignore_ascii_case("OAuth2") {
+        lookup_oauth2(&imap.hostname)
+    } else {
+        None
+    };
+
+    Some(MailServerConfig {
+        imap: ServerConfig::new(imap.hostname.clone(), port, encryption),
+        oauth2,
+    })
+}
+
+pub async fn resolve_autoconfig(email: impl AsRef<str>) -> BichonResult<Option<MailServerConfig>> {
+    let email = email.as_ref();
+    let email_address = EmailAddress::from_str(email).map_err(|error| {
+        raise_error!(
+            format!("Invalid email address: {email:#?}. {error:#?}"),
+            ErrorCode::InvalidParameter
+        )
+    })?;
+
+    let domain = email_address.domain();
+    // Try local cache first
+    if let Some(cached_entity) = CachedMailSettings::get(domain)? {
+        return Ok(Some(cached_entity.config));
+    }
+
+    let config = client::fetch(domain).await.map_err(|e| {
+        error!(
+            email = %email,
+            domain = %domain,
+            error = ?e,
+            "Autoconfig fetch failed"
+        );
+        raise_error!(
+            format!(
+                "Failed to fetch autoconfig for email '{}': {:#?}",
+                email_address.email(),
+                e
+            ),
+            ErrorCode::AutoconfigFetchFailed
+        )
+    })?;
+
+    let result = mail_config_to_server_config(&config).ok_or_else(|| {
+        raise_error!(
+            format!(
+                "No IMAP server found in autoconfig for email: {}",
+                email_address.email()
+            ),
+            ErrorCode::ResourceNotFound
+        )
+    })?;
+
+    CachedMailSettings::add(domain.into(), result.clone())?;
+    Ok(Some(result))
+}

crates/core/src/autoconfig/mod.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -16,65 +16,56 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
-use crate::modules::database::manager::DB_MANAGER;
-use crate::modules::database::{delete_impl, async_find_impl, upsert_impl};
-use crate::modules::error::code::ErrorCode;
-use crate::raise_error;
-use crate::{
-    modules::autoconfig::entity::MailServerConfig, modules::error::BichonResult, utc_now,
-};
-use native_db::*;
-use native_model::{native_model, Model};
+use crate::database::manager::DB_MANAGER;
+use crate::database::{delete_impl, upsert_impl};
+use crate::database::{find_impl, MemDbModel};
+use crate::{autoconfig::entity::MailServerConfig, error::BichonResult, utc_now};
 use serde::{Deserialize, Serialize};
 
+pub mod client;
 pub mod entity;
+pub mod guess;
 pub mod load;
+mod oauth2_providers;
 #[cfg(test)]
 mod tests;
 
 const EXPIRE_TIME_MS: i64 = 30 * 24 * 60 * 60 * 1000;
 
 #[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)]
-#[native_model(id = 3, version = 1)]
-#[native_db]
 pub struct CachedMailSettings {
-    #[primary_key]
     pub domain: String,
     pub config: MailServerConfig,
     pub created_at: i64,
 }
 
+impl MemDbModel for CachedMailSettings {
+    fn collection() -> &'static str {
+        "autoconfig"
+    }
+    fn key(&self) -> String {
+        self.domain.clone()
+    }
+}
+
 impl CachedMailSettings {
-    pub async fn add(domain: String, config: MailServerConfig) -> BichonResult<()> {
+    pub fn add(domain: String, config: MailServerConfig) -> BichonResult<()> {
         Self {
             domain,
             config,
             created_at: utc_now!(),
         }
         .save()
-        .await
     }
 
-    async fn save(&self) -> BichonResult<()> {
-        upsert_impl(DB_MANAGER.meta_db(), self.to_owned()).await
+    fn save(&self) -> BichonResult<()> {
+        upsert_impl(DB_MANAGER.db(), self.to_owned())
     }
 
-    pub async fn get(domain: &str) -> BichonResult<Option<CachedMailSettings>> {
-        if let Some(found) =
-            async_find_impl::<CachedMailSettings>(DB_MANAGER.meta_db(), domain.to_string()).await?
-        {
+    pub fn get(domain: &str) -> BichonResult<Option<CachedMailSettings>> {
+        if let Some(found) = find_impl::<CachedMailSettings>(DB_MANAGER.db(), domain)? {
             if (utc_now!() - found.created_at) > EXPIRE_TIME_MS {
-                let domain = domain.to_string();
-                delete_impl(DB_MANAGER.meta_db(), |rw| {
-                    rw.get()
-                        .primary::<CachedMailSettings>(domain)
-                        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
-                        .ok_or_else(|| {
-                            raise_error!("auto config cache miss".into(), ErrorCode::InternalError)
-                        })
-                })
-                .await?;
+                delete_impl::<CachedMailSettings>(DB_MANAGER.db(), domain)?;
                 Ok(None)
             } else {
                 Ok(Some(found))

crates/core/src/autoconfig/oauth2_providers.rs

@@ -0,0 +1,151 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::autoconfig::entity::OAuth2Config;
+
+/// Per-provider OAuth2 metadata, mirroring Thunderbird's `OAuth2Providers.sys.mjs`.
+///
+/// Each entry maps one or more IMAP hostname suffixes to a well-known OIDC issuer
+/// and the IMAP-specific OAuth2 scopes.
+struct Provider {
+    /// Suffixes matched case-insensitively against the end of the IMAP hostname.
+    host_suffixes: &'static [&'static str],
+    /// The OIDC issuer URL used by the provider.
+    issuer: &'static str,
+    /// OAuth2 scope(s) required for IMAP access.
+    scopes: &'static [&'static str],
+}
+
+const PROVIDERS: &[Provider] = &[
+    // Google
+    Provider {
+        host_suffixes: &["imap.gmail.com", ".gmail.com", ".googlemail.com"],
+        issuer: "https://accounts.google.com",
+        scopes: &["https://mail.google.com/"],
+    },
+    // Microsoft (Outlook / Office 365 / Hotmail / Live)
+    Provider {
+        host_suffixes: &[
+            "outlook.office365.com",
+            ".outlook.com",
+            ".hotmail.com",
+            ".live.com",
+            ".office365.com",
+        ],
+        issuer: "https://login.microsoftonline.com/common/v2.0",
+        scopes: &[
+            "https://outlook.office365.com/IMAP.AccessAsUser.All",
+            "offline_access",
+        ],
+    },
+    // Yahoo / AOL / ATT / Verizon
+    Provider {
+        host_suffixes: &[
+            "imap.mail.yahoo.com",
+            ".yahoo.com",
+            ".yahoodns.net",
+            ".aol.com",
+            "imap.aol.com",
+        ],
+        issuer: "https://login.yahoo.com",
+        scopes: &["mail-w"],
+    },
+    // Yandex
+    Provider {
+        host_suffixes: &["imap.yandex.ru", "imap.yandex.com", ".yandex.ru"],
+        issuer: "https://oauth.yandex.com",
+        scopes: &["imap:all"],
+    },
+    // Mail.ru
+    Provider {
+        host_suffixes: &["imap.mail.ru", ".mail.ru", ".bk.ru", ".list.ru", ".inbox.ru"],
+        issuer: "https://o2.mail.ru",
+        scopes: &["imap"],
+    },
+    // Fastmail
+    Provider {
+        host_suffixes: &["imap.fastmail.com", ".fastmail.com"],
+        issuer: "https://www.fastmail.com",
+        scopes: &[
+            "https://www.fastmail.com/dev/imap",
+            "offline_access",
+        ],
+    },
+    // Comcast
+    Provider {
+        host_suffixes: &["imap.comcast.net", ".comcast.net"],
+        issuer: "https://oauth.xfinity.com",
+        scopes: &["https://email.comcast.net/"],
+    },
+];
+
+/// Try to find an OAuth2 provider that matches the given IMAP hostname.
+///
+/// Matching is case-insensitive and done by suffix: a hostname "imap.gmail.com"
+/// matches the suffix ".gmail.com".
+pub fn lookup_oauth2(hostname: &str) -> Option<OAuth2Config> {
+    let host = hostname.to_ascii_lowercase();
+    for provider in PROVIDERS {
+        if provider
+            .host_suffixes
+            .iter()
+            .any(|suffix| host.ends_with(&suffix.to_ascii_lowercase()))
+        {
+            return Some(OAuth2Config {
+                issuer: provider.issuer.to_string(),
+                scope: provider.scopes.iter().map(|s| s.to_string()).collect(),
+                auth_url: String::new(),
+                token_url: String::new(),
+            });
+        }
+    }
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_known_providers() {
+        let cases = [
+            ("imap.gmail.com", Some("https://accounts.google.com")),
+            ("imap.gmail.com", Some("https://accounts.google.com")),
+            ("outlook.office365.com", Some("https://login.microsoftonline.com/common/v2.0")),
+            ("imap.mail.yahoo.com", Some("https://login.yahoo.com")),
+            ("imap.aol.com", Some("https://login.yahoo.com")),
+            ("imap.yandex.ru", Some("https://oauth.yandex.com")),
+            ("imap.mail.ru", Some("https://o2.mail.ru")),
+            ("imap.fastmail.com", Some("https://www.fastmail.com")),
+            ("imap.comcast.net", Some("https://oauth.xfinity.com")),
+        ];
+        for (hostname, expected_issuer) in &cases {
+            let result = lookup_oauth2(hostname);
+            assert_eq!(
+                result.map(|c| c.issuer),
+                expected_issuer.map(|s| s.to_string()),
+                "failed for hostname: {hostname}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_unknown_provider() {
+        assert!(lookup_oauth2("mail.my-company.example").is_none());
+    }
+}

crates/core/src/autoconfig/tests.rs

@@ -0,0 +1,368 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::account::entity::Encryption;
+use crate::autoconfig::client::{self, IncomingServer, MailConfig};
+use crate::autoconfig::load::{mail_config_to_server_config, socket_type_to_encryption};
+
+// ---------------------------------------------------------------------------
+// XML parsing tests
+// ---------------------------------------------------------------------------
+
+fn make_valid_xml() -> String {
+    r#"<?xml version="1.0" encoding="UTF-8"?>
+<clientConfig version="1.1">
+  <emailProvider id="example.com">
+    <domain>example.com</domain>
+    <displayName>Example Mail</displayName>
+    <incomingServer type="imap">
+      <hostname>imap.example.com</hostname>
+      <port>993</port>
+      <socketType>SSL</socketType>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <outgoingServer type="smtp">
+      <hostname>smtp.example.com</hostname>
+      <port>587</port>
+      <socketType>STARTTLS</socketType>
+      <username>%EMAILADDRESS%</username>
+    </outgoingServer>
+  </emailProvider>
+</clientConfig>"#
+        .to_string()
+}
+
+#[test]
+fn parse_valid_xml() {
+    let xml = make_valid_xml();
+    let config = client::parse_autoconfig_xml(&xml).expect("should parse valid XML");
+
+    assert_eq!(config.incoming.len(), 1);
+    let imap = &config.incoming[0];
+    assert_eq!(imap.protocol, "imap");
+    assert_eq!(imap.hostname, "imap.example.com");
+    assert_eq!(imap.port, 993);
+    assert_eq!(imap.socket_type, "SSL");
+    assert_eq!(imap.username, "%EMAILADDRESS%");
+
+    assert_eq!(config.outgoing.len(), 1);
+    let smtp = &config.outgoing[0];
+    assert_eq!(smtp.protocol, "smtp");
+    assert_eq!(smtp.hostname, "smtp.example.com");
+    assert_eq!(smtp.port, 587);
+    assert_eq!(smtp.socket_type, "STARTTLS");
+}
+
+#[test]
+fn parse_xml_empty_body() {
+    let xml = r#"<?xml version="1.0"?><clientConfig></clientConfig>"#;
+    let config = client::parse_autoconfig_xml(xml);
+    assert!(config.is_none(), "no emailProvider → None");
+}
+
+#[test]
+fn parse_xml_no_incoming_servers() {
+    let xml = r#"<?xml version="1.0"?>
+<clientConfig version="1.1">
+  <emailProvider id="example.com">
+    <domain>example.com</domain>
+  </emailProvider>
+</clientConfig>"#;
+    let config = client::parse_autoconfig_xml(xml).expect("should parse");
+    assert!(config.incoming.is_empty());
+    assert!(config.outgoing.is_empty());
+}
+
+#[test]
+fn parse_xml_garbage() {
+    let config = client::parse_autoconfig_xml("not xml at all");
+    assert!(config.is_none());
+}
+
+#[test]
+fn parse_xml_missing_port_defaults_to_zero() {
+    let xml = r#"<?xml version="1.0"?>
+<clientConfig version="1.1">
+  <emailProvider id="example.com">
+    <incomingServer type="imap">
+      <hostname>imap.example.com</hostname>
+      <socketType>SSL</socketType>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+  </emailProvider>
+</clientConfig>"#;
+    let config = client::parse_autoconfig_xml(xml).expect("should parse");
+    assert_eq!(config.incoming[0].port, 0);
+}
+
+#[test]
+fn parse_xml_multiple_providers_picks_first() {
+    let xml = r#"<?xml version="1.0"?>
+<clientConfig version="1.1">
+  <emailProvider id="first.example.com">
+    <incomingServer type="imap">
+      <hostname>imap.first.example.com</hostname>
+      <port>993</port>
+      <socketType>SSL</socketType>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+  </emailProvider>
+  <emailProvider id="second.example.com">
+    <incomingServer type="imap">
+      <hostname>imap.second.example.com</hostname>
+      <port>143</port>
+      <socketType>STARTTLS</socketType>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+  </emailProvider>
+</clientConfig>"#;
+    let config = client::parse_autoconfig_xml(xml).expect("should parse");
+    assert_eq!(config.incoming[0].hostname, "imap.first.example.com");
+}
+
+// ---------------------------------------------------------------------------
+// socket_type → Encryption mapping tests
+// ---------------------------------------------------------------------------
+
+#[test]
+fn encryption_ssl_uppercase() {
+    assert_eq!(socket_type_to_encryption("SSL"), Encryption::Ssl);
+}
+
+#[test]
+fn encryption_ssl_lowercase() {
+    assert_eq!(socket_type_to_encryption("ssl"), Encryption::Ssl);
+}
+
+#[test]
+fn encryption_tls() {
+    assert_eq!(socket_type_to_encryption("TLS"), Encryption::Ssl);
+}
+
+#[test]
+fn encryption_starttls() {
+    assert_eq!(socket_type_to_encryption("STARTTLS"), Encryption::StartTls);
+}
+
+#[test]
+fn encryption_starttls_lowercase() {
+    assert_eq!(socket_type_to_encryption("starttls"), Encryption::StartTls);
+}
+
+#[test]
+fn encryption_starttls_mixed_case() {
+    assert_eq!(socket_type_to_encryption("StartTls"), Encryption::StartTls);
+}
+
+#[test]
+fn encryption_plain() {
+    assert_eq!(socket_type_to_encryption("plain"), Encryption::None);
+}
+
+#[test]
+fn encryption_empty_string() {
+    assert_eq!(socket_type_to_encryption(""), Encryption::None);
+}
+
+#[test]
+fn encryption_unknown_value() {
+    assert_eq!(socket_type_to_encryption("WPA2-ENTERPRISE"), Encryption::None);
+}
+
+// ---------------------------------------------------------------------------
+// MailConfig → MailServerConfig conversion tests
+// ---------------------------------------------------------------------------
+
+fn make_imap_server(host: &str, port: u16, socket_type: &str) -> IncomingServer {
+    IncomingServer {
+        protocol: "imap".to_string(),
+        hostname: host.to_string(),
+        port,
+        socket_type: socket_type.to_string(),
+        username: "%EMAILADDRESS%".to_string(),
+        authentication: String::new(),
+    }
+}
+
+#[test]
+fn convert_basic_imap_ssl() {
+    let config = MailConfig {
+        incoming: vec![make_imap_server("imap.example.com", 993, "SSL")],
+        outgoing: vec![],
+    };
+    let result = mail_config_to_server_config(&config).expect("should convert");
+    assert_eq!(result.imap.host, "imap.example.com");
+    assert_eq!(result.imap.port, 993);
+    assert_eq!(result.imap.encryption, Encryption::Ssl);
+    assert!(result.oauth2.is_none());
+}
+
+#[test]
+fn convert_imap_starttls_with_default_port() {
+    let config = MailConfig {
+        incoming: vec![make_imap_server("imap.example.com", 0, "STARTTLS")],
+        outgoing: vec![],
+    };
+    let result = mail_config_to_server_config(&config).expect("should convert");
+    assert_eq!(result.imap.port, 143, "default port for STARTTLS → 143");
+    assert_eq!(result.imap.encryption, Encryption::StartTls);
+}
+
+#[test]
+fn convert_imap_ssl_with_default_port() {
+    let config = MailConfig {
+        incoming: vec![make_imap_server("imap.example.com", 0, "SSL")],
+        outgoing: vec![],
+    };
+    let result = mail_config_to_server_config(&config).expect("should convert");
+    assert_eq!(result.imap.port, 993, "default port for SSL → 993");
+}
+
+#[test]
+fn convert_no_imap_only_pop3() {
+    let config = MailConfig {
+        incoming: vec![IncomingServer {
+            protocol: "pop3".to_string(),
+            hostname: "pop.example.com".to_string(),
+            port: 995,
+            socket_type: "SSL".to_string(),
+            username: "%EMAILADDRESS%".to_string(),
+            authentication: String::new(),
+        }],
+        outgoing: vec![],
+    };
+    assert!(mail_config_to_server_config(&config).is_none());
+}
+
+#[test]
+fn convert_empty_incoming() {
+    let config = MailConfig {
+        incoming: vec![],
+        outgoing: vec![],
+    };
+    assert!(mail_config_to_server_config(&config).is_none());
+}
+
+#[test]
+fn convert_picks_imap_over_pop3() {
+    let config = MailConfig {
+        incoming: vec![
+            IncomingServer {
+                protocol: "pop3".to_string(),
+                hostname: "pop.example.com".to_string(),
+                port: 995,
+                socket_type: "SSL".to_string(),
+                username: "%EMAILADDRESS%".to_string(),
+                authentication: String::new(),
+            },
+            make_imap_server("imap.example.com", 993, "SSL"),
+        ],
+        outgoing: vec![],
+    };
+    let result = mail_config_to_server_config(&config).expect("should find IMAP");
+    assert_eq!(result.imap.host, "imap.example.com");
+}
+
+#[test]
+fn convert_imaps_protocol_variant() {
+    let config = MailConfig {
+        incoming: vec![IncomingServer {
+            protocol: "imaps".to_string(),
+            hostname: "imap.example.com".to_string(),
+            port: 993,
+            socket_type: "SSL".to_string(),
+            username: "%EMAILADDRESS%".to_string(),
+            authentication: String::new(),
+        }],
+        outgoing: vec![],
+    };
+    let result = mail_config_to_server_config(&config).expect("should recognize 'imaps'");
+    assert_eq!(result.imap.host, "imap.example.com");
+}
+
+#[test]
+fn convert_case_insensitive_protocol() {
+    let config = MailConfig {
+        incoming: vec![IncomingServer {
+            protocol: "IMAP".to_string(),
+            hostname: "imap.example.com".to_string(),
+            port: 143,
+            socket_type: "STARTTLS".to_string(),
+            username: "%EMAILADDRESS%".to_string(),
+            authentication: String::new(),
+        }],
+        outgoing: vec![],
+    };
+    let result = mail_config_to_server_config(&config).expect("should recognize 'IMAP'");
+    assert_eq!(result.imap.host, "imap.example.com");
+}
+
+#[test]
+fn convert_gmail_oauth2() {
+    let config = MailConfig {
+        incoming: vec![IncomingServer {
+            protocol: "imap".to_string(),
+            hostname: "imap.gmail.com".to_string(),
+            port: 993,
+            socket_type: "SSL".to_string(),
+            username: "%EMAILADDRESS%".to_string(),
+            authentication: "OAuth2".to_string(),
+        }],
+        outgoing: vec![],
+    };
+    let result = mail_config_to_server_config(&config).expect("should convert");
+    let oauth2 = result.oauth2.expect("Gmail should have OAuth2");
+    assert_eq!(oauth2.issuer, "https://accounts.google.com");
+    assert!(oauth2.scope.contains(&"https://mail.google.com/".to_string()));
+}
+
+#[test]
+fn convert_outlook_oauth2() {
+    let config = MailConfig {
+        incoming: vec![IncomingServer {
+            protocol: "imap".to_string(),
+            hostname: "outlook.office365.com".to_string(),
+            port: 993,
+            socket_type: "SSL".to_string(),
+            username: "%EMAILADDRESS%".to_string(),
+            authentication: "OAuth2".to_string(),
+        }],
+        outgoing: vec![],
+    };
+    let result = mail_config_to_server_config(&config).expect("should convert");
+    let oauth2 = result.oauth2.expect("Outlook should have OAuth2");
+    assert!(oauth2.issuer.contains("microsoftonline"));
+}
+
+#[test]
+fn convert_unknown_host_no_oauth2() {
+    // OAuth2 auth flag on an unknown hostname → no OAuth2 returned
+    let config = MailConfig {
+        incoming: vec![IncomingServer {
+            protocol: "imap".to_string(),
+            hostname: "mail.random-isp.example".to_string(),
+            port: 993,
+            socket_type: "SSL".to_string(),
+            username: "%EMAILADDRESS%".to_string(),
+            authentication: "OAuth2".to_string(),
+        }],
+        outgoing: vec![],
+    };
+    let result = mail_config_to_server_config(&config).expect("should convert");
+    assert!(result.oauth2.is_none(), "unknown hostname → no OAuth2 mapping");
+}

crates/core/src/cache/imap/download/download_folders.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -16,27 +16,28 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
 use std::collections::BTreeSet;
 
 use crate::{
-    decode_mailbox_name,
-    modules::{
+    decode_mailbox_name, raise_error,
+    {
         account::migration::{AccountModel, AccountType},
         cache::imap::mailbox::{AttributeEnum, MailBox},
-        context::executors::MAIL_CONTEXT,
+        cache::imap::mailbox_cache,
         error::{code::ErrorCode, BichonResult},
+        imap::{executor::ImapExecutor, session::SessionStream},
         mailbox::list::convert_names_to_mailboxes,
     },
-    raise_error,
 };
-use async_imap::types::Name;
+use async_imap::{types::Name, Session};
 use tracing::{debug, info, warn};
 
-pub async fn get_sync_folders(account: &AccountModel) -> BichonResult<Vec<MailBox>> {
+pub async fn get_download_folders(
+    account: &AccountModel,
+    session: &mut Session<Box<dyn SessionStream>>,
+) -> BichonResult<Vec<MailBox>> {
     assert_eq!(account.account_type, AccountType::IMAP);
-    let executor = MAIL_CONTEXT.imap(account.id).await?;
-    let names = executor.list_all_mailboxes().await?;
+    let names = ImapExecutor::list_all_mailboxes(session).await?;
     if names.is_empty() {
         warn!(
             "Account {}: No mailboxes returned from IMAP server.",
@@ -61,8 +62,8 @@ pub async fn get_sync_folders(account: &AccountModel) -> BichonResult<Vec<MailBo
         mailboxes.iter().map(|(m, _)| m.name.clone()).collect(),
     )
     .await?;
-    let account = AccountModel::get(account.id).await?;
-    let subscribed = &account.sync_folders.unwrap_or_default();
+    let account = AccountModel::get(account.id)?;
+    let subscribed = &account.download_folders.unwrap_or_default();
     let is_noselect = |mailbox: &MailBox| {
         mailbox
             .attributes
@@ -109,7 +110,7 @@ pub async fn get_sync_folders(account: &AccountModel) -> BichonResult<Vec<MailBo
                 .iter()
                 .map(|n| decode_mailbox_name!(n.name().to_string()))
                 .collect();
-            AccountModel::update_sync_folders(account.id, sync_folders).await?;
+            AccountModel::update_download_folders(account.id, sync_folders)?;
         } else {
             warn!(
                 "Account {}: No subscribed mailboxes found. This is unexpected — IMAP server should at least provide INBOX.",
@@ -121,7 +122,7 @@ pub async fn get_sync_folders(account: &AccountModel) -> BichonResult<Vec<MailBo
             ), ErrorCode::ImapUnexpectedResult));
         }
     }
-    convert_names_to_mailboxes(account.id, matched_mailboxes).await
+    convert_names_to_mailboxes(account.id, session, matched_mailboxes).await
 }
 
 pub async fn detect_mailbox_changes(
@@ -130,7 +131,7 @@ pub async fn detect_mailbox_changes(
 ) -> BichonResult<()> {
     if account.known_folders.is_none() {
         // First time sync: just save without comparing
-        AccountModel::update_known_folders(account.id, all_names).await?;
+        AccountModel::update_known_folders(account.id, all_names)?;
         return Ok(());
     }
     let known_folders = account.known_folders.clone().unwrap_or_default();
@@ -139,19 +140,19 @@ pub async fn detect_mailbox_changes(
     let deleted_folders: Vec<String> = known_folders.difference(&all_names).cloned().collect();
 
     let has_changes = !new_folders.is_empty() || !deleted_folders.is_empty();
-    let sync_folders = account.sync_folders.as_deref().unwrap_or_default();
+    let download_folders = account.download_folders.as_deref().unwrap_or_default();
     // Handle deleted folders in sync_folders
     if !deleted_folders.is_empty() {
         // Check if any deleted folders are in sync_folders
-        let remaining_sync_folders: Vec<String> = sync_folders
+        let remaining_sync_folders: Vec<String> = download_folders
             .iter()
             .filter(|folder| !deleted_folders.contains(folder))
             .cloned()
             .collect();
 
         // If sync_folders changed, update them
-        if remaining_sync_folders.len() != sync_folders.len() {
-            let removed_count = sync_folders.len() - remaining_sync_folders.len();
+        if remaining_sync_folders.len() != download_folders.len() {
+            let removed_count = download_folders.len() - remaining_sync_folders.len();
             info!(
                 "Account {}: Removed {} deleted folders from sync_folders",
                 account.id, removed_count
@@ -159,7 +160,7 @@ pub async fn detect_mailbox_changes(
             // Note: When all subscribed folders are deleted (remaining_sync_folders empty),
             // the system's default behavior is to automatically fall back to syncing
             // only the default folders (INBOX and Sent) in subsequent operations
-            AccountModel::update_sync_folders(account.id, remaining_sync_folders).await?;
+            AccountModel::update_download_folders(account.id, remaining_sync_folders)?;
         }
 
         info!(
@@ -174,11 +175,22 @@ pub async fn detect_mailbox_changes(
             "Account {}: New folders detected: {:?}",
             account.id, new_folders
         );
+        if account.auto_download_new_mailboxes.unwrap_or(false) {
+            let mut updated: Vec<String> = download_folders.to_vec();
+            updated.extend(new_folders.iter().cloned());
+            AccountModel::update_download_folders(account.id, updated)?;
+            info!(
+                "Account {}: Auto-added {} new folders to download list",
+                account.id,
+                new_folders.len()
+            );
+        }
     }
 
     // Update known folders only if there were changes
     if has_changes {
-        AccountModel::update_known_folders(account.id, all_names).await?;
+        AccountModel::update_known_folders(account.id, all_names)?;
+        mailbox_cache::invalidate(account.id).await;
     }
     Ok(())
 }

crates/core/src/cache/imap/download/download_type.rs

@@ -0,0 +1,161 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::str::FromStr;
+
+use chrono::{DateTime, Local, TimeZone, Utc};
+use cron::Schedule;
+
+use crate::{
+    utc_now,
+    {
+        account::{
+            migration::AccountModel,
+            state::{DownloadState, TriggerType},
+        },
+        error::BichonResult,
+    },
+};
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum DownloadTask {
+    FullFetch,
+    TraceFetch,
+    Idle,
+}
+
+pub async fn decide_next_download_task(
+    account: &AccountModel,
+    trigger_type: TriggerType,
+) -> BichonResult<DownloadTask> {
+    let state = match DownloadState::get(account.id)? {
+        None => {
+            DownloadState::init(account.id).await?;
+            return Ok(DownloadTask::FullFetch);
+        }
+        Some(s) => s,
+    };
+
+    let should_start = match trigger_type {
+        TriggerType::Manual => true,
+        TriggerType::SyncFull => true,
+        TriggerType::Scheduled => {
+            let now = utc_now!();
+            let cooldown_ok = now - state.last_finished_at.unwrap_or(0) > 60 * 1000;
+            if !cooldown_ok {
+                false
+            } else if let Some(ref schedule) = account.download_schedule {
+                should_trigger_scheduled(schedule, state.last_trigger_at)
+            } else {
+                should_trigger_next_download(
+                    state.last_trigger_at,
+                    account.download_interval_min.unwrap_or(60),
+                )
+            }
+        }
+    };
+
+    if should_start {
+        DownloadState::start_new_session(account.id, trigger_type)?;
+        Ok(DownloadTask::TraceFetch)
+    } else {
+        // Nothing to download right now. A Running active_session at this point
+        // is a leftover from an interrupted run (a real download would have
+        // been blocked by the busy guard before reaching here), so mark it
+        // Cancelled instead of leaving the UI showing a phantom "syncing".
+        DownloadState::finalize_stale_session(account.id)?;
+        Ok(DownloadTask::Idle)
+    }
+}
+fn should_trigger_next_download(last_trigger_at: i64, sync_interval_min: i64) -> bool {
+    let now = utc_now!();
+    now - last_trigger_at > (sync_interval_min * 60 * 1000)
+}
+
+fn should_trigger_scheduled(schedule_str: &str, last_trigger_at: i64) -> bool {
+    let schedule = match Schedule::from_str(schedule_str) {
+        Ok(s) => s,
+        Err(e) => {
+            tracing::warn!(
+                "Invalid cron expression '{}', falling back to no trigger: {}",
+                schedule_str,
+                e
+            );
+            return false;
+        }
+    };
+    // last_trigger_at is a UTC millis timestamp; convert to server local time
+    let last_utc = match Utc.timestamp_millis_opt(last_trigger_at) {
+        chrono::LocalResult::Single(dt) => dt,
+        _ => {
+            tracing::warn!("Invalid last_trigger_at timestamp: {}", last_trigger_at);
+            return false;
+        }
+    };
+    let last_dt: DateTime<Local> = last_utc.with_timezone(&Local);
+    let now = Local::now();
+    schedule
+        .after(&last_dt)
+        .next()
+        .map_or(false, |next| next <= now)
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn cron_every_minute_triggers_after_60s() {
+        // "0 * * * * *" = every minute at second 0. last_trigger 90s ago → should trigger
+        let now = Local::now();
+        let last_trigger = now.timestamp_millis() - 90_000;
+        assert!(should_trigger_scheduled("0 * * * * *", last_trigger));
+    }
+
+    #[test]
+    fn cron_daily_midnight_triggers_when_missed() {
+        // "0 0 0 * * *" = daily at midnight
+        // last_trigger was 25 hours ago → should trigger (we missed midnight)
+        let now = Local::now();
+        let last_trigger = now.timestamp_millis() - 25 * 60 * 60 * 1000;
+        assert!(should_trigger_scheduled("0 0 0 * * *", last_trigger));
+    }
+
+    #[test]
+    fn cron_daily_midnight_no_trigger_if_already_fired() {
+        // "0 0 0 * * *" = daily at midnight
+        // last_trigger was 1 minute ago → should NOT trigger
+        let now = Local::now();
+        let last_trigger = now.timestamp_millis() - 60_000;
+        assert!(!should_trigger_scheduled("0 0 0 * * *", last_trigger));
+    }
+
+    #[test]
+    fn invalid_cron_returns_false() {
+        assert!(!should_trigger_scheduled("invalid cron expression", 0));
+    }
+
+    #[test]
+    fn cron_every_hour_triggers() {
+        // "0 0 * * * *" = every hour at minute 0, second 0
+        // last_trigger was 61 minutes ago → should trigger
+        let now = Local::now();
+        let last_trigger = now.timestamp_millis() - 61 * 60 * 1000;
+        assert!(should_trigger_scheduled("0 0 * * * *", last_trigger));
+    }
+}

crates/core/src/cache/imap/download/gap_fill.rs

@@ -0,0 +1,472 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::{HashMap, HashSet};
+
+use tracing::{info, warn};
+
+use crate::error::code::ErrorCode;
+use crate::raise_error;
+use crate::{
+    account::{
+        migration::AccountModel,
+        state::{DownloadState, GapFillFolderStats, GapFillState},
+    },
+    cache::imap::mailbox::MailBox,
+    error::BichonResult,
+    imap::executor::{compress_uid_list, ImapExecutor, DEFAULT_BATCH_SIZE},
+    store::tantivy::envelope::EnvelopeSnapshot,
+};
+
+/// Number of times a batch download is retried with a fresh connection before
+/// it is counted as failed (mirrors the incremental sync path).
+const MAX_NETWORK_RETRIES: u32 = 3;
+
+/// Lightweight header metadata for one remote message, fetched via
+/// `FETCH (UID RFC822.SIZE INTERNALDATE BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])`.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct RemoteHeader {
+    pub uid: u32,
+    pub message_id: Option<String>,
+    pub size: u64,
+    /// Epoch millis (internal date).
+    pub internal_date: i64,
+}
+
+/// Which remote uids are missing locally. A remote message is "present" if its
+/// message-id exists locally. The (size, internal_date) fingerprint is a
+/// fallback for every remote message — not only those without a message-id —
+/// because some paths store a different message-id locally than the remote
+/// header carries (e.g. the SMTP path generates a random one), and servers
+/// like Zoho/163 reuse the same message-id across different messages. A
+/// duplicated local message-id still counts as present: re-downloading would
+/// be deduplicated away anyway, so it can never repair the duplication.
+pub fn compute_missing_uids(remote: &[RemoteHeader], local: &[EnvelopeSnapshot]) -> Vec<u32> {
+    let mut local_by_msg_id: HashMap<&str, usize> = HashMap::new();
+    let mut local_by_fingerprint: HashSet<(u64, i64)> = HashSet::new();
+    for snap in local {
+        if !snap.message_id.is_empty() {
+            *local_by_msg_id.entry(snap.message_id.as_str()).or_insert(0) += 1;
+        }
+        local_by_fingerprint.insert((snap.size, snap.internal_date));
+    }
+
+    let mut missing = Vec::new();
+    for header in remote {
+        let present = match &header.message_id {
+            Some(msg_id) => local_by_msg_id
+                .get(msg_id.as_str())
+                .is_some_and(|&c| c > 0),
+            None => false,
+        };
+        if !present {
+            // Fingerprint fallback for every remote message, not just those
+            // without a message-id: the remote message-id may not exist
+            // locally even though the message is already stored (random
+            // synthetic ids on the SMTP path).
+            let fp_present = local_by_fingerprint.contains(&(header.size, header.internal_date));
+            if !fp_present {
+                missing.push(header.uid);
+            }
+        }
+    }
+    missing
+}
+
+/// Runs the gap-fill phase for one mailbox: enumerates every remote UID,
+/// diffs against local envelopes, downloads the missing ones and returns the
+/// per-folder outcome. Handles per-batch network errors by counting them as
+/// failed (retryable on a later gap-fill run) instead of aborting the phase.
+pub async fn gap_fill_mailbox(
+    account: &AccountModel,
+    local_mailbox: &MailBox,
+    remote_mailbox: &MailBox,
+    token: tokio_util::sync::CancellationToken,
+) -> BichonResult<GapFillFolderStats> {
+    let account_id = account.id;
+    let mut stats = GapFillFolderStats::default();
+
+    let mut session = ImapExecutor::create_connection(account_id).await?;
+    session
+        .examine(&remote_mailbox.encoded_name())
+        .await
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+    // Phase 1: enumerate every remote UID. A huge mailbox can make the server
+    // take a while to answer `UID SEARCH ALL`; keep the UI informed instead of
+    // appearing stuck (the socket read timeout is the final backstop).
+    let search_started = std::time::Instant::now();
+    let results = loop {
+        match tokio::time::timeout(std::time::Duration::from_secs(5), session.uid_search("ALL"))
+            .await
+        {
+            Ok(res) => {
+                break res
+                    .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
+            }
+            Err(_) => {
+                let stall = search_started.elapsed().as_secs_f64();
+                if token.is_cancelled() {
+                    session.logout().await.ok();
+                    return Ok(stats);
+                }
+                let _ = GapFillState::update_folder_progress(
+                    account_id,
+                    remote_mailbox.name.clone(),
+                    0,
+                    0,
+                    crate::imap::executor::slow_server_message(None, Some(stall)),
+                );
+                tracing::warn!(
+                    account_id,
+                    mailbox = %remote_mailbox.name,
+                    stall_secs = format!("{:.0}", stall),
+                    "gap-fill: UID SEARCH ALL taking long, still waiting"
+                );
+            }
+        }
+    };
+    let mut remote_uids: Vec<u32> = results.into_iter().collect();
+    remote_uids.sort();
+    if remote_uids.is_empty() {
+        session.logout().await.ok();
+        return Ok(stats);
+    }
+
+    // Phase 2: fetch header metadata for all remote uids in batches.
+    // A failed batch counts its uids as failed (they cannot be diffed) but
+    // does not abort the phase. A cancellation, however, leaves the header
+    // list incomplete so the diff would be unreliable — return immediately.
+    let mut remote_headers: Vec<RemoteHeader> = Vec::with_capacity(remote_uids.len());
+    let batch_size = account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize;
+    let mut cancelled = false;
+    // Progress reported so far across all header batches, so the UI can show
+    // enumeration progress (and slow-server hints) while a huge mailbox is
+    // being scanned.
+    let headers_fetched = std::sync::Mutex::new(0u64);
+    for chunk in remote_uids.chunks(batch_size) {
+        if token.is_cancelled() {
+            cancelled = true;
+            break;
+        }
+        let seq_set = compress_uid_list(chunk.to_vec());
+        match ImapExecutor::fetch_uid_headers(
+            &mut session,
+            &seq_set,
+            token.clone(),
+            Some(&|count, stall_secs| {
+                *headers_fetched.lock().unwrap() = count;
+                GapFillState::update_folder_progress(
+                    account_id,
+                    remote_mailbox.name.clone(),
+                    remote_uids.len() as u64,
+                    count,
+                    crate::imap::executor::slow_server_message(None, stall_secs),
+                )
+            }),
+        )
+        .await
+        {
+            Ok(headers) => {
+                *headers_fetched.lock().unwrap() += headers.len() as u64;
+                remote_headers.extend(headers);
+            }
+            Err(e) => {
+                // Count the whole chunk as failed; the user can re-run
+                // gap-fill to retry. Do not abort the phase.
+                stats.failed += chunk.len() as u64;
+                let err_msg = format!("Gap-fill header batch failed: {:#?}", e);
+                warn!(account_id, mailbox = remote_mailbox.name, "{}", err_msg);
+                let _ = DownloadState::append_session_error(account_id, err_msg);
+            }
+        }
+    }
+    if cancelled {
+        session.logout().await.ok();
+        GapFillState::update_folder_progress(account_id, remote_mailbox.name.clone(), 0, 0, None)?;
+        return Ok(stats);
+    }
+    remote_headers.sort_by_key(|h| h.uid);
+    session.logout().await.ok();
+
+    // Phase 3: local snapshot
+    let local_snapshots = crate::store::tantivy::envelope::ENVELOPE_MANAGER
+        .get_envelope_snapshots_for_mailbox(account_id, local_mailbox.id)?;
+
+    // Phase 4: diff
+    let missing_uids = compute_missing_uids(&remote_headers, &local_snapshots);
+    stats.candidate_count = missing_uids.len() as u64;
+    if missing_uids.is_empty() {
+        info!(
+            account_id,
+            mailbox = remote_mailbox.name,
+            "Gap-fill: no missing emails"
+        );
+        GapFillState::update_folder_progress(account_id, remote_mailbox.name.clone(), 0, 0, None)?;
+        return Ok(stats);
+    }
+
+    // Phase 5: download missing in batches
+    let planned = missing_uids.len() as u64;
+    GapFillState::update_folder_progress(
+        account_id,
+        remote_mailbox.name.clone(),
+        planned,
+        0,
+        None,
+    )?;
+
+    let mut session2 = ImapExecutor::create_connection(account_id).await?;
+    session2
+        .examine(&remote_mailbox.encoded_name())
+        .await
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+    let batches =
+        crate::imap::executor::generate_uid_sequence_hashset(missing_uids.clone(), batch_size);
+    let mut downloaded = 0u64;
+    let mut failed = 0u64;
+    let mut cancelled = false;
+    for (index, batch) in batches.into_iter().enumerate() {
+        if token.is_cancelled() {
+            cancelled = true;
+            break;
+        }
+        // A slow server can stall a batch past the socket read timeout, same
+        // as in the incremental path. Retry such batches on a fresh connection
+        // instead of counting them as failed outright.
+        let mut retries = 0u32;
+        // Tracks the last cumulative count the progress callback reported. When
+        // a batch fails mid-stream the executor reports the already-stored
+        // count one final time before returning the error, so this is the
+        // number of emails of this batch that actually made it to disk.
+        let last_reported = std::sync::Mutex::new(0u64);
+        let batch_result = loop {
+            match ImapExecutor::uid_batch_retrieve_emails(
+                &mut session2,
+                account_id,
+                remote_mailbox.id,
+                &batch.0,
+                account.max_email_size_bytes,
+                token.clone(),
+                Some(&|cumulative, avg_secs, stall_secs| {
+                    *last_reported.lock().unwrap() = cumulative;
+                    let _ = GapFillState::update_folder_progress(
+                        account_id,
+                        remote_mailbox.name.clone(),
+                        planned,
+                        downloaded + cumulative,
+                        crate::imap::executor::slow_server_message(avg_secs, stall_secs),
+                    );
+                    Ok(())
+                }),
+            )
+            .await
+            {
+                Ok((processed, _throttled)) => break Ok(processed),
+                Err(e) if retries < MAX_NETWORK_RETRIES && e.code() == ErrorCode::NetworkError => {
+                    retries += 1;
+                    warn!(
+                        account_id,
+                        mailbox = remote_mailbox.name,
+                        index,
+                        retries,
+                        "Gap-fill: network error on batch, reconnecting ({}/{})",
+                        retries,
+                        MAX_NETWORK_RETRIES
+                    );
+                    match ImapExecutor::create_connection(account_id).await {
+                        Ok(new_session) => {
+                            session2 = new_session;
+                            if let Err(e2) = session2.examine(&remote_mailbox.encoded_name()).await
+                            {
+                                let err_msg = format!(
+                                    "Gap-fill: re-examine failed after reconnect: {:#?}",
+                                    e2
+                                );
+                                DownloadState::append_session_error(account_id, err_msg)?;
+                                break Err(e);
+                            }
+                            // Longer backoff than the original 1s/2s/4s: a
+                            // throttling server needs time to recover.
+                            let backoff = [5u64, 15, 30][(retries - 1) as usize];
+                            warn!(
+                                account_id,
+                                mailbox = remote_mailbox.name,
+                                "Gap-fill: backing off {}s before retrying batch",
+                                backoff
+                            );
+                            tokio::time::sleep(std::time::Duration::from_secs(backoff)).await;
+                            continue;
+                        }
+                        Err(e2) => {
+                            tracing::error!(account_id, "Gap-fill: reconnection failed: {:#?}", e2);
+                            break Err(e);
+                        }
+                    }
+                }
+                Err(e) => break Err(e),
+            }
+        };
+        match batch_result {
+            Ok(processed) => {
+                downloaded += processed;
+                GapFillState::update_folder_progress(
+                    account_id,
+                    remote_mailbox.name.clone(),
+                    planned,
+                    downloaded,
+                    None,
+                )?;
+            }
+            Err(e) => {
+                // The batch may have partially succeeded: emails already stored
+                // before the failure are counted as downloaded, only the rest
+                // of the batch is failed. The user can re-run gap-fill to
+                // retry the remainder; dedup makes re-downloading the stored
+                // ones harmless. Do not abort the phase.
+                let processed = *last_reported.lock().unwrap();
+                downloaded += processed;
+                let remaining = batch.1.saturating_sub(processed);
+                failed += remaining;
+                let err_msg = format!(
+                    "Gap-fill batch {} failed after {} processed: {:#?}",
+                    index, processed, e
+                );
+                warn!(account_id, mailbox = remote_mailbox.name, "{}", err_msg);
+                let _ = DownloadState::append_session_error(account_id, err_msg);
+            }
+        }
+    }
+    session2.logout().await.ok();
+
+    stats.downloaded = downloaded;
+    // Accumulate rather than overwrite: phase 2 (header batch) failures already
+    // counted into stats.failed and must survive alongside phase 5 failures.
+    stats.failed += failed;
+    // Final progress write into the independent gap-fill state (the folder is
+    // done; the outcome lands in GapFillRun.folders via add_folder_result).
+    GapFillState::update_folder_progress(
+        account_id,
+        remote_mailbox.name.clone(),
+        planned,
+        downloaded,
+        None,
+    )?;
+
+    // Advance the mailbox's highest_uid so subsequent incremental syncs
+    // start after the newly downloaded messages. Only do this on a complete,
+    // uncancelled run where every planned message was downloaded and nothing
+    // failed (phase-2 header batch failures leave uids outside `planned` that
+    // must still be picked up by a later gap-fill run).
+    if !cancelled && downloaded == planned && stats.failed == 0 {
+        if let Some(max_uid) = missing_uids.last().copied() {
+            let mut updated = remote_mailbox.clone();
+            updated.highest_uid = Some(max_uid.max(local_mailbox.highest_uid.unwrap_or(0)));
+            crate::cache::imap::mailbox::MailBox::batch_upsert(&[updated])?;
+        }
+    }
+
+    Ok(stats)
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    fn rh(uid: u32, message_id: Option<&str>, size: u64, internal_date: i64) -> RemoteHeader {
+        RemoteHeader {
+            uid,
+            message_id: message_id.map(|s| s.to_string()),
+            size,
+            internal_date,
+        }
+    }
+
+    fn snap(message_id: &str, uid: u64, size: u64, internal_date: i64) -> EnvelopeSnapshot {
+        EnvelopeSnapshot {
+            message_id: message_id.to_string(),
+            uid,
+            size,
+            internal_date,
+            //subject: String::new(),
+        }
+    }
+
+    #[test]
+    fn compute_missing_uids_message_id_diff() {
+        let remote = vec![
+            rh(1, Some("a"), 10, 1000),
+            rh(2, Some("b"), 20, 2000),
+            rh(3, Some("c"), 30, 3000),
+        ];
+        let local = vec![snap("a", 1, 10, 1000), snap("c", 3, 30, 3000)];
+        let missing = compute_missing_uids(&remote, &local);
+        assert_eq!(missing, vec![2]);
+    }
+
+    #[test]
+    fn compute_missing_uids_fingerprint_fallback() {
+        let remote = vec![rh(1, None, 10, 1000), rh(2, None, 20, 2000)];
+        let local = vec![snap("generated-x", 1, 10, 1000)];
+        let missing = compute_missing_uids(&remote, &local);
+        assert_eq!(missing, vec![2]);
+    }
+
+    #[test]
+    fn compute_missing_uids_remote_duplicates_all_present() {
+        let remote = vec![rh(1, Some("dup"), 10, 1000), rh(2, Some("dup"), 10, 1000)];
+        let local = vec![snap("dup", 1, 10, 1000)];
+        let missing = compute_missing_uids(&remote, &local);
+        assert!(missing.is_empty());
+    }
+
+    #[test]
+    fn compute_missing_uids_local_duplicate_not_re_downloaded() {
+        // A message-id appearing more than once locally is NOT a reason to
+        // re-download: servers (Zoho, 163) legitimately reuse message-ids
+        // across different messages, and re-downloading would be deduplicated
+        // away anyway, so it can never repair the duplication.
+        let remote = vec![rh(1, Some("dup"), 10, 1000), rh(2, Some("dup"), 10, 1000)];
+        let local = vec![snap("dup", 1, 10, 1000), snap("dup", 2, 10, 1000)];
+        let missing = compute_missing_uids(&remote, &local);
+        assert!(missing.is_empty());
+    }
+
+    #[test]
+    fn compute_missing_uids_msgid_mismatch_but_fingerprint_hit() {
+        // The remote message-id does not exist locally (e.g. a different
+        // message-id was stored by the SMTP path) but the fingerprint matches:
+        // the message is already stored and must NOT be re-downloaded.
+        let remote = vec![rh(1, Some("remote-id@x.com"), 10, 1000)];
+        let local = vec![snap("generated-random-id", 1, 10, 1000)];
+        let missing = compute_missing_uids(&remote, &local);
+        assert!(missing.is_empty());
+    }
+
+    #[test]
+    fn compute_missing_uids_msgid_mismatch_and_fingerprint_miss() {
+        let remote = vec![
+            rh(1, Some("remote-id@x.com"), 10, 1000),
+            rh(2, Some("remote-id-2@x.com"), 20, 2000),
+        ];
+        let local = vec![snap("generated-random-id", 1, 10, 1000)];
+        let missing = compute_missing_uids(&remote, &local);
+        assert_eq!(missing, vec![2]);
+    }
+}

crates/core/src/cache/imap/download/mod.rs

@@ -0,0 +1,254 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{
+    account::{
+        migration::{AccountModel, AccountType},
+        state::{
+            DownloadState, DownloadStatus, GapFillFolderStats, GapFillState, GapFillStatus,
+            TriggerType,
+        },
+    },
+    cache::imap::{download::flow::FetchDirection, mailbox::MailBox},
+    error::BichonResult,
+    imap::executor::ImapExecutor,
+};
+use download_folders::get_download_folders;
+use download_type::{decide_next_download_task, DownloadTask};
+use flow::reconcile_mailboxes;
+use rebuild::{rebuild_cache, rebuild_cache_by_date};
+use std::time::Instant;
+use tokio_util::sync::CancellationToken;
+use tracing::{debug, info, warn};
+
+pub mod download_folders;
+pub mod download_type;
+pub mod gap_fill;
+pub mod flow;
+pub mod rebuild;
+
+pub async fn process_imap_download(
+    account: &AccountModel,
+    token: CancellationToken,
+    trigger_type: TriggerType,
+    run_gap_fill: bool,
+) -> BichonResult<()> {
+    assert_eq!(account.account_type, AccountType::IMAP);
+    let start_time = Instant::now();
+    let account_id = account.id;
+    let download_task = decide_next_download_task(account, trigger_type).await?;
+    if matches!(download_task, DownloadTask::Idle) {
+        return Ok(());
+    }
+    let mut session = match ImapExecutor::create_connection(account_id).await {
+        Ok(session) => session,
+        Err(e) => {
+            let err_msg = format!("Failed to connect to IMAP server: {:#?}", e);
+            DownloadState::append_session_error(account_id, err_msg.clone())?;
+            DownloadState::update_session_status(
+                account_id,
+                DownloadStatus::Failed,
+                Some(err_msg),
+            )?;
+            return Err(e);
+        }
+    };
+    let remote_mailboxes = match get_download_folders(account, &mut session).await {
+        Ok(mailboxes) => mailboxes,
+        Err(err) => {
+            let err_msg = format!("Failed to fetch mailboxes: {:#?}", err);
+            warn!(account_id = account.id, error = %err, "{}", err_msg);
+            DownloadState::append_session_error(account_id, err_msg.clone())?;
+            DownloadState::update_session_status(
+                account_id,
+                DownloadStatus::Failed,
+                Some(err_msg),
+            )?;
+            return Ok(());
+        }
+    };
+    session.logout().await.ok();
+    if matches!(download_task, DownloadTask::FullFetch) {
+        let result = match &account.date_since {
+            Some(date_since) => {
+                rebuild_cache_by_date(
+                    account,
+                    &remote_mailboxes,
+                    &date_since.since_date()?,
+                    FetchDirection::Since,
+                    token,
+                )
+                .await
+            }
+            None => match &account.date_before {
+                Some(r) => {
+                    rebuild_cache_by_date(
+                        account,
+                        &remote_mailboxes,
+                        &r.calculate_date()?,
+                        FetchDirection::Before,
+                        token,
+                    )
+                    .await
+                }
+                None => rebuild_cache(account, &remote_mailboxes, token).await,
+            },
+        };
+        match result {
+            Ok(_) => {
+                DownloadState::update_session_status(account_id, DownloadStatus::Success, None)?;
+            }
+            Err(e) => {
+                let err_msg = format!("Email Download interrupted: {:#?}", e);
+                DownloadState::append_session_error(account_id, err_msg.clone())?;
+                DownloadState::update_session_status(
+                    account_id,
+                    DownloadStatus::Failed,
+                    Some(err_msg),
+                )?;
+            }
+        }
+        return Ok(());
+    }
+
+    let local_mailboxes = MailBox::list_all(account_id)?;
+    let reconcile_result =
+        reconcile_mailboxes(account, &remote_mailboxes, &local_mailboxes, token.clone()).await;
+
+    // Gap-fill phase: only on explicit user request (manual download with
+    // "run gap-fill" checked). Enumerate every UID in the download folders and
+    // download anything missing locally. Not run on scheduled syncs. Gap-fill
+    // runs are tracked in their own state (independent of the download session)
+    // because they are repeatable until `failed == 0`.
+    if run_gap_fill {
+        GapFillState::start_run(account_id)?;
+        // Run inside a helper so a failure anywhere still finalizes the run:
+        // an abandoned active run would otherwise show as Running forever.
+        let run_outcome = gap_fill_phase(
+            account,
+            &local_mailboxes,
+            &remote_mailboxes,
+            token,
+            account_id,
+        )
+        .await;
+        let (cancelled, total_downloaded, total_failed) = match run_outcome {
+            Ok(v) => v,
+            Err(e) => {
+                warn!(account_id = account_id, "Gap-fill phase error: {:#?}", e);
+                (false, 0, 1)
+            }
+        };
+        let status = if cancelled {
+            GapFillStatus::Cancelled
+        } else if total_failed > 0 {
+            GapFillStatus::Failed
+        } else {
+            GapFillStatus::Success
+        };
+        GapFillState::finish_run(account_id, status, total_downloaded, total_failed)?;
+        let summary = if cancelled {
+            format!(
+                "Gap-fill cancelled: {} downloaded, {} failed",
+                total_downloaded, total_failed
+            )
+        } else {
+            format!(
+                "Gap-fill finished: {} downloaded, {} failed",
+                total_downloaded, total_failed
+            )
+        };
+        DownloadState::update_session_message(account_id, summary.clone())?;
+        info!(account_id = account_id, "{}", summary);
+    }
+
+    // Finalize session status AFTER all phases so stats/progress written
+    // during gap-fill are not dropped (update_session_status closes the active
+    // session, moving it into history).
+    match reconcile_result {
+        Ok(_) => DownloadState::update_session_status(account_id, DownloadStatus::Success, None)?,
+        Err(e) => {
+            let err_msg = format!("Email Download interrupted: {:#?}", e);
+            DownloadState::append_session_error(account_id, err_msg.clone())?;
+            DownloadState::update_session_status(
+                account_id,
+                DownloadStatus::Failed,
+                Some(err_msg),
+            )?;
+        }
+    }
+
+    let elapsed_time = start_time.elapsed().as_secs();
+    debug!(
+        "Account{{{}}} Incremental sync completed: {} seconds elapsed.",
+        account.email, elapsed_time
+    );
+    Ok(())
+}
+
+/// Runs the gap-fill phase across all download-folder mailboxes. Errors inside
+/// are converted into a failed-run outcome instead of propagating, so the
+/// caller can always finalize the active run.
+async fn gap_fill_phase(
+    account: &AccountModel,
+    local_mailboxes: &[MailBox],
+    remote_mailboxes: &[MailBox],
+    token: CancellationToken,
+    account_id: u64,
+) -> BichonResult<(bool, u64, u64)> {
+    let mut total_downloaded = 0u64;
+    let mut total_failed = 0u64;
+    let mut cancelled = false;
+    for local_mailbox in local_mailboxes {
+        let Some(remote) = remote_mailboxes.iter().find(|r| r.name == local_mailbox.name) else {
+            continue;
+        };
+        if token.is_cancelled() {
+            cancelled = true;
+            break;
+        }
+        DownloadState::set_current_folder(account_id, local_mailbox.name.clone())?;
+        match gap_fill::gap_fill_mailbox(account, local_mailbox, remote, token.clone()).await {
+            Ok(stats) => {
+                total_downloaded += stats.downloaded;
+                total_failed += stats.failed;
+                GapFillState::add_folder_result(account_id, local_mailbox.name.clone(), stats)?;
+            }
+            Err(e) => {
+                let err_msg = format!(
+                    "Gap-fill failed for mailbox '{}': {:#?}",
+                    local_mailbox.name, e
+                );
+                warn!(account_id = account_id, "{}", err_msg);
+                DownloadState::append_session_error(account_id, err_msg)?;
+                total_failed += 1; // count the mailbox as a failed unit
+                GapFillState::add_folder_result(
+                    account_id,
+                    local_mailbox.name.clone(),
+                    GapFillFolderStats {
+                        downloaded: 0,
+                        failed: 1,
+                        candidate_count: 0,
+                        message: None,
+                    },
+                )?;
+            }
+        }
+    }
+    Ok((cancelled, total_downloaded, total_failed))
+}

crates/core/src/cache/imap/download/rebuild.rs

@@ -0,0 +1,268 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{
+    account::{
+        migration::AccountModel,
+        state::{DownloadState, DownloadStatus, FolderStatus},
+    },
+    cache::{
+        imap::{
+            download::flow::{fetch_and_save_by_date, fetch_and_save_full_mailbox, FetchDirection},
+            mailbox::MailBox,
+        },
+        SEMAPHORE,
+    },
+    error::{code::ErrorCode, BichonResult},
+    raise_error,
+    store::tantivy::{attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER},
+};
+
+use tokio_util::sync::CancellationToken;
+use tracing::{error, info};
+
+pub async fn rebuild_cache(
+    account: &AccountModel,
+    remote_mailboxes: &[MailBox],
+    token: CancellationToken,
+) -> BichonResult<()> {
+    MailBox::batch_insert(remote_mailboxes)?;
+    DownloadState::init_folder_details(
+        account.id,
+        remote_mailboxes.iter().map(|m| m.name.clone()).collect(),
+    )?;
+
+    let mut has_error = false;
+    let mut last_err = None;
+
+    for mailbox in remote_mailboxes {
+        if token.is_cancelled() {
+            DownloadState::update_session_status(
+                account.id,
+                DownloadStatus::Cancelled,
+                Some("Received termination signal (User stop or System shutdown)".to_string()),
+            )?;
+            break;
+        }
+        if mailbox.exists == 0 {
+            info!(
+                "Account {}: Mailbox '{}' on the remote server has no emails. Skipping fetch for this mailbox.",
+                account.id, &mailbox.name
+            );
+            DownloadState::update_folder_progress(
+                account.id,
+                mailbox.name.clone(),
+                0,
+                0,
+                FolderStatus::Success,
+                None,
+            )?;
+            continue;
+        }
+        let account = account.clone();
+        let mailbox = mailbox.clone();
+
+        let _global_permit = match SEMAPHORE.clone().acquire_owned().await {
+            Ok(permit) => permit,
+            Err(err) => {
+                error!(
+                    "Failed to acquire global semaphore permit for account {} mailbox '{}': {:#?}",
+                    account.id, &mailbox.name, err
+                );
+                continue;
+            }
+        };
+
+        match fetch_and_save_full_mailbox(&account, &mailbox, token.clone()).await {
+            Ok(new_highest_uid) => {
+                let mut updated = mailbox.clone();
+                updated.highest_uid = new_highest_uid;
+                MailBox::batch_upsert(&[updated])?;
+            }
+            Err(err) => {
+                has_error = true;
+                tracing::error!("Folder sync task failed: {:#?}", err);
+                last_err = Some(err);
+            }
+        }
+    }
+
+    if has_error {
+        if let Some(e) = last_err {
+            return Err(e);
+        }
+        return Err(raise_error!(
+            "Some tasks failed".into(),
+            ErrorCode::InternalError
+        ));
+    }
+    Ok(())
+}
+
+pub async fn rebuild_cache_by_date(
+    account: &AccountModel,
+    remote_mailboxes: &[MailBox],
+    date: &str,
+    direction: FetchDirection,
+    token: CancellationToken,
+) -> BichonResult<()> {
+    MailBox::batch_insert(remote_mailboxes)?;
+    DownloadState::init_folder_details(
+        account.id,
+        remote_mailboxes.iter().map(|m| m.name.clone()).collect(),
+    )?;
+
+    let mut has_error = false;
+    let mut last_err = None;
+
+    for mailbox in remote_mailboxes {
+        if token.is_cancelled() {
+            DownloadState::update_session_status(
+                account.id,
+                DownloadStatus::Cancelled,
+                Some("Received termination signal (User stop or System shutdown)".to_string()),
+            )?;
+            break;
+        }
+        if mailbox.exists == 0 {
+            info!(
+                "Account {}: Mailbox '{}' on the remote server has no emails. Skipping fetch for this mailbox.",
+                account.id, &mailbox.name
+            );
+
+            DownloadState::update_folder_progress(
+                account.id,
+                mailbox.name.clone(),
+                0,
+                0,
+                FolderStatus::Success,
+                None,
+            )?;
+            continue;
+        }
+        let account = account.clone();
+        let mailbox = mailbox.clone();
+        let date = date.to_string();
+        let direction = direction.clone();
+
+        let _global_permit = match SEMAPHORE.clone().acquire_owned().await {
+            Ok(permit) => permit,
+            Err(err) => {
+                error!(
+                    "Failed to acquire global semaphore permit for account {} mailbox '{}': {:#?}",
+                    account.id, &mailbox.name, err
+                );
+                continue;
+            }
+        };
+        match fetch_and_save_by_date(&account, date.as_str(), &mailbox, direction, token.clone())
+            .await
+        {
+            Ok(new_highest_uid) => {
+                let mut updated = mailbox.clone();
+                updated.highest_uid = new_highest_uid;
+                MailBox::batch_upsert(&[updated])?;
+            }
+            Err(err) => {
+                has_error = true;
+                tracing::error!("Folder sync task failed: {:#?}", err);
+                last_err = Some(err);
+            }
+        }
+    }
+
+    if has_error {
+        if let Some(e) = last_err {
+            return Err(e);
+        }
+        return Err(raise_error!(
+            "Some tasks failed".into(),
+            ErrorCode::InternalError
+        ));
+    }
+
+    Ok(())
+}
+
+pub async fn rebuild_mailbox_cache(
+    account: &AccountModel,
+    local_mailbox: &MailBox,
+    remote_mailbox: &MailBox,
+    token: CancellationToken,
+) -> BichonResult<Option<u32>> {
+    ENVELOPE_MANAGER
+        .delete_mailbox_envelopes(account.id, vec![local_mailbox.id])
+        .await?;
+    ATTACHMENT_MANAGER
+        .delete_mailbox_attachments(account.id, vec![local_mailbox.id])
+        .await?;
+    if remote_mailbox.exists == 0 {
+        info!(
+            "Account {}: Mailbox '{}' has no emails on the remote server. The mailbox is empty, no envelopes to fetch.",
+            account.id,
+            &local_mailbox.name
+        );
+        DownloadState::update_folder_progress(
+            account.id,
+            remote_mailbox.name.clone(),
+            0,
+            0,
+            FolderStatus::Success,
+            None,
+        )?;
+        return Ok(None);
+    }
+
+    let result = fetch_and_save_full_mailbox(account, remote_mailbox, token).await?;
+    Ok(result)
+}
+
+pub async fn rebuild_mailbox_cache_by_date(
+    account: &AccountModel,
+    local_mailbox_id: u64,
+    date: &str,
+    remote: &MailBox,
+    direction: FetchDirection,
+    token: CancellationToken,
+) -> BichonResult<Option<u32>> {
+    ENVELOPE_MANAGER
+        .delete_mailbox_envelopes(account.id, vec![local_mailbox_id])
+        .await?;
+    ATTACHMENT_MANAGER
+        .delete_mailbox_attachments(account.id, vec![local_mailbox_id])
+        .await?;
+    if remote.exists == 0 {
+        info!(
+            "Account {}: Mailbox '{}' has no emails on the remote server. The mailbox is empty, no envelopes to fetch.",
+            account.id,
+            &remote.name
+        );
+        DownloadState::update_folder_progress(
+            account.id,
+            remote.name.clone(),
+            0,
+            0,
+            FolderStatus::Success,
+            None,
+        )?;
+        return Ok(None);
+    }
+
+    let result = fetch_and_save_by_date(account, date, remote, direction, token).await?;
+    Ok(result)
+}

crates/core/src/cache/imap/mailbox.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -16,34 +16,25 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
 use crate::{
-    decode_mailbox_name, encode_mailbox_name,
-    modules::{
+    decode_mailbox_name, encode_mailbox_name, raise_error,
+    {
         database::{
-            batch_delete_impl, batch_insert_impl, batch_upsert_impl, filter_by_secondary_key_impl,
-            manager::DB_MANAGER,
+            batch_delete_impl, batch_insert_impl, batch_upsert_impl, delete_impl, filter_impl,
+            find_impl, manager::DB_MANAGER, MemDbModel,
         },
         error::{code::ErrorCode, BichonResult},
     },
-    raise_error,
 };
 use async_imap::types::{Name, NameAttribute};
-use itertools::Itertools;
-use native_db::*;
-use native_model::{native_model, Model};
-use poem_openapi::{Enum, Object};
 use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)]
-#[native_model(id = 1, version = 1)]
-#[native_db]
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct MailBox {
     /// The unique identifier for the mailbox
-    #[primary_key]
     pub id: u64,
     /// The ID of the account associated with the mailbox
-    #[secondary_key]
     pub account_id: u64,
     /// The unique, decoded, human-readable name of the mailbox (e.g., "INBOX", "Sent Items").
     /// This is the decoded name as presented to users, derived from the IMAP server's mailbox name
@@ -65,6 +56,19 @@ pub struct MailBox {
     /// The validity identifier for UIDs in this mailbox, used to ensure UID consistency across sessions.
     /// If `None`, the IMAP server has not provided this information.
     pub uid_validity: Option<u32>,
+    /// The highest UID that has been successfully downloaded and stored locally.
+    /// Used for incremental sync: next fetch starts from `highest_uid + 1`.
+    /// If `None`, a fallback query against the Tantivy index will be performed once.
+    pub highest_uid: Option<u32>,
+}
+
+impl MemDbModel for MailBox {
+    fn collection() -> &'static str {
+        "mailboxes"
+    }
+    fn key(&self) -> String {
+        self.id.to_string()
+    }
 }
 
 impl MailBox {
@@ -72,75 +76,50 @@ impl MailBox {
         encode_mailbox_name!(&self.name)
     }
 
-    // pub async fn batch_delete(mailboxes: Vec<MailBox>) -> BichonResult<()> {
-    //     batch_delete_impl(DB_MANAGER.envelope_db(), move |rw| {
-    //         let mut to_deleted = Vec::new();
-    //         for mailbox in mailboxes {
-    //             let retrived = rw
-    //                 .get()
-    //                 .primary::<MailBox>(mailbox.id)
-    //                 .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-    //             if let Some(retrived) = retrived {
-    //                 to_deleted.push(retrived);
-    //             }
-    //         }
-    //         Ok(to_deleted)
-    //     })
-    //     .await?;
-    //     Ok(())
-    // }
-
-    // pub async fn get(id: u64) -> RustMailerResult<MailBox> {
-    //     let result = async_find_impl::<MailBox>(DB_MANAGER.envelope_db(), id).await?;
-    //     Ok(result.ok_or_else(|| {
-    //         raise_error!(
-    //             format!("mailbox {} not found", id),
-    //             ErrorCode::InternalError
-    //         )
-    //     })?)
-    // }
-
-    // pub async fn delete(id: u64) -> BichonResult<()> {
-    //     delete_impl(DB_MANAGER.envelope_db(), move |rw| {
-    //         rw.get()
-    //             .primary::<MailBox>(id)
-    //             .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
-    //             .ok_or_else(|| raise_error!("mailbox missing".into(), ErrorCode::InternalError))
-    //     })
-    //     .await
-    // }
-
-    pub async fn list_all(account_id: u64) -> BichonResult<Vec<MailBox>> {
-        filter_by_secondary_key_impl(DB_MANAGER.envelope_db(), MailBoxKey::account_id, account_id)
-            .await
+    pub fn get(id: u64) -> BichonResult<MailBox> {
+        let result = find_impl::<MailBox>(DB_MANAGER.db(), &id.to_string())?;
+        Ok(result.ok_or_else(|| {
+            raise_error!(
+                format!("mailbox {} not found", id),
+                ErrorCode::InternalError
+            )
+        })?)
     }
 
-    pub async fn batch_insert(mailboxes: &[MailBox]) -> BichonResult<()> {
-        batch_insert_impl(DB_MANAGER.envelope_db(), mailboxes.to_vec()).await
+    pub fn delete(id: u64) -> BichonResult<()> {
+        delete_impl::<MailBox>(DB_MANAGER.db(), &id.to_string())
     }
 
-    pub async fn batch_upsert(mailboxes: &[MailBox]) -> BichonResult<()> {
-        batch_upsert_impl(DB_MANAGER.envelope_db(), mailboxes.to_vec()).await
+    pub fn list_all(account_id: u64) -> BichonResult<Vec<MailBox>> {
+        filter_impl::<MailBox, _>(DB_MANAGER.db(), move |m| m.account_id == account_id)
     }
 
-    pub async fn clean(account_id: u64) -> BichonResult<()> {
-        batch_delete_impl(DB_MANAGER.envelope_db(), move |rw| {
-            let mailboxes: Vec<MailBox> = rw
-                .scan()
-                .secondary::<MailBox>(MailBoxKey::account_id)
-                .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
-                .start_with(account_id)
-                .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
-                .try_collect()
-                .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-            Ok(mailboxes)
-        })
-        .await?;
+    pub fn find_mailbox(account_id: u64, mailbox_id: u64) -> BichonResult<Option<MailBox>> {
+        let all = filter_impl::<MailBox, _>(DB_MANAGER.db(), move |m| m.account_id == account_id)?;
+        Ok(all.into_iter().find(|m| m.id == mailbox_id))
+    }
+
+    pub fn batch_insert(mailboxes: &[MailBox]) -> BichonResult<()> {
+        batch_insert_impl(DB_MANAGER.db(), mailboxes.to_vec())
+    }
+
+    pub fn batch_upsert(mailboxes: &[MailBox]) -> BichonResult<()> {
+        batch_upsert_impl(DB_MANAGER.db(), mailboxes.to_vec())
+    }
+
+    pub fn clean(account_id: u64) -> BichonResult<()> {
+        let mailboxes =
+            filter_impl::<MailBox, _>(DB_MANAGER.db(), move |m| m.account_id == account_id)?;
+        let keys: Vec<String> = mailboxes.iter().map(|m| m.id.to_string()).collect();
+        if !keys.is_empty() {
+            batch_delete_impl::<MailBox>(DB_MANAGER.db(), keys)?;
+        }
         Ok(())
     }
 }
 
-#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, Object)]
+#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct Attribute {
     pub attr: AttributeEnum,
     pub extension: Option<String>,
@@ -152,7 +131,8 @@ impl Attribute {
     }
 }
 
-#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, Enum)]
+#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))]
 pub enum AttributeEnum {
     NoInferiors,
     NoSelect,

crates/core/src/cache/imap/mailbox_cache.rs

@@ -0,0 +1,114 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::cache::imap::mailbox::MailBox;
+use crate::utc_now;
+use lru::LruCache;
+use std::collections::HashMap;
+use std::num::NonZeroUsize;
+use std::sync::LazyLock;
+use tokio::sync::Mutex;
+
+struct CacheEntry {
+    mailboxes: Vec<MailBox>,
+    fetched_at: i64,
+}
+
+static CACHE: LazyLock<Mutex<LruCache<u64, CacheEntry>>> = LazyLock::new(|| {
+    Mutex::new(LruCache::new(NonZeroUsize::new(64).unwrap()))
+});
+
+const TTL_MS: i64 = 10 * 60 * 1000; // 10 minutes
+
+pub async fn get(account_id: u64) -> Option<Vec<MailBox>> {
+    let mut guard = CACHE.lock().await;
+    if let Some(entry) = guard.get(&account_id) {
+        if utc_now!() - entry.fetched_at < TTL_MS {
+            return Some(entry.mailboxes.clone());
+        }
+        guard.pop(&account_id);
+    }
+    None
+}
+
+pub async fn set(account_id: u64, mailboxes: Vec<MailBox>) {
+    let mut guard = CACHE.lock().await;
+    guard.put(
+        account_id,
+        CacheEntry {
+            mailboxes,
+            fetched_at: utc_now!(),
+        },
+    );
+}
+
+pub async fn invalidate(account_id: u64) {
+    let mut guard = CACHE.lock().await;
+    guard.pop(&account_id);
+}
+
+// Background fetch state tracking
+#[derive(Clone, Debug)]
+pub enum FetchStatus {
+    Fetching { examined: usize, total: usize },
+    Ready,
+    Error(String),
+}
+
+static FETCH_STATES: LazyLock<Mutex<HashMap<u64, FetchStatus>>> =
+    LazyLock::new(|| Mutex::new(HashMap::new()));
+
+pub async fn fetch_status(account_id: u64) -> Option<FetchStatus> {
+    FETCH_STATES.lock().await.get(&account_id).cloned()
+}
+
+pub async fn set_fetching(account_id: u64) {
+    FETCH_STATES.lock().await.insert(
+        account_id,
+        FetchStatus::Fetching {
+            examined: 0,
+            total: 0,
+        },
+    );
+}
+
+pub async fn update_fetch_progress(account_id: u64, examined: usize, total: usize) {
+    let mut guard = FETCH_STATES.lock().await;
+    guard.insert(
+        account_id,
+        FetchStatus::Fetching { examined, total },
+    );
+}
+
+pub async fn set_fetch_ready(account_id: u64) {
+    FETCH_STATES
+        .lock()
+        .await
+        .insert(account_id, FetchStatus::Ready);
+}
+
+pub async fn set_fetch_error(account_id: u64, error: String) {
+    FETCH_STATES
+        .lock()
+        .await
+        .insert(account_id, FetchStatus::Error(error));
+}
+
+pub async fn clear_fetch_state(account_id: u64) {
+    FETCH_STATES.lock().await.remove(&account_id);
+}

crates/core/src/cache/imap/mod.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -16,30 +16,20 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::collections::{HashMap, HashSet};
 
-use std::sync::LazyLock;
-
-use crate::modules::{account::state::AccountRunningState, database::ModelsAdapter};
-use ahash::{AHashMap, AHashSet};
 use mailbox::MailBox;
-use native_db::Models;
 
+pub mod download;
 pub mod mailbox;
-pub mod sync;
+pub mod mailbox_cache;
 pub mod task;
 
-pub static MAILBOX_MODELS: LazyLock<Models> = LazyLock::new(|| {
-    let mut adapter = ModelsAdapter::new();
-    adapter.register_model::<MailBox>();
-    adapter.register_model::<AccountRunningState>();
-    adapter.models
-});
-
 pub fn find_missing_mailboxes(
     local_mailboxes: &[MailBox],
     server_mailboxes: &[MailBox],
 ) -> Vec<MailBox> {
-    let local_names: AHashSet<_> = local_mailboxes.iter().map(|m| &m.name).collect();
+    let local_names: HashSet<_> = local_mailboxes.iter().map(|m| &m.name).collect();
     server_mailboxes
         .iter()
         .filter(|m| !local_names.contains(&m.name))
@@ -51,7 +41,7 @@ pub fn find_intersecting_mailboxes(
     local_mailboxes: &[MailBox],
     remote_mailboxes: &[MailBox],
 ) -> Vec<(MailBox, MailBox)> {
-    let local_map: AHashMap<_, _> = local_mailboxes
+    let local_map: HashMap<_, _> = local_mailboxes
         .iter()
         .map(|m| (m.name.clone(), m.clone()))
         .collect();

crates/core/src/cache/imap/task.rs

@@ -0,0 +1,286 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::account::entity::AuthType;
+use crate::account::state::{DownloadState, TriggerType};
+use crate::cache::imap::download::process_imap_download;
+use crate::common::periodic::{PeriodicTask, TaskHandle};
+use crate::error::code::ErrorCode;
+use crate::oauth2::token::OAuth2AccessToken;
+use crate::{account::migration::AccountModel, error::BichonResult};
+use crate::{raise_error, utc_now};
+use std::collections::{HashMap, HashSet};
+use std::sync::atomic::{AtomicI64, Ordering};
+use std::{sync::LazyLock, time::Duration};
+use tokio::sync::Mutex;
+use tokio::task::JoinHandle;
+use tokio_util::sync::CancellationToken;
+use tracing::{debug, error, info, warn};
+
+static _DESCRIPTION: &str = "This task periodically synchronizes mailbox data for a specified account, ensuring that all local data is up-to-date.";
+const TASK_INTERVAL: Duration = Duration::from_secs(10);
+pub static SYNC_TASKS: LazyLock<AccountDownTask> = LazyLock::new(AccountDownTask::new);
+static LAST_WARN_TIME: AtomicI64 = AtomicI64::new(0);
+const WARN_INTERVAL_MS: i64 = 600_000;
+
+pub struct AccountDownTask {
+    tasks: Mutex<Option<HashMap<u64, (TaskHandle, CancellationToken)>>>,
+    manual_tasks: Mutex<HashMap<u64, (JoinHandle<()>, CancellationToken)>>,
+    busy_accounts: Mutex<HashSet<u64>>,
+}
+
+impl AccountDownTask {
+    pub fn new() -> Self {
+        Self {
+            tasks: Mutex::new(Some(HashMap::new())),
+            manual_tasks: Mutex::new(HashMap::new()),
+            busy_accounts: Mutex::new(HashSet::new()),
+        }
+    }
+
+    async fn set_busy(&self, account_id: u64, is_busy: bool) {
+        let mut guard = self.busy_accounts.lock().await;
+        if is_busy {
+            guard.insert(account_id);
+        } else {
+            guard.remove(&account_id);
+        }
+    }
+
+    /// Atomically check and set busy. Returns true if we claimed the slot,
+    /// false if another task is already busy on this account.
+    async fn try_set_busy(&self, account_id: u64) -> bool {
+        let mut guard = self.busy_accounts.lock().await;
+        if guard.contains(&account_id) {
+            false
+        } else {
+            guard.insert(account_id);
+            true
+        }
+    }
+
+    // async fn is_busy(&self, account_id: u64) -> bool {
+    //     self.busy_accounts.lock().await.contains(&account_id)
+    // }
+
+    pub async fn start_download_task(&self, account_id: u64, email: String) {
+        let task_name = format!("account-download-task-{}-{}", account_id, &email);
+        let periodic_task = PeriodicTask::new(&task_name);
+
+        let cancel_token = CancellationToken::new();
+        let task_token = cancel_token.clone();
+
+        let task = move |param: Option<u64>| {
+            let account_id = param.unwrap();
+            let internal_token = task_token.clone();
+            Box::pin(async move {
+                if SYNC_TASKS.is_manual_running(account_id).await {
+                    debug!(
+                        "Account {}: Scheduled task skipped (Manual task is running).",
+                        account_id
+                    );
+                    return Ok(());
+                }
+
+                if !SYNC_TASKS.try_set_busy(account_id).await {
+                    debug!(
+                        "Account {}: Scheduled task skipped (Previous sync still active).",
+                        account_id
+                    );
+                    return Ok(());
+                }
+
+                let _busy_guard = scopeguard::guard(account_id, |id| {
+                    tokio::spawn(async move {
+                        SYNC_TASKS.set_busy(id, false).await;
+                    });
+                });
+                let account = AccountModel::get(account_id).ok();
+                match account {
+                    Some(account) => {
+                        if account.deleting {
+                            return Ok(());
+                        }
+                        if !account.enabled {
+                            let last = LAST_WARN_TIME.load(Ordering::Relaxed);
+                            let now = utc_now!();
+                            if now - last >= WARN_INTERVAL_MS {
+                                LAST_WARN_TIME.store(now, Ordering::Relaxed);
+                                warn!(
+                                    "Account {}: download aborted. Account is currently disabled.",
+                                    account_id
+                                );
+                            }
+                        } else {
+                            if let Some(imap) = &account.imap {
+                                if let AuthType::OAuth2 = imap.auth.auth_type {
+                                    if OAuth2AccessToken::get(account.id)?.is_none() {
+                                        if utc_now!() % 300_000 == 0 {
+                                            warn!("Account {}: download aborted. OAuth2 authorization not completed. Please visit the rustmailer admin page to authorize this account.", account_id);
+                                        }
+                                        return Ok(());
+                                    }
+                                }
+                            }
+                            if let Err(e) = process_imap_download(
+                                &account,
+                                internal_token,
+                                TriggerType::Scheduled,
+                                false,
+                            )
+                            .await
+                            {
+                                DownloadState::append_session_error(
+                                    account.id,
+                                    format!("error in account download task: {:#?}", e),
+                                )?;
+                                error!(
+                                    "Failed to download mailbox data for '{}': {:?}",
+                                    account_id, e
+                                )
+                            }
+                        }
+                    }
+                    None => {
+                        error!(
+                            "Account {}: download aborted. Account entity not found.",
+                            account_id
+                        );
+                    }
+                }
+                Ok(())
+            })
+        };
+        let handler = periodic_task.start(task, Some(account_id), TASK_INTERVAL, true, true);
+        self.add_task(account_id, (handler, cancel_token)).await;
+    }
+
+    pub async fn add_task(&self, account_id: u64, handler: (TaskHandle, CancellationToken)) {
+        let mut guard = self.tasks.lock().await;
+        if let Some(map) = guard.as_mut() {
+            map.insert(account_id, handler);
+        } else {
+            tracing::error!("Failed to add task: HashMap has been taken during shutdown.");
+        }
+    }
+
+    pub async fn stop(&self, account_id: u64) -> BichonResult<()> {
+        let mut guard = self.tasks.lock().await;
+        if let Some(map) = guard.as_mut() {
+            if let Some((handler, token)) = map.remove(&account_id) {
+                drop(guard);
+                token.cancel();
+                handler.cancel().await;
+            }
+        }
+        Ok(())
+    }
+
+    pub async fn shutdown(&self) {
+        let mut guard = self.tasks.lock().await;
+        if let Some(map) = guard.take() {
+            drop(guard);
+            for (account_id, (handler, token)) in map {
+                info!(
+                    "Shutdown: Sending cancel signal to account {}...",
+                    account_id
+                );
+                token.cancel();
+                if let Err(_) = tokio::time::timeout(Duration::from_secs(5), handler.stop()).await {
+                    error!(
+                        "Shutdown: Account {} download task forced timeout.",
+                        account_id
+                    );
+                }
+            }
+            info!("Shutdown: All download tasks processed.");
+        }
+    }
+
+    pub async fn start_manual_task(&self, account_id: u64, run_gap_fill: bool) -> BichonResult<()> {
+        {
+            if self.is_manual_running(account_id).await {
+                return Err(raise_error!(
+                    "Manual task already running.".into(),
+                    ErrorCode::Forbidden
+                ));
+            }
+            if !self.try_set_busy(account_id).await {
+                return Err(raise_error!(
+                    "The background synchronization is currently active. Please try again in a few seconds.".into(),
+                    ErrorCode::Forbidden
+                ));
+            }
+        }
+
+        let cancel_token = CancellationToken::new();
+        let token_clone = cancel_token.clone();
+        let handle = tokio::spawn(async move {
+            // busy already claimed by caller via try_set_busy
+            let _cleanup = scopeguard::guard(account_id, |id| {
+                tokio::spawn(async move {
+                    SYNC_TASKS.set_busy(id, false).await;
+                    let mut guard = SYNC_TASKS.manual_tasks.lock().await;
+                    guard.remove(&id);
+                });
+            });
+            if token_clone.is_cancelled() {
+                return;
+            }
+            let account = match AccountModel::get(account_id) {
+                Ok(acc) => acc,
+                Err(e) => {
+                    error!("Failed to fetch account {}: {:?}", account_id, e);
+                    return;
+                }
+            };
+
+            if account.deleting {
+                return;
+            }
+
+            if let Err(e) =
+                process_imap_download(&account, token_clone, TriggerType::Manual, run_gap_fill)
+                    .await
+            {
+                error!("Manual download failed for {}: {:?}", account_id, e);
+                let error_msg = format!("error in account download task: {:#?}", e);
+                let _ = DownloadState::append_session_error(account.id, error_msg);
+            }
+        });
+        {
+            let mut guard = self.manual_tasks.lock().await;
+            guard.insert(account_id, (handle, cancel_token));
+        }
+
+        Ok(())
+    }
+
+    pub async fn cancel_manual_task(&self, account_id: u64) {
+        let mut guard = self.manual_tasks.lock().await;
+        if let Some((handle, token)) = guard.remove(&account_id) {
+            token.cancel();
+            let _ = handle.await;
+        }
+    }
+
+    pub async fn is_manual_running(&self, account_id: u64) -> bool {
+        let guard = self.manual_tasks.lock().await;
+        guard.contains_key(&account_id)
+    }
+}

crates/core/src/cache/mod.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -17,7 +17,7 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
-use crate::modules::settings::cli::SETTINGS;
+use crate::settings::cli::SETTINGS;
 use std::sync::{Arc, LazyLock};
 use tokio::sync::Semaphore;
 

crates/core/src/common/auth.rs

@@ -0,0 +1,143 @@
+use std::{
+    collections::{BTreeSet, HashSet},
+    net::IpAddr,
+};
+
+use crate::{
+    error::{code::ErrorCode, BichonResult},
+    raise_error,
+    users::{permissions::Permission, role::UserRole, UserModel},
+};
+
+#[derive(Clone, Debug)]
+pub struct ClientContext {
+    pub ip_addr: Option<IpAddr>,
+    pub user: UserModel,
+}
+
+impl ClientContext {
+    pub fn require_any_permission(
+        &self,
+        requirements: Vec<(Option<u64>, &str)>,
+    ) -> BichonResult<()> {
+        for (account_id, permission) in requirements {
+            if self.has_permission(account_id, permission) {
+                return Ok(());
+            }
+        }
+        Err(raise_error!(
+            "Access denied: Insufficient permissions to perform this action.".into(),
+            ErrorCode::Forbidden
+        ))
+    }
+
+    pub fn check_has_permission(
+        user: &UserModel,
+        account_id: Option<u64>,
+        permission: &str,
+    ) -> bool {
+        if user.is_admin() {
+            return true;
+        }
+
+        let mut global_perms = HashSet::new();
+        for rid in &user.global_roles {
+            if let Some(role) = UserRole::find(*rid).ok().flatten() {
+                global_perms.extend(role.permissions);
+            }
+        }
+
+        if Self::check_global_logic(&global_perms, permission) {
+            return true;
+        }
+
+        if let Some(aid) = account_id {
+            if let Some(role_id) = user.account_access_map.get(&aid) {
+                if let Some(role) = UserRole::find(*role_id).ok().flatten() {
+                    if role.permissions.contains(&permission.to_string())
+                        || Self::check_account_logic(&role.permissions, permission)
+                    {
+                        return true;
+                    }
+                }
+            }
+        }
+
+        false
+    }
+
+    pub fn has_permission(&self, account_id: Option<u64>, permission: &str) -> bool {
+        if self.user.is_admin() {
+            return true;
+        }
+
+        let mut global_perms = HashSet::new();
+        for rid in &self.user.global_roles {
+            if let Some(role) = UserRole::find(*rid).ok().flatten() {
+                global_perms.extend(role.permissions);
+            }
+        }
+
+        if Self::check_global_logic(&global_perms, permission) {
+            return true;
+        }
+
+        if let Some(aid) = account_id {
+            if let Some(role_id) = self.user.account_access_map.get(&aid) {
+                if let Some(role) = UserRole::find(*role_id).ok().flatten() {
+                    if role.permissions.contains(&permission.to_string())
+                        || Self::check_account_logic(&role.permissions, permission)
+                    {
+                        return true;
+                    }
+                }
+            }
+        }
+
+        false
+    }
+
+    fn check_global_logic(global: &HashSet<String>, perm: &str) -> bool {
+        if global.contains(perm) {
+            return true;
+        }
+
+        match perm {
+            Permission::DATA_READ => global.contains(Permission::DATA_READ_ALL),
+            Permission::DATA_DELETE => global.contains(Permission::DATA_DELETE_ALL),
+            Permission::DATA_RAW_DOWNLOAD => global.contains(Permission::DATA_RAW_DOWNLOAD_ALL),
+            Permission::DATA_EXPORT_BATCH => global.contains(Permission::DATA_EXPORT_BATCH_ALL),
+            Permission::ACCOUNT_MANAGE | Permission::ACCOUNT_READ_DETAILS => {
+                global.contains(Permission::ACCOUNT_MANAGE_ALL)
+            }
+            _ => false,
+        }
+    }
+
+    fn check_account_logic(scoped_perms: &BTreeSet<String>, perm: &str) -> bool {
+        if scoped_perms.contains(perm) {
+            return true;
+        }
+        match perm {
+            Permission::DATA_READ | Permission::ACCOUNT_READ_DETAILS => {
+                scoped_perms.contains(Permission::ACCOUNT_MANAGE)
+            }
+            _ => false,
+        }
+    }
+
+    pub fn require_permission(
+        &self,
+        account_id: Option<u64>,
+        permission: &str,
+    ) -> BichonResult<()> {
+        if self.has_permission(account_id, permission) {
+            Ok(())
+        } else {
+            Err(raise_error!(
+                format!("Access Denied: Missing permission '{}'", permission),
+                ErrorCode::Forbidden
+            ))
+        }
+    }
+}

crates/core/src/common/mod.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -16,29 +16,19 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-use super::error::code::ErrorCode;
-use super::error::BichonError;
-use mail_parser::{Addr as ImapAddr, Address as ImapAddress};
-use poem::error::ResponseError;
-use poem::Body;
-use poem::{http::StatusCode, Error, Response};
-use poem_openapi::Object;
-use serde::{Deserialize, Serialize};
 use std::ops::Deref;
-use tracing::error;
 
+use mail_parser::{Addr as ImapAddr, Address as ImapAddress};
+use serde::{Deserialize, Serialize};
 pub mod auth;
-pub mod error;
-pub mod log;
 pub mod paginated;
 pub mod periodic;
 pub mod rustls;
 pub mod signal;
-pub mod timeout;
-pub mod tls;
+#[cfg(feature = "web-api")]
 pub mod validator;
 
-#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Object)]
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
 pub struct Addr {
     /// The optional display name associated with the email address (e.g., "John Doe").
     /// If `None`, no display name is specified.
@@ -91,62 +81,3 @@ impl<'x> From<&ImapAddress<'x>> for AddrVec {
         AddrVec(vec)
     }
 }
-
-// #[derive(Serialize)]
-// pub struct ErrorResponse {
-//     pub message: String,
-// }
-
-#[inline]
-fn create_rust_mailer_error(message: &str, code: ErrorCode) -> BichonError {
-    BichonError::Generic {
-        message: message.into(),
-        location: snafu::Location::default(),
-        code,
-    }
-}
-
-#[inline]
-pub fn create_api_error_response(message: &str, code: ErrorCode) -> Error {
-    let rust_mailer_error = create_rust_mailer_error(message, code);
-    rust_mailer_error.into()
-}
-
-impl ResponseError for BichonError {
-    fn status(&self) -> StatusCode {
-        match self {
-            BichonError::Generic {
-                message: _,
-                location: _,
-                code,
-            } => code.status(),
-        }
-    }
-
-    fn as_response(&self) -> Response
-    where
-        Self: std::error::Error + Send + Sync + 'static,
-    {
-        match self {
-            BichonError::Generic {
-                message,
-                location,
-                code,
-            } => {
-                error!(
-                    error_code = *code as u32,
-                    error_message = %message,
-                    error_location = ?location
-                );
-
-                let body = Body::from_json(serde_json::json!({
-                    "code": *code as u32,
-                    "message": message.to_string(),
-                }))
-                .unwrap();
-
-                Response::builder().status(self.status()).body(body)
-            }
-        }
-    }
-}

crates/core/src/common/paginated.rs

@@ -0,0 +1,260 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{
+    error::{code::ErrorCode, BichonResult},
+    raise_error,
+};
+use serde::{Deserialize, Serialize};
+use std::cmp::min;
+
+pub fn paginate_vec<T: Clone>(
+    items: &Vec<T>,
+    page: Option<u64>,
+    page_size: Option<u64>,
+) -> BichonResult<Paginated<T>> {
+    let total_items = items.len() as u64;
+
+    let (offset, total_pages) = match (page, page_size) {
+        (Some(p), Some(s)) if p > 0 && s > 0 => {
+            let offset = (p - 1) * s;
+            let total_pages = if total_items > 0 {
+                (total_items + s - 1) / s
+            } else {
+                0
+            };
+            (Some(offset), Some(total_pages))
+        }
+        (Some(0), _) | (_, Some(0)) => {
+            return Err(raise_error!(
+                "'page' and 'page_size' must be greater than 0.".into(),
+                ErrorCode::InvalidParameter
+            ));
+        }
+        _ => (None, None),
+    };
+
+    let data = match offset {
+        Some(offset) if offset >= total_items => vec![],
+        Some(offset) => {
+            let end = min(offset + page_size.unwrap_or(total_items), total_items) as usize;
+            items[offset as usize..end].to_vec()
+        }
+        None => items.clone(),
+    };
+
+    Ok(Paginated::new(
+        page,
+        page_size,
+        total_items,
+        total_pages,
+        data,
+    ))
+}
+
+
+#[cfg(not(feature = "web-api"))]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct DataPage<S>
+where
+    S: Serialize + std::fmt::Debug + std::marker::Unpin + Send + Sync,
+{
+    /// The current page number (starting from 1).
+    pub current_page: Option<u64>,
+    /// The number of items per page.
+    pub page_size: Option<u64>,
+    /// The total number of items across all pages.
+    pub total_items: u64,
+    /// The list of items returned on the current page.
+    pub items: Vec<S>,
+    /// The total number of pages. This is optional and may not be set if not calculated.
+    pub total_pages: Option<u64>,
+}
+#[cfg(not(feature = "web-api"))]
+impl<S: Serialize + std::fmt::Debug + std::marker::Unpin + Send + Sync> From<Paginated<S>>
+    for DataPage<S>
+{
+    fn from(paginated: Paginated<S>) -> Self {
+        DataPage {
+            current_page: paginated.page,
+            page_size: paginated.page_size,
+            total_items: paginated.total_items,
+            total_pages: paginated.total_pages,
+            items: paginated.items,
+        }
+    }
+}
+
+
+
+
+#[cfg(feature = "web-api")]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, poem_openapi::Object)]
+pub struct DataPage<S>
+where
+    S: Serialize
+        + std::fmt::Debug
+        + std::marker::Unpin
+        + Send
+        + Sync
+        + poem_openapi::types::Type
+        + poem_openapi::types::ParseFromJSON
+        + poem_openapi::types::ToJSON,
+{
+    /// The current page number (starting from 1).
+    pub current_page: Option<u64>,
+    /// The number of items per page.
+    pub page_size: Option<u64>,
+    /// The total number of items across all pages.
+    pub total_items: u64,
+    /// The list of items returned on the current page.
+    pub items: Vec<S>,
+    /// The total number of pages. This is optional and may not be set if not calculated.
+    pub total_pages: Option<u64>,
+}
+
+#[cfg(feature = "web-api")]
+impl<
+        S: Serialize
+            + std::fmt::Debug
+            + std::marker::Unpin
+            + Send
+            + Sync
+            + poem_openapi::types::Type
+            + poem_openapi::types::ParseFromJSON
+            + poem_openapi::types::ToJSON,
+    > From<Paginated<S>> for DataPage<S>
+{
+    fn from(paginated: Paginated<S>) -> Self {
+        DataPage {
+            current_page: paginated.page,
+            page_size: paginated.page_size,
+            total_items: paginated.total_items,
+            total_pages: paginated.total_pages,
+            items: paginated.items,
+        }
+    }
+}
+
+
+#[derive(Debug)]
+pub struct Paginated<T> {
+    pub page: Option<u64>,
+    pub page_size: Option<u64>,
+    pub total_items: u64,
+    pub total_pages: Option<u64>,
+    pub items: Vec<T>,
+}
+
+impl<T> Paginated<T> {
+    pub fn new(
+        page: Option<u64>,
+        page_size: Option<u64>,
+        total_items: u64,
+        total_pages: Option<u64>,
+        items: Vec<T>,
+    ) -> Self {
+        Paginated {
+            page,
+            page_size,
+            total_items,
+            total_pages,
+            items,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn paginate_vec_full_list_without_pagination() {
+        let items: Vec<i32> = (1..=10).collect();
+        let result = paginate_vec(&items, None, None).unwrap();
+        assert_eq!(result.items.len(), 10);
+        assert_eq!(result.total_items, 10);
+        assert_eq!(result.page, None);
+        assert_eq!(result.total_pages, None);
+    }
+
+    #[test]
+    fn paginate_vec_first_page() {
+        let items: Vec<i32> = (1..=25).collect();
+        let result = paginate_vec(&items, Some(1), Some(10)).unwrap();
+        assert_eq!(result.items, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
+        assert_eq!(result.total_items, 25);
+        assert_eq!(result.total_pages, Some(3));
+        assert_eq!(result.page, Some(1));
+    }
+
+    #[test]
+    fn paginate_vec_last_partial_page() {
+        let items: Vec<i32> = (1..=25).collect();
+        let result = paginate_vec(&items, Some(3), Some(10)).unwrap();
+        assert_eq!(result.items, vec![21, 22, 23, 24, 25]);
+        assert_eq!(result.total_items, 25);
+        assert_eq!(result.total_pages, Some(3));
+    }
+
+    #[test]
+    fn paginate_vec_page_beyond_range_returns_empty() {
+        let items: Vec<i32> = (1..=10).collect();
+        let result = paginate_vec(&items, Some(5), Some(10)).unwrap();
+        assert_eq!(result.items.len(), 0);
+        assert_eq!(result.total_items, 10);
+    }
+
+    #[test]
+    fn paginate_vec_empty_list() {
+        let items: Vec<i32> = vec![];
+        let result = paginate_vec(&items, Some(1), Some(10)).unwrap();
+        assert_eq!(result.items.len(), 0);
+        assert_eq!(result.total_items, 0);
+        assert_eq!(result.total_pages, Some(0));
+    }
+
+    #[test]
+    fn paginate_vec_zero_page_returns_error() {
+        let items: Vec<i32> = (1..=10).collect();
+        assert!(paginate_vec(&items, Some(0), Some(10)).is_err());
+    }
+
+    #[test]
+    fn paginate_vec_zero_page_size_returns_error() {
+        let items: Vec<i32> = (1..=10).collect();
+        assert!(paginate_vec(&items, Some(1), Some(0)).is_err());
+    }
+
+    #[test]
+    fn paginate_vec_single_item() {
+        let items = vec![42];
+        let result = paginate_vec(&items, Some(1), Some(10)).unwrap();
+        assert_eq!(result.items, vec![42]);
+        assert_eq!(result.total_items, 1);
+        assert_eq!(result.total_pages, Some(1));
+    }
+
+    #[test]
+    fn paginate_vec_exact_page_boundary() {
+        let items: Vec<i32> = (1..=20).collect();
+        let result = paginate_vec(&items, Some(2), Some(10)).unwrap();
+        assert_eq!(result.items, vec![11, 12, 13, 14, 15, 16, 17, 18, 19, 20]);
+        assert_eq!(result.total_pages, Some(2));
+    }
+}

crates/core/src/common/periodic.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -16,10 +16,9 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
-use crate::modules::{common::signal::SIGNAL_MANAGER, error::BichonResult};
+use crate::{common::signal::SIGNAL_MANAGER, error::BichonResult};
 use std::{future::Future, time::Duration};
-use tokio::{sync::oneshot, time::MissedTickBehavior};
+use tokio::{sync::oneshot, task::JoinHandle, time::MissedTickBehavior};
 use tracing::{info, warn};
 
 pub struct PeriodicTask {
@@ -28,7 +27,7 @@ pub struct PeriodicTask {
 
 pub struct TaskHandle {
     cancel_sender: Option<oneshot::Sender<()>>,
-    join_handle: tokio::task::JoinHandle<()>,
+    join_handle: JoinHandle<()>,
 }
 
 impl TaskHandle {
@@ -38,6 +37,10 @@ impl TaskHandle {
         }
         let _ = self.join_handle.await;
     }
+
+    pub async fn stop(self) {
+        let _ = self.join_handle.await;
+    }
 }
 
 impl PeriodicTask {
@@ -82,6 +85,14 @@ impl PeriodicTask {
             let mut cancel_receiver = cancel_receiver_opt;
 
             loop {
+                let cancel_fut = async {
+                    if let Some(ref mut rx) = cancel_receiver {
+                        rx.await.ok();
+                    } else {
+                        std::future::pending::<()>().await;
+                    }
+                };
+
                 tokio::select! {
                     _ = interval.tick() => {
                         match task(param).await {
@@ -92,13 +103,7 @@ impl PeriodicTask {
                         }
                     }
                     // only enabled if cancel_receiver is Some
-                    _ = async {
-                        if let Some(ref mut rx) = cancel_receiver {
-                            rx.await.ok()
-                        } else {
-                            futures::future::pending().await
-                        }
-                    } => {
+                    _ = cancel_fut => {
                         info!("Task '{}' received cancellation signal", name_clone);
                         break;
                     }

crates/core/src/common/rustls.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -16,18 +16,17 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
 use crate::{
-    modules::{
+    {
         context::Initialize,
         error::{code::ErrorCode, BichonResult},
     },
     raise_error,
 };
 
-pub struct RustMailerTls;
+pub struct BichonTls;
 
-impl Initialize for RustMailerTls {
+impl Initialize for BichonTls {
     async fn initialize() -> BichonResult<()> {
         rustls::crypto::CryptoProvider::install_default(rustls::crypto::ring::default_provider())
             .map_err(|_| {
@@ -38,4 +37,3 @@ impl Initialize for RustMailerTls {
             })
     }
 }
- 

crates/core/src/common/signal.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -16,12 +16,9 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
 use std::sync::LazyLock;
 
-use crate::modules::{
-    context::Initialize, error::BichonResult, utils::shutdown::shutdown_signal,
-};
+use crate::{context::Initialize, error::BichonResult, utils::shutdown::shutdown_signal};
 use tokio::sync::broadcast;
 
 pub static SIGNAL_MANAGER: LazyLock<SignalManager> = LazyLock::new(SignalManager::new);

crates/core/src/common/validator.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -23,6 +23,7 @@ use std::{
 };
 
 use email_address::EmailAddress;
+
 use poem_openapi::Validator;
 
 pub struct EmailValidator;
@@ -33,6 +34,7 @@ impl Display for EmailValidator {
     }
 }
 
+
 impl Validator<String> for EmailValidator {
     fn check(&self, value: &String) -> bool {
         match EmailAddress::from_str(value) {
@@ -40,4 +42,4 @@ impl Validator<String> for EmailValidator {
             Err(_) => false,
         }
     }
-}
+}

crates/core/src/context/controller.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -16,30 +16,29 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
-use crate::modules::{cache::imap::task::SYNC_TASKS, error::BichonResult};
+use crate::{cache::imap::task::SYNC_TASKS, error::BichonResult};
 use std::{sync::LazyLock, time::Duration};
 use tokio::sync::mpsc;
 use tracing::{error, info};
 
-pub static SYNC_CONTROLLER: LazyLock<SyncController> = LazyLock::new(SyncController::new);
+pub static DOWNLOAD_CONTROLLER: LazyLock<DownloadController> =
+    LazyLock::new(DownloadController::new);
 
-pub struct SyncController {
-    channel: mpsc::Sender<(u64, String)>, // Channel to trigger account sync by account ID
+pub struct DownloadController {
+    channel: mpsc::Sender<(u64, String)>, // Channel to trigger account download by account ID
 }
 
-impl SyncController {
+impl DownloadController {
     pub fn new() -> Self {
         let (tx, mut rx) = mpsc::channel::<(u64, String)>(100);
 
         tokio::spawn(async move {
             while let Some((account_id, email)) = rx.recv().await {
-                match Self::start_syncer(account_id, email.clone()).await {
-                    Ok(Some(_)) => {}
-                    Ok(None) => {}
+                match Self::start_download(account_id, email.clone()).await {
+                    Ok(_) => {}
                     Err(err) => {
                         error!(
-                            "Failed to prepare and start syncer of account {{{}-{}}}, error: {:#?}",
+                            "Failed to prepare and start scheduled download of account {{{}-{}}}, error: {:#?}",
                             &account_id, &email, err
                         );
                     }
@@ -47,26 +46,26 @@ impl SyncController {
             }
         });
 
-        SyncController { channel: tx }
+        DownloadController { channel: tx }
     }
 
     /// Trigger synchronization for a specific account
-    pub async fn trigger_start(&self, account_id: u64, email: String) {
+    pub async fn trigger_schedule(&self, account_id: u64, email: String) {
         if let Err(e) = self.channel.send((account_id, email)).await {
             error!(
-                "Failed to trigger synchronization for account={{{}}}, error: {:?}",
+                "Failed to trigger download for account={{{}}}, error: {:?}",
                 account_id, e
             );
         }
     }
 
-    async fn start_syncer(account_id: u64, email: String) -> BichonResult<Option<()>> {
+    async fn start_download(account_id: u64, email: String) -> BichonResult<()> {
         info!(
-            "Account syncer starting for account: {}-{}.",
+            "Account download starting for account: {}-{}.",
             account_id, email
         );
-        SYNC_TASKS.start_account_sync_task(account_id, email).await;
+        SYNC_TASKS.start_download_task(account_id, email).await;
         tokio::time::sleep(Duration::from_millis(100)).await;
-        Ok(Some(()))
+        Ok(())
     }
 }

crates/core/src/context/executors.rs

@@ -0,0 +1,115 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::account::migration::AccountType;
+use crate::account::state::{DownloadState, GapFillState};
+use crate::context::Initialize;
+use crate::{
+    {
+        account::migration::AccountModel, context::controller::DOWNLOAD_CONTROLLER, error::BichonResult,
+    },
+    utc_now,
+};
+use std::sync::LazyLock;
+use tracing::{info, warn};
+
+pub static BICHON_CONTEXT: LazyLock<BichonContext> = LazyLock::new(BichonContext::new);
+
+pub struct BichonContext {
+    start_at: i64,
+}
+
+impl Initialize for BichonContext {
+    async fn initialize() -> BichonResult<()> {
+        BICHON_CONTEXT.start_account_downloader().await
+    }
+}
+
+impl BichonContext {
+    pub fn new() -> Self {
+        Self {
+            start_at: utc_now!(),
+        }
+    }
+    pub fn uptime_ms(&self) -> i64 {
+        utc_now!() - self.start_at
+    }
+
+    pub async fn start_account_downloader(&self) -> BichonResult<()> {
+        let accounts = AccountModel::list_all()?;
+        let active_accounts: Vec<AccountModel> = accounts
+            .into_iter()
+            .filter(|a| a.enabled && matches!(a.account_type, AccountType::IMAP))
+            .collect();
+
+        if active_accounts.is_empty() {
+            info!("No active accounts found for account initialization.");
+            return Ok(());
+        }
+        info!(
+            "System has {} active IMAP accounts to initialize.",
+            active_accounts.len()
+        );
+        for account in active_accounts {
+            // A Running session surviving startup is a leftover from a previous
+            // interrupted run; nothing is downloading yet at this point. Mark it
+            // Cancelled so the UI doesn't show a phantom "syncing" state. The
+            // scheduler starts regardless — its first tick runs immediately, so
+            // the interrupted run is caught up on, and the session's trigger
+            // stays Scheduled rather than showing a "Manual" the user never
+            // initiated.
+            match DownloadState::finalize_stale_session(account.id) {
+                Ok(true) => {
+                    info!(
+                        "Account {}: stale sync session finalized on startup.",
+                        account.id
+                    );
+                }
+                Err(e) => {
+                    warn!(
+                        "Failed to finalize stale session for account {}: {:#?}",
+                        account.id, e
+                    );
+                }
+                Ok(false) => {}
+            }
+            // Same for a leftover gap-fill run: a Running active run surviving
+            // startup is a phantom — nothing is scanning at this point.
+            match GapFillState::finalize_stale_run(account.id) {
+                Ok(true) => {
+                    info!(
+                        "Account {}: stale gap-fill run finalized on startup.",
+                        account.id
+                    );
+                }
+                Err(e) => {
+                    warn!(
+                        "Failed to finalize stale gap-fill run for account {}: {:#?}",
+                        account.id, e
+                    );
+                }
+                Ok(false) => {}
+            }
+            DOWNLOAD_CONTROLLER
+                .trigger_schedule(account.id, account.email)
+                .await
+        }
+
+        Ok(())
+    }
+}

crates/core/src/context/mod.rs

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
 //
@@ -16,17 +16,16 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
-use crate::modules::error::BichonResult;
+use crate::{common::periodic::TaskHandle, error::BichonResult};
 
 pub mod controller;
 pub mod executors;
-pub mod status;
 
+#[allow(async_fn_in_trait)]
 pub trait Initialize {
     async fn initialize() -> BichonResult<()>;
 }
 
-pub trait RustMailTask {
-    fn start();
+pub trait BichonTask {
+    fn start() -> TaskHandle;
 }

crates/core/src/dashboard/mod.rs

@@ -0,0 +1,227 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{
+    store::tantivy::{
+        attachment::ATTACHMENT_MANAGER,
+        envelope::ENVELOPE_MANAGER,
+        fields::{F_CONTENT_HASH, F_ID},
+        schema::SchemaTools,
+    },
+    users::permissions::Permission,
+};
+//use poem_openapi::Object;
+use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
+use tantivy::{schema::Value, TantivyDocument};
+
+use crate::{
+    bichon_version, raise_error,
+    {
+        account::migration::AccountModel,
+        common::auth::ClientContext,
+        error::{code::ErrorCode, BichonResult},
+        settings::dir::DATA_DIR_MANAGER,
+        utils::get_total_size,
+    },
+};
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct DashboardStats {
+    pub account_count: usize,                            // Number of accounts
+    pub email_count: u64,                                // Total number of emails
+    pub attachment_count: u64,                           // Total number of attachments
+    pub total_size_bytes: u64,                           // Total size of all emails (in bytes)
+    pub storage_usage_bytes: u64,                        // Actual storage used (in bytes)
+    pub index_usage_bytes: u64,                          // Index storage size (in bytes)
+    pub recent_activity: Vec<TimeBucket>,                // Email activity over recent days
+    pub top_senders: Vec<Group>,                         // Top 10 senders
+    pub top_accounts: Vec<Group>,                        // Top 10 accounts
+    pub with_attachment_count: u64,                      // Emails with attachments
+    pub without_attachment_count: u64,                   // Emails without attachments
+    pub top_largest_emails: Vec<LargestEmail>,           // Top 10 largest emails
+    pub top_largest_attachments: Vec<LargestAttachment>, // Top 10 largest attachments
+    pub system_version: String, // The semantic version string of the currently running backend service
+}
+
+impl DashboardStats {
+    pub async fn get(context: ClientContext) -> BichonResult<Self> {
+        let has_all_accounts = context.has_permission(None, Permission::ACCOUNT_MANAGE_ALL);
+        let authorized_ids: Option<HashSet<u64>> = if has_all_accounts {
+            None
+        } else {
+            Some(context.user.account_access_map.keys().cloned().collect())
+        };
+
+        let mut stat = ENVELOPE_MANAGER.get_dashboard_stats(&authorized_ids)?;
+
+        stat.top_largest_emails = ENVELOPE_MANAGER.top_10_largest_emails(&authorized_ids)?;
+        stat.top_largest_attachments =
+            ATTACHMENT_MANAGER.top_10_largest_attachments(&authorized_ids)?;
+
+        stat.account_count = if has_all_accounts {
+            AccountModel::count()?
+        } else {
+            authorized_ids.as_ref().map(|ids| ids.len()).unwrap_or(0)
+        };
+
+        stat.email_count = ENVELOPE_MANAGER.total_emails(&authorized_ids)?;
+        stat.attachment_count = ATTACHMENT_MANAGER.total_attachments(&authorized_ids)?;
+        stat.storage_usage_bytes = get_total_size(&DATA_DIR_MANAGER.storage_dir)
+            .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+        stat.index_usage_bytes = get_total_size(&&DATA_DIR_MANAGER.envelope_dir)
+            .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+        stat.system_version = bichon_version!().to_string();
+
+        Ok(stat)
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct TimeBucket {
+    pub timestamp_ms: i64, // Timestamp in milliseconds
+    pub count: u64,        // Number of emails in this time bucket
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct Group {
+    pub key: String,
+    pub count: u64, // Number of emails from this sender
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct LargestEmail {
+    pub subject: String, // Email subject
+    pub size_bytes: u64, // Email size in bytes
+    pub id: String,
+}
+
+impl LargestEmail {
+    pub fn from_tantivy_doc(document: &TantivyDocument) -> BichonResult<Self> {
+        let fields = SchemaTools::email_fields();
+        let value = document.get_first(fields.f_size).ok_or_else(|| {
+            raise_error!(
+                "miss 'size' field in tantivy document".into(),
+                ErrorCode::InternalError
+            )
+        })?;
+        let size_bytes = value.as_u64().ok_or_else(|| {
+            raise_error!("'size' field is not a u64".into(), ErrorCode::InternalError)
+        })?;
+        let value = document.get_first(fields.f_subject).ok_or_else(|| {
+            raise_error!("'subject' field not found".into(), ErrorCode::InternalError)
+        })?;
+        let subject = value.as_str().map(|s| s.to_string()).ok_or_else(|| {
+            raise_error!(
+                "'subject' field is not a string".into(),
+                ErrorCode::InternalError
+            )
+        })?;
+
+        let value = document.get_first(fields.f_id).ok_or_else(|| {
+            raise_error!(
+                format!("'{}' field not found", F_ID),
+                ErrorCode::InternalError
+            )
+        })?;
+        let id = value.as_str().map(|s| s.to_string()).ok_or_else(|| {
+            raise_error!(
+                format!("'{}' field is not a string", F_ID),
+                ErrorCode::InternalError
+            )
+        })?;
+
+        let envelope = LargestEmail {
+            subject,
+            size_bytes,
+            id,
+        };
+
+        Ok(envelope)
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct LargestAttachment {
+    pub name: String,    // Attachment name
+    pub size_bytes: u64, // Attachment size in bytes
+    pub id: String,
+    pub content_hash: String,
+}
+
+impl LargestAttachment {
+    pub fn from_tantivy_doc(document: &TantivyDocument) -> BichonResult<Self> {
+        let fields = SchemaTools::attachment_fields();
+        let value = document.get_first(fields.f_size).ok_or_else(|| {
+            raise_error!(
+                "miss 'size' field in tantivy document".into(),
+                ErrorCode::InternalError
+            )
+        })?;
+        let size_bytes = value.as_u64().ok_or_else(|| {
+            raise_error!("'size' field is not a u64".into(), ErrorCode::InternalError)
+        })?;
+        let name = document
+            .get_first(fields.f_name_exact)
+            .and_then(|v| v.as_str())
+            .map(|s| s.to_string())
+            .unwrap_or_else(|| "Unknown".to_string());
+
+        let value = document.get_first(fields.f_id).ok_or_else(|| {
+            raise_error!(
+                format!("'{}' field not found", F_ID),
+                ErrorCode::InternalError
+            )
+        })?;
+        let id = value.as_str().map(|s| s.to_string()).ok_or_else(|| {
+            raise_error!(
+                format!("'{}' field is not a string", F_ID),
+                ErrorCode::InternalError
+            )
+        })?;
+
+        let value = document.get_first(fields.f_content_hash).ok_or_else(|| {
+            raise_error!(
+                format!("'{}' field not found", F_CONTENT_HASH),
+                ErrorCode::InternalError
+            )
+        })?;
+        let content_hash = value.as_str().map(|s| s.to_string()).ok_or_else(|| {
+            raise_error!(
+                format!("'{}' field is not a string", F_CONTENT_HASH),
+                ErrorCode::InternalError
+            )
+        })?;
+
+        let attachment = LargestAttachment {
+            name,
+            size_bytes,
+            id,
+            content_hash,
+        };
+
+        Ok(attachment)
+    }
+}

crates/core/src/database/manager.rs

@@ -0,0 +1,60 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::settings::dir::DATA_DIR_MANAGER;
+use bichon_memdb::{Durability, MemDb};
+use std::sync::LazyLock;
+use std::time::Duration;
+
+pub static DB_MANAGER: LazyLock<DatabaseManager> = LazyLock::new(DatabaseManager::new);
+
+pub struct DatabaseManager {
+    db: MemDb,
+}
+
+impl DatabaseManager {
+    fn new() -> Self {
+        let db_path = &DATA_DIR_MANAGER.memdb_dir;
+        std::fs::create_dir_all(db_path).expect("Failed to create memdb data directory");
+
+        let db = MemDb::open_with(db_path, Durability::Batch { max_ops: 100 })
+            .expect("Failed to open memdb database");
+
+        // Start periodic snapshot worker (every 5 minutes)
+        db.start_snapshot_worker(Duration::from_secs(300));
+
+        // Start periodic flush worker (every 10 seconds) so buffered writes
+        // are flushed regularly and not only at the batch threshold.
+        db.start_flush_worker(Duration::from_secs(10));
+
+        DatabaseManager { db }
+    }
+
+    /// Get a reference to the MemDb instance.
+    pub fn db(&self) -> &MemDb {
+        &self.db
+    }
+
+    /// Flush any buffered WAL entries to disk. Must be called before shutdown
+    /// to avoid losing writes that haven't hit the batch threshold yet.
+    pub fn flush(&self) {
+        if let Err(e) = self.db.flush() {
+            eprintln!("[memdb] flush error on shutdown: {e}");
+        }
+    }
+}

crates/core/src/database/mod.rs

@@ -0,0 +1,225 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::common::paginated::Paginated;
+use crate::error::code::ErrorCode;
+use crate::error::BichonResult;
+use crate::raise_error;
+use bichon_memdb::{MemDb, Transaction};
+use serde::de::DeserializeOwned;
+use serde::Serialize;
+
+pub mod manager;
+
+/// Trait for models that can be stored in MemDb collections.
+pub trait MemDbModel: Serialize + DeserializeOwned + Clone + Send + 'static {
+    /// The collection name this model is stored under.
+    fn collection() -> &'static str;
+    /// The primary key as a string for MemDb storage.
+    fn key(&self) -> String;
+}
+
+// ─── Insert ───────────────────────────────────────────────────────────────
+
+pub fn insert_impl<M: MemDbModel>(db: &MemDb, item: M) -> BichonResult<()> {
+    let coll = db.collection(M::collection());
+    let key = item.key();
+    coll.insert(key, &item)
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
+}
+
+pub fn batch_insert_impl<M: MemDbModel>(db: &MemDb, items: Vec<M>) -> BichonResult<()> {
+    let txn = db.transaction();
+    let mut txn = txn;
+    for item in &items {
+        txn = txn
+            .insert(M::collection(), item.key(), item)
+            .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+    }
+    txn.commit()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
+}
+
+// ─── Upsert ────────────────────────────────────────────────────────────────
+
+pub fn upsert_impl<M: MemDbModel>(db: &MemDb, item: M) -> BichonResult<()> {
+    let coll = db.collection(M::collection());
+    coll.upsert(item.key(), &item)
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
+}
+
+pub fn batch_upsert_impl<M: MemDbModel>(db: &MemDb, items: Vec<M>) -> BichonResult<()> {
+    let txn = db.transaction();
+    let mut txn = txn;
+    for item in &items {
+        txn = txn
+            .upsert(M::collection(), item.key(), item)
+            .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+    }
+    txn.commit()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
+}
+
+// ─── Find ──────────────────────────────────────────────────────────────────
+
+pub fn find_impl<M: MemDbModel>(db: &MemDb, key: &str) -> BichonResult<Option<M>> {
+    let coll = db.collection(M::collection());
+    coll.get(key)
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
+}
+
+// ─── Filter (replaces secondary key queries) ──────────────────────────────
+
+pub fn filter_impl<M, F>(db: &MemDb, predicate: F) -> BichonResult<Vec<M>>
+where
+    M: MemDbModel,
+    F: Fn(&M) -> bool + Send + 'static,
+{
+    let coll = db.collection(M::collection());
+    coll.filter(predicate)
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
+}
+
+// ─── Update (read-modify-write under a single spawn_blocking) ─────────────
+
+pub fn update_impl<M: MemDbModel>(
+    db: &MemDb,
+    key: &str,
+    update_fn: impl FnOnce(M) -> BichonResult<M> + Send + 'static,
+) -> BichonResult<M> {
+    let coll = db.collection(M::collection());
+    let current: M = coll
+        .get_required(key)
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+    let updated = update_fn(current)?;
+    coll.upsert(key, &updated)
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+    Ok(updated)
+}
+
+// ─── Delete ────────────────────────────────────────────────────────────────
+
+pub fn delete_impl<M: MemDbModel>(db: &MemDb, key: &str) -> BichonResult<()> {
+    let coll = db.collection(M::collection());
+    let existed = coll
+        .delete(key)
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+    if !existed {
+        return Err(raise_error!(
+            format!("{} '{}' not found for deletion", M::collection(), key),
+            ErrorCode::ResourceNotFound
+        ));
+    }
+    Ok(())
+}
+
+pub fn batch_delete_impl<M: MemDbModel>(db: &MemDb, keys: Vec<String>) -> BichonResult<usize> {
+    let txn = db.transaction();
+    let mut txn = txn;
+    let mut count = 0usize;
+    for key in &keys {
+        txn = txn.delete(M::collection(), key.clone());
+        count += 1;
+    }
+    txn.commit()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+    Ok(count)
+}
+
+// ─── List / Count ──────────────────────────────────────────────────────────
+
+pub fn list_all_impl<M: MemDbModel>(db: &MemDb) -> BichonResult<Vec<M>> {
+    let coll = db.collection(M::collection());
+    coll.list_all()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
+}
+
+pub fn count_impl<M: MemDbModel>(db: &MemDb) -> BichonResult<usize> {
+    let coll = db.collection(M::collection());
+    Ok(coll.count())
+}
+
+// ─── Paginate ──────────────────────────────────────────────────────────────
+
+pub fn paginate_impl<M: MemDbModel>(
+    db: &MemDb,
+    page: Option<u64>,
+    page_size: Option<u64>,
+    desc: Option<bool>,
+) -> BichonResult<Paginated<M>> {
+    let coll = db.collection(M::collection());
+    let total_items = coll.count() as u64;
+
+    let (offset, total_pages) = match (page, page_size) {
+        (Some(p), Some(s)) if p > 0 && s > 0 => {
+            let offset = (p - 1) * s;
+            let total_pages = if total_items > 0 {
+                (total_items as f64 / s as f64).ceil() as u64
+            } else {
+                0
+            };
+            (Some(offset), Some(total_pages))
+        }
+        (Some(0), _) | (_, Some(0)) => {
+            return Err(raise_error!(
+                "'page' and 'page_size' must be greater than 0.".into(),
+                ErrorCode::InvalidParameter
+            ));
+        }
+        _ => (None, None),
+    };
+
+    let all: Vec<M> = coll
+        .list_all()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+    let items: Vec<M> = match desc {
+        Some(true) => {
+            let iter: Vec<M> = all.into_iter().rev().collect();
+            let skip = offset.unwrap_or(0) as usize;
+            let take = page_size.unwrap_or(total_items) as usize;
+            iter.into_iter().skip(skip).take(take).collect()
+        }
+        _ => {
+            let skip = offset.unwrap_or(0) as usize;
+            let take = page_size.unwrap_or(total_items) as usize;
+            all.into_iter().skip(skip).take(take).collect()
+        }
+    };
+
+    Ok(Paginated::new(
+        page,
+        page_size,
+        total_items,
+        total_pages,
+        items,
+    ))
+}
+
+// ─── Transaction ───────────────────────────────────────────────────────────
+
+/// Execute operations within a single atomic transaction (one WAL entry).
+pub fn with_transaction(
+    db: &MemDb,
+    f: impl FnOnce(Transaction) -> BichonResult<Transaction> + Send + 'static,
+) -> BichonResult<()> {
+    let txn = db.transaction();
+    let txn = f(txn)?;
+    txn.commit()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
+}

crates/core/src/envelope/extractor.rs

@@ -0,0 +1,902 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::account::migration::AccountModel;
+use crate::cache::imap::mailbox::MailBox;
+use crate::common::AddrVec;
+use crate::envelope::meta::parse_bichon_metadata;
+use crate::envelope::utils::normalize_subject;
+use crate::error::code::ErrorCode;
+use crate::error::BichonResult;
+use crate::imap::executor::ImapExecutor;
+use crate::message::content::AttachmentInfo;
+use crate::store::blob::{DetachedEmail, BLOB_MANAGER};
+use crate::store::tantivy::attachment::ATTACHMENT_MANAGER;
+use crate::store::tantivy::dedup_cache::DEDUP_CACHE;
+use crate::store::tantivy::envelope::ENVELOPE_MANAGER;
+use crate::store::tantivy::model::{AttachmentModel, EnvelopeWithAttachments};
+use crate::utils::html::extract_text;
+use crate::utils::{compute_content_hash, hex_hash};
+use crate::{id, store::envelope::Envelope};
+use crate::{raise_error, utc_now};
+use async_imap::types::Fetch;
+use bytes::Bytes;
+use mail_parser::{Address, HeaderName, Message, MessageParser, MimeHeaders};
+use tantivy::TantivyDocument;
+use tantivy::schema::Facet;
+use tracing::error;
+use uuid::Uuid;
+
+pub async fn extract_envelope_and_store_it(
+    fetch: Fetch,
+    account_id: u64,
+    mailbox_id: u64,
+) -> BichonResult<()> {
+    let internal_date = fetch
+        .internal_date()
+        .map(|d| d.timestamp_millis())
+        .unwrap_or(0);
+    let uid = fetch.uid.unwrap_or(0);
+    let body = match fetch.body() {
+        Some(b) => b,
+        None => {
+            tracing::warn!(
+                account_id,
+                uid = fetch.uid,
+                "FETCH response has no body, skipping message"
+            );
+            return Ok(());
+        }
+    };
+    let size = fetch.size.unwrap_or(body.len() as u32);
+    extract_envelope_core(body, uid, size, internal_date, account_id, mailbox_id).await
+}
+
+pub async fn extract_envelope_from_eml(
+    body: &[u8],
+    account_id: u64,
+    mailbox_id: u64,
+) -> BichonResult<()> {
+    extract_envelope_core(body, 0, body.len() as u32, 0, account_id, mailbox_id).await
+}
+
+pub async fn extract_envelope_from_smtp(
+    body: &[u8],
+    account_id: u64,
+    mailbox_id: u64,
+) -> BichonResult<()> {
+    extract_envelope_core(
+        body,
+        0,
+        body.len() as u32,
+        utc_now!(),
+        account_id,
+        mailbox_id,
+    )
+    .await
+}
+
+async fn extract_envelope_core(
+    body: &[u8],
+    uid: u32,
+    size: u32,
+    internal_date: i64,
+    account_id: u64,
+    mailbox_id: u64,
+) -> BichonResult<()> {
+    //The content hash of the original raw EML
+    let email_content_hash = compute_content_hash(body);
+    if DEDUP_CACHE.contains(account_id, mailbox_id, &email_content_hash) {
+        tracing::debug!("Duplicate email detected");
+        //println!("Duplicate email detected");
+        return Ok(());
+    }
+    let message: Message<'_> = MessageParser::new().parse(body).ok_or_else(|| {
+        raise_error!(
+            "Email header parse result is not available".into(),
+            ErrorCode::InternalError
+        )
+    })?;
+
+    if let Ok(account) = AccountModel::get(account_id) {
+        if let Some(ref rules) = account.archive_rules {
+            let sender = message.from().and_then(|addr| {
+                AddrVec::from(addr).0.into_iter().next().and_then(|a| a.address)
+            });
+            let subject = message.subject().map(|s| s.to_string());
+
+            let is_spam = !rules.spam_headers.is_empty()
+                && rules.spam_headers.iter().any(|h| {
+                    message
+                        .header_raw(h.clone())
+                        .map(|v| matches!(v.trim().to_lowercase().as_str(), "yes" | "true"))
+                        .unwrap_or(false)
+                });
+
+            if !rules.should_archive(sender.as_deref(), subject.as_deref(), size, is_spam) {
+                tracing::debug!(
+                    account_id,
+                    uid,
+                    sender = sender.as_deref().unwrap_or("?"),
+                    subject = subject.as_deref().unwrap_or("?"),
+                    "Email filtered out by archive rules"
+                );
+                return Ok(());
+            }
+        }
+    }
+
+    let preview_limit = 100;
+    let text = if let Some(text) = message.body_text(0).map(|cow| cow.into_owned()) {
+        text
+    } else if let Some(html) = message.body_html(0).map(|cow| cow.into_owned()) {
+        extract_text(html)
+    } else {
+        String::new()
+    };
+
+    let text = text.split_whitespace().collect::<Vec<_>>().join(" ");
+
+    let preview = if text.chars().count() > preview_limit {
+        text.chars().take(preview_limit).collect::<String>() + "..."
+    } else {
+        text.clone()
+    };
+
+    let body_text = text;
+
+    let message_id = message
+        .message_id()
+        .map(String::from)
+        .unwrap_or_else(generate_message_id);
+
+    let in_reply_to = message.in_reply_to().as_text().map(String::from);
+    let references = extract_references(&message);
+    let thread_id = compute_thread_id(in_reply_to, references, &message_id);
+
+    let mut subject = message.subject().map(String::from).unwrap_or_default();
+    if subject.contains('\u{FFFD}') {
+        subject = normalize_subject(message.header_raw(HeaderName::Subject));
+    }
+
+    let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0);
+    let internal_date = if internal_date == 0 {
+        date
+    } else {
+        internal_date
+    };
+    let parse_addrs = |addrs: Option<&Address<'_>>| {
+        addrs
+            .map(|addr| {
+                AddrVec::from(addr)
+                    .0
+                    .into_iter()
+                    .filter_map(|a| a.address)
+                    .collect()
+            })
+            .unwrap_or_default()
+    };
+
+    let bcc = parse_addrs(message.bcc());
+    let cc = parse_addrs(message.cc());
+    let to = parse_addrs(message.to());
+
+    let from = message
+        .from()
+        .and_then(|addr| AddrVec::from(addr).0.into_iter().next())
+        .and_then(|add| add.address)
+        .unwrap_or_else(|| "unknown".to_string());
+    let attachment_count = message.attachment_count();
+    let attachments = detach_and_store_attachments(body, &message, &email_content_hash, account_id, mailbox_id).await;
+
+    let envelope_id = Uuid::new_v4().to_string();
+    let now = utc_now!();
+
+
+    let mut final_tags = Vec::new();
+
+    if let Some(meta_header) = message.header_raw("X-Bichon-Metadata") {
+        if let Some(bmd) = parse_bichon_metadata(meta_header) {
+            if let Some(tags) = bmd.tags {
+                let validated_tags: Result<Vec<String>, _> = tags
+                    .iter()
+                    .map(|tag| {
+                        Facet::from_text(tag)
+                            .map(|_| tag.clone()) 
+                            .map_err(|e| e)
+                    })
+                    .collect();
+
+                match validated_tags {
+                    Ok(valid_list) => {
+                        final_tags = valid_list;
+                    }
+                    Err(e) => {
+                        eprintln!(
+                            "Tag validation failed, ignoring all tags: {:#?}",
+                            e
+                        );
+                    }
+                }
+            }
+        }
+    }
+
+    let attachment_docs: Vec<TantivyDocument> = attachments
+        .iter()
+        .filter(|a| !a.inline || a.content_id.is_none())
+        .map(|a| {
+            let has_text = a.extracted_text.is_some();
+            AttachmentModel {
+                id: Uuid::new_v4().to_string(),
+                envelope_id: envelope_id.clone(),
+                account_id,
+                account_email: None,
+                mailbox_id,
+                mailbox_name: None,
+                subject: subject.clone(),
+                content_hash: a.content_hash.clone(),
+                from: from.clone(),
+                date,
+                ingest_at: now,
+                size: a.size as u64,
+                ext: a.get_extension(),
+                category: a.get_category().to_string(),
+                content_type: a.file_type.clone(),
+                shard_id: 0,
+                text: a.extracted_text.clone(),
+                has_text,
+                is_ocr: a.extracted_is_ocr,
+                page_count: a.extracted_page_count.map(|n| n as u64),
+                is_indexed: has_text,
+                is_message: a.is_message,
+                name: a.filename.clone(),
+                tags: None,
+                auto_tags: None,
+            }
+        })
+        .map(|a| a.into_document())
+        .collect();
+
+    let envelope = Envelope {
+        id: envelope_id,
+        message_id,
+        account_id,
+        mailbox_id,
+        uid,
+        subject,
+        preview,
+        from,
+        to,
+        cc,
+        bcc,
+        date,
+        internal_date,
+        ingest_at: now,
+        size,
+        thread_id,
+        attachment_count,
+        regular_attachment_count: attachment_docs.len(),
+        tags: (!final_tags.is_empty()).then_some(final_tags),
+        account_email: None,
+        mailbox_name: None,
+        content_hash: email_content_hash.clone(),
+        account_name: None,
+    };
+    // 'attachments' contains both regular and inline attachments
+    let ea = EnvelopeWithAttachments {
+        envelope,
+        attachments: Some(attachments),
+    };
+    let doc = ea.to_document(&body_text, 0)?;
+    tracing::debug!(
+        "[account {}][mailbox {}] extract: uid={} msg_id={} content_hash={}",
+        account_id,
+        mailbox_id,
+        uid,
+        &ea.envelope.message_id,
+        &ea.envelope.content_hash,
+    );
+    ENVELOPE_MANAGER.queue(doc).await;
+    DEDUP_CACHE.insert(account_id, mailbox_id, &email_content_hash);
+    for doc in attachment_docs {
+        ATTACHMENT_MANAGER.queue(doc).await;
+    }
+    Ok(())
+}
+
+pub fn extract_envelope_from_nested_message(
+    message: Message<'_>,
+    account_id: u64,
+) -> BichonResult<Envelope> {
+    let text = if let Some(text) = message.body_text(0).map(|cow| cow.into_owned()) {
+        text
+    } else if let Some(html) = message.body_html(0).map(|cow| cow.into_owned()) {
+        extract_text(html)
+    } else {
+        String::new()
+    };
+
+    let message_id = message
+        .message_id()
+        .map(String::from)
+        .unwrap_or_else(generate_message_id);
+
+    let in_reply_to = message.in_reply_to().as_text().map(String::from);
+    let references = extract_references(&message);
+    let thread_id = compute_thread_id(in_reply_to, references, &message_id);
+
+    let mut subject = message.subject().map(String::from).unwrap_or_default();
+    if subject.contains('\u{FFFD}') {
+        subject = normalize_subject(message.header_raw(HeaderName::Subject));
+    }
+
+    let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0);
+
+    let parse_addrs = |addrs: Option<&Address<'_>>| {
+        addrs
+            .map(|addr| {
+                AddrVec::from(addr)
+                    .0
+                    .into_iter()
+                    .filter_map(|a| a.address)
+                    .collect()
+            })
+            .unwrap_or_default()
+    };
+
+    let bcc = parse_addrs(message.bcc());
+    let cc = parse_addrs(message.cc());
+    let to = parse_addrs(message.to());
+
+    let from = message
+        .from()
+        .and_then(|addr| AddrVec::from(addr).0.into_iter().next())
+        .and_then(|add| add.address)
+        .unwrap_or_else(|| "unknown".to_string());
+
+    let envelope = Envelope {
+        id: Default::default(),
+        message_id,
+        account_id,
+        mailbox_id: Default::default(),
+        uid: Default::default(),
+        subject,
+        preview: text,
+        from,
+        to,
+        cc,
+        bcc,
+        date,
+        internal_date: Default::default(),
+        ingest_at: Default::default(),
+        size: Default::default(),
+        thread_id,
+        attachment_count: Default::default(),
+        regular_attachment_count: Default::default(),
+        tags: Default::default(),
+        account_email: Default::default(),
+        account_name: Default::default(),
+        mailbox_name: Default::default(),
+        content_hash: Default::default(),
+    };
+
+    Ok(envelope)
+}
+
+pub fn compute_thread_id(
+    in_reply_to: Option<String>,
+    references: Option<Vec<String>>,
+    message_id: &str,
+) -> String {
+    if in_reply_to.is_some() && references.as_ref().map_or(false, |r| !r.is_empty()) {
+        return hex_hash(&references.as_ref().unwrap()[0]);
+    }
+    hex_hash(message_id)
+}
+
+pub fn generate_message_id() -> String {
+    let ts = utc_now!();
+    let pid = std::process::id();
+    format!("<{:016x}.{}.{}@{}>", id!(128), ts, pid, "bichon")
+}
+
+pub fn extract_references(message: &Message<'_>) -> Option<Vec<String>> {
+    match message.references() {
+        mail_parser::HeaderValue::Text(cow) => Some(vec![cow.to_string()]),
+        mail_parser::HeaderValue::TextList(vec) => {
+            Some(vec.iter().map(|cow| cow.to_string()).collect())
+        }
+        _ => None,
+    }
+}
+
+pub async fn detach_and_store_attachments(
+    original_body: &[u8],
+    message: &Message<'_>,
+    eml_content_hash: &str,
+    account_id: u64,
+    mailbox_id: u64,
+) -> Vec<AttachmentInfo> {
+    let rules = if account_id > 0 {
+        AccountModel::get(account_id)
+            .ok()
+            .and_then(|a| a.extraction_rules)
+    } else {
+        None
+    };
+
+    let mailbox_name = match rules.as_ref().map(|r| !r.folders.is_empty()) {
+        Some(true) => MailBox::get(mailbox_id).ok().map(|mb| mb.name),
+        _ => None,
+    };
+
+    let sender = message
+        .from()
+        .and_then(|addr| AddrVec::from(addr).0.into_iter().next())
+        .and_then(|add| add.address);
+
+    let mut stripped_eml = original_body.to_vec();
+    let mut attachment_infos = Vec::new();
+    // Step 1: Collect and sort attachment ranges in reverse to maintain offset integrity
+    let mut ranges: Vec<_> = message
+        .attachments()
+        .map(|att| {
+            (
+                att.raw_body_offset() as usize,
+                att.raw_end_offset() as usize,
+                att,
+            )
+        })
+        .collect();
+
+    ranges.sort_by(|a, b| b.0.cmp(&a.0));
+    let mut attachments = Vec::with_capacity(ranges.len());
+
+    // Collect candidates for text extraction (non-inline, known document types).
+    struct TextCandidate {
+        content_hash: String,
+        file_type: String,
+        ext: String,
+        bytes: Vec<u8>,
+    }
+    let mut text_candidates: Vec<TextCandidate> = Vec::new();
+
+    for (raw_start, raw_end, att) in ranges {
+        // mail-parser may report attachment offsets past the body end for
+        // malformed messages; clamp the range to avoid a slice panic.
+        let body_len = original_body.len();
+        let raw_start = raw_start.min(body_len);
+        let raw_end = raw_end.min(body_len);
+        let range_valid = raw_start < raw_end;
+
+        // content hash is computed from the decoded attachment contents,
+        // which is always available regardless of raw offset validity.
+        let content_hash = compute_content_hash(att.contents());
+
+        if range_valid {
+            let raw_bytes = &original_body[raw_start..raw_end];
+            // The actual content stored in the blob is the raw undecoded data.
+            attachments.push((content_hash.clone(), Bytes::copy_from_slice(raw_bytes)));
+
+            // Replace raw attachment content with a hash-based placeholder
+            let placeholder = format!("<<BICHON_DETACH_HASH:{}>>", &content_hash);
+            stripped_eml.splice(raw_start..raw_end, placeholder.as_bytes().iter().cloned());
+        } else {
+            // Invalid range: store a zero-length blob so the consistency
+            // check passes; reattachment will log a warning for the missing
+            // blob data but won't panic.
+            attachments.push((content_hash.clone(), Bytes::new()));
+        }
+
+        let inline = att
+            .content_disposition()
+            .map(|d| d.is_inline())
+            .unwrap_or_else(|| att.content_id().is_some());
+        let file_type = att
+            .content_type()
+            .map(|ct| {
+                format!(
+                    "{}/{}",
+                    ct.c_type.as_ref(),
+                    ct.c_subtype.as_deref().unwrap_or("")
+                )
+            })
+            .unwrap_or_else(|| "application/octet-stream".to_string());
+        let has_cid = att.content_id().is_some();
+        let att_name = att.attachment_name().map(|n| n.to_string());
+        let ext = att_name
+            .as_deref()
+            .and_then(|n| {
+                std::path::Path::new(n)
+                    .extension()
+                    .and_then(|e| e.to_str())
+                    .map(|s| s.to_ascii_lowercase())
+            })
+            .unwrap_or_default();
+
+        let should_extract = rules.as_ref().map_or(true, |r| {
+            r.should_extract(
+                &ext,
+                mailbox_name.as_deref(),
+                att_name.as_deref(),
+                sender.as_deref(),
+            )
+        });
+
+        if !inline || !has_cid {
+            let decoded_len = att.contents().len();
+            if should_extract
+                && decoded_len <= crate::ext::text_extractor::MAX_EXTRACT_BYTES
+                && crate::ext::text_extractor::should_try_extract(&file_type, &ext)
+            {
+                text_candidates.push(TextCandidate {
+                    content_hash: content_hash.clone(),
+                    file_type: file_type.clone(),
+                    ext: ext.clone(),
+                    bytes: att.contents().to_vec(),
+                });
+            }
+        }
+
+        let info = AttachmentInfo {
+            filename: att.attachment_name().map(|n| n.to_string()),
+            size: att.contents().len(),
+            inline,
+            file_type,
+            content_id: att.content_id().map(|id| id.to_string()),
+            content_hash: content_hash.clone(),
+            is_message: att.is_message(),
+            extracted_text: None,
+            extracted_page_count: None,
+            extracted_is_ocr: false,
+        };
+
+        attachment_infos.push(info);
+    }
+
+    // Run text extraction in a single spawn_blocking batch.
+    if !text_candidates.is_empty() {
+        if let Ok(mut extracted_map) = tokio::task::spawn_blocking(move || {
+            let mut map: std::collections::HashMap<
+                String,
+                (String, Option<u32>, bool),
+            > = std::collections::HashMap::new();
+            for c in text_candidates {
+                if let Some(r) =
+                    crate::ext::text_extractor::extract_text(&c.file_type, &c.ext, &c.bytes)
+                {
+                    map.insert(c.content_hash, (r.text, r.page_count, r.is_ocr));
+                }
+            }
+            map
+        })
+        .await
+        {
+            for info in &mut attachment_infos {
+                if let Some((text, pages, is_ocr)) = extracted_map.remove(&info.content_hash) {
+                    info.extracted_text = Some(text);
+                    info.extracted_page_count = pages;
+                    info.extracted_is_ocr = is_ocr;
+                }
+            }
+        }
+    }
+    // Step 4: Store the final stripped EML content
+    BLOB_MANAGER
+        .queue(DetachedEmail {
+            email: (eml_content_hash.to_string(), Bytes::from(stripped_eml)),
+            attachments: Some(attachments),
+        })
+        .await;
+
+    attachment_infos
+}
+
+pub fn reattach_eml_content(
+    account_id: u64,
+    envelope_id: String,
+) -> BichonResult<(Envelope, Bytes)> {
+    let e = ENVELOPE_MANAGER
+        .get_envelope_by_id(account_id, &envelope_id)
+        ?
+        .ok_or_else(|| {
+            raise_error!(
+                format!(
+                    "Envelope not found: account_id={} envelope_id={}",
+                    account_id, &envelope_id
+                ),
+                ErrorCode::ResourceNotFound
+            )
+        })?;
+
+    let restored_eml = BLOB_MANAGER
+        .get_email(&e.envelope.content_hash)?
+        .ok_or_else(|| {
+            raise_error!(
+                format!(
+                "Original email content not found: account_id={} envelope_id={} content_hash={}",
+                account_id, &envelope_id, &e.envelope.content_hash
+            ),
+                ErrorCode::ResourceNotFound
+            )
+        })?;
+
+    if !e.envelope.has_any_attachments() {
+        return Ok((e.envelope, restored_eml));
+    }
+
+    let mut restored_eml = restored_eml.to_vec();
+    let actual_count = e.attachments.as_ref().map(|a| a.len()).unwrap_or(0);
+    if e.envelope.attachment_count != actual_count {
+        return Err(raise_error!(
+            format!(
+                "Consistency check failed: envelope.attachment_count ({}) does not match attachments.len ({})",
+                e.envelope.attachment_count, 
+                actual_count
+            ),
+            ErrorCode::InternalError
+        ));
+    }
+
+    let mut tasks = Vec::new();
+    for detail in e.attachments.unwrap() {
+        let placeholder_str = format!("<<BICHON_DETACH_HASH:{}>>", &detail.content_hash);
+        let pattern = placeholder_str.as_bytes();
+        let pattern_len = pattern.len();
+
+        let mut search_cursor = 0;
+        while let Some(pos) = restored_eml[search_cursor..]
+            .windows(pattern_len)
+            .position(|window| window == pattern)
+        {
+            let absolute_start = search_cursor + pos;
+            let absolute_end = absolute_start + pattern_len;
+
+            tasks.push((
+                absolute_start,
+                absolute_end,
+                detail.content_hash.clone(),
+            ));
+            search_cursor = absolute_end;
+        }
+    }
+
+    tasks.sort_by(|a, b| b.0.cmp(&a.0));
+
+    for (start, end, hash) in tasks {
+        if let Some(original_data) = BLOB_MANAGER.get_attachment(&hash)? {
+            restored_eml.splice(start..end, original_data.iter().cloned());
+        } else {
+            error!("[ERROR] Missing attachment blob for hash: {}", hash);
+        }
+    }
+
+    Ok((e.envelope, Bytes::from(restored_eml)))
+}
+
+/// Returns the raw EML for an indexed message, self-healing a missing content blob.
+///
+/// Behaves like [`reattach_eml_content`], but when the message's content blob is
+/// absent from the blob store it fetches that single message on demand from the
+/// IMAP server (`UID FETCH <uid> (BODY.PEEK[])`), persists it for future requests,
+/// and returns it. If the on-demand fetch itself fails, the original "content not
+/// found" error from [`reattach_eml_content`] is surfaced unchanged so the caller
+/// still produces its 404.
+pub async fn reattach_eml_content_self_healing(
+    account_id: u64,
+    envelope_id: String,
+) -> BichonResult<(Envelope, Bytes)> {
+    let envelope = ENVELOPE_MANAGER
+        .get_envelope_by_id(account_id, &envelope_id)?
+        .ok_or_else(|| {
+            raise_error!(
+                format!(
+                    "Envelope not found: account_id={} envelope_id={}",
+                    account_id, &envelope_id
+                ),
+                ErrorCode::ResourceNotFound
+            )
+        })?
+        .envelope;
+
+    // Fast path: the content blob is present, reuse the regular reattach logic.
+    if BLOB_MANAGER.get_email(&envelope.content_hash)?.is_some() {
+        return reattach_eml_content(account_id, envelope_id);
+    }
+
+    // The blob is missing. Try to recover it directly from the IMAP server.
+    match recover_message_blob(&envelope).await {
+        Ok(raw_body) => {
+            tracing::info!(
+                account_id,
+                envelope_id = %envelope_id,
+                uid = envelope.uid,
+                "Self-healed missing email content blob via on-demand IMAP fetch"
+            );
+            Ok((envelope, raw_body))
+        }
+        Err(e) => {
+            tracing::warn!(
+                account_id,
+                envelope_id = %envelope_id,
+                uid = envelope.uid,
+                error = %e,
+                "On-demand IMAP fetch for missing content blob failed; returning not-found"
+            );
+            Err(e)
+        }
+    }
+}
+
+/// Fetches one message from IMAP and re-stores its detached blob.
+///
+/// On success the freshly fetched raw RFC822 body is returned; it is also queued
+/// (in detached form) into the blob store so subsequent requests hit the cache.
+/// Fails if the message cannot be fetched, or if the fetched bytes do not match
+/// the archived `content_hash` (the server-side message no longer matches what
+/// Bichon archived, so it cannot be treated as a recovery of that blob).
+async fn recover_message_blob(envelope: &Envelope) -> BichonResult<Bytes> {
+    let mailbox = MailBox::find_mailbox(envelope.account_id, envelope.mailbox_id)?
+        .ok_or_else(|| {
+            raise_error!(
+                format!(
+                    "Mailbox not found: account_id={} mailbox_id={}",
+                    envelope.account_id, envelope.mailbox_id
+                ),
+                ErrorCode::ResourceNotFound
+            )
+        })?;
+
+    let mut session = ImapExecutor::create_connection(envelope.account_id).await?;
+    let result = ImapExecutor::fetch_single_message_body(
+        &mut session,
+        &mailbox.encoded_name(),
+        envelope.uid,
+    )
+    .await;
+    session.logout().await.ok();
+    let raw_body = result?;
+
+    let fetched_hash = compute_content_hash(&raw_body);
+    if fetched_hash != envelope.content_hash {
+        return Err(raise_error!(
+            format!(
+                "Fetched message does not match archived content: expected content_hash={} got={}",
+                envelope.content_hash, fetched_hash
+            ),
+            ErrorCode::ImapUnexpectedResult
+        ));
+    }
+
+    // Re-create the detached blob (stripped EML + attachments) so the missing
+    // blob is repopulated for future requests. The detached EML is queued under
+    // `fetched_hash`, which equals `envelope.content_hash`.
+    let message = MessageParser::new().parse(raw_body.as_slice()).ok_or_else(|| {
+        raise_error!(
+            "Failed to parse fetched email content".into(),
+            ErrorCode::InternalError
+        )
+    })?;
+    detach_and_store_attachments(&raw_body, &message, &fetched_hash, envelope.account_id, envelope.mailbox_id).await;
+
+    Ok(Bytes::from(raw_body))
+}
+
+#[cfg(test)]
+mod test {
+    use html2text::config;
+
+    #[test]
+    fn test_various_html_with_overflow_enabled() {
+        let cases = [
+            ("<p>Hello World</p>", "Simple paragraph"),
+            ("<h1>Title</h1><p>Content</p>", "Heading + paragraph"),
+            ("<ul><li>Item1</li><li>Item2</li></ul>", "Unordered list"),
+            (
+                "<strong>Bold</strong> and <em>italic</em>",
+                "Inline formatting",
+            ),
+            (
+                "<div><span>Nested</span> elements</div>",
+                "Nested inline elements inside block",
+            ),
+            (
+                "<table><tr><td>A</td><td>B</td></tr></table>",
+                "Simple table",
+            ),
+            (
+                "<pre>  preformatted text\n  line2</pre>",
+                "Preformatted block",
+            ),
+            ("😃 emoji test", "Wide emoji"),
+            ("<a href=\"#\">link</a>", "Anchor tag"),
+            (
+                "<blockquote><p>Quoted text</p></blockquote>",
+                "Blockquote with paragraph",
+            ),
+        ];
+
+        for (html, desc) in cases {
+            let result = config::plain()
+                .allow_width_overflow()
+                .string_from_read(html.as_bytes(), 100);
+
+            match result {
+                Ok(output) => {
+                    println!("✓ Rendered ({}) =>\n{}", desc, output);
+                }
+                Err(e) => panic!("Unexpected error for {}: {:?}", desc, e),
+            }
+        }
+    }
+
+    /// Verifies that [`super::detach_and_store_attachments`] does not panic
+    /// when mail-parser reports attachment offsets past the raw body length.
+    ///
+    /// Regression test for: "range end index X out of range for slice of
+    /// length Y" panic caused by a malformed email whose attachment
+    /// `raw_end_offset` exceeded the actual body size.
+    #[tokio::test]
+    async fn detach_attachments_bounds_check() {
+        let raw = concat!(
+            "From: sender@example.com\r\n",
+            "To: recipient@example.com\r\n",
+            "Subject: Test\r\n",
+            "MIME-Version: 1.0\r\n",
+            "Content-Type: multipart/mixed; boundary=\"bnd\"\r\n",
+            "\r\n",
+            "--bnd\r\n",
+            "Content-Type: text/plain\r\n",
+            "\r\n",
+            "Hello\r\n",
+            "--bnd\r\n",
+            "Content-Type: application/octet-stream\r\n",
+            "Content-Disposition: attachment; filename=\"test.bin\"\r\n",
+            "\r\n",
+            "AAAAABBBBBCCCCCDDDDDEEEEEAAAAABBBBBCCCCCDDDDDEEEEE\r\n",
+            "--bnd--\r\n",
+        )
+        .as_bytes()
+        .to_vec();
+
+        let message = mail_parser::MessageParser::new()
+            .parse(&raw)
+            .expect("parse valid MIME message");
+        assert_eq!(message.attachment_count(), 1);
+
+        // Truncate the raw body so the attachment's raw_end_offset lies
+        // past the body end — exactly the scenario reported by users.
+        let truncated = &raw[..raw.len() - 20];
+        assert!(truncated.len() < raw.len());
+
+        // Must not panic.
+        let infos = super::detach_and_store_attachments(
+            truncated,
+            &message,
+            "test_content_hash",
+            0,
+            0,
+        )
+        .await;
+
+        // The attachment count must still match so the consistency check
+        // in reattach_eml_content doesn't fail later.
+        assert_eq!(infos.len(), 1);
+    }
+}