update

fix(core): count deduplicated imports as duplicates, not successes

.github/workflows/release.yml

@@ -4,6 +4,7 @@ on:
   push:
     tags:
       - '[0-9]+.[0-9]+.[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+-*'
 env:
   BINARY_NAME: bichon-server
   BINARY_CLI: bichon-cli

.gitignore

@@ -3,4 +3,5 @@
 .idea
 config.toml
 node_modules
-dedup_report.txt
+dedup_report.txt
+crates/*/target

CLA.md

@@ -0,0 +1,56 @@
+# Contributor License Agreement
+
+Thank you for contributing to Bichon ("the Project"), maintained by rustmailer ("the Maintainer").
+
+This Agreement clarifies the intellectual property rights granted by You to the Maintainer when You submit a Contribution. By signing this Agreement, You accept and agree to the following terms.
+
+## 1. Definitions
+
+- **"You"** means the individual or legal entity (corporation, organization, etc.) on whose behalf the Contribution is submitted.
+- **"Contribution"** means any original work of authorship, including modifications or additions to existing work, that You intentionally submit to the Project (via Pull Request, patch, email, or any other means).
+
+## 2. Copyright License
+
+You grant the Maintainer a **perpetual, worldwide, non-exclusive, royalty-free, irrevocable** copyright license to:
+
+- Reproduce, modify, adapt, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Your Contribution and any derivative works thereof.
+- **Re-license** Your Contribution under different terms, including but not limited to proprietary or commercial licenses, in any medium now known or later developed.
+
+This license survives termination of this Agreement and continues even if You stop contributing.
+
+## 3. Patent License
+
+You grant the Maintainer a **perpetual, worldwide, non-exclusive, royalty-free, irrevocable** patent license to make, have made, use, sell, offer for sale, import, and otherwise transfer Your Contribution, where such license applies only to patent claims licensable by You that are necessarily infringed by Your Contribution alone or in combination with the Project.
+
+## 4. Moral Rights
+
+To the fullest extent permitted by applicable law, You waive, and agree not to assert, any moral rights (including rights of attribution, integrity, or withdrawal) in Your Contribution against the Maintainer or the Maintainer's licensees. If under Your jurisdiction moral rights cannot be waived, You agree not to enforce them against the Maintainer.
+
+## 5. Employer / Entity Disclaimer
+
+If You are employed by or acting on behalf of an entity, You represent that:
+
+- You have received permission from Your employer or entity to submit the Contribution under the terms of this Agreement.
+- Your employer or entity has waived any rights to the Contribution.
+
+If Your employer or entity has not provided such permission, You must not submit a Contribution.
+
+## 6. Representations
+
+You represent and warrant that:
+
+- You are legally entitled to make the Contribution and grant the licenses under this Agreement.
+- Your Contribution is Your original work, and You have the necessary rights from any third parties whose material is included.
+- You are not aware of any claims, pending litigation, or other issues that would affect the rights granted under this Agreement.
+
+## 7. No Obligation
+
+The Maintainer is under no obligation to accept or use Your Contribution. You are under no obligation to make further Contributions.
+
+## 8. Governing Law
+
+This Agreement is governed by the laws of the State of Delaware, United States of America, without regard to its conflict of law principles. Any dispute arising from this Agreement shall be resolved exclusively in the courts of Delaware.
+
+## 9. Signing
+
+You indicate Your acceptance of this Agreement by signing the [cla-assistant](https://github.com/cla-assistant/cla-assistant) prompt when submitting a Pull Request to the Project. Electronic acceptance constitutes a binding signature under applicable law.

Cargo.toml

@@ -3,6 +3,7 @@
 members = [
     "crates/memdb",
     "crates/core",
+    "crates/blob",
     "crates/server",
     "crates/cli",
     "crates/admin",
@@ -12,23 +13,24 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "1.4.2"
+version = "2.0.2"
 edition = "2021"
 
 [workspace.dependencies]
-chrono = "0.4.44"
-clap = { version = "4.6.1", features = ["derive", "env"] }
-memdb = { path = "crates/memdb" }
-itertools = "0.14.0"
+chrono = "0.4.45"
+clap = { version = "4.6", features = ["derive", "env"] }
+bichon-memdb = { path = "crates/memdb" }
+bichon-blob = { path = "crates/blob" }
+itertools = "0.15.0"
 ring = { version = "0.17.14", features = ["std"] }
-serde = { version = "1.0.228", features = ["derive"] }
-serde_json = "1.0.150"
-tokio = { version = "1.52.3", features = ["full"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+tokio = { version = "1.53", features = ["full"] }
 tracing = "0.1.44"
 tracing-appender = "0.2.3"
 tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] }
-base64 = "0.22.1"
-snafu = "0.9.0"
+base64 = "0.23"
+snafu = "0.9"
 reqwest = { version = "0.12.24", default-features = false, features = [
     "json",
     "stream",
@@ -37,58 +39,58 @@ reqwest = { version = "0.12.24", default-features = false, features = [
     "blocking",
     "socks",
 ] }
-tokio-socks = "0.5.2"
-http = "1.4.1"
-regex = "1.12.3"
+tokio-socks = "0.5.3"
+http = "1.5"
+regex = "1.13"
 email_address = "0.2.9"
-futures = "0.3.32"
+futures = "0.3"
 utf7-imap = "0.3.2"
-mail-parser = { version = '0.11.3', features = ["serde"] }
+mail-parser = { version = '0.11', features = ["serde"] }
 # mail-send = "0.5.2"
 tokio-rustls = { version = "0.26.4", default-features = false, features = [
     "ring",
     "tls12",
 ] }
-timeago = "0.6.0"
+timeago = "0.6.1"
 oauth2 = { version = "5.0.0", features = ["reqwest-blocking"] }
 url = { version = "2.5.8", features = ["serde"] }
-sysinfo = "0.39.2"
+sysinfo = "0.39"
 num_cpus = "1.17.0"
-rand = "0.10.1"
+rand = "0.10.2"
 encoding_rs = "0.8.35"
-webpki-roots = "1.0.7"
-rustls = { version = "0.23.40", default-features = false, features = ["ring"] }
-rustls-pki-types = "1.14.1"
+webpki-roots = "1.0"
+rustls = { version = "0.23", default-features = false, features = ["ring"] }
+rustls-pki-types = "1.15"
 tokio-io-timeout = "1.2.1"
 semver = "1.0.28"
 governor = "0.10.4"
-lru = "0.18.0"
+lru = "0.18.1"
 mime_guess = "2.0.5"
 hex = "0.4.3"
-time = { version = "0.3.47", features = [
+time = { version = "0.3", features = [
     "formatting",
     "parsing",
     "local-offset",
 ] }
-rust-embed = "8.11.0"
+rust-embed = "8.12"
 murmur3 = "0.5.2"
 urlencoding = "2.1.3"
 dashmap = "6.2.1"
 gethostname = "1.1.0"
 itoa = "1.0.18"
 html2text = "0.17.1"
-bytes = "1.11.1"
+bytes = "1.12"
 dialoguer = "0.12.0"
-console = "0.16.3"
-mail-send = "0.6.0"
+console = "0.16.4"
+mail-send = "0.6.1"
 rcgen = "0.14.8"
 rustls-pemfile = "2.2.0"
 blake3 = "1.8.5"
-uuid = { version = "1.23.1", features = ["v4", "serde"] }
-fjall = { version = "3.1.4", features = ["lz4", "metrics", "bytes_1"] }
+uuid = { version = "1.24", features = ["v4", "serde"] }
+fjall = { version = "3.1", features = ["lz4", "metrics", "bytes_1"] }
 tracing-log = "0.2.0"
-tokio-util = "0.7.18"
-indicatif = "0.18.4"
+tokio-util = "0.7"
+indicatif = "0.18.6"
 
 [profile.release]
 strip = true

README.md

@@ -93,17 +93,17 @@
 - **Attachment Search**: Browse and filter attachments by sender, file type, size, and other attachment properties.
 - **Faceted Tags**: Add, remove, or overwrite tags on messages and attachments. Filter by tag combinations with real-time count updates.
 - **Contacts View**: Extracted and deduplicated sender/recipient address book across all authorized accounts.
-- **Three-Layer Storage**: Tantivy for full-text indexing (Zstd compression), Fjall with LZ4 for compressed blob storage, and memdb for relational metadata. All embedded — zero external dependencies.
+- **Three-Layer Storage**: Tantivy for full-text indexing (Zstd compression), bichon-blob with Zstd for compressed blob storage, and memdb for relational metadata. All embedded — zero external dependencies.
 - **Content Deduplication**: Identical email bodies and attachments stored once via BLAKE3 content hashing. Folder moves update metadata only.
 - **Dashboard Analytics**: Email volume trends, top senders, storage usage breakdown, attachment statistics, and per-account activity. Scoped by user permissions.
 - **OpenAPI 3.0**: Interactive API documentation at `/api-docs` (Swagger UI, ReDoc, Scalar). All endpoints documented with request/response schemas.
 - **Multi-User RBAC**: 5 built-in roles (Admin, Manager, Member, AccountManager, AccountViewer) plus custom roles with 22 granular permissions.
 - **Account-Level Isolation**: Grant users access to specific accounts with scoped roles. Permissions enforced at the API layer.
-- **CLI Import Tools**: Import from EML directories, MBOX files (including Gmail variants), Thunderbird profiles, and Outlook PST files.
+- **CLI & WebUI Import Tools**: Import from EML directories, MBOX files (including Gmail variants), Thunderbird profiles, and Outlook PST files via CLI. Import EML files directly from the WebUI.
 - **CLI Export**: Download account data as MBOX via `bichon-cli`.
 - **Bulk Restore**: Restore emails in bulk back to their original IMAP accounts.
 - **Embedded SMTP Server**: Receive emails directly at the gateway level. STARTTLS or TLS encryption. AUTH PLAIN/LOGIN with API token authentication.
-- **Admin Tooling**: Password reset for locked-out admins. Non-destructive v0.3.7 to v1.0 data migration.
+- **Admin Tooling**: Password reset for locked-out admins. Non-destructive migration from v0.3.7 and v1.x to v2.x.
 - **API Token Management**: Create, list, and revoke long-lived API tokens for programmatic access.
 - **SOCKS5 Proxy Management**: Configure and manage proxy profiles for routing IMAP traffic per account.
 - **Scheduled Download**: Configure per-account download schedules using cron expressions. Run syncs at specific times or intervals — for example, nightly-only or business-hours-only archiving.
@@ -185,10 +185,7 @@ Download from the [Releases](https://github.com/rustmailer/bichon/releases) page
 git clone https://github.com/rustmailer/bichon.git
 cd bichon
 
-# Build the WebUI (required before building the server)
-cd web && pnpm install && pnpm run build && cd ..
-
-# Build and run
+# Build and run — frontend dependencies are installed and built automatically via build.rs
 export BICHON_ENCRYPT_PASSWORD=dev-password
 cargo run -- --bichon-root-dir /tmp/bichon-data
 ```
@@ -199,9 +196,6 @@ For frontend development:
 cd web && pnpm run dev   # Vite dev server with API proxy to Rust backend
 ```
 
-> [!TIP]
-> The WebUI must be built at least once (`pnpm run build`) for the server to serve the frontend. In dev mode (`pnpm run dev`), Vite proxies API calls to the Rust server automatically.
-
 ## Configuration Reference
 
 All settings accept both CLI flags (`--bichon-http-port`) and environment variables (`BICHON_HTTP_PORT`). CLI flags take precedence over environment variables.
@@ -270,7 +264,7 @@ All settings accept both CLI flags (`--bichon-http-port`) and environment variab
 | Variable | Default | Description |
 |----------|---------|-------------|
 | `BICHON_INDEX_DIR` | `{root}/bichon-indices` | Tantivy full-text index directory |
-| `BICHON_DATA_DIR` | `{root}/bichon-storage` | Fjall blob storage directory |
+| `BICHON_DATA_DIR` | `{root}/bichon-storage` | bichon-blob storage directory |
 
 > [!TIP]
 > Place `BICHON_INDEX_DIR` on fast SSD storage for responsive search, and `BICHON_DATA_DIR` on high-capacity HDD for cost-effective blob storage.
@@ -378,12 +372,13 @@ All imports are processed server-side — the server handles MIME parsing, index
 ./bichon-admin
 ```
 
-Interactive menu with two operations:
+Interactive menu with three operations:
 
 | Operation | Description |
 |-----------|-------------|
 | **Reset Admin Password** | Reset the built-in admin password when locked out |
-| **Migrate v0.3.7 → v1.0** | Non-destructive migration from legacy storage layout to v1.0 architecture |
+| **Migrate v0.3.7 → v2.x** | Non-destructive migration from legacy Tantivy-based storage to v2.x |
+| **Migrate v1.x → v2.x** | Blob-only migration from Fjall to bichon-blob (indexes and metadata untouched) |
 
 ## API Reference
 
@@ -409,10 +404,11 @@ All `/api/v1/*` endpoints require `Authorization: Bearer <token>`.
 | **MBOX** | `bichon-cli` | Single-file streaming import; supports Gmail's MBOX variant |
 | **Thunderbird** | `bichon-cli` | Reads directly from local Thunderbird profile directory |
 | **PST** | `bichon-cli` | Outlook Personal Storage (`.pst`) file parsing |
+| **WebUI Import** | WebUI | Upload `.eml` files directly from the browser |
 | **API Import** | `POST /api/v1/import` | Base64-encoded EML payloads for programmatic use |
 | **MBOX Export** | `bichon-cli` | Download account data as `.mbox` file |
 
-All imports flow through the Bichon REST API. The server parses MIME, extracts metadata, indexes content into Tantivy, deduplicates by BLAKE3 content hash, and stores raw blobs in Fjall.
+All imports flow through the Bichon REST API. The server parses MIME, extracts metadata, indexes content into Tantivy, deduplicates by BLAKE3 content hash, and stores raw blobs in bichon-blob.
 
 ## Architecture
 
@@ -438,12 +434,12 @@ Request Layer
 Storage Layer         │
                       │
   ┌──────────────┐  ┌──────────────┐  ┌──────────────┐
-  │    memdb     │  │   Tantivy    │  │    Fjall     │
+  │    memdb     │  │   Tantivy    │  │ bichon-blob  │
   │  (metadata)  │  │  (full-text) │  │   (blobs)    │
   │              │  │              │  │              │
   │ • accounts   │  │ • envelope   │  │ • raw emails │
   │ • users      │  │ • attachment │  │ • attachments│
-  │ • roles      │  │ • tags       │  │   LZ4 compr. │
+  │ • roles      │  │ • tags       │  │   Zstd compr.│
   │ • config     │  │ • contacts   │  │              │
   │ • proxies    │  │   Zstd compr.│  │ BLAKE3 hash  │
   └──────────────┘  └──────────────┘  └──────────────┘
@@ -451,7 +447,7 @@ Storage Layer         │
 
 - **memdb**: Key-value metadata store. Houses accounts, users, roles, OAuth2 configs, proxy settings, and system configuration. 
 - **Tantivy**: Full-text search indices with Zstd compression support. Two separate indices: envelope (email metadata + body text) and attachment (file metadata + extracted text). Batch-committed every 1,000 documents or 60 seconds.
-- **Fjall**: LZ4-compressed LSM tree key-value store. Two keyspaces — `email_keyspace` and `attachments_keyspace`. Content-hash addressed (BLAKE3) with insert-time deduplication. Values larger than 1 KB stored as separate files (KV separation).
+- **bichon-blob**: Zstd-compressed log-structured storage engine. Append-only segment files (1&nbsp;GB each) with a redb-backed index for O(1) key lookup. Content-hash addressed (BLAKE3) with insert-time deduplication. Supports global dedup, online GC, and crash-safe recovery.
 
 ### IMAP Download Pipeline
 
@@ -477,7 +473,7 @@ extract_envelope_and_store_it()
         │
    ┌────┼────┐
    ▼    ▼    ▼
-Tantivy Fjall memdb
+Tantivy bichon-blob memdb
 ```
 
 - Per-account background tasks managed by a global download-task singleton
@@ -513,7 +509,7 @@ Tantivy Fjall memdb
                │  attachment     │   │  → attachment_content_hash   │
                │  bytes with     │   │                              │
                │  placeholder:   │   │  Store raw undecoded bytes   │
-               │                 │   │  in Fjall attachments_ks     │
+               │                 │   │  in bichon-blob               │
                │  <<BICHON_      │   │  (skip if hash exists)       │
                │   DETACH_HASH:  │   │                              │
                │   xxx>>         │   │  Extract text for indexing   │
@@ -523,7 +519,7 @@ Tantivy Fjall memdb
                        ▼                            │
                ┌──────────────────────────────┐     │
                │  Stripped EML stored in      │     │
-               │  Fjall email_keyspace        │     │
+               │  bichon-blob                 │     │
                │  keyed by email_content_hash │     │
                │  (skip if hash exists)       │     │
                └──────────────┬───────────────┘     │
@@ -538,8 +534,8 @@ Tantivy Fjall memdb
 
    Dedup layers
    ┌─────────────────────────────────────────────────────────────────┐
-   │ Fjall (insert-time)                                             │
-   │   contains_key(hash)? → skip : store with LZ4 compression       │
+   │ bichon-blob (insert-time)                                       │
+   │   contains_key(hash)? → skip : store with Zstd compression       │
    │                                                                 │
    │ Tantivy (periodic, every 12 h)                                  │
    │   Group by (account, mailbox, content_hash)                     │
@@ -549,14 +545,14 @@ Tantivy Fjall memdb
 
    Reconstruction
    ┌─────────────────────────────────────────────────────────────────┐
-   │ Fetch stripped EML by content_hash from Fjall                  │
+   │ Fetch stripped EML by content_hash from bichon-blob            │
    │ Find <<BICHON_DETACH_HASH:xxx>> placeholders                    │
-   │ Replace each with raw attachment blob from Fjall                │
+   │ Replace each with raw attachment blob from bichon-blob          │
    │ Result → byte-identical original EML                            │
    └─────────────────────────────────────────────────────────────────┘
 ```
 
-Every ingested email is hashed with BLAKE3. Attachments are detached from the MIME tree, hashed independently (decoded content), and stored as raw undecoded bytes in Fjall's `attachments_keyspace`. The email body is patched with hash-based placeholders and stored in `email_keyspace`. Both keyspaces check for existing hashes before writing — identical content is never stored twice, regardless of which account or folder it arrives in. A periodic index dedup task (every 12 hours) scans Tantivy for duplicate `(account, mailbox, content_hash)` tuples, keeps the most recently ingested copy, and cascade-deletes orphaned attachment entries so UID-based incremental sync remains accurate. The original EML reconstructs byte-for-byte by swapping placeholders back with their attachment blobs.
+Every ingested email is hashed with BLAKE3. Attachments are detached from the MIME tree, hashed independently (decoded content), and stored as raw undecoded bytes in bichon-blob. The email body is patched with hash-based placeholders and stored separately. Both email and attachment blobs are deduplicated by content hash — identical content is never stored twice, regardless of which account or folder it arrives in. A periodic index dedup task (every 12 hours) scans Tantivy for duplicate `(account, mailbox, content_hash)` tuples, keeps the most recently ingested copy, and cascade-deletes orphaned attachment entries so UID-based incremental sync remains accurate. The original EML reconstructs byte-for-byte by swapping placeholders back with their attachment blobs.
 
 ## Storage & Backup
 
@@ -565,7 +561,7 @@ Every ingested email is hashed with BLAKE3. Attachments are detached from the MI
 ```
 {root}/
 ├── bichon-indices/         Tantivy full-text index (envelope + attachment)
-├── bichon-storage/         Fjall LZ4-compressed blob store
+├── bichon-storage/         bichon-blob Zstd-compressed blob store
 ├── memdb/                  Metadata database (accounts, users, roles, config)
 ├── logs/                   Server logs (when BICHON_LOG_TO_FILE=true)
 ```
@@ -605,25 +601,36 @@ The WebUI is available in **18 languages**:
 
 Language preference and UI theme are saved to your user profile and can be changed anytime from the WebUI settings.
 
-## Data Migration (v0.3.7 → v1.x)
+## Data Migration
 
-Bichon v1.x introduced a redesigned storage architecture:
+Bichon v2.x replaces the Fjall blob engine with bichon-blob. Two migration paths are available:
 
-| Layer | v0.3.7 (Legacy) | v1.x |
-| :--- | :--- | :--- |
-| **Index** | Tantivy (shared instance, no full attachments) | Tantivy (separate envelope + attachment indices) |
-| **Raw data** | Tantivy (inline, stored in another Tantivy instance) | Fjall (LZ4-compressed LSM-tree key-value store) |
-| **Metadata** | Native_DB (shared, disk-based DB powered by redb) | memdb (dedicated, in-house in-memory DB) |
+| Layer | v0.3.7 (Legacy) | v1.x | v2.x |
+| :--- | :--- | :--- | :--- |
+| **Index** | Tantivy (inline) | Tantivy (separate envelope + attachment) | Tantivy (unchanged from v1.x) |
+| **Blobs** | Tantivy (inline) | Fjall (LZ4-compressed LSM tree) | bichon-blob (Zstd-compressed log-structured) |
+| **Metadata** | native_db (redb-backed) | memdb | memdb (unchanged from v1.x) |
 
-If you ran Bichon prior to v1.x, migrate your data:
+**v0.3.7 → v2.x** (full migration):
 
 ```bash
 ./bichon-admin
-# Select "Migrate Legacy v0.3.7 Storage to v1.x"
+# Select "Migrate Legacy v0.3.7 Storage to v2.x"
 ```
 
+Rebuilds Tantivy indexes, migrates metadata to memdb, and converts blobs to bichon-blob.
+
+**v1.x → v2.x** (blob-only):
+
+```bash
+./bichon-admin
+# Select "Migrate v1.x Storage to v2.x"
+```
+
+Copies blobs from Fjall to bichon-blob. Tantivy indexes and memdb are left untouched.
+
 > [!NOTE]
-> The migration is **non-destructive** — original v0.3.7 files remain in place and are not modified. You can safely remove them manually after verifying the migration was successful.
+> Both migrations are **non-destructive** — legacy files are never modified. After verifying the migration was successful, see the [Migration Guide](https://github.com/rustmailer/bichon/wiki/Bichon-v2.x-Migration-Guide) for cleanup instructions.
 
 ## FAQ
 
@@ -636,7 +643,7 @@ If you ran Bichon prior to v1.x, migrate your data:
 
 ### "Legacy data layout detected" error on startup
 
-Your data was created by Bichon v0.3.7 and must be migrated. Run `./bichon-admin` and select the migration option.
+Your data was created by an older version of Bichon and must be migrated. Run `./bichon-admin` and select the appropriate migration option.
 
 ### How do I run Bichon behind a reverse proxy?
 
@@ -684,7 +691,7 @@ No. Bichon is an **archiver**, not an email client. The optional SMTP server **r
 - [x] CLI import: EML, MBOX, Thunderbird, PST
 - [x] CLI export: MBOX
 - [x] Embedded SMTP server
-- [x] Data migration tooling (v0.3.7 → v1.0)
+- [x] Data migration tooling (v0.3.7 / v1.x → v2.x)
 - [x] On-demand manual download controls
 - [ ] Post-download server cleanup (free remote mailbox space)
 - [ ] Account-to-account email merge / migration
@@ -696,14 +703,14 @@ No. Bichon is an **archiver**, not an email client. The optional SMTP server **r
 
 Contributions of all kinds are welcome — code, bug reports, documentation, or feature suggestions.
 
+> [!IMPORTANT]
+> By submitting a Pull Request, you agree to the terms of the [Contributor License Agreement](CLA.md).
+
 ```bash
 git clone https://github.com/rustmailer/bichon.git
 cd bichon
 
-# Build WebUI
-cd web && pnpm install && pnpm run build && cd ..
-
-# Build backend
+# Build backend — frontend dependencies and build are handled automatically via build.rs
 cargo build
 
 # Run tests
@@ -711,17 +718,46 @@ cargo test
 ```
 
 > [!IMPORTANT]
-> Before implementing a new feature or making significant changes, please **open an issue first** to discuss your idea with the maintainer and ensure it aligns with the project's scope.
+> **For new features:** Please **open a feature request issue first** before starting implementation. PRs that introduce new functionality without a prior issue may be **rejected** to avoid unnecessary wasted effort.
+>
+> **For major bug fixes** with wide-ranging impact, please **open an issue and discuss with the maintainer** before acting and submitting. This ensures the fix approach is aligned and avoids duplicate or conflicting work.
+>
+> **Large, hard-to-review PRs** that touch many modules or contain substantial changes may be **rejected outright**. Break your work into smaller, focused PRs — one logical change per PR.
 
 Feel free to open an [Issue](https://github.com/rustmailer/bichon/issues) or join the [Discord](https://discord.gg/Bq4M2cDmF4) to discuss ideas.
 
+#### Guidelines
+
+1. **AI-assisted, not AI-authored.** Use AI to help analyze, debug, or draft code when unsure — but understand and review every change yourself before submitting. Don't submit unreviewed AI-generated content.
+2. **Keep PRs scoped to one issue.** Don't bundle unrelated changes (CI config, dependency bumps, fixes to other modules) into the same PR. Split them into separate PRs.
+3. **Frontend/backend changes go together.** If a change affects an API, data structure, or behavior with a frontend consumer, update the frontend in the same PR (or a clearly linked companion PR).
+4. **Unit tests are required.** New or fixed logic must include tests that reproduce the original issue and verify the fix. PRs without tests won't be merged.
+5. **Maintain backward compatibility.** Changes to data formats, protocols, configs, or APIs must state whether they're backward compatible. If not, include a migration plan.
+6. **State the blast radius.** PR descriptions must specify which modules/APIs/data are affected and any downstream impact.
+
+### Commit Messages
+
+Format: `<type>(<scope>): <subject>`
+
+- **type**: `fix`, `feat`, `refactor`, `ci`, `test`, `docs`, `chore`
+- **scope**: affected module/component (e.g. `rustmailer#286`, `dedup_cache`)
+- **subject**: imperative, present tense, no period
+
+Rules:
+- One logical change per commit — don't mix a fix with CI tweaks or unrelated module changes.
+- Reference the issue number when applicable (e.g. `fix(#286): ...`).
+- Body explains *why*, not just *what* — include root cause and how it was verified for non-trivial fixes.
+- Rebase before submitting — squash WIP/fixup commits into a clean, logical sequence.
+- No vague messages like `update`, `fix bug`, `wip`.
+
+
 ## Tech Stack
 
 | Layer | Technology |
 |-------|-----------|
 | **Backend** | Rust, Tokio, Poem + Poem OpenAPI |
 | **Full-text search** | Tantivy (Zstd compression) |
-| **Blob storage** | Fjall (LSM tree, LZ4 compression, KV separation) |
+| **Blob storage** | bichon-blob (log-structured, Zstd compression, BLAKE3 dedup) |
 | **Metadata DB** | memdb (embedded key-value store with WAL) |
 | **IMAP** | async-imap, rustls (ring), SOCKS5 proxy support |
 | **SMTP** | Embedded receiver (AUTH PLAIN/LOGIN, STARTTLS/TLS) |

crates/admin/Cargo.toml

@@ -17,4 +17,16 @@ serde_json.workspace = true
 
 itertools.workspace = true
 snafu.workspace = true
-memdb.workspace = true
+bichon-memdb.workspace = true
+fjall.workspace = true
+hex.workspace = true
+bichon-blob.workspace = true
+mail-parser.workspace = true
+bytes.workspace = true
+uuid.workspace = true
+tantivy = { version = "0.26.1", features = ["zstd-compression", "quickwit"] }
+chrono.workspace = true
+tracing.workspace = true
+
+[dev-dependencies]
+tempfile = "3"

crates/admin/src/legacy/schema.rs

@@ -1,6 +1,6 @@
 use tantivy::schema::{FacetOptions, Field, Schema, FAST, INDEXED, STORED, STRING, TEXT};
 
-use crate::migrate::legacy::fields::{EmlFields, EnvelopeFields, *};
+use crate::legacy::fields::{EmlFields, EnvelopeFields, *};
 
 pub struct SchemaTools;
 

crates/admin/src/main.rs

@@ -19,10 +19,13 @@
 use console::style;
 use dialoguer::{theme::ColorfulTheme, Select};
 
-use crate::{migrate::handle_migration, reset::handle_reset_password};
+use crate::{migrate_v037::handle_migration_v037, migrate_v1::handle_migrate_v1, reset::handle_reset_password};
 
+pub mod legacy;
 pub mod meta;
-pub mod migrate;
+pub mod migrate_store_v2;
+pub mod migrate_v037;
+pub mod migrate_v1;
 pub mod reset;
 
 
@@ -40,7 +43,8 @@ async fn run_interactive() {
 
     let main_options = vec![
         "Reset Admin Password",
-        "Migrate Legacy v0.3.7 Storage to v1.x",
+        "Migrate Legacy v0.3.7 Storage to v2.x (bichon-blob)",
+        "Migrate v1.x Storage to v2.x (Fjall → bichon-blob)",
         "Exit",
     ];
 
@@ -53,7 +57,8 @@ async fn run_interactive() {
 
     match selection {
         0 => handle_reset_password(&theme),
-        1 => handle_migration(&theme),
+        1 => handle_migration_v037(&theme),
+        2 => handle_migrate_v1(&theme),
         _ => {
             println!("{}", style("Exiting...").dim());
         }

crates/admin/src/meta.rs

@@ -11,16 +11,16 @@ use bichon_core::{
         since::{DateSince, RelativeDate},
     },
     autoconfig::entity::MailServerConfig,
-    cache::imap::mailbox::Attribute,
+    archive::imap::mailbox::Attribute,
     database::batch_insert_impl,
     error::{code::ErrorCode, BichonError, BichonResult},
     raise_error,
     token::TokenType,
     users::{acl::AccessControl, role::RoleType},
 };
+use bichon_memdb::{Durability, MemDb};
 use console::style;
 use itertools::Itertools;
-use memdb::{Durability, MemDb};
 use native_db::*;
 use native_model::{native_model, Model};
 use serde::{Deserialize, Serialize};
@@ -250,17 +250,20 @@ impl From<AccountV3> for AccountModel {
             account_type: value.account_type,
             download_interval_min: value.sync_interval_min,
             download_batch_size: value.sync_batch_size,
+            max_email_size_bytes: None,
             known_folders: value.known_folders,
             created_at: value.created_at,
             updated_at: value.updated_at,
             created_by: value.created_by,
-            use_proxy: value.use_proxy,
             use_dangerous: value.use_dangerous,
             pgp_key: value.pgp_key,
             imap_quota_window: None,
             imap_quota_bytes: None,
             auto_download_new_mailboxes: None,
             download_schedule: None,
+            deleting: false,
+            archive_rules: None,
+            extraction_rules: None,
         }
     }
 }
@@ -572,6 +575,8 @@ impl From<BichonUserV2> for bichon_core::users::BichonUserV2 {
             acl: value.acl,
             theme: value.theme,
             language: value.language,
+            sso_id: None,
+            sso_provider: None,
         }
     }
 }
@@ -648,7 +653,7 @@ pub struct MailBox {
     pub uid_validity: Option<u32>,
 }
 
-impl From<MailBox> for bichon_core::cache::imap::mailbox::MailBox {
+impl From<MailBox> for bichon_core::archive::imap::mailbox::MailBox {
     fn from(value: MailBox) -> Self {
         Self {
             id: value.id,
@@ -893,7 +898,7 @@ pub fn migrate_metadata(root_path: &PathBuf) -> Result<(), Box<dyn std::error::E
     migrate_collection!(
         "Mailboxes",
         MailBox,
-        bichon_core::cache::imap::mailbox::MailBox,
+        bichon_core::archive::imap::mailbox::MailBox,
         &envelope_db
     );
 

crates/admin/src/migrate_store_v2.rs

@@ -3,20 +3,17 @@ use std::{path::PathBuf, time::Instant};
 use bytes::Bytes;
 use mail_parser::MimeHeaders;
 
-use crate::{
+use bichon_core::{
     envelope::extractor::extract_references, message::content::AttachmentInfo,
     store::tantivy::tokenizers::EuroTokenizer, utils::compute_content_hash,
 };
 
-use fjall::{
-    config::{BlockSizePolicy, CompressionPolicy},
-    CompressionType, Database, Keyspace, KeyspaceCreateOptions, KvSeparationOptions,
-};
+use bichon_blob::{Codec, Config, Engine};
 use mail_parser::MessageParser;
 use tantivy::{indexer::NoMergePolicy, Index, IndexWriter, TantivyDocument};
 use uuid::Uuid;
 
-use crate::{
+use bichon_core::{
     common::AddrVec,
     envelope::extractor::{compute_thread_id, generate_message_id},
     error::{code::ErrorCode, BichonResult},
@@ -64,6 +61,17 @@ pub struct DetachOutput {
     pub blobs: Vec<(String, Bytes)>,
 }
 
+fn hex_to_raw_key(hex: &str) -> BichonResult<[u8; 32]> {
+    let mut key = [0u8; 32];
+    hex::decode_to_slice(hex, &mut key).map_err(|e| {
+        raise_error!(
+            format!("invalid content hash: {e:#?}"),
+            ErrorCode::InternalError
+        )
+    })?;
+    Ok(key)
+}
+
 pub fn detach_attachments_standalone(
     original_body: &[u8],
     message: &mail_parser::Message<'_>,
@@ -132,19 +140,16 @@ pub fn detach_attachments_standalone(
     (stripped_eml, DetachOutput { infos, blobs })
 }
 
-pub struct NewIndexWriter {
+pub struct NewIndexWriterV2 {
     pub envelope_writer: Option<IndexWriter>,
     pub attachment_writer: Option<IndexWriter>,
-    pub email_ks: Keyspace,
-    pub attachment_ks: Keyspace,
+    pub engine: Engine,
     pending: usize,
-    email_buf: Vec<(String, Vec<u8>)>,
-    attachment_buf: Vec<(String, Vec<u8>)>,
+    email_buf: Vec<([u8; 32], Vec<u8>)>,
+    attachment_buf: Vec<([u8; 32], Vec<u8>)>,
 }
 
-//const COMMIT_THRESHOLD: usize = 500;
-
-impl NewIndexWriter {
+impl NewIndexWriterV2 {
     pub fn open(dirs: NewDirs) -> BichonResult<Self> {
         // ── envelope index ──────────────────────────────────────────────
         std::fs::create_dir_all(&dirs.envelope_dir)
@@ -170,11 +175,6 @@ impl NewIndexWriter {
             .writer_with_num_threads(3, 256 * 1024 * 1024)
             .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
 
-        // let mut merge_policy = LogMergePolicy::default();
-        // merge_policy.set_min_num_segments(25);
-        // merge_policy.set_min_layer_size(10_000);
-        // merge_policy.set_max_docs_before_merge(100_000);
-
         envelope_writer.set_merge_policy(Box::new(NoMergePolicy));
         // ── attachment index ─────────────────────────────────────────────
         std::fs::create_dir_all(&dirs.attachment_dir)
@@ -199,58 +199,26 @@ impl NewIndexWriter {
             .writer_with_num_threads(3, 256 * 1024 * 1024)
             .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
 
-        // let mut merge_policy = LogMergePolicy::default();
-        // merge_policy.set_min_num_segments(25);
-        // merge_policy.set_min_layer_size(10_000);
-        // merge_policy.set_max_docs_before_merge(100_000);
-
         attachment_writer.set_merge_policy(Box::new(NoMergePolicy));
 
-        // ── blob store ───────────────────────────────────────────────────
+        // ── blob store (bichon-blob, not fjall) ───────────────────────────
         std::fs::create_dir_all(&dirs.storage_dir)
             .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
-        let db = Database::builder(&dirs.storage_dir)
-            .cache_size(8 * 1024 * 1024)
-            .journal_compression(CompressionType::None)
-            .max_journaling_size(64 * 1024 * 1024)
-            .open()
-            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        let blob_dir = dirs.storage_dir.join("blobs");
 
-        let email_ks = db
-            .keyspace("email", || {
-                KeyspaceCreateOptions::default()
-                    .max_memtable_size(4 * 1024 * 1024)
-                    .data_block_size_policy(BlockSizePolicy::all(4 * 1024))
-                    .data_block_compression_policy(CompressionPolicy::all(CompressionType::Lz4))
-                    .with_kv_separation(Some(
-                        KvSeparationOptions::default()
-                            .separation_threshold(1024)
-                            .compression(CompressionType::Lz4)
-                            .file_target_size(512 * 1024 * 1024),
-                    ))
-            })
-            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        let mut config = Config::default();
+        config.default_codec = Codec::Zstd;
+        config.compress_threshold = 1024;
+        config.flush_interval_secs = 0;
+        config.gc_interval_secs = 0;
 
-        let attachment_ks = db
-            .keyspace("attachments", || {
-                KeyspaceCreateOptions::default()
-                    .max_memtable_size(4 * 1024 * 1024)
-                    .data_block_size_policy(BlockSizePolicy::all(4 * 1024))
-                    .data_block_compression_policy(CompressionPolicy::all(CompressionType::Lz4))
-                    .with_kv_separation(Some(
-                        KvSeparationOptions::default()
-                            .separation_threshold(1024)
-                            .compression(CompressionType::Lz4)
-                            .file_target_size(512 * 1024 * 1024),
-                    ))
-            })
+        let engine = Engine::open(&blob_dir, config)
             .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
 
         Ok(Self {
             envelope_writer: Some(envelope_writer),
             attachment_writer: Some(attachment_writer),
-            email_ks,
-            attachment_ks,
+            engine,
             pending: 0,
             email_buf: Vec::new(),
             attachment_buf: Vec::new(),
@@ -266,6 +234,7 @@ impl NewIndexWriter {
         internal_date: i64,
     ) -> BichonResult<()> {
         let email_content_hash = compute_content_hash(eml_bytes);
+        let email_raw_key = hex_to_raw_key(&email_content_hash)?;
 
         let message = MessageParser::new()
             .parse(eml_bytes)
@@ -284,7 +253,7 @@ impl NewIndexWriter {
             .or_else(|| {
                 message
                     .body_html(0)
-                    .map(|html| crate::utils::html::extract_text(html.into_owned()))
+                    .map(|html| bichon_core::utils::html::extract_text(html.into_owned()))
             })
             .unwrap_or_default();
         let text = text.split_whitespace().collect::<Vec<_>>().join(" ");
@@ -336,11 +305,13 @@ impl NewIndexWriter {
         // ── detach attachments → blob ──────────────────────────────────────
         let (stripped_eml, attachment_output) = detach_attachments_standalone(eml_bytes, &message);
 
-        // Buffer for bulk ingestion — sorted + flushed later.
+        // Buffer for bulk write — sorted + flushed later.
+        // Key is the raw 32-byte hash (not the hex string).
         self.email_buf
-            .push((email_content_hash.clone(), stripped_eml));
+            .push((email_raw_key, stripped_eml));
         for (hash, data) in &attachment_output.blobs {
-            self.attachment_buf.push((hash.clone(), data.to_vec()));
+            let raw_key = hex_to_raw_key(hash)?;
+            self.attachment_buf.push((raw_key, data.to_vec()));
         }
 
         // ── build envelope doc ────────────────────────────────────────────
@@ -404,6 +375,7 @@ impl NewIndexWriter {
             regular_attachment_count: attachment_docs.len(),
             tags: None,
             account_email: None,
+            account_name: None,
             mailbox_name: None,
             content_hash: email_content_hash,
         };
@@ -429,9 +401,6 @@ impl NewIndexWriter {
         }
 
         self.pending += 1;
-        // if self.pending >= COMMIT_THRESHOLD {
-        //     self.commit()?;
-        // }
 
         Ok(())
     }
@@ -472,7 +441,7 @@ impl NewIndexWriter {
                 let seg_ids = writer
                     .index()
                     .searchable_segment_ids()
-                    .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+                    .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
                 println!("merging {} {} segments...", seg_ids.len(), name);
                 if seg_ids.len() > 1 {
                     let _ = writer.merge(&seg_ids);
@@ -490,70 +459,111 @@ impl NewIndexWriter {
         Ok(())
     }
 
-    /// Sort buffered (hash, data) pairs, dedup, and write via Fjall's
-    /// ingestion API — writes SSTables directly, bypassing memtable and WAL.
+    /// Write buffered blobs to the bichon-blob engine.
     /// Also commits the Tantivy writers to bound their in-memory state.
-    pub fn flush_fjall_buffers(&mut self) -> BichonResult<()> {
+    pub fn flush_blob_buffers(&mut self) -> BichonResult<()> {
         self.commit_tantivy()?;
 
         if !self.email_buf.is_empty() {
-            self.email_buf.sort_by(|a, b| a.0.cmp(&b.0));
-            self.email_buf.dedup_by(|a, b| a.0 == b.0);
+            let mut buf = std::mem::take(&mut self.email_buf);
+            buf.sort_by(|a, b| a.0.cmp(&b.0));
+            buf.dedup_by(|a, b| a.0 == b.0);
 
-            let mut ingestion = self.email_ks.start_ingestion().map_err(|e| {
-                raise_error!(
-                    format!("email ingestion start: {e:#?}"),
-                    ErrorCode::InternalError
-                )
-            })?;
-            for (hash, data) in &self.email_buf {
-                ingestion
-                    .write(hash.as_bytes(), data.as_slice())
-                    .map_err(|e| {
-                        raise_error!(
-                            format!("email ingestion write: {e:#?}"),
-                            ErrorCode::InternalError
-                        )
-                    })?;
+            let count = buf.len();
+            let mut skipped = 0usize;
+            let mut batch: Vec<([u8; 32], Vec<u8>, Codec)> = Vec::with_capacity(buf.len());
+            for (key, data) in buf {
+                if data.len() > 100 * 1024 * 1024 {
+                    eprintln!(
+                        "{}",
+                        console::style(format!(
+                            "WARN: skipping oversized email blob key={} ({} bytes)",
+                            hex::encode(key),
+                            data.len()
+                        ))
+                        .yellow()
+                    );
+                    skipped += 1;
+                    continue;
+                }
+                batch.push((key, data, Codec::Zstd));
+            }
+            if !batch.is_empty() {
+                self.engine.put_batch(&batch).map_err(|e| {
+                    raise_error!(
+                        format!("blob engine put_batch error: {e:#?}"),
+                        ErrorCode::InternalError
+                    )
+                })?;
+            }
+            println!("flushed {} email blobs to engine", count - skipped);
+            if skipped > 0 {
+                eprintln!(
+                    "{}",
+                    console::style(format!("skipped {} oversized email blobs", skipped)).yellow()
+                );
             }
-            ingestion.finish().map_err(|e| {
-                raise_error!(
-                    format!("email ingestion finish: {e:#?}"),
-                    ErrorCode::InternalError
-                )
-            })?;
-            self.email_buf.clear();
         }
 
         if !self.attachment_buf.is_empty() {
-            self.attachment_buf.sort_by(|a, b| a.0.cmp(&b.0));
-            self.attachment_buf.dedup_by(|a, b| a.0 == b.0);
+            let mut buf = std::mem::take(&mut self.attachment_buf);
+            buf.sort_by(|a, b| a.0.cmp(&b.0));
+            buf.dedup_by(|a, b| a.0 == b.0);
 
-            let mut ingestion = self.attachment_ks.start_ingestion().map_err(|e| {
-                raise_error!(
-                    format!("attachment ingestion start: {e:#?}"),
-                    ErrorCode::InternalError
-                )
-            })?;
-            for (hash, data) in &self.attachment_buf {
-                ingestion
-                    .write(hash.as_bytes(), data.as_slice())
-                    .map_err(|e| {
-                        raise_error!(
-                            format!("attachment ingestion write: {e:#?}"),
-                            ErrorCode::InternalError
-                        )
-                    })?;
+            let count = buf.len();
+            let mut skipped = 0usize;
+            let mut batch: Vec<([u8; 32], Vec<u8>, Codec)> = Vec::with_capacity(buf.len());
+            for (key, data) in buf {
+                if data.len() > 100 * 1024 * 1024 {
+                    eprintln!(
+                        "{}",
+                        console::style(format!(
+                            "WARN: skipping oversized attachment blob key={} ({} bytes)",
+                            hex::encode(key),
+                            data.len()
+                        ))
+                        .yellow()
+                    );
+                    skipped += 1;
+                    continue;
+                }
+                batch.push((key, data, Codec::Zstd));
+            }
+            if !batch.is_empty() {
+                self.engine.put_batch(&batch).map_err(|e| {
+                    raise_error!(
+                        format!("blob engine put_batch error: {e:#?}"),
+                        ErrorCode::InternalError
+                    )
+                })?;
+            }
+            println!("flushed {} attachment blobs to engine", count - skipped);
+            if skipped > 0 {
+                eprintln!(
+                    "{}",
+                    console::style(format!("skipped {} oversized attachment blobs", skipped))
+                        .yellow()
+                );
             }
-            ingestion.finish().map_err(|e| {
-                raise_error!(
-                    format!("attachment ingestion finish: {e:#?}"),
-                    ErrorCode::InternalError
-                )
-            })?;
-            self.attachment_buf.clear();
         }
 
         Ok(())
     }
+
+    /// Flush and shutdown the blob engine (called once at the very end).
+    pub fn shutdown_engine(&mut self) -> BichonResult<()> {
+        self.engine.flush().map_err(|e| {
+            raise_error!(
+                format!("engine flush error: {e:#?}"),
+                ErrorCode::InternalError
+            )
+        })?;
+        self.engine.shutdown().map_err(|e| {
+            raise_error!(
+                format!("engine shutdown error: {e:#?}"),
+                ErrorCode::InternalError
+            )
+        })?;
+        Ok(())
+    }
 }

crates/admin/src/migrate_v037.rs

@@ -1,17 +1,62 @@
-use std::path::{Path, PathBuf};
+use std::{collections::HashMap, path::PathBuf};
 
-use bichon_core::migrate::{
-    count_eml_segments, do_migrate_segment, is_tantivy_index_dir,
-    store::{LegacyDirs, NewDirs},
+use bichon_core::{
+    error::{code::ErrorCode, BichonResult},
+    migrate::{is_tantivy_index_dir, write_storage_version},
+    raise_error,
 };
 use console::style;
 use dialoguer::{theme::ColorfulTheme, Confirm, Input};
 use indicatif::{ProgressBar, ProgressStyle};
+use tantivy::{
+    collector::TopDocs,
+    columnar::Column,
+    query::TermQuery,
+    schema::{IndexRecordOption, Value},
+    DocAddress, Index, TantivyDocument, Term,
+};
 
-pub fn handle_migration(theme: &ColorfulTheme) {
+use crate::legacy::schema::SchemaTools;
+use crate::migrate_store_v2::{NewDirs, NewIndexWriterV2};
+
+pub struct LegacyDirs {
+    pub envelope_dir: PathBuf,
+    pub eml_dir: PathBuf,
+}
+
+impl LegacyDirs {
+    pub fn new(index: PathBuf, data: PathBuf) -> Self {
+        Self {
+            envelope_dir: index,
+            eml_dir: data,
+        }
+    }
+}
+
+pub fn is_legacy_data_layout_with_paths(
+    envelope_dir: &PathBuf,
+    eml_dir: &PathBuf,
+) -> std::io::Result<bool> {
+    let envelope_result = is_tantivy_index_dir(envelope_dir)?;
+    let eml_result = is_tantivy_index_dir(eml_dir)?;
+    Ok(envelope_result || eml_result)
+}
+
+/// Return the number of segments in the legacy EML Tantivy index.
+pub fn count_eml_segments(legacy: &LegacyDirs) -> BichonResult<usize> {
+    let eml_index = Index::open_in_dir(&legacy.eml_dir)
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+    let reader = eml_index
+        .reader()
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+    let searcher = reader.searcher();
+    Ok(searcher.segment_readers().len())
+}
+
+pub fn handle_migration_v037(theme: &ColorfulTheme) {
     println!(
         "\n{}",
-        style("MIGRATION: Bichon v0.3.7 Storage Architecture → v1.x")
+        style("MIGRATION: Bichon v0.3.7 Storage → v2.x (bichon-blob)")
             .bold()
             .yellow()
     );
@@ -20,8 +65,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
         "{}",
         style(
             "This tool migrates data from the legacy v0.3.7 Tantivy-based storage \
-            architecture to the new v1.x \
-            separated index and Fjall-backed storage format."
+            architecture directly to the v2.x bichon-blob storage format."
         )
         .dim()
     );
@@ -32,11 +76,11 @@ pub fn handle_migration(theme: &ColorfulTheme) {
             "Legacy v0.3.7 architecture:\n\
             • envelope metadata stored in Tantivy\n\
             • message data stored in Tantivy\n\n\
-                New v1.x architecture:\n\
+                New v2.x architecture:\n\
             • mail indexes stored in Tantivy\n\
             • attachment indexes stored in Tantivy\n\
-            • raw message data stored in Fjall\n\
-            • attachment blobs stored in Fjall"
+            • raw message data stored in bichon-blob engine\n\
+            • attachment blobs stored in bichon-blob engine"
         )
         .dim()
     );
@@ -54,7 +98,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
     let root_dir_str: String = Input::with_theme(theme)
         .with_prompt("Enter --bichon-root-dir (same value used by the old server)")
         .validate_with(|input: &String| -> Result<(), &str> {
-            let path = Path::new(input);
+            let path = PathBuf::from(input);
             if !path.is_absolute() {
                 return Err("Path must be absolute.");
             }
@@ -81,7 +125,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
             if input.is_empty() {
                 return Ok(());
             }
-            let path = Path::new(input);
+            let path = PathBuf::from(input);
             if !path.is_absolute() {
                 return Err("Path must be absolute.");
             }
@@ -119,7 +163,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
             if input.is_empty() {
                 return Ok(());
             }
-            let path = Path::new(input);
+            let path = PathBuf::from(input);
             if !path.is_absolute() {
                 return Err("Path must be absolute.");
             }
@@ -172,7 +216,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
             println!(
                 "{} {}",
                 style("✔").green(),
-                style("Legacy v0.3.7 Tantivy-based storage detected. Migration to v1.x is required.")
+                style("Legacy v0.3.7 Tantivy-based storage detected. Migration to v2.x is required.")
                     .yellow()
             );
         }
@@ -186,7 +230,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
             println!(
                 "{}",
                 style(
-                    "The selected directories may already be using the v1.x storage architecture."
+                    "The selected directories may already be using a newer storage architecture."
                 )
                 .dim()
             );
@@ -326,6 +370,18 @@ pub fn handle_migration(theme: &ColorfulTheme) {
             .progress_chars("#>-"),
     );
 
+    let mut writer = match NewIndexWriterV2::open(NewDirs::new(
+        new_index_path.clone(),
+        new_data_path.clone(),
+    )) {
+        Ok(w) => w,
+        Err(e) => {
+            pb.finish_with_message(format!("{}", style("Migration failed.").red()));
+            eprintln!("\n{} {:?}", style("✘").red().bold(), e);
+            return;
+        }
+    };
+
     let mut grand_total_migrated: usize = 0;
     let mut grand_total_skipped: usize = 0;
 
@@ -334,10 +390,10 @@ pub fn handle_migration(theme: &ColorfulTheme) {
 
         pb.set_message(format!("Segment {}/{}", seg_idx + 1, total_segments));
         let legacy = LegacyDirs::new(index_path.clone(), data_path.clone());
-        match do_migrate_segment(
+        match do_migrate_segment_v2(
             batch_size,
             legacy,
-            NewDirs::new(new_index_path.clone(), new_data_path.clone()),
+            &mut writer,
             seg_idx,
             |msg| {
                 if let Some(data) = msg.strip_prefix("TOTAL:") {
@@ -407,6 +463,31 @@ pub fn handle_migration(theme: &ColorfulTheme) {
         pb.set_position((seg_idx + 1) as u64);
     }
 
+    pb.set_message(style("Finalizing indexes...").dim().to_string());
+    if let Err(e) = writer.finish_writers() {
+        pb.finish_with_message(format!("{}", style("Migration failed.").red()));
+        eprintln!("\n{} {:?}", style("✘").red().bold(), e);
+        return;
+    }
+
+    pb.set_message(style("Shutting down blob engine...").dim().to_string());
+    if let Err(e) = writer.shutdown_engine() {
+        pb.finish_with_message(format!("{}", style("Migration failed.").red()));
+        eprintln!("\n{} {:?}", style("✘").red().bold(), e);
+        return;
+    }
+
+    // Write STORAGE_VERSION = 2 to mark the data as v2.x compatible
+    if let Err(e) = write_storage_version(&root_path, 2) {
+        pb.finish_with_message(format!("{}", style("Migration failed.").red()));
+        eprintln!(
+            "\n{} Failed to write STORAGE_VERSION: {:?}",
+            style("✘").red().bold(),
+            e
+        );
+        return;
+    }
+
     pb.finish_with_message(format!(
         "Migration finished. Total: {}, Skipped: {}",
         grand_total_migrated, grand_total_skipped
@@ -415,16 +496,196 @@ pub fn handle_migration(theme: &ColorfulTheme) {
     println!(
         "{} {}",
         style("✔").green(),
-        style("Migration completed successfully!").bold()
+        style("Migration to v2.x completed successfully!").bold()
     );
 }
 
-pub fn is_legacy_data_layout_with_paths(
-    envelope_dir: &PathBuf,
-    eml_dir: &PathBuf,
-) -> std::io::Result<bool> {
-    let envelope_result = is_tantivy_index_dir(envelope_dir)?;
-    let eml_result = is_tantivy_index_dir(eml_dir)?;
+/// Migrate all documents from a single EML segment to the v2.x storage layout.
+fn do_migrate_segment_v2<F>(
+    batch_size: u32,
+    legacy: LegacyDirs,
+    writer: &mut NewIndexWriterV2,
+    segment_index: usize,
+    mut on_progress: F,
+) -> BichonResult<()>
+where
+    F: FnMut(&str),
+{
+    // ── open legacy indices ────────────────────────────────────────────
+    let envelope_index = Index::open_in_dir(&legacy.envelope_dir)
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+    let eml_index = Index::open_in_dir(&legacy.eml_dir)
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
 
-    Ok(envelope_result || eml_result)
+    let envelope_reader = envelope_index
+        .reader()
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+    let eml_reader = eml_index
+        .reader()
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+    let envelope_searcher = envelope_reader.searcher();
+    let eml_searcher = eml_reader.searcher();
+
+    let ef = SchemaTools::envelope_fields();
+    let mf = SchemaTools::eml_fields();
+
+    let eml_segments = eml_searcher.segment_readers();
+    let eml_segment = eml_segments.get(segment_index).ok_or_else(|| {
+        raise_error!(
+            format!(
+                "segment index {} out of range ({} segments)",
+                segment_index,
+                eml_segments.len()
+            ),
+            ErrorCode::InternalError
+        )
+    })?;
+
+    let num_docs = eml_segment.num_docs();
+    if num_docs == 0 {
+        on_progress("TOTAL:0");
+        on_progress("DONE:0:0");
+        return Ok(());
+    }
+
+    on_progress(&format!("TOTAL:{}", num_docs));
+
+    let max_doc = eml_segment.max_doc();
+    let ff = eml_segment.fast_fields();
+    let f_id_col: Column<u64> = ff.u64("id").map_err(|e| {
+        raise_error!(
+            format!("failed to open f_id fast field: {e:#?}"),
+            ErrorCode::InternalError
+        )
+    })?;
+
+    // ── Phase 1: build eid → (uid, internal_date) from envelope, then drop it ──
+    let mut envelope_map: HashMap<u64, (u32, i64)> = HashMap::with_capacity(num_docs as usize);
+
+    let mut env_scanned = 0u32;
+    let mut env_skipped = 0u32;
+    for doc_id in 0..max_doc {
+        if eml_segment.is_deleted(doc_id) {
+            continue;
+        }
+        let eid = f_id_col.values.get_val(doc_id);
+
+        let term = Term::from_field_u64(ef.f_id, eid);
+        let query = TermQuery::new(term, IndexRecordOption::Basic);
+        let hits: Vec<(_, DocAddress)> = envelope_searcher
+            .search(&query, &TopDocs::with_limit(1).order_by_score())
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        if let Some((_, addr)) = hits.first() {
+            let env_doc: TantivyDocument = envelope_searcher
+                .doc(*addr)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+            let uid = env_doc
+                .get_first(ef.f_uid)
+                .and_then(|v| v.as_u64())
+                .unwrap_or(0) as u32;
+            let internal_date = env_doc
+                .get_first(ef.f_internal_date)
+                .and_then(|v| v.as_i64())
+                .unwrap_or(0);
+            envelope_map.insert(eid, (uid, internal_date));
+            env_scanned += 1;
+        } else {
+            env_skipped += 1;
+        }
+
+        if env_scanned % 10 == 0 {
+            on_progress(&format!(
+                "PHASE1:{}/{} skipped:{}",
+                env_scanned, max_doc, env_skipped
+            ));
+        }
+    }
+
+    // Free the envelope index before the heavy EML processing.
+    drop(envelope_searcher);
+    drop(envelope_reader);
+    drop(envelope_index);
+
+    // ── Phase 2: process EML docs, streaming one at a time ─────────────
+    let mut total_migrated = 0usize;
+    let mut total_skipped = 0usize;
+
+    let mut chunk_start = 0u32;
+
+    while chunk_start < max_doc {
+        let chunk_end = (chunk_start + batch_size).min(max_doc);
+        let store_reader = eml_segment
+            .get_store_reader(2)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        for doc_id in chunk_start..chunk_end {
+            if eml_segment.is_deleted(doc_id) {
+                continue;
+            }
+
+            let eid = f_id_col.values.get_val(doc_id);
+
+            let (uid, internal_date) = match envelope_map.get(&eid) {
+                Some(v) => *v,
+                None => {
+                    on_progress(&format!("WARN: eid {} envelope not found", eid));
+                    total_skipped += 1;
+                    continue;
+                }
+            };
+
+            let eml_doc: TantivyDocument = store_reader
+                .get(doc_id)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+            let account_id = match eml_doc.get_first(mf.f_account_id).and_then(|v| v.as_u64()) {
+                Some(v) => v,
+                None => {
+                    on_progress(&format!("WARN: eid {} account_id missing", eid));
+                    total_skipped += 1;
+                    continue;
+                }
+            };
+            let mailbox_id = eml_doc
+                .get_first(mf.f_mailbox_id)
+                .and_then(|v| v.as_u64())
+                .unwrap_or(0);
+
+            let eml_bytes = match eml_doc.get_first(mf.f_eml).and_then(|v| v.as_bytes()) {
+                Some(b) => b,
+                None => {
+                    on_progress(&format!("WARN: eid {} eml bytes missing", eid));
+                    total_skipped += 1;
+                    continue;
+                }
+            };
+
+            if let Err(e) = writer.ingest(eml_bytes, account_id, mailbox_id, uid, internal_date) {
+                on_progress(&format!(
+                    "ERROR: Account {} eid {} ingest failed: {}",
+                    account_id, eid, e
+                ));
+                total_skipped += 1;
+                continue;
+            }
+
+            total_migrated += 1;
+
+            if total_migrated % 10 == 0 || total_migrated as u32 == num_docs {
+                on_progress(&format!("PROGRESS:{}:{}", total_migrated, num_docs));
+            }
+        }
+
+        drop(store_reader);
+
+        // Flush blob buffers to bichon-blob engine.
+        writer.flush_blob_buffers()?;
+
+        chunk_start = chunk_end;
+    }
+
+    on_progress(&format!("DONE:{}:{}", total_migrated, total_skipped));
+    Ok(())
 }

crates/admin/src/migrate_v1.rs

@@ -0,0 +1,472 @@
+use std::path::PathBuf;
+
+use bichon_blob::{Codec, Config, Engine};
+use bichon_core::{
+    error::{code::ErrorCode, BichonResult},
+    migrate::write_storage_version,
+    raise_error,
+};
+use console::style;
+use dialoguer::{theme::ColorfulTheme, Input};
+use fjall::{Config as FjallConfig, Database};
+use indicatif::{ProgressBar, ProgressStyle};
+
+fn hex_key_to_raw(hex_bytes: &[u8]) -> BichonResult<[u8; 32]> {
+    let hex_str = std::str::from_utf8(hex_bytes).map_err(|e| {
+        raise_error!(
+            format!("invalid UTF-8 in fjall key: {e:#?}"),
+            ErrorCode::InternalError
+        )
+    })?;
+    let mut raw = [0u8; 32];
+    hex::decode_to_slice(hex_str, &mut raw).map_err(|e| {
+        raise_error!(
+            format!("invalid hex in fjall key '{hex_str}': {e:#?}"),
+            ErrorCode::InternalError
+        )
+    })?;
+    Ok(raw)
+}
+
+fn migrate_keyspace(
+    engine: &Engine,
+    db: &Database,
+    ks_name: &str,
+    label: &str,
+    batch_size: usize,
+) -> BichonResult<u64> {
+    let ks = db
+        .keyspace(ks_name, || {
+            panic!("{ks_name} keyspace not found in fjall database")
+        })
+        .map_err(|e| {
+            raise_error!(
+                format!("failed to open fjall keyspace '{ks_name}': {e:#?}"),
+                ErrorCode::InternalError
+            )
+        })?;
+
+    let pb = ProgressBar::new_spinner();
+    pb.set_style(
+        ProgressStyle::with_template("{spinner:.cyan} {msg} [{elapsed_precise}]").unwrap(),
+    );
+    pb.set_message(format!("Scanning {label} blobs..."));
+
+    let mut count: u64 = 0;
+    let mut batch: Vec<([u8; 32], Vec<u8>, Codec)> = Vec::with_capacity(batch_size);
+
+    for item in ks.iter() {
+        let (key_bytes, value) = item.into_inner().map_err(|e| {
+            raise_error!(
+                format!("fjall iter error in '{ks_name}': {e:#?}"),
+                ErrorCode::InternalError
+            )
+        })?;
+
+        if value.is_empty() {
+            continue;
+        }
+        // MAX_VALUE_SIZE = 100 MB (bichon_blob::types)
+        if value.len() > 100 * 1024 * 1024 {
+            let raw_key = hex_key_to_raw(&key_bytes)?;
+            eprintln!(
+                "{}",
+                console::style(format!(
+                    "WARN: skipping oversized blob key={} ({} bytes)",
+                    hex::encode(raw_key),
+                    value.len()
+                ))
+                .yellow()
+            );
+            continue;
+        }
+        let raw_key = hex_key_to_raw(&key_bytes)?;
+        batch.push((raw_key, value.to_vec(), Codec::Zstd));
+
+        if batch.len() >= batch_size {
+            engine
+                .put_batch(&batch)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+            count += batch.len() as u64;
+            pb.set_message(format!("{label}: {} blobs migrated...", count));
+            batch.clear();
+        }
+    }
+
+    if !batch.is_empty() {
+        engine
+            .put_batch(&batch)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        count += batch.len() as u64;
+    }
+
+    pb.finish_with_message(format!("{label}: {} blobs migrated", count));
+    Ok(count)
+}
+
+pub fn handle_migrate_v1(theme: &ColorfulTheme) {
+    println!(
+        "\n{}",
+        style("MIGRATION: Bichon v1.x Storage → v2.x (Fjall → bichon-blob)")
+            .bold()
+            .yellow()
+    );
+    println!(
+        "{}\n",
+        style(
+            "This migrates blob storage from the fjall engine to bichon-blob.\n\
+              Tantivy indexes and metadata (memdb) are NOT affected."
+        )
+        .dim()
+    );
+
+    let root_dir: String = Input::with_theme(theme)
+        .with_prompt("Enter --bichon-root-dir (same value used by the old server)")
+        .validate_with(|input: &String| -> Result<(), &str> {
+            let path = PathBuf::from(input);
+            if !path.is_absolute() {
+                return Err("Path must be absolute.");
+            }
+            if !path.exists() {
+                return Err("Directory does not exist.");
+            }
+            Ok(())
+        })
+        .interact_text()
+        .unwrap();
+    let root_dir = PathBuf::from(root_dir.trim());
+
+    let data_base = {
+        let input: String = Input::with_theme(theme)
+            .with_prompt("Enter --bichon-data-dir (leave blank to use root directory)")
+            .allow_empty(true)
+            .interact_text()
+            .unwrap();
+        if input.trim().is_empty() {
+            root_dir.clone()
+        } else {
+            let path = PathBuf::from(input.trim());
+            if !path.exists() {
+                eprintln!(
+                    "{}",
+                    style(format!("Data directory does not exist: {}", path.display())).red()
+                );
+                return;
+            }
+            path
+        }
+    };
+
+    let fjall_path = data_base.join("bichon-storage");
+    let blob_path = fjall_path.join("blobs");
+
+    if !fjall_path.exists() {
+        println!(
+            "{}",
+            style(format!(
+                "Fjall database not found at '{}'. Is this really a v1.x install?",
+                fjall_path.display()
+            ))
+            .red()
+        );
+        return;
+    }
+
+    if blob_path.exists() {
+        println!(
+            "{}",
+            style(format!(
+                "Target blob directory '{}' already exists.\n\
+                 If you have already migrated, you can remove the old fjall files manually.\n\
+                 Otherwise, delete this directory and re-run the migration.",
+                blob_path.display()
+            ))
+            .yellow()
+        );
+        return;
+    }
+
+    let batch_size: usize = {
+        let input: String = Input::with_theme(theme)
+            .with_prompt(
+                "Enter batch size (affects memory usage, higher = faster but uses more RAM)",
+            )
+            .default("1000".to_string())
+            .validate_with(|s: &String| match s.trim().parse::<usize>() {
+                Ok(n) if n > 0 => Ok(()),
+                _ => Err("Please enter a valid positive number"),
+            })
+            .interact_text()
+            .unwrap_or("1000".to_string());
+        input.trim().parse::<usize>().unwrap_or(1000)
+    };
+
+    println!(
+        "{} Using batch size: {}\n",
+        style("✓").green(),
+        style(batch_size).cyan().bold()
+    );
+
+    // Open old Fjall database (read-only by nature of the iter API)
+    println!("\n{}", style("Opening fjall database...").dim());
+    let db = match Database::open(FjallConfig::new(&fjall_path)) {
+        Ok(db) => db,
+        Err(e) => {
+            println!(
+                "{}",
+                style(format!("Failed to open fjall database: {e:#?}")).red()
+            );
+            return;
+        }
+    };
+
+    // Open new bichon-blob engine
+    println!("{}", style("Initializing bichon-blob engine...").dim());
+    let mut config = Config::default();
+    config.default_codec = Codec::Zstd;
+    config.compress_threshold = 1024;
+    config.flush_interval_secs = 0;
+    config.gc_interval_secs = 0;
+
+    let engine = match Engine::open(&blob_path, config) {
+        Ok(e) => e,
+        Err(e) => {
+            println!(
+                "{}",
+                style(format!("Failed to open bichon-blob engine: {e:#?}")).red()
+            );
+            return;
+        }
+    };
+
+    // Migrate email blobs
+    let email_count = match migrate_keyspace(&engine, &db, "email", "Email", batch_size) {
+        Ok(n) => n,
+        Err(e) => {
+            println!("{}", style(format!("Email migration failed: {e:#?}")).red());
+            let _ = engine.shutdown();
+            return;
+        }
+    };
+
+    // Migrate attachment blobs
+    let attach_count = match migrate_keyspace(&engine, &db, "attachments", "Attachment", batch_size)
+    {
+        Ok(n) => n,
+        Err(e) => {
+            println!(
+                "{}",
+                style(format!("Attachment migration failed: {e:#?}")).red()
+            );
+            let _ = engine.shutdown();
+            return;
+        }
+    };
+
+    // Flush and shutdown
+    println!(
+        "\n{}",
+        style("Flushing and shutting down blob engine...").dim()
+    );
+    if let Err(e) = engine.flush() {
+        println!("{}", style(format!("flush warning: {e:#?}")).yellow());
+    }
+    if let Err(e) = engine.shutdown() {
+        println!("{}", style(format!("shutdown error: {e:#?}")).red());
+        return;
+    }
+
+    // Write STORAGE_VERSION = 2
+    if let Err(e) = write_storage_version(&root_dir, 2) {
+        println!(
+            "{}",
+            style(format!("Failed to write STORAGE_VERSION: {e:#?}")).red()
+        );
+        return;
+    }
+
+    println!(
+        "\n{}",
+        style(format!(
+            "✅ Migration complete!\n\
+         \n\
+         📊 {} email blobs, {} attachment blobs migrated\n\
+         \n\
+         📖 **Next steps:**\n\
+         Refer to the official migration guide for:\n\
+         • How to verify the new storage\n\
+         • Cleanup commands for legacy files\n\
+         • Rollback instructions if needed\n\
+         \n\
+         🔗 {}\n\
+         \n\
+         ⚠️  **Important:** Old data is preserved until you manually remove it.\n\
+         Do not delete anything until you have verified the new server works correctly.",
+            email_count,
+            attach_count,
+            "https://github.com/rustmailer/bichon/wiki/Bichon-v2.x-Migration-Guide"
+        ))
+        .green()
+        .bold()
+    );
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use bichon_core::utils::compute_content_hash;
+
+    // ── hex_key_to_raw ────────────────────────────────────────────────
+
+    #[test]
+    fn hex_key_to_raw_valid() {
+        // "hello" blake3 hex = 64 chars
+        let hash_hex = compute_content_hash(b"hello");
+        assert_eq!(hash_hex.len(), 64);
+
+        let raw = hex_key_to_raw(hash_hex.as_bytes()).unwrap();
+        // Decoding 64 hex chars → 32 bytes
+        assert_eq!(raw.len(), 32);
+        // Round-trip: raw → hex should match original
+        assert_eq!(hex::encode(raw), hash_hex);
+    }
+
+    #[test]
+    fn hex_key_to_raw_invalid_utf8() {
+        // 0xFF is not valid UTF-8
+        let invalid = vec![0xFFu8; 64];
+        let err = hex_key_to_raw(&invalid).unwrap_err();
+        assert!(err.to_string().contains("invalid UTF-8"));
+    }
+
+    #[test]
+    fn hex_key_to_raw_invalid_hex() {
+        // "zz" is valid UTF-8 but not valid hex
+        let invalid = b"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz";
+        let err = hex_key_to_raw(invalid).unwrap_err();
+        assert!(err.to_string().contains("invalid hex"));
+    }
+
+    #[test]
+    fn hex_key_to_raw_wrong_length() {
+        let short = b"abcd";
+        let err = hex_key_to_raw(short).unwrap_err();
+        assert!(err.to_string().contains("invalid hex"));
+    }
+
+    #[test]
+    fn hex_key_to_raw_different_content() {
+        let a = hex_key_to_raw(compute_content_hash(b"a").as_bytes()).unwrap();
+        let b = hex_key_to_raw(compute_content_hash(b"b").as_bytes()).unwrap();
+        assert_ne!(a, b);
+    }
+
+    // ── migrate_keyspace integration ──────────────────────────────────
+
+    #[test]
+    fn migrate_keyspace_end_to_end() {
+        let tmp = tempfile::tempdir().unwrap();
+        let fjall_path = tmp.path().join("fjall");
+        let blob_path = tmp.path().join("blobs");
+
+        use fjall::KeyspaceCreateOptions;
+
+        // --- Setup: create a Fjall database with test blobs ---
+        let fjall_db = Database::open(FjallConfig::new(&fjall_path)).unwrap();
+        let ks = fjall_db
+            .keyspace("test_ks", KeyspaceCreateOptions::default)
+            .unwrap();
+
+        // Insert test blobs using hex string keys (matching v1.x convention)
+        let mut expected: Vec<(String, Vec<u8>)> = Vec::new();
+        for i in 0..10 {
+            let data = format!("blob data {}", i).into_bytes();
+            let hash = compute_content_hash(&data);
+            ks.insert(hash.as_bytes(), data.clone()).unwrap();
+            expected.push((hash, data));
+        }
+
+        // --- Setup: create bichon-blob engine and run migration ---
+        {
+            let mut config = Config::default();
+            config.flush_interval_secs = 0;
+            config.gc_interval_secs = 0;
+            let engine = Engine::open(&blob_path, config).unwrap();
+
+            let count = migrate_keyspace(&engine, &fjall_db, "test_ks", "Test", 100).unwrap();
+            assert_eq!(count, expected.len() as u64);
+
+            engine.flush().unwrap();
+            engine.shutdown().unwrap();
+            // engine dropped here → LOCK released
+        }
+
+        // --- Verify: re-open engine and check all blobs ---
+        let mut config = Config::default();
+        config.flush_interval_secs = 0;
+        config.gc_interval_secs = 0;
+        let engine2 = Engine::open(&blob_path, config).unwrap();
+
+        for (hex_hash, expected_data) in &expected {
+            let mut raw_key = [0u8; 32];
+            hex::decode_to_slice(hex_hash, &mut raw_key).unwrap();
+            let got = engine2.get(&raw_key).unwrap();
+            assert_eq!(
+                got.as_deref(),
+                Some(expected_data.as_slice()),
+                "mismatch for key {}",
+                hex_hash
+            );
+        }
+
+        // Verify non-existent key returns None
+        let fake_hash = compute_content_hash(b"nonexistent");
+        let mut fake_key = [0u8; 32];
+        hex::decode_to_slice(&fake_hash, &mut fake_key).unwrap();
+        assert!(engine2.get(&fake_key).unwrap().is_none());
+
+        engine2.shutdown().unwrap();
+    }
+
+    #[test]
+    fn migrate_empty_keyspace() {
+        let tmp = tempfile::tempdir().unwrap();
+        let fjall_path = tmp.path().join("fjall");
+        let blob_path = tmp.path().join("blobs");
+
+        let fjall_db = Database::open(FjallConfig::new(&fjall_path)).unwrap();
+        let _ks = fjall_db
+            .keyspace("empty_ks", fjall::KeyspaceCreateOptions::default)
+            .unwrap();
+
+        let mut config = Config::default();
+        config.flush_interval_secs = 0;
+        config.gc_interval_secs = 0;
+        let engine = Engine::open(&blob_path, config).unwrap();
+
+        let count = migrate_keyspace(&engine, &fjall_db, "empty_ks", "Empty", 100).unwrap();
+        assert_eq!(count, 0);
+
+        engine.shutdown().unwrap();
+    }
+
+    #[test]
+    fn hex_key_roundtrip_with_real_content_hash() {
+        // Simulate the exact data flow from v1.x to v2.x
+        let eml_data = b"From: sender@example.com\r\nSubject: Test\r\n\r\nHello world";
+        let hash_hex = compute_content_hash(eml_data); // 64-char hex string
+
+        // v1.x: key stored as hash_hex.as_bytes()
+        let fjall_key = hash_hex.as_bytes().to_vec();
+        assert_eq!(fjall_key.len(), 64);
+
+        // Migration: hex decode → raw 32 bytes
+        let raw_key = hex_key_to_raw(&fjall_key).unwrap();
+        assert_eq!(raw_key.len(), 32);
+
+        // v2.x: engine.put(raw_key, data)
+        // Verify round-trip: raw_key → hex → compare
+        let hex_roundtrip = hex::encode(raw_key);
+        assert_eq!(hex_roundtrip, hash_hex);
+    }
+}

crates/blob/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "bichon-blob"
+version = "0.1.0"
+edition = "2021"
+description = "Embedded KV storage engine for email"
+
+[dependencies]
+crc32fast = "1.4"
+zstd = "0.13"
+lz4_flex = "0.13.1"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+bincode = { package = "bincode_reloaded", version = "3.1.10", default-features = false, features = ["serde", "alloc"] }
+tracing = "0.1"
+thiserror = "2"
+redb = "4.1"
+fs2 = "0.4"
+
+[dev-dependencies]
+tempfile = "3"
+rand = "0.10.2"
+criterion = { version = "0.6", features = ["html_reports"] }
+
+[[bench]]
+name = "benchmark"
+harness = false

crates/blob/README.md

@@ -0,0 +1,177 @@
+# bichon-blob
+
+Content-addressable blob store for [Bichon](https://github.com/rustmailer/bichon) email archival.
+
+All data is keyed by a 32-byte content hash — identical content is stored only once. The caller (Bichon) strips attachments from emails and stores the "holed" raw email + attachments into this store, tracking reference counts in its own schema.
+
+## On-disk layout
+
+```
+<root>/
+├── meta.bin              # global metadata (bincode + CRC32)
+├── index.redb            # key → (segment_id, offset, size) index (redb B-tree)
+├── segments/
+│   ├── 00000001.seg      # append-only segment files (≤ 1 GB each)
+│   ├── 00000002.seg
+│   └── ...
+```
+
+### Segment entry format (50-byte fixed header + variable data)
+
+```
+magic(4)  crc32(4)  flags(1)  codec(1)  key(32)  raw_size(4)  data_size(4)  data(*)
+```
+
+- `magic`: `0xB3DB_0001` — entry boundary validation
+- `crc32`: covers everything after this field
+- `flags`: `0` = live, `1` = tombstone
+- `codec`: `0` = none, `1` = Zstd, `2` = Lz4
+- `key`: 32-byte content hash (BLAKE3 / SHA-256)
+- `raw_size`: original uncompressed size
+- `data_size`: on-disk data size (after compression)
+
+### Index store
+
+The key → (segment_id, offset, data_size, flags) mapping is stored in a single [redb](https://github.com/cberner/redb) database (`index.redb`). redb provides:
+
+- **B-tree + mmap**: O(log N) point lookups with zero heap allocation — pages are faulted in on demand.
+- **ACID transactions**: every index write is durable and atomic.
+- **Crash recovery**: handled transparently by redb's WAL — no manual reload or rebuild logic.
+- **O(1) startup**: only the B-tree root page is read at open time.
+
+Records are stored as fixed-size 56-byte blobs, each carrying an internal CRC32 checksum.
+
+## Read path
+
+```
+get(key)
+  → index_store.get(key)           # redb B-tree lookup, zero-copy
+  → IndexRecord CRC32 verify       # (segment_id, offset, data_size, flags)
+  → pread entry from segment file
+  → entry CRC32 verify → decompress → return value
+```
+
+The index record and the segment entry carry independent CRC32 checksums. Corruption in one record or entry is contained — it never affects other keys.
+
+## Write path
+
+```
+put(key, value, codec)
+  → compress value (Zstd/Lz4 if ≥ 4 KB, else store raw)
+  → append entry to active segment file
+  → insert IndexRecord into redb (single write txn)
+  → update metadata (indexed_up_to_offset)
+  → if segment ≥ 1 GB → seal it, create new segment
+```
+
+## Delete
+
+Deletes are **tombstones** — an entry with `flags=1` and empty data is appended to the active segment. Before writing the tombstone, the existing index record is consulted to increment `deleted_bytes` on the **original** segment (the one that holds the live data). This drives the GC threshold.
+
+```
+delete(key)
+  → index_store.get(key) → find original (segment_id, data_size)
+  → original_segment.deleted_bytes += data_size
+  → recompute deleted_ratio on original segment
+  → append tombstone entry to active segment
+  → insert tombstone IndexRecord into redb
+  → index_store.get(key) now returns None
+```
+
+## GC
+
+**Segment GC:** two-phase, driven by the `deleted_ratio` tracked per segment.
+
+### Trigger
+
+- **Background**: the `blob-gc` thread wakes up every `gc_interval_secs` (default 300s), checks whether any sealed segment's `deleted_ratio ≥ gc_deleted_ratio` (default 0.30), and runs GC on the worst segment if so.
+- **Manual**: `engine.gc_if_needed()` or `engine.gc()`.
+
+### Phase 1 — scan & compact (read-only, no write lock)
+
+1. Pick the sealed segment with the highest `deleted_ratio`.
+2. Scan only that segment's entries.
+3. For each entry, ask the index: "is this entry still the latest version for its key?"
+   - If the index points to this exact `(segment_id, offset)` → **keep**, write to a temp segment file.
+   - If the index points elsewhere (overwritten by a later segment, or a tombstone) → **skip** (stale).
+4. Fsync the temp file.
+
+Phase 1 holds only the read lock — `put` / `delete` continue uninterrupted.
+
+### Phase 2 — commit & update index (write lock)
+
+1. Atomically rename the temp file over the original segment.
+2. Batch-insert new `IndexRecord`s (now at new offsets) into redb. Old records with the same key are naturally overwritten.
+3. Reset the segment's `deleted_bytes` and `deleted_ratio` to zero.
+4. Persist metadata.
+
+Phase 2 holds the write lock, but is fast — no full segment scan, no full index rebuild.
+
+## Data integrity
+
+Every record on disk is independently checksummed:
+
+| Layer | Format | Protection |
+|---|---|---|
+| Segment entry | 50-byte header + data | CRC32 covers all fields + data |
+| Index record | 56 bytes | CRC32 covers key + segment_id + offset + data_size + flags |
+| Global metadata | bincode blob | CRC32 + version header |
+
+Corruption is **contained** — a bad segment entry or index record produces an error for that key only. Recovery and GC skip corrupt records (with a warning) rather than aborting. The index is backed by redb's B-tree which maintains its own internal integrity.
+
+## Crash recovery
+
+- Temp files from interrupted GC are cleaned up on open.
+- Any segment data beyond `indexed_up_to_offset` is scanned and inserted into the index.
+- Partial writes at the tail of a segment (detected via CRC32 mismatch near EOF) are truncated.
+- redb's WAL ensures the index is always consistent — no manual reload or rebuild needed.
+
+## Background threads
+
+Set `Config.flush_interval_secs` and `Config.gc_interval_secs` to positive values to enable periodic background work:
+
+| Thread | Config | Default | What it does |
+|---|---|---|---|
+| `blob-flush` | `flush_interval_secs` | `0` (off) | Fsync the active segment and save metadata |
+| `blob-gc` | `gc_interval_secs` | `0` (off) | Check deleted-ratio, compact one segment if needed |
+
+The two threads are independent — a long GC run never blocks fsync.
+
+## Config
+
+| Field | Default | Notes |
+|---|---|---|
+| `compress_threshold` | 4096 | Bytes; smaller values stored uncompressed |
+| `default_codec` | Zstd | Also supports Lz4 |
+| `compression_level` | 0 | Zstd compression level |
+| `gc_deleted_ratio` | 0.30 | Trigger GC when a sealed segment exceeds this |
+| `flush_interval_secs` | 0 | 0 = disabled; ≥ 5 for periodic background fsync |
+| `gc_interval_secs` | 0 | 0 = disabled; ≥ 10 for periodic background GC |
+
+## Basic usage
+
+```rust
+use bichon_blob::{Codec, Config, Engine};
+
+let engine = Engine::open(path, Config::default())?;
+
+// Store
+let hash = blake3::hash(b"email body").into();
+engine.put(hash, b"email body", Codec::Zstd)?;
+
+// Retrieve
+let value = engine.get(&hash)?; // Some(Vec<u8>) or None
+
+// Delete (caller must track refcounts)
+engine.delete(&hash)?;
+
+// Batch operations
+engine.put_batch(&[(hash1, data1, Codec::Zstd), (hash2, data2, Codec::Lz4)])?;
+engine.delete_batch(&[hash1, hash2])?;
+
+// GC
+engine.gc_if_needed()?;
+
+// Clean shutdown
+engine.shutdown()?;
+```

crates/blob/benches/benchmark.rs

@@ -0,0 +1,293 @@
+use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput};
+use std::time::Duration;
+use tempfile::TempDir;
+
+use bichon_blob::{Codec, Config, Engine};
+
+fn make_key(seed: u64) -> [u8; 32] {
+    let mut key = [0u8; 32];
+    key[0..8].copy_from_slice(&seed.to_le_bytes());
+    key
+}
+
+fn make_value(size: usize) -> Vec<u8> {
+    let mut v = Vec::with_capacity(size);
+    let pattern = b"The quick brown fox jumps over the lazy dog. ";
+    while v.len() < size {
+        let rem = size - v.len();
+        let n = rem.min(pattern.len());
+        v.extend_from_slice(&pattern[..n]);
+    }
+    v
+}
+
+pub fn bench_write_small(c: &mut Criterion) {
+    let mut group = c.benchmark_group("write");
+    group.throughput(Throughput::Elements(1));
+    group.measurement_time(Duration::from_secs(10));
+
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let value = make_value(1024); // 1 KB
+    let mut counter = 0u64;
+
+    group.bench_function("1KB", |b| {
+        b.iter_batched(
+            || {
+                counter += 1;
+                (make_key(counter), value.clone())
+            },
+            |(key, val)| {
+                engine.put(key, &val, Codec::Zstd).unwrap()
+            },
+            BatchSize::SmallInput,
+        )
+    });
+    group.finish();
+}
+
+pub fn bench_write_medium(c: &mut Criterion) {
+    let mut group = c.benchmark_group("write");
+    group.throughput(Throughput::Bytes(64 * 1024));
+    group.measurement_time(Duration::from_secs(10));
+
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let value = make_value(64 * 1024); // 64 KB
+    let mut counter = 0u64;
+
+    group.bench_function("64KB", |b| {
+        b.iter_batched(
+            || {
+                counter += 1;
+                (make_key(counter), value.clone())
+            },
+            |(key, val)| {
+                engine.put(key, &val, Codec::Zstd).unwrap()
+            },
+            BatchSize::SmallInput,
+        )
+    });
+    group.finish();
+}
+
+pub fn bench_write_large(c: &mut Criterion) {
+    let mut group = c.benchmark_group("write");
+    group.throughput(Throughput::Bytes(1024 * 1024));
+    group.measurement_time(Duration::from_secs(15));
+
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let value = make_value(1024 * 1024); // 1 MB
+    let mut counter = 0u64;
+
+    group.bench_function("1MB", |b| {
+        b.iter_batched(
+            || {
+                counter += 1;
+                (make_key(counter), value.clone())
+            },
+            |(key, val)| {
+                engine.put(key, &val, Codec::Zstd).unwrap()
+            },
+            BatchSize::SmallInput,
+        )
+    });
+    group.finish();
+}
+
+pub fn bench_read_hot(c: &mut Criterion) {
+    let mut group = c.benchmark_group("read");
+    group.throughput(Throughput::Elements(1));
+    group.measurement_time(Duration::from_secs(10));
+
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    // Pre-populate: 10 keys
+    let value = make_value(4096);
+    for i in 0..10u64 {
+        engine
+            .put(make_key(i), &value, Codec::Zstd)
+            .unwrap();
+    }
+
+    let mut counter = 0u64;
+    group.bench_function("hot", |b| {
+        b.iter(|| {
+            let key = make_key(counter % 10);
+            counter += 1;
+            std::hint::black_box(engine.get(&key).unwrap());
+        })
+    });
+    group.finish();
+}
+
+pub fn bench_read_cold(c: &mut Criterion) {
+    let mut group = c.benchmark_group("read");
+    group.throughput(Throughput::Elements(1));
+    group.measurement_time(Duration::from_secs(10));
+
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let value = make_value(4096);
+    // Write 5000 keys across all 256 buckets — mmap page faults will occur
+    for i in 0..5000u64 {
+        engine
+            .put(make_key(i), &value, Codec::Zstd)
+            .unwrap();
+    }
+
+    let mut counter = 0u64;
+    group.bench_function("cold", |b| {
+        b.iter(|| {
+            let key = make_key(counter % 5000);
+            counter += 1;
+            std::hint::black_box(engine.get(&key).unwrap());
+        })
+    });
+    group.finish();
+}
+
+pub fn bench_read_large_value(c: &mut Criterion) {
+    let mut group = c.benchmark_group("read");
+    group.throughput(Throughput::Bytes(1024 * 1024));
+    group.measurement_time(Duration::from_secs(10));
+
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let value = make_value(1024 * 1024); // 1 MB
+    for i in 0..5u64 {
+        engine
+            .put(make_key(i), &value, Codec::Zstd)
+            .unwrap();
+    }
+
+    let mut counter = 0u64;
+    group.bench_function("1MB", |b| {
+        b.iter(|| {
+            let key = make_key(counter % 5);
+            counter += 1;
+            std::hint::black_box(engine.get(&key).unwrap());
+        })
+    });
+    group.finish();
+}
+
+pub fn bench_delete(c: &mut Criterion) {
+    let mut group = c.benchmark_group("delete");
+    group.throughput(Throughput::Elements(1));
+    group.measurement_time(Duration::from_secs(10));
+
+    group.bench_function("delete", |b| {
+        let dir = TempDir::new().unwrap();
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+        let value = make_value(4096);
+        let mut counter = 0u64;
+
+        b.iter_batched(
+            || {
+                counter += 1;
+                let key = make_key(counter);
+                engine.put(key, &value, Codec::Zstd).unwrap();
+                key
+            },
+            |key| {
+                engine.delete(&key).unwrap();
+            },
+            BatchSize::SmallInput,
+        )
+    });
+    group.finish();
+}
+
+pub fn bench_mixed_workload(c: &mut Criterion) {
+    let mut group = c.benchmark_group("mixed");
+    group.throughput(Throughput::Elements(1));
+    group.measurement_time(Duration::from_secs(15));
+
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    // Pre-populate with 500 entries
+    let value = make_value(8192);
+    for i in 0..500u64 {
+        engine
+            .put(make_key(i), &value, Codec::Zstd)
+            .unwrap();
+    }
+
+    let mut counter: u64 = 500;
+    group.bench_function("80w_15r_5d", |b| {
+        b.iter(|| {
+            counter += 1;
+            let op = counter % 100;
+            match op {
+                0..=79 => {
+                    let key = make_key(counter);
+                    let val = make_value(4096);
+                    engine.put(key, &val, Codec::Zstd).unwrap();
+                }
+                80..=94 => {
+                    std::hint::black_box(
+                        engine.get(&make_key(counter % 500)).unwrap(),
+                    );
+                }
+                _ => {
+                    if counter % 2 == 0 {
+                        let key = make_key(counter % 500);
+                        let _ = engine.delete(&key);
+                    }
+                }
+            }
+        })
+    });
+    group.finish();
+}
+
+pub fn bench_gc(c: &mut Criterion) {
+    let mut group = c.benchmark_group("gc");
+    group.measurement_time(Duration::from_secs(30));
+    group.sample_size(10);
+
+    group.bench_function("gc_30pct_deleted", |b| {
+        let dir = TempDir::new().unwrap();
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+        let value = make_value(200_000);
+        let n = 1200u64;
+        for i in 0..n {
+            engine
+                .put(make_key(i), &value, Codec::None)
+                .unwrap();
+        }
+        for i in (0..n).step_by(3) {
+            engine.delete(&make_key(i)).unwrap();
+        }
+
+        b.iter(|| {
+            engine.gc().unwrap();
+        })
+    });
+    group.finish();
+}
+
+criterion_group!(
+    benches,
+    bench_write_small,
+    bench_write_medium,
+    bench_write_large,
+    bench_read_hot,
+    bench_read_cold,
+    bench_read_large_value,
+    bench_delete,
+    bench_mixed_workload,
+    bench_gc,
+);
+criterion_main!(benches);

crates/blob/examples/email_archive.rs

@@ -0,0 +1,125 @@
+/// bichon-blob usage example: email archival with content-addressable storage.
+///
+/// This example simulates a mail archival system where multiple accounts
+/// may receive the same email (e.g. CC'd or forwarded).  The blob store
+/// deduplicates by content hash, and the upper application layer tracks
+/// which accounts reference each hash.
+///
+/// Run: cargo run --example email_archive
+
+use std::collections::HashMap;
+
+use bichon_blob::{Codec, Config, Engine};
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // ── Setup ──────────────────────────────────────────────────────────
+
+    let store_path = std::path::Path::new("target/example_blob_store");
+    let _ = std::fs::remove_dir_all(store_path); // clean up from previous run
+    let engine = Engine::open(store_path, Config::default())?;
+
+    println!("Store opened at: {:?}\n", store_path);
+
+    // ── Simulated upper-layer reference tracker ────────────────────────
+    let mut refs: HashMap<String, Vec<[u8; 32]>> = HashMap::new();
+
+    // ── 1. Store emails ────────────────────────────────────────────────
+
+    // Simulate three emails arriving.  Email #2 is a newsletter that
+    // both alice and bob received — identical content, same hash.
+    let emails = vec![
+        ("alice", "Welcome to Bichon Mail!"),
+        ("alice", "Weekly Newsletter: Rust Edition"),
+        ("bob", "Weekly Newsletter: Rust Edition"), // same content as above
+    ];
+
+    for (account, body) in &emails {
+        let hash = mock_content_hash(body.as_bytes());
+        let account_refs = refs.entry(account.to_string()).or_default();
+
+        // Check if this content already exists in the blob store
+        if engine.exists(&hash)? {
+            println!(
+                "[dedup] {}: hash {:02x?}... already stored, skipping",
+                account,
+                &hash[..4]
+            );
+        } else {
+            engine.put(hash, body.as_bytes(), Codec::Zstd)?;
+            println!(
+                "[store] {}: hash {:02x?}..., {} bytes",
+                account,
+                &hash[..4],
+                body.len()
+            );
+        }
+
+        account_refs.push(hash);
+    }
+
+    println!();
+
+    // ── 2. Read back emails ────────────────────────────────────────────
+
+    let hash = mock_content_hash(b"Weekly Newsletter: Rust Edition");
+    let stored = engine.get(&hash)?;
+    println!(
+        "Read newsletter: {:?}",
+        stored.map(|v| String::from_utf8_lossy(&v).to_string())
+    );
+
+    // ── 3. Batch store attachments ─────────────────────────────────────
+
+    let attachments: Vec<([u8; 32], Vec<u8>, Codec)> = (0..10)
+        .map(|i| {
+            let body = format!("Attachment #{}: {}", i, "X".repeat(5000));
+            let hash = mock_content_hash(body.as_bytes());
+            (hash, body.into_bytes(), Codec::Zstd)
+        })
+        .collect();
+
+    engine.put_batch(&attachments)?;
+    println!("\nBatch-stored {} attachments", attachments.len());
+
+    // ── 4. Stats ───────────────────────────────────────────────────────
+
+    let stats = engine.stats()?;
+    println!(
+        "Stats: {} keys, {} bytes, {} segments",
+        stats.total_keys, stats.total_bytes, stats.segment_count
+    );
+
+    // ── 5. Delete an email (simulating: last reference removed) ─────────
+
+    // In production, before calling engine.delete(), you'd check:
+    //   SELECT COUNT(*) FROM email_refs WHERE content_hash = ? AND account_id != ?
+    // If count == 0, it's safe to delete from blob.
+
+    let welcome_hash = mock_content_hash(b"Welcome to Bichon Mail!");
+    engine.delete(&welcome_hash)?;
+    println!("\nDeleted welcome email, still exists? {}", engine.exists(&welcome_hash)?);
+
+    // ── 6. Shutdown ────────────────────────────────────────────────────
+
+    engine.shutdown()?;
+    println!("\nClean shutdown complete.");
+
+    Ok(())
+}
+
+/// In production, use BLAKE3 or SHA-256.
+/// Here we use a trivial hash for demonstration.
+fn mock_content_hash(data: &[u8]) -> [u8; 32] {
+    use std::collections::hash_map::DefaultHasher;
+    use std::hash::{Hash, Hasher};
+
+    let mut hasher = DefaultHasher::new();
+    data.hash(&mut hasher);
+    let h = hasher.finish();
+
+    let mut key = [0u8; 32];
+    key[0..8].copy_from_slice(&h.to_le_bytes());
+    // Mix in the length so different-sized content gets different hashes
+    key[8..16].copy_from_slice(&(data.len() as u64).to_le_bytes());
+    key
+}

crates/blob/src/bucket.rs

@@ -0,0 +1,257 @@
+use std::path::Path;
+
+use redb::{Database, ReadableTable, TableDefinition};
+use redb::ReadableDatabase;
+
+use crate::error::Result;
+use crate::types::INDEX_RECORD_SIZE;
+
+// ── IndexRecord ──────────────────────────────────────────────────────────────
+
+/// On-disk format: 52 bytes per record + 4 bytes CRC = 56 bytes total.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct IndexRecord {
+    pub key: [u8; 32],
+    pub segment_id: u32,
+    pub offset: u64,
+    pub data_size: u32,
+    pub flags: u8,
+}
+
+impl IndexRecord {
+    pub fn new(key: [u8; 32], segment_id: u32, offset: u64, data_size: u32, flags: u8) -> Self {
+        Self {
+            key,
+            segment_id,
+            offset,
+            data_size,
+            flags,
+        }
+    }
+
+    pub fn is_tombstone(&self) -> bool {
+        self.flags == 1
+    }
+
+    pub fn encode(&self) -> [u8; INDEX_RECORD_SIZE] {
+        let mut buf = [0u8; INDEX_RECORD_SIZE];
+        buf[0..32].copy_from_slice(&self.key);
+        buf[32..36].copy_from_slice(&self.segment_id.to_le_bytes());
+        buf[36..44].copy_from_slice(&self.offset.to_le_bytes());
+        buf[44..48].copy_from_slice(&self.data_size.to_le_bytes());
+        buf[48] = self.flags;
+        let crc = crate::checksum::crc32(&buf[..52]);
+        buf[52..56].copy_from_slice(&crc.to_le_bytes());
+        buf
+    }
+
+    pub fn decode(buf: &[u8; INDEX_RECORD_SIZE]) -> crate::error::Result<Self> {
+        let stored_crc = u32::from_le_bytes(buf[52..56].try_into().unwrap());
+        let computed = crate::checksum::crc32(&buf[..52]);
+        if stored_crc != computed {
+            return Err(crate::error::Error::BucketIndexCorrupt {
+                path: std::path::PathBuf::new(),
+                reason: format!(
+                    "CRC mismatch: stored=0x{:08X} computed=0x{:08X}",
+                    stored_crc, computed
+                ),
+            });
+        }
+        let mut key = [0u8; 32];
+        key.copy_from_slice(&buf[0..32]);
+        let segment_id = u32::from_le_bytes(buf[32..36].try_into().unwrap());
+        let offset = u64::from_le_bytes(buf[36..44].try_into().unwrap());
+        let data_size = u32::from_le_bytes(buf[44..48].try_into().unwrap());
+        let flags = buf[48];
+        Ok(Self {
+            key,
+            segment_id,
+            offset,
+            data_size,
+            flags,
+        })
+    }
+}
+
+// ── redb Value impl for fixed-size record bytes ──────────────────────────────
+
+/// Newtype wrapper so we can implement `redb::Value` for `[u8; INDEX_RECORD_SIZE]`.
+#[derive(Debug, Clone, Copy)]
+struct RecordBytes([u8; INDEX_RECORD_SIZE]);
+
+impl redb::Value for RecordBytes {
+    type SelfType<'a> = RecordBytes;
+    type AsBytes<'a> = [u8; INDEX_RECORD_SIZE];
+
+    fn fixed_width() -> Option<usize> {
+        Some(INDEX_RECORD_SIZE)
+    }
+
+    fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a>
+    where
+        Self: 'a
+    {
+        let mut arr = [0u8; INDEX_RECORD_SIZE];
+        arr.copy_from_slice(data);
+        RecordBytes(arr)
+    }
+
+    fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> {
+        value.0
+    }
+
+    fn type_name() -> redb::TypeName {
+        redb::TypeName::new("IndexRecord")
+    }
+}
+
+// ── IndexStore ───────────────────────────────────────────────────────────────
+
+const INDEX_TABLE: TableDefinition<[u8; 32], RecordBytes> = TableDefinition::new("blob_index");
+
+/// Zero-heap key-value index backed by redb.
+///
+/// B-tree + mmap + ACID.  No manual compact / sort / dedup / per-bucket mmap
+/// management.  Startup is O(1) — redb reads only its root page.
+pub struct IndexStore {
+    db: Database,
+}
+
+impl IndexStore {
+    /// Open (or create) the index database at `dir/index.redb`.
+    pub fn open(dir: &Path) -> Result<Self> {
+        let path = dir.join("index.redb");
+        let db = Database::create(&path)
+            .map_err(|e| crate::error::Error::IndexDb(format!("failed to create index: {}", e)))?;
+        // Ensure the table exists so reads on a fresh database don't fail.
+        {
+            let txn = db
+                .begin_write()
+                .map_err(|e| crate::error::Error::IndexDb(format!("init write txn: {}", e)))?;
+            txn.open_table(INDEX_TABLE)
+                .map_err(|e| crate::error::Error::IndexDb(format!("init table: {}", e)))?;
+            txn.commit()
+                .map_err(|e| crate::error::Error::IndexDb(format!("init commit: {}", e)))?;
+        }
+        Ok(Self { db })
+    }
+
+    /// Look up a key. Returns the latest IndexRecord, or None if absent/tombstone.
+    pub fn get(&self, key: &[u8; 32]) -> Result<Option<IndexRecord>> {
+        let txn = self
+            .db
+            .begin_read()
+            .map_err(|e| crate::error::Error::IndexDb(format!("read txn: {}", e)))?;
+        let table = txn
+            .open_table(INDEX_TABLE)
+            .map_err(|e| crate::error::Error::IndexDb(format!("open table: {}", e)))?;
+
+        match table
+            .get(key)
+            .map_err(|e| crate::error::Error::IndexDb(format!("get: {}", e)))?
+        {
+            Some(guard) => {
+                let record = IndexRecord::decode(&guard.value().0)?;
+                Ok(if record.is_tombstone() { None } else { Some(record) })
+            }
+            None => Ok(None),
+        }
+    }
+
+    /// Check whether a key exists (non-tombstone) in the store.
+    pub fn exists(&self, key: &[u8; 32]) -> Result<bool> {
+        self.get(key).map(|r| r.is_some())
+    }
+
+    /// Insert or update a record for a key.  Committed in a single write txn.
+    pub fn insert(&self, record: &IndexRecord) -> Result<()> {
+        let txn = self
+            .db
+            .begin_write()
+            .map_err(|e| crate::error::Error::IndexDb(format!("write txn: {}", e)))?;
+        {
+            let mut table = txn
+                .open_table(INDEX_TABLE)
+                .map_err(|e| crate::error::Error::IndexDb(format!("open table: {}", e)))?;
+            table
+                .insert(&record.key, RecordBytes(record.encode()))
+                .map_err(|e| crate::error::Error::IndexDb(format!("insert: {}", e)))?;
+        }
+        txn.commit()
+            .map_err(|e| crate::error::Error::IndexDb(format!("commit: {}", e)))?;
+        Ok(())
+    }
+
+    /// Batch insert multiple records in a single write transaction.
+    pub fn insert_batch(&self, records: &[IndexRecord]) -> Result<()> {
+        if records.is_empty() {
+            return Ok(());
+        }
+        let txn = self
+            .db
+            .begin_write()
+            .map_err(|e| crate::error::Error::IndexDb(format!("write txn: {}", e)))?;
+        {
+            let mut table = txn
+                .open_table(INDEX_TABLE)
+                .map_err(|e| crate::error::Error::IndexDb(format!("open table: {}", e)))?;
+            for record in records {
+                table
+                    .insert(&record.key, RecordBytes(record.encode()))
+                    .map_err(|e| crate::error::Error::IndexDb(format!("insert: {}", e)))?;
+            }
+        }
+        txn.commit()
+            .map_err(|e| crate::error::Error::IndexDb(format!("commit: {}", e)))?;
+        Ok(())
+    }
+
+    /// Remove keys from the index in a single write transaction.
+    pub fn delete_batch(&self, keys: &[[u8; 32]]) -> Result<()> {
+        if keys.is_empty() {
+            return Ok(());
+        }
+        let txn = self
+            .db
+            .begin_write()
+            .map_err(|e| crate::error::Error::IndexDb(format!("write txn: {}", e)))?;
+        {
+            let mut table = txn
+                .open_table(INDEX_TABLE)
+                .map_err(|e| crate::error::Error::IndexDb(format!("open table: {}", e)))?;
+            for key in keys {
+                table
+                    .remove(key)
+                    .map_err(|e| crate::error::Error::IndexDb(format!("remove: {}", e)))?;
+            }
+        }
+        txn.commit()
+            .map_err(|e| crate::error::Error::IndexDb(format!("commit: {}", e)))?;
+        Ok(())
+    }
+
+    /// Total number of live (non-tombstone) keys.
+    pub fn total_keys(&self) -> Result<usize> {
+        let txn = self
+            .db
+            .begin_read()
+            .map_err(|e| crate::error::Error::IndexDb(format!("read txn: {}", e)))?;
+        let table = txn
+            .open_table(INDEX_TABLE)
+            .map_err(|e| crate::error::Error::IndexDb(format!("open table: {}", e)))?;
+
+        let mut count = 0usize;
+        let iter = table
+            .iter()
+            .map_err(|e| crate::error::Error::IndexDb(format!("iter: {}", e)))?;
+        for item in iter {
+            let (_, guard) =
+                item.map_err(|e| crate::error::Error::IndexDb(format!("iter next: {}", e)))?;
+            let record = IndexRecord::decode(&guard.value().0)?;
+            if !record.is_tombstone() {
+                count += 1;
+            }
+        }
+        Ok(count)
+    }
+}

crates/blob/src/checksum.rs

@@ -0,0 +1,65 @@
+use crc32fast::Hasher;
+
+pub fn crc32(data: &[u8]) -> u32 {
+    let mut h = Hasher::new();
+    h.update(data);
+    h.finalize()
+}
+
+pub struct CrcWriter {
+    hasher: Hasher,
+}
+
+impl Default for CrcWriter {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl CrcWriter {
+    pub fn new() -> Self {
+        Self {
+            hasher: Hasher::new(),
+        }
+    }
+
+    pub fn update(&mut self, data: &[u8]) {
+        self.hasher.update(data);
+    }
+
+    pub fn finalize(self) -> u32 {
+        self.hasher.finalize()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_crc32_deterministic() {
+        let a = crc32(b"hello");
+        let b = crc32(b"hello");
+        assert_eq!(a, b);
+    }
+
+    #[test]
+    fn test_crc32_different() {
+        let a = crc32(b"hello");
+        let b = crc32(b"world");
+        assert!(a != b);
+    }
+
+    #[test]
+    fn test_crc_writer_matches_crc32() {
+        let mut w = CrcWriter::new();
+        w.update(b"hello");
+        w.update(b" world");
+        assert_eq!(w.finalize(), crc32(b"hello world"));
+    }
+
+    #[test]
+    fn test_crc32_empty() {
+        assert_eq!(crc32(b""), 0);
+    }
+}

crates/blob/src/compress.rs

@@ -0,0 +1,97 @@
+use crate::types::Codec;
+
+pub fn compress(data: &[u8], codec: Codec, threshold: usize, level: i32) -> (Vec<u8>, Codec) {
+    if data.len() < threshold {
+        return (data.to_vec(), Codec::None);
+    }
+    let (compressed, actual_codec) = match codec {
+        Codec::Zstd => {
+            match zstd::encode_all(data, level) {
+                Ok(out) => (out, Codec::Zstd),
+                Err(e) => {
+                    tracing::warn!("zstd compression failed, storing uncompressed: {}", e);
+                    (data.to_vec(), Codec::None)
+                }
+            }
+        }
+        Codec::Lz4 => {
+            let out = lz4_flex::compress(data);
+            (out, Codec::Lz4)
+        }
+        Codec::None => (data.to_vec(), Codec::None),
+    };
+    // If compression made it larger, store uncompressed
+    if compressed.len() >= data.len() {
+        (data.to_vec(), Codec::None)
+    } else {
+        (compressed, actual_codec)
+    }
+}
+
+pub fn decompress(data: &[u8], codec: Codec, raw_size: usize) -> crate::error::Result<Vec<u8>> {
+    match codec {
+        Codec::None => Ok(data.to_vec()),
+        Codec::Zstd => {
+            zstd::decode_all(data)
+                .map_err(|e| crate::error::Error::Compression(format!("zstd decompress: {}", e)))
+        }
+        Codec::Lz4 => {
+            lz4_flex::decompress(data, raw_size)
+                .map_err(|e| crate::error::Error::Compression(format!("lz4 decompress: {}", e)))
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_small_data_not_compressed() {
+        let data = b"hi";
+        let (out, codec) = compress(data, Codec::Zstd, 4096, 0);
+        assert_eq!(out, b"hi");
+        assert_eq!(codec, Codec::None);
+    }
+
+    #[test]
+    fn test_large_data_compressed_zstd() {
+        let data = vec![b'A'; 5000];
+        let (out, codec) = compress(&data, Codec::Zstd, 4096, 0);
+        assert_eq!(codec, Codec::Zstd);
+        assert!(out.len() < data.len());
+    }
+
+    #[test]
+    fn test_roundtrip_zstd() {
+        let data = vec![b'B'; 10000];
+        let (compressed, codec) = compress(&data, Codec::Zstd, 4096, 0);
+        let decompressed = decompress(&compressed, codec, data.len()).unwrap();
+        assert_eq!(decompressed, data);
+    }
+
+    #[test]
+    fn test_roundtrip_lz4() {
+        let data = vec![b'C'; 10000];
+        let (compressed, codec) = compress(&data, Codec::Lz4, 4096, 0);
+        let decompressed = decompress(&compressed, codec, data.len()).unwrap();
+        assert_eq!(decompressed, data);
+    }
+
+    #[test]
+    fn test_roundtrip_none() {
+        let data = vec![b'D'; 100];
+        let (compressed, codec) = compress(&data, Codec::None, 4096, 0);
+        assert_eq!(codec, Codec::None);
+        let decompressed = decompress(&compressed, codec, data.len()).unwrap();
+        assert_eq!(decompressed, data);
+    }
+
+    #[test]
+    fn test_threshold_zero_always_compresses() {
+        let data = vec![b'E'; 100];
+        let (out, codec) = compress(&data, Codec::Zstd, 0, 0);
+        assert_eq!(codec, Codec::Zstd);
+        assert!(out.len() < data.len());
+    }
+}

crates/blob/src/engine.rs

@@ -0,0 +1,697 @@
+use std::fs;
+use std::fs::File;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex, RwLock};
+use std::thread::{self, JoinHandle};
+use std::time::Duration;
+
+use fs2::FileExt;
+
+use crate::bucket::{IndexRecord, IndexStore};
+use crate::compress;
+use crate::error::{Error, Result};
+use crate::file_pool::FilePool;
+use crate::gc::{self, GcStats};
+use crate::meta::{GlobalMeta, SegmentStats};
+use crate::segment::{self, SegmentReader, SegmentWriter};
+use crate::types::{Codec, Config, ENTRY_HEADER_SIZE};
+
+/// Global content-addressable blob store.
+///
+/// All data is keyed by a 32-byte content hash.  Identical content is stored
+/// only once.  The caller is responsible for tracking which entities reference
+/// which content hashes — this crate is a pure content-addressable KV engine.
+pub struct Engine {
+    shared: Arc<EngineShared>,
+    flush_handle: Mutex<Option<FlushHandle>>,
+    gc_handle: Mutex<Option<GcHandle>>,
+}
+
+struct EngineShared {
+    config: Config,
+    inner: RwLock<EngineInner>,
+    index_store: IndexStore,
+    write_mutex: Mutex<()>,
+    file_pool: FilePool,
+    #[allow(dead_code)]
+    lock_file: File,
+}
+
+struct FlushHandle {
+    handle: JoinHandle<()>,
+    stop: Arc<AtomicBool>,
+}
+
+struct GcHandle {
+    handle: JoinHandle<()>,
+    stop: Arc<AtomicBool>,
+}
+
+struct EngineInner {
+    root: PathBuf,
+    meta: GlobalMeta,
+    active_writer: SegmentWriter,
+}
+
+#[derive(Debug, Clone)]
+pub struct Stats {
+    pub total_keys: u64,
+    pub total_bytes: u64,
+    pub deleted_bytes: u64,
+    pub segment_count: usize,
+}
+
+impl Engine {
+    pub fn open(path: &Path, config: Config) -> Result<Self> {
+        config.validate()?;
+
+        fs::create_dir_all(path)?;
+        fs::create_dir_all(path.join("segments"))?;
+
+        // Acquire an exclusive file lock so that no two processes can open
+        // the same database directory concurrently.
+        let lock_path = path.join("LOCK");
+        let lock_file = File::create(&lock_path)?;
+        lock_file.try_lock_exclusive().map_err(|_| Error::AlreadyOpen {
+            path: path.display().to_string(),
+        })?;
+
+        let (index_store, index_rebuilt) = match IndexStore::open(path) {
+            Ok(s) => (s, false),
+            Err(e) => {
+                tracing::warn!("Index database unreadable, rebuilding from segments: {}", e);
+                let index_path = path.join("index.redb");
+                let _ = std::fs::remove_file(&index_path);
+                (IndexStore::open(path)?, true)
+            }
+        };
+
+        let mut meta = GlobalMeta::load(path)?;
+
+        // Discover existing segments on disk
+        let seg_dir = path.join("segments");
+        let mut disk_segments: Vec<u32> = Vec::new();
+        if seg_dir.exists() {
+            for entry in fs::read_dir(&seg_dir)? {
+                let entry = entry?;
+                let name = entry.file_name();
+                let name_str = name.to_string_lossy();
+                if name_str.ends_with(".seg") && !name_str.contains("temp_") {
+                    if let Some(id_str) = name_str.strip_suffix(".seg") {
+                        if let Ok(id) = id_str.parse::<u32>() {
+                            disk_segments.push(id);
+                        }
+                    }
+                }
+            }
+        }
+        disk_segments.sort_unstable();
+
+        for &seg_id in &disk_segments {
+            if !meta.segments.contains_key(&seg_id) {
+                meta.segments.insert(seg_id, SegmentStats::new(seg_id));
+            }
+        }
+
+        let max_disk_id = disk_segments.last().copied().unwrap_or(0);
+        if max_disk_id > meta.active_segment_id {
+            meta.active_segment_id = max_disk_id;
+        }
+        if meta.active_segment_id == 0 {
+            meta.active_segment_id = 1;
+        }
+
+        crate::recovery::cleanup_temp_files(path)?;
+        let recovered_records = if index_rebuilt {
+            crate::recovery::rebuild_index(path, &mut meta)?
+        } else {
+            crate::recovery::recover(path, &mut meta)?
+        };
+        index_store.insert_batch(&recovered_records)?;
+
+        let seg_path = path
+            .join("segments")
+            .join(segment::segment_filename(meta.active_segment_id));
+        let active_writer = if seg_path.exists() {
+            SegmentWriter::open_append(seg_path, meta.active_segment_id)?
+        } else {
+            SegmentWriter::create(seg_path, meta.active_segment_id)?
+        };
+
+        meta.save(path)?;
+
+        let shared = Arc::new(EngineShared {
+            config: config.clone(),
+            inner: RwLock::new(EngineInner {
+                root: path.to_path_buf(),
+                meta,
+                active_writer,
+            }),
+            index_store,
+            write_mutex: Mutex::new(()),
+            file_pool: FilePool::new(8),
+            lock_file,
+        });
+
+        let flush_handle = if config.flush_interval_secs > 0 {
+            let shared2 = Arc::clone(&shared);
+            let stop = Arc::new(AtomicBool::new(false));
+            let stop2 = Arc::clone(&stop);
+            let interval = Duration::from_secs(config.flush_interval_secs);
+
+            let handle = thread::Builder::new()
+                .name("blob-flush".into())
+                .spawn(move || {
+                    while !stop2.load(Ordering::Acquire) {
+                        thread::park_timeout(interval);
+                        if stop2.load(Ordering::Acquire) {
+                            break;
+                        }
+                        let _lock = shared2.write_mutex.lock().unwrap();
+                        let mut inner = shared2.inner.write().unwrap();
+                        if let Err(e) = inner.flush_active() {
+                            tracing::error!("background flush failed: {}", e);
+                        }
+                    }
+                })
+                .expect("failed to spawn blob-flush thread");
+
+            Some(FlushHandle { handle, stop })
+        } else {
+            None
+        };
+
+        let gc_handle = if config.gc_interval_secs > 0 {
+            let shared3 = Arc::clone(&shared);
+            let stop = Arc::new(AtomicBool::new(false));
+            let stop2 = Arc::clone(&stop);
+            let interval = Duration::from_secs(config.gc_interval_secs);
+
+            let handle = thread::Builder::new()
+                .name("blob-gc".into())
+                .spawn(move || {
+                    while !stop2.load(Ordering::Acquire) {
+                        thread::park_timeout(interval);
+                        if stop2.load(Ordering::Acquire) {
+                            break;
+                        }
+                        // Seal the active segment first so that the data from
+                        // this GC cycle becomes eligible for compaction.
+                        // Without this, the active segment never seals until
+                        // it reaches SEGMENT_MAX_SIZE (1 GB), and GC would
+                        // have no candidates on small databases.
+                        if let Err(e) = shared3.ensure_sealed() {
+                            tracing::error!("background GC: seal failed: {}", e);
+                        }
+                        match shared3.gc_if_needed() {
+                            Ok(Some(stats)) => {
+                                tracing::info!(
+                                    "GC compacted segment {}: {} → {} bytes (kept {}, skipped {})",
+                                    stats.segment_id,
+                                    stats.bytes_before,
+                                    stats.bytes_after,
+                                    stats.entries_kept,
+                                    stats.entries_skipped,
+                                );
+                            }
+                            Ok(None) => {
+                                tracing::debug!("GC check: no segment exceeds deleted-ratio threshold");
+                            }
+                            Err(e) => {
+                                tracing::error!("background GC failed: {}", e);
+                            }
+                        }
+                    }
+                })
+                .expect("failed to spawn blob-gc thread");
+
+            Some(GcHandle { handle, stop })
+        } else {
+            None
+        };
+
+        Ok(Self {
+            shared,
+            flush_handle: Mutex::new(flush_handle),
+            gc_handle: Mutex::new(gc_handle),
+        })
+    }
+
+    // ── Read / Write / Delete ───────────────────────────────────────────
+
+    pub fn put(&self, key: [u8; 32], value: &[u8], codec: Codec) -> Result<()> {
+        if value.len() > crate::types::MAX_VALUE_SIZE {
+            return Err(Error::ValueTooLarge { size: value.len() });
+        }
+
+        let _write_lock = self.shared.write_mutex.lock().unwrap();
+        let mut inner = self.shared.inner.write().unwrap();
+
+        let (data, actual_codec) = compress::compress(
+            value,
+            codec,
+            self.shared.config.compress_threshold,
+            self.shared.config.compression_level,
+        );
+
+        let original_len = value.len() as u32;
+        let (segment_id, offset, data_size) =
+            inner.append_entry(key, &data, original_len, 0, actual_codec)?;
+
+        let record = IndexRecord::new(key, segment_id, offset, data_size, 0);
+        self.shared.index_store.insert(&record)?;
+
+        let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
+        inner.mark_indexed(segment_id, entry_end)?;
+
+        Ok(())
+    }
+
+    pub fn get(&self, key: &[u8; 32]) -> Result<Option<Vec<u8>>> {
+        let record = match self.shared.index_store.get(key)? {
+            Some(r) => r,
+            None => return Ok(None),
+        };
+
+        let inner = self.shared.inner.read().unwrap();
+        let seg_path = inner.segment_path(record.segment_id)?;
+
+        if !seg_path.exists() {
+            return Err(Error::SegmentNotFound(record.segment_id));
+        }
+
+        let reader = SegmentReader::open(seg_path.clone(), record.segment_id)?;
+        let file = self.shared.file_pool.get(record.segment_id, &seg_path)?;
+        let (entry, _) = reader.read_entry_at_file(record.offset, &file)?;
+
+        let value = compress::decompress(&entry.data, entry.codec, entry.raw_size as usize)?;
+        Ok(Some(value))
+    }
+
+    pub fn delete(&self, key: &[u8; 32]) -> Result<()> {
+        let _write_lock = self.shared.write_mutex.lock().unwrap();
+        let mut inner = self.shared.inner.write().unwrap();
+
+        // Look up the existing record so we can account deleted_bytes on the
+        // segment that holds the original data — this is what drives GC.
+        if let Some(rec) = self.shared.index_store.get(key)? {
+            if !rec.is_tombstone() {
+                if let Some(stats) = inner.meta.segments.get_mut(&rec.segment_id) {
+                    stats.deleted_bytes += rec.data_size as u64;
+                    stats.recompute_ratio();
+                }
+            }
+        }
+
+        let (segment_id, offset, data_size) =
+            inner.append_entry(*key, &[], 0, 1, Codec::None)?;
+
+        let record = IndexRecord::new(*key, segment_id, offset, data_size, 1);
+        self.shared.index_store.insert(&record)?;
+
+        let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
+        inner.mark_indexed(segment_id, entry_end)?;
+
+        Ok(())
+    }
+
+    pub fn exists(&self, key: &[u8; 32]) -> Result<bool> {
+        self.shared.index_store.exists(key)
+    }
+
+    // ── Batch delete ─────────────────────────────────────────────────────
+
+    pub fn delete_batch(&self, keys: &[[u8; 32]]) -> Result<()> {
+        if keys.is_empty() {
+            return Ok(());
+        }
+
+        let _write_lock = self.shared.write_mutex.lock().unwrap();
+        let mut inner = self.shared.inner.write().unwrap();
+
+        let mut records: Vec<IndexRecord> = Vec::with_capacity(keys.len());
+        let mut ends: Vec<(u32, u64)> = Vec::with_capacity(keys.len());
+
+        for key in keys {
+            // Track deleted_bytes for GC threshold on the original segment.
+            if let Some(rec) = self.shared.index_store.get(key)? {
+                if !rec.is_tombstone() {
+                    if let Some(stats) = inner.meta.segments.get_mut(&rec.segment_id) {
+                        stats.deleted_bytes += rec.data_size as u64;
+                        stats.recompute_ratio();
+                    }
+                }
+            }
+
+            let (segment_id, offset, data_size) =
+                inner.append_entry(*key, &[], 0, 1, Codec::None)?;
+
+            let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
+            records.push(IndexRecord::new(*key, segment_id, offset, data_size, 1));
+            ends.push((segment_id, entry_end));
+        }
+
+        inner.flush_active()?;
+
+        self.shared.index_store.insert_batch(&records)?;
+
+        for (segment_id, entry_end) in &ends {
+            inner.mark_indexed(*segment_id, *entry_end)?;
+        }
+
+        Ok(())
+    }
+
+    // ── Batch write ─────────────────────────────────────────────────────
+
+    pub fn put_batch(&self, entries: &[([u8; 32], Vec<u8>, Codec)]) -> Result<()> {
+        if entries.is_empty() {
+            return Ok(());
+        }
+
+        let _write_lock = self.shared.write_mutex.lock().unwrap();
+        let mut inner = self.shared.inner.write().unwrap();
+
+        let mut records: Vec<IndexRecord> = Vec::with_capacity(entries.len());
+        let mut ends: Vec<(u32, u64)> = Vec::with_capacity(entries.len());
+
+        for (key, value, codec) in entries {
+            if value.len() > crate::types::MAX_VALUE_SIZE {
+                return Err(Error::ValueTooLarge { size: value.len() });
+            }
+            let (data, actual_codec) = compress::compress(
+                value,
+                *codec,
+                self.shared.config.compress_threshold,
+                self.shared.config.compression_level,
+            );
+
+            let original_len = value.len() as u32;
+            let (segment_id, offset, data_size) =
+                inner.append_entry(*key, &data, original_len, 0, actual_codec)?;
+
+            let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
+            records.push(IndexRecord::new(*key, segment_id, offset, data_size, 0));
+            ends.push((segment_id, entry_end));
+        }
+
+        inner.flush_active()?;
+
+        self.shared.index_store.insert_batch(&records)?;
+
+        // Deduplicate: keep only the max offset per segment.
+        ends.sort_by(|a, b| a.0.cmp(&b.0).then(b.1.cmp(&a.1)));
+        ends.dedup_by(|a, b| a.0 == b.0);
+        for (segment_id, entry_end) in &ends {
+            inner.mark_indexed(*segment_id, *entry_end)?;
+        }
+
+        Ok(())
+    }
+
+    // ── GC ──────────────────────────────────────────────────────────────
+
+    pub fn gc(&self) -> Result<Option<GcStats>> {
+        self.shared.gc()
+    }
+
+    /// Run GC only if some segment exceeds the configured deleted-ratio threshold.
+    /// Returns `Ok(None)` immediately without acquiring the write lock when no
+    /// segment qualifies.
+    pub fn gc_if_needed(&self) -> Result<Option<GcStats>> {
+        self.shared.gc_if_needed()
+    }
+
+    // ── Flush / Stats / Shutdown ────────────────────────────────────────
+
+    /// Fsync the active segment and save metadata without compacting buckets.
+    /// Lightweight checkpoint suitable for periodic calls from the background
+    /// flush thread or external schedulers.
+    pub fn flush(&self) -> Result<()> {
+        let _write_lock = self.shared.write_mutex.lock().unwrap();
+        let mut inner = self.shared.inner.write().unwrap();
+        inner.flush_active()
+    }
+
+    pub fn stats(&self) -> Result<Stats> {
+        let inner = self.shared.inner.read().unwrap();
+        let meta = &inner.meta;
+
+        let mut total_bytes = 0u64;
+        let mut deleted_bytes = 0u64;
+        for seg in meta.segments.values() {
+            total_bytes += seg.total_bytes;
+            deleted_bytes += seg.deleted_bytes;
+        }
+
+        let total_keys = self.shared.index_store.total_keys()? as u64;
+
+        Ok(Stats {
+            total_keys,
+            total_bytes,
+            deleted_bytes,
+            segment_count: meta.segments.len(),
+        })
+    }
+
+    /// Seal the active segment, forcing it to become a GC candidate.
+    #[doc(hidden)]
+    pub fn seal_active_segment(&self) -> Result<u32> {
+        let _write_lock = self.shared.write_mutex.lock().unwrap();
+        let mut inner = self.shared.inner.write().unwrap();
+        let id = inner.active_writer.id();
+        inner.seal_active()?;
+        Ok(id)
+    }
+
+    pub fn shutdown(&self) -> Result<()> {
+        // Stop background threads first
+        if let Some(fh) = self.flush_handle.lock().unwrap().take() {
+            fh.stop.store(true, Ordering::Release);
+            fh.handle.thread().unpark();
+            let _ = fh.handle.join();
+        }
+        if let Some(gh) = self.gc_handle.lock().unwrap().take() {
+            gh.stop.store(true, Ordering::Release);
+            gh.handle.thread().unpark();
+            let _ = gh.handle.join();
+        }
+
+        let mut inner = self.shared.inner.write().unwrap();
+        inner.flush_active()?;
+
+        inner.meta.save(&inner.root)?;
+        tracing::info!("bichon-blob shut down cleanly");
+        Ok(())
+    }
+}
+
+impl Drop for Engine {
+    fn drop(&mut self) {
+        // Best-effort shutdown that is panic-safe: only signal threads to
+        // stop — don't try to acquire write_mutex or inner.write(), which
+        // would deadlock if we're unwinding from a panic that happened while
+        // one of those locks was held.
+        if let Some(fh) = self.flush_handle.lock().ok().and_then(|mut g| g.take()) {
+            fh.stop.store(true, Ordering::Release);
+            fh.handle.thread().unpark();
+            let _ = fh.handle.join();
+        }
+        if let Some(gh) = self.gc_handle.lock().ok().and_then(|mut g| g.take()) {
+            gh.stop.store(true, Ordering::Release);
+            gh.handle.thread().unpark();
+            let _ = gh.handle.join();
+        }
+    }
+}
+
+// ── EngineShared ────────────────────────────────────────────────────────────
+
+impl EngineShared {
+    /// Seal the active segment if its deleted-ratio exceeds the GC threshold,
+    /// so that the upcoming GC pass can compact it.  Called periodically by
+    /// the background GC thread.
+    fn ensure_sealed(&self) -> Result<()> {
+        let mut inner = self.inner.write().unwrap();
+        let active_id = inner.active_writer.id();
+        if let Some(stats) = inner.meta.segments.get(&active_id) {
+            if stats.deleted_ratio >= self.config.gc_deleted_ratio {
+                inner.seal_active()?;
+            }
+        }
+        Ok(())
+    }
+
+    fn gc_if_needed(&self) -> Result<Option<GcStats>> {
+        let inner = self.inner.read().unwrap();
+        let needs_gc = inner
+            .meta
+            .segments
+            .values()
+            .any(|s| s.sealed && s.deleted_ratio >= self.config.gc_deleted_ratio);
+        drop(inner);
+
+        if needs_gc {
+            self.gc()
+        } else {
+            Ok(None)
+        }
+    }
+
+    fn gc(&self) -> Result<Option<GcStats>> {
+        // Phase 1: scan only the target segment, consult bucket index per entry.
+        // Read-only with respect to Engine state — no write_mutex needed.
+        let prep = {
+            let inner = self.inner.read().unwrap();
+            gc::gc_prepare(
+                &inner.root,
+                &inner.meta,
+                self.config.gc_deleted_ratio,
+                &self.index_store,
+            )?
+        };
+
+        let mut prep = match prep {
+            Some(p) => p,
+            None => return Ok(None),
+        };
+
+        // Phase 2: rename temp file + update bucket index.
+        let _write_lock = self.write_mutex.lock().unwrap();
+
+        let kept_records = std::mem::take(&mut prep.kept_records);
+        let deleted_keys = std::mem::take(&mut prep.deleted_keys);
+        let stats = gc::gc_finish(prep)?;
+        self.file_pool.invalidate(stats.segment_id);
+
+        // Insert new index records with updated offsets for kept entries.
+        if !kept_records.is_empty() {
+            self.index_store.insert_batch(&kept_records)?;
+        }
+
+        // Remove tombstone IndexRecords that pointed to entries in this
+        // compacted segment — they are gone now and would accumulate forever.
+        if !deleted_keys.is_empty() {
+            self.index_store.delete_batch(&deleted_keys)?;
+        }
+
+        {
+            let mut inner = self.inner.write().unwrap();
+
+            if stats.bytes_after == 0 {
+                // Segment was completely emptied — remove it from meta first,
+                // then delete the file.  If we crash between the two steps the
+                // orphaned file is rediscovered on next open and retried.
+                inner.meta.segments.remove(&stats.segment_id);
+                inner.meta.save(&inner.root)?;
+                drop(inner);
+
+                let seg_path = self.inner.read().unwrap()
+                    .root
+                    .join("segments")
+                    .join(segment::segment_filename(stats.segment_id));
+                let _ = fs::remove_file(&seg_path);
+            } else {
+                // Update segment stats: now smaller and clean.
+                if let Some(seg_stats) = inner.meta.segments.get_mut(&stats.segment_id) {
+                    seg_stats.total_bytes = stats.bytes_after;
+                    seg_stats.deleted_bytes = 0;
+                    seg_stats.deleted_ratio = 0.0;
+                    seg_stats.indexed_up_to_offset = stats.bytes_after;
+                }
+                inner.meta.save(&inner.root)?;
+            }
+        }
+
+        Ok(Some(stats))
+    }
+}
+
+// ── EngineInner ───────────────────────────────────────────────────────────
+
+impl EngineInner {
+    fn append_entry(
+        &mut self,
+        key: [u8; 32],
+        data: &[u8],
+        raw_size: u32,
+        flags: u8,
+        codec: Codec,
+    ) -> Result<(u32, u64, u32)> {
+        if self.active_writer.is_full() {
+            self.seal_active()?;
+        }
+
+        use crate::segment::Entry;
+        let entry = if flags == 1 {
+            Entry::tombstone(key)
+        } else {
+            Entry::new(key, data, raw_size, flags, codec)
+        };
+
+        let data_size = entry.data.len() as u32;
+        let segment_id = self.active_writer.id();
+        let offset = self.active_writer.append(&entry)?;
+
+        let stats = self
+            .meta
+            .segments
+            .entry(segment_id)
+            .or_insert_with(|| SegmentStats::new(segment_id));
+        stats.total_bytes += data_size as u64;
+        if flags == 1 {
+            stats.deleted_bytes += entry.raw_size as u64;
+        }
+        stats.recompute_ratio();
+
+        Ok((segment_id, offset, data_size))
+    }
+
+    fn flush_active(&mut self) -> Result<()> {
+        self.active_writer.fsync()?;
+        self.meta.save(&self.root)
+    }
+
+    fn mark_indexed(&mut self, segment_id: u32, offset: u64) -> Result<()> {
+        if let Some(stats) = self.meta.segments.get_mut(&segment_id) {
+            if offset > stats.indexed_up_to_offset {
+                stats.indexed_up_to_offset = offset;
+            }
+        }
+        self.meta.save(&self.root)
+    }
+
+    fn seal_active(&mut self) -> Result<()> {
+        let old_id = self.active_writer.id();
+        let old_stats = self
+            .meta
+            .segments
+            .entry(old_id)
+            .or_insert_with(|| SegmentStats::new(old_id));
+        old_stats.sealed = true;
+
+        let new_id = old_id + 1;
+        self.meta.active_segment_id = new_id;
+        let new_path = self
+            .root
+            .join("segments")
+            .join(segment::segment_filename(new_id));
+        self.active_writer = SegmentWriter::create(new_path, new_id)?;
+        self.meta.save(&self.root)?;
+
+        Ok(())
+    }
+
+    fn segment_path(&self, segment_id: u32) -> Result<PathBuf> {
+        let path = self
+            .root
+            .join("segments")
+            .join(segment::segment_filename(segment_id));
+        if path.exists() {
+            Ok(path)
+        } else {
+            Err(Error::SegmentNotFound(segment_id))
+        }
+    }
+}

crates/blob/src/error.rs

@@ -0,0 +1,59 @@
+use std::{io, path::PathBuf};
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("I/O error: {0}")]
+    Io(#[from] io::Error),
+
+    #[error("JSON error: {0}")]
+    Json(#[from] serde_json::Error),
+
+    #[error("CRC32 mismatch at {path}:{offset}")]
+    CrcMismatch { path: PathBuf, offset: u64 },
+
+    #[error("Corrupt entry at {path}:{offset}: {reason}")]
+    CorruptEntry {
+        path: PathBuf,
+        offset: u64,
+        reason: String,
+    },
+
+    #[error("Segment not found: {0}")]
+    SegmentNotFound(u32),
+
+    #[error("Value too large: {size} bytes (max 100 MB)")]
+    ValueTooLarge { size: usize },
+
+    #[error("Compression error: {0}")]
+    Compression(String),
+
+    #[error("Disk full: {0}")]
+    DiskFull(String),
+
+    #[error("Invalid config: {0}")]
+    InvalidConfig(String),
+
+    #[error("Bucket index corrupt at {path}: {reason}")]
+    BucketIndexCorrupt { path: PathBuf, reason: String },
+
+    #[error("Segment file truncated at {path}: expected {expected}, got {actual}")]
+    SegmentTruncated {
+        path: PathBuf,
+        expected: u64,
+        actual: u64,
+    },
+
+    #[error("Corrupt metadata file: {0}")]
+    CorruptMeta(String),
+
+    #[error("Unsupported metadata version {version} in {path}")]
+    UnsupportedMetaVersion { path: PathBuf, version: u32 },
+
+    #[error("Index database error: {0}")]
+    IndexDb(String),
+
+    #[error("Database is already open by another process at {path}")]
+    AlreadyOpen { path: String },
+}

crates/blob/src/file_pool.rs

@@ -0,0 +1,54 @@
+use std::collections::VecDeque;
+use std::fs::File;
+use std::path::Path;
+use std::sync::{Arc, Mutex};
+
+use crate::error::Result;
+use crate::fs as fs_util;
+
+/// Simple LRU pool of open file handles, keyed by segment_id.
+/// Uses pread-based reads so a single `Arc<File>` supports concurrent access.
+pub struct FilePool {
+    max_entries: usize,
+    entries: Mutex<VecDeque<(u32, Arc<File>)>>,
+}
+
+impl FilePool {
+    pub fn new(max_entries: usize) -> Self {
+        Self {
+            max_entries: max_entries.max(1),
+            entries: Mutex::new(VecDeque::new()),
+        }
+    }
+
+    /// Get an open File for the given segment. Reuses cached handle if available.
+    pub fn get(&self, seg_id: u32, path: &Path) -> Result<Arc<File>> {
+        let mut entries = self.entries.lock().unwrap();
+
+        // Check for existing entry
+        for (i, (id, _)) in entries.iter().enumerate() {
+            if *id == seg_id {
+                let (_, file) = entries.remove(i).unwrap();
+                entries.push_front((seg_id, file.clone()));
+                return Ok(file);
+            }
+        }
+
+        // Open new file
+        let file = Arc::new(fs_util::open_read(path)?);
+
+        // Evict oldest if full
+        if entries.len() >= self.max_entries {
+            entries.pop_back();
+        }
+
+        entries.push_front((seg_id, file.clone()));
+        Ok(file)
+    }
+
+    /// Remove a cached file handle (e.g. after GC rewrites a segment).
+    pub fn invalidate(&self, seg_id: u32) {
+        let mut entries = self.entries.lock().unwrap();
+        entries.retain(|(id, _)| *id != seg_id);
+    }
+}

crates/blob/src/fs.rs

@@ -0,0 +1,167 @@
+use std::fs::{self, File, OpenOptions};
+use std::io::{self, Write};
+use std::path::Path;
+use std::time::Duration;
+
+use crate::error::Result;
+
+/// Max retries for transient filesystem errors (NFS ESTALE, CIFS sharing violations, etc.)
+const MAX_RETRIES: u32 = 5;
+const RETRY_DELAY: Duration = Duration::from_millis(20);
+
+/// Check if an I/O error is transient (retryable).
+fn is_transient(err: &io::Error) -> bool {
+    use std::io::ErrorKind;
+    matches!(
+        err.kind(),
+        ErrorKind::TimedOut
+            | ErrorKind::Interrupted
+            | ErrorKind::WouldBlock
+            | ErrorKind::UnexpectedEof
+    ) || err.raw_os_error() == Some(116) // ESTALE on Linux
+}
+
+/// Open an existing file for reading, with retry on transient errors (NFS ESTALE etc.).
+pub fn open_read(path: &Path) -> Result<File> {
+    let mut last_err = None;
+    for attempt in 0..MAX_RETRIES {
+        match File::open(path) {
+            Ok(f) => return Ok(f),
+            Err(e) if is_transient(&e) => {
+                last_err = Some(e);
+                if attempt > 0 {
+                    std::thread::sleep(RETRY_DELAY * attempt);
+                }
+                continue;
+            }
+            Err(e) => return Err(e.into()),
+        }
+    }
+    Err(crate::error::Error::Io(last_err.unwrap()))
+}
+
+/// Open an existing file for writing, with retry on transient errors.
+pub fn open_write(path: &Path) -> Result<File> {
+    let mut last_err = None;
+    for attempt in 0..MAX_RETRIES {
+        match OpenOptions::new().write(true).open(path) {
+            Ok(f) => return Ok(f),
+            Err(e) if is_transient(&e) => {
+                last_err = Some(e);
+                if attempt > 0 {
+                    std::thread::sleep(RETRY_DELAY * attempt);
+                }
+                continue;
+            }
+            Err(e) => return Err(e.into()),
+        }
+    }
+    Err(crate::error::Error::Io(last_err.unwrap()))
+}
+
+/// Create a new file atomically: write content to a temp file, fsync, then rename.
+/// Avoids `create_new(true)` which is racy on NFS.
+pub fn create_atomic(path: &Path, content: &[u8]) -> Result<()> {
+    let tmp = path.with_extension(
+        path.extension()
+            .map(|e| format!("{}.tmp", e.to_string_lossy()))
+            .unwrap_or_else(|| "tmp".to_string()),
+    );
+
+    {
+        let mut f = File::create(&tmp)?;
+        f.write_all(content)?;
+        f.sync_all()?;
+    }
+
+    fs::rename(&tmp, path)?;
+    Ok(())
+}
+
+/// Truncate an existing file to the given size, with retry.
+pub fn truncate(path: &Path, size: u64) -> Result<()> {
+    let f = open_write(path)?;
+    f.set_len(size)?;
+    f.sync_all()?;
+    Ok(())
+}
+
+/// Positional read: read `buf.len()` bytes at `offset` from `file`.
+/// Uses platform-specific pread so `&File` (shared ref) suffices —
+/// no Mutex needed for concurrent reads.
+pub fn pread_exact(file: &File, offset: u64, buf: &mut [u8]) -> Result<()> {
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::FileExt;
+        file.read_exact_at(buf, offset)?;
+    }
+    #[cfg(windows)]
+    {
+        use std::os::windows::fs::FileExt;
+        file.seek_read(buf, offset)?;
+    }
+    #[cfg(not(any(unix, windows)))]
+    {
+        // Fallback: seek+read (requires &mut, so this is best-effort on exotic platforms)
+        use std::io::{Read, Seek, SeekFrom};
+        let mut tmp = file.try_clone()?;
+        tmp.seek(SeekFrom::Start(offset))?;
+        tmp.read_exact(buf)?;
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::TempDir;
+
+    #[test]
+    fn test_open_read_existing() {
+        let dir = TempDir::new().unwrap();
+        let path = dir.path().join("test.txt");
+        std::fs::write(&path, b"hello").unwrap();
+
+        let mut f = open_read(&path).unwrap();
+        let mut s = String::new();
+        std::io::Read::read_to_string(&mut f, &mut s).unwrap();
+        assert_eq!(s, "hello");
+    }
+
+    #[test]
+    fn test_open_read_missing() {
+        let dir = TempDir::new().unwrap();
+        let result = open_read(&dir.path().join("nope.txt"));
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_create_atomic_success() {
+        let dir = TempDir::new().unwrap();
+        let path = dir.path().join("data.bin");
+        create_atomic(&path, b"hello world").unwrap();
+
+        let content = std::fs::read(&path).unwrap();
+        assert_eq!(content, b"hello world");
+        // Temp file should not exist
+        assert!(!dir.path().join("data.bin.tmp").exists());
+    }
+
+    #[test]
+    fn test_create_atomic_overwrites() {
+        let dir = TempDir::new().unwrap();
+        let path = dir.path().join("data.bin");
+        create_atomic(&path, b"first").unwrap();
+        create_atomic(&path, b"second").unwrap();
+        assert_eq!(std::fs::read(&path).unwrap(), b"second");
+    }
+
+    #[test]
+    fn test_truncate() {
+        let dir = TempDir::new().unwrap();
+        let path = dir.path().join("trunc.bin");
+        std::fs::write(&path, b"1234567890").unwrap();
+        truncate(&path, 5).unwrap();
+        assert_eq!(std::fs::metadata(&path).unwrap().len(), 5);
+    }
+}

crates/blob/src/gc.rs

@@ -0,0 +1,148 @@
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use crate::bucket::{IndexRecord, IndexStore};
+use crate::error::Result;
+use crate::meta::GlobalMeta;
+use crate::segment::{self, SegmentReader, SegmentWriter};
+
+/// Result of a GC run.
+#[derive(Debug)]
+pub struct GcStats {
+    pub segment_id: u32,
+    pub bytes_before: u64,
+    pub bytes_after: u64,
+    pub entries_kept: usize,
+    pub entries_skipped: usize,
+}
+
+/// Prepared GC result — the compacted segment has been written to a temp file
+/// but not yet renamed over the original.  `gc_finish` must be called to commit.
+pub struct GcPrepare {
+    pub segment_id: u32,
+    pub bytes_before: u64,
+    pub bytes_after: u64,
+    pub entries_kept: usize,
+    pub entries_skipped: usize,
+    /// Index records for kept entries with their new offsets in the compacted segment.
+    pub kept_records: Vec<IndexRecord>,
+    /// Keys whose tombstone IndexRecord should be removed from redb after
+    /// this segment is compacted (the tombstone entries they pointed to are gone).
+    pub deleted_keys: Vec<[u8; 32]>,
+    temp_path: PathBuf,
+    seg_path: PathBuf,
+}
+
+/// Phase 1: pick the sealed segment with the highest deleted_ratio, then for
+/// each entry in that segment consult the bucket index to decide whether it is
+/// still the latest version.  Live entries are written to a temp file; stale
+/// entries and tombstones are skipped.  Does NOT rename — the caller should
+/// hold the write lock only during `gc_finish`.
+pub fn gc_prepare(
+    store_root: &Path,
+    meta: &GlobalMeta,
+    deleted_ratio_threshold: f64,
+    index_store: &IndexStore,
+) -> Result<Option<GcPrepare>> {
+    let candidate = meta
+        .segments
+        .values()
+        .filter(|s| s.sealed && s.deleted_ratio >= deleted_ratio_threshold)
+        .max_by(|a, b| a.deleted_ratio.partial_cmp(&b.deleted_ratio).unwrap());
+
+    let target = match candidate {
+        Some(s) => s.clone(),
+        None => return Ok(None),
+    };
+
+    let seg_path = store_root
+        .join("segments")
+        .join(segment::segment_filename(target.segment_id));
+
+    // Guard against stale meta entries: if the segment file was deleted
+    // (e.g. after a prior GC emptied it), skip this candidate.
+    if !seg_path.exists() {
+        return Ok(None);
+    }
+
+    let reader = SegmentReader::open(seg_path.clone(), target.segment_id)?;
+
+    // Create temp segment (not renamed yet)
+    let timestamp = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_nanos();
+    let temp_name = format!("temp_{:016x}.seg", timestamp);
+    let temp_path = store_root.join("segments").join(&temp_name);
+    let mut writer = SegmentWriter::create(temp_path.clone(), target.segment_id)?;
+
+    let mut kept_records: Vec<IndexRecord> = Vec::new();
+    let mut deleted_keys: Vec<[u8; 32]> = Vec::new();
+    let mut bytes_after: u64 = 0;
+    let mut entries_kept: usize = 0;
+    let mut entries_skipped: usize = 0;
+
+    reader.scan_entries(0, |entry, offset| {
+        if entry.is_tombstone() {
+            // If this tombstone is the latest version in the index, the key
+            // must be removed from redb after compaction — otherwise it
+            // accumulates forever.
+            match index_store.get(&entry.key)? {
+                Some(rec)
+                    if rec.segment_id == target.segment_id && rec.offset == offset =>
+                {
+                    deleted_keys.push(entry.key);
+                }
+                _ => {}
+            }
+            entries_skipped += 1;
+            return Ok(());
+        }
+
+        // Ask the bucket index whether this entry is still the latest version.
+        match index_store.get(&entry.key)? {
+            Some(rec) if rec.segment_id == target.segment_id && rec.offset == offset => {
+                let new_offset = writer.append(entry)?;
+                kept_records.push(IndexRecord::new(
+                    entry.key,
+                    target.segment_id,
+                    new_offset,
+                    entry.data.len() as u32,
+                    entry.flags,
+                ));
+                bytes_after += entry.data.len() as u64;
+                entries_kept += 1;
+            }
+            _ => {
+                entries_skipped += 1;
+            }
+        }
+        Ok(())
+    })?;
+
+    writer.fsync()?;
+
+    Ok(Some(GcPrepare {
+        segment_id: target.segment_id,
+        bytes_before: target.total_bytes,
+        bytes_after,
+        entries_kept,
+        entries_skipped,
+        kept_records,
+        deleted_keys,
+        temp_path,
+        seg_path,
+    }))
+}
+
+/// Phase 2: atomically replace the old segment with the compacted one.
+pub fn gc_finish(prep: GcPrepare) -> Result<GcStats> {
+    fs::rename(&prep.temp_path, &prep.seg_path)?;
+    Ok(GcStats {
+        segment_id: prep.segment_id,
+        bytes_before: prep.bytes_before,
+        bytes_after: prep.bytes_after,
+        entries_kept: prep.entries_kept,
+        entries_skipped: prep.entries_skipped,
+    })
+}

crates/blob/src/lib.rs

@@ -0,0 +1,16 @@
+pub mod bucket;
+pub mod checksum;
+pub mod compress;
+pub mod engine;
+pub mod error;
+pub mod file_pool;
+pub mod fs;
+pub mod gc;
+pub mod meta;
+pub mod recovery;
+pub mod segment;
+pub mod types;
+
+pub use engine::{Engine, Stats};
+pub use error::{Error, Result};
+pub use types::{Codec, Config};

crates/blob/src/meta.rs

@@ -0,0 +1,180 @@
+use std::collections::BTreeMap;
+use std::path::Path;
+
+use crate::checksum;
+use crate::error::Result;
+use serde::{Deserialize, Serialize};
+
+const META_VERSION: u32 = 2;
+
+// ── Helpers ────────────────────────────────────────────────────────────────
+
+fn write_bin<T: Serialize>(path: &Path, value: &T) -> Result<()> {
+    let payload =
+        bincode::serde::encode_to_vec(value, bincode::config::standard()).map_err(|e| {
+            crate::error::Error::CorruptMeta(format!("{}: bincode encode: {}", path.display(), e))
+        })?;
+    let crc = checksum::crc32(&payload);
+    let mut buf = Vec::with_capacity(8 + payload.len());
+    buf.extend_from_slice(&crc.to_le_bytes());
+    buf.extend_from_slice(&META_VERSION.to_le_bytes());
+    buf.extend_from_slice(&payload);
+
+    crate::fs::create_atomic(path, &buf)?;
+    Ok(())
+}
+
+fn read_bin<T: for<'de> Deserialize<'de>>(path: &Path) -> Result<T> {
+    let data = std::fs::read(path)?;
+    if data.len() < 8 {
+        return Err(crate::error::Error::CorruptMeta(path.display().to_string()));
+    }
+    let stored_crc = u32::from_le_bytes(data[0..4].try_into().unwrap());
+    let version = u32::from_le_bytes(data[4..8].try_into().unwrap());
+    if version != META_VERSION {
+        return Err(crate::error::Error::UnsupportedMetaVersion {
+            path: path.to_path_buf(),
+            version,
+        });
+    }
+    let computed = checksum::crc32(&data[8..]);
+    if stored_crc != computed {
+        return Err(crate::error::Error::CorruptMeta(path.display().to_string()));
+    }
+    bincode::serde::decode_from_slice(&data[8..], bincode::config::standard())
+        .map(|(v, _)| v)
+        .map_err(|e| {
+            crate::error::Error::CorruptMeta(format!("{}: bincode decode: {}", path.display(), e))
+        })
+}
+
+// ── SegmentStats ───────────────────────────────────────────────────────────
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SegmentStats {
+    pub segment_id: u32,
+    pub total_bytes: u64,
+    pub deleted_bytes: u64,
+    pub deleted_ratio: f64,
+    pub sealed: bool,
+    /// Byte offset up to which entries have been indexed in bucket files.
+    pub indexed_up_to_offset: u64,
+    /// Number of compacted (sorted, deduped) records in each bucket file for this segment.
+    /// Used by BucketStore on recovery to know where the clean portion ends.
+    pub bucket_compacted: u64,
+}
+
+impl SegmentStats {
+    pub fn new(segment_id: u32) -> Self {
+        Self {
+            segment_id,
+            total_bytes: 0,
+            deleted_bytes: 0,
+            deleted_ratio: 0.0,
+            sealed: false,
+            indexed_up_to_offset: 0,
+            bucket_compacted: 0,
+        }
+    }
+
+    pub fn recompute_ratio(&mut self) {
+        if self.total_bytes > 0 {
+            self.deleted_ratio = self.deleted_bytes as f64 / self.total_bytes as f64;
+        } else {
+            self.deleted_ratio = 0.0;
+        }
+    }
+}
+
+// ── GlobalMeta ─────────────────────────────────────────────────────────────
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GlobalMeta {
+    pub version: u32,
+    pub active_segment_id: u32,
+    pub segments: BTreeMap<u32, SegmentStats>,
+}
+
+impl GlobalMeta {
+    pub fn new() -> Self {
+        Self {
+            version: META_VERSION,
+            active_segment_id: 1,
+            segments: BTreeMap::new(),
+        }
+    }
+
+    pub fn load(store_root: &Path) -> Result<Self> {
+        let bin_path = store_root.join("meta.bin");
+        if bin_path.exists() {
+            return read_bin(&bin_path);
+        }
+        Ok(Self::new())
+    }
+
+    pub fn save(&self, store_root: &Path) -> Result<()> {
+        let path = store_root.join("meta.bin");
+        write_bin(&path, self)
+    }
+}
+
+impl Default for GlobalMeta {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::TempDir;
+
+    #[test]
+    fn test_global_meta_roundtrip() {
+        let dir = TempDir::new().unwrap();
+        let mut meta = GlobalMeta::new();
+        meta.segments.insert(
+            1,
+            SegmentStats {
+                segment_id: 1,
+                total_bytes: 1000,
+                deleted_bytes: 300,
+                deleted_ratio: 0.3,
+                sealed: false,
+                indexed_up_to_offset: 500,
+                bucket_compacted: 0,
+            },
+        );
+        meta.save(dir.path()).unwrap();
+
+        let loaded = GlobalMeta::load(dir.path()).unwrap();
+        assert_eq!(loaded.active_segment_id, 1);
+        assert_eq!(loaded.segments[&1].total_bytes, 1000);
+        assert_eq!(loaded.segments[&1].indexed_up_to_offset, 500);
+    }
+
+    #[test]
+    fn test_global_meta_default_when_missing() {
+        let dir = TempDir::new().unwrap();
+        let meta = GlobalMeta::load(dir.path()).unwrap();
+        assert_eq!(meta.active_segment_id, 1);
+        assert!(meta.segments.is_empty());
+    }
+
+    #[test]
+    fn test_corrupt_bin_detected() {
+        let dir = TempDir::new().unwrap();
+        std::fs::write(dir.path().join("meta.bin"), vec![0xFFu8; 100]).unwrap();
+        let result = GlobalMeta::load(dir.path());
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_segment_stats_recompute() {
+        let mut s = SegmentStats::new(1);
+        s.total_bytes = 1000;
+        s.deleted_bytes = 250;
+        s.recompute_ratio();
+        assert!((s.deleted_ratio - 0.25).abs() < 0.001);
+    }
+}

crates/blob/src/recovery.rs

@@ -0,0 +1,144 @@
+use std::fs;
+use std::path::Path;
+
+use crate::bucket::IndexRecord;
+use crate::error::Result;
+use crate::meta::{GlobalMeta, SegmentStats};
+use crate::segment::{self, SegmentReader};
+
+/// Recover after a crash: scan any unindexed portions of segments,
+/// update segment stats, and return newly discovered index records.
+///
+/// The caller is responsible for inserting the returned records into
+/// the index store.
+pub fn recover(store_root: &Path, meta: &mut GlobalMeta) -> Result<Vec<IndexRecord>> {
+    scan_segments(store_root, meta, false)
+}
+
+/// Rebuild the entire index from scratch by scanning all segments from
+/// offset 0.  Used when the index database is corrupted or lost.
+///
+/// Resets all segment stats and returns every entry found on disk.
+/// The caller should replace the index database before calling this.
+pub fn rebuild_index(store_root: &Path, meta: &mut GlobalMeta) -> Result<Vec<IndexRecord>> {
+    // Reset all segment stats — they'll be recomputed during the scan.
+    // Also reset indexed_up_to_offset so we scan from 0.
+    for stats in meta.segments.values_mut() {
+        stats.total_bytes = 0;
+        stats.deleted_bytes = 0;
+        stats.deleted_ratio = 0.0;
+        stats.indexed_up_to_offset = 0;
+    }
+    scan_segments(store_root, meta, true)
+}
+
+/// Common implementation: scan segments and collect index records.
+/// When `full_scan` is true, every segment is scanned from offset 0.
+fn scan_segments(
+    store_root: &Path,
+    meta: &mut GlobalMeta,
+    full_scan: bool,
+) -> Result<Vec<IndexRecord>> {
+    let seg_dir = store_root.join("segments");
+    if !seg_dir.exists() {
+        fs::create_dir_all(&seg_dir)?;
+    }
+
+    // Discover all segment files on disk
+    let mut disk_segments: Vec<u32> = Vec::new();
+    if seg_dir.exists() {
+        for entry in fs::read_dir(&seg_dir)? {
+            let entry = entry?;
+            let name = entry.file_name();
+            let name_str = name.to_string_lossy();
+            if name_str.ends_with(".seg") && !name_str.contains("temp_") {
+                if let Some(id_str) = name_str.strip_suffix(".seg") {
+                    if let Ok(id) = id_str.parse::<u32>() {
+                        disk_segments.push(id);
+                    }
+                }
+            }
+        }
+    }
+    disk_segments.sort_unstable();
+
+    let mut all_records: Vec<IndexRecord> = Vec::new();
+
+    // For each segment, scan unindexed portions
+    for &seg_id in &disk_segments {
+        let seg_path = seg_dir.join(segment::segment_filename(seg_id));
+        let file_size = fs::metadata(&seg_path)?.len();
+
+        let mut stats = meta
+            .segments
+            .remove(&seg_id)
+            .unwrap_or_else(|| SegmentStats::new(seg_id));
+        let is_sealed = seg_id != meta.active_segment_id;
+        stats.sealed = is_sealed;
+
+        // Determine scan range.
+        let scan_start = if full_scan {
+            0
+        } else if stats.indexed_up_to_offset <= file_size {
+            stats.indexed_up_to_offset
+        } else {
+            // Segment was replaced (interrupted GC) — full rescan needed.
+            0
+        };
+
+        if scan_start >= file_size {
+            meta.segments.insert(seg_id, stats);
+            continue;
+        }
+
+        let reader = SegmentReader::open(seg_path.clone(), seg_id)?;
+
+        let truncation_point = reader.scan_entries(scan_start, |entry, offset| {
+            all_records.push(IndexRecord::new(
+                entry.key,
+                seg_id,
+                offset,
+                entry.data.len() as u32,
+                entry.flags,
+            ));
+
+            stats.total_bytes += entry.data.len() as u64;
+            if entry.is_tombstone() {
+                stats.deleted_bytes += entry.raw_size as u64;
+            }
+
+            Ok(())
+        })?;
+
+        // Truncate if tail corruption found
+        if truncation_point < file_size {
+            segment::truncate_segment(&seg_path, truncation_point)?;
+        }
+
+        stats.indexed_up_to_offset = truncation_point;
+        stats.recompute_ratio();
+        meta.segments.insert(seg_id, stats);
+    }
+
+    meta.save(store_root)?;
+
+    Ok(all_records)
+}
+
+/// Clean up leftover temp files from interrupted GC.
+pub fn cleanup_temp_files(store_root: &Path) -> Result<()> {
+    let seg_dir = store_root.join("segments");
+    if seg_dir.exists() {
+        for entry in fs::read_dir(&seg_dir)? {
+            let entry = entry?;
+            let name = entry.file_name();
+            let name_str = name.to_string_lossy();
+            if name_str.starts_with("temp_") {
+                let path = entry.path();
+                tracing::warn!("Removing leftover temp file: {:?}", path);
+                fs::remove_file(&path)?;
+            }
+        }
+    }
+    Ok(())
+}

crates/blob/src/segment.rs

@@ -0,0 +1,551 @@
+use std::fs::{self, File};
+use std::io::{Read, Seek, SeekFrom, Write};
+use std::path::{Path, PathBuf};
+
+use crate::checksum;
+use crate::error::{Error, Result};
+use crate::fs as fs_util;
+use crate::types::{Codec, ENTRY_HEADER_SIZE, ENTRY_MAGIC, SEGMENT_MAX_SIZE};
+
+/// In-memory representation of a stored entry.
+#[derive(Debug, Clone)]
+pub struct Entry {
+    pub flags: u8,
+    pub codec: Codec,
+    pub key: [u8; 32],
+    pub raw_size: u32,
+    pub data: Vec<u8>,
+}
+
+impl Entry {
+    /// Create a normal data entry.
+    /// `raw_size` is the original uncompressed size; `data` is what goes to disk.
+    pub fn new(key: [u8; 32], data: &[u8], raw_size: u32, flags: u8, codec: Codec) -> Self {
+        Self {
+            flags,
+            codec,
+            key,
+            raw_size,
+            data: data.to_vec(),
+        }
+    }
+
+    /// Create a tombstone entry.
+    pub fn tombstone(key: [u8; 32]) -> Self {
+        Self {
+            flags: 1,
+            codec: Codec::None,
+            key,
+            raw_size: 0,
+            data: Vec::new(),
+        }
+    }
+
+    pub fn is_tombstone(&self) -> bool {
+        self.flags == 1
+    }
+
+    /// Total on-disk size: header + data
+    pub fn disk_size(&self) -> usize {
+        ENTRY_HEADER_SIZE + self.data.len()
+    }
+}
+
+/// Write entries sequentially to a segment file.
+pub struct SegmentWriter {
+    file: File,
+    path: PathBuf,
+    id: u32,
+    bytes_written: u64,
+}
+
+impl SegmentWriter {
+    pub fn create(path: PathBuf, id: u32) -> Result<Self> {
+        // Use create+truncate instead of create_new to avoid NFS O_EXCL issues.
+        let file = File::create(&path)?;
+        Ok(Self {
+            file,
+            path,
+            id,
+            bytes_written: 0,
+        })
+    }
+
+    pub fn open_append(path: PathBuf, id: u32) -> Result<Self> {
+        let mut file = fs_util::open_write(&path)?;
+        file.seek(SeekFrom::End(0))?;
+        let bytes_written = file.stream_position()?;
+        Ok(Self {
+            file,
+            path,
+            id,
+            bytes_written,
+        })
+    }
+
+    pub fn id(&self) -> u32 {
+        self.id
+    }
+
+    pub fn path(&self) -> &Path {
+        &self.path
+    }
+
+    pub fn bytes_written(&self) -> u64 {
+        self.bytes_written
+    }
+
+    pub fn is_full(&self) -> bool {
+        self.bytes_written >= SEGMENT_MAX_SIZE
+    }
+
+    /// Append an entry. Returns the offset where it was written.
+    pub fn append(&mut self, entry: &Entry) -> Result<u64> {
+        let offset = self.bytes_written;
+        self.write_entry(entry)
+            .map_err(|e| map_io_err(e, &self.path))?;
+        Ok(offset)
+    }
+
+    fn write_entry(&mut self, entry: &Entry) -> Result<()> {
+        let data_size = entry.data.len() as u32;
+
+        // Write magic
+        self.file.write_all(&ENTRY_MAGIC.to_le_bytes())?;
+
+        // CRC32 placeholder: write zeros, remember position
+        let crc_pos = self.file.stream_position()?;
+        self.file.write_all(&0u32.to_le_bytes())?;
+
+        // Write flags, codec, key, raw_size, data_size
+        self.file.write_all(&[entry.flags])?;
+        self.file.write_all(&[entry.codec as u8])?;
+        self.file.write_all(&entry.key)?;
+        self.file.write_all(&entry.raw_size.to_le_bytes())?;
+        self.file.write_all(&data_size.to_le_bytes())?;
+
+        // Write data
+        self.file.write_all(&entry.data)?;
+
+        // Calculate CRC32 over everything after the crc32 field
+        let crc = {
+            let mut hasher = checksum::CrcWriter::new();
+            hasher.update(&[entry.flags]);
+            hasher.update(&[entry.codec as u8]);
+            hasher.update(&entry.key);
+            hasher.update(&entry.raw_size.to_le_bytes());
+            hasher.update(&data_size.to_le_bytes());
+            hasher.update(&entry.data);
+            hasher.finalize()
+        };
+
+        // Seek back and write the real CRC32
+        self.file.seek(SeekFrom::Start(crc_pos))?;
+        self.file.write_all(&crc.to_le_bytes())?;
+
+        // Seek back to end
+        self.file.seek(SeekFrom::End(0))?;
+
+        self.bytes_written += entry.disk_size() as u64;
+        Ok(())
+    }
+
+    pub fn fsync(&self) -> Result<()> {
+        self.file.sync_all().map_err(|e| {
+            if e.kind() == std::io::ErrorKind::StorageFull {
+                Error::DiskFull(format!("{}: {}", self.path.display(), e))
+            } else {
+                Error::Io(e)
+            }
+        })?;
+        Ok(())
+    }
+}
+
+/// Read entries from a segment file.
+pub struct SegmentReader {
+    path: PathBuf,
+    id: u32,
+}
+
+impl SegmentReader {
+    pub fn open(path: PathBuf, id: u32) -> Result<Self> {
+        Ok(Self { path, id })
+    }
+
+    pub fn id(&self) -> u32 {
+        self.id
+    }
+
+    pub fn path(&self) -> &Path {
+        &self.path
+    }
+
+    pub fn file_size(&self) -> Result<u64> {
+        Ok(fs::metadata(&self.path)?.len())
+    }
+
+    /// Read a single entry at the given offset. Returns the entry and the offset of the next entry.
+    pub fn read_entry_at(&self, offset: u64) -> Result<(Entry, u64)> {
+        let mut file = fs_util::open_read(&self.path)?;
+        file.seek(SeekFrom::Start(offset))?;
+
+        // Read magic
+        let mut magic_buf = [0u8; 4];
+        file.read_exact(&mut magic_buf)?;
+        let magic = u32::from_le_bytes(magic_buf);
+        if magic != ENTRY_MAGIC {
+            return Err(Error::CorruptEntry {
+                path: self.path.clone(),
+                offset,
+                reason: format!("bad magic: 0x{:08X}", magic),
+            });
+        }
+
+        // Read CRC32
+        let mut crc_buf = [0u8; 4];
+        file.read_exact(&mut crc_buf)?;
+        let stored_crc = u32::from_le_bytes(crc_buf);
+
+        // Read flags, codec
+        let mut flags_buf = [0u8; 1];
+        file.read_exact(&mut flags_buf)?;
+        let flags = flags_buf[0];
+
+        let mut codec_buf = [0u8; 1];
+        file.read_exact(&mut codec_buf)?;
+        let codec = Codec::from_u8(codec_buf[0]).ok_or_else(|| Error::CorruptEntry {
+            path: self.path.clone(),
+            offset,
+            reason: format!("unknown codec: {}", codec_buf[0]),
+        })?;
+
+        // Read key, raw_size, data_size
+        let mut key = [0u8; 32];
+        file.read_exact(&mut key)?;
+
+        let mut raw_size_buf = [0u8; 4];
+        file.read_exact(&mut raw_size_buf)?;
+        let raw_size = u32::from_le_bytes(raw_size_buf);
+
+        let mut data_size_buf = [0u8; 4];
+        file.read_exact(&mut data_size_buf)?;
+        let data_size = u32::from_le_bytes(data_size_buf);
+
+        if data_size as usize > crate::types::MAX_VALUE_SIZE {
+            return Err(Error::CorruptEntry {
+                path: self.path.clone(),
+                offset,
+                reason: format!("data_size {} exceeds max {}", data_size, crate::types::MAX_VALUE_SIZE),
+            });
+        }
+
+        // Read data
+        let mut data = vec![0u8; data_size as usize];
+        file.read_exact(&mut data)?;
+
+        // Verify CRC32 (over everything after the crc32 field)
+        let computed_crc = {
+            let mut hasher = checksum::CrcWriter::new();
+            hasher.update(&[flags]);
+            hasher.update(&[codec as u8]);
+            hasher.update(&key);
+            hasher.update(&raw_size.to_le_bytes());
+            hasher.update(&data_size.to_le_bytes());
+            hasher.update(&data);
+            hasher.finalize()
+        };
+
+        if stored_crc != computed_crc {
+            return Err(Error::CrcMismatch {
+                path: self.path.clone(),
+                offset,
+            });
+        }
+
+        let next_offset = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
+
+        Ok((
+            Entry {
+                flags,
+                codec,
+                key,
+                raw_size,
+                data,
+            },
+            next_offset,
+        ))
+    }
+
+    /// Read a single entry at the given offset using a pre-opened File.
+    /// Uses pread so concurrent reads on the same segment don't block each other.
+    pub fn read_entry_at_file(&self, offset: u64, file: &File) -> Result<(Entry, u64)> {
+        // Read header (50 bytes)
+        let mut header = [0u8; ENTRY_HEADER_SIZE];
+        fs_util::pread_exact(file, offset, &mut header)?;
+
+        let mut pos = 0;
+
+        // Magic
+        let magic = u32::from_le_bytes(header[pos..pos+4].try_into().unwrap());
+        pos += 4;
+        if magic != ENTRY_MAGIC {
+            return Err(Error::CorruptEntry {
+                path: self.path.clone(),
+                offset,
+                reason: format!("bad magic: 0x{:08X}", magic),
+            });
+        }
+
+        // CRC32
+        let stored_crc = u32::from_le_bytes(header[pos..pos+4].try_into().unwrap());
+        pos += 4;
+
+        // Flags, codec
+        let flags = header[pos];
+        pos += 1;
+        let codec = Codec::from_u8(header[pos]).ok_or_else(|| Error::CorruptEntry {
+            path: self.path.clone(),
+            offset,
+            reason: format!("unknown codec: {}", header[pos]),
+        })?;
+        pos += 1;
+
+        // Key
+        let mut key = [0u8; 32];
+        key.copy_from_slice(&header[pos..pos+32]);
+        pos += 32;
+
+        // Raw size, data size
+        let raw_size = u32::from_le_bytes(header[pos..pos+4].try_into().unwrap());
+        pos += 4;
+        let data_size = u32::from_le_bytes(header[pos..pos+4].try_into().unwrap());
+
+        // Defense against header corruption: refuse absurdly large allocations
+        if data_size as usize > crate::types::MAX_VALUE_SIZE {
+            return Err(Error::CorruptEntry {
+                path: self.path.clone(),
+                offset,
+                reason: format!("data_size {} exceeds max {}", data_size, crate::types::MAX_VALUE_SIZE),
+            });
+        }
+
+        // Read data
+        let data_offset = offset + ENTRY_HEADER_SIZE as u64;
+        let mut data = vec![0u8; data_size as usize];
+        fs_util::pread_exact(file, data_offset, &mut data)?;
+
+        // Verify CRC32 (over everything after the crc32 field: flags+codec+key+raw_size+data_size+data)
+        let computed_crc = {
+            let mut hasher = crate::checksum::CrcWriter::new();
+            hasher.update(&[flags]);
+            hasher.update(&[codec as u8]);
+            hasher.update(&key);
+            hasher.update(&raw_size.to_le_bytes());
+            hasher.update(&data_size.to_le_bytes());
+            hasher.update(&data);
+            hasher.finalize()
+        };
+
+        if stored_crc != computed_crc {
+            return Err(Error::CrcMismatch {
+                path: self.path.clone(),
+                offset,
+            });
+        }
+
+        let next_offset = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
+
+        Ok((
+            Entry {
+                flags,
+                codec,
+                key,
+                raw_size,
+                data,
+            },
+            next_offset,
+        ))
+    }
+
+    /// Read data portion of an entry (for pread-style reads when you already know offset + data_size).
+    pub fn read_data(&self, offset: u64, data_size: u32) -> Result<Vec<u8>> {
+        let mut file = fs_util::open_read(&self.path)?;
+        // Skip magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4) = 50 bytes
+        let data_start = offset + ENTRY_HEADER_SIZE as u64;
+        file.seek(SeekFrom::Start(data_start))?;
+        let mut buf = vec![0u8; data_size as usize];
+        file.read_exact(&mut buf)?;
+        Ok(buf)
+    }
+
+    /// Read the full entry header + data for verification (used by recovery and GC).
+    pub fn read_full_entry(&self, offset: u64, data_size: u32) -> Result<Vec<u8>> {
+        let mut file = fs_util::open_read(&self.path)?;
+        file.seek(SeekFrom::Start(offset))?;
+        let total = ENTRY_HEADER_SIZE + data_size as usize;
+        let mut buf = vec![0u8; total];
+        file.read_exact(&mut buf)?;
+        Ok(buf)
+    }
+
+    /// Iterate over all valid entries in the segment, calling f for each.
+    /// Stops when hitting a corrupt/incomplete entry at the tail.
+    pub fn scan_entries<F>(&self, start_offset: u64, mut f: F) -> Result<u64>
+    where
+        F: FnMut(&Entry, u64) -> Result<()>,
+    {
+        let file_size = self.file_size()?;
+        let mut offset = start_offset;
+
+        while offset + ENTRY_HEADER_SIZE as u64 <= file_size {
+            match self.read_entry_at(offset) {
+                Ok((entry, next)) => {
+                    f(&entry, offset)?;
+                    offset = next;
+                }
+                Err(Error::CrcMismatch { .. }) | Err(Error::CorruptEntry { .. }) => {
+                    // If near end of file (within one max entry), truncate
+                    if file_size - offset < ENTRY_HEADER_SIZE as u64 + 100 * 1024 * 1024 {
+                        // Likely a partial write at tail, stop here
+                        break;
+                    } else {
+                        return Err(Error::CorruptEntry {
+                            path: self.path.clone(),
+                            offset,
+                            reason: "mid-file corruption detected".into(),
+                        });
+                    }
+                }
+                Err(e) => return Err(e),
+            }
+        }
+
+        Ok(offset) // return the truncation point
+    }
+}
+
+/// Truncate a segment file to the given size.
+pub fn truncate_segment(path: &Path, size: u64) -> Result<()> {
+    fs_util::truncate(path, size)
+}
+
+/// Map an Error, converting Io(StorageFull) to DiskFull with path context.
+fn map_io_err(e: Error, path: &Path) -> Error {
+    match e {
+        Error::Io(io) if io.kind() == std::io::ErrorKind::StorageFull => {
+            Error::DiskFull(format!("{}: {}", path.display(), io))
+        }
+        _ => e,
+    }
+}
+
+/// Segment file name from id: "00000001.seg"
+pub fn segment_filename(id: u32) -> String {
+    format!("{:08}.seg", id)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::TempDir;
+
+    fn temp_segment_path(dir: &TempDir, id: u32) -> PathBuf {
+        dir.path().join(segment_filename(id))
+    }
+
+    #[test]
+    fn test_write_and_read_entry() {
+        let dir = TempDir::new().unwrap();
+        let path = temp_segment_path(&dir, 1);
+        let key = [0xAAu8; 32];
+        let data = b"hello world".to_vec();
+
+        let entry = Entry::new(key, &data, data.len() as u32, 0, Codec::None);
+        {
+            let mut writer = SegmentWriter::create(path.clone(), 1).unwrap();
+            writer.append(&entry).unwrap();
+            writer.fsync().unwrap();
+        }
+
+        let reader = SegmentReader::open(path, 1).unwrap();
+        let (read_entry, next) = reader.read_entry_at(0).unwrap();
+
+        assert_eq!(read_entry.key, key);
+        assert_eq!(read_entry.data, data);
+        assert_eq!(read_entry.flags, 0);
+        assert_eq!(read_entry.raw_size, 11);
+        assert!(next > 0);
+    }
+
+    #[test]
+    fn test_tombstone_entry() {
+        let dir = TempDir::new().unwrap();
+        let path = temp_segment_path(&dir, 1);
+        let key = [0xBBu8; 32];
+
+        let entry = Entry::tombstone(key);
+        {
+            let mut writer = SegmentWriter::create(path.clone(), 1).unwrap();
+            writer.append(&entry).unwrap();
+            writer.fsync().unwrap();
+        }
+
+        let reader = SegmentReader::open(path, 1).unwrap();
+        let (read_entry, _) = reader.read_entry_at(0).unwrap();
+
+        assert!(read_entry.is_tombstone());
+        assert_eq!(read_entry.data.len(), 0);
+    }
+
+    #[test]
+    fn test_multiple_entries() {
+        let dir = TempDir::new().unwrap();
+        let path = temp_segment_path(&dir, 1);
+
+        let entries: Vec<_> = (0..10)
+            .map(|i| {
+                let mut key = [0u8; 32];
+                key[0] = i;
+                Entry::new(key, &vec![i; 100], 100, 0, Codec::None)
+            })
+            .collect();
+
+        {
+            let mut writer = SegmentWriter::create(path.clone(), 1).unwrap();
+            for e in &entries {
+                writer.append(e).unwrap();
+            }
+            writer.fsync().unwrap();
+        }
+
+        let reader = SegmentReader::open(path, 1).unwrap();
+        let mut offset = 0u64;
+        for (i, expected) in entries.iter().enumerate() {
+            let (entry, next) = reader.read_entry_at(offset).unwrap();
+            assert_eq!(entry.key[0], i as u8);
+            assert_eq!(entry.data, expected.data);
+            offset = next;
+        }
+    }
+
+    #[test]
+    fn test_bad_magic_detected() {
+        let dir = TempDir::new().unwrap();
+        let path = temp_segment_path(&dir, 1);
+        // Write garbage
+        std::fs::write(&path, vec![0xFFu8; 100]).unwrap();
+
+        let reader = SegmentReader::open(path, 1).unwrap();
+        let result = reader.read_entry_at(0);
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_is_full() {
+        let dir = TempDir::new().unwrap();
+        let path = temp_segment_path(&dir, 1);
+        let writer = SegmentWriter::create(path, 1).unwrap();
+        assert!(!writer.is_full());
+    }
+}

crates/blob/src/types.rs

@@ -0,0 +1,98 @@
+use serde::{Deserialize, Serialize};
+
+/// Magic number for entry identification
+pub const ENTRY_MAGIC: u32 = 0xB3DB_0001;
+
+/// Fixed header size: magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4)
+pub const ENTRY_HEADER_SIZE: usize = 50;
+
+/// Index record size: key(32) + segment_id(4) + offset(8) + data_size(4) + flags(1) + _pad(3) + crc32(4)
+pub const INDEX_RECORD_SIZE: usize = 56;
+
+/// Maximum segment size (1 GB)
+pub const SEGMENT_MAX_SIZE: u64 = 1024 * 1024 * 1024;
+
+/// Maximum value size (100 MB)
+pub const MAX_VALUE_SIZE: usize = 100 * 1024 * 1024;
+
+/// Default compression threshold (4 KB)
+pub const DEFAULT_COMPRESS_THRESHOLD: usize = 4096;
+
+/// Default GC deleted ratio threshold
+pub const DEFAULT_GC_DELETED_RATIO: f64 = 0.30;
+
+/// Default GC interval in seconds (5 minutes)
+pub const DEFAULT_GC_INTERVAL_SECS: u64 = 300;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+pub enum Codec {
+    None = 0,
+    Zstd = 1,
+    Lz4 = 2,
+}
+
+impl Codec {
+    pub fn from_u8(v: u8) -> Option<Self> {
+        match v {
+            0 => Some(Codec::None),
+            1 => Some(Codec::Zstd),
+            2 => Some(Codec::Lz4),
+            _ => None,
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Config {
+    pub compress_threshold: usize,
+    pub default_codec: Codec,
+    pub compression_level: i32,
+    pub gc_deleted_ratio: f64,
+    /// Interval in seconds for periodic background flush (0 = disabled).
+    /// When set, a background thread fsyncs the active segment and saves
+    /// metadata at this interval, bounding recovery time after a crash.
+    pub flush_interval_secs: u64,
+    /// Interval in seconds for periodic background GC (0 = disabled).
+    /// When set, a background thread checks whether any sealed segment
+    /// exceeds the deleted-ratio threshold and compacts it if needed.
+    pub gc_interval_secs: u64,
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            compress_threshold: DEFAULT_COMPRESS_THRESHOLD,
+            default_codec: Codec::Zstd,
+            compression_level: 0,
+            gc_deleted_ratio: DEFAULT_GC_DELETED_RATIO,
+            flush_interval_secs: 0,
+            gc_interval_secs: 0,
+        }
+    }
+}
+
+impl Config {
+    pub fn validate(&self) -> crate::error::Result<()> {
+        if self.gc_deleted_ratio <= 0.0 || self.gc_deleted_ratio >= 1.0 {
+            return Err(crate::error::Error::InvalidConfig(
+                "gc_deleted_ratio must be in (0.0, 1.0)".into(),
+            ));
+        }
+        if self.compression_level < 0 {
+            return Err(crate::error::Error::InvalidConfig(
+                "compression_level must be >= 0".into(),
+            ));
+        }
+        if self.flush_interval_secs > 0 && self.flush_interval_secs < 5 {
+            return Err(crate::error::Error::InvalidConfig(
+                "flush_interval_secs must be 0 (disabled) or >= 5".into(),
+            ));
+        }
+        if self.gc_interval_secs > 0 && self.gc_interval_secs < 10 {
+            return Err(crate::error::Error::InvalidConfig(
+                "gc_interval_secs must be 0 (disabled) or >= 10".into(),
+            ));
+        }
+        Ok(())
+    }
+}

crates/blob/tests/acid_test.rs

@@ -0,0 +1,514 @@
+/// Crash-consistency and ACID property tests for bichon-blob.
+///
+/// Since we can't kill the process mid-write in an inline test, we simulate crashes
+/// by dropping the Engine without calling any cleanup (close/drop is the "crash"),
+/// then re-opening and verifying recovery produced consistent state.
+
+use std::fs;
+use std::path::Path;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::thread;
+
+use bichon_blob::{Codec, Config, Engine};
+use tempfile::TempDir;
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+fn make_key(seed: u64) -> [u8; 32] {
+    let mut key = [0u8; 32];
+    key[0..8].copy_from_slice(&seed.to_le_bytes());
+    key
+}
+
+fn make_value(size: usize) -> Vec<u8> {
+    let pattern = b"The quick brown fox jumps over the lazy dog. ";
+    let mut v = Vec::with_capacity(size);
+    while v.len() < size {
+        let rem = size - v.len();
+        let n = rem.min(pattern.len());
+        v.extend_from_slice(&pattern[..n]);
+    }
+    v
+}
+
+// ---------------------------------------------------------------------------
+// 1. Durability: committed data survives crash
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_durability_single_write_survives_crash() {
+    let dir = TempDir::new().unwrap();
+    let key = make_key(42);
+    let value = make_value(8192);
+
+    // Write
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        engine.put(key, &value, Codec::Zstd).unwrap();
+    } // <-- Engine dropped = simulated crash
+
+    // Recover
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        let result = engine.get(&key).unwrap();
+        assert_eq!(result, Some(value));
+    }
+}
+
+#[test]
+fn test_durability_many_writes_survive_crash() {
+    let dir = TempDir::new().unwrap();
+    let n = 500;
+    let value = make_value(2048);
+    let mut keys = Vec::new();
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for i in 0..n {
+            let key = make_key(i as u64);
+            keys.push(key);
+            engine.put(key, &value, Codec::Zstd).unwrap();
+        }
+    } // crash
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for (i, key) in keys.iter().enumerate() {
+            let result = engine.get(key).unwrap();
+            assert_eq!(
+                result,
+                Some(value.clone()),
+                "missing key at index {}",
+                i
+            );
+        }
+    }
+}
+
+#[test]
+fn test_durability_delete_survives_crash() {
+    let dir = TempDir::new().unwrap();
+    let key = make_key(99);
+    let value = make_value(4096);
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        engine.put(key, &value, Codec::Zstd).unwrap();
+    } // crash after write
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        engine.delete(&key).unwrap();
+    } // crash after delete
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        let result = engine.get(&key).unwrap();
+        assert_eq!(result, None, "delete should persist across crash");
+    }
+}
+
+// ---------------------------------------------------------------------------
+// 2. Atomicity: no partial writes visible after crash
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_atomicity_no_partial_entries_after_crash() {
+    let dir = TempDir::new().unwrap();
+
+    // Write enough entries to fill part of a segment, then crash
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        let value = make_value(50_000);
+        for i in 0..200u64 {
+            engine
+                .put(make_key(i), &value, Codec::None)
+                .unwrap();
+        }
+    } // crash
+
+    // Recovery should clean up any partial tail entries
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        let value = make_value(50_000);
+        for i in 0..200u64 {
+            let result = engine.get(&make_key(i)).unwrap();
+            assert_eq!(
+                result,
+                Some(value.clone()),
+                "committed key {} should be intact",
+                i
+            );
+        }
+    }
+}
+
+#[test]
+fn test_atomicity_crash_during_segment_roll() {
+    let dir = TempDir::new().unwrap();
+    let big_value = make_value(2 * 1024 * 1024); // 2 MB each entry
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        // Write enough to cross at least one segment boundary (256 MB)
+        for i in 0..140u64 {
+            engine
+                .put(make_key(i), &big_value, Codec::None)
+                .unwrap();
+        }
+    } // crash mid-way or after multiple segments
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for i in 0..140u64 {
+            let result = engine.get(&make_key(i)).unwrap();
+            assert!(
+                result.is_some(),
+                "key {} should exist after segment roll recovery",
+                i
+            );
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// 3. Consistency: CRC detects corruption, no silent data loss
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_consistency_crc_detects_corruption() {
+    let dir = TempDir::new().unwrap();
+    let key = make_key(77);
+    let value = make_value(8192);
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        engine.put(key, &value, Codec::Zstd).unwrap();
+    }
+
+    // Corrupt the segment file by flipping a byte
+    let seg_path = find_first_segment(dir.path());
+    let mut data = fs::read(&seg_path).unwrap();
+    let flip_pos = data.len() - 100;
+    data[flip_pos] ^= 0xFF;
+    fs::write(&seg_path, &data).unwrap();
+
+    // Reading should detect CRC mismatch
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        let result = engine.get(&key);
+        match result {
+            Err(_) => {} // CRC mismatch detected - good
+            Ok(None) => {} // index may point to truncated/removed data
+            Ok(Some(v)) => {
+                if v == value {
+                    panic!("CRC corruption was NOT detected - silent data corruption!");
+                }
+            }
+        }
+    }
+}
+
+#[test]
+fn test_consistency_corrupt_magic_truncated_on_recovery() {
+    let dir = TempDir::new().unwrap();
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for i in 0..10u64 {
+            engine
+                .put(make_key(i), &make_value(4096), Codec::Zstd)
+                .unwrap();
+        }
+    }
+
+    // Append garbage to the segment file (simulating partial write from crash)
+    let seg_path = find_first_segment(dir.path());
+    let mut data = fs::read(&seg_path).unwrap();
+    let orig_len = data.len();
+    data.extend_from_slice(&[0xFF; 200]);
+    fs::write(&seg_path, &data).unwrap();
+
+    // Recovery should truncate the garbage
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for i in 0..10u64 {
+            let result = engine.get(&make_key(i)).unwrap();
+            assert!(
+                result.is_some(),
+                "committed key {} should survive tail truncation",
+                i
+            );
+        }
+    }
+
+    // Verify file was actually truncated
+    let truncated_len = fs::metadata(&seg_path).unwrap().len();
+    assert!(
+        truncated_len <= orig_len as u64,
+        "garbage should have been truncated"
+    );
+}
+
+// ---------------------------------------------------------------------------
+// 4. Isolation: concurrent reader sees consistent snapshot
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_isolation_reader_sees_snapshot_not_partial_write() {
+    let dir = TempDir::new().unwrap();
+    let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
+
+    // Pre-populate a known key
+    let original_value = make_value(4096);
+    let key = make_key(100);
+    engine.put(key, &original_value, Codec::Zstd).unwrap();
+
+    let running = Arc::new(AtomicBool::new(true));
+
+    // Spawn a writer that continuously overwrites the same key
+    let writer_engine = engine.clone();
+    let writer_running = running.clone();
+    let writer_key = key;
+
+    let writer = thread::spawn(move || {
+        for i in 0..1000u64 {
+            if !writer_running.load(Ordering::Relaxed) {
+                break;
+            }
+            let val = make_value(4096 + (i as usize % 100));
+            writer_engine
+                .put(writer_key, &val, Codec::Zstd)
+                .unwrap();
+            thread::yield_now();
+        }
+    });
+
+    // Concurrent reader: reads should never panic or hang
+    let reader_engine = engine.clone();
+    let reader_running = running.clone();
+    let reader = thread::spawn(move || {
+        let mut reads = 0;
+        while reads < 500 {
+            if !reader_running.load(Ordering::Relaxed) && reads > 0 {
+                break;
+            }
+            let result = reader_engine.get(&key);
+            match result {
+                Ok(Some(_)) | Ok(None) => {}
+                Err(e) => {
+                    eprintln!("reader saw error: {:?}", e);
+                }
+            }
+            reads += 1;
+            thread::yield_now();
+        }
+    });
+
+    reader.join().unwrap();
+    running.store(false, Ordering::SeqCst);
+    writer.join().unwrap();
+
+    let final_result = engine.get(&key).unwrap();
+    assert!(final_result.is_some(), "final read should find a value");
+}
+
+// ---------------------------------------------------------------------------
+// 5. Crash during GC: old data intact, no corruption
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_crash_during_gc_leaves_data_intact() {
+    let dir = TempDir::new().unwrap();
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+        let value = make_value(500_000);
+        for i in 0..500u64 {
+            engine
+                .put(make_key(i), &value, Codec::None)
+                .unwrap();
+        }
+        // Delete ~40%
+        for i in (0..500u64).step_by(5) {
+            engine.delete(&make_key(i)).unwrap();
+        }
+        let _ = engine.gc();
+    } // crash after GC
+
+    // All non-deleted entries must still be readable
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        let value = make_value(500_000);
+        for i in 0..500u64 {
+            let key = make_key(i);
+            let result = engine.get(&key).unwrap();
+            if i % 5 == 0 {
+                assert_eq!(result, None, "key {} should be deleted", i);
+            } else {
+                assert_eq!(
+                    result,
+                    Some(value.clone()),
+                    "key {} should survive GC+crash",
+                    i
+                );
+            }
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// 6. Multiple crash-reopen cycles (torture test)
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_multiple_crash_reopen_cycles() {
+    use std::collections::HashSet;
+
+    let dir = TempDir::new().unwrap();
+    let value = make_value(4096);
+    let mut alive: HashSet<u64> = HashSet::new();
+
+    // Populate and crash
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for i in 0..50u64 {
+            engine.put(make_key(i), &value, Codec::Zstd).unwrap();
+            alive.insert(i);
+        }
+    }
+
+    // Reopen, verify all exist, write more, crash
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for &k in &alive {
+            assert!(engine.get(&make_key(k)).unwrap().is_some());
+        }
+        for i in 100..150u64 {
+            engine
+                .put(make_key(i), &value, Codec::Zstd)
+                .unwrap();
+            alive.insert(i);
+        }
+    }
+
+    // Reopen, verify all exist, delete some, crash
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for &k in &alive {
+            assert!(engine.get(&make_key(k)).unwrap().is_some());
+        }
+        for i in 0..10u64 {
+            engine.delete(&make_key(i)).unwrap();
+            alive.remove(&i);
+        }
+    }
+
+    // Final reopen: survivors exist, deleted gone
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for &k in &alive {
+            assert!(
+                engine.get(&make_key(k)).unwrap().is_some(),
+                "key {} should exist",
+                k
+            );
+        }
+        for i in 0..10u64 {
+            assert_eq!(
+                engine.get(&make_key(i)).unwrap(),
+                None,
+                "key {} should be deleted",
+                i
+            );
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// 7. Global dedup: same content stored once
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_global_dedup_after_crash() {
+    let dir = TempDir::new().unwrap();
+    let key = make_key(42);
+    let value = make_value(8192);
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        // Write same key twice (simulating two sources with same content)
+        engine.put(key, &value, Codec::Zstd).unwrap();
+        engine.put(key, &value, Codec::Zstd).unwrap();
+    }
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        let result = engine.get(&key).unwrap();
+        assert_eq!(result, Some(value));
+    }
+}
+
+// ---------------------------------------------------------------------------
+// 8. Recovery + mmap consistency: entries re-indexed by recovery are readable
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_recovery_reloads_bucket_mmaps() {
+    use bichon_blob::meta::GlobalMeta;
+
+    let dir = TempDir::new().unwrap();
+    let value = make_value(8192);
+    let n = 50u64;
+
+    // Phase 1: write data with clean shutdown, then corrupt meta to force
+    // re-indexing on next open (simulates crash where mark_indexed didn't run).
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for i in 0..n {
+            engine.put(make_key(i), &value, Codec::None).unwrap();
+        }
+    } // clean shutdown — everything is indexed and fsynced
+
+    // Corrupt meta: zero out indexed_up_to_offset so recovery re-scans.
+    let mut meta = GlobalMeta::load(dir.path()).expect("failed to load meta");
+    for seg in meta.segments.values_mut() {
+        seg.indexed_up_to_offset = 0;
+    }
+    meta.save(dir.path()).expect("failed to save corrupted meta");
+
+    // Phase 2: reopen — recovery must re-scan the segment and append to bucket
+    // files. After the fix, reload_all() ensures the mmaps include recovered data.
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for i in 0..n {
+            let result = engine.get(&make_key(i)).unwrap();
+            assert_eq!(
+                result,
+                Some(value.clone()),
+                "key {} should be readable after recovery reload",
+                i
+            );
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+fn find_first_segment(store_root: &Path) -> std::path::PathBuf {
+    let seg_dir = store_root.join("segments");
+    for entry in fs::read_dir(&seg_dir).unwrap() {
+        let entry = entry.unwrap();
+        let name = entry.file_name().to_string_lossy().into_owned();
+        if name.ends_with(".seg") && !name.contains("temp_") {
+            return entry.path();
+        }
+    }
+    panic!("no segment found in {:?}", seg_dir);
+}

crates/blob/tests/fuzz_test.rs

@@ -0,0 +1,380 @@
+/// Fuzz-style tests: randomized operation sequences, corruption injection,
+/// and crash-recovery stress testing.
+use bichon_blob::{Codec, Config, Engine};
+use rand::RngExt;
+use std::collections::HashMap;
+use tempfile::TempDir;
+
+/// Number of iterations for each randomized test.
+const FUZZ_OPS: usize = 2000;
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+fn make_key(i: u64) -> [u8; 32] {
+    let mut key = [0u8; 32];
+    key[0..8].copy_from_slice(&i.to_le_bytes());
+    key
+}
+
+fn make_value(rng: &mut impl RngExt) -> Vec<u8> {
+    let size = match rng.random_range(0..100) {
+        0..=4   => rng.random_range(0..64),              // tiny
+        5..=9   => 0,                                  // empty
+        10..=79 => rng.random_range(64..4096),            // small
+        80..=89 => rng.random_range(4096..65536),         // medium
+        90..=94 => rng.random_range(65536..500_000),      // large
+        _       => rng.random_range(500_000..2_000_000),  // xl (near max)
+    };
+    let mut v = vec![0u8; size];
+    rng.fill(&mut v[..]);
+    v
+}
+
+fn random_codec(rng: &mut impl RngExt) -> Codec {
+    match rng.random_range(0..4) {
+        0 => Codec::None,
+        1 => Codec::Zstd,
+        _ => Codec::Lz4,
+    }
+}
+
+// ── Fuzz: random operation sequence ─────────────────────────────────────────
+
+#[test]
+fn fuzz_random_ops() {
+    let mut rng = rand::rng();
+    let dir = TempDir::new().unwrap();
+    let mut config = Config::default();
+    config.flush_interval_secs = 0; // manual flush only
+    config.gc_interval_secs = 0;
+
+    let engine = Engine::open(dir.path(), config).unwrap();
+
+    // Oracle: track expected values in memory
+    let mut oracle: HashMap<[u8; 32], Vec<u8>> = HashMap::new();
+    let mut next_key = 0u64;
+
+    for _ in 0..FUZZ_OPS {
+        match rng.random_range(0..100) {
+            // 45%: put new key
+            0..=44 => {
+                let key = make_key(next_key);
+                next_key += 1;
+                let value = make_value(&mut rng);
+                let codec = random_codec(&mut rng);
+                engine.put(key, &value, codec).unwrap();
+                oracle.insert(key, value);
+            }
+            // 20%: put overwrite existing key
+            45..=64 => {
+                if oracle.is_empty() { continue; }
+                let idx = rng.random_range(0..oracle.len());
+                let key = *oracle.iter().nth(idx).unwrap().0;
+                let value = make_value(&mut rng);
+                let codec = random_codec(&mut rng);
+                engine.put(key, &value, codec).unwrap();
+                oracle.insert(key, value);
+            }
+            // 15%: read and verify
+            65..=79 => {
+                if oracle.is_empty() { continue; }
+                let idx = rng.random_range(0..oracle.len());
+                let key = *oracle.iter().nth(idx).unwrap().0;
+                let expected = oracle.get(&key).unwrap();
+                let got = engine.get(&key).unwrap();
+                assert_eq!(got.as_ref(), Some(expected), "key mismatch on read");
+            }
+            // 10%: delete
+            80..=89 => {
+                if oracle.is_empty() { continue; }
+                let idx = rng.random_range(0..oracle.len());
+                let key = *oracle.iter().nth(idx).unwrap().0;
+                engine.delete(&key).unwrap();
+                oracle.remove(&key);
+                let got = engine.get(&key).unwrap();
+                assert_eq!(got, None, "deleted key should return None");
+            }
+            // 5%: read non-existent key
+            90..=94 => {
+                let key = make_key(next_key + rng.random_range(1000u64..10000));
+                let got = engine.get(&key).unwrap();
+                assert_eq!(got, None, "non-existent key should return None");
+            }
+            // 5%: flush
+            _ => {
+                engine.flush().unwrap();
+            }
+        }
+    }
+
+    // Final verification: all oracle entries must match
+    for (key, expected) in &oracle {
+        let got = engine.get(key).unwrap();
+        assert_eq!(got.as_ref(), Some(expected), "final verification: key mismatch");
+    }
+}
+
+// ── Fuzz: crash + reopen cycle ──────────────────────────────────────────────
+
+#[test]
+fn fuzz_crash_reopen_cycles() {
+    let mut rng = rand::rng();
+    let dir = TempDir::new().unwrap();
+    let dir_path = dir.path().to_path_buf();
+
+    let mut oracle: HashMap<[u8; 32], Vec<u8>> = HashMap::new();
+    let mut next_key = 0u64;
+    let cycles = 20;
+
+    for _cycle in 0..cycles {
+        // Open database
+        let mut config = Config::default();
+        config.flush_interval_secs = 0;
+        config.gc_interval_secs = 0;
+        let engine = Engine::open(&dir_path, config).unwrap();
+
+        // Do some work
+        let ops = rng.random_range(50..200);
+        for _ in 0..ops {
+            match rng.random_range(0..100) {
+                0..=50 => {
+                    let key = make_key(next_key);
+                    next_key += 1;
+                    let value = make_value(&mut rng);
+                    if engine.put(key, &value, Codec::Zstd).is_ok() {
+                        oracle.insert(key, value);
+                    }
+                }
+                51..=65 => {
+                    if oracle.is_empty() { continue; }
+                    let idx = rng.random_range(0..oracle.len());
+                    let key = *oracle.iter().nth(idx).unwrap().0;
+                    engine.delete(&key).unwrap();
+                    oracle.remove(&key);
+                }
+                66..=85 => {
+                    if oracle.is_empty() { continue; }
+                    let idx = rng.random_range(0..oracle.len());
+                    let key = *oracle.iter().nth(idx).unwrap().0;
+                    let expected = oracle.get(&key).unwrap();
+                    if let Ok(Some(got)) = engine.get(&key) {
+                        assert_eq!(&got, expected, "pre-crash read mismatch");
+                    }
+                }
+                _ => {
+                    let _ = engine.flush();
+                }
+            }
+        }
+
+        // Simulate crash: drop without shutdown
+        drop(engine);
+    }
+
+    // Final reopen: all oracle entries must be intact
+    let config = Config::default();
+    let engine = Engine::open(&dir_path, config).unwrap();
+    for (key, expected) in &oracle {
+        let got = engine.get(key).unwrap();
+        assert_eq!(got.as_ref(), Some(expected), "after {} crash cycles", cycles);
+    }
+}
+
+// ── Fuzz: batch operations ──────────────────────────────────────────────────
+
+#[test]
+fn fuzz_batch_ops() {
+    let mut rng = rand::rng();
+    let dir = TempDir::new().unwrap();
+
+    let config = Config::default();
+    let engine = Engine::open(dir.path(), config).unwrap();
+
+    let mut oracle: HashMap<[u8; 32], Vec<u8>> = HashMap::new();
+    let mut next_key = 0u64;
+
+    for _ in 0..200 {
+        match rng.random_range(0..100) {
+            // 50%: batch write
+            0..=49 => {
+                let batch_size = rng.random_range(1..30);
+                let entries: Vec<_> = (0..batch_size)
+                    .map(|_| {
+                        let key = make_key(next_key);
+                        next_key += 1;
+                        let value = make_value(&mut rng);
+                        oracle.insert(key, value.clone());
+                        (key, value, Codec::Zstd)
+                    })
+                    .collect();
+                engine.put_batch(&entries).unwrap();
+            }
+            // 30%: verify random subset
+            50..=79 => {
+                if oracle.is_empty() { continue; }
+                let n = rng.random_range(1..=20.min(oracle.len()));
+                for _ in 0..n {
+                    let idx = rng.random_range(0..oracle.len());
+                    let (key, expected) = oracle.iter().nth(idx).unwrap();
+                    let got = engine.get(key).unwrap();
+                    assert_eq!(got.as_ref(), Some(expected));
+                }
+            }
+            // 20%: batch delete
+            _ => {
+                if oracle.is_empty() { continue; }
+                let n = rng.random_range(1..=20.min(oracle.len()));
+                let keys: Vec<[u8; 32]> = (0..n)
+                    .map(|_| {
+                        let idx = rng.random_range(0..oracle.len());
+                        let key = *oracle.iter().nth(idx).unwrap().0;
+                        oracle.remove(&key);
+                        key
+                    })
+                    .collect();
+                engine.delete_batch(&keys).unwrap();
+            }
+        }
+    }
+
+    for (key, expected) in &oracle {
+        let got = engine.get(key).unwrap();
+        assert_eq!(got.as_ref(), Some(expected));
+    }
+}
+
+// ── Fuzz: GC stress ─────────────────────────────────────────────────────────
+
+#[test]
+fn fuzz_gc_stress() {
+    let mut rng = rand::rng();
+    let dir = TempDir::new().unwrap();
+
+    let mut config = Config::default();
+    config.gc_deleted_ratio = 0.1; // aggressive GC trigger
+    let engine = Engine::open(dir.path(), config).unwrap();
+
+    let mut oracle: HashMap<[u8; 32], Vec<u8>> = HashMap::new();
+    let mut next_key = 0u64;
+
+    for round in 0..10 {
+        // Write a batch of keys
+        let n = rng.random_range(50..150);
+        let mut round_keys: Vec<[u8; 32]> = Vec::new();
+        let value_size = rng.random_range(100..10000);
+        let value: Vec<u8> = (0..value_size).map(|_| rng.random::<u8>()).collect();
+
+        for _ in 0..n {
+            let key = make_key(next_key);
+            next_key += 1;
+            engine.put(key, &value, Codec::None).unwrap();
+            oracle.insert(key, value.clone());
+            round_keys.push(key);
+        }
+
+        // Delete some fraction
+        let delete_frac: f64 = rng.random_range(0.2..0.8);
+        let delete_count = (round_keys.len() as f64 * delete_frac) as usize;
+        for _ in 0..delete_count {
+            let idx = rng.random_range(0..round_keys.len());
+            let key = round_keys.swap_remove(idx);
+            engine.delete(&key).unwrap();
+            oracle.remove(&key);
+        }
+
+        // Seal and run GC
+        engine.seal_active_segment().unwrap();
+        let _ = engine.gc().unwrap();
+
+        // Verify all remaining oracle entries
+        for (key, expected) in &oracle {
+            let got = engine.get(key).unwrap();
+            assert_eq!(
+                got.as_ref(),
+                Some(expected),
+                "GC round {}: key mismatch",
+                round
+            );
+        }
+    }
+}
+
+// ── Fuzz: bitflip corruption resilience ─────────────────────────────────────
+
+#[test]
+fn fuzz_corruption_resilience() {
+    let mut rng = rand::rng();
+    let dir = TempDir::new().unwrap();
+    let dir_path = dir.path().to_path_buf();
+
+    let config = Config::default();
+    let engine = Engine::open(&dir_path, config).unwrap();
+
+    // Write known data
+    let mut good_keys: Vec<[u8; 32]> = Vec::new();
+    let n = 100u64;
+    for i in 0..n {
+        let key = make_key(i);
+        let value = vec![i as u8; 2048];
+        engine.put(key, &value, Codec::None).unwrap();
+        good_keys.push(key);
+    }
+    engine.flush().unwrap();
+
+    // Seal so segment file is durable on disk
+    engine.seal_active_segment().unwrap();
+    engine.shutdown().unwrap();
+    drop(engine);
+
+    // Find segment files and corrupt random bytes
+    let seg_dir = dir_path.join("segments");
+    let mut seg_files: Vec<_> = std::fs::read_dir(&seg_dir)
+        .unwrap()
+        .filter_map(|e| e.ok())
+        .filter(|e| {
+            e.file_name()
+                .to_string_lossy()
+                .ends_with(".seg")
+        })
+        .collect();
+    seg_files.sort_by_key(|e| e.file_name());
+
+    // Corrupt 3 random bytes in the first segment
+    if let Some(seg) = seg_files.first() {
+        let path = seg.path();
+        let mut data = std::fs::read(&path).unwrap();
+        if data.len() > 100 {
+            for _ in 0..3 {
+                let pos = rng.random_range(50..data.len());
+                data[pos] ^= 0xFF; // flip all bits
+            }
+            std::fs::write(&path, &data).unwrap();
+        }
+    }
+
+    // Reopen: recovery must succeed (not panic), even if some keys are lost
+    let config = Config::default();
+    let engine = Engine::open(&dir_path, config).unwrap();
+
+    // At least some uncorrupted keys should still be readable
+    let mut readable = 0;
+    let mut corrupted = 0;
+    for key in &good_keys {
+        match engine.get(key) {
+            Ok(Some(_)) => readable += 1,
+            Ok(None) => { /* key might be lost due to corruption */ }
+            Err(_) => corrupted += 1,
+        }
+    }
+    // The vast majority should still be ok (corruption only hit 3 bytes in one segment)
+    assert!(
+        readable + corrupted > 0,
+        "at least some outcomes should be observable"
+    );
+    assert!(
+        readable > n as usize / 2,
+        "majority of keys should survive localized corruption ({} of {})",
+        readable,
+        n
+    );
+}

crates/blob/tests/integration_test.rs

@@ -0,0 +1,445 @@
+use bichon_blob::{Codec, Config, Engine};
+use tempfile::TempDir;
+
+#[test]
+fn test_write_and_read() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let key = [0xAA; 32];
+    let value = b"Hello, this is a test email!".to_vec();
+
+    engine.put(key, &value, Codec::Zstd).unwrap();
+
+    let result = engine.get(&key).unwrap();
+    assert_eq!(result, Some(value));
+}
+
+#[test]
+fn test_read_missing_key() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let key = [0xFF; 32];
+    let result = engine.get(&key).unwrap();
+    assert_eq!(result, None);
+}
+
+#[test]
+fn test_delete() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let key = [0xBB; 32];
+    let value = b"Some email content".to_vec();
+
+    engine.put(key, &value, Codec::Zstd).unwrap();
+    engine.delete(&key).unwrap();
+
+    let result = engine.get(&key).unwrap();
+    assert_eq!(result, None);
+}
+
+#[test]
+fn test_exists() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let key = [0xCC; 32];
+    assert!(!engine.exists(&key).unwrap());
+
+    engine.put(key, b"data", Codec::None).unwrap();
+    assert!(engine.exists(&key).unwrap());
+
+    engine.delete(&key).unwrap();
+    assert!(!engine.exists(&key).unwrap());
+}
+
+#[test]
+fn test_small_value_not_compressed() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let key = [0xCC; 32];
+    let value = b"hi"; // Smaller than 4KB threshold
+
+    engine.put(key, value, Codec::Zstd).unwrap();
+
+    let result = engine.get(&key).unwrap();
+    assert_eq!(result, Some(value.to_vec()));
+}
+
+#[test]
+fn test_large_value() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let key = [0xDD; 32];
+    let value = vec![b'X'; 100_000]; // 100KB
+
+    engine.put(key, &value, Codec::Zstd).unwrap();
+
+    let result = engine.get(&key).unwrap();
+    assert_eq!(result, Some(value));
+}
+
+#[test]
+fn test_multiple_keys() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let n = 100;
+    for i in 0..n {
+        let mut key = [0u8; 32];
+        key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
+        let value = format!("email number {}", i).into_bytes();
+        engine.put(key, &value, Codec::Zstd).unwrap();
+    }
+
+    for i in 0..n {
+        let mut key = [0u8; 32];
+        key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
+        let result = engine.get(&key).unwrap();
+        assert_eq!(result, Some(format!("email number {}", i).into_bytes()));
+    }
+}
+
+#[test]
+fn test_gc() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    // Write many entries
+    let value = vec![b'Y'; 5000];
+    let n = 100;
+
+    for i in 0..n {
+        let mut key = [0u8; 32];
+        key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
+        engine.put(key, &value, Codec::None).unwrap();
+    }
+
+    // Delete even-numbered keys
+    for i in (0..n).step_by(2) {
+        let mut key = [0u8; 32];
+        key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
+        engine.delete(&key).unwrap();
+    }
+
+    // Run GC
+    let _result = engine.gc().unwrap();
+
+    // Verify remaining keys still readable
+    for i in (1..n).step_by(2) {
+        let mut key = [0u8; 32];
+        key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
+        let result = engine.get(&key).unwrap();
+        assert_eq!(result, Some(value.clone()));
+    }
+
+    // Deleted keys should not exist
+    for i in (0..n).step_by(2) {
+        let mut key = [0u8; 32];
+        key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
+        let result = engine.get(&key).unwrap();
+        assert_eq!(result, None);
+    }
+}
+
+#[test]
+fn test_reopen_persistence() {
+    let dir = TempDir::new().unwrap();
+    let key = [0xEE; 32];
+    let value = b"persistent data".to_vec();
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        engine.put(key, &value, Codec::Zstd).unwrap();
+    }
+
+    // Reopen
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        let result = engine.get(&key).unwrap();
+        assert_eq!(result, Some(value));
+    }
+}
+
+#[test]
+fn test_stats() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    engine.put([1u8; 32], b"hello", Codec::None).unwrap();
+
+    let stats = engine.stats().unwrap();
+    assert!(stats.total_bytes > 0);
+    assert!(stats.total_keys > 0);
+}
+
+#[test]
+fn test_batch_write() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let n = 50;
+    let entries: Vec<_> = (0..n)
+        .map(|i: u64| {
+            let mut key = [0u8; 32];
+            key[0..8].copy_from_slice(&i.to_le_bytes());
+            let value = format!("batch email {}", i).into_bytes();
+            (key, value, Codec::Zstd)
+        })
+        .collect();
+
+    engine.put_batch(&entries).unwrap();
+
+    for (key, value, _) in &entries {
+        let result = engine.get(key).unwrap();
+        assert_eq!(result.as_ref(), Some(value));
+    }
+}
+
+#[test]
+fn test_batch_write_persistence() {
+    let dir = TempDir::new().unwrap();
+    let entries: Vec<_> = (0..30u64)
+        .map(|i| {
+            let mut key = [0u8; 32];
+            key[0..8].copy_from_slice(&i.to_le_bytes());
+            (key, format!("persist {}", i).into_bytes(), Codec::Zstd)
+        })
+        .collect();
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        engine.put_batch(&entries).unwrap();
+    }
+
+    {
+        let engine = Engine::open(dir.path(), Config::default()).unwrap();
+        for (key, value, _) in &entries {
+            let result = engine.get(key).unwrap();
+            assert_eq!(result.as_ref(), Some(value));
+        }
+    }
+}
+
+#[test]
+fn test_invalid_config_rejected() {
+    let dir = TempDir::new().unwrap();
+    let mut config = Config::default();
+    config.compression_level = -1;
+    assert!(Engine::open(dir.path(), config).is_err());
+
+    let mut config = Config::default();
+    config.gc_deleted_ratio = 1.5;
+    assert!(Engine::open(dir.path(), config).is_err());
+}
+
+#[test]
+fn test_concurrent_reads() {
+    use std::sync::Arc;
+    use std::thread;
+
+    let dir = TempDir::new().unwrap();
+    let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
+
+    // Write some data
+    for i in 0..50u32 {
+        let mut key = [0u8; 32];
+        key[0..4].copy_from_slice(&i.to_le_bytes());
+        engine
+            .put(key, &vec![i as u8; 1024], Codec::None)
+            .unwrap();
+    }
+
+    // Spawn 4 threads, each reading a different subset
+    let mut handles = vec![];
+    for t in 0..4 {
+        let engine = engine.clone();
+        handles.push(thread::spawn(move || {
+            for i in (t * 12)..((t + 1) * 12) {
+                let mut key = [0u8; 32];
+                key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
+                let read = engine.get(&key).unwrap();
+                assert!(read.is_some(), "key {} should exist", i);
+            }
+        }));
+    }
+    for h in handles {
+        h.join().unwrap();
+    }
+}
+
+#[test]
+fn test_global_dedup() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    let key = [0x42; 32];
+    let value = b"same content across what would be accounts".to_vec();
+
+    // Write same key twice (simulating two accounts ingesting the same email)
+    engine.put(key, &value, Codec::Zstd).unwrap();
+    engine.put(key, &value, Codec::Zstd).unwrap();
+
+    // Should still be readable
+    let result = engine.get(&key).unwrap();
+    assert_eq!(result, Some(value));
+
+    // Stats should reflect dedup (not double count)
+    let stats = engine.stats().unwrap();
+    // The key appears once in the bucket store
+    assert!(stats.total_keys > 0);
+}
+
+#[test]
+fn test_crash_recovery() {
+    let dir = TempDir::new().unwrap();
+    let dir_path = dir.path().to_path_buf();
+
+    // Phase 1: write data, then drop without shutdown (simulates crash)
+    {
+        let engine = Engine::open(&dir_path, Config::default()).unwrap();
+
+        for i in 0..50u32 {
+            let mut key = [0u8; 32];
+            key[0..4].copy_from_slice(&i.to_le_bytes());
+            engine
+                .put(key, &vec![i as u8; 512], Codec::None)
+                .unwrap();
+        }
+        // Engine dropped here without calling shutdown()
+    }
+
+    // Phase 2: reopen - recovery should run, data should be intact
+    let engine = Engine::open(&dir_path, Config::default()).unwrap();
+    let stats = engine.stats().unwrap();
+    assert!(stats.total_keys > 0, "recovery should preserve data");
+
+    // Verify reads work
+    for i in 0..50u32 {
+        let mut key = [0u8; 32];
+        key[0..4].copy_from_slice(&i.to_le_bytes());
+        let read = engine.get(&key).unwrap();
+        assert!(read.is_some(), "key {} should survive crash recovery", i);
+    }
+}
+
+#[test]
+fn test_meta_bin_durability() {
+    let dir = TempDir::new().unwrap();
+    let dir_path = dir.path().to_path_buf();
+
+    {
+        let engine = Engine::open(&dir_path, Config::default()).unwrap();
+
+        let key = [0x42u8; 32];
+        engine.put(key, b"durable", Codec::None).unwrap();
+    }
+    // Engine dropped -> shutdown() called -> meta saved
+
+    // Verify meta.bin exists
+    let meta_path = dir_path.join("meta.bin");
+    assert!(meta_path.exists(), "meta.bin should exist after clean shutdown");
+
+    let data = std::fs::read(&meta_path).unwrap();
+    assert!(
+        data.len() >= 8,
+        "meta.bin should have at least 8 bytes"
+    );
+
+    let stored_crc = u32::from_le_bytes(data[0..4].try_into().unwrap());
+    assert_ne!(stored_crc, 0, "stored CRC should be non-zero");
+
+    // Reopen and verify data is intact
+    let engine = Engine::open(&dir_path, Config::default()).unwrap();
+    let read = engine.get(&[0x42u8; 32]).unwrap();
+    assert_eq!(read, Some(b"durable".to_vec()));
+}
+
+#[test]
+fn test_gc_deletes_empty_segment() {
+    let dir = TempDir::new().unwrap();
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+
+    // Write data, seal, then delete everything so the sealed segment
+    // becomes 100% garbage.  GC should remove the segment file entirely.
+    let n = 50u64;
+    for i in 0..n {
+        let mut key = [0u8; 32];
+        key[0..8].copy_from_slice(&i.to_le_bytes());
+        engine.put(key, &vec![b'X'; 4096], Codec::None).unwrap();
+    }
+
+    // Force seal so the segment becomes a GC candidate.
+    let sealed_id = engine.seal_active_segment().unwrap();
+
+    // Delete all keys — the sealed segment is now entirely garbage.
+    let keys: Vec<[u8; 32]> = (0..n)
+        .map(|i| {
+            let mut key = [0u8; 32];
+            key[0..8].copy_from_slice(&i.to_le_bytes());
+            key
+        })
+        .collect();
+    engine.delete_batch(&keys).unwrap();
+
+    // Run GC — this should empty and then delete the sealed segment.
+    let result = engine.gc().unwrap();
+    assert!(result.is_some(), "GC should have found a candidate");
+    let stats = result.unwrap();
+    assert_eq!(stats.segment_id, sealed_id);
+    assert_eq!(stats.bytes_after, 0);
+
+    // Segment file must be deleted.
+    let seg_path = dir
+        .path()
+        .join("segments")
+        .join(format!("{:08}.seg", sealed_id));
+    assert!(
+        !seg_path.exists(),
+        "emptied segment file should have been deleted, but {:?} exists",
+        seg_path
+    );
+
+    // Subsequent reads / writes must still work (no corruption).
+    let new_key = [0x99u8; 32];
+    engine
+        .put(new_key, b"post-gc data", Codec::None)
+        .unwrap();
+    let result = engine.get(&new_key).unwrap();
+    assert_eq!(result, Some(b"post-gc data".to_vec()));
+
+    // Engine stats should be consistent.
+    let stats = engine.stats().unwrap();
+    assert_eq!(stats.total_keys, 1);
+
+    // Shutdown and reopen — persistence must be intact.
+    drop(engine);
+    let engine = Engine::open(dir.path(), Config::default()).unwrap();
+    let result = engine.get(&new_key).unwrap();
+    assert_eq!(result, Some(b"post-gc data".to_vec()));
+    let result = engine.get(&keys[0]).unwrap();
+    assert_eq!(result, None);
+}
+
+#[test]
+fn test_file_lock_prevents_concurrent_open() {
+    let dir = TempDir::new().unwrap();
+    let _engine1 = Engine::open(dir.path(), Config::default()).unwrap();
+    let result = Engine::open(dir.path(), Config::default());
+    assert!(result.is_err(), "second open on same directory must fail");
+}
+
+#[test]
+fn test_file_lock_released_after_close() {
+    let dir = TempDir::new().unwrap();
+    {
+        let _engine = Engine::open(dir.path(), Config::default()).unwrap();
+    }
+    // Lock should be released after engine is dropped
+    let engine = Engine::open(dir.path(), Config::default());
+    assert!(engine.is_ok(), "reopen after close must succeed");
+}

crates/cli/Cargo.toml

@@ -16,12 +16,8 @@ reqwest.workspace = true
 toml = "0.9.8"
 memmap2 = "0.9.10"
 outlook-pst = { git = "https://github.com/rustmailer/outlook-pst-rs.git", branch = "main" }
-compressed-rtf = "1.0.1"
 chrono.workspace = true
-mail-send.workspace = true
 base64.workspace = true
-codepage-strings = "1.0.2"
-hex = "0.4.3"
 sysinfo.workspace = true
 indicatif.workspace = true
 serde_json.workspace = true

crates/cli/src/mbox/mod.rs

@@ -21,7 +21,7 @@ use std::path::PathBuf;
 
 use crate::api::sender::send_batch_request;
 use crate::mbox::gmail::determine_folder;
-use crate::mbox::reader::MboxFile;
+use bichon_core::import::reader::MboxFile;
 use crate::BichonCliConfig;
 use bichon_core::base64_encode_url_safe;
 use bichon_core::envelope::meta::{parse_bichon_metadata, BichonMetadata};
@@ -37,7 +37,6 @@ const MAX_EMAIL_BYTES: usize = 100 * 1024 * 1024;
 const MAX_BUFFER_BYTES: usize = 200 * 1024 * 1024;
 
 pub mod gmail;
-pub mod reader;
 
 pub async fn handle_mbox_single_file_import(
     config: &BichonCliConfig,

crates/cli/src/pst/mod.rs

@@ -16,21 +16,12 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-use chrono::{DateTime, TimeZone, Utc};
-use dialoguer::theme::ColorfulTheme;
-use dialoguer::Input;
-use mail_send::mail_builder::headers::text::Text;
-use mail_send::mail_builder::MessageBuilder;
-use outlook_pst::ltp::prop_context::PropertyValue;
-
 use crate::api::sender::send_batch_request;
-use crate::pst::encoding::decode_subject;
 use crate::BichonCliConfig;
-use bichon_core::base64_encode_url_safe;
-use dialoguer::Confirm;
-use outlook_pst::messaging::attachment::AttachmentProperties;
+use bichon_core::import::pst::build_eml_base64;
+use dialoguer::theme::ColorfulTheme;
+use dialoguer::{Confirm, Input};
 use outlook_pst::messaging::folder::Folder;
-use outlook_pst::messaging::message::{Message, MessageProperties};
 use outlook_pst::ndb::node_id::NodeId;
 use reqwest::Client;
 use std::future::Future;
@@ -38,28 +29,6 @@ use std::path::PathBuf;
 use std::pin::Pin;
 use std::rc::Rc;
 
-mod encoding;
-
-#[derive(Debug, Default)]
-pub struct EmailMetadata {
-    pub message_id: Option<String>,
-    pub subject: Option<String>,
-    pub from: Option<String>,
-    pub to: Option<Vec<String>>,
-    pub cc: Option<Vec<String>>,
-    pub bcc: Option<Vec<String>>,
-    pub html: Option<String>,
-    pub text: Option<String>,
-    pub in_reply_to: Option<String>,
-}
-
-#[derive(Debug, Default)]
-pub struct EmailAttachment {
-    pub name: Option<String>,
-    pub mime_type: Option<String>,
-    pub data: Option<Vec<u8>>,
-}
-
 pub async fn handle_pst_import(config: &BichonCliConfig, account_id: u64, theme: &ColorfulTheme) {
     let path_str: String = Input::with_theme(theme)
         .with_prompt("Enter the path to your SINGLE .pst file")
@@ -244,167 +213,6 @@ fn process_folder_recursively<'a>(
     })
 }
 
-fn build_eml_base64(message: Rc<dyn Message>) -> Option<String> {
-    let properties = message.properties();
-
-    let mut builder = MessageBuilder::new();
-    if let Some(sub) = extract_subject(properties) {
-        builder = builder.subject(sub);
-    }
-    if let Some(mid) = extract_string_property(properties, 0x1035) {
-        builder = builder.message_id(mid);
-    }
-    if let Some(irt) = extract_string_property(properties, 0x1042) {
-        builder = builder.in_reply_to(irt);
-    }
-
-    if let Some(refs) = extract_string_property(properties, 0x1039) {
-        builder = builder.header("References", Text::new(refs));
-    }
-
-    if let Some(cid_val) = properties.get(0x3013) {
-        if let PropertyValue::Binary(bin) = cid_val {
-            builder = builder.header(
-                "X-Bichon-Conversation-ID",
-                Text::new(hex::encode(bin.buffer())),
-            );
-        }
-    }
-
-    let from = extract_string_property(properties, 0x5D01)
-        .or_else(|| extract_string_property(properties, 0x5D02))
-        .or_else(|| extract_string_property(properties, 0x0C1F));
-
-    if let Some(f) = from {
-        builder = builder.from(f);
-    }
-
-    if let Some(filetime) = extract_i64_property(properties, &[0x0039, 0x0E06]) {
-        let dt = filetime_to_datetime(filetime).timestamp();
-        builder = builder.date(dt);
-    }
-
-    let (to, cc, bcc) = extract_recipients_list(&message);
-    if !to.is_empty() {
-        builder = builder.to(to.iter().map(|s| s.as_str()).collect::<Vec<_>>());
-    }
-    if !cc.is_empty() {
-        builder = builder.cc(cc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
-    }
-    if !bcc.is_empty() {
-        builder = builder.bcc(bcc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
-    }
-
-    if let Some(html) = extract_html(properties) {
-        builder = builder.html_body(html);
-    }
-
-    if let Some(text) = extract_text(properties) {
-        builder = builder.text_body(text);
-    }
-
-    if let Some(attachment_table) = message.attachment_table() {
-        for row in attachment_table.rows_matrix() {
-            let node_id = NodeId::from(u32::from(row.id()));
-            if let Ok(attachment) = message.clone().read_attachment(node_id, None) {
-                let att_props = attachment.properties();
-                let name = extract_attachment_string_property(att_props, 0x3707);
-                let mime = extract_attachment_string_property(att_props, 0x370E)
-                    .unwrap_or_else(|| "application/octet-stream".into());
-                let cid = extract_attachment_string_property(att_props, 0x3712);
-                let is_inline = att_props
-                    .get(0x3714)
-                    .and_then(|val| {
-                        if let PropertyValue::Integer32(f) = val {
-                            Some(f)
-                        } else {
-                            None
-                        }
-                    })
-                    .map(|flag| (flag & 0x4) != 0)
-                    .unwrap_or(false);
-
-                if let Some(PropertyValue::Binary(bin)) = att_props.get(0x3701) {
-                    let data = bin.buffer().to_vec();
-                    let file_name = name.unwrap_or_else(|| "unnamed_attachment".to_string());
-
-                    if is_inline && cid.is_some() {
-                        let content_id = cid.unwrap();
-                        builder = builder.inline(mime, content_id, data);
-                    } else {
-                        builder = builder.attachment(mime, file_name, data);
-                    }
-                }
-            }
-        }
-    }
-
-    match builder.write_to_vec() {
-        Ok(eml_vec) => Some(base64_encode_url_safe!(eml_vec)),
-        Err(e) => {
-            eprintln!("Failed to generate EML: {:?}", e);
-            None
-        }
-    }
-}
-
-fn filetime_to_datetime(filetime: i64) -> DateTime<Utc> {
-    let unix_secs = (filetime / 10_000_000) - 11_644_473_600;
-    let nsecs = (filetime % 10_000_000) * 100;
-    Utc.timestamp_opt(unix_secs, nsecs as u32).unwrap()
-}
-
-fn extract_recipients_list(message: &Rc<dyn Message>) -> (Vec<String>, Vec<String>, Vec<String>) {
-    let mut to = Vec::new();
-    let mut cc = Vec::new();
-    let mut bcc = Vec::new();
-
-    let recipient_table = message.recipient_table();
-    if let Some(recipient_table) = recipient_table {
-        let context = recipient_table.context();
-        for row in recipient_table.rows_matrix() {
-            if let Ok(cols) = row.columns(context) {
-                let mut r_type = 0;
-                let mut email = String::new();
-
-                for (col, val) in context.columns().iter().zip(cols) {
-                    let prop_val = val
-                        .as_ref()
-                        .and_then(|v| recipient_table.read_column(v, col.prop_type()).ok());
-                    match col.prop_id() {
-                        0x0C15 => {
-                            if let Some(PropertyValue::Integer32(t)) = prop_val {
-                                r_type = t;
-                            }
-                        }
-                        0x39FE | 0x3003 => {
-                            if let Some(s) = prop_val.and_then(|v| extract_string(&v)) {
-                                email = s;
-                            }
-                        }
-                        _ => {}
-                    }
-                }
-
-                if !email.is_empty() {
-                    match r_type {
-                        1 => to.push(email),
-                        2 => cc.push(email),
-                        3 => bcc.push(email),
-                        _ => {}
-                    }
-                }
-            }
-        }
-    } else {
-        let receiver = extract_string_property(message.properties(), 0x0076);
-        if let Some(receiver) = receiver {
-            to.push(receiver);
-        }
-    }
-    (to, cc, bcc)
-}
-
 async fn send_to_bichon(
     client: &Client,
     config: &BichonCliConfig,
@@ -414,69 +222,3 @@ async fn send_to_bichon(
 ) {
     send_batch_request(client, config, account_id, folder_path, emls).await;
 }
-
-fn extract_subject(props: &MessageProperties) -> Option<String> {
-    props.get(0x0037).and_then(|val| decode_subject(val))
-}
-
-fn extract_string_property(properties: &MessageProperties, prop_id: u16) -> Option<String> {
-    properties
-        .get(prop_id)
-        .and_then(|value| extract_string(value))
-}
-
-fn extract_attachment_string_property(
-    properties: &AttachmentProperties,
-    prop_id: u16,
-) -> Option<String> {
-    properties
-        .get(prop_id)
-        .and_then(|value| extract_string(value))
-}
-
-fn extract_string(value: &PropertyValue) -> Option<String> {
-    match value {
-        PropertyValue::String8(value) => Some(value.to_string()),
-        PropertyValue::Unicode(value) => Some(value.to_string()),
-        _ => None,
-    }
-}
-
-fn extract_text(properties: &MessageProperties) -> Option<String> {
-    properties.get(0x1000).and_then(extract_string).or_else(|| {
-        properties.get(0x1009).and_then(|value| match value {
-            PropertyValue::Binary(value) => encoding::decode_rtf_compressed(value.buffer()),
-            _ => None,
-        })
-    })
-}
-
-fn extract_html(properties: &MessageProperties) -> Option<String> {
-    properties.get(0x1013).and_then(|value| match value {
-        PropertyValue::Binary(value) => {
-            let code_page = properties
-                .get(0x3FDE)
-                .and_then(|v| {
-                    if let PropertyValue::Integer32(cpid) = v {
-                        Some(*cpid as u16)
-                    } else {
-                        None
-                    }
-                })
-                .unwrap_or(65001);
-            encoding::decode_html_body(value.buffer(), code_page)
-        }
-        PropertyValue::String8(value) => Some(value.to_string()),
-        PropertyValue::Unicode(value) => Some(value.to_string()),
-        _ => None,
-    })
-}
-
-fn extract_i64_property(properties: &MessageProperties, prop_ids: &[u16]) -> Option<i64> {
-    for &prop_id in prop_ids {
-        if let Some(PropertyValue::Time(value)) = properties.get(prop_id) {
-            return Some(*value);
-        }
-    }
-    None
-}

crates/core/Cargo.toml

@@ -19,7 +19,7 @@ poem-openapi = { version = "5.1.16", features = [
 ], optional = true }
 chrono.workspace = true
 clap.workspace = true
-memdb.workspace = true
+bichon-memdb.workspace = true
 itertools.workspace = true
 ring.workspace = true
 serde.workspace = true
@@ -63,12 +63,17 @@ bytes.workspace = true
 mail-send.workspace = true
 blake3.workspace = true
 uuid.workspace = true
-fjall.workspace = true
+bichon-blob.workspace = true
 tracing-log.workspace = true
 tokio-util.workspace = true
 whichlang = "0.1.1"
 deunicode = "1.6.2"
 scopeguard = "1.2.0"
-cron = "0.15"
-quick-xml = { version = "0.40.0", features = ["serialize"] }
-hickory-resolver = "0.26.0-alpha.1"
+cron = "0.17"
+quick-xml = { version = "0.41.0", features = ["serialize"] }
+hickory-resolver = "0.26.1"
+memmap2 = "0.9.10"
+outlook-pst = { git = "https://github.com/rustmailer/outlook-pst-rs.git", branch = "main" }
+compressed-rtf = "1.0.1"
+codepage-strings = "1.0.2"
+hex.workspace = true

crates/core/src/account/migration.rs

@@ -27,7 +27,7 @@ use crate::{
         since::{DateSince, RelativeDate},
         state::DownloadState,
     },
-    cache::imap::{mailbox::MailBox, task::SYNC_TASKS},
+    archive::imap::{mailbox::MailBox, task::SYNC_TASKS},
     common::paginated::DataPage,
     context::controller::DOWNLOAD_CONTROLLER,
     database::{
@@ -64,6 +64,216 @@ pub enum QuotaWindow {
     Monthly,
 }
 
+/// Include/exclude filter rule.
+///
+/// - `include` non-empty: only values matching these patterns pass.
+/// - `exclude` non-empty: values matching these patterns are rejected.
+/// - Both empty: all values pass.
+/// - Both set: include checked first, then exclude.
+///
+/// Extension patterns use case-insensitive exact match; all others use regex.
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct FilterRule {
+    #[serde(default)]
+    pub include: Vec<String>,
+    #[serde(default)]
+    pub exclude: Vec<String>,
+}
+
+impl FilterRule {
+    pub fn is_empty(&self) -> bool {
+        self.include.is_empty() && self.exclude.is_empty()
+    }
+
+    fn matches_exact(&self, value: &str) -> bool {
+        if !self.include.is_empty() && !self.include.iter().any(|e| e.eq_ignore_ascii_case(value)) {
+            return false;
+        }
+        if !self.exclude.is_empty() && self.exclude.iter().any(|e| e.eq_ignore_ascii_case(value)) {
+            return false;
+        }
+        true
+    }
+
+    fn matches_regex(&self, value: &str) -> bool {
+        if !self.include.is_empty() && !matches_any_regex(&self.include, value) {
+            return false;
+        }
+        if !self.exclude.is_empty() && matches_any_regex(&self.exclude, value) {
+            return false;
+        }
+        true
+    }
+
+    fn validate_regex(&self, field: &str) -> Result<(), String> {
+        validate_patterns(&self.include, &format!("{field}.include"))?;
+        validate_patterns(&self.exclude, &format!("{field}.exclude"))?;
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct ExtractionRules {
+    /// Type 0: Master switch.
+    #[serde(default)]
+    pub enabled: bool,
+    /// Type 1: File extensions (exact match, e.g. `{"include": ["pdf","docx"]}`).
+    #[serde(default)]
+    pub extensions: FilterRule,
+    /// Type 2: Folder patterns (regex, e.g. `{"include": ["^INBOX/Invoices"]}`).
+    #[serde(default)]
+    pub folders: FilterRule,
+    /// Type 3: Attachment filename patterns (regex).
+    #[serde(default)]
+    pub attachment_names: FilterRule,
+    /// Type 4: Sender patterns (regex).
+    #[serde(default)]
+    pub senders: FilterRule,
+}
+
+impl ExtractionRules {
+    /// Returns `true` if the attachment should be extracted under these rules.
+    pub fn should_extract(
+        &self,
+        ext: &str,
+        folder: Option<&str>,
+        attachment_name: Option<&str>,
+        sender: Option<&str>,
+    ) -> bool {
+        if !self.enabled {
+            return false;
+        }
+        if !self.extensions.matches_exact(ext) {
+            return false;
+        }
+        if !self.folders.is_empty() {
+            if let Some(folder) = folder {
+                if !self.folders.matches_regex(folder) {
+                    return false;
+                }
+            }
+        }
+        if !self.attachment_names.is_empty() {
+            if let Some(name) = attachment_name {
+                if !self.attachment_names.matches_regex(name) {
+                    return false;
+                }
+            }
+        }
+        if !self.senders.is_empty() {
+            if let Some(sender) = sender {
+                if !self.senders.matches_regex(sender) {
+                    return false;
+                }
+            }
+        }
+        true
+    }
+
+    pub fn validate(&self) -> Result<(), String> {
+        self.folders.validate_regex("folders")?;
+        self.attachment_names.validate_regex("attachment_names")?;
+        self.senders.validate_regex("senders")?;
+        Ok(())
+    }
+}
+
+/// Archive filtering rules — skip unwanted emails before storage.
+///
+/// Rule types:
+///   0 — Master switch
+///   1 — Sender filter (regex)
+///   2 — Subject filter (regex)
+///   3 — Skip emails larger than this (bytes)
+///   4 — Skip emails with spam headers (X-Spam-Flag, X-Spam)
+///
+/// `None` = archive everything (backward compatible).
+#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct ArchiveRules {
+    /// Type 0: Master switch. `false` = archive everything.
+    #[serde(default)]
+    pub enabled: bool,
+    /// Type 1: Sender filter (regex, include/exclude).
+    #[serde(default)]
+    pub senders: FilterRule,
+    /// Type 2: Subject filter (regex, include/exclude).
+    #[serde(default)]
+    pub subjects: FilterRule,
+    /// Type 3: Skip emails larger than this (bytes). `None` = no size limit.
+    #[serde(default)]
+    pub skip_larger_than: Option<u64>,
+    /// Type 4: Spam header names to check (e.g. `["X-Spam-Flag", "X-Spam"]`).
+    /// When the value is `yes` or `true` (case-insensitive), the email is skipped.
+    /// Empty = don't check. Common headers: `X-Spam-Flag` (SpamAssassin),
+    /// `X-Spam` (rspamd), `X-MS-Exchange-Organization-SCL` (Exchange).
+    #[serde(default)]
+    pub spam_headers: Vec<String>,
+}
+
+impl ArchiveRules {
+    /// Returns `true` if the email should be archived under these rules.
+    pub fn should_archive(
+        &self,
+        sender: Option<&str>,
+        subject: Option<&str>,
+        size: u32,
+        is_spam: bool,
+    ) -> bool {
+        if !self.enabled {
+            return true;
+        }
+        if !self.senders.is_empty() {
+            if let Some(sender) = sender {
+                if !self.senders.matches_regex(sender) {
+                    return false;
+                }
+            }
+        }
+        if !self.subjects.is_empty() {
+            if let Some(subject) = subject {
+                if !self.subjects.matches_regex(subject) {
+                    return false;
+                }
+            }
+        }
+        if let Some(limit) = self.skip_larger_than {
+            if size as u64 > limit {
+                return false;
+            }
+        }
+        if !self.spam_headers.is_empty() && is_spam {
+            return false;
+        }
+        true
+    }
+
+    /// Validate all regex patterns are well-formed.
+    pub fn validate(&self) -> Result<(), String> {
+        self.senders.validate_regex("senders")?;
+        self.subjects.validate_regex("subjects")?;
+        Ok(())
+    }
+}
+
+fn matches_any_regex(patterns: &[String], value: &str) -> bool {
+    patterns.iter().any(|p| {
+        regex::Regex::new(p)
+            .map(|re| re.is_match(value))
+            .unwrap_or(false)
+    })
+}
+
+fn validate_patterns(patterns: &[String], field_name: &str) -> Result<(), String> {
+    for p in patterns {
+        regex::Regex::new(p)
+            .map_err(|e| format!("{} pattern '{}' is invalid regex: {}", field_name, p, e))?;
+    }
+    Ok(())
+}
+
 #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
 #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct Account {
@@ -84,17 +294,28 @@ pub struct Account {
     pub account_type: AccountType,
     pub download_interval_min: Option<i64>,
     pub download_batch_size: Option<u32>,
+    #[serde(default)]
+    pub max_email_size_bytes: Option<u64>,
     pub known_folders: Option<BTreeSet<String>>,
     pub created_at: i64,
     pub updated_at: i64,
     pub created_by: u64, //user id
-    pub use_proxy: Option<u64>,
     pub use_dangerous: bool,
     pub pgp_key: Option<String>,
     pub imap_quota_bytes: Option<u64>,
     pub imap_quota_window: Option<QuotaWindow>,
     pub auto_download_new_mailboxes: Option<bool>,
     pub download_schedule: Option<String>,
+    #[serde(default)]
+    pub deleting: bool,
+    /// Email-level filtering rules (Pro feature).
+    /// `None` = archive everything (backward compatible).
+    #[serde(default)]
+    pub archive_rules: Option<ArchiveRules>,
+    /// Attachment text extraction rules (Pro feature).
+    /// `None` = extract everything (backward compatible).
+    #[serde(default)]
+    pub extraction_rules: Option<ExtractionRules>,
 }
 
 impl MemDbModel for Account {
@@ -123,16 +344,19 @@ impl Account {
             download_interval_min: request.download_interval_min,
             created_at: utc_now!(),
             updated_at: utc_now!(),
-            use_proxy: request.use_proxy,
             use_dangerous: request.use_dangerous,
             pgp_key: request.pgp_key,
             created_by: user_id,
             download_batch_size: request.download_batch_size,
+            max_email_size_bytes: request.max_email_size_bytes,
             date_before: request.date_before,
             auto_download_new_mailboxes: request.auto_download_new_mailboxes,
             imap_quota_bytes: request.imap_quota_bytes,
             imap_quota_window: request.imap_quota_window,
             download_schedule: request.download_schedule,
+            deleting: false,
+            archive_rules: request.archive_rules,
+            extraction_rules: request.extraction_rules,
         })
     }
 
@@ -220,14 +444,46 @@ impl Account {
 
     pub async fn delete(account_id: u64) -> BichonResult<()> {
         let account = Self::get(account_id)?;
-        if let Err(error) = Self::cleanup_account_resources_sequential(&account).await {
-            tracing::error!(
-                "[CLEANUP_ACCOUNT_ERROR] Account {}: failed to cleanup resources: {:#?}",
-                account_id,
-                error
-            );
-            return Err(error);
+
+        // Immediately stop scheduling to prevent new downloads
+        if matches!(account.account_type, AccountType::IMAP) {
+            SYNC_TASKS.stop(account.id).await?;
         }
+
+        // Mark as deleting and disabled so frontend shows status and download tasks skip it
+        update_impl(
+            DB_MANAGER.db(),
+            &account_id.to_string(),
+            move |current: Account| {
+                let mut updated = current.clone();
+                updated.deleting = true;
+                updated.enabled = false;
+                Ok(updated)
+            },
+        )?;
+
+        // Spawn background cleanup — heavy work (Tantivy, attachments) runs off the request path
+        tokio::spawn(async move {
+            if let Err(error) = Self::cleanup_account_resources_sequential(&account).await {
+                tracing::error!(
+                    "[CLEANUP_ACCOUNT_ERROR] Account {}: cleanup failed, reverting deleting flag: {:#?}",
+                    account_id,
+                    error
+                );
+                // Revert deleting flag so the user can retry (only if account record still exists)
+                let _ = update_impl(
+                    DB_MANAGER.db(),
+                    &account_id.to_string(),
+                    move |current: Account| {
+                        let mut updated = current.clone();
+                        updated.deleting = false;
+                        updated.enabled = true;
+                        Ok(updated)
+                    },
+                );
+            }
+        });
+
         Ok(())
     }
 
@@ -236,8 +492,8 @@ impl Account {
     }
 
     async fn cleanup_account_resources_sequential(account: &AccountModel) -> BichonResult<()> {
+        // Sync task already stopped in delete() before spawning this background task
         if matches!(account.account_type, AccountType::IMAP) {
-            SYNC_TASKS.stop(account.id).await?;
             DownloadState::delete(account.id)?;
         }
         OAuth2AccessToken::try_delete(account.id)?;
@@ -324,6 +580,7 @@ impl Account {
             .map(|account: AccountModel| MinimalAccount {
                 id: account.id,
                 email: account.email,
+                name: account.account_name,
             })
             .collect::<Vec<MinimalAccount>>();
         Ok(result)
@@ -395,8 +652,8 @@ impl Account {
                 new.download_batch_size = Some(*download_batch_size);
             }
 
-            if let Some(use_proxy) = request.use_proxy {
-                new.use_proxy = Some(use_proxy);
+            if let Some(max_email_size_bytes) = request.max_email_size_bytes {
+                new.max_email_size_bytes = Some(max_email_size_bytes);
             }
         }
 
@@ -435,7 +692,299 @@ impl Account {
         if request.clear_download_schedule == Some(true) {
             new.download_schedule = None;
         }
+        if request.extraction_rules.is_some() {
+            new.extraction_rules = request.extraction_rules;
+        }
+        if request.archive_rules.is_some() {
+            new.archive_rules = request.archive_rules;
+        }
         new.updated_at = utc_now!();
         Ok(new)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // ── FilterRule ───────────────────────────────────────────────────
+
+    #[test]
+    fn filter_rule_include_only() {
+        let r = FilterRule {
+            include: vec![r"@ok\.com$".into()],
+            ..Default::default()
+        };
+        assert!(r.matches_regex("bob@ok.com"));
+        assert!(!r.matches_regex("spam@bad.com"));
+    }
+
+    #[test]
+    fn filter_rule_exclude_only() {
+        let r = FilterRule {
+            exclude: vec![r"@spam\.com$".into()],
+            ..Default::default()
+        };
+        assert!(r.matches_regex("bob@ok.com"));
+        assert!(!r.matches_regex("bot@spam.com"));
+    }
+
+    #[test]
+    fn filter_rule_include_then_exclude() {
+        let r = FilterRule {
+            include: vec![r"@company\.com$".into()],
+            exclude: vec![r"noreply@company\.com$".into()],
+            ..Default::default()
+        };
+        assert!(r.matches_regex("bob@company.com"));
+        assert!(!r.matches_regex("noreply@company.com"));
+        assert!(!r.matches_regex("spam@other.com"));
+    }
+
+    #[test]
+    fn filter_rule_exact_match() {
+        let r = FilterRule {
+            include: vec!["pdf".into(), "docx".into()],
+            exclude: vec!["xlsx".into()],
+            ..Default::default()
+        };
+        assert!(r.matches_exact("pdf"));
+        assert!(r.matches_exact("docx"));
+        assert!(r.matches_exact("DOCX")); // case-insensitive
+        assert!(!r.matches_exact("xlsx"));
+        assert!(!r.matches_exact("txt"));
+    }
+
+    // ── ExtractionRules ─────────────────────────────────────────────
+
+    #[test]
+    fn extraction_rules_master_switch() {
+        let rules = ExtractionRules {
+            enabled: false,
+            ..Default::default()
+        };
+        assert!(!rules.should_extract("pdf", None, None, None));
+    }
+
+    #[test]
+    fn extraction_rules_extension_include() {
+        let rules = ExtractionRules {
+            enabled: true,
+            extensions: FilterRule {
+                include: vec!["pdf".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.should_extract("pdf", None, None, None));
+        assert!(!rules.should_extract("docx", None, None, None));
+    }
+
+    #[test]
+    fn extraction_rules_extension_exclude() {
+        let rules = ExtractionRules {
+            enabled: true,
+            extensions: FilterRule {
+                exclude: vec!["xlsx".into(), "pptx".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.should_extract("pdf", None, None, None));
+        assert!(!rules.should_extract("xlsx", None, None, None));
+    }
+
+    #[test]
+    fn extraction_rules_folder_regex() {
+        let rules = ExtractionRules {
+            enabled: true,
+            folders: FilterRule {
+                include: vec![r"^INBOX/Invoices".into(), r"Contracts$".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.should_extract("pdf", Some("INBOX/Invoices"), None, None));
+        assert!(rules.should_extract("pdf", Some("Finance/Contracts"), None, None));
+        assert!(!rules.should_extract("pdf", Some("INBOX/Junk"), None, None));
+    }
+
+    #[test]
+    fn extraction_rules_attachment_name_regex() {
+        let rules = ExtractionRules {
+            enabled: true,
+            attachment_names: FilterRule {
+                include: vec![r"^invoice-.*\.pdf$".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.should_extract("pdf", None, Some("invoice-2024.pdf"), None));
+        assert!(!rules.should_extract("pdf", None, Some("newsletter.pdf"), None));
+    }
+
+    #[test]
+    fn extraction_rules_sender_regex() {
+        let rules = ExtractionRules {
+            enabled: true,
+            senders: FilterRule {
+                exclude: vec![r"@noreply\.com$".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        // non-excluded sender passes
+        assert!(rules.should_extract("pdf", None, None, Some("bob@ok.com")));
+        // excluded sender blocked
+        assert!(!rules.should_extract("pdf", None, None, Some("bot@noreply.com")));
+    }
+
+    #[test]
+    fn extraction_rules_empty_filters_pass_everything() {
+        let rules = ExtractionRules {
+            enabled: true,
+            ..Default::default()
+        };
+        assert!(rules.should_extract(
+            "anything",
+            Some("any/folder"),
+            Some("any.pdf"),
+            Some("any@x.com")
+        ));
+    }
+
+    // ── ArchiveRules ────────────────────────────────────────────────
+
+    #[test]
+    fn archive_rules_disabled_archives_everything() {
+        let rules = ArchiveRules {
+            enabled: false,
+            ..Default::default()
+        };
+        assert!(rules.should_archive(Some("spam@x.com"), Some("BUY NOW"), 999, false));
+    }
+
+    #[test]
+    fn archive_rules_sender_exclude() {
+        let rules = ArchiveRules {
+            enabled: true,
+            senders: FilterRule {
+                exclude: vec![r"@spam\.com$".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(!rules.should_archive(Some("bot@spam.com"), None, 100, false));
+        assert!(rules.should_archive(Some("friend@ok.com"), None, 100, false));
+    }
+
+    #[test]
+    fn archive_rules_subject_exclude() {
+        let rules = ArchiveRules {
+            enabled: true,
+            subjects: FilterRule {
+                exclude: vec![r"(?i)unsubscribe|buy now|limited offer".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(!rules.should_archive(None, Some("UNSUBSCRIBE NOW"), 100, false));
+        assert!(!rules.should_archive(None, Some("Limited Offer!!"), 100, false));
+        assert!(rules.should_archive(None, Some("Meeting tomorrow"), 100, false));
+    }
+
+    #[test]
+    fn archive_rules_sender_include() {
+        // Only archive emails from specific senders
+        let rules = ArchiveRules {
+            enabled: true,
+            senders: FilterRule {
+                include: vec![r"@partner\.com$".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.should_archive(Some("bob@partner.com"), None, 100, false));
+        assert!(!rules.should_archive(Some("spam@random.com"), None, 100, false));
+    }
+
+    #[test]
+    fn archive_rules_skip_larger_than() {
+        let rules = ArchiveRules {
+            enabled: true,
+            skip_larger_than: Some(50_000_000),
+            ..Default::default()
+        };
+        assert!(rules.should_archive(None, None, 1_000_000, false));
+        assert!(!rules.should_archive(None, None, 60_000_000, false));
+    }
+
+    #[test]
+    fn archive_rules_skip_spam_headers() {
+        let rules = ArchiveRules {
+            enabled: true,
+            spam_headers: vec!["X-Spam-Flag".into()],
+            ..Default::default()
+        };
+        assert!(!rules.should_archive(None, None, 100, true));
+        assert!(rules.should_archive(None, None, 100, false));
+    }
+
+    // ── Validation ──────────────────────────────────────────────────
+
+    #[test]
+    fn validate_extraction_rules_valid() {
+        let rules = ExtractionRules {
+            folders: FilterRule {
+                include: vec![r"^INBOX/.*".into()],
+                ..Default::default()
+            },
+            senders: FilterRule {
+                exclude: vec![r"@spam\.com$".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.validate().is_ok());
+    }
+
+    #[test]
+    fn validate_extraction_rules_invalid_regex() {
+        let rules = ExtractionRules {
+            folders: FilterRule {
+                include: vec!["***bad[".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.validate().is_err());
+    }
+
+    #[test]
+    fn validate_archive_rules_valid() {
+        let rules = ArchiveRules {
+            senders: FilterRule {
+                exclude: vec![r"@spam\.com$".into()],
+                ..Default::default()
+            },
+            subjects: FilterRule {
+                include: vec![r"(?i)invoice".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.validate().is_ok());
+    }
+
+    #[test]
+    fn validate_archive_rules_invalid_regex() {
+        let rules = ArchiveRules {
+            senders: FilterRule {
+                include: vec!["[unclosed".into()],
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        assert!(rules.validate().is_err());
+    }
+}

crates/core/src/account/payload.rs

@@ -19,7 +19,9 @@
 use std::str::FromStr;
 
 use crate::account::entity::ImapConfig;
-use crate::account::migration::{AccountModel, AccountType, QuotaWindow};
+use crate::account::migration::{
+    AccountModel, AccountType, ArchiveRules, ExtractionRules, QuotaWindow,
+};
 use crate::account::since::{DateSince, RelativeDate};
 use crate::error::code::ErrorCode;
 use crate::error::BichonResult;
@@ -41,20 +43,26 @@ pub struct AccountCreateRequest {
     pub date_since: Option<DateSince>,
     pub date_before: Option<RelativeDate>,
     pub account_type: AccountType,
-    #[cfg_attr(feature = "web-api", oai(validator(minimum(value = "10"))))]
+    #[cfg_attr(feature = "web-api", oai(validator(minimum(value = "1"))))]
     pub download_interval_min: Option<i64>,
     #[cfg_attr(
         feature = "web-api",
         oai(validator(minimum(value = "10"), maximum(value = "200")))
     )]
     pub download_batch_size: Option<u32>,
-    pub use_proxy: Option<u64>,
+    pub max_email_size_bytes: Option<u64>,
     pub use_dangerous: bool,
     pub pgp_key: Option<String>,
     pub imap_quota_bytes: Option<u64>,
     pub imap_quota_window: Option<QuotaWindow>,
     pub auto_download_new_mailboxes: Option<bool>,
     pub download_schedule: Option<String>,
+    /// Email archive filtering rules (Pro feature).
+    /// `None` = archive everything (backward compatible).
+    pub archive_rules: Option<ArchiveRules>,
+    /// Attachment text extraction rules (Pro feature).
+    /// `None` = extract everything (backward compatible).
+    pub extraction_rules: Option<ExtractionRules>,
 }
 
 impl AccountCreateRequest {
@@ -105,6 +113,19 @@ impl AccountCreateRequest {
             }
             AccountType::NoSync => {}
         }
+        if let Some(ref rules) = self.extraction_rules {
+            rules.validate().map_err(|e| {
+                raise_error!(
+                    format!("extraction_rules: {}", e),
+                    ErrorCode::InvalidParameter
+                )
+            })?;
+        }
+        if let Some(ref rules) = self.archive_rules {
+            rules.validate().map_err(|e| {
+                raise_error!(format!("archive_rules: {}", e), ErrorCode::InvalidParameter)
+            })?;
+        }
         Ok(AccountModel::new(user_id, self)?)
     }
 
@@ -158,18 +179,14 @@ pub struct AccountUpdateRequest {
     /// Modified folders will be automatically synced on the next update.
     pub sync_folders: Option<Vec<String>>,
     /// Incremental download interval (seconds)
-    #[cfg_attr(feature = "web-api", oai(validator(minimum(value = "10"))))]
+    #[cfg_attr(feature = "web-api", oai(validator(minimum(value = "1"))))]
     pub download_interval_min: Option<i64>,
     #[cfg_attr(
         feature = "web-api",
         oai(validator(minimum(value = "10"), maximum(value = "200")))
     )]
     pub download_batch_size: Option<u32>,
-    /// Optional proxy ID for establishing the connection to external APIs (e.g., Gmail, Outlook).
-    /// - If `None` or not provided, the client will connect directly to the API server.
-    /// - If `Some(proxy_id)`, the client will use the pre-configured proxy with the given ID for API requests.
-    pub use_proxy: Option<u64>,
-
+    pub max_email_size_bytes: Option<u64>,
     pub use_dangerous: Option<bool>,
 
     pub pgp_key: Option<String>,
@@ -178,6 +195,12 @@ pub struct AccountUpdateRequest {
     pub auto_download_new_mailboxes: Option<bool>,
     pub download_schedule: Option<String>,
     pub clear_download_schedule: Option<bool>,
+    /// Email archive filtering rules (Pro feature).
+    /// `None` = no change. Use `Some(ArchiveRules { .. })` to set.
+    pub archive_rules: Option<ArchiveRules>,
+    /// Attachment text extraction rules (Pro feature).
+    /// `None` = no change. Use `Some(ExtractionRules { .. })` to set.
+    pub extraction_rules: Option<ExtractionRules>,
 }
 
 impl AccountUpdateRequest {
@@ -233,6 +256,19 @@ impl AccountUpdateRequest {
                 validate_cron_expression(schedule)?;
             }
         }
+        if let Some(ref rules) = self.extraction_rules {
+            rules.validate().map_err(|e| {
+                raise_error!(
+                    format!("extraction_rules: {}", e),
+                    ErrorCode::InvalidParameter
+                )
+            })?;
+        }
+        if let Some(ref rules) = self.archive_rules {
+            rules.validate().map_err(|e| {
+                raise_error!(format!("archive_rules: {}", e), ErrorCode::InvalidParameter)
+            })?;
+        }
         Ok(())
     }
 }
@@ -259,6 +295,7 @@ fn validate_cron_expression(expr: &str) -> BichonResult<()> {
 pub struct MinimalAccount {
     pub id: u64,
     pub email: String,
+    pub name: Option<String>,
 }
 
 pub fn filter_accessible_accounts<'a>(

crates/core/src/account/state.rs

@@ -40,6 +40,10 @@ pub enum TriggerType {
     Manual,
     #[default]
     Scheduled,
+    /// Full re-sync (UID SEARCH ALL) invoked explicitly, e.g. to repair a
+    /// mailbox whose incremental download was interrupted. Semantically a
+    /// manual trigger, tracked distinctly for diagnostics.
+    SyncFull,
 }
 
 #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
@@ -63,6 +67,64 @@ pub struct FolderProgress {
     pub message: Option<String>,
 }
 
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))]
+pub enum GapFillStatus {
+    #[default]
+    Running,
+    Success,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct GapFillFolderStats {
+    pub downloaded: u64,
+    pub failed: u64,
+    pub candidate_count: u64,
+    /// Live progress hint (e.g. "IMAP server is slow...") shown while the
+    /// folder is being scanned; usually `None` once the folder is done.
+    pub message: Option<String>,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct GapFillRun {
+    pub started_at: i64,
+    pub finished_at: Option<i64>,
+    pub status: GapFillStatus,
+    /// Per-mailbox gap-fill outcome, keyed by mailbox name.
+    pub folders: BTreeMap<String, GapFillFolderStats>,
+    /// Total emails newly downloaded by gap-fill.
+    pub downloaded: u64,
+    /// Total emails that failed to download during gap-fill.
+    pub failed: u64,
+}
+
+/// Independent, repeatable gap-fill history for an account. Gap-fill is a
+/// distinct operation from downloading (it can be run again and again until
+/// `failed == 0`), so its runs are tracked separately from `DownloadState`
+/// instead of being mixed into download sessions.
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct GapFillState {
+    pub account_id: u64,
+    /// The gap-fill run currently in progress, if any.
+    pub active: Option<GapFillRun>,
+    /// Finished runs, most recent last.
+    pub history: Vec<GapFillRun>,
+}
+
+impl MemDbModel for GapFillState {
+    fn collection() -> &'static str {
+        "gap_fill_states"
+    }
+    fn key(&self) -> String {
+        self.account_id.to_string()
+    }
+}
+
 #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
 #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct DownloadSession {
@@ -189,6 +251,47 @@ impl DownloadState {
         })
     }
 
+    /// Moves a stale Running session into history as Cancelled.
+    ///
+    /// A Running `active_session` that survives an Idle decision means the
+    /// previous run was interrupted without a clean shutdown (e.g. process
+    /// killed mid-download). Leaving it in place makes the UI show a phantom
+    /// "syncing" state even though nothing is downloading. Callers invoke this
+    /// only when no download is actually running for the account, so a
+    /// legitimately active session is never touched.
+    ///
+    /// Returns `true` if a stale session was finalized (i.e. the previous sync
+    /// did not finish) and `false` otherwise.
+    pub fn finalize_stale_session(account_id: u64) -> BichonResult<bool> {
+        let stale = Self::get(account_id)?
+            .and_then(|s| s.active_session)
+            .map_or(false, |s| s.status == DownloadStatus::Running);
+        if stale {
+            Self::update_state(account_id, move |current| {
+                let mut updated = current.clone();
+                if updated
+                    .active_session
+                    .as_ref()
+                    .map_or(false, |s| s.status == DownloadStatus::Running)
+                {
+                    if let Some(mut session) = updated.active_session.take() {
+                        session.status = DownloadStatus::Cancelled;
+                        session.end_time = Some(utc_now!());
+                        session.message = Some(
+                            "Previous sync did not finish cleanly; marked as cancelled on startup."
+                                .into(),
+                        );
+                        updated.history.push(session);
+                        updated.last_finished_at = Some(utc_now!());
+                        updated.active_session = None;
+                    }
+                }
+                Ok(updated)
+            })?;
+        }
+        Ok(stale)
+    }
+
     pub fn update_folder_progress(
         account_id: u64,
         folder_name: String,
@@ -220,6 +323,19 @@ impl DownloadState {
         })
     }
 
+    /// Touches only `current_folder` without rewriting folder progress. Lets
+    /// long-running IMAP operations (e.g. waiting on a slow server mid-batch)
+    /// keep the UI's "last activity" indicator fresh without spamming writes.
+    pub fn set_current_folder(account_id: u64, folder_name: String) -> BichonResult<()> {
+        Self::update_state(account_id, move |state| {
+            let mut updated = state.clone();
+            if let Some(ref mut session) = updated.active_session {
+                session.current_folder = Some(folder_name);
+            }
+            Ok(updated)
+        })
+    }
+
     pub fn init_folder_details(account_id: u64, folders: Vec<String>) -> BichonResult<()> {
         Self::update_state(account_id, move |state| {
             let mut updated = state.clone();
@@ -263,6 +379,18 @@ impl DownloadState {
         })
     }
 
+    /// Appends/updates a free-form message on the active session without
+    /// changing its status. Used to record the gap-fill summary.
+    pub fn update_session_message(account_id: u64, message: String) -> BichonResult<()> {
+        Self::update_state(account_id, move |current| {
+            let mut updated = current.clone();
+            if let Some(ref mut session) = updated.active_session {
+                session.message = Some(message);
+            }
+            Ok(updated)
+        })
+    }
+
     fn update_state(
         account_id: u64,
         updater: impl FnOnce(DownloadState) -> BichonResult<DownloadState> + Send + 'static,
@@ -281,3 +409,161 @@ impl DownloadState {
         delete_impl::<DownloadState>(DB_MANAGER.db(), &account_id.to_string())
     }
 }
+
+impl GapFillState {
+    pub fn get(account_id: u64) -> BichonResult<Option<GapFillState>> {
+        find_impl::<GapFillState>(DB_MANAGER.db(), &account_id.to_string())
+    }
+
+    /// Starts a new gap-fill run, moving any stale active run into history as
+    /// Cancelled. Creates the state record on first use.
+    pub fn start_run(account_id: u64) -> BichonResult<()> {
+        let now = utc_now!();
+        let run = GapFillRun {
+            started_at: now,
+            status: GapFillStatus::Running,
+            ..Default::default()
+        };
+        if Self::get(account_id)?.is_none() {
+            let state = GapFillState {
+                account_id,
+                active: Some(run),
+                history: Vec::new(),
+            };
+            return upsert_impl(DB_MANAGER.db(), state);
+        }
+        Self::update_state(account_id, move |current| {
+            let mut updated = current.clone();
+            if let Some(mut old) = updated.active.take() {
+                if old.status == GapFillStatus::Running {
+                    old.status = GapFillStatus::Cancelled;
+                    old.finished_at = Some(utc_now!());
+                }
+                updated.history.push(old);
+                let keep = updated.history.len().saturating_sub(10);
+                if keep > 0 {
+                    updated.history.drain(0..keep);
+                }
+            }
+            updated.active = Some(run);
+            Ok(updated)
+        })
+    }
+
+    /// Accumulates a per-folder outcome into the active run.
+    pub fn add_folder_result(
+        account_id: u64,
+        folder_name: String,
+        stats: GapFillFolderStats,
+    ) -> BichonResult<()> {
+        Self::update_state(account_id, move |current| {
+            let mut updated = current.clone();
+            if let Some(ref mut run) = updated.active {
+                run.downloaded += stats.downloaded;
+                run.failed += stats.failed;
+                run.folders.insert(folder_name, stats);
+            }
+            Ok(updated)
+        })
+    }
+    /// Updates the live per-folder progress of the active run (used during a
+    /// gap-fill scan so the UI can show per-folder progress without waiting
+    /// for the folder to finish). `candidate_count` is the planned total,
+    /// `downloaded` the current count, `message` an optional live hint
+    /// (e.g. slow-server notice).
+    pub fn update_folder_progress(
+        account_id: u64,
+        folder_name: String,
+        candidate_count: u64,
+        downloaded: u64,
+        message: Option<String>,
+    ) -> BichonResult<()> {
+        Self::update_state(account_id, move |current| {
+            let mut updated = current.clone();
+            if let Some(ref mut run) = updated.active {
+                let entry = run
+                    .folders
+                    .entry(folder_name.clone())
+                    .or_insert(GapFillFolderStats {
+                        downloaded: 0,
+                        failed: 0,
+                        candidate_count,
+                        message: None,
+                    });
+                entry.candidate_count = candidate_count;
+                entry.downloaded = downloaded;
+                entry.message = message;
+            }
+            Ok(updated)
+        })
+    }
+
+    /// Finalizes the active run: moves it to history with the given status and
+    /// totals. `failed`/`downloaded` are the authoritative accumulated values.
+    pub fn finish_run(
+        account_id: u64,
+        status: GapFillStatus,
+        downloaded: u64,
+        failed: u64,
+    ) -> BichonResult<()> {
+        Self::update_state(account_id, move |current| {
+            let mut updated = current.clone();
+            if let Some(mut run) = updated.active.take() {
+                run.status = status;
+                run.finished_at = Some(utc_now!());
+                run.downloaded = downloaded;
+                run.failed = failed;
+                updated.history.push(run);
+                let keep = updated.history.len().saturating_sub(10);
+                if keep > 0 {
+                    updated.history.drain(0..keep);
+                }
+            }
+            Ok(updated)
+        })
+    }
+
+    /// Moves a stale Running active run into history as Cancelled.
+    ///
+    /// A Running `active` that survives a restart means the previous gap-fill
+    /// run was interrupted without finishing (process killed mid-scan). Leaving
+    /// it in place makes the UI show a phantom "Running" gap-fill. Callers
+    /// invoke this on startup, when no gap-fill is actually running.
+    ///
+    /// Returns `true` if a stale run was finalized.
+    pub fn finalize_stale_run(account_id: u64) -> BichonResult<bool> {
+        let stale = Self::get(account_id)?
+            .and_then(|s| s.active)
+            .map_or(false, |r| r.status == GapFillStatus::Running);
+        if stale {
+            Self::update_state(account_id, move |current| {
+                let mut updated = current.clone();
+                if let Some(mut run) = updated.active.take() {
+                    if run.status == GapFillStatus::Running {
+                        run.status = GapFillStatus::Cancelled;
+                        run.finished_at = Some(utc_now!());
+                        updated.history.push(run);
+                        let keep = updated.history.len().saturating_sub(10);
+                        if keep > 0 {
+                            updated.history.drain(0..keep);
+                        }
+                    } else {
+                        updated.active = Some(run);
+                    }
+                }
+                Ok(updated)
+            })?;
+        }
+        Ok(stale)
+    }
+
+    fn update_state(
+        account_id: u64,
+        updater: impl FnOnce(GapFillState) -> BichonResult<GapFillState> + Send + 'static,
+    ) -> BichonResult<()> {
+        if Self::get(account_id)?.is_some() {
+            update_impl(DB_MANAGER.db(), &account_id.to_string(), updater)?;
+        }
+        Ok(())
+    }
+}

crates/core/src/account/view.rs

@@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize};
 use crate::{
     account::{
         entity::ImapConfig,
-        migration::{AccountModel, AccountType, QuotaWindow},
+        migration::{AccountModel, AccountType, ArchiveRules, QuotaWindow},
         since::{DateSince, RelativeDate},
     },
     users::UserModel,
@@ -44,19 +44,21 @@ pub struct AccountResp {
     pub account_type: AccountType,
     pub download_interval_min: Option<i64>,
     pub download_batch_size: Option<u32>,
+    pub max_email_size_bytes: Option<u64>,
     pub known_folders: Option<BTreeSet<String>>,
     pub created_at: i64,
     pub updated_at: i64,
     pub created_by: u64, //user id
     pub created_user_name: String,
     pub created_user_email: String,
-    pub use_proxy: Option<u64>,
     pub use_dangerous: bool,
     pub pgp_key: Option<String>,
     pub imap_quota_bytes: Option<u64>,
     pub imap_quota_window: Option<QuotaWindow>,
     pub auto_download_new_mailboxes: Option<bool>,
     pub download_schedule: Option<String>,
+    pub archive_rules: Option<ArchiveRules>,
+    pub deleting: bool,
 }
 
 impl AccountResp {
@@ -76,6 +78,7 @@ impl AccountResp {
             account_type: account.account_type,
             download_interval_min: account.download_interval_min,
             download_batch_size: account.download_batch_size,
+            max_email_size_bytes: account.max_email_size_bytes,
             known_folders: account.known_folders,
             created_at: account.created_at,
             updated_at: account.updated_at,
@@ -86,13 +89,14 @@ impl AccountResp {
             created_user_email: user
                 .map(|u| u.email.clone())
                 .unwrap_or_else(|| "N/A".to_string()),
-            use_proxy: account.use_proxy,
             use_dangerous: account.use_dangerous,
             pgp_key: account.pgp_key,
             imap_quota_bytes: account.imap_quota_bytes,
             imap_quota_window: account.imap_quota_window,
             auto_download_new_mailboxes: account.auto_download_new_mailboxes,
             download_schedule: account.download_schedule,
+            archive_rules: account.archive_rules,
+            deleting: account.deleting,
         }
     }
 }

crates/core/src/admin/meta.rs

@@ -18,7 +18,7 @@
 
 use std::path::Path;
 
-use memdb::{Durability, MemDb};
+use bichon_memdb::{Durability, MemDb};
 
 use crate::{
     database::MemDbModel,

crates/core/src/archive/imap/download/download_folders.rs

@@ -22,8 +22,8 @@ use crate::{
     decode_mailbox_name, raise_error,
     {
         account::migration::{AccountModel, AccountType},
-        cache::imap::mailbox::{AttributeEnum, MailBox},
-        cache::imap::mailbox_cache,
+        archive::imap::mailbox::{AttributeEnum, MailBox},
+        archive::imap::mailbox_cache,
         error::{code::ErrorCode, BichonResult},
         imap::{executor::ImapExecutor, session::SessionStream},
         mailbox::list::convert_names_to_mailboxes,

crates/core/src/archive/imap/download/download_type.rs

@@ -53,6 +53,7 @@ pub async fn decide_next_download_task(
 
     let should_start = match trigger_type {
         TriggerType::Manual => true,
+        TriggerType::SyncFull => true,
         TriggerType::Scheduled => {
             let now = utc_now!();
             let cooldown_ok = now - state.last_finished_at.unwrap_or(0) > 60 * 1000;
@@ -73,10 +74,14 @@ pub async fn decide_next_download_task(
         DownloadState::start_new_session(account.id, trigger_type)?;
         Ok(DownloadTask::TraceFetch)
     } else {
+        // Nothing to download right now. A Running active_session at this point
+        // is a leftover from an interrupted run (a real download would have
+        // been blocked by the busy guard before reaching here), so mark it
+        // Cancelled instead of leaving the UI showing a phantom "syncing".
+        DownloadState::finalize_stale_session(account.id)?;
         Ok(DownloadTask::Idle)
     }
 }
-
 fn should_trigger_next_download(last_trigger_at: i64, sync_interval_min: i64) -> bool {
     let now = utc_now!();
     now - last_trigger_at > (sync_interval_min * 60 * 1000)

crates/core/src/archive/imap/download/gap_fill.rs

@@ -0,0 +1,472 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::{HashMap, HashSet};
+
+use tracing::{info, warn};
+
+use crate::error::code::ErrorCode;
+use crate::raise_error;
+use crate::{
+    account::{
+        migration::AccountModel,
+        state::{DownloadState, GapFillFolderStats, GapFillState},
+    },
+    archive::imap::mailbox::MailBox,
+    error::BichonResult,
+    imap::executor::{compress_uid_list, ImapExecutor, DEFAULT_BATCH_SIZE},
+    store::tantivy::envelope::EnvelopeSnapshot,
+};
+
+/// Number of times a batch download is retried with a fresh connection before
+/// it is counted as failed (mirrors the incremental sync path).
+const MAX_NETWORK_RETRIES: u32 = 3;
+
+/// Lightweight header metadata for one remote message, fetched via
+/// `FETCH (UID RFC822.SIZE INTERNALDATE BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])`.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct RemoteHeader {
+    pub uid: u32,
+    pub message_id: Option<String>,
+    pub size: u64,
+    /// Epoch millis (internal date).
+    pub internal_date: i64,
+}
+
+/// Which remote uids are missing locally. A remote message is "present" if its
+/// message-id exists locally. The (size, internal_date) fingerprint is a
+/// fallback for every remote message — not only those without a message-id —
+/// because some paths store a different message-id locally than the remote
+/// header carries (e.g. the SMTP path generates a random one), and servers
+/// like Zoho/163 reuse the same message-id across different messages. A
+/// duplicated local message-id still counts as present: re-downloading would
+/// be deduplicated away anyway, so it can never repair the duplication.
+pub fn compute_missing_uids(remote: &[RemoteHeader], local: &[EnvelopeSnapshot]) -> Vec<u32> {
+    let mut local_by_msg_id: HashMap<&str, usize> = HashMap::new();
+    let mut local_by_fingerprint: HashSet<(u64, i64)> = HashSet::new();
+    for snap in local {
+        if !snap.message_id.is_empty() {
+            *local_by_msg_id.entry(snap.message_id.as_str()).or_insert(0) += 1;
+        }
+        local_by_fingerprint.insert((snap.size, snap.internal_date));
+    }
+
+    let mut missing = Vec::new();
+    for header in remote {
+        let present = match &header.message_id {
+            Some(msg_id) => local_by_msg_id
+                .get(msg_id.as_str())
+                .is_some_and(|&c| c > 0),
+            None => false,
+        };
+        if !present {
+            // Fingerprint fallback for every remote message, not just those
+            // without a message-id: the remote message-id may not exist
+            // locally even though the message is already stored (random
+            // synthetic ids on the SMTP path).
+            let fp_present = local_by_fingerprint.contains(&(header.size, header.internal_date));
+            if !fp_present {
+                missing.push(header.uid);
+            }
+        }
+    }
+    missing
+}
+
+/// Runs the gap-fill phase for one mailbox: enumerates every remote UID,
+/// diffs against local envelopes, downloads the missing ones and returns the
+/// per-folder outcome. Handles per-batch network errors by counting them as
+/// failed (retryable on a later gap-fill run) instead of aborting the phase.
+pub async fn gap_fill_mailbox(
+    account: &AccountModel,
+    local_mailbox: &MailBox,
+    remote_mailbox: &MailBox,
+    token: tokio_util::sync::CancellationToken,
+) -> BichonResult<GapFillFolderStats> {
+    let account_id = account.id;
+    let mut stats = GapFillFolderStats::default();
+
+    let mut session = ImapExecutor::create_connection(account_id).await?;
+    session
+        .examine(&remote_mailbox.encoded_name())
+        .await
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+    // Phase 1: enumerate every remote UID. A huge mailbox can make the server
+    // take a while to answer `UID SEARCH ALL`; keep the UI informed instead of
+    // appearing stuck (the socket read timeout is the final backstop).
+    let search_started = std::time::Instant::now();
+    let results = loop {
+        match tokio::time::timeout(std::time::Duration::from_secs(5), session.uid_search("ALL"))
+            .await
+        {
+            Ok(res) => {
+                break res
+                    .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
+            }
+            Err(_) => {
+                let stall = search_started.elapsed().as_secs_f64();
+                if token.is_cancelled() {
+                    session.logout().await.ok();
+                    return Ok(stats);
+                }
+                let _ = GapFillState::update_folder_progress(
+                    account_id,
+                    remote_mailbox.name.clone(),
+                    0,
+                    0,
+                    crate::imap::executor::slow_server_message(None, Some(stall)),
+                );
+                tracing::warn!(
+                    account_id,
+                    mailbox = %remote_mailbox.name,
+                    stall_secs = format!("{:.0}", stall),
+                    "gap-fill: UID SEARCH ALL taking long, still waiting"
+                );
+            }
+        }
+    };
+    let mut remote_uids: Vec<u32> = results.into_iter().collect();
+    remote_uids.sort();
+    if remote_uids.is_empty() {
+        session.logout().await.ok();
+        return Ok(stats);
+    }
+
+    // Phase 2: fetch header metadata for all remote uids in batches.
+    // A failed batch counts its uids as failed (they cannot be diffed) but
+    // does not abort the phase. A cancellation, however, leaves the header
+    // list incomplete so the diff would be unreliable — return immediately.
+    let mut remote_headers: Vec<RemoteHeader> = Vec::with_capacity(remote_uids.len());
+    let batch_size = account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize;
+    let mut cancelled = false;
+    // Progress reported so far across all header batches, so the UI can show
+    // enumeration progress (and slow-server hints) while a huge mailbox is
+    // being scanned.
+    let headers_fetched = std::sync::Mutex::new(0u64);
+    for chunk in remote_uids.chunks(batch_size) {
+        if token.is_cancelled() {
+            cancelled = true;
+            break;
+        }
+        let seq_set = compress_uid_list(chunk.to_vec());
+        match ImapExecutor::fetch_uid_headers(
+            &mut session,
+            &seq_set,
+            token.clone(),
+            Some(&|count, stall_secs| {
+                *headers_fetched.lock().unwrap() = count;
+                GapFillState::update_folder_progress(
+                    account_id,
+                    remote_mailbox.name.clone(),
+                    remote_uids.len() as u64,
+                    count,
+                    crate::imap::executor::slow_server_message(None, stall_secs),
+                )
+            }),
+        )
+        .await
+        {
+            Ok(headers) => {
+                *headers_fetched.lock().unwrap() += headers.len() as u64;
+                remote_headers.extend(headers);
+            }
+            Err(e) => {
+                // Count the whole chunk as failed; the user can re-run
+                // gap-fill to retry. Do not abort the phase.
+                stats.failed += chunk.len() as u64;
+                let err_msg = format!("Gap-fill header batch failed: {:#?}", e);
+                warn!(account_id, mailbox = remote_mailbox.name, "{}", err_msg);
+                let _ = DownloadState::append_session_error(account_id, err_msg);
+            }
+        }
+    }
+    if cancelled {
+        session.logout().await.ok();
+        GapFillState::update_folder_progress(account_id, remote_mailbox.name.clone(), 0, 0, None)?;
+        return Ok(stats);
+    }
+    remote_headers.sort_by_key(|h| h.uid);
+    session.logout().await.ok();
+
+    // Phase 3: local snapshot
+    let local_snapshots = crate::store::tantivy::envelope::ENVELOPE_MANAGER
+        .get_envelope_snapshots_for_mailbox(account_id, local_mailbox.id)?;
+
+    // Phase 4: diff
+    let missing_uids = compute_missing_uids(&remote_headers, &local_snapshots);
+    stats.candidate_count = missing_uids.len() as u64;
+    if missing_uids.is_empty() {
+        info!(
+            account_id,
+            mailbox = remote_mailbox.name,
+            "Gap-fill: no missing emails"
+        );
+        GapFillState::update_folder_progress(account_id, remote_mailbox.name.clone(), 0, 0, None)?;
+        return Ok(stats);
+    }
+
+    // Phase 5: download missing in batches
+    let planned = missing_uids.len() as u64;
+    GapFillState::update_folder_progress(
+        account_id,
+        remote_mailbox.name.clone(),
+        planned,
+        0,
+        None,
+    )?;
+
+    let mut session2 = ImapExecutor::create_connection(account_id).await?;
+    session2
+        .examine(&remote_mailbox.encoded_name())
+        .await
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+    let batches =
+        crate::imap::executor::generate_uid_sequence_hashset(missing_uids.clone(), batch_size);
+    let mut downloaded = 0u64;
+    let mut failed = 0u64;
+    let mut cancelled = false;
+    for (index, batch) in batches.into_iter().enumerate() {
+        if token.is_cancelled() {
+            cancelled = true;
+            break;
+        }
+        // A slow server can stall a batch past the socket read timeout, same
+        // as in the incremental path. Retry such batches on a fresh connection
+        // instead of counting them as failed outright.
+        let mut retries = 0u32;
+        // Tracks the last cumulative count the progress callback reported. When
+        // a batch fails mid-stream the executor reports the already-stored
+        // count one final time before returning the error, so this is the
+        // number of emails of this batch that actually made it to disk.
+        let last_reported = std::sync::Mutex::new(0u64);
+        let batch_result = loop {
+            match ImapExecutor::uid_batch_retrieve_emails(
+                &mut session2,
+                account_id,
+                remote_mailbox.id,
+                &batch.0,
+                account.max_email_size_bytes,
+                token.clone(),
+                Some(&|cumulative, avg_secs, stall_secs| {
+                    *last_reported.lock().unwrap() = cumulative;
+                    let _ = GapFillState::update_folder_progress(
+                        account_id,
+                        remote_mailbox.name.clone(),
+                        planned,
+                        downloaded + cumulative,
+                        crate::imap::executor::slow_server_message(avg_secs, stall_secs),
+                    );
+                    Ok(())
+                }),
+            )
+            .await
+            {
+                Ok((processed, _throttled)) => break Ok(processed),
+                Err(e) if retries < MAX_NETWORK_RETRIES && e.code() == ErrorCode::NetworkError => {
+                    retries += 1;
+                    warn!(
+                        account_id,
+                        mailbox = remote_mailbox.name,
+                        index,
+                        retries,
+                        "Gap-fill: network error on batch, reconnecting ({}/{})",
+                        retries,
+                        MAX_NETWORK_RETRIES
+                    );
+                    match ImapExecutor::create_connection(account_id).await {
+                        Ok(new_session) => {
+                            session2 = new_session;
+                            if let Err(e2) = session2.examine(&remote_mailbox.encoded_name()).await
+                            {
+                                let err_msg = format!(
+                                    "Gap-fill: re-examine failed after reconnect: {:#?}",
+                                    e2
+                                );
+                                DownloadState::append_session_error(account_id, err_msg)?;
+                                break Err(e);
+                            }
+                            // Longer backoff than the original 1s/2s/4s: a
+                            // throttling server needs time to recover.
+                            let backoff = [5u64, 15, 30][(retries - 1) as usize];
+                            warn!(
+                                account_id,
+                                mailbox = remote_mailbox.name,
+                                "Gap-fill: backing off {}s before retrying batch",
+                                backoff
+                            );
+                            tokio::time::sleep(std::time::Duration::from_secs(backoff)).await;
+                            continue;
+                        }
+                        Err(e2) => {
+                            tracing::error!(account_id, "Gap-fill: reconnection failed: {:#?}", e2);
+                            break Err(e);
+                        }
+                    }
+                }
+                Err(e) => break Err(e),
+            }
+        };
+        match batch_result {
+            Ok(processed) => {
+                downloaded += processed;
+                GapFillState::update_folder_progress(
+                    account_id,
+                    remote_mailbox.name.clone(),
+                    planned,
+                    downloaded,
+                    None,
+                )?;
+            }
+            Err(e) => {
+                // The batch may have partially succeeded: emails already stored
+                // before the failure are counted as downloaded, only the rest
+                // of the batch is failed. The user can re-run gap-fill to
+                // retry the remainder; dedup makes re-downloading the stored
+                // ones harmless. Do not abort the phase.
+                let processed = *last_reported.lock().unwrap();
+                downloaded += processed;
+                let remaining = batch.1.saturating_sub(processed);
+                failed += remaining;
+                let err_msg = format!(
+                    "Gap-fill batch {} failed after {} processed: {:#?}",
+                    index, processed, e
+                );
+                warn!(account_id, mailbox = remote_mailbox.name, "{}", err_msg);
+                let _ = DownloadState::append_session_error(account_id, err_msg);
+            }
+        }
+    }
+    session2.logout().await.ok();
+
+    stats.downloaded = downloaded;
+    // Accumulate rather than overwrite: phase 2 (header batch) failures already
+    // counted into stats.failed and must survive alongside phase 5 failures.
+    stats.failed += failed;
+    // Final progress write into the independent gap-fill state (the folder is
+    // done; the outcome lands in GapFillRun.folders via add_folder_result).
+    GapFillState::update_folder_progress(
+        account_id,
+        remote_mailbox.name.clone(),
+        planned,
+        downloaded,
+        None,
+    )?;
+
+    // Advance the mailbox's highest_uid so subsequent incremental syncs
+    // start after the newly downloaded messages. Only do this on a complete,
+    // uncancelled run where every planned message was downloaded and nothing
+    // failed (phase-2 header batch failures leave uids outside `planned` that
+    // must still be picked up by a later gap-fill run).
+    if !cancelled && downloaded == planned && stats.failed == 0 {
+        if let Some(max_uid) = missing_uids.last().copied() {
+            let mut updated = remote_mailbox.clone();
+            updated.highest_uid = Some(max_uid.max(local_mailbox.highest_uid.unwrap_or(0)));
+            crate::archive::imap::mailbox::MailBox::batch_upsert(&[updated])?;
+        }
+    }
+
+    Ok(stats)
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    fn rh(uid: u32, message_id: Option<&str>, size: u64, internal_date: i64) -> RemoteHeader {
+        RemoteHeader {
+            uid,
+            message_id: message_id.map(|s| s.to_string()),
+            size,
+            internal_date,
+        }
+    }
+
+    fn snap(message_id: &str, uid: u64, size: u64, internal_date: i64) -> EnvelopeSnapshot {
+        EnvelopeSnapshot {
+            message_id: message_id.to_string(),
+            uid,
+            size,
+            internal_date,
+            //subject: String::new(),
+        }
+    }
+
+    #[test]
+    fn compute_missing_uids_message_id_diff() {
+        let remote = vec![
+            rh(1, Some("a"), 10, 1000),
+            rh(2, Some("b"), 20, 2000),
+            rh(3, Some("c"), 30, 3000),
+        ];
+        let local = vec![snap("a", 1, 10, 1000), snap("c", 3, 30, 3000)];
+        let missing = compute_missing_uids(&remote, &local);
+        assert_eq!(missing, vec![2]);
+    }
+
+    #[test]
+    fn compute_missing_uids_fingerprint_fallback() {
+        let remote = vec![rh(1, None, 10, 1000), rh(2, None, 20, 2000)];
+        let local = vec![snap("generated-x", 1, 10, 1000)];
+        let missing = compute_missing_uids(&remote, &local);
+        assert_eq!(missing, vec![2]);
+    }
+
+    #[test]
+    fn compute_missing_uids_remote_duplicates_all_present() {
+        let remote = vec![rh(1, Some("dup"), 10, 1000), rh(2, Some("dup"), 10, 1000)];
+        let local = vec![snap("dup", 1, 10, 1000)];
+        let missing = compute_missing_uids(&remote, &local);
+        assert!(missing.is_empty());
+    }
+
+    #[test]
+    fn compute_missing_uids_local_duplicate_not_re_downloaded() {
+        // A message-id appearing more than once locally is NOT a reason to
+        // re-download: servers (Zoho, 163) legitimately reuse message-ids
+        // across different messages, and re-downloading would be deduplicated
+        // away anyway, so it can never repair the duplication.
+        let remote = vec![rh(1, Some("dup"), 10, 1000), rh(2, Some("dup"), 10, 1000)];
+        let local = vec![snap("dup", 1, 10, 1000), snap("dup", 2, 10, 1000)];
+        let missing = compute_missing_uids(&remote, &local);
+        assert!(missing.is_empty());
+    }
+
+    #[test]
+    fn compute_missing_uids_msgid_mismatch_but_fingerprint_hit() {
+        // The remote message-id does not exist locally (e.g. a different
+        // message-id was stored by the SMTP path) but the fingerprint matches:
+        // the message is already stored and must NOT be re-downloaded.
+        let remote = vec![rh(1, Some("remote-id@x.com"), 10, 1000)];
+        let local = vec![snap("generated-random-id", 1, 10, 1000)];
+        let missing = compute_missing_uids(&remote, &local);
+        assert!(missing.is_empty());
+    }
+
+    #[test]
+    fn compute_missing_uids_msgid_mismatch_and_fingerprint_miss() {
+        let remote = vec![
+            rh(1, Some("remote-id@x.com"), 10, 1000),
+            rh(2, Some("remote-id-2@x.com"), 20, 2000),
+        ];
+        let local = vec![snap("generated-random-id", 1, 10, 1000)];
+        let missing = compute_missing_uids(&remote, &local);
+        assert_eq!(missing, vec![2]);
+    }
+}

crates/core/src/archive/imap/download/mod.rs

@@ -19,9 +19,12 @@
 use crate::{
     account::{
         migration::{AccountModel, AccountType},
-        state::{DownloadState, DownloadStatus, TriggerType},
+        state::{
+            DownloadState, DownloadStatus, GapFillFolderStats, GapFillState, GapFillStatus,
+            TriggerType,
+        },
     },
-    cache::imap::{download::flow::FetchDirection, mailbox::MailBox},
+    archive::imap::{download::flow::FetchDirection, mailbox::MailBox},
     error::BichonResult,
     imap::executor::ImapExecutor,
 };
@@ -31,10 +34,11 @@ use flow::reconcile_mailboxes;
 use rebuild::{rebuild_cache, rebuild_cache_by_date};
 use std::time::Instant;
 use tokio_util::sync::CancellationToken;
-use tracing::{debug, warn};
+use tracing::{debug, info, warn};
 
 pub mod download_folders;
 pub mod download_type;
+pub mod gap_fill;
 pub mod flow;
 pub mod rebuild;
 
@@ -42,6 +46,7 @@ pub async fn process_imap_download(
     account: &AccountModel,
     token: CancellationToken,
     trigger_type: TriggerType,
+    run_gap_fill: bool,
 ) -> BichonResult<()> {
     assert_eq!(account.account_type, AccountType::IMAP);
     let start_time = Instant::now();
@@ -122,7 +127,60 @@ pub async fn process_imap_download(
     }
 
     let local_mailboxes = MailBox::list_all(account_id)?;
-    match reconcile_mailboxes(account, &remote_mailboxes, &local_mailboxes, token).await {
+    let reconcile_result =
+        reconcile_mailboxes(account, &remote_mailboxes, &local_mailboxes, token.clone()).await;
+
+    // Gap-fill phase: only on explicit user request (manual download with
+    // "run gap-fill" checked). Enumerate every UID in the download folders and
+    // download anything missing locally. Not run on scheduled syncs. Gap-fill
+    // runs are tracked in their own state (independent of the download session)
+    // because they are repeatable until `failed == 0`.
+    if run_gap_fill {
+        GapFillState::start_run(account_id)?;
+        // Run inside a helper so a failure anywhere still finalizes the run:
+        // an abandoned active run would otherwise show as Running forever.
+        let run_outcome = gap_fill_phase(
+            account,
+            &local_mailboxes,
+            &remote_mailboxes,
+            token,
+            account_id,
+        )
+        .await;
+        let (cancelled, total_downloaded, total_failed) = match run_outcome {
+            Ok(v) => v,
+            Err(e) => {
+                warn!(account_id = account_id, "Gap-fill phase error: {:#?}", e);
+                (false, 0, 1)
+            }
+        };
+        let status = if cancelled {
+            GapFillStatus::Cancelled
+        } else if total_failed > 0 {
+            GapFillStatus::Failed
+        } else {
+            GapFillStatus::Success
+        };
+        GapFillState::finish_run(account_id, status, total_downloaded, total_failed)?;
+        let summary = if cancelled {
+            format!(
+                "Gap-fill cancelled: {} downloaded, {} failed",
+                total_downloaded, total_failed
+            )
+        } else {
+            format!(
+                "Gap-fill finished: {} downloaded, {} failed",
+                total_downloaded, total_failed
+            )
+        };
+        DownloadState::update_session_message(account_id, summary.clone())?;
+        info!(account_id = account_id, "{}", summary);
+    }
+
+    // Finalize session status AFTER all phases so stats/progress written
+    // during gap-fill are not dropped (update_session_status closes the active
+    // session, moving it into history).
+    match reconcile_result {
         Ok(_) => DownloadState::update_session_status(account_id, DownloadStatus::Success, None)?,
         Err(e) => {
             let err_msg = format!("Email Download interrupted: {:#?}", e);
@@ -134,6 +192,7 @@ pub async fn process_imap_download(
             )?;
         }
     }
+
     let elapsed_time = start_time.elapsed().as_secs();
     debug!(
         "Account{{{}}} Incremental sync completed: {} seconds elapsed.",
@@ -141,3 +200,55 @@ pub async fn process_imap_download(
     );
     Ok(())
 }
+
+/// Runs the gap-fill phase across all download-folder mailboxes. Errors inside
+/// are converted into a failed-run outcome instead of propagating, so the
+/// caller can always finalize the active run.
+async fn gap_fill_phase(
+    account: &AccountModel,
+    local_mailboxes: &[MailBox],
+    remote_mailboxes: &[MailBox],
+    token: CancellationToken,
+    account_id: u64,
+) -> BichonResult<(bool, u64, u64)> {
+    let mut total_downloaded = 0u64;
+    let mut total_failed = 0u64;
+    let mut cancelled = false;
+    for local_mailbox in local_mailboxes {
+        let Some(remote) = remote_mailboxes.iter().find(|r| r.name == local_mailbox.name) else {
+            continue;
+        };
+        if token.is_cancelled() {
+            cancelled = true;
+            break;
+        }
+        DownloadState::set_current_folder(account_id, local_mailbox.name.clone())?;
+        match gap_fill::gap_fill_mailbox(account, local_mailbox, remote, token.clone()).await {
+            Ok(stats) => {
+                total_downloaded += stats.downloaded;
+                total_failed += stats.failed;
+                GapFillState::add_folder_result(account_id, local_mailbox.name.clone(), stats)?;
+            }
+            Err(e) => {
+                let err_msg = format!(
+                    "Gap-fill failed for mailbox '{}': {:#?}",
+                    local_mailbox.name, e
+                );
+                warn!(account_id = account_id, "{}", err_msg);
+                DownloadState::append_session_error(account_id, err_msg)?;
+                total_failed += 1; // count the mailbox as a failed unit
+                GapFillState::add_folder_result(
+                    account_id,
+                    local_mailbox.name.clone(),
+                    GapFillFolderStats {
+                        downloaded: 0,
+                        failed: 1,
+                        candidate_count: 0,
+                        message: None,
+                    },
+                )?;
+            }
+        }
+    }
+    Ok((cancelled, total_downloaded, total_failed))
+}

crates/core/src/archive/imap/download/rebuild.rs

@@ -21,7 +21,7 @@ use crate::{
         migration::AccountModel,
         state::{DownloadState, DownloadStatus, FolderStatus},
     },
-    cache::{
+    archive::{
         imap::{
             download::flow::{fetch_and_save_by_date, fetch_and_save_full_mailbox, FetchDirection},
             mailbox::MailBox,

crates/core/src/archive/imap/mailbox_cache.rs

@@ -16,7 +16,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-use crate::cache::imap::mailbox::MailBox;
+use crate::archive::imap::mailbox::MailBox;
 use crate::utc_now;
 use lru::LruCache;
 use std::collections::HashMap;

crates/core/src/archive/imap/task.rs

@@ -18,7 +18,7 @@
 
 use crate::account::entity::AuthType;
 use crate::account::state::{DownloadState, TriggerType};
-use crate::cache::imap::download::process_imap_download;
+use crate::archive::imap::download::process_imap_download;
 use crate::common::periodic::{PeriodicTask, TaskHandle};
 use crate::error::code::ErrorCode;
 use crate::oauth2::token::OAuth2AccessToken;
@@ -30,7 +30,7 @@ use std::{sync::LazyLock, time::Duration};
 use tokio::sync::Mutex;
 use tokio::task::JoinHandle;
 use tokio_util::sync::CancellationToken;
-use tracing::{error, info, warn};
+use tracing::{debug, error, info, warn};
 
 static _DESCRIPTION: &str = "This task periodically synchronizes mailbox data for a specified account, ensuring that all local data is up-to-date.";
 const TASK_INTERVAL: Duration = Duration::from_secs(10);
@@ -90,7 +90,7 @@ impl AccountDownTask {
             let internal_token = task_token.clone();
             Box::pin(async move {
                 if SYNC_TASKS.is_manual_running(account_id).await {
-                    info!(
+                    debug!(
                         "Account {}: Scheduled task skipped (Manual task is running).",
                         account_id
                     );
@@ -98,7 +98,7 @@ impl AccountDownTask {
                 }
 
                 if !SYNC_TASKS.try_set_busy(account_id).await {
-                    warn!(
+                    debug!(
                         "Account {}: Scheduled task skipped (Previous sync still active).",
                         account_id
                     );
@@ -113,6 +113,9 @@ impl AccountDownTask {
                 let account = AccountModel::get(account_id).ok();
                 match account {
                     Some(account) => {
+                        if account.deleting {
+                            return Ok(());
+                        }
                         if !account.enabled {
                             let last = LAST_WARN_TIME.load(Ordering::Relaxed);
                             let now = utc_now!();
@@ -138,6 +141,7 @@ impl AccountDownTask {
                                 &account,
                                 internal_token,
                                 TriggerType::Scheduled,
+                                false,
                             )
                             .await
                             {
@@ -208,7 +212,7 @@ impl AccountDownTask {
         }
     }
 
-    pub async fn start_manual_task(&self, account_id: u64) -> BichonResult<()> {
+    pub async fn start_manual_task(&self, account_id: u64, run_gap_fill: bool) -> BichonResult<()> {
         {
             if self.is_manual_running(account_id).await {
                 return Err(raise_error!(
@@ -246,7 +250,13 @@ impl AccountDownTask {
                 }
             };
 
-            if let Err(e) = process_imap_download(&account, token_clone, TriggerType::Manual).await
+            if account.deleting {
+                return;
+            }
+
+            if let Err(e) =
+                process_imap_download(&account, token_clone, TriggerType::Manual, run_gap_fill)
+                    .await
             {
                 error!("Manual download failed for {}: {:?}", account_id, e);
                 let error_msg = format!("error in account download task: {:#?}", e);

crates/core/src/autoconfig/client.rs

@@ -16,9 +16,8 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-use hickory_resolver::name_server::TokioConnectionProvider;
+use hickory_resolver::net::runtime::TokioRuntimeProvider;
 use hickory_resolver::proto::rr::RData;
-use hickory_resolver::proto::rr::RecordType;
 use hickory_resolver::TokioResolver;
 use quick_xml::de::from_str;
 use reqwest::Client;
@@ -108,28 +107,29 @@ async fn fetch_xml(client: &Client, url: &str) -> Option<MailConfig> {
 }
 
 async fn lookup_srv(domain: &str) -> Option<MailConfig> {
-    let resolver = TokioResolver::builder(TokioConnectionProvider::default())
+    let resolver = TokioResolver::builder(TokioRuntimeProvider::default())
         .ok()?
-        .build();
+        .build()
+        .ok()?;
 
     let imap_srv = format!("_imaps._tcp.{}.", domain);
-    let imap_lookup = resolver.lookup(imap_srv, RecordType::SRV).await.ok()?;
-    let imap_record = imap_lookup.iter().next()?;
-    let (imap_host, imap_port) = match imap_record {
+    let imap_lookup = resolver.srv_lookup(imap_srv).await.ok()?;
+    let imap_record = imap_lookup.answers().first()?;
+    let (imap_host, imap_port) = match &imap_record.data {
         RData::SRV(srv) => {
-            let host = srv.target().to_string().trim_end_matches('.').to_string();
-            (host, srv.port())
+            let host = srv.target.to_string().trim_end_matches('.').to_string();
+            (host, srv.port)
         }
         _ => return None,
     };
 
     let smtp_srv = format!("_submission._tcp.{}.", domain);
-    let smtp_lookup = resolver.lookup(smtp_srv, RecordType::SRV).await.ok()?;
-    let smtp_record = smtp_lookup.iter().next()?;
-    let (smtp_host, smtp_port) = match smtp_record {
+    let smtp_lookup = resolver.srv_lookup(smtp_srv).await.ok()?;
+    let smtp_record = smtp_lookup.answers().first()?;
+    let (smtp_host, smtp_port) = match &smtp_record.data {
         RData::SRV(srv) => {
-            let host = srv.target().to_string().trim_end_matches('.').to_string();
-            (host, srv.port())
+            let host = srv.target.to_string().trim_end_matches('.').to_string();
+            (host, srv.port)
         }
         _ => return None,
     };
@@ -177,13 +177,19 @@ pub async fn fetch(domain: &str) -> BichonResult<MailConfig> {
         .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
 
     // ── ISP autoconfig (HTTPS, then HTTP) ──────────────────────────
-    if let Some(config) =
-        fetch_xml(&client, &format!("https://autoconfig.{domain}/mail/config-v1.1.xml")).await
+    if let Some(config) = fetch_xml(
+        &client,
+        &format!("https://autoconfig.{domain}/mail/config-v1.1.xml"),
+    )
+    .await
     {
         return Ok(config);
     }
-    if let Some(config) =
-        fetch_xml(&client, &format!("http://autoconfig.{domain}/mail/config-v1.1.xml")).await
+    if let Some(config) = fetch_xml(
+        &client,
+        &format!("http://autoconfig.{domain}/mail/config-v1.1.xml"),
+    )
+    .await
     {
         return Ok(config);
     }
@@ -212,8 +218,11 @@ pub async fn fetch(domain: &str) -> BichonResult<MailConfig> {
     }
 
     // ── Thunderbird central ISPDB ──────────────────────────────────
-    if let Some(config) =
-        fetch_xml(&client, &format!("https://autoconfig.thunderbird.net/v1.1/{domain}")).await
+    if let Some(config) = fetch_xml(
+        &client,
+        &format!("https://autoconfig.thunderbird.net/v1.1/{domain}"),
+    )
+    .await
     {
         return Ok(config);
     }
@@ -245,20 +254,29 @@ async fn fetch_for_mx(client: &Client, domain: &str) -> Option<MailConfig> {
     }
 
     // Try ISPDB for the MX domain
-    if let Some(config) =
-        fetch_xml(client, &format!("https://autoconfig.thunderbird.net/v1.1/{mx_domain}")).await
+    if let Some(config) = fetch_xml(
+        client,
+        &format!("https://autoconfig.thunderbird.net/v1.1/{mx_domain}"),
+    )
+    .await
     {
         return Some(config);
     }
 
     // Try ISP autoconfig for the MX domain (HTTPS then HTTP)
-    if let Some(config) =
-        fetch_xml(client, &format!("https://autoconfig.{mx_domain}/mail/config-v1.1.xml")).await
+    if let Some(config) = fetch_xml(
+        client,
+        &format!("https://autoconfig.{mx_domain}/mail/config-v1.1.xml"),
+    )
+    .await
     {
         return Some(config);
     }
-    if let Some(config) =
-        fetch_xml(client, &format!("http://autoconfig.{mx_domain}/mail/config-v1.1.xml")).await
+    if let Some(config) = fetch_xml(
+        client,
+        &format!("http://autoconfig.{mx_domain}/mail/config-v1.1.xml"),
+    )
+    .await
     {
         return Some(config);
     }
@@ -268,12 +286,16 @@ async fn fetch_for_mx(client: &Client, domain: &str) -> Option<MailConfig> {
 
 /// DNS MX lookup → extract the second-level domain of the first MX hostname.
 async fn lookup_mx_domain(domain: &str) -> Option<String> {
-    let resolver = TokioResolver::builder(TokioConnectionProvider::default())
+    let resolver = TokioResolver::builder(TokioRuntimeProvider::default())
         .ok()?
-        .build();
+        .build()
+        .ok()?;
     let lookup = resolver.mx_lookup(domain).await.ok()?;
-    let record = lookup.iter().next()?;
-    let mx_host = record.to_string().trim_end_matches('.').to_string();
+    let record = lookup.answers().first()?;
+    let mx_host = match &record.data {
+        RData::MX(mx) => mx.exchange.to_string().trim_end_matches('.').to_string(),
+        _ => return None,
+    };
 
     // Extract a reasonable base domain from the MX hostname.
     // E.g., "aspmx.l.google.com" → "google.com"
@@ -336,4 +358,95 @@ mod tests {
             }
         }
     }
+
+    #[test]
+    fn test_parse_autoconfig_xml() {
+        let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
+<clientConfig version="1.1">
+    <emailProvider id="example.com">
+        <domain>example.com</domain>
+        <incomingServer type="imap">
+            <hostname>imap.example.com</hostname>
+            <port>993</port>
+            <socketType>SSL</socketType>
+            <username>%EMAILADDRESS%</username>
+            <authentication>password-cleartext</authentication>
+        </incomingServer>
+        <outgoingServer type="smtp">
+            <hostname>smtp.example.com</hostname>
+            <port>587</port>
+            <socketType>STARTTLS</socketType>
+            <username>%EMAILADDRESS%</username>
+        </outgoingServer>
+    </emailProvider>
+</clientConfig>"#;
+
+        let config = parse_autoconfig_xml(xml).expect("should parse valid XML");
+        assert_eq!(config.incoming.len(), 1);
+        assert_eq!(config.incoming[0].protocol, "imap");
+        assert_eq!(config.incoming[0].hostname, "imap.example.com");
+        assert_eq!(config.incoming[0].port, 993);
+        assert_eq!(config.incoming[0].socket_type, "SSL");
+        assert_eq!(config.incoming[0].username, "%EMAILADDRESS%");
+        assert_eq!(config.incoming[0].authentication, "password-cleartext");
+
+        assert_eq!(config.outgoing.len(), 1);
+        assert_eq!(config.outgoing[0].protocol, "smtp");
+        assert_eq!(config.outgoing[0].hostname, "smtp.example.com");
+        assert_eq!(config.outgoing[0].port, 587);
+        assert_eq!(config.outgoing[0].socket_type, "STARTTLS");
+        assert_eq!(config.outgoing[0].username, "%EMAILADDRESS%");
+    }
+
+    #[test]
+    fn test_parse_autoconfig_xml_invalid() {
+        assert!(parse_autoconfig_xml("not xml").is_none());
+        assert!(parse_autoconfig_xml("<clientConfig></clientConfig>").is_none());
+    }
+
+    #[test]
+    fn test_extract_base_domain() {
+        assert_eq!(
+            extract_base_domain("aspmx.l.google.com"),
+            Some("google.com".to_string())
+        );
+        assert_eq!(
+            extract_base_domain("company.mail.protection.outlook.com"),
+            Some("outlook.com".to_string())
+        );
+        assert_eq!(
+            extract_base_domain("mx.example.com"),
+            Some("example.com".to_string())
+        );
+        assert_eq!(
+            extract_base_domain("example.com"),
+            Some("example.com".to_string())
+        );
+        assert_eq!(extract_base_domain("localhost"), None);
+    }
+
+    #[tokio::test]
+    async fn test_lookup_srv_gmail() {
+        // Gmail should have SRV records for IMAPS and SMTP submission
+        let config = lookup_srv("gmail.com").await;
+        assert!(config.is_some(), "Gmail should have SRV records");
+        let config = config.unwrap();
+        assert_eq!(config.incoming.len(), 1);
+        assert_eq!(config.incoming[0].protocol, "imap");
+        assert_eq!(config.incoming[0].socket_type, "SSL");
+        assert!(!config.incoming[0].hostname.is_empty());
+        assert!(config.incoming[0].port > 0);
+        assert_eq!(config.outgoing.len(), 1);
+        assert_eq!(config.outgoing[0].protocol, "smtp");
+        assert_eq!(config.outgoing[0].socket_type, "STARTTLS");
+        assert!(!config.outgoing[0].hostname.is_empty());
+        assert!(config.outgoing[0].port > 0);
+    }
+
+    #[tokio::test]
+    async fn test_lookup_srv_nonexistent() {
+        // A domain without SRV records should return None
+        let config = lookup_srv("this-domain-definitely-does-not-exist-12345.com").await;
+        assert!(config.is_none());
+    }
 }

crates/core/src/cache/imap/download/flow.rs

@@ -1,645 +0,0 @@
-//
-// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
-//
-// This file is part of the Bichon Email Archiving Project
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-use crate::{
-    raise_error,
-    {
-        account::{
-            migration::AccountModel,
-            state::{DownloadState, DownloadStatus, FolderStatus},
-        },
-        cache::{
-            imap::{
-                download::rebuild::{rebuild_mailbox_cache, rebuild_mailbox_cache_by_date},
-                find_intersecting_mailboxes, find_missing_mailboxes,
-                mailbox::MailBox,
-            },
-            SEMAPHORE,
-        },
-        error::{code::ErrorCode, BichonResult},
-        imap::executor::{
-            generate_uid_sequence_hashset, ImapExecutor, DEFAULT_BATCH_SIZE,
-        },
-        store::tantivy::envelope::ENVELOPE_MANAGER,
-    },
-};
-use std::time::Instant;
-use tokio_util::sync::CancellationToken;
-use tracing::{debug, error, info, warn};
-
-
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub enum FetchDirection {
-    Since,
-    Before,
-}
-
-pub async fn fetch_and_save_by_date(
-    account: &AccountModel,
-    date: &str,
-    mailbox: &MailBox,
-    direction: FetchDirection,
-    token: CancellationToken,
-) -> BichonResult<Option<u32>> {
-    let account_id = account.id;
-    let mut session = match ImapExecutor::create_connection(account_id).await {
-        Ok(session) => session,
-        Err(e) => {
-            let err_msg = format!("Connection failed for this folder: {:#?}", e);
-            DownloadState::update_folder_progress(
-                account_id,
-                mailbox.name.clone(),
-                0,
-                0,
-                FolderStatus::Failed,
-                Some(err_msg.clone()),
-            )?;
-            DownloadState::append_session_error(account_id, err_msg)?;
-            return Err(e);
-        }
-    };
-
-    let search_criteria = match direction {
-        FetchDirection::Since => format!("SINCE {date}"),
-        FetchDirection::Before => format!("BEFORE {date}"),
-    };
-
-    let uid_list =
-        match ImapExecutor::uid_search(&mut session, &mailbox.encoded_name(), &search_criteria)
-            .await
-        {
-            Ok(uid_list) => uid_list,
-            Err(e) => {
-                let err_msg = format!("UID search failed in [{}]: {:#?}", mailbox.name, e);
-                DownloadState::update_folder_progress(
-                    account_id,
-                    mailbox.name.clone(),
-                    0,
-                    0,
-                    FolderStatus::Failed,
-                    Some(err_msg.clone()),
-                )?;
-                DownloadState::append_session_error(account_id, err_msg)?;
-                return Err(e);
-            }
-        };
-
-    let len = uid_list.len();
-    if len == 0 {
-        DownloadState::update_folder_progress(
-            account_id,
-            mailbox.name.clone(),
-            0,
-            0,
-            FolderStatus::Success,
-            None,
-        )?;
-        return Ok(None);
-    }
-
-    // sort small -> bigger
-    let mut uid_vec: Vec<u32> = uid_list.into_iter().collect();
-    uid_vec.sort();
-
-    let max_uid = uid_vec.last().copied();
-    let planned = uid_vec.len() as u64;
-    let uid_batches = generate_uid_sequence_hashset(
-        uid_vec,
-        account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize,
-    );
-    DownloadState::update_folder_progress(
-        account_id,
-        mailbox.name.clone(),
-        planned,
-        0,
-        FolderStatus::Pending,
-        None,
-    )?;
-
-    let mut current_processed = 0u64;
-    let mut has_error_or_cancel = false;
-    for (index, batch) in uid_batches.into_iter().enumerate() {
-        if token.is_cancelled() {
-            DownloadState::update_session_status(
-                account_id,
-                DownloadStatus::Cancelled,
-                Some("User stopped or system shutdown".to_string()),
-            )?;
-            DownloadState::update_folder_progress(
-                account_id,
-                mailbox.name.clone(),
-                planned,
-                current_processed,
-                FolderStatus::Cancelled,
-                None,
-            )?;
-            has_error_or_cancel = true;
-            break;
-        }
-        // Fetch metadata for the current batch of UIDs
-        match ImapExecutor::uid_batch_retrieve_emails(
-            &mut session,
-            account_id,
-            mailbox.id,
-            &batch.0,
-            token.clone(),
-        )
-        .await
-        {
-            Ok(_) => {
-                current_processed += batch.1;
-                DownloadState::update_folder_progress(
-                    account_id,
-                    mailbox.name.clone(),
-                    planned,
-                    current_processed,
-                    FolderStatus::Downloading,
-                    None,
-                )?;
-            }
-            Err(e) => {
-                let err_msg = format!("Batch {} failed: {:#?}", index, e);
-                DownloadState::append_session_error(account_id, err_msg.clone())?;
-                DownloadState::update_folder_progress(
-                    account_id,
-                    mailbox.name.clone(),
-                    planned,
-                    current_processed,
-                    FolderStatus::Failed,
-                    Some(err_msg),
-                )?;
-                has_error_or_cancel = true;
-                break;
-            }
-        }
-    }
-    if !has_error_or_cancel {
-        DownloadState::update_folder_progress(
-            account_id,
-            mailbox.name.clone(),
-            planned,
-            current_processed,
-            FolderStatus::Success,
-            None,
-        )?;
-    }
-    session.logout().await.ok();
-    Ok(max_uid)
-}
-
-/// Fetches all messages from a mailbox.
-/// Returns `Ok(Some(max_uid))` with the highest UID stored, or `Ok(None)` if empty.
-pub async fn fetch_and_save_full_mailbox(
-    account: &AccountModel,
-    mailbox: &MailBox,
-    token: CancellationToken,
-) -> BichonResult<Option<u32>> {
-    let mailbox_id = mailbox.id;
-    let account_id = account.id;
-
-    let mut session = match ImapExecutor::create_connection(account_id).await {
-        Ok(session) => session,
-        Err(e) => {
-            let err_msg = format!("Connection failed for this folder: {:#?}", e);
-            DownloadState::update_folder_progress(
-                account_id,
-                mailbox.name.clone(),
-                0,
-                0,
-                FolderStatus::Failed,
-                Some(err_msg.clone()),
-            )?;
-            DownloadState::append_session_error(account_id, err_msg)?;
-            return Err(e);
-        }
-    };
-
-    let total = match session.examine(&mailbox.encoded_name()).await {
-        Ok(mailbox) => mailbox.exists as u64,
-        Err(e) => {
-            let err_msg = format!("Failed to examine folder [{}]: {:#?}", mailbox.name, e);
-            DownloadState::update_folder_progress(
-                account_id,
-                mailbox.name.clone(),
-                mailbox.exists as u64,
-                0,
-                FolderStatus::Failed,
-                Some(err_msg.clone()),
-            )?;
-
-            DownloadState::append_session_error(account_id, err_msg)?;
-            session.logout().await.ok();
-            return Err(raise_error!(
-                format!("{:#?}", e),
-                ErrorCode::ImapCommandFailed
-            ));
-        }
-    };
-
-    let page_size = account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
-    let total_batches = total.div_ceil(page_size as u64);
-
-    info!(
-        "Starting full mailbox download for '{}', total={}, batches={}",
-        mailbox.name, total, total_batches
-    );
-
-    let mut current_processed = 0u64;
-    let mut has_error_or_cancel = false;
-    let mut max_uid: Option<u32> = None;
-
-    for page in 1..=total_batches {
-        if token.is_cancelled() {
-            DownloadState::update_session_status(
-                account_id,
-                DownloadStatus::Cancelled,
-                Some("User stopped or system shutdown".to_string()),
-            )?;
-            DownloadState::update_folder_progress(
-                account_id,
-                mailbox.name.clone(),
-                total,
-                current_processed,
-                FolderStatus::Cancelled,
-                None,
-            )?;
-            has_error_or_cancel = true;
-            break;
-        }
-
-        match ImapExecutor::batch_retrieve_emails(
-            &mut session,
-            account_id,
-            mailbox_id,
-            total,
-            page as u64,
-            page_size as u64,
-            &mailbox.encoded_name(),
-            token.clone(),
-            &mut max_uid,
-        )
-        .await
-        {
-            Ok(count) => {
-                current_processed += count as u64;
-                DownloadState::update_folder_progress(
-                    account_id,
-                    mailbox.name.clone(),
-                    total,
-                    current_processed,
-                    FolderStatus::Downloading,
-                    None,
-                )?;
-            }
-            Err(e) => {
-                let err_msg = format!("Batch {} failed: {:#?}", page, e);
-                DownloadState::append_session_error(account_id, err_msg.clone())?;
-                DownloadState::update_folder_progress(
-                    account_id,
-                    mailbox.name.clone(),
-                    total,
-                    current_processed,
-                    FolderStatus::Failed,
-                    Some(err_msg),
-                )?;
-                has_error_or_cancel = true;
-                break;
-            }
-        };
-    }
-
-    if !has_error_or_cancel {
-        DownloadState::update_folder_progress(
-            account_id,
-            mailbox.name.clone(),
-            total,
-            current_processed,
-            FolderStatus::Success,
-            None,
-        )?;
-    }
-    session.logout().await.ok();
-    Ok(max_uid)
-}
-
-pub async fn reconcile_mailboxes(
-    account: &AccountModel,
-    remote_mailboxes: &[MailBox],
-    local_mailboxes: &[MailBox],
-    token: CancellationToken,
-) -> BichonResult<()> {
-    let start_time = Instant::now();
-    let existing_mailboxes = find_intersecting_mailboxes(local_mailboxes, remote_mailboxes);
-    let account_id = account.id;
-    if !existing_mailboxes.is_empty() {
-        let mut mailboxes_to_update = Vec::with_capacity(existing_mailboxes.len());
-
-        DownloadState::init_folder_details(
-            account.id,
-            remote_mailboxes.iter().map(|m| m.name.clone()).collect(),
-        )?;
-
-        for (local_mailbox, remote_mailbox) in &existing_mailboxes {
-            if token.is_cancelled() {
-                DownloadState::update_session_status(
-                    account.id,
-                    DownloadStatus::Cancelled,
-                    Some("Received termination signal (User stop or System shutdown)".to_string()),
-                )?;
-                break;
-            }
-
-            let new_highest_uid = if local_mailbox.uid_validity != remote_mailbox.uid_validity {
-                if remote_mailbox.uid_validity.is_none() {
-                    let err_msg = format!(
-                        "Mailbox '{}' logic error: Server did not provide UIDVALIDITY.",
-                        local_mailbox.name
-                    );
-
-                    warn!("Account {}: {}", account_id, err_msg);
-
-                    DownloadState::update_folder_progress(
-                        account_id,
-                        remote_mailbox.name.clone(),
-                        0,
-                        0,
-                        FolderStatus::Failed,
-                        Some(err_msg.clone()),
-                    )?;
-                    DownloadState::append_session_error(account_id, err_msg)?;
-                    continue;
-                }
-                info!(
-                    "Account {}: Mailbox '{}' detected with changed uid_validity (local: {:#?}, remote: {:#?}). \
-                    The mailbox data may be invalid, resetting its envelopes and rebuilding the cache.",
-                    account_id, local_mailbox.name, &local_mailbox.uid_validity, &remote_mailbox.uid_validity
-                );
-
-                DownloadState::update_folder_progress(
-                    account_id,
-                    local_mailbox.name.clone(),
-                    remote_mailbox.exists as u64,
-                    0,
-                    FolderStatus::Downloading,
-                    Some("UID validity changed, rebuilding...".into()),
-                )?;
-
-                match &account.date_since {
-                    Some(date_since) => {
-                        rebuild_mailbox_cache_by_date(
-                            account,
-                            local_mailbox.id,
-                            &date_since.since_date()?,
-                            remote_mailbox,
-                            FetchDirection::Since,
-                            token.clone(),
-                        )
-                        .await?
-                    }
-                    None => match &account.date_before {
-                        Some(r) => {
-                            rebuild_mailbox_cache_by_date(
-                                account,
-                                local_mailbox.id,
-                                &r.calculate_date()?,
-                                remote_mailbox,
-                                FetchDirection::Before,
-                                token.clone(),
-                            )
-                            .await?
-                        }
-                        None => {
-                            rebuild_mailbox_cache(
-                                account,
-                                local_mailbox,
-                                remote_mailbox,
-                                token.clone(),
-                            )
-                            .await?
-                        }
-                    },
-                }
-            } else {
-                perform_incremental_sync(account, local_mailbox, remote_mailbox, token.clone())
-                    .await?
-            };
-
-            let mut updated = remote_mailbox.clone();
-            updated.highest_uid = new_highest_uid;
-            mailboxes_to_update.push(updated);
-        }
-        //The metadata of this mailbox must only be updated after a successful synchronization;
-        //otherwise, it may cause synchronization errors and result in missing emails in the local sync results.
-        MailBox::batch_upsert(&mailboxes_to_update)?;
-    }
-
-    debug!(
-        "Checked mailbox folders for account ID: {}. Compared local and server folders to identify changes. Elapsed time: {} seconds",
-        account.id,
-        start_time.elapsed().as_secs()
-    );
-
-    let missing_mailboxes = find_missing_mailboxes(local_mailboxes, remote_mailboxes);
-    //Mail folders that are not locally need to be downloaded.
-    if !missing_mailboxes.is_empty() {
-        MailBox::batch_insert(&missing_mailboxes)?;
-
-        let mut has_error = false;
-        let mut last_err = None;
-        for mailbox in &missing_mailboxes {
-            if token.is_cancelled() {
-                DownloadState::update_session_status(
-                    account.id,
-                    DownloadStatus::Cancelled,
-                    Some("Received termination signal (User stop or System shutdown)".to_string()),
-                )?;
-                break;
-            }
-            if mailbox.exists > 0 {
-                let account = account.clone();
-                let mailbox = mailbox.clone();
-
-                let _global_permit = match SEMAPHORE.clone().acquire_owned().await {
-                    Ok(permit) => permit,
-                    Err(err) => {
-                        error!(
-                            "Failed to acquire global semaphore permit for account {} mailbox '{}': {:#?}",
-                            account.id, &mailbox.name, err
-                        );
-                        continue;
-                    }
-                };
-
-                let result = match &account.date_since {
-                    Some(date_since) => {
-                        rebuild_mailbox_cache_by_date(
-                            &account,
-                            mailbox.id,
-                            &date_since.since_date()?,
-                            &mailbox,
-                            FetchDirection::Since,
-                            token.clone(),
-                        )
-                        .await
-                    }
-                    None => match &account.date_before {
-                        Some(r) => {
-                            rebuild_mailbox_cache_by_date(
-                                &account,
-                                mailbox.id,
-                                &r.calculate_date()?,
-                                &mailbox,
-                                FetchDirection::Before,
-                                token.clone(),
-                            )
-                            .await
-                        }
-                        None => {
-                            rebuild_mailbox_cache(&account, &mailbox, &mailbox, token.clone()).await
-                        }
-                    },
-                };
-
-                match result {
-                    Ok(new_highest_uid) => {
-                        let mut updated = mailbox.clone();
-                        updated.highest_uid = new_highest_uid;
-                        MailBox::batch_upsert(&[updated])?;
-                    }
-                    Err(err) => {
-                        has_error = true;
-                        tracing::error!("Folder sync task failed: {:#?}", err);
-                        last_err = Some(err);
-                    }
-                }
-            }
-        }
-
-        if has_error {
-            if let Some(e) = last_err {
-                return Err(e);
-            }
-            return Err(raise_error!(
-                "Some tasks failed".into(),
-                ErrorCode::InternalError
-            ));
-        }
-    }
-    Ok(())
-}
-
-//only check new emails and sync
-/// Incrementally syncs a mailbox.
-/// Returns the new highest UID after sync, or `None` if nothing changed.
-async fn perform_incremental_sync(
-    account: &AccountModel,
-    local_mailbox: &MailBox,
-    remote_mailbox: &MailBox,
-    token: CancellationToken,
-) -> BichonResult<Option<u32>> {
-    if remote_mailbox.exists > 0 {
-        // Use stored highest_uid if available; otherwise fall back to Tantivy
-        // query once (backward compatibility with pre-existing databases).
-        let start_uid = match local_mailbox.highest_uid {
-            Some(uid) => {
-                tracing::info!(
-                    "[account {}][mailbox {}] perform_incremental_sync: stored highest_uid={}, remote.exists={}",
-                    account.id,
-                    local_mailbox.name,
-                    uid,
-                    remote_mailbox.exists
-                );
-                uid as u64 + 1
-            }
-            None => {
-                let local_max_uid =
-                    ENVELOPE_MANAGER.get_max_uid(account.id, local_mailbox.id)?;
-                tracing::info!(
-                    "[account {}][mailbox {}] perform_incremental_sync: highest_uid unset, Tantivy max_uid={:?}, remote.exists={}",
-                    account.id,
-                    local_mailbox.name,
-                    local_max_uid,
-                    remote_mailbox.exists
-                );
-                match local_max_uid {
-                    Some(uid) => uid + 1,
-                    None => {
-                        info!(
-                            "No maximum UID found in index for mailbox, assuming local storage is missing."
-                        );
-
-                        let result = match &account.date_since {
-                            Some(date_since) => {
-                                fetch_and_save_by_date(
-                                    account,
-                                    date_since.since_date()?.as_str(),
-                                    remote_mailbox,
-                                    FetchDirection::Since,
-                                    token,
-                                )
-                                .await?
-                            }
-                            None => match &account.date_before {
-                                Some(r) => {
-                                    fetch_and_save_by_date(
-                                        account,
-                                        &r.calculate_date()?,
-                                        remote_mailbox,
-                                        FetchDirection::Before,
-                                        token,
-                                    )
-                                    .await?
-                                }
-                                None => {
-                                    fetch_and_save_full_mailbox(
-                                        account, remote_mailbox, token,
-                                    )
-                                    .await?
-                                }
-                            },
-                        };
-                        return Ok(result);
-                    }
-                }
-            }
-        };
-
-        let mut session = ImapExecutor::create_connection(account.id).await?;
-        let before_date = account
-            .date_before
-            .as_ref()
-            .map(|r| r.calculate_date())
-            .transpose()?;
-
-        let new_max_uid = ImapExecutor::fetch_new_mail(
-            &mut session,
-            account,
-            local_mailbox,
-            start_uid,
-            before_date.as_deref(),
-            token,
-        )
-        .await?;
-        session.logout().await.ok();
-
-        // Keep existing highest_uid if no new mail was fetched.
-        Ok(new_max_uid.or(local_mailbox.highest_uid))
-    } else {
-        Ok(local_mailbox.highest_uid)
-    }
-}

crates/core/src/context/controller.rs

@@ -16,7 +16,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-use crate::{cache::imap::task::SYNC_TASKS, error::BichonResult};
+use crate::{archive::imap::task::SYNC_TASKS, error::BichonResult};
 use std::{sync::LazyLock, time::Duration};
 use tokio::sync::mpsc;
 use tracing::{error, info};

crates/core/src/context/executors.rs

@@ -17,6 +17,7 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::account::migration::AccountType;
+use crate::account::state::{DownloadState, GapFillState};
 use crate::context::Initialize;
 use crate::{
     {
@@ -25,7 +26,7 @@ use crate::{
     utc_now,
 };
 use std::sync::LazyLock;
-use tracing::info;
+use tracing::{info, warn};
 
 pub static BICHON_CONTEXT: LazyLock<BichonContext> = LazyLock::new(BichonContext::new);
 
@@ -65,6 +66,45 @@ impl BichonContext {
             active_accounts.len()
         );
         for account in active_accounts {
+            // A Running session surviving startup is a leftover from a previous
+            // interrupted run; nothing is downloading yet at this point. Mark it
+            // Cancelled so the UI doesn't show a phantom "syncing" state. The
+            // scheduler starts regardless — its first tick runs immediately, so
+            // the interrupted run is caught up on, and the session's trigger
+            // stays Scheduled rather than showing a "Manual" the user never
+            // initiated.
+            match DownloadState::finalize_stale_session(account.id) {
+                Ok(true) => {
+                    info!(
+                        "Account {}: stale sync session finalized on startup.",
+                        account.id
+                    );
+                }
+                Err(e) => {
+                    warn!(
+                        "Failed to finalize stale session for account {}: {:#?}",
+                        account.id, e
+                    );
+                }
+                Ok(false) => {}
+            }
+            // Same for a leftover gap-fill run: a Running active run surviving
+            // startup is a phantom — nothing is scanning at this point.
+            match GapFillState::finalize_stale_run(account.id) {
+                Ok(true) => {
+                    info!(
+                        "Account {}: stale gap-fill run finalized on startup.",
+                        account.id
+                    );
+                }
+                Err(e) => {
+                    warn!(
+                        "Failed to finalize stale gap-fill run for account {}: {:#?}",
+                        account.id, e
+                    );
+                }
+                Ok(false) => {}
+            }
             DOWNLOAD_CONTROLLER
                 .trigger_schedule(account.id, account.email)
                 .await

crates/core/src/database/manager.rs

@@ -17,7 +17,7 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::settings::dir::DATA_DIR_MANAGER;
-use memdb::{Durability, MemDb};
+use bichon_memdb::{Durability, MemDb};
 use std::sync::LazyLock;
 use std::time::Duration;
 

crates/core/src/database/mod.rs

@@ -20,7 +20,7 @@ use crate::common::paginated::Paginated;
 use crate::error::code::ErrorCode;
 use crate::error::BichonResult;
 use crate::raise_error;
-use memdb::{MemDb, Transaction};
+use bichon_memdb::{MemDb, Transaction};
 use serde::de::DeserializeOwned;
 use serde::Serialize;
 

crates/core/src/envelope/extractor.rs

@@ -1,4 +1,3 @@
-//
 // Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
@@ -16,30 +15,48 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-use crate::cache::imap::mailbox::MailBox;
-use crate::common::AddrVec;
-use crate::envelope::meta::parse_bichon_metadata;
-use crate::envelope::utils::normalize_subject;
-use crate::error::code::ErrorCode;
-use crate::error::BichonResult;
-use crate::imap::executor::ImapExecutor;
-use crate::message::content::AttachmentInfo;
-use crate::store::blob::{DetachedEmail, BLOB_MANAGER};
-use crate::store::tantivy::attachment::ATTACHMENT_MANAGER;
-use crate::store::tantivy::envelope::ENVELOPE_MANAGER;
-use crate::store::tantivy::model::{AttachmentModel, EnvelopeWithAttachments};
-use crate::utils::html::extract_text;
-use crate::utils::{compute_content_hash, hex_hash};
-use crate::{id, store::envelope::Envelope};
-use crate::{raise_error, utc_now};
 use async_imap::types::Fetch;
 use bytes::Bytes;
 use mail_parser::{Address, HeaderName, Message, MessageParser, MimeHeaders};
-use tantivy::TantivyDocument;
-use tantivy::schema::Facet;
+use tantivy::{schema::Facet, TantivyDocument};
 use tracing::error;
 use uuid::Uuid;
 
+use crate::{
+    account::migration::AccountModel,
+    archive::imap::mailbox::MailBox,
+    common::AddrVec,
+    envelope::{meta::parse_bichon_metadata, utils::normalize_subject},
+    error::{code::ErrorCode, BichonResult},
+    id,
+    imap::executor::ImapExecutor,
+    message::content::AttachmentInfo,
+    raise_error,
+    store::{
+        blob::{DetachedEmail, BLOB_MANAGER},
+        envelope::Envelope,
+        tantivy::{
+            attachment::ATTACHMENT_MANAGER,
+            dedup_cache::DEDUP_CACHE,
+            envelope::ENVELOPE_MANAGER,
+            model::{AttachmentModel, EnvelopeWithAttachments},
+        },
+    },
+    utc_now,
+    utils::{compute_content_hash, hex_hash, html::extract_text},
+};
+
+/// The outcome of extracting an envelope. `Duplicate` means the message was
+/// skipped because its content hash was already archived. `Imported` covers
+/// every other processed message, including mail dropped by archive rules,
+/// which has always counted as a success on the import surfaces.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[must_use]
+pub enum ExtractOutcome {
+    Imported,
+    Duplicate,
+}
+
 pub async fn extract_envelope_and_store_it(
     fetch: Fetch,
     account_id: u64,
@@ -50,18 +67,28 @@ pub async fn extract_envelope_and_store_it(
         .map(|d| d.timestamp_millis())
         .unwrap_or(0);
     let uid = fetch.uid.unwrap_or(0);
-    let body = fetch
-        .body()
-        .ok_or_else(|| raise_error!("No body available".into(), ErrorCode::InternalError))?;
+    let body = match fetch.body() {
+        Some(b) => b,
+        None => {
+            tracing::warn!(
+                account_id,
+                uid = fetch.uid,
+                "FETCH response has no body, skipping message"
+            );
+            return Ok(());
+        }
+    };
     let size = fetch.size.unwrap_or(body.len() as u32);
-    extract_envelope_core(body, uid, size, internal_date, account_id, mailbox_id).await
+    extract_envelope_core(body, uid, size, internal_date, account_id, mailbox_id)
+        .await
+        .map(|_| ())
 }
 
 pub async fn extract_envelope_from_eml(
     body: &[u8],
     account_id: u64,
     mailbox_id: u64,
-) -> BichonResult<()> {
+) -> BichonResult<ExtractOutcome> {
     extract_envelope_core(body, 0, body.len() as u32, 0, account_id, mailbox_id).await
 }
 
@@ -69,7 +96,7 @@ pub async fn extract_envelope_from_smtp(
     body: &[u8],
     account_id: u64,
     mailbox_id: u64,
-) -> BichonResult<()> {
+) -> BichonResult<ExtractOutcome> {
     extract_envelope_core(
         body,
         0,
@@ -88,9 +115,14 @@ async fn extract_envelope_core(
     internal_date: i64,
     account_id: u64,
     mailbox_id: u64,
-) -> BichonResult<()> {
-    //The content hash of the original raw EML
+) -> BichonResult<ExtractOutcome> {
+    // The content hash of the original raw EML
     let email_content_hash = compute_content_hash(body);
+    if DEDUP_CACHE.contains(account_id, mailbox_id, &email_content_hash) {
+        tracing::debug!("Duplicate email detected");
+        // println!("Duplicate email detected");
+        return Ok(ExtractOutcome::Duplicate);
+    }
     let message: Message<'_> = MessageParser::new().parse(body).ok_or_else(|| {
         raise_error!(
             "Email header parse result is not available".into(),
@@ -98,6 +130,38 @@ async fn extract_envelope_core(
         )
     })?;
 
+    if let Ok(account) = AccountModel::get(account_id) {
+        if let Some(ref rules) = account.archive_rules {
+            let sender = message.from().and_then(|addr| {
+                AddrVec::from(addr)
+                    .0
+                    .into_iter()
+                    .next()
+                    .and_then(|a| a.address)
+            });
+            let subject = message.subject().map(|s| s.to_string());
+
+            let is_spam = !rules.spam_headers.is_empty()
+                && rules.spam_headers.iter().any(|h| {
+                    message
+                        .header_raw(h.clone())
+                        .map(|v| matches!(v.trim().to_lowercase().as_str(), "yes" | "true"))
+                        .unwrap_or(false)
+                });
+
+            if !rules.should_archive(sender.as_deref(), subject.as_deref(), size, is_spam) {
+                tracing::debug!(
+                    account_id,
+                    uid,
+                    sender = sender.as_deref().unwrap_or("?"),
+                    subject = subject.as_deref().unwrap_or("?"),
+                    "Email filtered out by archive rules"
+                );
+                return Ok(ExtractOutcome::Imported);
+            }
+        }
+    }
+
     let preview_limit = 100;
     let text = if let Some(text) = message.body_text(0).map(|cow| cow.into_owned()) {
         text
@@ -159,12 +223,13 @@ async fn extract_envelope_core(
         .and_then(|add| add.address)
         .unwrap_or_else(|| "unknown".to_string());
     let attachment_count = message.attachment_count();
-    let attachments = detach_and_store_attachments(body, &message, &email_content_hash).await;
+    let attachments =
+        detach_and_store_attachments(body, &message, &email_content_hash, account_id, mailbox_id)
+            .await;
 
     let envelope_id = Uuid::new_v4().to_string();
     let now = utc_now!();
 
-
     let mut final_tags = Vec::new();
 
     if let Some(meta_header) = message.header_raw("X-Bichon-Metadata") {
@@ -172,11 +237,7 @@ async fn extract_envelope_core(
             if let Some(tags) = bmd.tags {
                 let validated_tags: Result<Vec<String>, _> = tags
                     .iter()
-                    .map(|tag| {
-                        Facet::from_text(tag)
-                            .map(|_| tag.clone()) 
-                            .map_err(|e| e)
-                    })
+                    .map(|tag| Facet::from_text(tag).map(|_| tag.clone()).map_err(|e| e))
                     .collect();
 
                 match validated_tags {
@@ -184,10 +245,7 @@ async fn extract_envelope_core(
                         final_tags = valid_list;
                     }
                     Err(e) => {
-                        eprintln!(
-                            "Tag validation failed, ignoring all tags: {:#?}",
-                            e
-                        );
+                        eprintln!("Tag validation failed, ignoring all tags: {:#?}", e);
                     }
                 }
             }
@@ -252,7 +310,8 @@ async fn extract_envelope_core(
         tags: (!final_tags.is_empty()).then_some(final_tags),
         account_email: None,
         mailbox_name: None,
-        content_hash: email_content_hash,
+        content_hash: email_content_hash.clone(),
+        account_name: None,
     };
     // 'attachments' contains both regular and inline attachments
     let ea = EnvelopeWithAttachments {
@@ -269,10 +328,11 @@ async fn extract_envelope_core(
         &ea.envelope.content_hash,
     );
     ENVELOPE_MANAGER.queue(doc).await;
+    DEDUP_CACHE.insert(account_id, mailbox_id, &email_content_hash);
     for doc in attachment_docs {
         ATTACHMENT_MANAGER.queue(doc).await;
     }
-    Ok(())
+    Ok(ExtractOutcome::Imported)
 }
 
 pub fn extract_envelope_from_nested_message(
@@ -346,6 +406,7 @@ pub fn extract_envelope_from_nested_message(
         regular_attachment_count: Default::default(),
         tags: Default::default(),
         account_email: Default::default(),
+        account_name: Default::default(),
         mailbox_name: Default::default(),
         content_hash: Default::default(),
     };
@@ -384,10 +445,31 @@ pub async fn detach_and_store_attachments(
     original_body: &[u8],
     message: &Message<'_>,
     eml_content_hash: &str,
+    account_id: u64,
+    mailbox_id: u64,
 ) -> Vec<AttachmentInfo> {
+    let rules = if account_id > 0 {
+        AccountModel::get(account_id)
+            .ok()
+            .and_then(|a| a.extraction_rules)
+    } else {
+        None
+    };
+
+    let mailbox_name = match rules.as_ref().map(|r| !r.folders.is_empty()) {
+        Some(true) => MailBox::get(mailbox_id).ok().map(|mb| mb.name),
+        _ => None,
+    };
+
+    let sender = message
+        .from()
+        .and_then(|addr| AddrVec::from(addr).0.into_iter().next())
+        .and_then(|add| add.address);
+
     let mut stripped_eml = original_body.to_vec();
     let mut attachment_infos = Vec::new();
-    // Step 1: Collect and sort attachment ranges in reverse to maintain offset integrity
+    // Step 1: Collect and sort attachment ranges in reverse to maintain offset
+    // integrity
     let mut ranges: Vec<_> = message
         .attachments()
         .map(|att| {
@@ -453,19 +535,30 @@ pub async fn detach_and_store_attachments(
             })
             .unwrap_or_else(|| "application/octet-stream".to_string());
         let has_cid = att.content_id().is_some();
-        let ext = att
-            .attachment_name()
+        let att_name = att.attachment_name().map(|n| n.to_string());
+        let ext = att_name
+            .as_deref()
             .and_then(|n| {
-                std::path::Path::new(&n)
+                std::path::Path::new(n)
                     .extension()
                     .and_then(|e| e.to_str())
                     .map(|s| s.to_ascii_lowercase())
             })
             .unwrap_or_default();
 
+        let should_extract = rules.as_ref().map_or(true, |r| {
+            r.should_extract(
+                &ext,
+                mailbox_name.as_deref(),
+                att_name.as_deref(),
+                sender.as_deref(),
+            )
+        });
+
         if !inline || !has_cid {
             let decoded_len = att.contents().len();
-            if decoded_len <= crate::ext::text_extractor::MAX_EXTRACT_BYTES
+            if should_extract
+                && decoded_len <= crate::ext::text_extractor::MAX_EXTRACT_BYTES
                 && crate::ext::text_extractor::should_try_extract(&file_type, &ext)
             {
                 text_candidates.push(TextCandidate {
@@ -496,10 +589,8 @@ pub async fn detach_and_store_attachments(
     // Run text extraction in a single spawn_blocking batch.
     if !text_candidates.is_empty() {
         if let Ok(mut extracted_map) = tokio::task::spawn_blocking(move || {
-            let mut map: std::collections::HashMap<
-                String,
-                (String, Option<u32>, bool),
-            > = std::collections::HashMap::new();
+            let mut map: std::collections::HashMap<String, (String, Option<u32>, bool)> =
+                std::collections::HashMap::new();
             for c in text_candidates {
                 if let Some(r) =
                     crate::ext::text_extractor::extract_text(&c.file_type, &c.ext, &c.bytes)
@@ -536,8 +627,7 @@ pub fn reattach_eml_content(
     envelope_id: String,
 ) -> BichonResult<(Envelope, Bytes)> {
     let e = ENVELOPE_MANAGER
-        .get_envelope_by_id(account_id, &envelope_id)
-        ?
+        .get_envelope_by_id(account_id, &envelope_id)?
         .ok_or_else(|| {
             raise_error!(
                 format!(
@@ -570,7 +660,7 @@ pub fn reattach_eml_content(
         return Err(raise_error!(
             format!(
                 "Consistency check failed: envelope.attachment_count ({}) does not match attachments.len ({})",
-                e.envelope.attachment_count, 
+                e.envelope.attachment_count,
                 actual_count
             ),
             ErrorCode::InternalError
@@ -591,11 +681,7 @@ pub fn reattach_eml_content(
             let absolute_start = search_cursor + pos;
             let absolute_end = absolute_start + pattern_len;
 
-            tasks.push((
-                absolute_start,
-                absolute_end,
-                detail.content_hash.clone(),
-            ));
+            tasks.push((absolute_start, absolute_end, detail.content_hash.clone()));
             search_cursor = absolute_end;
         }
     }
@@ -613,14 +699,15 @@ pub fn reattach_eml_content(
     Ok((e.envelope, Bytes::from(restored_eml)))
 }
 
-/// Returns the raw EML for an indexed message, self-healing a missing content blob.
+/// Returns the raw EML for an indexed message, self-healing a missing content
+/// blob.
 ///
-/// Behaves like [`reattach_eml_content`], but when the message's content blob is
-/// absent from the blob store it fetches that single message on demand from the
-/// IMAP server (`UID FETCH <uid> (BODY.PEEK[])`), persists it for future requests,
-/// and returns it. If the on-demand fetch itself fails, the original "content not
-/// found" error from [`reattach_eml_content`] is surfaced unchanged so the caller
-/// still produces its 404.
+/// Behaves like [`reattach_eml_content`], but when the message's content blob
+/// is absent from the blob store it fetches that single message on demand from
+/// the IMAP server (`UID FETCH <uid> (BODY.PEEK[])`), persists it for future
+/// requests, and returns it. If the on-demand fetch itself fails, the original
+/// "content not found" error from [`reattach_eml_content`] is surfaced
+/// unchanged so the caller still produces its 404.
 pub async fn reattach_eml_content_self_healing(
     account_id: u64,
     envelope_id: String,
@@ -669,14 +756,15 @@ pub async fn reattach_eml_content_self_healing(
 
 /// Fetches one message from IMAP and re-stores its detached blob.
 ///
-/// On success the freshly fetched raw RFC822 body is returned; it is also queued
-/// (in detached form) into the blob store so subsequent requests hit the cache.
-/// Fails if the message cannot be fetched, or if the fetched bytes do not match
-/// the archived `content_hash` (the server-side message no longer matches what
-/// Bichon archived, so it cannot be treated as a recovery of that blob).
+/// On success the freshly fetched raw RFC822 body is returned; it is also
+/// queued (in detached form) into the blob store so subsequent requests hit the
+/// cache. Fails if the message cannot be fetched, or if the fetched bytes do
+/// not match the archived `content_hash` (the server-side message no longer
+/// matches what Bichon archived, so it cannot be treated as a recovery of that
+/// blob).
 async fn recover_message_blob(envelope: &Envelope) -> BichonResult<Bytes> {
-    let mailbox = MailBox::find_mailbox(envelope.account_id, envelope.mailbox_id)?
-        .ok_or_else(|| {
+    let mailbox =
+        MailBox::find_mailbox(envelope.account_id, envelope.mailbox_id)?.ok_or_else(|| {
             raise_error!(
                 format!(
                     "Mailbox not found: account_id={} mailbox_id={}",
@@ -710,13 +798,22 @@ async fn recover_message_blob(envelope: &Envelope) -> BichonResult<Bytes> {
     // Re-create the detached blob (stripped EML + attachments) so the missing
     // blob is repopulated for future requests. The detached EML is queued under
     // `fetched_hash`, which equals `envelope.content_hash`.
-    let message = MessageParser::new().parse(raw_body.as_slice()).ok_or_else(|| {
-        raise_error!(
-            "Failed to parse fetched email content".into(),
-            ErrorCode::InternalError
-        )
-    })?;
-    detach_and_store_attachments(&raw_body, &message, &fetched_hash).await;
+    let message = MessageParser::new()
+        .parse(raw_body.as_slice())
+        .ok_or_else(|| {
+            raise_error!(
+                "Failed to parse fetched email content".into(),
+                ErrorCode::InternalError
+            )
+        })?;
+    detach_and_store_attachments(
+        &raw_body,
+        &message,
+        &fetched_hash,
+        envelope.account_id,
+        envelope.mailbox_id,
+    )
+    .await;
 
     Ok(Bytes::from(raw_body))
 }
@@ -809,12 +906,9 @@ mod test {
         assert!(truncated.len() < raw.len());
 
         // Must not panic.
-        let infos = super::detach_and_store_attachments(
-            truncated,
-            &message,
-            "test_content_hash",
-        )
-        .await;
+        let infos =
+            super::detach_and_store_attachments(truncated, &message, "test_content_hash", 0, 0)
+                .await;
 
         // The attachment count must still match so the consistency check
         // in reattach_eml_content doesn't fail later.

crates/core/src/error/mod.rs

@@ -16,4 +16,12 @@ pub enum BichonError {
     },
 }
 
+impl BichonError {
+    pub fn code(&self) -> ErrorCode {
+        match self {
+            BichonError::Generic { code, .. } => *code,
+        }
+    }
+}
+
 pub type BichonResult<T, E = BichonError> = std::result::Result<T, E>;

crates/core/src/ext/event_bus.rs

@@ -26,7 +26,13 @@
 // It never reads from the event bus — events are fire-and-forget.
 
 use std::net::IpAddr;
-use std::sync::{LazyLock, RwLock};
+use std::sync::{LazyLock, Mutex, RwLock};
+use std::time::{Duration, Instant};
+
+/// Free-form JSON value carried by events that need a content snapshot
+/// (e.g. the subject / attachment names of a message being deleted, so the
+/// audit trail stays self-describing after the content is gone).
+pub type EventPayload = serde_json::Map<String, serde_json::Value>;
 
 #[derive(Debug, Clone)]
 pub enum Event {
@@ -34,10 +40,58 @@ pub enum Event {
         email_id: String,
         user: String,
         ip: IpAddr,
+        account_id: u64,
+        mailbox_id: u64,
+        /// Subject of the viewed message, so the audit trail is readable
+        /// without resolving the envelope again.
+        subject: Option<String>,
     },
     EmailDeleted {
         email_id: String,
         user: String,
+        account_id: u64,
+        mailbox_id: u64,
+        /// Subject at deletion time (the message is gone afterwards).
+        subject: Option<String>,
+        /// Snapshot of the deleted message (attachment names, from/to, ...).
+        snapshot: Option<EventPayload>,
+    },
+    /// Raw EML file downloaded (export).
+    EmailExported {
+        email_id: String,
+        user: String,
+        account_id: u64,
+        subject: Option<String>,
+    },
+    /// Email restored back to the source IMAP server.
+    EmailRestored {
+        email_id: String,
+        user: String,
+        account_id: u64,
+        subject: Option<String>,
+    },
+    /// Facet tags added/removed on emails.
+    EmailTagged {
+        user: String,
+        account_id: u64,
+        /// Number of emails whose tags were changed.
+        count: u64,
+    },
+    /// Facet tags added/removed on attachments.
+    AttachmentTagged {
+        user: String,
+        account_id: u64,
+        /// Number of attachments whose tags were changed.
+        count: u64,
+    },
+    /// Attachment content streamed for in-browser preview (not a download).
+    AttachmentPreviewed {
+        email_id: String,
+        content_hash: String,
+        user: String,
+        account_id: u64,
+        mailbox_id: u64,
+        filename: Option<String>,
     },
     UserLoggedIn {
         user: String,
@@ -47,6 +101,26 @@ pub enum Event {
         created_by: String,
         new_user: String,
     },
+    UserUpdated {
+        updated_by: String,
+        target_user: String,
+    },
+    UserRemoved {
+        removed_by: String,
+        target_user: String,
+    },
+    RoleCreated {
+        created_by: String,
+        role_name: String,
+    },
+    RoleUpdated {
+        updated_by: String,
+        role_name: String,
+    },
+    RoleRemoved {
+        removed_by: String,
+        role_name: String,
+    },
     SearchPerformed {
         query: String,
         user: String,
@@ -57,8 +131,116 @@ pub enum Event {
     },
     AttachmentDownloaded {
         email_id: String,
+        /// The attachment's own content hash.
         content_hash: String,
         user: String,
+        account_id: u64,
+        mailbox_id: u64,
+        filename: Option<String>,
+        size: Option<u64>,
+        ext: Option<String>,
+        /// Content hash of the parent email (EML), when known.
+        parent_content_hash: Option<String>,
+    },
+    AccountCreated {
+        created_by: String,
+        account_id: u64,
+        email: String,
+    },
+    AccountUpdated {
+        updated_by: String,
+        account_id: u64,
+        email: String,
+    },
+    AccountRemoved {
+        removed_by: String,
+        account_id: u64,
+        email: String,
+    },
+    AccountDownloadStarted {
+        user: String,
+        account_id: u64,
+        run_gap_fill: bool,
+    },
+    AccountDownloadStopped {
+        user: String,
+        account_id: u64,
+    },
+    /// Batch role / access assignment on one or more accounts.
+    AccountRoleAssigned {
+        user: String,
+        target_user: String,
+        account_count: usize,
+        roles: Vec<String>,
+    },
+    AccessTokenCreated {
+        user: String,
+        target_user: String,
+        name: Option<String>,
+    },
+    AccessTokenRemoved {
+        user: String,
+        token_user: String,
+        name: Option<String>,
+    },
+    OAuth2ConfigCreated {
+        user: String,
+        oauth2_id: u64,
+        name: String,
+    },
+    OAuth2ConfigUpdated {
+        user: String,
+        oauth2_id: u64,
+        name: String,
+    },
+    OAuth2ConfigRemoved {
+        user: String,
+        oauth2_id: u64,
+        name: String,
+    },
+    /// External OAuth2 token stored / refreshed for an account.
+    OAuth2TokenStored {
+        user: String,
+        account_id: u64,
+    },
+    ImportPerformed {
+        user: String,
+        account_id: u64,
+        format: String,
+        total: u64,
+        success: u64,
+        duplicates: u64,
+        failed: u64,
+    },
+    MailboxRemoved {
+        user: String,
+        account_id: u64,
+        mailbox_id: u64,
+    },
+    ProxyCreated {
+        user: String,
+        url: String,
+    },
+    ProxyUpdated {
+        user: String,
+        url: String,
+    },
+    ProxyRemoved {
+        user: String,
+        url: String,
+    },
+    /// Pro edition: SSO (OIDC) login, logout, or license upload.
+    SsoLogin {
+        user: String,
+        ip: Option<IpAddr>,
+    },
+    SsoLogout {
+        user: String,
+    },
+    LicenseUploaded {
+        user: String,
+        email: String,
+        edition: String,
     },
 }
 
@@ -75,6 +257,53 @@ impl EventBus for NoopEventBus {
 static EVENT_BUS: LazyLock<RwLock<Box<dyn EventBus>>> =
     LazyLock::new(|| RwLock::new(Box::new(NoopEventBus)));
 
+/// Short-window dedup of view/download events.
+///
+/// The web UI can fire duplicate `message-content` requests for the same
+/// email (React StrictMode double-effects, remote-content toggle, thread
+/// expansion). Deduping here keeps the audit trail to one record per
+/// intentional view without hiding repeated deliberate accesses.
+static VIEW_DEDUP: LazyLock<Mutex<Vec<(String, Instant)>>> =
+    LazyLock::new(|| Mutex::new(Vec::new()));
+
+const VIEW_DEDUP_WINDOW: Duration = Duration::from_secs(10);
+
+fn is_duplicate_view(event: &Event) -> bool {
+    let key = match event {
+        Event::EmailViewed {
+            user, email_id, ..
+        } => Some(format!("email.viewed|{user}|{email_id}")),
+        Event::EmailDeleted {
+            user, email_id, ..
+        } => Some(format!("email.deleted|{user}|{email_id}")),
+        Event::AttachmentDownloaded {
+            user,
+            email_id,
+            content_hash,
+            ..
+        } => Some(format!("attachment.downloaded|{user}|{email_id}|{content_hash}")),
+        Event::AttachmentPreviewed {
+            user,
+            email_id,
+            content_hash,
+            ..
+        } => Some(format!("attachment.previewed|{user}|{email_id}|{content_hash}")),
+        _ => None,
+    };
+    let Some(key) = key else {
+        return false;
+    };
+
+    let mut entries = VIEW_DEDUP.lock().unwrap();
+    let now = Instant::now();
+    entries.retain(|(_, at)| now.duration_since(*at) < VIEW_DEDUP_WINDOW);
+    if entries.iter().any(|(k, _)| *k == key) {
+        return true;
+    }
+    entries.push((key, now));
+    false
+}
+
 /// Called by Pro/Enterprise at startup to replace the noop default.
 pub fn set_event_bus(bus: Box<dyn EventBus>) {
     *EVENT_BUS.write().unwrap() = bus;
@@ -82,5 +311,8 @@ pub fn set_event_bus(bus: Box<dyn EventBus>) {
 
 /// Fire-and-forget. Called by the server at key points.
 pub fn emit(event: Event) {
+    if is_duplicate_view(&event) {
+        return;
+    }
     EVENT_BUS.read().unwrap().emit(event);
 }

crates/core/src/ext/text_extractor.rs

@@ -63,8 +63,18 @@ pub const MAX_EXTRACT_BYTES: usize = 10 * 1024 * 1024;
 pub fn should_try_extract(content_type: &str, ext: &str) -> bool {
     matches!(
         ext,
-        "pdf" | "doc" | "docx" | "xls" | "xlsx" | "ppt" | "pptx"
-            | "txt" | "rtf" | "odt" | "ods" | "odp"
+        "pdf"
+            | "doc"
+            | "docx"
+            | "xls"
+            | "xlsx"
+            | "ppt"
+            | "pptx"
+            | "txt"
+            | "rtf"
+            | "odt"
+            | "ods"
+            | "odp"
     ) || content_type.starts_with("text/")
 }
 

crates/core/src/imap/client.rs

@@ -34,6 +34,25 @@ use std::ops::DerefMut;
 use tokio::io::BufWriter;
 use tracing::debug;
 
+/// Classify an `io::Error` (from TLS stream I/O) for IMAP connection errors.
+/// `UnexpectedEof` is treated as a network error because many servers skip
+/// the TLS `close_notify` alert, causing rustls to emit this error when the
+/// TCP connection is dropped normally.
+fn classify_io_error(e: &std::io::Error) -> ErrorCode {
+    use std::io::ErrorKind;
+    matches!(
+        e.kind(),
+        ErrorKind::BrokenPipe
+            | ErrorKind::ConnectionReset
+            | ErrorKind::ConnectionAborted
+            | ErrorKind::TimedOut
+            | ErrorKind::UnexpectedEof
+            | ErrorKind::NotConnected
+    )
+    .then_some(ErrorCode::NetworkError)
+    .unwrap_or(ErrorCode::ImapCommandFailed)
+}
+
 #[derive(Debug)]
 pub(crate) struct Client {
     inner: ImapClient<Box<dyn SessionStream>>,
@@ -141,7 +160,7 @@ impl Client {
         let _greeting = client
             .read_response()
             .await
-            .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
+            .map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))?
             .ok_or_else(|| {
                 raise_error!(
                     "Failed to read IMAP greeting — this usually indicates an incorrect encryption setting (SSL vs. STARTTLS). Your current setting is SSL.".into(),
@@ -171,7 +190,7 @@ impl Client {
         let _greeting = client
             .read_response()
             .await
-            .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
+            .map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))?
             .ok_or_else(|| {
                 raise_error!(
                     "failed to read greeting".into(),
@@ -202,7 +221,7 @@ impl Client {
         let _greeting = client
             .read_response()
             .await
-            .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
+            .map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))?
             .ok_or_else(|| {
                 raise_error!(
                     "Failed to read IMAP greeting — this usually indicates an incorrect encryption setting (SSL vs. STARTTLS). Your current setting is STARTTLS.".into(),

crates/core/src/imap/manager.rs

@@ -95,16 +95,40 @@ impl ImapConnectionManager {
 
     pub async fn build(account_id: u64) -> BichonResult<Session<Box<dyn SessionStream>>> {
         let account = AccountModel::get(account_id)?;
-        let client = match Self::create_client(&account).await {
-            Ok(client) => client,
-            Err(error) => {
-                error!(
-                    "Failed to create IMAP {}'s client: {:#?}",
-                    &account.email, error
-                );
-                return Err(error);
+        let account_email = account.email.clone();
+
+        let mut client = None;
+        for attempt in 0..3u32 {
+            match Self::create_client(&account).await {
+                Ok(c) => {
+                    client = Some(c);
+                    break;
+                }
+                Err(error) if error.code() == ErrorCode::NetworkError && attempt < 2 => {
+                    warn!(
+                        "IMAP connection attempt {}/3 to {} failed (network error), retrying...",
+                        attempt + 1,
+                        account_email
+                    );
+                    tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+                    continue;
+                }
+                Err(error) => {
+                    error!(
+                        "Failed to create IMAP {}'s client: {:#?}",
+                        account_email, error
+                    );
+                    return Err(error);
+                }
             }
-        };
+        }
+
+        let client = client.ok_or_else(|| {
+            raise_error!(
+                format!("Failed to create IMAP {}'s client after 3 attempts", account_email),
+                ErrorCode::NetworkError
+            )
+        })?;
 
         let mut session = match Self::authenticate(client, &account).await {
             Ok(session) => session,

crates/core/src/imap/mock_server.rs

@@ -0,0 +1,552 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//! A minimal scriptable IMAP server for integration testing.
+//!
+//! Each instance listens on a random localhost port and responds to a
+//! pre-configured script of (expected_command, response) pairs. Commands
+//! are matched by substring — the first matching pattern wins.
+//!
+//! # Example
+//! ```ignore
+//! let server = MockImapServer::new()
+//!     .greeting("* OK ready\r\n")
+//!     .respond("LOGIN", "A0 OK logged in\r\n")
+//!     .respond("CAPABILITY", "* CAPABILITY IMAP4rev1\r\nA0 OK done\r\n")
+//!     .respond("STATUS", "* STATUS INBOX (MESSAGES 10 UIDVALIDITY 42)\r\nA0 OK\r\n")
+//!     .respond("LOGOUT", "* BYE\r\nA0 OK\r\n")
+//!     .start()
+//!     .await;
+//!
+//! let (host, port) = server.addr();
+//! // connect to host:port with Encryption::None
+//! ```
+
+use std::net::SocketAddr;
+use std::sync::Arc;
+use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
+use tokio::net::{TcpListener, TcpStream};
+
+type Response = Vec<u8>;
+
+pub struct MockImapServer {
+    greeting: Vec<u8>,
+    script: Vec<(String, Response)>,
+}
+
+impl MockImapServer {
+    pub fn new() -> Self {
+        Self {
+            greeting: b"* OK Mock IMAP server ready\r\n".to_vec(),
+            script: Vec::new(),
+        }
+    }
+
+    /// Set the greeting banner sent immediately after connection.
+    pub fn greeting(mut self, banner: impl Into<Vec<u8>>) -> Self {
+        self.greeting = banner.into();
+        self
+    }
+
+    /// Add a script step: when a client command *contains* `pattern` (case-insensitive),
+    /// respond with `response`. Steps are checked in insertion order.
+    pub fn respond(mut self, pattern: impl Into<String>, response: impl Into<Vec<u8>>) -> Self {
+        self.script.push((pattern.into(), response.into()));
+        self
+    }
+
+    /// Start the server on a random port. Returns a handle whose `addr()` gives
+    /// the `(host, port)` to connect to.
+    pub async fn start(self) -> MockImapServerHandle {
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
+        let addr = listener.local_addr().expect("local_addr");
+
+        let server = Arc::new(self);
+
+        tokio::spawn(async move {
+            loop {
+                match listener.accept().await {
+                    Ok((stream, _)) => {
+                        let srv = server.clone();
+                        tokio::spawn(async move {
+                            srv.handle_connection(stream).await;
+                        });
+                    }
+                    Err(_) => break,
+                }
+            }
+        });
+
+        MockImapServerHandle { addr }
+    }
+
+    async fn handle_connection(&self, mut stream: TcpStream) {
+        let (reader, mut writer) = stream.split();
+        let mut reader = BufReader::new(reader);
+
+        // Send greeting
+        if writer.write_all(&self.greeting).await.is_err() {
+            return;
+        }
+
+        let mut line = String::new();
+        loop {
+            line.clear();
+            match reader.read_line(&mut line).await {
+                Ok(0) => break, // EOF
+                Ok(_) => {}
+                Err(_) => break,
+            }
+
+            let tag = extract_tag(&line).unwrap_or("A0");
+            let matched = self.find_match(&line);
+            if let Some(response) = matched {
+                let substituted = substitute_tag(response, tag);
+                if writer.write_all(&substituted).await.is_err() {
+                    break;
+                }
+            } else {
+                // Default: send tagged OK for commands we don't handle
+                let fallback = format!("{tag} OK done\r\n");
+                if writer.write_all(fallback.as_bytes()).await.is_err() {
+                    break;
+                }
+            }
+        }
+    }
+
+    fn find_match(&self, line: &str) -> Option<&[u8]> {
+        let line_lower = line.to_lowercase();
+        for (pattern, response) in &self.script {
+            if line_lower.contains(&pattern.to_lowercase()) {
+                return Some(response);
+            }
+        }
+        None
+    }
+}
+
+impl Default for MockImapServer {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Handle to a running mock IMAP server. The server stops when this handle
+/// is dropped.
+pub struct MockImapServerHandle {
+    addr: SocketAddr,
+}
+
+impl MockImapServerHandle {
+    pub fn host(&self) -> String {
+        self.addr.ip().to_string()
+    }
+
+    pub fn port(&self) -> u16 {
+        self.addr.port()
+    }
+}
+
+fn extract_tag(line: &str) -> Option<&str> {
+    line.split_whitespace().next()
+}
+
+/// Replace `{TAG}` placeholders in `response` with `tag`.
+fn substitute_tag(response: &[u8], tag: &str) -> Vec<u8> {
+    let placeholder = b"{TAG}";
+    if response.is_empty() || !contains_slice(response, placeholder) {
+        return response.to_vec();
+    }
+    let tag_bytes = tag.as_bytes();
+    let mut result = Vec::with_capacity(response.len());
+    let mut pos = 0;
+    while let Some(idx) = find_slice(&response[pos..], placeholder) {
+        result.extend_from_slice(&response[pos..pos + idx]);
+        result.extend_from_slice(tag_bytes);
+        pos += idx + placeholder.len();
+    }
+    result.extend_from_slice(&response[pos..]);
+    result
+}
+
+fn contains_slice(haystack: &[u8], needle: &[u8]) -> bool {
+    haystack.windows(needle.len()).any(|w| w == needle)
+}
+
+fn find_slice(haystack: &[u8], needle: &[u8]) -> Option<usize> {
+    haystack
+        .windows(needle.len())
+        .position(|w| w == needle)
+}
+
+// ============================================================
+// Pre-built response helpers
+// ============================================================
+
+/// Build a tagged OK response.
+pub fn ok(tag: impl AsRef<str>, msg: impl AsRef<str>) -> Vec<u8> {
+    format!("{} OK {}\r\n", tag.as_ref(), msg.as_ref()).into_bytes()
+}
+
+/// Build a STATUS response line.
+pub fn status_response(
+    mailbox: &str,
+    messages: u32,
+    unseen: u32,
+    uid_next: u32,
+    uid_validity: Option<u32>,
+) -> Vec<u8> {
+    let uv = uid_validity
+        .map(|v| format!(" UIDVALIDITY {v}"))
+        .unwrap_or_default();
+    let text = format!(
+        "* STATUS \"{mailbox}\" (MESSAGES {messages} UNSEEN {unseen} UIDNEXT {uid_next}{uv})\r\n"
+    );
+    // Clients expect a tagged response after the untagged STATUS line.
+    // We produce a generic OK that works for any tag.
+    let mut out = text.into_bytes();
+    out.extend_from_slice(b"{TAG} OK STATUS completed\r\n");
+    out
+}
+
+/// Build an EXAMINE response with mailbox data.
+pub fn examine_response(
+    _mailbox: &str,
+    exists: u32,
+    uid_validity: u32,
+    uid_next: u32,
+) -> Vec<u8> {
+    format!(
+        "* FLAGS (\\Seen \\Answered \\Flagged \\Deleted \\Draft)\r\n\
+         * OK [PERMANENTFLAGS ()]\r\n\
+         * {exists} EXISTS\r\n\
+         * 0 RECENT\r\n\
+         * OK [UIDVALIDITY {uid_validity}]\r\n\
+         * OK [UIDNEXT {uid_next}]\r\n\
+         * OK [HIGHESTMODSEQ 1]\r\n\
+         {{TAG}} OK [READ-ONLY] EXAMINE completed\r\n"
+    )
+    .into_bytes()
+}
+
+/// Build a UID SEARCH response for the given UID list.
+pub fn uid_search_response(uids: &[u32]) -> Vec<u8> {
+    let uid_str = uids
+        .iter()
+        .map(|u| u.to_string())
+        .collect::<Vec<_>>()
+        .join(" ");
+    format!("* SEARCH {uid_str}\r\n{{TAG}} OK SEARCH completed\r\n").into_bytes()
+}
+
+/// Build a UID FETCH response returning UID + RFC822.SIZE + INTERNALDATE
+/// (no body). Each entry: (uid, size)
+pub fn uid_fetch_size_response(entries: &[(u32, u32)]) -> Vec<u8> {
+    let mut out = Vec::new();
+    for (uid, size) in entries {
+        let line = format!(
+            "* {uid} FETCH (UID {uid} RFC822.SIZE {size} INTERNALDATE \"01-Jan-2025 00:00:00 +0000\")\r\n"
+        );
+        out.extend_from_slice(line.as_bytes());
+    }
+    out.extend_from_slice(b"{TAG} OK FETCH completed\r\n");
+    out
+}
+
+/// Build a UID FETCH response returning full headers (for BODY[HEADER]).
+/// Each entry: (uid, message_id)
+pub fn uid_fetch_metadata_response(entries: &[(u32, &str)]) -> Vec<u8> {
+    let mut out = Vec::new();
+    for (uid, msg_id) in entries {
+        // Build a minimal header that contains the Message-ID line.
+        let header_data = format!(
+            "From: sender@example.com\r\n\
+To: recipient@example.com\r\n\
+Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\
+Subject: test\r\n\
+Message-ID: {msg_id}\r\n\r\n"
+        );
+        let header_len = header_data.len();
+        let line = format!(
+            "* {uid} FETCH (UID {uid} BODY[HEADER] {{{header_len}}}\r\n\
+{header_data}\
+)\r\n",
+        );
+        out.extend_from_slice(line.as_bytes());
+    }
+    out.extend_from_slice(b"{TAG} OK FETCH completed\r\n");
+    out
+}
+
+/// Build a UID FETCH RFC822 response with a full email body.
+pub fn uid_fetch_rfc822_response(uid: u32, eml: &[u8]) -> Vec<u8> {
+    let header = format!(
+        "* {uid} FETCH (UID {uid} RFC822 {{{len}}}\r\n",
+        len = eml.len()
+    );
+    let mut out = header.into_bytes();
+    out.extend_from_slice(eml);
+    out.extend_from_slice(b")\r\n{TAG} OK FETCH completed\r\n");
+    out
+}
+
+/// A minimal RFC822 email fixture for testing.
+pub fn minimal_eml(subject: &str, message_id: &str) -> Vec<u8> {
+    format!(
+        "From: sender@example.com\r\n\
+         To: recipient@example.com\r\n\
+         Subject: {subject}\r\n\
+         Message-ID: <{message_id}>\r\n\
+         Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\
+         MIME-Version: 1.0\r\n\
+         Content-Type: text/plain; charset=utf-8\r\n\
+         \r\n\
+         This is a test email: {subject}.\r\n"
+    )
+    .into_bytes()
+}
+
+// ============================================================
+// Self-tests for the mock server itself
+// ============================================================
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    async fn connect_and_read_greeting(host: &str, port: u16) -> String {
+        let mut stream = TcpStream::connect((host, port)).await.unwrap();
+        let (reader, _writer) = stream.split();
+        let mut reader = BufReader::new(reader);
+        let mut line = String::new();
+        reader.read_line(&mut line).await.unwrap();
+        line
+    }
+
+    async fn send_and_recv(host: &str, port: u16, cmd: &str) -> String {
+        let mut stream = TcpStream::connect((host, port)).await.unwrap();
+        let (reader, mut writer) = stream.split();
+        let mut reader = BufReader::new(reader);
+
+        // Read greeting
+        let mut line = String::new();
+        reader.read_line(&mut line).await.unwrap();
+
+        // Send command
+        writer.write_all(cmd.as_bytes()).await.unwrap();
+        writer.write_all(b"\r\n").await.unwrap();
+
+        // Read response (may be multi-line; read until tagged response)
+        let mut out = String::new();
+        loop {
+            line.clear();
+            reader.read_line(&mut line).await.unwrap();
+            out.push_str(&line);
+            if line.starts_with("A0") || line.starts_with("A1") {
+                break;
+            }
+        }
+        out
+    }
+
+    #[tokio::test]
+    async fn test_mock_greeting() {
+        let handle = MockImapServer::new().start().await;
+        let greeting = connect_and_read_greeting(&handle.host(), handle.port()).await;
+        assert!(greeting.starts_with("* OK"));
+    }
+
+    #[tokio::test]
+    async fn test_mock_scripted_response() {
+        let handle = MockImapServer::new()
+            .respond(
+                "LOGIN",
+                "A0 OK LOGIN completed\r\n",
+            )
+            .start()
+            .await;
+
+        let resp = send_and_recv(&handle.host(), handle.port(), "A0 LOGIN u p").await;
+        assert!(resp.contains("LOGIN completed"));
+    }
+
+    #[tokio::test]
+    async fn test_mock_fallback_on_unmatched() {
+        let handle = MockImapServer::new().start().await;
+
+        // Send a command that has no scripted response
+        let resp = send_and_recv(&handle.host(), handle.port(), "A0 NOOP").await;
+        assert!(resp.contains("OK done"), "unmatched command should get fallback OK");
+    }
+
+    #[tokio::test]
+    async fn test_status_response_helper() {
+        let resp = status_response("INBOX", 10, 2, 11, Some(42));
+        let text = String::from_utf8(resp).unwrap();
+        assert!(text.contains("MESSAGES 10"));
+        assert!(text.contains("UNSEEN 2"));
+        assert!(text.contains("UIDNEXT 11"));
+        assert!(text.contains("UIDVALIDITY 42"));
+    }
+
+    #[tokio::test]
+    async fn test_status_response_without_uidvalidity() {
+        let resp = status_response("INBOX", 10, 2, 11, None);
+        let text = String::from_utf8(resp).unwrap();
+        assert!(!text.contains("UIDVALIDITY"));
+        assert!(text.contains("MESSAGES 10"));
+    }
+
+    #[tokio::test]
+    async fn test_examine_response() {
+        let resp = examine_response("INBOX", 10, 42, 11);
+        let text = String::from_utf8(resp).unwrap();
+        assert!(text.contains("UIDVALIDITY 42"));
+        assert!(text.contains("10 EXISTS"));
+    }
+
+    #[tokio::test]
+    async fn test_uid_search_response() {
+        let resp = uid_search_response(&[1, 3, 5]);
+        let text = String::from_utf8(resp).unwrap();
+        assert!(text.contains("SEARCH 1 3 5"));
+    }
+
+    #[tokio::test]
+    async fn test_uid_fetch_metadata_response() {
+        let resp = uid_fetch_metadata_response(&[(1, "msg-a@x.com"), (2, "msg-b@x.com")]);
+        let text = String::from_utf8(resp).unwrap();
+        assert!(text.contains("Message-ID: msg-a@x.com"));
+        assert!(text.contains("Message-ID: msg-b@x.com"));
+    }
+
+    #[tokio::test]
+    async fn test_multiple_commands_in_sequence() {
+        let handle = MockImapServer::new()
+            .respond("LOGIN", "A0 OK LOGIN\r\n")
+            .respond("STATUS", status_response("INBOX", 5, 1, 6, Some(99)))
+            .respond("LOGOUT", "* BYE\r\nA0 OK\r\n")
+            .start()
+            .await;
+
+        let mut stream = TcpStream::connect((handle.host(), handle.port()))
+            .await
+            .unwrap();
+        let (reader, mut writer) = stream.split();
+        let mut reader = BufReader::new(reader);
+
+        // Read greeting
+        let mut buf = String::new();
+        reader.read_line(&mut buf).await.unwrap();
+
+        // LOGIN
+        writer.write_all(b"A0 LOGIN u p\r\n").await.unwrap();
+        buf.clear();
+        reader.read_line(&mut buf).await.unwrap();
+        assert!(buf.contains("LOGIN"));
+
+        // STATUS
+        writer
+            .write_all(b"A0 STATUS INBOX (MESSAGES UNSEEN UIDNEXT UIDVALIDITY)\r\n")
+            .await
+            .unwrap();
+        buf.clear();
+        // Read multi-line STATUS response (untagged line + tagged OK)
+        loop {
+            reader.read_line(&mut buf).await.unwrap();
+            if buf.contains("UIDVALIDITY 99") {
+                // Consume the tagged OK line that follows
+                buf.clear();
+                reader.read_line(&mut buf).await.unwrap();
+                break;
+            }
+        }
+
+        // LOGOUT
+        writer.write_all(b"A0 LOGOUT\r\n").await.unwrap();
+        buf.clear();
+        reader.read_line(&mut buf).await.unwrap();
+        assert!(buf.contains("BYE"));
+    }
+
+    #[tokio::test]
+    async fn test_tag_substitution_in_response() {
+        // Use {TAG} placeholder in the response and verify it gets the
+        // client's actual tag ("A5") substituted in.
+        let handle = MockImapServer::new()
+            .respond("LOGIN", "{TAG} OK LOGIN succeeded\r\n")
+            .start()
+            .await;
+
+        let mut stream = TcpStream::connect((handle.host(), handle.port()))
+            .await
+            .unwrap();
+        let (reader, mut writer) = stream.split();
+        let mut reader = BufReader::new(reader);
+
+        // Read greeting
+        let mut buf = String::new();
+        reader.read_line(&mut buf).await.unwrap();
+
+        // Send LOGIN with non-standard tag
+        writer.write_all(b"A5 LOGIN u p\r\n").await.unwrap();
+        buf.clear();
+        reader.read_line(&mut buf).await.unwrap();
+
+        assert!(
+            buf.contains("A5 OK LOGIN succeeded"),
+            "expected 'A5 OK LOGIN succeeded', got '{buf}'"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_tag_substitution_multiple_placeholders() {
+        let handle = MockImapServer::new()
+            .respond("NOOP", "* 0 RECENT\r\n{TAG} OK NOOP done\r\n")
+            .start()
+            .await;
+
+        let mut stream = TcpStream::connect((handle.host(), handle.port()))
+            .await
+            .unwrap();
+        let (reader, mut writer) = stream.split();
+        let mut reader = BufReader::new(reader);
+
+        // Read greeting
+        let mut buf = String::new();
+        reader.read_line(&mut buf).await.unwrap();
+
+        // Send with tag "B99"
+        writer.write_all(b"B99 NOOP\r\n").await.unwrap();
+
+        // Read all lines
+        let mut all = String::new();
+        loop {
+            buf.clear();
+            reader.read_line(&mut buf).await.unwrap();
+            all.push_str(&buf);
+            if buf.starts_with("B99") {
+                break;
+            }
+        }
+
+        assert!(all.contains("* 0 RECENT\r\n"));
+        assert!(all.contains("B99 OK NOOP done\r\n"));
+    }
+}

crates/core/src/imap/mod.rs

@@ -26,3 +26,5 @@ pub mod session;
 pub mod stats;
 #[cfg(test)]
 mod tests;
+#[cfg(test)]
+pub mod mock_server;

crates/core/src/import/history.rs

@@ -0,0 +1,127 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+
+use crate::database::MemDbModel;
+use crate::import::{ImportProgress, ImportStatus};
+use serde::{Deserialize, Serialize};
+
+/// Maximum number of import history entries to keep per user.
+pub const MAX_HISTORY_PER_USER: usize = 5;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct ImportHistory {
+    /// Composite key: "{user_id}:{import_id}"
+    pub id: String,
+    pub user_id: u64,
+    pub import_id: String,
+    pub account_id: u64,
+    pub folder: String,
+    pub format: String,
+    pub status: String,
+    pub total: usize,
+    pub success: usize,
+    pub duplicates: usize,
+    pub failed: usize,
+    pub failed_details: Vec<crate::import::FailedItemDetail>,
+    /// Unix timestamp in milliseconds.
+    pub created_at: i64,
+}
+
+impl MemDbModel for ImportHistory {
+    fn collection() -> &'static str {
+        "import_history"
+    }
+    fn key(&self) -> String {
+        self.id.clone()
+    }
+}
+
+impl ImportHistory {
+    pub fn from_progress(
+        user_id: u64,
+        import_id: &str,
+        account_id: u64,
+        folder: &str,
+        progress: &ImportProgress,
+    ) -> Self {
+        Self {
+            id: format!("{}:{}", user_id, import_id),
+            user_id,
+            import_id: import_id.to_string(),
+            account_id,
+            folder: folder.to_string(),
+            format: progress.format.clone(),
+            status: match progress.status {
+                ImportStatus::Pending => "pending",
+                ImportStatus::Processing => "processing",
+                ImportStatus::Completed => "completed",
+                ImportStatus::Failed => "failed",
+            }
+            .to_string(),
+            total: progress.total,
+            success: progress.success,
+            duplicates: progress.duplicates,
+            failed: progress.failed,
+            failed_details: progress.failed_details.clone(),
+            created_at: crate::utc_now!(),
+        }
+    }
+}
+
+/// Prune old entries for a user so only the latest `MAX_HISTORY_PER_USER` remain.
+pub fn prune_user_history(user_id: u64) -> crate::error::BichonResult<()> {
+    use crate::database::manager::DB_MANAGER;
+    use crate::database::batch_delete_impl;
+    use crate::raise_error;
+    use crate::error::code::ErrorCode;
+    let db = DB_MANAGER.db();
+    let coll = db.collection(ImportHistory::collection());
+    let prefix = format!("{}:", user_id);
+
+    let mut entries: Vec<ImportHistory> = coll
+        .scan_prefix(&prefix)
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+    if entries.len() <= MAX_HISTORY_PER_USER {
+        return Ok(());
+    }
+
+    // Sort by created_at descending (newest first), keep the first N
+    entries.sort_by(|a, b| b.created_at.cmp(&a.created_at));
+    let to_delete: Vec<String> = entries
+        .iter()
+        .skip(MAX_HISTORY_PER_USER)
+        .map(|e| e.id.clone())
+        .collect();
+
+    if !to_delete.is_empty() {
+        batch_delete_impl::<ImportHistory>(db, to_delete)?;
+    }
+
+    Ok(())
+}
+
+/// Save an import history record and prune old entries for the user.
+pub fn save_import_history(
+    user_id: u64,
+    account_id: u64,
+    folder: &str,
+    progress: &ImportProgress,
+) {
+    use crate::database::manager::DB_MANAGER;
+    use crate::database::upsert_impl;
+
+    let entry = ImportHistory::from_progress(user_id, &progress.import_id, account_id, folder, progress);
+    let db = DB_MANAGER.db();
+
+    if let Err(e) = upsert_impl::<ImportHistory>(db, entry) {
+        tracing::error!("Failed to save import history: {:?}", e);
+        return;
+    }
+
+    if let Err(e) = prune_user_history(user_id) {
+        tracing::warn!("Failed to prune import history: {:?}", e);
+    }
+}

crates/core/src/import/mod.rs

@@ -1,4 +1,3 @@
-//
 // Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
 //
 // This file is part of the Bichon Email Archiving Project
@@ -16,54 +15,65 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+// use poem_openapi::Object;
+pub mod history;
+pub mod pst;
+pub mod reader;
+use std::{collections::HashMap, path::Path, sync::RwLock};
 
-//use poem_openapi::Object;
+pub use history::ImportHistory;
 use serde::{Deserialize, Serialize};
 
 use crate::{
+    account::migration::{AccountModel, AccountType},
     base64_decode_url_safe,
-    {
-        account::migration::{AccountModel, AccountType},
-        cache::imap::mailbox::{Attribute, AttributeEnum, MailBox},
-        envelope::extractor::extract_envelope_from_eml,
-        error::{BichonResult, code::ErrorCode},
-        utils::create_hash,
-    },
+    archive::imap::mailbox::{Attribute, AttributeEnum, MailBox},
+    envelope::extractor::{extract_envelope_from_eml, ExtractOutcome},
+    error::{code::ErrorCode, BichonResult},
     raise_error,
+    settings::dir::DATA_DIR_MANAGER,
+    utils::create_hash,
 };
 
-/// Skip individual emails larger than this after decoding (100 MB).
+/// Maximum byte size of an individual email message after splitting (100 MB).
 const MAX_SINGLE_EML_BYTES: usize = 100 * 1024 * 1024;
 
-#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
+/// Max file size accepted via the web upload endpoint.
+pub const MAX_WEB_EML_BYTES: usize = 100 * 1024 * 1024; // 100 MB
+pub const MAX_WEB_MBOX_BYTES: usize = 1024 * 1024 * 1024; // 1 GB
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct BatchEmlRequest {
     pub account_id: u64,
     pub mail_folder: String,
-    /// A list of emails in base64-encoded format. Each element represents one .eml file.
+    /// A list of emails in base64-encoded format. Each element represents one
+    /// .eml file.
     pub emls: Vec<String>,
 }
 
 #[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
 #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
-pub struct FailedEmlDetail {
-    /// The 0-based index of the failed EML in the request list
+pub struct FailedItemDetail {
+    /// The index (0-based) of the failed item.
     pub index: usize,
-    /// The error message that caused the import to fail
+    /// The error message that caused the import to fail.
     pub error_message: String,
 }
 
 #[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
 #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct BatchEmlResult {
-    /// Total number of emails processed
+    /// Total number of emails processed.
     pub total: usize,
-    /// Number of emails successfully imported
+    /// Number of emails successfully imported.
     pub success: usize,
-    /// Number of emails failed to import
+    /// Number of duplicate emails skipped (content hash already existed).
+    pub duplicates: usize,
+    /// Number of emails failed to import.
     pub failed: usize,
-    /// A list of details for failed imports
-    pub failed_details: Vec<FailedEmlDetail>,
+    /// A list of details for failed imports.
+    pub failed_details: Vec<FailedItemDetail>,
 }
 
 pub struct ImportEmls;
@@ -71,26 +81,31 @@ pub struct ImportEmls;
 impl ImportEmls {
     pub async fn do_import(mut request: BatchEmlRequest) -> BichonResult<BatchEmlResult> {
         let account = AccountModel::check_account_exists(request.account_id)?;
-        
+
         if !account.enabled {
-            return Err(raise_error!("The account is disabled and cannot be used for this operation.".into(), ErrorCode::InvalidParameter));
+            return Err(raise_error!(
+                "The account is disabled and cannot be used for this operation.".into(),
+                ErrorCode::InvalidParameter
+            ));
         }
 
         let mailbox_id = match account.account_type {
             AccountType::IMAP => {
                 let all_mailboxes = MailBox::list_all(account.id)?;
-                let mailbox = all_mailboxes.into_iter().find(|m| m.name == request.mail_folder);
-                
+                let mailbox = all_mailboxes
+                    .into_iter()
+                    .find(|m| m.name == request.mail_folder);
+
                 match mailbox {
                     Some(mailbox) => mailbox.id,
                     None => return Err(raise_error!(
-                        format!("Mail folder '{}' not found for account ID {}. The target folder must exist before importing.", 
-                                request.mail_folder, 
+                        format!("Mail folder '{}' not found for account ID {}. The target folder must exist before importing.",
+                                request.mail_folder,
                                 request.account_id).into(),
                         ErrorCode::ResourceNotFound
                     )),
                 }
-            },
+            }
             AccountType::NoSync => {
                 let mailbox = MailBox {
                     id: create_hash(request.account_id, &request.mail_folder),
@@ -111,12 +126,13 @@ impl ImportEmls {
                 // Upsert the mailbox, creating it if it doesn't exist
                 MailBox::batch_upsert(&[mailbox])?;
                 mailbox_id
-            },
+            }
         };
 
         let account_id = account.id;
         let mut success_count = 0;
-        let mut failed_details: Vec<FailedEmlDetail> = Vec::new(); // Store failure details
+        let mut duplicate_count = 0;
+        let mut failed_details: Vec<FailedItemDetail> = Vec::new(); // Store failure details
 
         let total = request.emls.len();
         let mut index: usize = 0;
@@ -127,7 +143,7 @@ impl ImportEmls {
                     let error_msg =
                         format!("Failed to decode base64 EML at index {}: {:?}", index, e);
                     tracing::error!("{}", error_msg);
-                    failed_details.push(FailedEmlDetail {
+                    failed_details.push(FailedItemDetail {
                         index,
                         error_message: error_msg,
                     });
@@ -144,7 +160,7 @@ impl ImportEmls {
                     index, size_mb,
                 );
                 tracing::warn!("{}", error_msg);
-                failed_details.push(FailedEmlDetail {
+                failed_details.push(FailedItemDetail {
                     index,
                     error_message: error_msg,
                 });
@@ -153,16 +169,19 @@ impl ImportEmls {
             }
 
             match extract_envelope_from_eml(&decoded, account_id, mailbox_id).await {
-                Ok(_) => {
+                Ok(ExtractOutcome::Imported) => {
                     success_count += 1;
-                },
+                }
+                Ok(ExtractOutcome::Duplicate) => {
+                    duplicate_count += 1;
+                }
                 Err(e) => {
                     let error_msg = format!(
                         "Failed to extract envelope from EML at index {}: {:?}",
                         index, e
                     );
                     tracing::error!("{}", error_msg);
-                    failed_details.push(FailedEmlDetail {
+                    failed_details.push(FailedItemDetail {
                         index,
                         error_message: error_msg,
                     });
@@ -178,8 +197,742 @@ impl ImportEmls {
         Ok(BatchEmlResult {
             total,
             success: success_count,
+            duplicates: duplicate_count,
             failed: failed_count,
             failed_details, // Return the list of failure details
         })
     }
 }
+
+// ── File upload import ──────────────────────────────────────────────
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))]
+pub enum ImportStatus {
+    Pending,
+    Processing,
+    Completed,
+    Failed,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct ImportProgress {
+    pub import_id: String,
+    pub status: ImportStatus,
+    pub format: String,
+    pub total: usize,
+    pub success: usize,
+    pub duplicates: usize,
+    pub failed: usize,
+    pub failed_details: Vec<FailedItemDetail>,
+}
+
+static PROGRESS_STORE: std::sync::LazyLock<RwLock<HashMap<String, ImportProgress>>> =
+    std::sync::LazyLock::new(|| RwLock::new(HashMap::new()));
+
+pub fn get_import_progress(import_id: &str) -> Option<ImportProgress> {
+    PROGRESS_STORE.read().ok()?.get(import_id).cloned()
+}
+
+pub fn update_progress(import_id: &str, progress: ImportProgress) {
+    if let Ok(mut store) = PROGRESS_STORE.write() {
+        store.insert(import_id.to_string(), progress);
+    }
+}
+
+/// Check free disk space (in bytes) on the temp directory's filesystem.
+pub fn check_temp_disk_space() -> BichonResult<u64> {
+    use sysinfo::Disks;
+    let disks = Disks::new_with_refreshed_list();
+    let temp_path = &DATA_DIR_MANAGER.temp_dir;
+    // Use the canonical path so we can match mount points
+    let canonical = std::fs::canonicalize(temp_path).unwrap_or_else(|_| temp_path.clone());
+    for disk in disks.list() {
+        if canonical.starts_with(disk.mount_point()) {
+            return Ok(disk.available_space());
+        }
+    }
+    // Fallback: if we can't find the mount point, report plenty of space
+    Ok(u64::MAX)
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FileFormat {
+    Eml,
+    Mbox,
+    Pst,
+}
+
+pub fn detect_format(bytes: &[u8], file_name: &str) -> Option<FileFormat> {
+    // PST files start with OLE2 compound document magic bytes
+    if bytes.len() >= 8 && &bytes[..8] == b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1" {
+        return Some(FileFormat::Pst);
+    }
+
+    // MBOX files start with "From " (note the trailing space after From)
+    if bytes.starts_with(b"From ") {
+        // Double-check: look for a valid date after the first "From " line
+        // MBOX format: "From sender@host DayOfWeek Mon DD HH:MM:SS YYYY"
+        if let Some(first_newline) = bytes.iter().position(|&b| b == b'\n') {
+            let from_line = std::str::from_utf8(&bytes[..first_newline]).unwrap_or("");
+            let parts: Vec<&str> = from_line.split_whitespace().collect();
+            if parts.len() >= 7 {
+                return Some(FileFormat::Mbox);
+            }
+        }
+    }
+    // EML: starts with a header line or "Return-Path:", "Received:", "From:",
+    // "Date:", etc. Or check extension
+    if bytes.starts_with(b"Return-Path:")
+        || bytes.starts_with(b"Received:")
+        || bytes.starts_with(b"Date:")
+        || bytes.starts_with(b"From:")
+        || bytes.starts_with(b"Subject:")
+        || bytes.starts_with(b"To:")
+        || bytes.starts_with(b"Message-ID:")
+    {
+        return Some(FileFormat::Eml);
+    }
+    // Fallback: check file extension
+    let lower = file_name.to_lowercase();
+    if lower.ends_with(".eml") {
+        Some(FileFormat::Eml)
+    } else if lower.ends_with(".mbox") {
+        Some(FileFormat::Mbox)
+    } else if lower.ends_with(".pst") {
+        Some(FileFormat::Pst)
+    } else {
+        None
+    }
+}
+
+/// Check whether `bytes` looks like a text file by inspecting the first chunk.
+/// Returns `true` if it passes, `false` if it appears to be binary (video,
+/// executable, etc.).
+///
+/// Email files (EML/MBOX) are text-based with printable ASCII, whitespace, and
+/// optional UTF-8. Binary files like video contain null bytes and high ratios
+/// of non-printable control characters.
+pub fn detect_text_file(bytes: &[u8]) -> bool {
+    let check_len = bytes.len().min(8192);
+    if check_len == 0 {
+        return false;
+    }
+    let sample = &bytes[..check_len];
+
+    // Null bytes are a strong binary indicator
+    if sample.contains(&0x00) {
+        return false;
+    }
+
+    let mut printable = 0usize;
+    let mut total = 0usize;
+
+    let mut i = 0;
+    while i < sample.len() {
+        total += 1;
+        let b = sample[i];
+
+        if b.is_ascii_graphic() || b.is_ascii_whitespace() {
+            // Printable ASCII + whitespace (space, tab, CR, LF)
+            printable += 1;
+        } else if b == 0x1b {
+            // ESC — common in terminal sequences, rare in email
+            // Count as printable to avoid false positives
+            printable += 1;
+        } else if b >= 0x80 {
+            // UTF-8 continuation or multi-byte lead byte — allow.
+            // Check that we have a valid UTF-8 sequence ahead.
+            let seq_len = match b {
+                b if b & 0xE0 == 0xC0 => 2,
+                b if b & 0xF0 == 0xE0 => 3,
+                b if b & 0xF8 == 0xF0 => 4,
+                _ => 0,
+            };
+            if seq_len > 0 && i + seq_len <= sample.len() {
+                let valid = std::str::from_utf8(&sample[i..i + seq_len]).is_ok();
+                if valid {
+                    printable += 1;
+                    i += 1; // lead byte counted, continuations counted in loop
+                }
+                // if invalid, don't count as printable
+            }
+            // standalone continuation byte — not printable
+        }
+        // Other control characters (0x01-0x1F except whitespace/Esc) are not counted as
+        // printable
+
+        i += 1;
+    }
+
+    // Require at least 90% printable characters
+    printable as f64 / total as f64 >= 0.90
+}
+
+/// Validate that the target account exists, is enabled, and is NoSync type.
+fn validate_import_account(account_id: u64) -> BichonResult<AccountModel> {
+    let account = AccountModel::check_account_exists(account_id)?;
+    if !account.enabled {
+        return Err(raise_error!(
+            "The account is disabled.".into(),
+            ErrorCode::InvalidParameter
+        ));
+    }
+    if !matches!(account.account_type, AccountType::NoSync) {
+        return Err(raise_error!(
+            "Import is only allowed for NoSync accounts. IMAP accounts sync from the server."
+                .into(),
+            ErrorCode::InvalidParameter
+        ));
+    }
+    Ok(account)
+}
+
+/// Resolve or create a mailbox/folder for the given account.
+pub(super) fn resolve_mailbox(account: &AccountModel, folder: &str) -> BichonResult<u64> {
+    match account.account_type {
+        AccountType::IMAP => {
+            // Shouldn't reach here (validated above), but handle gracefully
+            let all_mailboxes = MailBox::list_all(account.id)?;
+            let mailbox = all_mailboxes.into_iter().find(|m| m.name == folder);
+            match mailbox {
+                Some(m) => Ok(m.id),
+                None => Err(raise_error!(
+                    format!("Mail folder '{}' not found.", folder).into(),
+                    ErrorCode::ResourceNotFound
+                )),
+            }
+        }
+        AccountType::NoSync => {
+            let mailbox = MailBox {
+                id: create_hash(account.id, folder),
+                account_id: account.id,
+                name: folder.to_string(),
+                delimiter: Some("/".to_string()),
+                attributes: vec![Attribute {
+                    attr: AttributeEnum::Extension,
+                    extension: Some("CreatedByBichon".into()),
+                }],
+                exists: 0,
+                unseen: None,
+                uid_next: None,
+                uid_validity: None,
+                highest_uid: None,
+            };
+            let mailbox_id = mailbox.id;
+            MailBox::batch_upsert(&[mailbox])?;
+            Ok(mailbox_id)
+        }
+    }
+}
+
+/// Resolve or create a mailbox for a given account_id and folder name.
+/// Used by PST import to create per-folder mailboxes.
+pub fn resolve_mailbox_by_account_id(account_id: u64, folder: &str) -> BichonResult<u64> {
+    let account = AccountModel::check_account_exists(account_id)?;
+    resolve_mailbox(&account, folder)
+}
+
+/// Process an uploaded file (EML or MBOX) and import into the given
+/// account/folder. This runs synchronously and should be spawned on a
+/// background thread.
+///
+/// For MBOX files, the file is memory-mapped via `memmap2` and messages are
+/// yielded one at a time — the full file is never loaded into RAM. Individual
+/// messages exceeding `MAX_SINGLE_EML_BYTES` (100 MB) are skipped.
+pub fn process_uploaded_file(
+    import_id: &str,
+    file_path: &Path,
+    file_name: &str,
+    account_id: u64,
+    folder: &str,
+    user_id: u64,
+) {
+    let account = match validate_import_account(account_id) {
+        Ok(a) => a,
+        Err(e) => {
+            let progress = ImportProgress {
+                import_id: import_id.to_string(),
+                status: ImportStatus::Failed,
+                format: "unknown".to_string(),
+                total: 0,
+                success: 0,
+                duplicates: 0,
+                failed: 0,
+                failed_details: vec![FailedItemDetail {
+                    index: 0,
+                    error_message: format!("Account validation failed: {:?}", e),
+                }],
+            };
+            update_progress(import_id, progress.clone());
+            history::save_import_history(user_id, account_id, folder, &progress);
+            return;
+        }
+    };
+
+    let mailbox_id = match resolve_mailbox(&account, folder) {
+        Ok(id) => id,
+        Err(e) => {
+            let progress = ImportProgress {
+                import_id: import_id.to_string(),
+                status: ImportStatus::Failed,
+                format: "unknown".to_string(),
+                total: 0,
+                success: 0,
+                duplicates: 0,
+                failed: 0,
+                failed_details: vec![FailedItemDetail {
+                    index: 0,
+                    error_message: format!("Mailbox resolution failed: {:?}", e),
+                }],
+            };
+            update_progress(import_id, progress.clone());
+            history::save_import_history(user_id, account_id, folder, &progress);
+            return;
+        }
+    };
+
+    // Read a small prefix for format detection
+    let format = match detect_format_from_file(file_path, file_name) {
+        Ok(f) => f,
+        Err(e) => {
+            let progress = ImportProgress {
+                import_id: import_id.to_string(),
+                status: ImportStatus::Failed,
+                format: "unknown".to_string(),
+                total: 0,
+                success: 0,
+                duplicates: 0,
+                failed: 0,
+                failed_details: vec![FailedItemDetail {
+                    index: 0,
+                    error_message: format!("{:?}", e),
+                }],
+            };
+            update_progress(import_id, progress.clone());
+            history::save_import_history(user_id, account_id, folder, &progress);
+            let _ = std::fs::remove_file(file_path);
+            return;
+        }
+    };
+
+    match format {
+        FileFormat::Eml => process_eml_file(
+            import_id, file_path, account_id, mailbox_id, user_id, folder,
+        ),
+        FileFormat::Mbox => process_mbox_file(
+            import_id, file_path, account_id, mailbox_id, user_id, folder,
+        ),
+        FileFormat::Pst => process_pst_upload(
+            import_id, file_path, account_id, mailbox_id, user_id, folder,
+        ),
+    }
+}
+
+/// Detect format from a file by reading only the first few KB.
+fn detect_format_from_file(file_path: &Path, file_name: &str) -> BichonResult<FileFormat> {
+    use std::io::Read;
+    let mut file = std::fs::File::open(file_path).map_err(|e| {
+        raise_error!(
+            format!("Failed to open file: {}", e),
+            ErrorCode::InternalError
+        )
+    })?;
+    let mut buf = vec![0u8; 8192];
+    let n = file.read(&mut buf).unwrap_or(0);
+    buf.truncate(n);
+
+    detect_format(&buf, file_name).ok_or_else(|| {
+        raise_error!(
+            "Unknown file format. Supported: .eml, .mbox, .pst".into(),
+            ErrorCode::InvalidParameter
+        )
+    })
+}
+
+/// Process a single EML file. The file is at most `MAX_WEB_EML_BYTES` (100 MB),
+/// so reading it entirely is safe.
+fn process_eml_file(
+    import_id: &str,
+    file_path: &Path,
+    account_id: u64,
+    mailbox_id: u64,
+    user_id: u64,
+    folder: &str,
+) {
+    let file_bytes = match std::fs::read(file_path) {
+        Ok(b) => b,
+        Err(e) => {
+            fail_progress(
+                import_id,
+                "eml",
+                &format!("Failed to read file: {}", e),
+                user_id,
+                account_id,
+                folder,
+            );
+            let _ = std::fs::remove_file(file_path);
+            return;
+        }
+    };
+
+    let total = 1;
+    update_progress(
+        import_id,
+        ImportProgress {
+            import_id: import_id.to_string(),
+            status: ImportStatus::Processing,
+            format: "eml".to_string(),
+            total,
+            success: 0,
+            duplicates: 0,
+            failed: 0,
+            failed_details: vec![],
+        },
+    );
+
+    let (success_count, duplicate_count, failed_details) =
+        process_single_eml(&file_bytes, 0, account_id, mailbox_id);
+
+    // Clean up
+    let _ = std::fs::remove_file(file_path);
+
+    let final_progress = ImportProgress {
+        import_id: import_id.to_string(),
+        status: ImportStatus::Completed,
+        format: "eml".to_string(),
+        total,
+        success: success_count,
+        duplicates: duplicate_count,
+        failed: failed_details.len(),
+        failed_details,
+    };
+    history::save_import_history(user_id, account_id, folder, &final_progress);
+    update_progress(import_id, final_progress);
+}
+
+/// Process an MBOX file using memory-mapped I/O. Messages are yielded one at a
+/// time by `MboxReader` — the full file is never loaded into RAM.
+fn process_mbox_file(
+    import_id: &str,
+    file_path: &Path,
+    account_id: u64,
+    mailbox_id: u64,
+    user_id: u64,
+    folder: &str,
+) {
+    let mbox = match reader::MboxFile::from_file(file_path) {
+        Ok(m) => m,
+        Err(e) => {
+            fail_progress(
+                import_id,
+                "mbox",
+                &format!("Failed to open MBOX file: {}", e),
+                user_id,
+                account_id,
+                folder,
+            );
+            let _ = std::fs::remove_file(file_path);
+            return;
+        }
+    };
+
+    // First pass: count total messages (MboxReader is lazy, so this is O(n) but
+    // cheap)
+    let total = mbox.iter().count();
+
+    update_progress(
+        import_id,
+        ImportProgress {
+            import_id: import_id.to_string(),
+            status: ImportStatus::Processing,
+            format: "mbox".to_string(),
+            total,
+            success: 0,
+            duplicates: 0,
+            failed: 0,
+            failed_details: vec![],
+        },
+    );
+
+    let mut success_count = 0usize;
+    let mut duplicate_count = 0usize;
+    let mut failed_details: Vec<FailedItemDetail> = Vec::new();
+
+    for (index, entry) in mbox.iter().enumerate() {
+        let eml_bytes = entry.data;
+
+        if eml_bytes.len() > MAX_SINGLE_EML_BYTES {
+            let size_mb = eml_bytes.len() as f64 / 1024.0 / 1024.0;
+            failed_details.push(FailedItemDetail {
+                index,
+                error_message: format!(
+                    "Email at index {} is {:.1} MB (limit {} MB). Skipping.",
+                    index,
+                    size_mb,
+                    MAX_SINGLE_EML_BYTES / 1024 / 1024
+                ),
+            });
+            continue;
+        }
+
+        match futures::executor::block_on(extract_envelope_from_eml(
+            eml_bytes, account_id, mailbox_id,
+        )) {
+            Ok(ExtractOutcome::Imported) => {
+                success_count += 1;
+            }
+            Ok(ExtractOutcome::Duplicate) => {
+                duplicate_count += 1;
+            }
+            Err(e) => {
+                failed_details.push(FailedItemDetail {
+                    index,
+                    error_message: format!("{:?}", e),
+                });
+            }
+        };
+
+        // Update progress every 100 items
+        if index % 100 == 0 || index == total - 1 {
+            update_progress(
+                import_id,
+                ImportProgress {
+                    import_id: import_id.to_string(),
+                    status: ImportStatus::Processing,
+                    format: "mbox".to_string(),
+                    total,
+                    success: success_count,
+                    duplicates: duplicate_count,
+                    failed: failed_details.len(),
+                    failed_details: failed_details.clone(),
+                },
+            );
+        }
+    }
+
+    // Clean up temp file (drop the mmap first — MboxFile owns it)
+    drop(mbox);
+    let _ = std::fs::remove_file(file_path);
+
+    let final_progress = ImportProgress {
+        import_id: import_id.to_string(),
+        status: ImportStatus::Completed,
+        format: "mbox".to_string(),
+        total,
+        success: success_count,
+        duplicates: duplicate_count,
+        failed: failed_details.len(),
+        failed_details,
+    };
+    history::save_import_history(user_id, account_id, folder, &final_progress);
+    update_progress(import_id, final_progress);
+}
+
+/// Process a single EML byte slice and return (success_count, duplicate_count,
+/// failed_details).
+fn process_single_eml(
+    eml_bytes: &[u8],
+    index: usize,
+    account_id: u64,
+    mailbox_id: u64,
+) -> (usize, usize, Vec<FailedItemDetail>) {
+    if eml_bytes.len() > MAX_SINGLE_EML_BYTES {
+        let size_mb = eml_bytes.len() as f64 / 1024.0 / 1024.0;
+        return (
+            0,
+            0,
+            vec![FailedItemDetail {
+                index,
+                error_message: format!(
+                    "Email is {:.1} MB (limit {} MB). Skipping.",
+                    size_mb,
+                    MAX_SINGLE_EML_BYTES / 1024 / 1024
+                ),
+            }],
+        );
+    }
+
+    match futures::executor::block_on(extract_envelope_from_eml(eml_bytes, account_id, mailbox_id))
+    {
+        Ok(ExtractOutcome::Imported) => (1, 0, vec![]),
+        Ok(ExtractOutcome::Duplicate) => (0, 1, vec![]),
+        Err(e) => (
+            0,
+            0,
+            vec![FailedItemDetail {
+                index,
+                error_message: format!("{:?}", e),
+            }],
+        ),
+    }
+}
+
+/// Process a PST file uploaded via the web UI.
+/// Two-pass approach: count messages first, then process with periodic progress
+/// updates.
+fn process_pst_upload(
+    import_id: &str,
+    file_path: &Path,
+    account_id: u64,
+    _mailbox_id: u64, // ignored; PST creates its own mailboxes per folder
+    user_id: u64,
+    folder: &str,
+) {
+    // Pass 1: count total messages
+    let total = match pst::count_pst_messages(file_path) {
+        Ok(n) => n,
+        Err(e) => {
+            fail_progress(
+                import_id,
+                "pst",
+                &format!("{:?}", e),
+                user_id,
+                account_id,
+                folder,
+            );
+            let _ = std::fs::remove_file(file_path);
+            return;
+        }
+    };
+
+    update_progress(
+        import_id,
+        ImportProgress {
+            import_id: import_id.to_string(),
+            status: ImportStatus::Processing,
+            format: "pst".to_string(),
+            total,
+            success: 0,
+            duplicates: 0,
+            failed: 0,
+            failed_details: vec![],
+        },
+    );
+
+    // Pass 2: process messages with progress updates
+    let mut success_count: usize = 0;
+    let mut duplicate_count: usize = 0;
+    let mut failed_details: Vec<FailedItemDetail> = Vec::new();
+    let mut index: usize = 0;
+
+    let pst_store = match outlook_pst::open_store(file_path) {
+        Ok(s) => s,
+        Err(e) => {
+            fail_progress(
+                import_id,
+                "pst",
+                &format!("{:?}", e),
+                user_id,
+                account_id,
+                folder,
+            );
+            let _ = std::fs::remove_file(file_path);
+            return;
+        }
+    };
+
+    let ipm_sub_tree = match pst_store.properties().ipm_sub_tree_entry_id() {
+        Ok(id) => id,
+        Err(e) => {
+            fail_progress(
+                import_id,
+                "pst",
+                &format!("{:?}", e),
+                user_id,
+                account_id,
+                folder,
+            );
+            let _ = std::fs::remove_file(file_path);
+            return;
+        }
+    };
+
+    let ipm_subtree_folder = match pst_store.open_folder(&ipm_sub_tree) {
+        Ok(f) => f,
+        Err(e) => {
+            fail_progress(
+                import_id,
+                "pst",
+                &format!("{:?}", e),
+                user_id,
+                account_id,
+                folder,
+            );
+            let _ = std::fs::remove_file(file_path);
+            return;
+        }
+    };
+
+    // Progress callback: update progress every 50 messages
+    let import_id = import_id.to_string();
+    let format_str = "pst".to_string();
+    pst::process_folder_with_progress(
+        &ipm_subtree_folder,
+        "", // parent_path starts empty
+        account_id,
+        total, // pass pre-counted total for accurate progress
+        &mut success_count,
+        &mut duplicate_count,
+        &mut failed_details,
+        &mut index,
+        &|success, duplicates, actual_failed| {
+            update_progress(
+                &import_id,
+                ImportProgress {
+                    import_id: import_id.clone(),
+                    status: ImportStatus::Processing,
+                    format: format_str.clone(),
+                    total,
+                    success,
+                    duplicates,
+                    failed: actual_failed,
+                    failed_details: vec![],
+                },
+            );
+        },
+    );
+
+    // Clean up temp file
+    let _ = std::fs::remove_file(file_path);
+
+    let final_progress = ImportProgress {
+        import_id: import_id.to_string(),
+        status: ImportStatus::Completed,
+        format: "pst".to_string(),
+        total,
+        success: success_count,
+        duplicates: duplicate_count,
+        failed: failed_details.len(),
+        failed_details,
+    };
+    history::save_import_history(user_id, account_id, folder, &final_progress);
+    update_progress(&import_id, final_progress);
+}
+
+/// Record a fatal failure and save history.
+fn fail_progress(
+    import_id: &str,
+    format: &str,
+    message: &str,
+    user_id: u64,
+    account_id: u64,
+    folder: &str,
+) {
+    let progress = ImportProgress {
+        import_id: import_id.to_string(),
+        status: ImportStatus::Failed,
+        format: format.to_string(),
+        total: 0,
+        success: 0,
+        duplicates: 0,
+        failed: 0,
+        failed_details: vec![FailedItemDetail {
+            index: 0,
+            error_message: message.to_string(),
+        }],
+    };
+    update_progress(import_id, progress.clone());
+    history::save_import_history(user_id, account_id, folder, &progress);
+}

crates/core/src/import/pst/encoding/mod.rs

@@ -16,8 +16,6 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
-use compressed_rtf::*;
 use outlook_pst::ltp::prop_context::PropertyValue;
 
 pub fn decode_subject(value: &PropertyValue) -> Option<String> {
@@ -60,5 +58,5 @@ pub fn decode_html_body(buffer: &[u8], code_page: u16) -> Option<String> {
 }
 
 pub fn decode_rtf_compressed(buffer: &[u8]) -> Option<String> {
-    decompress_rtf(buffer).ok()
+    compressed_rtf::decompress_rtf(buffer).ok()
 }

crates/core/src/import/pst/mod.rs

@@ -0,0 +1,505 @@
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::rc::Rc;
+
+use chrono::{DateTime, TimeZone, Utc};
+use mail_send::mail_builder::{headers::text::Text, MessageBuilder};
+use outlook_pst::{
+    ltp::prop_context::PropertyValue,
+    messaging::{
+        attachment::AttachmentProperties,
+        folder::Folder,
+        message::{Message, MessageProperties},
+    },
+    ndb::node_id::NodeId,
+};
+
+use crate::{
+    base64_encode_url_safe,
+    envelope::extractor::{extract_envelope_from_eml, ExtractOutcome},
+};
+
+mod encoding;
+
+/// Convert a PST Message into a base64-encoded EML string.
+pub fn build_eml_base64(message: Rc<dyn Message>) -> Option<String> {
+    let properties = message.properties();
+
+    let mut builder = MessageBuilder::new();
+    if let Some(sub) = extract_subject(properties) {
+        builder = builder.subject(sub);
+    }
+    if let Some(mid) = extract_string_property(properties, 0x1035) {
+        builder = builder.message_id(mid);
+    }
+    if let Some(irt) = extract_string_property(properties, 0x1042) {
+        builder = builder.in_reply_to(irt);
+    }
+
+    if let Some(refs) = extract_string_property(properties, 0x1039) {
+        builder = builder.header("References", Text::new(refs));
+    }
+
+    if let Some(cid_val) = properties.get(0x3013) {
+        if let PropertyValue::Binary(bin) = cid_val {
+            builder = builder.header(
+                "X-Bichon-Conversation-ID",
+                Text::new(hex::encode(bin.buffer())),
+            );
+        }
+    }
+
+    let from = extract_string_property(properties, 0x5D01)
+        .or_else(|| extract_string_property(properties, 0x5D02))
+        .or_else(|| extract_string_property(properties, 0x0C1F));
+
+    if let Some(f) = from {
+        builder = builder.from(f);
+    }
+
+    if let Some(filetime) = extract_i64_property(properties, &[0x0039, 0x0E06]) {
+        let dt = filetime_to_datetime(filetime).timestamp();
+        builder = builder.date(dt);
+    }
+
+    let (to, cc, bcc) = extract_recipients_list(&message);
+    if !to.is_empty() {
+        builder = builder.to(to.iter().map(|s| s.as_str()).collect::<Vec<_>>());
+    }
+    if !cc.is_empty() {
+        builder = builder.cc(cc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
+    }
+    if !bcc.is_empty() {
+        builder = builder.bcc(bcc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
+    }
+
+    if let Some(html) = extract_html(properties) {
+        builder = builder.html_body(html);
+    }
+
+    if let Some(text) = extract_text(properties) {
+        builder = builder.text_body(text);
+    }
+
+    if let Some(attachment_table) = message.attachment_table() {
+        for row in attachment_table.rows_matrix() {
+            let node_id = NodeId::from(u32::from(row.id()));
+            if let Ok(attachment) = message.clone().read_attachment(node_id, None) {
+                let att_props = attachment.properties();
+                let name = extract_attachment_string_property(att_props, 0x3707);
+                let mime = extract_attachment_string_property(att_props, 0x370E)
+                    .unwrap_or_else(|| "application/octet-stream".into());
+                let cid = extract_attachment_string_property(att_props, 0x3712);
+                let is_inline = att_props
+                    .get(0x3714)
+                    .and_then(|val| {
+                        if let PropertyValue::Integer32(f) = val {
+                            Some(f)
+                        } else {
+                            None
+                        }
+                    })
+                    .map(|flag| (flag & 0x4) != 0)
+                    .unwrap_or(false);
+
+                if let Some(PropertyValue::Binary(bin)) = att_props.get(0x3701) {
+                    let data = bin.buffer().to_vec();
+                    let file_name = name.unwrap_or_else(|| "unnamed_attachment".to_string());
+
+                    if is_inline && cid.is_some() {
+                        let content_id = cid.unwrap();
+                        builder = builder.inline(mime, content_id, data);
+                    } else {
+                        builder = builder.attachment(mime, file_name, data);
+                    }
+                }
+            }
+        }
+    }
+
+    match builder.write_to_vec() {
+        Ok(eml_vec) => Some(base64_encode_url_safe!(eml_vec)),
+        Err(e) => {
+            tracing::error!("Failed to generate EML from PST message: {:?}", e);
+            None
+        }
+    }
+}
+
+fn filetime_to_datetime(filetime: i64) -> DateTime<Utc> {
+    let unix_secs = (filetime / 10_000_000) - 11_644_473_600;
+    let nsecs = (filetime % 10_000_000) * 100;
+    Utc.timestamp_opt(unix_secs, nsecs as u32).unwrap()
+}
+
+fn extract_recipients_list(message: &Rc<dyn Message>) -> (Vec<String>, Vec<String>, Vec<String>) {
+    let mut to = Vec::new();
+    let mut cc = Vec::new();
+    let mut bcc = Vec::new();
+
+    let recipient_table = message.recipient_table();
+    if let Some(recipient_table) = recipient_table {
+        let context = recipient_table.context();
+        for row in recipient_table.rows_matrix() {
+            if let Ok(cols) = row.columns(context) {
+                let mut r_type = 0;
+                let mut email = String::new();
+
+                for (col, val) in context.columns().iter().zip(cols) {
+                    let prop_val = val
+                        .as_ref()
+                        .and_then(|v| recipient_table.read_column(v, col.prop_type()).ok());
+                    match col.prop_id() {
+                        0x0C15 => {
+                            if let Some(PropertyValue::Integer32(t)) = prop_val {
+                                r_type = t;
+                            }
+                        }
+                        0x39FE | 0x3003 => {
+                            if let Some(s) = prop_val.and_then(|v| extract_string(&v)) {
+                                email = s;
+                            }
+                        }
+                        _ => {}
+                    }
+                }
+
+                if !email.is_empty() {
+                    match r_type {
+                        1 => to.push(email),
+                        2 => cc.push(email),
+                        3 => bcc.push(email),
+                        _ => {}
+                    }
+                }
+            }
+        }
+    } else {
+        let receiver = extract_string_property(message.properties(), 0x0076);
+        if let Some(receiver) = receiver {
+            to.push(receiver);
+        }
+    }
+    (to, cc, bcc)
+}
+
+fn extract_subject(props: &MessageProperties) -> Option<String> {
+    props
+        .get(0x0037)
+        .and_then(|val| encoding::decode_subject(val))
+}
+
+fn extract_string_property(properties: &MessageProperties, prop_id: u16) -> Option<String> {
+    properties
+        .get(prop_id)
+        .and_then(|value| extract_string(value))
+}
+
+fn extract_attachment_string_property(
+    properties: &AttachmentProperties,
+    prop_id: u16,
+) -> Option<String> {
+    properties
+        .get(prop_id)
+        .and_then(|value| extract_string(value))
+}
+
+fn extract_string(value: &PropertyValue) -> Option<String> {
+    match value {
+        PropertyValue::String8(value) => Some(value.to_string()),
+        PropertyValue::Unicode(value) => Some(value.to_string()),
+        _ => None,
+    }
+}
+
+fn extract_text(properties: &MessageProperties) -> Option<String> {
+    properties.get(0x1000).and_then(extract_string).or_else(|| {
+        properties.get(0x1009).and_then(|value| match value {
+            PropertyValue::Binary(value) => encoding::decode_rtf_compressed(value.buffer()),
+            _ => None,
+        })
+    })
+}
+
+fn extract_html(properties: &MessageProperties) -> Option<String> {
+    properties.get(0x1013).and_then(|value| match value {
+        PropertyValue::Binary(value) => {
+            let code_page = properties
+                .get(0x3FDE)
+                .and_then(|v| {
+                    if let PropertyValue::Integer32(cpid) = v {
+                        Some(*cpid as u16)
+                    } else {
+                        None
+                    }
+                })
+                .unwrap_or(65001);
+            encoding::decode_html_body(value.buffer(), code_page)
+        }
+        PropertyValue::String8(value) => Some(value.to_string()),
+        PropertyValue::Unicode(value) => Some(value.to_string()),
+        _ => None,
+    })
+}
+
+fn extract_i64_property(properties: &MessageProperties, prop_ids: &[u16]) -> Option<i64> {
+    for &prop_id in prop_ids {
+        if let Some(PropertyValue::Time(value)) = properties.get(prop_id) {
+            return Some(*value);
+        }
+    }
+    None
+}
+
+/// Open a PST file and count total messages across all folders.
+/// Called from the web upload flow to get the total before processing.
+pub fn count_pst_messages(pst_path: &std::path::Path) -> crate::error::BichonResult<usize> {
+    let pst_store = outlook_pst::open_store(pst_path).map_err(|e| {
+        crate::raise_error!(
+            format!("Failed to open PST file: {:?}", e),
+            crate::error::code::ErrorCode::InvalidParameter
+        )
+    })?;
+
+    let ipm_sub_tree = pst_store
+        .properties()
+        .ipm_sub_tree_entry_id()
+        .map_err(|e| {
+            crate::raise_error!(
+                format!("Could not find IPM_SUBTREE in PST: {:?}", e),
+                crate::error::code::ErrorCode::InvalidParameter
+            )
+        })?;
+
+    let ipm_subtree_folder = pst_store.open_folder(&ipm_sub_tree).map_err(|e| {
+        crate::raise_error!(
+            format!("Failed to open root mailbox folder: {:?}", e),
+            crate::error::code::ErrorCode::InvalidParameter
+        )
+    })?;
+
+    Ok(count_folder_messages(&ipm_subtree_folder))
+}
+
+fn count_folder_messages(folder: &Rc<dyn Folder>) -> usize {
+    let mut count = 0usize;
+
+    if let Some(contents_table) = folder.contents_table() {
+        for row in contents_table.rows_matrix() {
+            let store = folder.store().clone();
+            let entry_id = match store
+                .properties()
+                .make_entry_id(NodeId::from(u32::from(row.id())))
+            {
+                Ok(id) => id,
+                Err(_) => continue,
+            };
+
+            if store.open_message(&entry_id, None).is_ok() {
+                count += 1;
+            }
+        }
+    }
+
+    if let Some(hierarchy_table) = folder.hierarchy_table() {
+        for row in hierarchy_table.rows_matrix() {
+            let node = NodeId::from(u32::from(row.id()));
+            if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
+                if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
+                    count += count_folder_messages(&sub_folder);
+                }
+            }
+        }
+    }
+
+    count
+}
+
+/// Walk all folders and process messages, calling the progress callback
+/// every 50 messages. Used by the web upload flow.
+pub fn process_folder_with_progress<F>(
+    folder: &Rc<dyn Folder>,
+    parent_path: &str,
+    account_id: u64,
+    total: usize,
+    success_count: &mut usize,
+    duplicate_count: &mut usize,
+    failed_details: &mut Vec<super::FailedItemDetail>,
+    index: &mut usize,
+    progress_cb: &F,
+) where
+    F: Fn(usize, usize, usize), // (success, duplicates, failed)
+{
+    process_folder_with_progress_inner(
+        folder,
+        parent_path,
+        account_id,
+        total,
+        success_count,
+        duplicate_count,
+        failed_details,
+        index,
+        progress_cb,
+    );
+}
+
+fn process_folder_with_progress_inner<F>(
+    folder: &Rc<dyn Folder>,
+    parent_path: &str,
+    account_id: u64,
+    total: usize,
+    success_count: &mut usize,
+    duplicate_count: &mut usize,
+    failed_details: &mut Vec<super::FailedItemDetail>,
+    index: &mut usize,
+    progress_cb: &F,
+) where
+    F: Fn(usize, usize, usize),
+{
+    let folder_name = folder
+        .properties()
+        .display_name()
+        .unwrap_or_else(|_| "Unknown".to_string());
+
+    let mail_folder = if parent_path.is_empty() {
+        folder_name
+    } else {
+        format!("{}/{}", parent_path, folder_name)
+    };
+
+    tracing::debug!("Processing PST folder: {}", mail_folder);
+
+    let mailbox_id = match super::resolve_mailbox_by_account_id(account_id, &mail_folder) {
+        Ok(id) => id,
+        Err(e) => {
+            tracing::error!("Failed to resolve mailbox '{}': {:?}", mail_folder, e);
+            // Still recurse into sub-folders even if this folder's mailbox creation fails
+            if let Some(hierarchy_table) = folder.hierarchy_table() {
+                for row in hierarchy_table.rows_matrix() {
+                    let node = NodeId::from(u32::from(row.id()));
+                    if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
+                        if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
+                            process_folder_with_progress_inner(
+                                &sub_folder,
+                                &mail_folder,
+                                account_id,
+                                total,
+                                success_count,
+                                duplicate_count,
+                                failed_details,
+                                index,
+                                progress_cb,
+                            );
+                        }
+                    }
+                }
+            }
+            return;
+        }
+    };
+
+    let mut batch_size = 0usize;
+
+    if let Some(contents_table) = folder.contents_table() {
+        for row in contents_table.rows_matrix() {
+            let store = folder.store().clone();
+
+            let entry_id = match store
+                .properties()
+                .make_entry_id(NodeId::from(u32::from(row.id())))
+            {
+                Ok(id) => id,
+                Err(e) => {
+                    tracing::warn!("Skip PST row {}: {:?}", row.unique(), e);
+                    continue;
+                }
+            };
+
+            match store.open_message(&entry_id, None) {
+                Ok(message) => match build_eml_base64(message) {
+                    Some(base64_eml) => {
+                        let decoded = match crate::base64_decode_url_safe!(base64_eml.as_bytes()) {
+                            Ok(bytes) => bytes,
+                            Err(e) => {
+                                failed_details.push(super::FailedItemDetail {
+                                    index: *index,
+                                    error_message: format!(
+                                        "Failed to decode base64 EML at index {}: {:?}",
+                                        *index, e
+                                    ),
+                                });
+                                *index += 1;
+                                batch_size += 1;
+                                continue;
+                            }
+                        };
+
+                        match futures::executor::block_on(extract_envelope_from_eml(
+                            &decoded, account_id, mailbox_id,
+                        )) {
+                            Ok(ExtractOutcome::Imported) => {
+                                *success_count += 1;
+                            }
+                            Ok(ExtractOutcome::Duplicate) => {
+                                *duplicate_count += 1;
+                            }
+                            Err(e) => {
+                                failed_details.push(super::FailedItemDetail {
+                                    index: *index,
+                                    error_message: format!("{:?}", e),
+                                });
+                            }
+                        };
+                        *index += 1;
+                        batch_size += 1;
+                    }
+                    None => {}
+                },
+                Err(e) => {
+                    tracing::warn!("Open PST message error: {:?}", e);
+                }
+            }
+
+            // Report progress every 50 messages
+            if batch_size % 50 == 0 {
+                progress_cb(*success_count, *duplicate_count, failed_details.len());
+            }
+        }
+    }
+
+    if let Some(hierarchy_table) = folder.hierarchy_table() {
+        for row in hierarchy_table.rows_matrix() {
+            let node = NodeId::from(u32::from(row.id()));
+            if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
+                if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
+                    process_folder_with_progress_inner(
+                        &sub_folder,
+                        &mail_folder,
+                        account_id,
+                        total,
+                        success_count,
+                        duplicate_count,
+                        failed_details,
+                        index,
+                        progress_cb,
+                    );
+                }
+            }
+        }
+    }
+}

crates/core/src/import/reader.rs

@@ -21,6 +21,8 @@ use std::fs;
 use std::io;
 use std::path::Path;
 
+/// Memory-mapped MBOX file. Messages are yielded one at a time without
+/// loading the entire file into RAM.
 pub struct MboxFile {
     map: Mmap,
 }
@@ -127,10 +129,6 @@ impl<'a> Iterator for MboxReader<'a> {
 
 #[cfg(test)]
 mod tests {
-    use mail_parser::MessageParser;
-
-    use crate::mbox::gmail::determine_folder;
-
     use super::*;
 
     fn collect_entries(data: &[u8]) -> Vec<&[u8]> {
@@ -144,6 +142,7 @@ mod tests {
         let e = collect_entries(data);
         assert_eq!(e, vec![b"mail1\n", b"mail2\n"]);
     }
+
     #[test]
     fn no_trailing_newline() {
         let data = b"From a\nmail1";
@@ -204,22 +203,4 @@ mod tests {
         let e = collect_entries(&data);
         assert_eq!(e.len(), 1000);
     }
-
-    #[test]
-    fn test11() {
-        let mbox = MboxFile::from_file(Path::new("e:\\test.mbox")).unwrap();
-
-        for e in mbox.iter() {
-            let body = e.data;
-
-            let message = MessageParser::new().parse(body).unwrap();
-            let labels = message.header("X-Gmail-Labels").unwrap().as_text().unwrap();
-            //println!("offset={} X-Gmail-Labels={:?}", e.offset, labels);
-            println!(
-                "X-Gmail-Labels={:?}, determine_folder={}",
-                labels,
-                determine_folder(labels)
-            )
-        }
-    }
 }

crates/core/src/lib.rs

@@ -2,7 +2,7 @@ pub mod account;
 pub mod ext;
 pub mod admin;
 pub mod autoconfig;
-pub mod cache;
+pub mod archive;
 pub mod common;
 pub mod context;
 pub mod dashboard;

crates/core/src/mailbox/delete.rs

@@ -17,7 +17,7 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::{
-    cache::imap::mailbox::MailBox,
+    archive::imap::mailbox::MailBox,
     error::BichonResult,
     store::tantivy::{attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER},
 };

crates/core/src/mailbox/list.rs

@@ -17,8 +17,8 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::account::migration::{AccountModel, AccountType};
-use crate::cache::imap::mailbox::{Attribute, AttributeEnum, MailBox};
-use crate::cache::imap::mailbox_cache::{self, FetchStatus};
+use crate::archive::imap::mailbox::{Attribute, AttributeEnum, MailBox};
+use crate::archive::imap::mailbox_cache::{self, FetchStatus};
 use crate::error::code::ErrorCode;
 use crate::error::BichonResult;
 use crate::imap::executor::ImapExecutor;

crates/core/src/message/search.rs

@@ -43,6 +43,10 @@ pub struct EmailSearchFilter {
     pub to: Option<String>,
     pub cc: Option<String>,
     pub bcc: Option<String>,
+    /// Matches if the address appears in `to`, `cc`, or `bcc` (OR semantics).
+    pub any_recipient: Option<String>,
+    /// Matches if the address appears in `from`, `to`, `cc`, or `bcc` (OR semantics).
+    pub any_participant: Option<String>,
     pub since: Option<i64>,
     pub before: Option<i64>,
     /// Lower bound (inclusive) on the IMAP server INTERNALDATE timestamp.
@@ -165,6 +169,10 @@ pub struct AttachmentSearchRequest {
     desc: Option<bool>,
 }
 impl AttachmentSearchRequest {
+    pub fn filter(&self) -> &AttachmentSearchFilter {
+        &self.filter
+    }
+
     pub fn validate(&self) -> BichonResult<()> {
         if self.page == 0 || self.page_size == 0 {
             return Err(raise_error!(

crates/core/src/migrate/mod.rs

@@ -1,24 +1,24 @@
-use std::{collections::HashMap, path::PathBuf};
+use std::path::{Path, PathBuf};
 
-use crate::{
-    error::{code::ErrorCode, BichonResult},
-    migrate::{
-        legacy::schema::SchemaTools,
-        store::{LegacyDirs, NewDirs, NewIndexWriter},
-    },
-    raise_error,
-    settings::cli::SETTINGS,
-};
-use tantivy::{
-    collector::TopDocs,
-    columnar::Column,
-    query::TermQuery,
-    schema::{IndexRecordOption, Value},
-    DocAddress, Index, TantivyDocument, Term,
-};
+use crate::settings::cli::SETTINGS;
 
-pub mod legacy;
-pub mod store;
+/// Current storage layout version.
+/// - 1: fjall-based blob storage (post v0.3.7 migration)
+/// - 2: bichon-blob based storage
+pub const CURRENT_STORAGE_VERSION: u32 = 2;
+
+const VERSION_FILE: &str = "STORAGE_VERSION";
+
+/// Read the storage layout version from `root_dir/STORAGE_VERSION`.
+pub fn read_storage_version(root_dir: &Path) -> Option<u32> {
+    let content = std::fs::read_to_string(root_dir.join(VERSION_FILE)).ok()?;
+    content.trim().parse().ok()
+}
+
+/// Write the storage layout version to `root_dir/STORAGE_VERSION`.
+pub fn write_storage_version(root_dir: &Path, version: u32) -> std::io::Result<()> {
+    std::fs::write(root_dir.join(VERSION_FILE), format!("{}\n", version))
+}
 
 pub fn is_tantivy_index_dir(dir: &PathBuf) -> std::io::Result<bool> {
     if !dir.exists() || !dir.is_dir() {
@@ -47,28 +47,17 @@ pub fn is_tantivy_index_dir(dir: &PathBuf) -> std::io::Result<bool> {
     Ok(has_meta_json && match_count >= 3)
 }
 
-/// Return the number of segments in the legacy EML Tantivy index.
-/// Each segment can be passed to `do_migrate_segment` for bounded-memory batch migration.
-pub fn count_eml_segments(legacy: &LegacyDirs) -> BichonResult<usize> {
-    let eml_index = Index::open_in_dir(&legacy.eml_dir)
-        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
-    let reader = eml_index
-        .reader()
-        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
-    let searcher = reader.searcher();
-    Ok(searcher.segment_readers().len())
-}
-
+/// Check whether the data layout is compatible with the current server.
+/// Returns `false` when legacy data (v0.3.7 or v1.x) is detected and migration is required.
 pub fn check_data_status() -> std::io::Result<bool> {
     let root_dir = PathBuf::from(&SETTINGS.bichon_root_dir);
 
-    let new_indices_base = SETTINGS
-        .bichon_index_dir
-        .as_ref()
-        .map(PathBuf::from)
-        .unwrap_or_else(|| root_dir.clone());
-    let new_indices_path = new_indices_base.join("bichon-indices");
+    // 1. Version file takes precedence
+    if let Some(version) = read_storage_version(&root_dir) {
+        return Ok(version >= CURRENT_STORAGE_VERSION);
+    }
 
+    // 2. No version file — check for existing v1.x-style storage (fjall era)
     let new_data_base = SETTINGS
         .bichon_data_dir
         .as_ref()
@@ -76,14 +65,13 @@ pub fn check_data_status() -> std::io::Result<bool> {
         .unwrap_or_else(|| root_dir.clone());
     let new_storage_path = new_data_base.join("bichon-storage");
 
-    let has_new_indices = is_tantivy_index_dir(&new_indices_path.join("attachment_metadata"))?
-        && is_tantivy_index_dir(&new_indices_path.join("mail_metadata"))?;
-    let has_new_storage = is_dir_not_empty(&new_storage_path)?;
-
-    if has_new_indices && has_new_storage {
-        return Ok(true);
+    if is_dir_not_empty(&new_storage_path)? {
+        // Existing v1.x install predates version file — mark it as v1
+        let _ = write_storage_version(&root_dir, 1);
+        return Ok(false); // Needs migration: v1.x → v2.x
     }
 
+    // 3. Check for legacy v0.3.7 Tantivy layout
     let legacy_index_root = SETTINGS
         .bichon_index_dir
         .as_ref()
@@ -99,9 +87,9 @@ pub fn check_data_status() -> std::io::Result<bool> {
     let has_legacy_data = is_tantivy_index_dir(&legacy_data_root)?;
 
     if has_legacy_index || has_legacy_data {
-        Ok(false)
+        Ok(false) // Needs migration
     } else {
-        Ok(true)
+        Ok(true) // Fresh install
     }
 }
 
@@ -112,216 +100,3 @@ fn is_dir_not_empty(path: &PathBuf) -> std::io::Result<bool> {
     let mut entries = std::fs::read_dir(path)?;
     Ok(entries.next().is_some())
 }
-
-/// Migrate all documents from a single EML segment to the new storage layout.
-///
-/// This is the core of the batch migration strategy: each Process B invocation
-/// handles exactly one EML segment, so peak memory is bounded by that segment's
-/// size regardless of the total archive size.
-pub fn do_migrate_segment<F>(
-    batch_size: u32,
-    legacy: LegacyDirs,
-    new_dirs: NewDirs,
-    segment_index: usize,
-    mut on_progress: F,
-) -> BichonResult<()>
-where
-    F: FnMut(&str),
-{
-    // ── open legacy indices ────────────────────────────────────────────
-    let envelope_index = Index::open_in_dir(&legacy.envelope_dir)
-        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
-    let eml_index = Index::open_in_dir(&legacy.eml_dir)
-        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
-
-    let envelope_reader = envelope_index
-        .reader()
-        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
-    let eml_reader = eml_index
-        .reader()
-        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
-
-    let envelope_searcher = envelope_reader.searcher();
-    let eml_searcher = eml_reader.searcher();
-
-    let ef = SchemaTools::envelope_fields();
-    let mf = SchemaTools::eml_fields();
-
-    let eml_segments = eml_searcher.segment_readers();
-    let eml_segment = eml_segments.get(segment_index).ok_or_else(|| {
-        raise_error!(
-            format!(
-                "segment index {} out of range ({} segments)",
-                segment_index,
-                eml_segments.len()
-            ),
-            ErrorCode::InternalError
-        )
-    })?;
-
-    let num_docs = eml_segment.num_docs();
-    if num_docs == 0 {
-        on_progress("TOTAL:0");
-        on_progress("DONE:0:0");
-        return Ok(());
-    }
-
-    on_progress(&format!("TOTAL:{}", num_docs));
-
-    let max_doc = eml_segment.max_doc();
-    let ff = eml_segment.fast_fields();
-    let f_id_col: Column<u64> = ff.u64("id").map_err(|e| {
-        raise_error!(
-            format!("failed to open f_id fast field: {e:#?}"),
-            ErrorCode::InternalError
-        )
-    })?;
-
-    // ── Phase 1: build eid → (uid, internal_date) from envelope, then drop it ──
-    let mut envelope_map: HashMap<u64, (u32, i64)> = HashMap::with_capacity(num_docs as usize);
-
-    let mut env_scanned = 0u32;
-    let mut env_skipped = 0u32;
-    for doc_id in 0..max_doc {
-        if eml_segment.is_deleted(doc_id) {
-            continue;
-        }
-        let eid = f_id_col.values.get_val(doc_id);
-
-        let term = Term::from_field_u64(ef.f_id, eid);
-        let query = TermQuery::new(term, IndexRecordOption::Basic);
-        let hits: Vec<(_, DocAddress)> = envelope_searcher
-            .search(&query, &TopDocs::with_limit(1).order_by_score())
-            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
-
-        if let Some((_, addr)) = hits.first() {
-            let env_doc: TantivyDocument = envelope_searcher
-                .doc(*addr)
-                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
-            let uid = env_doc
-                .get_first(ef.f_uid)
-                .and_then(|v| v.as_u64())
-                .unwrap_or(0) as u32;
-            let internal_date = env_doc
-                .get_first(ef.f_internal_date)
-                .and_then(|v| v.as_i64())
-                .unwrap_or(0);
-            envelope_map.insert(eid, (uid, internal_date));
-            env_scanned += 1;
-        } else {
-            env_skipped += 1;
-        }
-
-        if env_scanned % 10 == 0 {
-            on_progress(&format!(
-                "PHASE1:{}/{} skipped:{}",
-                env_scanned, max_doc, env_skipped
-            ));
-        }
-    }
-
-    // Free the envelope index before the heavy EML processing.
-    drop(envelope_searcher);
-    drop(envelope_reader);
-    drop(envelope_index);
-
-    // ── Phase 2: process EML docs, streaming one at a time ─────────────
-    let mut writer = NewIndexWriter::open(new_dirs)?;
-
-    let mut total_migrated = 0usize;
-    let mut total_skipped = 0usize;
-
-    // Recreate the StoreReader periodically to bound any internal caches.
-    //const CHUNK_SIZE: u32 = 3000;
-    let mut chunk_start = 0u32;
-
-    while chunk_start < max_doc {
-        let chunk_end = (chunk_start + batch_size).min(max_doc);
-        let store_reader = eml_segment
-            .get_store_reader(2)
-            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
-
-        for doc_id in chunk_start..chunk_end {
-            if eml_segment.is_deleted(doc_id) {
-                continue;
-            }
-
-            let eid = f_id_col.values.get_val(doc_id);
-
-            let (uid, internal_date) = match envelope_map.get(&eid) {
-                Some(v) => *v,
-                None => {
-                    on_progress(&format!("WARN: eid {} envelope not found", eid));
-                    total_skipped += 1;
-                    continue;
-                }
-            };
-
-            let eml_doc: TantivyDocument = store_reader
-                .get(doc_id)
-                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
-
-            let account_id = match eml_doc.get_first(mf.f_account_id).and_then(|v| v.as_u64()) {
-                Some(v) => v,
-                None => {
-                    on_progress(&format!("WARN: eid {} account_id missing", eid));
-                    total_skipped += 1;
-                    continue;
-                }
-            };
-            let mailbox_id = eml_doc
-                .get_first(mf.f_mailbox_id)
-                .and_then(|v| v.as_u64())
-                .unwrap_or(0);
-
-            // Borrow directly from eml_doc — no .to_vec() clone.
-            let eml_bytes = match eml_doc.get_first(mf.f_eml).and_then(|v| v.as_bytes()) {
-                Some(b) => b,
-                None => {
-                    on_progress(&format!("WARN: eid {} eml bytes missing", eid));
-                    total_skipped += 1;
-                    continue;
-                }
-            };
-
-            if let Err(e) = writer.ingest(eml_bytes, account_id, mailbox_id, uid, internal_date) {
-                on_progress(&format!(
-                    "ERROR: Account {} eid {} ingest failed: {}",
-                    account_id, eid, e
-                ));
-                total_skipped += 1;
-                continue;
-            }
-
-            total_migrated += 1;
-
-            if total_migrated % 10 == 0 || total_migrated as u32 == num_docs {
-                on_progress(&format!("PROGRESS:{}:{}", total_migrated, num_docs));
-            }
-        }
-
-        drop(store_reader);
-
-        // Flush Fjall buffers via ingestion API — bypasses memtable/WAL.
-        writer.flush_fjall_buffers()?;
-
-        chunk_start = chunk_end;
-    }
-
-    writer.finish_writers()?;
-    on_progress(&format!("DONE:{}:{}", total_migrated, total_skipped));
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_real_tantivy_dir() {
-        let path = PathBuf::from(r"D:\test-data\envelope");
-        let result = is_tantivy_index_dir(&path).unwrap();
-        println!("is tantivy index dir: {}", result);
-        assert!(result);
-    }
-}

crates/core/src/oauth2/flow.rs

@@ -20,6 +20,7 @@ use crate::error::code::ErrorCode;
 use crate::error::BichonResult;
 use crate::oauth2::{entity::OAuth2, pending::OAuth2PendingEntity, token::OAuth2AccessToken};
 use crate::settings::proxy::Proxy;
+use crate::utils::net::parse_proxy_url;
 use crate::{decrypt, encrypt, raise_error};
 use oauth2::{
     basic::BasicClient, AuthUrl, AuthorizationCode, ClientId, ClientSecret, CsrfToken,
@@ -265,14 +266,12 @@ impl OAuth2Flow {
 fn build_http_client(use_proxy: Option<u64>) -> BichonResult<reqwest::Client> {
     if let Some(proxy_id) = use_proxy {
         let proxy = Proxy::get(proxy_id)?;
+        let proxy_url = parse_proxy_url(&proxy.url)?.standard_url();
         return oauth2::reqwest::ClientBuilder::new()
             .redirect(oauth2::reqwest::redirect::Policy::none())
-            .proxy(reqwest::Proxy::all(&proxy.url).map_err(|e| {
+            .proxy(reqwest::Proxy::all(&proxy_url).map_err(|_| {
                 raise_error!(
-                    format!(
-                        "Failed to configure SOCKS5 proxy ({}): {:#?}. Please check",
-                        &proxy.url, e
-                    ),
+                    "Failed to configure proxy. Please check the proxy configuration.".into(),
                     ErrorCode::InternalError
                 )
             })?)

crates/core/src/settings/cli.rs

@@ -242,6 +242,14 @@ pub struct Settings {
     )]
     pub bichon_sync_concurrency: Option<u16>,
 
+    #[clap(
+        long,
+        env,
+        default_value = "90",
+        help = "IMAP socket read timeout in seconds (0 disables the timeout). Servers that throttle or burst slowly (e.g. Zoho) can pause for 30-60s between responses; keep this above the longest expected server silence so throttling surfaces as progress delay, not a failed sync."
+    )]
+    pub bichon_imap_timeout_seconds: u64,
+
     #[clap(
         long,
         env,
@@ -265,7 +273,7 @@ pub struct Settings {
             Ok(s.to_string())
         })
     )]
-    pub bichon_smtp_tls_key_path: Option<String>,
+    pub bichon_tls_key_path: Option<String>,
 
     #[clap(
         long,
@@ -282,7 +290,7 @@ pub struct Settings {
             Ok(s.to_string())
         })
     )]
-    pub bichon_smtp_tls_cert_path: Option<String>,
+    pub bichon_tls_cert_path: Option<String>,
 
     #[clap(
         long,
@@ -299,7 +307,7 @@ pub struct Settings {
         default_value = "starttls",
         help = "Set the encryption mode for SMTP: 'none', 'starttls', or 'tls'"
     )]
-    pub bichon_smtp_encryption: SmtpEncryptionMode,
+    pub bichon_smtp_encryption: EncryptionMode,
 
     #[clap(
         long,
@@ -308,6 +316,103 @@ pub struct Settings {
         help = "Enable SMTP authentication requirement"
     )]
     pub bichon_smtp_auth_required: bool,
+
+    /// Enable the built-in IMAP server for read-only email access via standard
+    /// email clients (Thunderbird, Outlook, Apple Mail, etc.).
+    #[clap(
+        long,
+        default_value = "false",
+        env,
+        help = "Enable the embedded IMAP server"
+    )]
+    pub bichon_enable_imap: bool,
+
+    #[clap(
+        long,
+        default_value = "10143",
+        env,
+        help = "Set the IMAP port (STARTTLS or plaintext)",
+        value_parser = clap::value_parser!(u16).range(1..)
+    )]
+    pub bichon_imap_port: u16,
+
+    #[clap(
+        long,
+        default_value = "10993",
+        env,
+        help = "Set the IMAPS port (implicit TLS)",
+        value_parser = clap::value_parser!(u16).range(1..)
+    )]
+    pub bichon_imaps_port: u16,
+
+    #[clap(
+        long,
+        env,
+        default_value = "none",
+        help = "Set the encryption mode for IMAP: 'none', 'starttls', or 'tls'"
+    )]
+    pub bichon_imap_encryption: EncryptionMode,
+
+    /// Enable OIDC-based Single Sign-On (Pro/Enterprise feature).
+    #[clap(long, default_value = "false", env, help = "Enable OpenID Connect SSO")]
+    pub bichon_oidc_enabled: bool,
+
+    /// OIDC issuer URL (e.g. https://keycloak.example.com/realms/myorg).
+    #[clap(long, env, help = "OpenID Connect issuer URL")]
+    pub bichon_oidc_issuer_url: Option<String>,
+
+    /// OIDC client ID registered with the IdP.
+    #[clap(long, env, help = "OpenID Connect client ID")]
+    pub bichon_oidc_client_id: Option<String>,
+
+    /// OIDC client secret registered with the IdP.
+    #[clap(long, env, help = "OpenID Connect client secret")]
+    pub bichon_oidc_client_secret: Option<String>,
+
+    /// OIDC redirect URI (must match what's registered with the IdP).
+    #[clap(long, env, help = "OpenID Connect redirect URI")]
+    pub bichon_oidc_redirect_uri: Option<String>,
+
+    /// Maximum HTTP request body size in MB for file uploads (default: 1100 MB).
+    /// Requests exceeding this limit are rejected at the framework level before
+    /// the application reads the body, preventing memory exhaustion attacks.
+    #[clap(
+        long,
+        default_value = "1100",
+        env,
+        help = "Maximum HTTP request body size in MB for file uploads"
+    )]
+    pub bichon_upload_body_limit_mb: u64,
+
+    /// Maximum per-file size in MB for MBOX uploads via the web UI (default: 1024 MB = 1 GB).
+    /// Individual EML files are always capped at 100 MB regardless of this setting.
+    #[clap(
+        long,
+        default_value = "1024",
+        env,
+        help = "Maximum per-file size in MB for MBOX uploads via the web UI"
+    )]
+    pub bichon_web_mbox_upload_limit_mb: u64,
+
+    /// Maximum per-file size in MB for PST uploads via the web UI (default: 2048 MB = 2 GB).
+    #[clap(
+        long,
+        default_value = "2048",
+        env,
+        help = "Maximum per-file size in MB for PST uploads via the web UI"
+    )]
+    pub bichon_web_pst_upload_limit_mb: u64,
+
+    /// Audit log retention period in days (default: 90). Older audit records
+    /// are purged periodically by a background task. 0 disables the cleanup.
+    /// Pro edition only.
+    #[clap(
+        long,
+        default_value = "90",
+        env,
+        help = "Audit log retention period in days (0 disables cleanup). Pro edition only."
+    )]
+    pub bichon_audit_retention_days: u64,
 }
 
 impl Settings {
@@ -317,9 +422,8 @@ impl Settings {
         // rejects it, fall back to parsing with only the binary name so that
         // the settings come entirely from environment variables.
         let args: Vec<String> = std::env::args().collect();
-        let s = Self::try_parse_from(&args).unwrap_or_else(|_| {
-            Self::parse_from(std::iter::once(args[0].clone()))
-        });
+        let s = Self::try_parse_from(&args)
+            .unwrap_or_else(|_| Self::parse_from(std::iter::once(args[0].clone())));
         if s.bichon_encrypt_password.is_none() && s.bichon_encrypt_password_file.is_none() {
             panic!(
                 "One of --bichon_encrypt_password or --bichon_encrypt_password_file has to be set"
@@ -368,7 +472,7 @@ impl fmt::Display for CompressionAlgorithm {
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, ValueEnum)]
-pub enum SmtpEncryptionMode {
+pub enum EncryptionMode {
     #[clap(name = "none")]
     None,
     #[clap(name = "starttls")]
@@ -377,12 +481,12 @@ pub enum SmtpEncryptionMode {
     Tls,
 }
 
-impl fmt::Display for SmtpEncryptionMode {
+impl fmt::Display for EncryptionMode {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
-            SmtpEncryptionMode::None => write!(f, "none"),
-            SmtpEncryptionMode::Starttls => write!(f, "starttls"),
-            SmtpEncryptionMode::Tls => write!(f, "tls"),
+            EncryptionMode::None => write!(f, "none"),
+            EncryptionMode::Starttls => write!(f, "starttls"),
+            EncryptionMode::Tls => write!(f, "tls"),
         }
     }
 }

crates/core/src/settings/dir.rs

@@ -17,6 +17,7 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::context::Initialize;
+use crate::migrate::{write_storage_version, CURRENT_STORAGE_VERSION};
 use crate::settings::cli::SETTINGS;
 use crate::{
     error::{code::ErrorCode, BichonResult},
@@ -62,6 +63,14 @@ impl Initialize for DataDirManager {
             .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
         std::fs::create_dir_all(&DATA_DIR_MANAGER.storage_dir)
             .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+        // Write STORAGE_VERSION on fresh install (no existing data)
+        let version_path = DATA_DIR_MANAGER.root_dir.join("STORAGE_VERSION");
+        if !version_path.exists() && !DATA_DIR_MANAGER.storage_dir.join("blobs").exists() {
+            write_storage_version(&DATA_DIR_MANAGER.root_dir, CURRENT_STORAGE_VERSION)
+                .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+        }
+
         Ok(())
     }
 }

crates/core/src/settings/mod.rs

@@ -60,6 +60,17 @@ pub struct SystemConfigurations {
     pub bichon_smtp_auth_required: bool,
     pub bichon_smtp_tls_key_path: Option<String>,
     pub bichon_smtp_tls_cert_path: Option<String>,
+
+    pub bichon_oidc_enabled: bool,
+    pub bichon_oidc_issuer_url: Option<String>,
+    pub bichon_oidc_client_id: Option<String>,
+    pub bichon_oidc_redirect_uri: Option<String>,
+
+    pub bichon_upload_body_limit_mb: u64,
+
+    pub bichon_web_mbox_upload_limit_mb: u64,
+
+    pub bichon_web_pst_upload_limit_mb: u64,
 }
 
 impl From<&Settings> for SystemConfigurations {
@@ -92,8 +103,15 @@ impl From<&Settings> for SystemConfigurations {
             bichon_smtp_port: s.bichon_smtp_port,
             bichon_smtp_encryption: s.bichon_smtp_encryption.to_string(),
             bichon_smtp_auth_required: s.bichon_smtp_auth_required,
-            bichon_smtp_tls_key_path: s.bichon_smtp_tls_key_path.clone(),
-            bichon_smtp_tls_cert_path: s.bichon_smtp_tls_cert_path.clone(),
+            bichon_smtp_tls_key_path: s.bichon_tls_key_path.clone(),
+            bichon_smtp_tls_cert_path: s.bichon_tls_cert_path.clone(),
+            bichon_oidc_enabled: s.bichon_oidc_enabled,
+            bichon_oidc_issuer_url: s.bichon_oidc_issuer_url.clone(),
+            bichon_oidc_client_id: s.bichon_oidc_client_id.clone(),
+            bichon_oidc_redirect_uri: s.bichon_oidc_redirect_uri.clone(),
+            bichon_upload_body_limit_mb: s.bichon_upload_body_limit_mb,
+            bichon_web_mbox_upload_limit_mb: s.bichon_web_mbox_upload_limit_mb,
+            bichon_web_pst_upload_limit_mb: s.bichon_web_pst_upload_limit_mb,
         }
     }
 }

crates/core/src/settings/proxy.rs

@@ -18,6 +18,7 @@
 
 //use poem_openapi::Object;
 use serde::{Deserialize, Serialize};
+use std::{error::Error, time::Duration};
 
 use crate::{
     database::{
@@ -26,9 +27,30 @@ use crate::{
     },
     error::{code::ErrorCode, BichonResult},
     id, raise_error, utc_now,
-    utils::net::parse_proxy_addr,
+    utils::net::parse_proxy_url,
 };
 
+const PROXY_TEST_TIMEOUT: Duration = Duration::from_secs(8);
+const GEO_PROVIDERS: &[GeoProvider] = &[
+    GeoProvider {
+        name: "ip-api.com",
+        url: "http://ip-api.com/json/?fields=status,message,query,country,countryCode,regionName,city,isp,timezone,lat,lon",
+    },
+    GeoProvider {
+        name: "ipwho.is",
+        url: "https://ipwho.is/",
+    },
+    GeoProvider {
+        name: "ipapi.co",
+        url: "https://ipapi.co/json/",
+    },
+];
+
+struct GeoProvider {
+    name: &'static str,
+    url: &'static str,
+}
+
 #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
 #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
 pub struct Proxy {
@@ -45,6 +67,16 @@ pub struct Proxy {
     pub updated_at: i64,
 }
 
+#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
+pub struct ProxyTestResult {
+    pub ip: Option<String>,
+    pub country: Option<String>,
+    pub region: Option<String>,
+    pub city: Option<String>,
+    pub isp: Option<String>,
+}
+
 impl MemDbModel for Proxy {
     fn collection() -> &'static str {
         "proxies"
@@ -85,9 +117,10 @@ impl Proxy {
 
     pub fn update(id: u64, url: String) -> BichonResult<()> {
         update_impl(DB_MANAGER.db(), &id.to_string(), move |current: Proxy| {
-            let mut updated = current.clone();
+            let mut updated = current;
             updated.url = url;
             updated.updated_at = utc_now!();
+            updated.validate()?;
             Ok(updated)
         })?;
         Ok(())
@@ -98,11 +131,185 @@ impl Proxy {
         insert_impl(DB_MANAGER.db(), self.to_owned())
     }
 
-    /// Validate that the URL is a valid SOCKS5 proxy URL.
+    /// Validate that the URL is a valid proxy URL.
     pub fn validate(&self) -> BichonResult<()> {
-        parse_proxy_addr(&self.url)?;
+        parse_proxy_url(&self.url)?;
         Ok(())
     }
+
+    pub async fn test_connectivity(&self) -> BichonResult<ProxyTestResult> {
+        test_proxy_url(&self.url).await
+    }
+
+    pub async fn test(id: u64) -> BichonResult<ProxyTestResult> {
+        let proxy = Self::get(id)?;
+        proxy.test_connectivity().await
+    }
+}
+
+async fn test_proxy_url(url: &str) -> BichonResult<ProxyTestResult> {
+    let proxy_url = parse_proxy_url(url)?.standard_url();
+    let client = reqwest::Client::builder()
+        .timeout(PROXY_TEST_TIMEOUT)
+        .proxy(reqwest::Proxy::all(&proxy_url).map_err(|_| {
+            raise_error!(
+                "Failed to configure proxy. Please check the proxy configuration.".into(),
+                ErrorCode::InvalidParameter
+            )
+        })?)
+        .build()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+    let mut last_error = None;
+    for provider in GEO_PROVIDERS {
+        match test_geo_provider(&client, provider).await {
+            Ok(result) => return Ok(result),
+            Err(err) => last_error = Some(err.to_string()),
+        }
+    }
+
+    Err(raise_error!(
+        format!(
+            "Proxy check failed with all geo providers: {}",
+            last_error.unwrap_or_else(|| "unknown error".into())
+        ),
+        ErrorCode::NetworkError
+    ))
+}
+
+async fn test_geo_provider(
+    client: &reqwest::Client,
+    provider: &GeoProvider,
+) -> BichonResult<ProxyTestResult> {
+    let value = client
+        .get(provider.url)
+        .send()
+        .await
+        .map_err(|e| {
+            raise_error!(
+                proxy_request_error_message(
+                    &format!("Proxy check request failed via {}", provider.name),
+                    &e
+                ),
+                ErrorCode::NetworkError
+            )
+        })?
+        .error_for_status()
+        .map_err(|e| {
+            raise_error!(
+                proxy_request_error_message(
+                    &format!("Proxy check request failed via {}", provider.name),
+                    &e
+                ),
+                ErrorCode::NetworkError
+            )
+        })?
+        .json::<serde_json::Value>()
+        .await
+        .map_err(|e| {
+            raise_error!(
+                proxy_request_error_message(
+                    &format!("Failed to read proxy check response from {}", provider.name),
+                    &e
+                ),
+                ErrorCode::NetworkError
+            )
+        })?;
+
+    proxy_test_result_from_value(provider.name, &value)
+}
+
+fn proxy_test_result_from_value(
+    provider: &str,
+    value: &serde_json::Value,
+) -> BichonResult<ProxyTestResult> {
+    if provider == "ip-api.com" && value["status"].as_str() == Some("fail") {
+        return Err(raise_error!(
+            format!(
+                "ip-api.com proxy check failed: {}",
+                value["message"].as_str().unwrap_or("unknown error")
+            ),
+            ErrorCode::NetworkError
+        ));
+    }
+    if provider == "ipwho.is" && value["success"].as_bool() == Some(false) {
+        return Err(raise_error!(
+            format!(
+                "ipwho.is proxy check failed: {}",
+                value["message"].as_str().unwrap_or("unknown error")
+            ),
+            ErrorCode::NetworkError
+        ));
+    }
+    if provider == "ipapi.co" && value["error"].as_bool() == Some(true) {
+        return Err(raise_error!(
+            format!(
+                "ipapi.co proxy check failed: {}",
+                value["reason"].as_str().unwrap_or("unknown error")
+            ),
+            ErrorCode::NetworkError
+        ));
+    }
+
+    let ip_key = if provider == "ip-api.com" {
+        "query"
+    } else {
+        "ip"
+    };
+    let ip = value[ip_key].as_str().ok_or_else(|| {
+        raise_error!(
+            format!("{provider} did not return an IP address"),
+            ErrorCode::NetworkError
+        )
+    })?;
+    let connection = &value["connection"];
+
+    Ok(ProxyTestResult {
+        ip: Some(ip.to_string()),
+        country: value[if provider == "ipapi.co" {
+            "country_name"
+        } else {
+            "country"
+        }]
+        .as_str()
+        .map(str::to_string),
+        region: value[if provider == "ip-api.com" {
+            "regionName"
+        } else {
+            "region"
+        }]
+        .as_str()
+        .map(str::to_string),
+        city: value["city"].as_str().map(str::to_string),
+        isp: if provider == "ipapi.co" {
+            value["org"].as_str().map(str::to_string)
+        } else if provider == "ip-api.com" {
+            value["isp"].as_str().map(str::to_string)
+        } else {
+            connection["isp"].as_str().map(str::to_string)
+        },
+    })
+}
+
+fn proxy_request_error_message(context: &str, err: &reqwest::Error) -> String {
+    let kind = if err.is_timeout() {
+        "timed out"
+    } else if err.is_connect() {
+        "could not connect through the proxy"
+    } else if err.is_status() {
+        "received an error response"
+    } else {
+        "request failed"
+    };
+    let mut message = format!("{context}: {kind}: {err}");
+    let mut source = err.source();
+
+    while let Some(err) = source {
+        message.push_str(&format!(": {err}"));
+        source = err.source();
+    }
+
+    message
 }
 
 #[cfg(test)]
@@ -111,11 +318,54 @@ mod tests {
 
     #[test]
     fn test_valid_proxy_urls() {
-        let urls = vec!["socks5://127.0.0.1:1080", "http://127.0.0.1:8080"];
+        let urls = vec![
+            "socks5://127.0.0.1:1080",
+            "http://127.0.0.1:8080",
+            "socks5://proxy.example.com:1080",
+            "socks5://user:pass@proxy.example.com:1080",
+            "http://user:pass@proxy.example.com:8080",
+            "socks5://[::1]:1080",
+            "socks5://user:pass@[::1]:1080",
+            // Non-standard format: host:port:user:pass
+            "socks5://server.nodeprovider.com:8080:username123:passwordhere",
+            "http://server.nodeprovider.com:8080:username123:passwordhere",
+        ];
 
         for url in urls {
             let proxy = Proxy::new(url.to_string());
             assert!(proxy.validate().is_ok(), "URL should be valid: {}", url);
         }
     }
+
+    #[test]
+    fn test_invalid_proxy_urls() {
+        for url in ["socks5://user@proxy.example.com:1080", "socks5://::1:1080"] {
+            let proxy = Proxy::new(url.to_string());
+            assert!(proxy.validate().is_err(), "URL should be invalid: {}", url);
+        }
+    }
+
+    #[test]
+    fn test_ipv6_proxy_urls_render_with_brackets() {
+        let addr = parse_proxy_url("socks5://[::1]:1080").unwrap();
+        assert_eq!(addr.standard_url(), "socks5://[::1]:1080");
+
+        let addr = parse_proxy_url("socks5://user:pass@[::1]:1080").unwrap();
+        assert_eq!(addr.standard_url(), "socks5://user:pass@[::1]:1080");
+    }
+
+    #[test]
+    fn proxy_test_result_rejects_empty_provider_response() {
+        let result = proxy_test_result_from_value("ipwho.is", &serde_json::json!({}));
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn proxy_test_result_rejects_provider_error_response() {
+        let result = proxy_test_result_from_value(
+            "ipwho.is",
+            &serde_json::json!({ "success": false, "message": "reserved range" }),
+        );
+        assert!(result.is_err());
+    }
 }

crates/core/src/store/blob.rs

@@ -16,17 +16,17 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+use crate::raise_error;
 use crate::{
     common::signal::SIGNAL_MANAGER,
     envelope::extractor::reattach_eml_content_self_healing,
     error::{code::ErrorCode, BichonResult},
     settings::dir::DATA_DIR_MANAGER,
 };
-use crate::raise_error;
+use bichon_blob::{Codec, Config, Engine};
 use bytes::Bytes;
-use fjall::{CompressionType, Database, Keyspace, KeyspaceCreateOptions, KvSeparationOptions, config::{BlockSizePolicy, CompressionPolicy}};
 
-use std::{io::Cursor, sync::LazyLock};
+use std::{io::Cursor, sync::Arc, sync::LazyLock};
 use tokio::{
     sync::{mpsc, Mutex},
     task::{self, JoinHandle},
@@ -41,33 +41,48 @@ pub struct DetachedEmail {
 
 pub struct BlobManager {
     sender: mpsc::Sender<DetachedEmail>,
-    db: Database,
-    email_keyspace: Keyspace,
-    attachments_keyspace: Keyspace,
+    engine: Arc<Engine>,
     handle: Mutex<Option<JoinHandle<()>>>,
 }
 
+fn hex_to_key(hex: &str) -> BichonResult<[u8; 32]> {
+    let mut key = [0u8; 32];
+    hex::decode_to_slice(hex, &mut key).map_err(|e| {
+        raise_error!(
+            format!("invalid content hash '{hex}': {e:#?}"),
+            ErrorCode::InternalError
+        )
+    })?;
+    Ok(key)
+}
+
 impl BlobManager {
     pub async fn shutdown(&self) {
         let mut guard = self.handle.lock().await;
         if let Some(handle) = guard.take() {
             let _ = handle.await;
         }
+        if let Err(e) = self.engine.shutdown() {
+            tracing::error!("blob engine shutdown error: {}", e);
+        }
     }
 
-    fn process_detached_email(
-        eml: DetachedEmail,
-        email_ks: &Keyspace,
-        attach_ks: &Keyspace,
-    ) {
+    fn process_detached_email(eml: DetachedEmail, engine: &Engine) {
         let (email_hash, email_data) = eml.email;
-        match email_ks.contains_key(&email_hash) {
+        let email_key = match hex_to_key(&email_hash) {
+            Ok(k) => k,
+            Err(e) => {
+                tracing::error!("{:#?}", e);
+                return;
+            }
+        };
+        match engine.exists(&email_key) {
             Ok(false) => {
-                if let Err(e) = email_ks.insert(email_hash, email_data) {
-                    tracing::error!("CRITICAL: Failed to insert email: {:?}",  e);
+                if let Err(e) = engine.put(email_key, &email_data, Codec::Lz4) {
+                    tracing::error!("CRITICAL: Failed to insert email blob: {:?}", e);
                 }
             }
-            Err(e) => tracing::error!("Fjall email_ks error: {:?}", e),
+            Err(e) => tracing::error!("blob engine error: {:?}", e),
             Ok(true) => {
                 tracing::debug!("Email blob already exists (dedup): {}", &email_hash);
             }
@@ -75,13 +90,20 @@ impl BlobManager {
 
         if let Some(attachments) = eml.attachments {
             for (a_hash, a_data) in attachments {
-                match attach_ks.contains_key(&a_hash) {
+                let a_key = match hex_to_key(&a_hash) {
+                    Ok(k) => k,
+                    Err(e) => {
+                        tracing::error!("{:#?}", e);
+                        continue;
+                    }
+                };
+                match engine.exists(&a_key) {
                     Ok(false) => {
-                        if let Err(e) = attach_ks.insert(a_hash, a_data) {
-                            tracing::error!("CRITICAL: Failed to insert attachment: {:?}", e);
+                        if let Err(e) = engine.put(a_key, &a_data, Codec::Lz4) {
+                            tracing::error!("CRITICAL: Failed to insert attachment blob: {:?}", e);
                         }
                     }
-                    Err(e) => tracing::error!("Fjall attach_ks error: {:?}", e),
+                    Err(e) => tracing::error!("blob engine error: {:?}", e),
                     Ok(true) => {
                         tracing::debug!("Attachment blob already exists (dedup): {}", &a_hash);
                     }
@@ -91,57 +113,22 @@ impl BlobManager {
     }
 
     pub fn new() -> Self {
-        let db = Database::builder(&DATA_DIR_MANAGER.storage_dir)
-        .cache_size(64 * 1024 * 1024)
-        .max_cached_files(Some(400))
-        .journal_compression(CompressionType::None)
-        .max_journaling_size(64 * 1024 * 1024)
-            .open()
-            .expect("Failed to initialize Fjall database: Check if the directory exists and has write permissions.");
+        let blob_dir = DATA_DIR_MANAGER.storage_dir.join("blobs");
 
+        let mut config = Config::default();
+        config.default_codec = Codec::Zstd;
+        config.compress_threshold = 1024;
+        config.flush_interval_secs = 60;
+        config.gc_interval_secs = 300;
+
+        let engine = Engine::open(&blob_dir, config)
+            .expect("Failed to initialize blob engine: Check disk space and permissions.");
+
+        let engine = Arc::new(engine);
 
-        let email_keyspace = db
-            .keyspace("email", || {
-                KeyspaceCreateOptions::default()
-                .max_memtable_size(16 * 1024 * 1024)
-                .data_block_size_policy(BlockSizePolicy::all(4 * 1024))
-                .data_block_compression_policy(  
-                    CompressionPolicy::all(CompressionType::Lz4)  
-                )  
-                .with_kv_separation(Some(
-                    KvSeparationOptions::default()
-                        .separation_threshold(1024)
-                        .compression(CompressionType::Lz4)
-                        .file_target_size(512 * 1024 * 1024)
-                        .staleness_threshold(0.5)
-                        .age_cutoff(0.6),
-                ))
-            })
-            .expect("Failed to open 'email' keyspace: The partition metadata might be corrupted or inaccessible.");
-        
-        let attachments_keyspace = db
-            .keyspace("attachments", || {
-                KeyspaceCreateOptions::default()
-                .data_block_size_policy(BlockSizePolicy::all(4 * 1024))
-                .data_block_compression_policy(  
-                    CompressionPolicy::all(CompressionType::Lz4)  
-                )
-                .with_kv_separation(Some(
-                    KvSeparationOptions::default()
-                        .separation_threshold(1024)
-                        .compression(CompressionType::Lz4)
-                        .file_target_size(512 * 1024 * 1024)
-                        .staleness_threshold(0.5)
-                        .age_cutoff(0.6),
-                ))
-                .max_memtable_size(16 * 1024 * 1024)
-            })
-            .expect("Failed to open 'attachments' keyspace: Check disk space for blob storage initialization.");
-        
         let (sender, mut receiver) = mpsc::channel::<DetachedEmail>(100);
 
-        let email_ks = email_keyspace.clone();
-        let attach_ks = attachments_keyspace.clone();
+        let engine_bg = Arc::clone(&engine);
         let handler = task::spawn(async move {
             let mut shutdown = SIGNAL_MANAGER.subscribe();
             loop {
@@ -153,18 +140,17 @@ impl BlobManager {
                                 while let Ok(next_eml) = receiver.try_recv() {
                                     batch.push(next_eml);
                                 }
-                                let email_ks = email_ks.clone();
-                                let attach_ks = attach_ks.clone();
+                                let engine_bg = Arc::clone(&engine_bg);
                                 if let Err(e) = tokio::task::spawn_blocking(move || {
                                     for eml in batch {
-                                        Self::process_detached_email(eml, &email_ks, &attach_ks);
+                                        Self::process_detached_email(eml, &engine_bg);
                                     }
                                 }).await {
                                     tracing::error!("BlobManager: spawn_blocking join error: {:#?}", e);
                                 }
                             }
                             None => {
-                                tracing::info!("BlobManager: All senders dropped, closing storage.");
+                                tracing::info!("BlobManager: All senders dropped, closing blob storage.");
                                 break;
                             }
                         }
@@ -180,17 +166,16 @@ impl BlobManager {
                             remaining.len()
                         );
                         if !remaining.is_empty() {
-                            let email_ks = email_ks.clone();
-                            let attach_ks = attach_ks.clone();
+                            let engine_bg = Arc::clone(&engine_bg);
                             if let Err(e) = tokio::task::spawn_blocking(move || {
                                 for eml in remaining {
-                                    Self::process_detached_email(eml, &email_ks, &attach_ks);
+                                    Self::process_detached_email(eml, &engine_bg);
                                 }
                             }).await {
                                 tracing::error!("BlobManager: shutdown spawn_blocking join error: {:#?}", e);
                             }
                         }
-                        tracing::info!("BlobManager: All remaining tasks processed. Closing Fjall.");
+                        tracing::info!("BlobManager: All remaining tasks processed. Closing blob engine.");
                         break;
                     }
                 }
@@ -199,30 +184,30 @@ impl BlobManager {
 
         Self {
             sender,
-            db,
-            email_keyspace,
-            attachments_keyspace,
+            engine,
             handle: Mutex::new(Some(handler)),
         }
     }
 
     pub async fn queue(&self, email: DetachedEmail) {
         if let Err(e) = self.sender.send(email).await {
-          tracing::error!("BlobManager channel closed, email lost: {:#?}", e);
+            tracing::error!("BlobManager channel closed, email lost: {:#?}", e);
         }
     }
 
     pub fn get_email(&self, content_hash: &str) -> BichonResult<Option<Bytes>> {
-        self.email_keyspace
-            .get(content_hash)
-            .map(|user_value| user_value.map(|s| s.into()))
-            .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
+        self.get(content_hash)
     }
 
     pub fn get_attachment(&self, content_hash: &str) -> BichonResult<Option<Bytes>> {
-        self.attachments_keyspace
-            .get(content_hash)
-            .map(|user_value| user_value.map(|s| s.into()))
+        self.get(content_hash)
+    }
+
+    fn get(&self, content_hash: &str) -> BichonResult<Option<Bytes>> {
+        let key = hex_to_key(content_hash)?;
+        self.engine
+            .get(&key)
+            .map(|v| v.map(Bytes::from))
             .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
     }
 
@@ -235,17 +220,23 @@ impl BlobManager {
         I1: IntoIterator,
         I1::Item: AsRef<str>,
         I2: IntoIterator,
-        I2::Item: AsRef<str> {
-        let mut batch = self.db.batch();
-        for hash in email_content_hashes {
-            batch.remove(&self.email_keyspace, hash.as_ref());
+        I2::Item: AsRef<str>,
+    {
+        let mut keys: Vec<[u8; 32]> = email_content_hashes
+            .into_iter()
+            .map(|h| hex_to_key(h.as_ref()))
+            .collect::<BichonResult<_>>()?;
+
+        for h in attachment_content_hashes {
+            keys.push(hex_to_key(h.as_ref())?);
         }
-        for hash in attachment_content_hashes {
-            batch.remove(&self.attachments_keyspace, hash.as_ref());
+
+        if !keys.is_empty() {
+            self.engine
+                .delete_batch(&keys)
+                .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
         }
-        batch
-            .commit()
-            .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
         Ok(())
     }
 }

crates/core/src/store/envelope.rs

@@ -27,6 +27,7 @@ pub struct Envelope {
     pub message_id: String,
     pub account_id: u64,
     pub account_email: Option<String>,
+    pub account_name: Option<String>,
     pub mailbox_id: u64,
     pub mailbox_name: Option<String>,
     pub uid: u32,

crates/core/src/store/tantivy/dedup_cache.rs

@@ -0,0 +1,641 @@
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{LazyLock, Mutex};
+
+use crate::store::tantivy::envelope::ENVELOPE_MANAGER;
+use crate::store::tantivy::fields::{F_ACCOUNT_ID, F_CONTENT_HASH, F_INGEST_AT, F_MAILBOX_ID};
+use crate::utc_now;
+
+/// Max entries before evicting the oldest.
+/// At ~152 bytes/entry, 300_000 ≈ 45 MB, within the 50 MB budget.
+const MAX_ENTRIES: usize = 300_000;
+
+/// Fraction of entries to keep when evicting (newest 3/4).
+const KEEP_FRACTION_NUM: usize = 3;
+const KEEP_FRACTION_DEN: usize = 4;
+
+/// Populate only loads entries ingested within this window.
+const POPULATE_WINDOW_MS: i64 = 7 * 24 * 60 * 60 * 1000; // 7 days
+
+pub static DEDUP_CACHE: LazyLock<DedupCache> = LazyLock::new(DedupCache::new);
+
+pub struct DedupCache {
+    entries: Mutex<HashMap<(u64, u64, String), i64>>,
+    max_entries: usize,
+    populated: AtomicBool,
+}
+
+impl DedupCache {
+    fn new() -> Self {
+        Self {
+            entries: Mutex::new(HashMap::new()),
+            max_entries: MAX_ENTRIES,
+            populated: AtomicBool::new(false),
+        }
+    }
+
+    #[cfg(test)]
+    fn new_for_test() -> Self {
+        Self {
+            entries: Mutex::new(HashMap::new()),
+            max_entries: MAX_ENTRIES,
+            populated: AtomicBool::new(true),
+        }
+    }
+
+    #[cfg(test)]
+    fn new_for_test_small(max_entries: usize) -> Self {
+        Self {
+            entries: Mutex::new(HashMap::new()),
+            max_entries,
+            populated: AtomicBool::new(true),
+        }
+    }
+
+    /// Returns true if this `(account_id, mailbox_id, content_hash)` triple
+    /// has already been seen.
+    ///
+    /// On the very first call the cache is populated from the Tantivy index
+    /// FAST columns (only entries ingested within [`POPULATE_WINDOW_MS`]).
+    /// If that scan fails the cache starts empty and still operates correctly
+    /// for newly-arriving emails.
+    pub fn contains(&self, account_id: u64, mailbox_id: u64, hash: &str) -> bool {
+        self.ensure_populated();
+
+        let entries = self.entries.lock().unwrap();
+        entries.contains_key(&(account_id, mailbox_id, hash.to_string()))
+    }
+
+    /// Insert a triple into the cache after it has been queued for indexing.
+    ///
+    /// Each entry is stamped with the current time. When the cache exceeds
+    /// [`MAX_ENTRIES`], the oldest entries are evicted, keeping the newest
+    /// `MAX_ENTRIES * 3/4`.
+    pub fn insert(&self, account_id: u64, mailbox_id: u64, hash: &str) {
+        let mut entries = self.entries.lock().unwrap();
+        let now = utc_now!();
+        entries.insert((account_id, mailbox_id, hash.to_string()), now);
+
+        if entries.len() > self.max_entries {
+            let keep = self.max_entries * KEEP_FRACTION_NUM / KEEP_FRACTION_DEN;
+            let mut vec: Vec<_> = entries.drain().collect();
+            // Sort descending by timestamp (newest first)
+            vec.sort_by(|a, b| b.1.cmp(&a.1));
+            for (k, v) in vec.into_iter().take(keep) {
+                entries.insert(k, v);
+            }
+            tracing::warn!(
+                "DedupCache evicted oldest entries, kept {}/{}",
+                entries.len(),
+                keep
+            );
+        }
+    }
+
+    // ── private ──────────────────────────────────────────────────────────────
+
+    fn ensure_populated(&self) {
+        if self.populated.load(Ordering::Acquire) {
+            return;
+        }
+        self.do_populate();
+    }
+
+    fn do_populate(&self) {
+        if self
+            .populated
+            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed)
+            .is_err()
+        {
+            return;
+        }
+
+        let reader = match ENVELOPE_MANAGER.create_reader() {
+            Ok(r) => r,
+            Err(e) => {
+                tracing::warn!("DedupCache: failed to create reader for populate: {e}");
+                return;
+            }
+        };
+
+        let searcher = reader.searcher();
+        let cutoff = utc_now!() - POPULATE_WINDOW_MS;
+        let mut entries = self.entries.lock().unwrap();
+
+        for segment_reader in searcher.segment_readers() {
+            let account_col = match segment_reader.fast_fields().u64(F_ACCOUNT_ID) {
+                Ok(c) => c,
+                Err(_) => continue,
+            };
+            let mailbox_col = match segment_reader.fast_fields().u64(F_MAILBOX_ID) {
+                Ok(c) => c,
+                Err(_) => continue,
+            };
+            let hash_col = match segment_reader.fast_fields().str(F_CONTENT_HASH) {
+                Ok(Some(c)) => c,
+                _ => continue,
+            };
+            let ingest_col = match segment_reader.fast_fields().i64(F_INGEST_AT) {
+                Ok(c) => c,
+                Err(_) => continue,
+            };
+
+            let max_doc = segment_reader.max_doc();
+            for doc_id in 0..max_doc {
+                if segment_reader.is_deleted(doc_id) {
+                    continue;
+                }
+                let ingest_at = ingest_col.values.get_val(doc_id);
+                if ingest_at < cutoff {
+                    continue;
+                }
+                let account_id = account_col.values.get_val(doc_id);
+                let mailbox_id = mailbox_col.values.get_val(doc_id);
+
+                let hash_ord = hash_col
+                    .ords()
+                    .values_for_doc(doc_id as u32)
+                    .next()
+                    .unwrap_or(0);
+                let mut hash_buf = String::new();
+                if hash_col.ord_to_str(hash_ord, &mut hash_buf).is_err() {
+                    continue;
+                }
+
+                entries.insert((account_id, mailbox_id, hash_buf), ingest_at);
+            }
+        }
+
+        tracing::info!(
+            "DedupCache populated with {} entries from index (cutoff {}d ago)",
+            entries.len(),
+            POPULATE_WINDOW_MS / (24 * 60 * 60 * 1000),
+        );
+    }
+
+    /// Remove all entries for a specific account.
+    pub fn remove_by_account(&self, account_id: u64) {
+        let mut entries = self.entries.lock().unwrap();
+        let before = entries.len();
+        entries.retain(|(aid, _, _), _| *aid != account_id);
+        let removed = before - entries.len();
+        if removed > 0 {
+            tracing::info!(
+                "DedupCache: removed {} entries for account {}",
+                removed,
+                account_id
+            );
+        }
+    }
+
+    /// Remove all entries for a specific mailbox (across all accounts).
+    pub fn remove_by_mailbox(&self, mailbox_id: u64) {
+        let mut entries = self.entries.lock().unwrap();
+        let before = entries.len();
+        entries.retain(|(_, mid, _), _| *mid != mailbox_id);
+        let removed = before - entries.len();
+        if removed > 0 {
+            tracing::info!(
+                "DedupCache: removed {} entries for mailbox {}",
+                removed,
+                mailbox_id
+            );
+        }
+    }
+
+    /// Remove a specific triple (most precise removal).
+    pub fn remove(&self, account_id: u64, mailbox_id: u64, hash: &str) {
+        let mut entries = self.entries.lock().unwrap();
+        if entries
+            .remove(&(account_id, mailbox_id, hash.to_string()))
+            .is_some()
+        {
+            tracing::debug!(
+                "DedupCache: removed specific entry ({}, {}, {})",
+                account_id,
+                mailbox_id,
+                hash
+            );
+        }
+    }
+}
+
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::store::tantivy::fields::EmailFields;
+    use crate::store::tantivy::schema::SchemaTools;
+    use crate::store::tantivy::tokenizers::EuroTokenizer;
+    use std::fs;
+    use tantivy::{Index, TantivyDocument};
+
+    fn temp_dir(name: &str) -> std::path::PathBuf {
+        let dir = std::env::temp_dir()
+            .join("bichon-dedup-cache-test")
+            .join(name)
+            .join(uuid::Uuid::new_v4().to_string());
+        fs::create_dir_all(&dir).unwrap();
+        dir
+    }
+
+    // ── basic contains / insert ─────────────────────────────────────────────
+
+    #[test]
+    fn contains_after_insert() {
+        let cache = DedupCache::new_for_test();
+        assert!(!cache.contains(1, 10, "hash-aaa"));
+        cache.insert(1, 10, "hash-aaa");
+        assert!(cache.contains(1, 10, "hash-aaa"));
+    }
+
+    #[test]
+    fn different_hash_not_matched() {
+        let cache = DedupCache::new_for_test();
+        cache.insert(1, 10, "hash-aaa");
+        assert!(!cache.contains(1, 10, "hash-bbb"));
+    }
+
+    #[test]
+    fn different_account_not_matched() {
+        let cache = DedupCache::new_for_test();
+        cache.insert(1, 10, "hash-aaa");
+        assert!(!cache.contains(2, 10, "hash-aaa"));
+    }
+
+    #[test]
+    fn different_mailbox_not_matched() {
+        let cache = DedupCache::new_for_test();
+        cache.insert(1, 10, "hash-aaa");
+        assert!(!cache.contains(1, 20, "hash-aaa"));
+    }
+
+    #[test]
+    fn cross_account_allowed() {
+        let cache = DedupCache::new_for_test();
+        cache.insert(1, 10, "hash-aaa");
+        cache.insert(2, 10, "hash-aaa");
+        assert!(cache.contains(2, 10, "hash-aaa"));
+        assert!(cache.contains(1, 10, "hash-aaa"));
+    }
+
+    #[test]
+    fn cross_mailbox_allowed() {
+        let cache = DedupCache::new_for_test();
+        cache.insert(1, 10, "hash-aaa");
+        cache.insert(1, 20, "hash-aaa");
+        assert!(cache.contains(1, 20, "hash-aaa"));
+        assert!(cache.contains(1, 10, "hash-aaa"));
+    }
+
+    // ── time-based eviction ─────────────────────────────────────────────────
+
+    #[test]
+    fn eviction_keeps_newest() {
+        let cap = 100;
+        let cache = DedupCache::new_for_test_small(cap);
+
+        // Fill to exact capacity. Entry hash-0 is oldest.
+        for i in 0..cap {
+            cache.insert(1, 1, &format!("hash-{}", i));
+            std::thread::sleep(std::time::Duration::from_micros(100));
+        }
+        assert!(cache.contains(1, 1, "hash-0"));
+        assert!(cache.contains(1, 1, &format!("hash-{}", cap - 1)));
+
+        // One more triggers eviction
+        cache.insert(1, 1, "hash-overflow");
+
+        // Newest survives, oldest evicted
+        assert!(cache.contains(1, 1, "hash-overflow"));
+        assert!(cache.contains(1, 1, &format!("hash-{}", cap - 1)));
+        assert!(!cache.contains(1, 1, "hash-0"));
+
+        let keep = cap * KEEP_FRACTION_NUM / KEEP_FRACTION_DEN;
+        assert!(cache.entries.lock().unwrap().len() <= keep);
+    }
+
+    // ── memory bound ────────────────────────────────────────────────────────
+
+    #[test]
+    fn memory_bound_within_budget() {
+        let cache = DedupCache::new_for_test();
+
+        for i in 0..MAX_ENTRIES {
+            cache.insert(1, 1, &format!("{:064x}", i));
+        }
+
+        let entries = cache.entries.lock().unwrap();
+        assert_eq!(entries.len(), MAX_ENTRIES);
+
+        let capacity = entries.capacity();
+        // HashMap with (u64,u64,String) key + i64 value ≈ 112 + map overhead
+        let approx_bytes = capacity * (104 + 8 + 8);
+        let approx_mb = approx_bytes as f64 / (1024.0 * 1024.0);
+        println!(
+            "DedupCache: {} entries, {} buckets, ~{:.1} MB",
+            MAX_ENTRIES, capacity, approx_mb
+        );
+        assert!(
+            approx_mb < 55.0,
+            "memory estimate {:.1} MB exceeds 55 MB buffer",
+            approx_mb
+        );
+    }
+
+    // ── populate guard ──────────────────────────────────────────────────────
+
+    #[test]
+    fn populate_cas_is_idempotent() {
+        let cache = DedupCache::new_for_test();
+        assert!(cache.populated.load(Ordering::Acquire));
+        cache.ensure_populated();
+        assert!(cache.populated.load(Ordering::Acquire));
+        cache.do_populate();
+    }
+
+    // ── populate from test index ────────────────────────────────────────────
+
+    fn build_test_index() -> (Index, &'static EmailFields) {
+        let dir = temp_dir("populate");
+        let schema = SchemaTools::email_schema();
+        let fields = SchemaTools::email_fields();
+        let index = Index::create_in_dir(&dir, schema).unwrap();
+        index.tokenizers().register("euro", EuroTokenizer::new());
+        (index, fields)
+    }
+
+    fn add_email_doc(
+        fields: &EmailFields,
+        writer: &mut tantivy::IndexWriter,
+        account: u64,
+        mailbox: u64,
+        hash: &str,
+        ingest_at: i64,
+    ) {
+        let mut doc = TantivyDocument::new();
+        doc.add_u64(fields.f_account_id, account);
+        doc.add_u64(fields.f_mailbox_id, mailbox);
+        doc.add_text(fields.f_content_hash, hash);
+        doc.add_i64(fields.f_ingest_at, ingest_at);
+        doc.add_text(fields.f_id, &uuid::Uuid::new_v4().to_string());
+        doc.add_text(fields.f_subject, "test");
+        doc.add_text(fields.f_body, "test body");
+        doc.add_u64(fields.f_uid, 1);
+        doc.add_i64(fields.f_date, 1);
+        doc.add_i64(fields.f_internal_date, 1);
+        doc.add_u64(fields.f_size, 100);
+        writer.add_document(doc).unwrap();
+    }
+
+    #[test]
+    fn populate_reads_all_docs_in_window() {
+        let (index, fields) = build_test_index();
+        let mut writer = index.writer_with_num_threads(1, 50_000_000).unwrap();
+
+        let recent = utc_now!();
+        add_email_doc(&fields, &mut writer, 1, 10, "hash-recent", recent);
+        add_email_doc(&fields, &mut writer, 2, 10, "hash-recent", recent);
+        add_email_doc(&fields, &mut writer, 1, 20, "hash-recent", recent);
+        writer.commit().unwrap();
+        drop(writer);
+
+        let reader = index.reader().unwrap();
+        let cache = DedupCache::new_for_test();
+        {
+            let searcher = reader.searcher();
+            let cutoff = utc_now!() - POPULATE_WINDOW_MS;
+            let mut entries = cache.entries.lock().unwrap();
+            entries.clear();
+
+            for segment_reader in searcher.segment_readers() {
+                let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap();
+                let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap();
+                let hash_col = segment_reader
+                    .fast_fields()
+                    .str(F_CONTENT_HASH)
+                    .unwrap()
+                    .unwrap();
+                let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap();
+
+                let max_doc = segment_reader.max_doc();
+                for doc_id in 0..max_doc {
+                    if segment_reader.is_deleted(doc_id) {
+                        continue;
+                    }
+                    let ingest_at = ingest_col.values.get_val(doc_id);
+                    if ingest_at < cutoff {
+                        continue;
+                    }
+                    let account_id = account_col.values.get_val(doc_id);
+                    let mailbox_id = mailbox_col.values.get_val(doc_id);
+
+                    let hash_ord = hash_col
+                        .ords()
+                        .values_for_doc(doc_id as u32)
+                        .next()
+                        .unwrap_or(0);
+                    let mut hash_buf = String::new();
+                    hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap();
+
+                    entries.insert((account_id, mailbox_id, hash_buf), ingest_at);
+                }
+            }
+        }
+
+        assert_eq!(cache.entries.lock().unwrap().len(), 3);
+        assert!(cache.contains(1, 10, "hash-recent"));
+        assert!(cache.contains(2, 10, "hash-recent"));
+        assert!(cache.contains(1, 20, "hash-recent"));
+    }
+
+    #[test]
+    fn populate_skips_old_entries() {
+        let (index, fields) = build_test_index();
+        let mut writer = index.writer_with_num_threads(1, 50_000_000).unwrap();
+
+        let recent = utc_now!();
+        let old = recent - POPULATE_WINDOW_MS - 60_000; // 1 minute past the window
+        add_email_doc(&fields, &mut writer, 1, 10, "hash-recent", recent);
+        add_email_doc(&fields, &mut writer, 1, 10, "hash-old", old);
+        writer.commit().unwrap();
+        drop(writer);
+
+        let reader = index.reader().unwrap();
+        let cache = DedupCache::new_for_test();
+        {
+            let searcher = reader.searcher();
+            let cutoff = utc_now!() - POPULATE_WINDOW_MS;
+            let mut entries = cache.entries.lock().unwrap();
+            entries.clear();
+
+            for segment_reader in searcher.segment_readers() {
+                let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap();
+                let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap();
+                let hash_col = segment_reader
+                    .fast_fields()
+                    .str(F_CONTENT_HASH)
+                    .unwrap()
+                    .unwrap();
+                let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap();
+
+                let max_doc = segment_reader.max_doc();
+                for doc_id in 0..max_doc {
+                    if segment_reader.is_deleted(doc_id) {
+                        continue;
+                    }
+                    let ingest_at = ingest_col.values.get_val(doc_id);
+                    if ingest_at < cutoff {
+                        continue;
+                    }
+                    let account_id = account_col.values.get_val(doc_id);
+                    let mailbox_id = mailbox_col.values.get_val(doc_id);
+
+                    let hash_ord = hash_col
+                        .ords()
+                        .values_for_doc(doc_id as u32)
+                        .next()
+                        .unwrap_or(0);
+                    let mut hash_buf = String::new();
+                    hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap();
+
+                    entries.insert((account_id, mailbox_id, hash_buf), ingest_at);
+                }
+            }
+        }
+
+        assert!(cache.contains(1, 10, "hash-recent"));
+        assert!(!cache.contains(1, 10, "hash-old"));
+        assert_eq!(cache.entries.lock().unwrap().len(), 1);
+    }
+
+    #[test]
+    fn populate_skips_deleted_docs() {
+        let (index, fields) = build_test_index();
+        let mut writer = index.writer_with_num_threads(1, 50_000_000).unwrap();
+
+        let recent = utc_now!();
+        add_email_doc(&fields, &mut writer, 1, 10, "hash-keep", recent);
+        add_email_doc(&fields, &mut writer, 1, 10, "hash-delete", recent);
+        writer.commit().unwrap();
+
+        let term = tantivy::Term::from_field_text(fields.f_content_hash, "hash-delete");
+        writer.delete_term(term);
+        writer.commit().unwrap();
+        drop(writer);
+
+        let reader = index.reader().unwrap();
+        let cache = DedupCache::new_for_test();
+        {
+            let searcher = reader.searcher();
+            let cutoff = utc_now!() - POPULATE_WINDOW_MS;
+            let mut entries = cache.entries.lock().unwrap();
+            entries.clear();
+
+            for segment_reader in searcher.segment_readers() {
+                let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap();
+                let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap();
+                let hash_col = segment_reader
+                    .fast_fields()
+                    .str(F_CONTENT_HASH)
+                    .unwrap()
+                    .unwrap();
+                let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap();
+
+                let max_doc = segment_reader.max_doc();
+                for doc_id in 0..max_doc {
+                    if segment_reader.is_deleted(doc_id) {
+                        continue;
+                    }
+                    let ingest_at = ingest_col.values.get_val(doc_id);
+                    if ingest_at < cutoff {
+                        continue;
+                    }
+                    let account_id = account_col.values.get_val(doc_id);
+                    let mailbox_id = mailbox_col.values.get_val(doc_id);
+
+                    let hash_ord = hash_col
+                        .ords()
+                        .values_for_doc(doc_id as u32)
+                        .next()
+                        .unwrap_or(0);
+                    let mut hash_buf = String::new();
+                    hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap();
+
+                    entries.insert((account_id, mailbox_id, hash_buf), ingest_at);
+                }
+            }
+        }
+
+        assert!(cache.contains(1, 10, "hash-keep"));
+        assert!(!cache.contains(1, 10, "hash-delete"));
+    }
+
+    // ── removal methods ─────────────────────────────────────────────────────
+
+    #[test]
+    fn remove_by_account_works() {
+        let cache = DedupCache::new_for_test();
+
+        cache.insert(1, 10, "hash-a1");
+        cache.insert(1, 20, "hash-a2");
+        cache.insert(2, 10, "hash-b1");
+        cache.insert(2, 30, "hash-b2");
+        cache.insert(1, 10, "hash-a3");
+
+        assert_eq!(cache.entries.lock().unwrap().len(), 5);
+
+        cache.remove_by_account(1);
+
+        let entries = cache.entries.lock().unwrap();
+        assert_eq!(entries.len(), 2);
+        assert!(!entries.contains_key(&(1, 10, "hash-a1".to_string())));
+        assert!(!entries.contains_key(&(1, 20, "hash-a2".to_string())));
+        assert!(!entries.contains_key(&(1, 10, "hash-a3".to_string())));
+
+        assert!(entries.contains_key(&(2, 10, "hash-b1".to_string())));
+        assert!(entries.contains_key(&(2, 30, "hash-b2".to_string())));
+    }
+
+    #[test]
+    fn remove_by_mailbox_works() {
+        let cache = DedupCache::new_for_test();
+
+        cache.insert(1, 10, "hash-1");
+        cache.insert(1, 20, "hash-2");
+        cache.insert(2, 10, "hash-3");
+        cache.insert(3, 20, "hash-4");
+        cache.insert(1, 10, "hash-5");
+
+        cache.remove_by_mailbox(10);
+
+        let entries = cache.entries.lock().unwrap();
+        assert_eq!(entries.len(), 2);
+
+        assert!(entries.contains_key(&(1, 20, "hash-2".to_string())));
+        assert!(entries.contains_key(&(3, 20, "hash-4".to_string())));
+
+        assert!(!entries.contains_key(&(1, 10, "hash-1".to_string())));
+        assert!(!entries.contains_key(&(2, 10, "hash-3".to_string())));
+    }
+
+    #[test]
+    fn remove_specific_triple_works() {
+        let cache = DedupCache::new_for_test();
+
+        cache.insert(1, 10, "hash-aaa");
+        cache.insert(1, 10, "hash-bbb");
+        cache.insert(2, 20, "hash-aaa");
+
+        assert!(cache.contains(1, 10, "hash-aaa"));
+        assert!(cache.contains(1, 10, "hash-bbb"));
+        assert!(cache.contains(2, 20, "hash-aaa"));
+
+        cache.remove(1, 10, "hash-aaa");
+
+        assert!(!cache.contains(1, 10, "hash-aaa"));
+        assert!(cache.contains(1, 10, "hash-bbb"));
+        assert!(cache.contains(2, 20, "hash-aaa"));
+    }
+}

crates/core/src/store/tantivy/mod.rs

@@ -25,6 +25,7 @@ use crate::{
 
 pub mod attachment;
 pub mod dedup;
+pub mod dedup_cache;
 pub mod envelope;
 pub mod fields;
 pub mod filter;

crates/core/src/store/tantivy/model.rs

@@ -28,7 +28,7 @@ use crate::{
     raise_error,
     {
         account::migration::AccountModel,
-        cache::imap::mailbox::MailBox,
+        archive::imap::mailbox::MailBox,
         error::{code::ErrorCode, BichonResult},
         message::content::AttachmentInfo,
         store::{
@@ -155,7 +155,8 @@ impl EnvelopeWithAttachments {
             id: extract_string_field(doc, fields.f_id, F_ID)?,
             message_id: extract_string_field(doc, fields.f_message_id, F_MESSAGE_ID)?,
             account_id,
-            account_email: Some(account.email),
+            account_email: Some(account.email), //https://github.com/rustmailer/bichon/issues/306
+            account_name: account.account_name,
             mailbox_id,
             mailbox_name: Some(mailbox.name),
             uid: extract_u64_field(doc, fields.f_uid, F_UID)? as u32,

crates/core/src/token/mod.rs

@@ -156,10 +156,16 @@ impl AccessTokenModel {
             })?;
 
         if matches!(token_model.token_type, TokenType::WebUI) {
-            let life = utc_now!() - token_model.created_at;
-            let max_life = SETTINGS.bichon_webui_token_expiration_hours * 60 * 60 * 1000;
+            // Use last_access_at if set, otherwise fall back to created_at
+            let last_active = if token_model.last_access_at > 0 {
+                token_model.last_access_at
+            } else {
+                token_model.created_at
+            };
+            let idle = utc_now!() - last_active;
+            let max_life = SETTINGS.bichon_webui_token_expiration_hours as i64 * 60 * 60 * 1000;
 
-            if life > (max_life as i64) {
+            if idle > max_life {
                 return Err(raise_error!(
                     "Permission denied: the WebUI token has expired.".into(),
                     ErrorCode::PermissionDenied
@@ -176,13 +182,15 @@ impl AccessTokenModel {
                     ));
                 }
             }
-            update_impl(DB_MANAGER.db(), &token_str, |current: AccessTokenModel| {
-                let mut updated = current.clone();
-                updated.last_access_at = utc_now!();
-                Ok(updated)
-            })?;
         }
 
+        // Update last_access_at on every successful use for both token types
+        update_impl(DB_MANAGER.db(), &token_str, |current: AccessTokenModel| {
+            let mut updated = current.clone();
+            updated.last_access_at = utc_now!();
+            Ok(updated)
+        })?;
+
         let user = UserModel::find(token_model.user_id)
             ?
             .ok_or_else(|| raise_error!("The user associated with this access token does not exist or may have been deleted.".into(), ErrorCode::ResourceNotFound))?;

crates/core/src/users/mod.rs

@@ -87,6 +87,11 @@ pub struct BichonUserV2 {
 
     pub theme: Option<String>,
     pub language: Option<String>,
+
+    /// SSO identity: unique subject ID from the external IdP (e.g. OIDC `sub` claim).
+    pub sso_id: Option<String>,
+    /// SSO provider identifier: `"oidc"` or future `"saml"` / `"ldap"`.
+    pub sso_provider: Option<String>,
 }
 
 impl MemDbModel for BichonUserV2 {
@@ -192,6 +197,8 @@ impl BichonUserV2 {
             global_permissions,
             theme: self.theme,
             language: self.language,
+            sso_id: self.sso_id,
+            sso_provider: self.sso_provider,
         }
     }
 
@@ -226,6 +233,8 @@ impl BichonUserV2 {
                 acl: None,
                 theme: None,
                 language: None,
+                sso_id: None,
+                sso_provider: None,
             };
 
             // 3. Generate and insert an initial access token for the first-time setup
@@ -382,6 +391,8 @@ impl BichonUserV2 {
             account_access_map: request.account_access_map,
             theme: request.theme,
             language: request.language,
+            sso_id: None,
+            sso_provider: None,
         };
 
         let user_clone = user.clone();

crates/core/src/users/view.rs

@@ -52,4 +52,9 @@ pub struct UserView {
     pub acl: Option<AccessControl>,
     pub theme: Option<String>,
     pub language: Option<String>,
+
+    /// SSO identity: unique subject ID from the external IdP (e.g. OIDC `sub` claim).
+    pub sso_id: Option<String>,
+    /// SSO provider identifier: `"oidc"` or future `"saml"` / `"ldap"`.
+    pub sso_provider: Option<String>,
 }

crates/core/src/utils/mod.rs

@@ -138,7 +138,7 @@ macro_rules! generate_token {
     }};
 }
 
-pub(crate) fn generate_token_impl(bit_strength: usize) -> String {
+pub fn generate_token_impl(bit_strength: usize) -> String {
     let byte_length = (bit_strength + 23) / 24 * 3;
     let random_bytes: Vec<u8> = (0..byte_length).map(|_| rand::random::<u8>()).collect();
     let mut encoded = general_purpose::URL_SAFE.encode(&random_bytes);

crates/core/src/utils/net.rs

@@ -18,12 +18,14 @@
 
 use crate::error::code::ErrorCode;
 use crate::raise_error;
-use crate::settings::proxy::Proxy;
+use crate::settings::{cli::SETTINGS, proxy::Proxy};
 use crate::utils::tls::establish_tls_stream;
 use crate::{error::BichonResult, imap::session::SessionStream};
+use base64::{engine::general_purpose, Engine as _};
 use std::net::SocketAddr;
 use std::pin::Pin;
 use std::time::Duration;
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::net::TcpStream;
 use tokio::time::timeout;
 use tokio_io_timeout::TimeoutStream;
@@ -32,6 +34,54 @@ use tracing::error;
 
 pub(crate) const TIMEOUT: Duration = Duration::from_secs(30);
 
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+pub enum ProxyScheme {
+    Socks5,
+    Http,
+}
+
+impl ProxyScheme {
+    fn as_str(self) -> &'static str {
+        match self {
+            Self::Socks5 => "socks5",
+            Self::Http => "http",
+        }
+    }
+}
+
+/// Parsed proxy address components.
+#[derive(Debug, Clone)]
+pub struct ProxyAddr {
+    pub scheme: ProxyScheme,
+    pub host: String,
+    pub port: u16,
+    pub username: Option<String>,
+    pub password: Option<String>,
+}
+
+impl ProxyAddr {
+    pub fn standard_url(&self) -> String {
+        let host = if self.host.contains(':') {
+            format!("[{}]", self.host)
+        } else {
+            self.host.clone()
+        };
+
+        if let (Some(user), Some(pass)) = (&self.username, &self.password) {
+            format!(
+                "{}://{}:{}@{}:{}",
+                self.scheme.as_str(),
+                user,
+                pass,
+                host,
+                self.port
+            )
+        } else {
+            format!("{}://{}:{}", self.scheme.as_str(), host, self.port)
+        }
+    }
+}
+
 pub(crate) async fn establish_tcp_connection_with_timeout(
     address: SocketAddr,
     use_proxy: Option<u64>,
@@ -40,9 +90,16 @@ pub(crate) async fn establish_tcp_connection_with_timeout(
     let tcp_stream = connect_with_optional_proxy(use_proxy, address).await?;
     let mut timeout_stream = TimeoutStream::new(tcp_stream);
 
-    // Set read and write timeouts
+    // Set read and write timeouts. The read timeout bounds how long the sync
+    // task blocks waiting for a slow IMAP server between commands; a hung
+    // server therefore surfaces as a network error (and retry) instead of a
+    // silently stuck download. 0 disables the read timeout (server decides).
+    let read_timeout = SETTINGS
+        .bichon_imap_timeout_seconds
+        .checked_sub(1)
+        .map(|seconds| Duration::from_secs(seconds.max(1)));
     timeout_stream.set_write_timeout(Some(Duration::from_secs(15)));
-    timeout_stream.set_read_timeout(Some(Duration::from_secs(30)));
+    timeout_stream.set_read_timeout(read_timeout);
 
     // Return the timeout-wrapped TCP stream as a Pin
     Ok(Box::pin(timeout_stream))
@@ -66,74 +123,247 @@ pub async fn establish_tls_connection(
     Ok(tls_stream)
 }
 
-pub fn parse_proxy_addr(input: &str) -> BichonResult<SocketAddr> {
-    // Normalize and check protocol prefix
+/// Parse a proxy URL into its components.
+///
+/// Supports two formats:
+/// - **Standard**: `[scheme://][user:pass@]host:port`
+/// - **Non-standard** (some proxy providers): `[scheme://]host:port:username:password`
+///
+/// The distinguishing feature is the `@` sign in the standard format.
+pub fn parse_proxy_url(input: &str) -> BichonResult<ProxyAddr> {
+    // Normalize and strip scheme prefix
     let (scheme, stripped) = if let Some(rest) = input
         .strip_prefix("socks5://")
         .or_else(|| input.strip_prefix("SOCKS5://"))
         .or_else(|| input.strip_prefix("Socks5://"))
     {
-        ("socks5", rest)
+        (ProxyScheme::Socks5, rest)
     } else if let Some(rest) = input
         .strip_prefix("http://")
         .or_else(|| input.strip_prefix("HTTP://"))
         .or_else(|| input.strip_prefix("Http://"))
     {
-        ("http", rest)
+        (ProxyScheme::Http, rest)
     } else {
         return Err(raise_error!(
-            format!(
-                "Invalid proxy URL: must start with 'http://' or 'socks5://', got '{}'",
-                input
-            ),
+            "Invalid proxy URL: must start with 'http://' or 'socks5://'".into(),
             ErrorCode::InvalidParameter
         ));
     };
 
-    // Parse the remaining address
-    let addr = stripped.parse::<SocketAddr>().map_err(|e| {
+    if stripped.is_empty() {
+        return Err(raise_error!(
+            "Proxy URL has empty address after scheme.".into(),
+            ErrorCode::InvalidParameter
+        ));
+    }
+
+    // Check for standard format: user:pass@host:port
+    if let Some(at_pos) = stripped.rfind('@') {
+        let userinfo = &stripped[..at_pos];
+        let hostport = &stripped[at_pos + 1..];
+
+        let (username, password) = split_userinfo(userinfo)?;
+        let (host, port) = split_hostport(hostport)?;
+
+        return Ok(ProxyAddr {
+            scheme,
+            host,
+            port,
+            username,
+            password,
+        });
+    }
+
+    // No '@' — check for non-standard format: host:port:user:pass
+    if stripped.starts_with('[') {
+        let (host, port) = split_hostport(stripped)?;
+        return Ok(ProxyAddr {
+            scheme,
+            host,
+            port,
+            username: None,
+            password: None,
+        });
+    }
+
+    let mut parts = stripped.split(':');
+    match (
+        parts.next(),
+        parts.next(),
+        parts.next(),
+        parts.next(),
+        parts.next(),
+    ) {
+        (Some(_), Some(_), None, None, None) => {
+            // host:port, no auth
+            let (host, port) = split_hostport(stripped)?;
+            Ok(ProxyAddr {
+                scheme,
+                host,
+                port,
+                username: None,
+                password: None,
+            })
+        }
+        (Some(host), Some(port), Some(username), Some(password), None) => {
+            // Non-standard: host:port:username:password
+            let port = port.parse::<u16>().map_err(|_| {
+                raise_error!(
+                    format!("Invalid port '{}' in proxy URL.", port),
+                    ErrorCode::InvalidParameter
+                )
+            })?;
+
+            if host.is_empty() {
+                return Err(raise_error!(
+                    "Empty hostname in proxy URL.".into(),
+                    ErrorCode::InvalidParameter
+                ));
+            }
+            if host.contains(':') || host.contains('[') || host.contains(']') {
+                return Err(raise_error!(
+                    "IPv6 proxy hosts are not supported.".into(),
+                    ErrorCode::InvalidParameter
+                ));
+            }
+            if username.is_empty() {
+                return Err(raise_error!(
+                    "Empty username in proxy URL.".into(),
+                    ErrorCode::InvalidParameter
+                ));
+            }
+            if password.is_empty() {
+                return Err(raise_error!(
+                    "Empty password in proxy URL.".into(),
+                    ErrorCode::InvalidParameter
+                ));
+            }
+
+            Ok(ProxyAddr {
+                scheme,
+                host: host.to_string(),
+                port,
+                username: Some(username.to_string()),
+                password: Some(password.to_string()),
+            })
+        }
+        _ => Err(raise_error!(
+            "Invalid proxy URL format. Expected '[scheme://][user:pass@]host:port' or 'scheme://host:port:user:pass'.".into(),
+            ErrorCode::InvalidParameter
+        )),
+    }
+}
+
+/// Split "user:pass" into (Some(user), Some(pass)).
+fn split_userinfo(userinfo: &str) -> BichonResult<(Option<String>, Option<String>)> {
+    if userinfo.is_empty() {
+        return Ok((None, None));
+    }
+    if let Some(colon_pos) = userinfo.find(':') {
+        let user = &userinfo[..colon_pos];
+        let pass = &userinfo[colon_pos + 1..];
+        if user.is_empty() {
+            return Err(raise_error!(
+                "Empty username in proxy URL credentials.".into(),
+                ErrorCode::InvalidParameter
+            ));
+        }
+        if pass.is_empty() {
+            return Err(raise_error!(
+                "Empty password in proxy URL credentials.".into(),
+                ErrorCode::InvalidParameter
+            ));
+        }
+        Ok((Some(user.to_string()), Some(pass.to_string())))
+    } else {
+        Err(raise_error!(
+            "Password cannot be empty when username is provided.".into(),
+            ErrorCode::InvalidParameter
+        ))
+    }
+}
+
+/// Split "host:port" into (host, port). Bracketed IPv6 is accepted.
+fn split_hostport(hostport: &str) -> BichonResult<(String, u16)> {
+    if hostport.is_empty() {
+        return Err(raise_error!(
+            "Empty host:port in proxy URL.".into(),
+            ErrorCode::InvalidParameter
+        ));
+    }
+
+    if let Some(rest) = hostport.strip_prefix('[') {
+        let Some(close_bracket) = rest.find(']') else {
+            return Err(raise_error!(
+                format!("Invalid IPv6 address in proxy URL: '{}'.", hostport),
+                ErrorCode::InvalidParameter
+            ));
+        };
+        let host = &rest[..close_bracket];
+        let port_text = rest[close_bracket + 1..].strip_prefix(':').ok_or_else(|| {
+            raise_error!(
+                format!(
+                    "Missing port after IPv6 address in proxy URL: '{}'.",
+                    hostport
+                ),
+                ErrorCode::InvalidParameter
+            )
+        })?;
+        let port = port_text.parse::<u16>().map_err(|_| {
+            raise_error!(
+                format!("Invalid port in proxy URL: '{}'.", hostport),
+                ErrorCode::InvalidParameter
+            )
+        })?;
+
+        return Ok((host.to_string(), port));
+    }
+
+    // hostname:port or ip:port — split from right
+    let last_colon = hostport.rfind(':').ok_or_else(|| {
         raise_error!(
-            format!(
-                "Failed to parse {} proxy address '{}': {}",
-                scheme, stripped, e
-            ),
+            format!("Missing port in proxy URL: '{}'.", hostport),
+            ErrorCode::InvalidParameter
+        )
+    })?;
+    let host = hostport[..last_colon].to_string();
+    let port = hostport[last_colon + 1..].parse::<u16>().map_err(|_| {
+        raise_error!(
+            format!("Invalid port in proxy URL: '{}'.", hostport),
             ErrorCode::InvalidParameter
         )
     })?;
 
-    Ok(addr)
+    if host.is_empty() {
+        return Err(raise_error!(
+            "Empty hostname in proxy URL.".into(),
+            ErrorCode::InvalidParameter
+        ));
+    }
+    if host.contains(':') || host.contains('[') || host.contains(']') {
+        return Err(raise_error!(
+            "IPv6 proxy hosts are not supported.".into(),
+            ErrorCode::InvalidParameter
+        ));
+    }
+
+    Ok((host, port))
 }
 
-/// Try to connect via SOCKS5 proxy or TCP with timeout
+/// Try to connect via SOCKS5 proxy or TCP with timeout.
 async fn connect_with_optional_proxy(
     use_proxy: Option<u64>,
     address: SocketAddr,
 ) -> BichonResult<TcpStream> {
-    // Try if proxy is enabled
     if let Some(proxy_id) = use_proxy {
         let proxy = Proxy::get(proxy_id)?;
-        let proxy = parse_proxy_addr(&proxy.url)?;
-        return timeout(TIMEOUT, Socks5Stream::connect(proxy, address))
-            .await
-            .map_err(|_| {
-                error!(
-                    "SOCKS5 proxy connection to {} via {} timed out after {}s",
-                    address,
-                    proxy,
-                    TIMEOUT.as_secs()
-                );
-                raise_error!(
-                    format!(
-                        "SOCKS5 proxy connection to {} via {} timed out after {}s",
-                        address,
-                        proxy,
-                        TIMEOUT.as_secs()
-                    ),
-                    ErrorCode::ConnectionTimeout
-                )
-            })?
-            .map(|s| s.into_inner())
-            .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError));
+        let addr = parse_proxy_url(&proxy.url)?;
+        return if addr.scheme == ProxyScheme::Http {
+            connect_via_http_proxy(&addr, address).await
+        } else {
+            connect_via_socks5_proxy(&addr, address).await
+        };
     }
     // Fallback to direct TCP connection
     timeout(TIMEOUT, TcpStream::connect(address))
@@ -155,3 +385,135 @@ async fn connect_with_optional_proxy(
         })?
         .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError))
 }
+
+async fn connect_via_socks5_proxy(
+    addr: &ProxyAddr,
+    address: SocketAddr,
+) -> BichonResult<TcpStream> {
+    let proxy_addr = (addr.host.as_str(), addr.port);
+    let result = if let (Some(user), Some(pass)) = (&addr.username, &addr.password) {
+        timeout(
+            TIMEOUT,
+            Socks5Stream::connect_with_password(proxy_addr, address, user.as_str(), pass.as_str()),
+        )
+        .await
+    } else {
+        timeout(TIMEOUT, Socks5Stream::connect(proxy_addr, address)).await
+    };
+
+    result
+        .map_err(|_| {
+            error!(
+                "SOCKS5 proxy connection to {} via {}:{} timed out after {}s",
+                address,
+                addr.host,
+                addr.port,
+                TIMEOUT.as_secs()
+            );
+            raise_error!(
+                format!(
+                    "SOCKS5 proxy connection to {} via {}:{} timed out after {}s",
+                    address,
+                    addr.host,
+                    addr.port,
+                    TIMEOUT.as_secs()
+                ),
+                ErrorCode::ConnectionTimeout
+            )
+        })?
+        .map(|s| s.into_inner())
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError))
+}
+
+async fn connect_via_http_proxy(addr: &ProxyAddr, address: SocketAddr) -> BichonResult<TcpStream> {
+    let mut stream = timeout(TIMEOUT, TcpStream::connect((addr.host.as_str(), addr.port)))
+        .await
+        .map_err(|_| {
+            raise_error!(
+                format!(
+                    "HTTP proxy connection to {}:{} timed out after {}s",
+                    addr.host,
+                    addr.port,
+                    TIMEOUT.as_secs()
+                ),
+                ErrorCode::ConnectionTimeout
+            )
+        })?
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError))?;
+
+    let mut request = format!(
+        "CONNECT {address} HTTP/1.1\r\nHost: {address}\r\nProxy-Connection: keep-alive\r\n"
+    );
+    if let (Some(user), Some(pass)) = (&addr.username, &addr.password) {
+        let auth = general_purpose::STANDARD.encode(format!("{user}:{pass}"));
+        request.push_str(&format!("Proxy-Authorization: Basic {auth}\r\n"));
+    }
+    request.push_str("\r\n");
+
+    timeout(TIMEOUT, stream.write_all(request.as_bytes()))
+        .await
+        .map_err(|_| {
+            raise_error!(
+                format!(
+                    "HTTP proxy CONNECT to {} via {}:{} timed out after {}s",
+                    address,
+                    addr.host,
+                    addr.port,
+                    TIMEOUT.as_secs()
+                ),
+                ErrorCode::ConnectionTimeout
+            )
+        })?
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError))?;
+
+    let mut response = Vec::new();
+    timeout(TIMEOUT, async {
+        let mut byte = [0u8; 1];
+        while !response.ends_with(b"\r\n\r\n") {
+            stream.read_exact(&mut byte).await?;
+            response.push(byte[0]);
+            if response.len() > 8192 {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::InvalidData,
+                    "HTTP proxy CONNECT response headers are too large",
+                ));
+            }
+        }
+        Ok::<(), std::io::Error>(())
+    })
+    .await
+    .map_err(|_| {
+        raise_error!(
+            format!(
+                "HTTP proxy CONNECT response from {}:{} timed out after {}s",
+                addr.host,
+                addr.port,
+                TIMEOUT.as_secs()
+            ),
+            ErrorCode::ConnectionTimeout
+        )
+    })?
+    .map_err(|e| {
+        if e.kind() == std::io::ErrorKind::InvalidData {
+            raise_error!(e.to_string(), ErrorCode::NetworkError)
+        } else {
+            raise_error!(format!("{:#?}", e), ErrorCode::NetworkError)
+        }
+    })?;
+
+    let response = String::from_utf8_lossy(&response);
+    if response.starts_with("HTTP/1.1 200") || response.starts_with("HTTP/1.0 200") {
+        Ok(stream)
+    } else {
+        Err(raise_error!(
+            format!(
+                "HTTP proxy CONNECT to {} via {}:{} failed: {}",
+                address,
+                addr.host,
+                addr.port,
+                response.lines().next().unwrap_or("invalid response")
+            ),
+            ErrorCode::NetworkError
+        ))
+    }
+}

crates/memdb/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "memdb"
+name = "bichon-memdb"
 version = "0.1.0"
 edition = "2021"
 

crates/memdb/tests/integration.rs

@@ -1,4 +1,4 @@
-use memdb::{DbError, MemDb, Page};
+use bichon_memdb::{DbError, MemDb, Page};
 use serde::{Deserialize, Serialize};
 use tempfile::TempDir;
 
@@ -458,7 +458,7 @@ fn test_wal_seq_skips_already_snapshotted_entries() {
 
     // Verify WAL only contains seq=3.
     let wal_path = dir.path().join("wal.jsonl");
-    let entries = memdb::wal::read_after(&wal_path, 2).unwrap();
+    let entries = bichon_memdb::wal::read_after(&wal_path, 2).unwrap();
     assert_eq!(entries.len(), 1);
     assert_eq!(entries[0].seq, 3);
 
@@ -696,7 +696,7 @@ async fn test_concurrent_writes_wal_seq_monotonic() {
 
     // Verify WAL seq is strictly monotonic.
     let wal_path = dir.path().join("wal.jsonl");
-    let entries = memdb::wal::read_after(&wal_path, 0).unwrap();
+    let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap();
     assert_eq!(entries.len(), 50);
     let mut last = 0u64;
     for e in &entries {

crates/memdb/tests/stress.rs

@@ -1,4 +1,4 @@
-use memdb::{Durability, MemDb, Page};
+use bichon_memdb::{Durability, MemDb, Page};
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
@@ -548,7 +548,7 @@ fn stress_wal_seq_monotonic_under_load() {
     }
 
     let wal_path = dir.path().join("wal.jsonl");
-    let entries = memdb::wal::read_after(&wal_path, 0).unwrap();
+    let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap();
     assert_eq!(entries.len(), n);
     let mut last = 0u64;
     for e in &entries {
@@ -842,7 +842,7 @@ async fn wal_concurrent_persistent_writes() {
 
     // Verify strict seq ordering in WAL under concurrent load.
     let wal_path = db_path.join("wal.jsonl");
-    let entries = memdb::wal::read_after(&wal_path, 0).unwrap();
+    let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap();
     assert_eq!(entries.len(), total as usize);
     let mut last = 0u64;
     for e in &entries {
@@ -1153,7 +1153,7 @@ fn wal_large_transaction_batch() {
 
     // The entire transaction should be a single WAL entry.
     let wal_path = dir.path().join("wal.jsonl");
-    let entries = memdb::wal::read_after(&wal_path, 0).unwrap();
+    let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap();
     assert_eq!(entries.len(), 1);
     assert_eq!(entries[0].ops.len(), n as usize);