diff --git a/Cargo.lock b/Cargo.lock index a5cddcd..12b2365 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -308,7 +308,7 @@ dependencies = [ "imap-proto", "log", "nom 7.1.3", - "pin-project", + "pin-project 1.1.11", "pin-utils", "self_cell", "stop-token", @@ -465,6 +465,13 @@ dependencies = [ "console", "dialoguer", "indicatif", + "itertools", + "memdb", + "native_db", + "native_model", + "serde", + "serde_json", + "snafu", "tokio", ] @@ -518,9 +525,8 @@ dependencies = [ "lru 0.18.0", "mail-parser", "mail-send", + "memdb", "murmur3", - "native_db", - "native_model", "num_cpus", "oauth2", "poem-openapi", @@ -570,6 +576,7 @@ dependencies = [ "serde", "serde_json", "snafu", + "tempfile", "timeago", "tokio", "tracing", @@ -605,6 +612,15 @@ dependencies = [ "serde", ] +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "2.11.1" @@ -1378,6 +1394,16 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags", + "objc2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -1794,6 +1820,18 @@ dependencies = [ "slab", ] +[[package]] +name = "futures_codec" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce54d63f8b0c75023ed920d46fd71d0cbbb830b0ee012726b5b4f506fb6dea5b" +dependencies = [ + "bytes 0.5.6", + "futures", + "memchr", + "pin-project 0.4.30", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1911,7 +1949,7 @@ dependencies = [ "parking_lot", "portable-atomic", "quanta", - "rand 0.9.4", + "rand 0.9.2", "smallvec", "spinning_top 0.3.0", "web-time", @@ -2052,7 +2090,7 @@ dependencies = [ "idna 1.1.0", "ipnet", "once_cell", - "rand 0.9.4", + "rand 0.9.2", "ring", "rustls", "rustls-pki-types", @@ -2078,7 +2116,7 @@ dependencies = [ "moka", "once_cell", "parking_lot", - "rand 0.9.4", + "rand 0.9.2", "resolv-conf", "rustls", "smallvec", @@ -2909,7 +2947,7 @@ dependencies = [ "mail-auth", "mail-builder", "md5", - "rand 0.9.4", + "rand 0.9.2", "rustls", "rustls-pki-types", "rustls-platform-verifier", @@ -2965,6 +3003,18 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memdb" +version = "0.1.0" +dependencies = [ + "rand 0.9.2", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", +] + [[package]] name = "memmap2" version = "0.9.10" @@ -3277,6 +3327,15 @@ dependencies = [ "url", ] +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + [[package]] name = "objc2-core-foundation" version = "0.3.2" @@ -3284,6 +3343,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags", + "objc2", ] [[package]] @@ -3296,6 +3373,17 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-open-directory" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb82bed227edf5201dfedf072bba4015a33d3d4a98519837295a90f0a23f676d" +dependencies = [ + "objc2", + "objc2-core-foundation", + "objc2-foundation", +] + [[package]] name = "oem_cp" version = "1.3.0" @@ -3523,13 +3611,33 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ef0f924a5ee7ea9cbcea77529dba45f8a9ba9f622419fe3386ca581a3ae9d5a" +dependencies = [ + "pin-project-internal 0.4.30", +] + [[package]] name = "pin-project" version = "1.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" dependencies = [ - "pin-project-internal", + "pin-project-internal 1.1.11", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851c8d0ce9bebe43790dedfc86614c23494ac9f423dd618d3a61fc693eafe61e" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -3605,6 +3713,7 @@ dependencies = [ "serde_urlencoded", "serde_yaml", "smallvec", + "sse-codec", "sync_wrapper", "tempfile", "thiserror 2.0.18", @@ -3852,7 +3961,7 @@ dependencies = [ "bytes 1.11.1", "getrandom 0.3.4", "lru-slab", - "rand 0.9.4", + "rand 0.9.2", "ring", "rustc-hash", "rustls", @@ -3931,9 +4040,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -4053,9 +4162,9 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.14.7" +version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10b99e0098aa4082912d4c649628623db6aba77335e4f4569ff5083a6448b32e" +checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" dependencies = [ "pem", "ring", @@ -4800,6 +4909,18 @@ dependencies = [ "lock_api", ] +[[package]] +name = "sse-codec" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a59f811350c44b4a037aabeb72dc6a9591fc22aa95a036db9a96297c58085a" +dependencies = [ + "bytes 0.5.6", + "futures-io", + "futures_codec", + "memchr", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -4985,15 +5106,16 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.38.4" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +checksum = "a4deba334e1190ba7cb498327affa11e5ece10d26a30ab2f27fcf09504b8d8b6" dependencies = [ "libc", "memchr", "ntapi", "objc2-core-foundation", "objc2-io-kit", + "objc2-open-directory", "windows", ] @@ -5335,9 +5457,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.1" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes 1.11.1", "libc", @@ -5412,6 +5534,7 @@ checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes 1.11.1", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -5574,7 +5697,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ - "pin-project", + "pin-project 1.1.11", "tracing", ] @@ -6592,10 +6715,11 @@ checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] name = "yasna" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" dependencies = [ + "bit-vec", "time 0.3.47", ] diff --git a/Cargo.toml b/Cargo.toml index 4c458e0..cab5e88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,12 @@ [workspace] -members = ["crates/core", "crates/server", "crates/cli", "crates/admin"] +members = [ + "crates/memdb", + "crates/core", + "crates/server", + "crates/cli", + "crates/admin", +] resolver = "2" @@ -12,13 +18,12 @@ edition = "2021" chrono = "0.4.44" clap = { version = "4.6.1", features = ["derive", "env"] } mimalloc = "0.1.50" -native_db = "0.8.2" +memdb = { path = "crates/memdb" } itertools = "0.14.0" -native_model = "0.4.20" ring = { version = "0.17.14", features = ["std"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.149" -tokio = { version = "1.52.1", features = ["full"] } +tokio = { version = "1.52.3", features = ["full"] } tracing = "0.1.44" tracing-appender = "0.2.3" tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] } @@ -47,7 +52,7 @@ tokio-rustls = { version = "0.26.4", default-features = false, features = [ timeago = "0.6.0" oauth2 = { version = "5.0.0", features = ["reqwest-blocking"] } url = { version = "2.5.8", features = ["serde"] } -sysinfo = "0.38.4" +sysinfo = "0.39.1" num_cpus = "1.17.0" rand = "0.10.1" encoding_rs = "0.8.35" @@ -77,7 +82,7 @@ bytes = "1.11.1" dialoguer = "0.12.0" console = "0.16.3" mail-send = "0.6.0" -rcgen = "0.14.7" +rcgen = "0.14.8" rustls-pemfile = "2.2.0" blake3 = "1.8.5" uuid = { version = "1.23.1", features = ["v4", "serde"] } diff --git a/crates/admin/Cargo.toml b/crates/admin/Cargo.toml index b1c91db..eddbfb2 100644 --- a/crates/admin/Cargo.toml +++ b/crates/admin/Cargo.toml @@ -10,3 +10,11 @@ tokio.workspace = true dialoguer.workspace = true console.workspace = true indicatif.workspace = true +native_db = "0.8.2" +native_model = "0.4.20" +serde.workspace = true +serde_json.workspace = true + +itertools.workspace = true +snafu.workspace = true +memdb.workspace = true \ No newline at end of file diff --git a/crates/admin/src/main.rs b/crates/admin/src/main.rs index 2e15120..00364e2 100644 --- a/crates/admin/src/main.rs +++ b/crates/admin/src/main.rs @@ -21,6 +21,7 @@ use dialoguer::{theme::ColorfulTheme, Select}; use crate::{migrate::handle_migration, reset::handle_reset_password}; +pub mod meta; pub mod migrate; pub mod reset; @@ -34,7 +35,7 @@ async fn main() { let main_options = vec![ "Reset Admin Password", - "Migrate Legacy v0.x Storage to v1.0", + "Migrate Legacy v0.3.7 Storage to v1.0.0", "Exit", ]; diff --git a/crates/admin/src/meta.rs b/crates/admin/src/meta.rs new file mode 100644 index 0000000..9992a07 --- /dev/null +++ b/crates/admin/src/meta.rs @@ -0,0 +1,910 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + path::PathBuf, + sync::{Arc, LazyLock}, +}; + +use bichon_core::{ + account::{ + entity::ImapConfig, + migration::{AccountModel, AccountType}, + since::{DateSince, RelativeDate}, + }, + autoconfig::entity::MailServerConfig, + cache::imap::mailbox::Attribute, + database::batch_insert_impl, + error::{code::ErrorCode, BichonError, BichonResult}, + raise_error, + token::TokenType, + users::{acl::AccessControl, role::RoleType}, +}; +use console::style; +use itertools::Itertools; +use memdb::{Durability, MemDb}; +use native_db::*; +use native_model::{native_model, Model}; +use serde::{Deserialize, Serialize}; + +pub const DEFAULT_ADMIN_USER_ID: u64 = 100000000000000; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] +#[native_model(id = 3, version = 1)] +#[native_db] +pub struct CachedMailSettings { + #[primary_key] + pub domain: String, + pub config: MailServerConfig, + pub created_at: i64, +} + +impl From for bichon_core::autoconfig::CachedMailSettings { + fn from(value: CachedMailSettings) -> Self { + Self { + domain: value.domain, + config: value.config, + created_at: value.created_at, + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] +#[native_model(id = 4, version = 1)] +#[native_db(primary_key(pk -> String))] +pub struct AccountV1 { + #[secondary_key(unique)] + pub id: u64, + pub imap: Option, + pub enabled: bool, + pub email: String, + pub name: Option, + pub capabilities: Option>, + pub date_since: Option, + pub folder_limit: Option, + pub sync_folders: Option>, + pub account_type: AccountType, + pub sync_interval_min: Option, + pub known_folders: Option>, + pub created_at: i64, + pub updated_at: i64, + pub use_proxy: Option, +} +impl AccountV1 { + fn pk(&self) -> String { + format!("{}_{}", self.created_at, self.id) + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] +#[native_model(id = 4, version = 2, from = AccountV1)] +#[native_db(primary_key(pk -> String))] +pub struct AccountV2 { + #[secondary_key(unique)] + pub id: u64, + pub imap: Option, + pub enabled: bool, + pub email: String, + pub name: Option, + pub capabilities: Option>, + pub date_since: Option, + pub folder_limit: Option, + pub sync_folders: Option>, + pub account_type: AccountType, + pub sync_interval_min: Option, + pub known_folders: Option>, + pub created_at: i64, + pub updated_at: i64, + pub use_proxy: Option, + pub use_dangerous: bool, + pub pgp_key: Option, +} + +impl AccountV2 { + fn pk(&self) -> String { + format!("{}_{}", self.created_at, self.id) + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] +#[native_model(id = 4, version = 3, from = AccountV2)] +#[native_db(primary_key(pk -> String))] +pub struct AccountV3 { + #[secondary_key(unique)] + pub id: u64, + pub imap: Option, + pub enabled: bool, + pub email: String, + pub name: Option, + pub capabilities: Option>, + pub date_since: Option, + pub date_before: Option, + pub folder_limit: Option, + pub sync_folders: Option>, + pub account_type: AccountType, + pub sync_interval_min: Option, + pub sync_batch_size: Option, + pub known_folders: Option>, + pub created_at: i64, + pub updated_at: i64, + pub created_by: u64, //user id + pub use_proxy: Option, + pub use_dangerous: bool, + pub pgp_key: Option, +} + +impl AccountV3 { + fn pk(&self) -> String { + format!("{}_{}", self.created_at, self.id) + } +} + +impl From for AccountV2 { + fn from(value: AccountV1) -> Self { + Self { + id: value.id, + imap: value.imap, + enabled: value.enabled, + email: value.email, + name: value.name, + capabilities: value.capabilities, + date_since: value.date_since, + folder_limit: value.folder_limit, + sync_folders: value.sync_folders, + account_type: value.account_type, + sync_interval_min: value.sync_interval_min, + known_folders: value.known_folders, + created_at: value.created_at, + updated_at: value.updated_at, + use_proxy: value.use_proxy, + use_dangerous: false, + pgp_key: None, + } + } +} + +impl From for AccountV1 { + fn from(value: AccountV2) -> Self { + Self { + id: value.id, + imap: value.imap, + enabled: value.enabled, + email: value.email, + name: value.name, + capabilities: value.capabilities, + date_since: value.date_since, + folder_limit: value.folder_limit, + sync_folders: value.sync_folders, + account_type: value.account_type, + sync_interval_min: value.sync_interval_min, + known_folders: value.known_folders, + created_at: value.created_at, + updated_at: value.updated_at, + use_proxy: value.use_proxy, + } + } +} + +impl From for AccountV2 { + fn from(value: AccountV3) -> Self { + Self { + id: value.id, + imap: value.imap, + enabled: value.enabled, + email: value.email, + name: value.name, + capabilities: value.capabilities, + date_since: value.date_since, + folder_limit: value.folder_limit, + sync_folders: value.sync_folders, + account_type: value.account_type, + sync_interval_min: value.sync_interval_min, + known_folders: value.known_folders, + created_at: value.created_at, + updated_at: value.updated_at, + use_proxy: value.use_proxy, + use_dangerous: value.use_dangerous, + pgp_key: value.pgp_key, + } + } +} + +impl From for AccountV3 { + fn from(value: AccountV2) -> Self { + Self { + id: value.id, + imap: value.imap, + enabled: value.enabled, + email: value.email, + name: value.name, + capabilities: value.capabilities, + date_since: value.date_since, + folder_limit: value.folder_limit, + sync_folders: value.sync_folders, + account_type: value.account_type, + sync_interval_min: value.sync_interval_min, + known_folders: value.known_folders, + created_at: value.created_at, + updated_at: value.updated_at, + created_by: DEFAULT_ADMIN_USER_ID, + use_proxy: value.use_proxy, + use_dangerous: value.use_dangerous, + pgp_key: value.pgp_key, + sync_batch_size: None, + date_before: None, + } + } +} + +impl From for AccountModel { + fn from(value: AccountV3) -> Self { + Self { + id: value.id, + imap: value.imap, + enabled: value.enabled, + email: value.email, + account_name: None, + login_name: value.name, + capabilities: value.capabilities, + date_since: value.date_since, + date_before: value.date_before, + folder_limit: value.folder_limit, + download_folders: value.sync_folders, + account_type: value.account_type, + download_interval_min: value.sync_interval_min, + download_batch_size: value.sync_batch_size, + known_folders: value.known_folders, + created_at: value.created_at, + updated_at: value.updated_at, + created_by: value.created_by, + use_proxy: value.use_proxy, + use_dangerous: value.use_dangerous, + pgp_key: value.pgp_key, + imap_quota_window: None, + imap_quota_bytes: None, + auto_download_new_mailboxes: None, + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[native_model(id = 5, version = 1)] +#[native_db(primary_key(pk -> String))] +pub struct OAuth2 { + /// A unique identifier for the OAuth2 configuration. + #[secondary_key(unique)] + pub id: u64, + /// A description of what this configuration is used for. + pub description: Option, + /// The client ID used for authenticating the application with the OAuth2 provider. + pub client_id: String, + /// The client secret used in conjunction with the client ID. + /// + /// Users should provide a plaintext secret. + /// The server will encrypt it using AES-256-GCM and securely store it. + /// The plaintext secret is never stored, so users must ensure it is valid for OAuth2 authentication. + pub client_secret: String, + /// The URL to redirect users to for OAuth2 authorization. + pub auth_url: String, + /// The URL to exchange authorization codes for access tokens. + pub token_url: String, + /// The URI where the OAuth2 provider will redirect to after authorization. + pub redirect_uri: String, + /// The scopes of access that are being requested (e.g., email, profile). + pub scopes: Option>, + /// Any additional parameters to include in the OAuth2 requests (e.g., access_type, prompt). + pub extra_params: Option>, + /// Indicates whether this configuration is enabled or disabled. + pub enabled: bool, + /// route OAuth through proxy (when direct access is blocked) + pub use_proxy: Option, + /// The timestamp when the configuration was created, in milliseconds since the Unix epoch. + pub created_at: i64, + /// The timestamp when the configuration was last updated, in milliseconds since the Unix epoch. + pub updated_at: i64, +} + +impl OAuth2 { + fn pk(&self) -> String { + format!("{}_{}", &self.created_at, &self.id) + } +} + +impl From for bichon_core::oauth2::entity::OAuth2 { + fn from(value: OAuth2) -> Self { + Self { + id: value.id, + description: value.description, + client_id: value.client_id, + client_secret: value.client_secret, + auth_url: value.auth_url, + token_url: value.token_url, + redirect_uri: value.redirect_uri, + scopes: value.scopes, + extra_params: value.extra_params, + enabled: value.enabled, + use_proxy: value.use_proxy, + created_at: value.created_at, + updated_at: value.updated_at, + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[native_model(id = 6, version = 1)] +#[native_db] +pub struct OAuth2PendingEntity { + /// Unique identifier for the OAuth2 request record + pub oauth2_id: u64, + + pub account_id: u64, + /// CSRF protection state parameter used to verify the integrity of the authorization request + #[primary_key] + pub state: String, + + /// PKCE code verifier used in the authorization code exchange process to ensure security + pub code_verifier: String, + + /// Timestamp when the OAuth2 request was created, used to determine request expiration + pub created_at: i64, +} + +impl From for bichon_core::oauth2::pending::OAuth2PendingEntity { + fn from(value: OAuth2PendingEntity) -> Self { + Self { + oauth2_id: value.oauth2_id, + account_id: value.account_id, + state: value.state, + code_verifier: value.code_verifier, + created_at: value.created_at, + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[native_model(id = 7, version = 1)] +#[native_db] +pub struct OAuth2AccessToken { + /// The ID of the account associated with this access token. + #[primary_key] + pub account_id: u64, + /// The id of the OAuth2 configuration associated with this access token. + #[secondary_key] + pub oauth2_id: u64, + /// The OAuth2 access token used to authenticate requests to the provider. + pub access_token: Option, + /// The OAuth2 refresh token used to obtain new access tokens. + pub refresh_token: Option, + /// The timestamp when the token record was created, in milliseconds since the Unix epoch. + pub created_at: i64, + /// The timestamp when the token record was last updated, in milliseconds since the Unix epoch. + pub updated_at: i64, +} + +impl From for bichon_core::oauth2::token::OAuth2AccessToken { + fn from(value: OAuth2AccessToken) -> Self { + Self { + account_id: value.account_id, + oauth2_id: value.oauth2_id, + access_token: value.access_token, + refresh_token: value.refresh_token, + created_at: value.created_at, + updated_at: value.updated_at, + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[native_model(id = 8, version = 1)] +#[native_db] +pub struct Proxy { + /// The unique identifier for this proxy configuration. + #[primary_key] + pub id: u64, + + /// The proxy URL (e.g., socks5://127.0.0.1:1080) used to route network requests. + pub url: String, + + /// The creation timestamp of this record, represented as milliseconds since the Unix epoch. + pub created_at: i64, + + /// The last update timestamp of this record, represented as milliseconds since the Unix epoch. + pub updated_at: i64, +} + +impl From for bichon_core::settings::proxy::Proxy { + fn from(value: Proxy) -> Self { + Self { + id: value.id, + url: value.url, + created_at: value.created_at, + updated_at: value.updated_at, + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[native_model(id = 9, version = 1)] +#[native_db] +pub struct UserRole { + #[primary_key] + pub id: u64, + pub name: String, + pub description: Option, + pub permissions: BTreeSet, + pub is_builtin: bool, + pub created_at: i64, + pub role_type: RoleType, + pub updated_at: i64, +} + +impl From for bichon_core::users::role::UserRole { + fn from(value: UserRole) -> Self { + Self { + id: value.id, + name: value.name, + description: value.description, + permissions: value.permissions, + is_builtin: value.is_builtin, + created_at: value.created_at, + role_type: value.role_type, + updated_at: value.updated_at, + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[native_model(id = 10, version = 1)] +#[native_db] +pub struct BichonUser { + #[primary_key] + pub id: u64, + #[secondary_key(unique)] + pub username: String, + #[secondary_key(unique)] + pub email: String, + + pub password: Option, + + /// Scoped Access: Defines per-account permissions. + /// Example: + /// { account_id: 1, role_id: role_manager_id } -> Manager on Account 1 + /// { account_id: 2, role_id: role_viewer_id } -> Viewer on Account 2 + pub account_access_map: BTreeMap, + + pub description: Option, + + /// System Roles: Permissions that apply to the whole system + /// (e.g., system settings, creating new users). + pub global_roles: Vec, + + pub avatar: Option, + pub created_at: i64, + pub updated_at: i64, + /// Optional access control settings + pub acl: Option, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[native_model(id = 10, version = 2, from = BichonUser)] +#[native_db] +pub struct BichonUserV2 { + #[primary_key] + pub id: u64, + #[secondary_key(unique)] + pub username: String, + #[secondary_key(unique)] + pub email: String, + + pub password: Option, + + /// Scoped Access: Defines per-account permissions. + /// Example: + /// { account_id: 1, role_id: role_manager_id } -> Manager on Account 1 + /// { account_id: 2, role_id: role_viewer_id } -> Viewer on Account 2 + pub account_access_map: BTreeMap, + + pub description: Option, + + /// System Roles: Permissions that apply to the whole system + /// (e.g., system settings, creating new users). + pub global_roles: Vec, + + pub avatar: Option, + pub created_at: i64, + pub updated_at: i64, + /// Optional access control settings + pub acl: Option, + + pub theme: Option, + pub language: Option, +} + +impl From for BichonUser { + fn from(value: BichonUserV2) -> Self { + BichonUser { + id: value.id, + username: value.username, + email: value.email, + password: value.password, + account_access_map: value.account_access_map, + description: value.description, + global_roles: value.global_roles, + avatar: value.avatar, + created_at: value.created_at, + updated_at: value.updated_at, + acl: value.acl, + } + } +} + +impl From for BichonUserV2 { + fn from(value: BichonUser) -> Self { + BichonUserV2 { + id: value.id, + username: value.username, + email: value.email, + password: value.password, + account_access_map: value.account_access_map, + description: value.description, + global_roles: value.global_roles, + avatar: value.avatar, + created_at: value.created_at, + updated_at: value.updated_at, + acl: value.acl, + theme: None, + language: None, + } + } +} + +impl From for bichon_core::users::BichonUserV2 { + fn from(value: BichonUserV2) -> Self { + Self { + id: value.id, + username: value.username, + email: value.email, + password: value.password, + account_access_map: value.account_access_map, + description: value.description, + global_roles: value.global_roles, + avatar: value.avatar, + created_at: value.created_at, + updated_at: value.updated_at, + acl: value.acl, + theme: value.theme, + language: value.language, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[native_model(id = 11, version = 1)] +#[native_db] +pub struct AccessTokenModel { + /// The ID of the user who owns this token + #[secondary_key] + pub user_id: u64, + /// The unique token string used for authentication + #[primary_key] + pub token: String, + /// An optional name of the token. + pub name: Option, + /// Token type: WebUI or API + pub token_type: TokenType, + /// The timestamp (in milliseconds since epoch) when the token was created. + pub created_at: i64, + /// The timestamp (in milliseconds since epoch) when the token was last updated. + pub updated_at: i64, + /// The timestamp (in milliseconds since epoch) when the token expires. + /// None means the token does not expire (this applies only to API tokens). + pub expire_at: Option, + /// The timestamp (in milliseconds since epoch) when the token was last used. + pub last_access_at: i64, +} + +impl From for bichon_core::token::AccessTokenModel { + fn from(value: AccessTokenModel) -> Self { + Self { + user_id: value.user_id, + token: value.token, + name: value.name, + token_type: value.token_type, + created_at: value.created_at, + updated_at: value.updated_at, + expire_at: value.expire_at, + last_access_at: value.last_access_at, + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] +#[native_model(id = 1, version = 1)] +#[native_db] +pub struct MailBox { + /// The unique identifier for the mailbox + #[primary_key] + pub id: u64, + /// The ID of the account associated with the mailbox + #[secondary_key] + pub account_id: u64, + /// The unique, decoded, human-readable name of the mailbox (e.g., "INBOX", "Sent Items"). + /// This is the decoded name as presented to users, derived from the IMAP server's mailbox name + /// (e.g., after decoding UTF-7 or other encodings per RFC 3501). + pub name: String, + /// Optional delimiter used to separate mailbox names in a hierarchy (e.g., "/" or "."). + /// Used in IMAP to structure nested mailboxes (e.g., "INBOX/Archive"). + pub delimiter: Option, + /// List of attributes associated with the mailbox (e.g., `\NoSelect`, `\Deleted`). + /// These indicate special properties, such as whether the mailbox can hold messages. + pub attributes: Vec, + /// The number of messages that currently exist in the mailbox. + pub exists: u32, + /// Optional number of unseen messages in the mailbox (i.e., messages without the `\Seen` flag). + pub unseen: Option, + /// The next unique identifier (UID) that will be assigned to a new message in the mailbox. + /// If `None`, the IMAP server has not provided this information. + pub uid_next: Option, + /// The validity identifier for UIDs in this mailbox, used to ensure UID consistency across sessions. + /// If `None`, the IMAP server has not provided this information. + pub uid_validity: Option, +} + +impl From for bichon_core::cache::imap::mailbox::MailBox { + fn from(value: MailBox) -> Self { + Self { + id: value.id, + account_id: value.account_id, + name: value.name, + delimiter: value.delimiter, + attributes: value.attributes, + exists: value.exists, + unseen: value.unseen, + uid_next: value.uid_next, + uid_validity: value.uid_validity, + } + } +} + +pub static META_MODELS: LazyLock = LazyLock::new(|| { + let mut adapter = ModelsAdapter::new(); + adapter.register_metadata_models(); + adapter.models +}); + +pub static MAILBOX_MODELS: LazyLock = LazyLock::new(|| { + let mut adapter = ModelsAdapter::new(); + adapter.register_model::(); + adapter.models +}); + +pub struct ModelsAdapter { + pub models: Models, +} + +impl ModelsAdapter { + pub fn new() -> Self { + ModelsAdapter { + models: Models::new(), + } + } + + pub fn register_model(&mut self) { + self.models.define::().expect("failed to define model "); + } + + pub fn register_metadata_models(&mut self) { + self.register_model::(); + self.register_model::(); + self.register_model::(); + self.register_model::(); + self.register_model::(); + self.register_model::(); + self.register_model::(); + self.register_model::(); + self.register_model::(); + self.register_model::(); + self.register_model::(); + self.register_model::(); + } +} + +fn init_meta_database(root_path: &PathBuf) -> BichonResult>> { + let mut database = Builder::new() + .set_cache_size(134217728) + .create(&META_MODELS, root_path.join("meta.db")) + .map_err(handle_database_error)?; + + let rw = database + .rw_transaction() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + rw.migrate::() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + rw.migrate::() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + rw.commit() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + database + .compact() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(Arc::new(database)) +} + +fn init_evenlope_database(root_path: &PathBuf) -> BichonResult>> { + let mut database = Builder::new() + .set_cache_size(1073741824) + .create(&MAILBOX_MODELS, root_path.join("mailbox.db")) + .map_err(handle_database_error)?; + + let rw = database + .rw_transaction() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + rw.commit() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + database + .compact() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + Ok(Arc::new(database)) +} + +fn handle_database_error(error: native_db::db_type::Error) -> BichonError { + raise_error!( + format!("Failed to create database: {:?}", error), + ErrorCode::InternalError + ) +} + +pub fn list_all_impl( + database: &Arc>, +) -> BichonResult> { + let r_transaction = database + .r_transaction() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let entities: Vec = r_transaction + .scan() + .primary() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .all() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .try_collect() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(entities) +} + +pub fn migrate_metadata(root_path: &PathBuf) -> Result<(), Box> { + // Pre-flight: verify old metadata databases exist + let meta_db_path = root_path.join("meta.db"); + if !meta_db_path.exists() { + return Err(format!( + "Legacy metadata database not found at '{}'. \ + Make sure the root directory points to a valid v0.3.7 installation.", + meta_db_path.display() + ) + .into()); + } + let mailbox_db_path = root_path.join("mailbox.db"); + if !mailbox_db_path.exists() { + return Err(format!( + "Legacy mailbox database not found at '{}'. \ + Make sure the root directory points to a valid v0.3.7 installation.", + mailbox_db_path.display() + ) + .into()); + } + + // Initialize legacy database connections + let meta_db = init_meta_database(root_path) + .map_err(|e| format!("Failed to initialize legacy metadata database: {}", e))?; + let envelope_db = init_evenlope_database(root_path) + .map_err(|e| format!("Failed to initialize legacy envelope database: {}", e))?; + + // Prepare new database directory + let db_path = root_path.join("memdb"); + if !db_path.exists() { + std::fs::create_dir_all(&db_path)?; + } + + // Open new database (disable full durability for faster bulk writes) + let db = MemDb::open_with(&db_path, Durability::Off) + .map_err(|e| format!("Failed to open new memdb database: {}", e))?; + + println!( + "{}", + style("Step 1: Migrating Metadata Entities...") + .bold() + .cyan() + ); + + // Migration helper macro to reduce boilerplate + macro_rules! migrate_collection { + ($name:expr, $old_type:ty, $new_type:ty, $source_db:expr) => { + print!(" > {:<25} ", $name); + let items = list_all_impl::<$old_type>($source_db)?; + let count = items.len(); + let converted: Vec<$new_type> = items.into_iter().map(|a| a.into()).collect(); + batch_insert_impl(&db, converted)?; + println!("{} ({} items)", style("done").green(), count); + }; + } + + // --- Migrate each entity type --- + + migrate_collection!( + "Mail Settings", + CachedMailSettings, + bichon_core::autoconfig::CachedMailSettings, + &meta_db + ); + + migrate_collection!("Accounts", AccountV3, AccountModel, &meta_db); + + migrate_collection!( + "OAuth2 Entities", + OAuth2, + bichon_core::oauth2::entity::OAuth2, + &meta_db + ); + + migrate_collection!( + "OAuth2 Pending", + OAuth2PendingEntity, + bichon_core::oauth2::pending::OAuth2PendingEntity, + &meta_db + ); + + migrate_collection!( + "OAuth2 Access Tokens", + OAuth2AccessToken, + bichon_core::oauth2::token::OAuth2AccessToken, + &meta_db + ); + + migrate_collection!( + "Proxy Settings", + Proxy, + bichon_core::settings::proxy::Proxy, + &meta_db + ); + + migrate_collection!( + "User Roles", + UserRole, + bichon_core::users::role::UserRole, + &meta_db + ); + + migrate_collection!( + "Users", + BichonUserV2, + bichon_core::users::BichonUserV2, + &meta_db + ); + + migrate_collection!( + "Access Tokens", + AccessTokenModel, + bichon_core::token::AccessTokenModel, + &meta_db + ); + + // Mailboxes (from envelope_db) + migrate_collection!( + "Mailboxes", + MailBox, + bichon_core::cache::imap::mailbox::MailBox, + &envelope_db + ); + + // Persist and finish + db.snapshot() + .map_err(|e| format!("Snapshot save failed: {}", e))?; + println!( + "{}", + style("Metadata migration completed successfully.") + .green() + .bold() + ); + + Ok(()) +} diff --git a/crates/admin/src/migrate.rs b/crates/admin/src/migrate.rs index 5e9da7c..4d870e3 100644 --- a/crates/admin/src/migrate.rs +++ b/crates/admin/src/migrate.rs @@ -11,7 +11,7 @@ use indicatif::{ProgressBar, ProgressStyle}; pub fn handle_migration(theme: &ColorfulTheme) { println!( "\n{}", - style("MIGRATION: Bichon v0.x Storage Architecture → v1.0") + style("MIGRATION: Bichon v0.3.7 Storage Architecture → v1.0.0") .bold() .yellow() ); @@ -19,8 +19,8 @@ pub fn handle_migration(theme: &ColorfulTheme) { println!( "{}", style( - "This tool migrates data from the legacy v0.x Tantivy-based storage \ - architecture (used in versions 0.0.1 through 0.3.7) to the new v1.0 \ + "This tool migrates data from the legacy v0.3.7 Tantivy-based storage \ + architecture to the new v1.0.0 \ separated index and Fjall-backed storage format." ) .dim() @@ -29,10 +29,10 @@ pub fn handle_migration(theme: &ColorfulTheme) { println!( "{}", style( - "Legacy v0.x architecture:\n\ + "Legacy v0.3.7 architecture:\n\ • envelope metadata stored in Tantivy\n\ • message data stored in Tantivy\n\n\ - New v1.0 architecture:\n\ + New v1.0.0 architecture:\n\ • mail indexes stored in Tantivy\n\ • attachment indexes stored in Tantivy\n\ • raw message data stored in Fjall\n\ @@ -223,7 +223,29 @@ pub fn handle_migration(theme: &ColorfulTheme) { return; } - println!("\n{} Migrating...", style("⌛").yellow()); + // Step 1: Migrate metadata (meta.db + mailbox.db → memdb) + match crate::meta::migrate_metadata(&root_path) { + Ok(()) => {} + Err(e) => { + eprintln!( + "\n{} Metadata migration failed:\n{}", + style("✘").red().bold(), + style(e).red() + ); + eprintln!( + "{}", + style("Aborting migration. No changes have been made to Tantivy data.") + .yellow() + ); + return; + } + } + + println!( + "\n{} {}", + style("⌛").yellow(), + style("Step 2: Migrating email index and blob data...").cyan() + ); let pb = ProgressBar::new(0); pb.set_style(ProgressStyle::default_bar() .template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta}) {msg}") diff --git a/crates/admin/src/reset.rs b/crates/admin/src/reset.rs index 050d5f5..f71c36b 100644 --- a/crates/admin/src/reset.rs +++ b/crates/admin/src/reset.rs @@ -1,8 +1,7 @@ use std::path::{Path, PathBuf}; use bichon_core::{ - admin::meta::{find_admin, init_meta_database, update_admin_password}, - error::BichonError, + admin::meta::{find_admin, open_database, update_admin_password}, utils::encrypt::internal_decrypt_string, }; use console::{style, Emoji}; @@ -19,9 +18,9 @@ pub fn handle_reset_password(theme: &ColorfulTheme) { if !path.exists() { return Err("Directory does not exist."); } - let has_metadata = path.join("meta.db").exists(); - if !has_metadata { - return Err("Invalid directory: 'meta.db' not found."); + let memdb_dir = path.join("memdb"); + if !memdb_dir.exists() || !memdb_dir.is_dir() { + return Err("Invalid directory: 'memdb' data directory not found."); } Ok(()) }) @@ -29,39 +28,11 @@ pub fn handle_reset_password(theme: &ColorfulTheme) { .unwrap(); let root_path = PathBuf::from(&root_dir_str); - let database = match init_meta_database(&root_path.join("meta.db")) { - Ok(database) => database, - Err(e) => match e { - BichonError::Generic { - message, - location, - code, - } => { - if message.contains("RedbDatabaseError(DatabaseAlreadyOpen") { - println!("\n{}", style("ERROR: Database is locked.").red().bold()); - println!( - "{}", - style("The Bichon service is likely still running.").yellow() - ); - println!( - "Since the database cannot be shared between multiple instances, \n\ - you must {} the Bichon service before proceeding.", - style("STOP").underlined().bold() - ); - std::process::exit(1); - } else { - eprintln!( - "\n{} (Code: {:#?})\nLocation: {}\nMessage: {}", - style("A database error occurred:").red().bold(), - code, - location, - message - ); - std::process::exit(1); - } - } - }, - }; + let database = open_database(&root_path.join("memdb")).unwrap_or_else(|e| { + eprintln!("\n{} Failed to open database.", style("ERROR:").red().bold()); + eprintln!("Details: {:?}", e); + std::process::exit(1); + }); let admin = find_admin(&database); diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 220090c..bffd804 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -19,9 +19,8 @@ poem-openapi = { version = "5.1.16", features = [ ], optional = true } chrono.workspace = true clap.workspace = true -native_db.workspace = true +memdb.workspace = true itertools.workspace = true -native_model.workspace = true ring.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/core/src/account/grant.rs b/crates/core/src/account/grant.rs index d7ad697..2d17599 100644 --- a/crates/core/src/account/grant.rs +++ b/crates/core/src/account/grant.rs @@ -24,7 +24,7 @@ use crate::{ { account::migration::AccountModel, common::auth::ClientContext, - database::{manager::DB_MANAGER, with_transaction}, + database::{manager::DB_MANAGER, with_transaction, MemDbModel}, error::{code::ErrorCode, BichonResult}, users::{ permissions::Permission, @@ -43,8 +43,8 @@ pub struct BatchAccountRoleRequest { } impl BatchAccountRoleRequest { - pub async fn validate_existence(&self) -> BichonResult<()> { - let role = UserRole::find(self.role_id).await?.ok_or_else(|| { + pub fn validate_existence(&self) -> BichonResult<()> { + let role = UserRole::find(self.role_id)?.ok_or_else(|| { raise_error!( format!("Role ID {} not found", self.role_id), ErrorCode::ResourceNotFound @@ -59,7 +59,7 @@ impl BatchAccountRoleRequest { } for id in &self.account_ids { - let exists = AccountModel::async_find(*id).await?; // Assuming an exists helper + let exists = AccountModel::find(*id)?; // Assuming an exists helper if exists.is_none() { return Err(raise_error!( format!("Account ID {} not found", id), @@ -69,7 +69,7 @@ impl BatchAccountRoleRequest { } for id in &self.user_ids { - let exists = UserModel::find(*id).await?; // Assuming an exists helper + let exists = UserModel::find(*id)?; // Assuming an exists helper if exists.is_none() { return Err(raise_error!( format!("User ID {} not found", id), @@ -81,44 +81,38 @@ impl BatchAccountRoleRequest { Ok(()) } - async fn grant_batch_account_access( + fn grant_batch_account_access( account_ids: Vec, user_ids: Vec, role_id: u64, ) -> BichonResult<()> { - with_transaction(DB_MANAGER.meta_db(), move |rw| { + with_transaction(DB_MANAGER.db(), move |txn| { + let mut txn = txn; for &uid in &user_ids { - // Fetch the current user record from the database - let user = rw - .get() - .primary::(uid) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| { - raise_error!( - format!("User with id={} not found.", uid), - ErrorCode::ResourceNotFound - ) - })?; + let db = DB_MANAGER.db(); + let coll = db.collection(UserModel::collection()); + let key = uid.to_string(); + let user: UserModel = coll + .get_required(&key) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; let mut updated_user = user.clone(); - // Apply the role to each specified account_id for &aid in &account_ids { updated_user.account_access_map.insert(aid, role_id); } updated_user.updated_at = utc_now!(); - // Save the updated user back to the database within the transaction - rw.update(user, updated_user) + txn = txn + .upsert(UserModel::collection(), key, &updated_user) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; } - Ok(()) + Ok(txn) }) - .await } - pub async fn do_assign(self, context: &ClientContext) -> BichonResult<()> { + pub fn do_assign(self, context: &ClientContext) -> BichonResult<()> { for account_id in &self.account_ids { // Get the user's specific access for this account let assigned_role_id = @@ -134,7 +128,7 @@ impl BatchAccountRoleRequest { })?; // Fetch the role definition from the database - let user_scoped_role = UserRole::find(*assigned_role_id).await?.ok_or_else(|| { + let user_scoped_role = UserRole::find(*assigned_role_id)?.ok_or_else(|| { raise_error!( "Assigned account role no longer exists".into(), ErrorCode::InternalError @@ -156,6 +150,6 @@ impl BatchAccountRoleRequest { // This is where you'd compare target_role.permissions vs manager's perms } - Self::grant_batch_account_access(self.account_ids, self.user_ids, self.role_id).await + Self::grant_batch_account_access(self.account_ids, self.user_ids, self.role_id) } } diff --git a/crates/core/src/account/migration.rs b/crates/core/src/account/migration.rs index 2b4d54d..83569b4 100644 --- a/crates/core/src/account/migration.rs +++ b/crates/core/src/account/migration.rs @@ -16,8 +16,6 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use native_db::*; -use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; use std::collections::BTreeSet; use tracing::info; @@ -25,36 +23,28 @@ use tracing::info; use crate::{ account::{ entity::ImapConfig, + payload::{AccountCreateRequest, AccountUpdateRequest, MinimalAccount}, since::{DateSince, RelativeDate}, state::DownloadState, }, - cache::imap::mailbox::MailBox, + cache::imap::{mailbox::MailBox, task::SYNC_TASKS}, common::paginated::DataPage, - database::{list_all_impl, secondary_find_impl, with_transaction}, + context::controller::DOWNLOAD_CONTROLLER, + database::{ + count_impl, delete_impl, find_impl, insert_impl, list_all_impl, manager::DB_MANAGER, + paginate_impl, update_impl, MemDbModel, + }, encrypt, - error::BichonResult, + error::{code::ErrorCode, BichonResult}, + id, + oauth2::token::OAuth2AccessToken, + raise_error, store::tantivy::{attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER}, - users::{role::DEFAULT_ACCOUNT_MANAGER_ROLE_ID, UserModel, DEFAULT_ADMIN_USER_ID}, + users::{payload::UserUpdateRequest, role::DEFAULT_ACCOUNT_MANAGER_ROLE_ID, UserModel}, utc_now, }; -use crate::account::payload::AccountCreateRequest; -use crate::account::payload::AccountUpdateRequest; -use crate::account::payload::MinimalAccount; -use crate::cache::imap::task::SYNC_TASKS; -use crate::context::controller::DOWNLOAD_CONTROLLER; -use crate::database::count_by_unique_secondary_key_impl; -use crate::database::delete_impl; -use crate::database::manager::DB_MANAGER; -use crate::database::{ - async_secondary_find_impl, paginate_query_primary_scan_all_impl, update_impl, -}; -use crate::error::code::ErrorCode; -use crate::id; -use crate::oauth2::token::OAuth2AccessToken; -use crate::raise_error; - -pub type AccountModel = AccountV4; +pub type AccountModel = Account; #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))] @@ -74,102 +64,9 @@ pub enum QuotaWindow { Monthly, } -#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] -#[native_model(id = 4, version = 1)] -#[native_db(primary_key(pk -> String))] -pub struct AccountV1 { - #[secondary_key(unique)] - pub id: u64, - pub imap: Option, - pub enabled: bool, - pub email: String, - pub name: Option, - pub capabilities: Option>, - pub date_since: Option, - pub folder_limit: Option, - pub sync_folders: Option>, - pub account_type: AccountType, - pub sync_interval_min: Option, - pub known_folders: Option>, - pub created_at: i64, - pub updated_at: i64, - pub use_proxy: Option, -} -impl AccountV1 { - fn pk(&self) -> String { - format!("{}_{}", self.created_at, self.id) - } -} - -#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] -#[native_model(id = 4, version = 2, from = AccountV1)] -#[native_db(primary_key(pk -> String))] -pub struct AccountV2 { - #[secondary_key(unique)] - pub id: u64, - pub imap: Option, - pub enabled: bool, - pub email: String, - pub name: Option, - pub capabilities: Option>, - pub date_since: Option, - pub folder_limit: Option, - pub sync_folders: Option>, - pub account_type: AccountType, - pub sync_interval_min: Option, - pub known_folders: Option>, - pub created_at: i64, - pub updated_at: i64, - pub use_proxy: Option, - pub use_dangerous: bool, - pub pgp_key: Option, -} - -impl AccountV2 { - fn pk(&self) -> String { - format!("{}_{}", self.created_at, self.id) - } -} - -#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] -#[native_model(id = 4, version = 3, from = AccountV2)] -#[native_db(primary_key(pk -> String))] -pub struct AccountV3 { - #[secondary_key(unique)] - pub id: u64, - pub imap: Option, - pub enabled: bool, - pub email: String, - pub name: Option, - pub capabilities: Option>, - pub date_since: Option, - pub date_before: Option, - pub folder_limit: Option, - pub sync_folders: Option>, - pub account_type: AccountType, - pub sync_interval_min: Option, - pub sync_batch_size: Option, - pub known_folders: Option>, - pub created_at: i64, - pub updated_at: i64, - pub created_by: u64, //user id - pub use_proxy: Option, - pub use_dangerous: bool, - pub pgp_key: Option, -} - -impl AccountV3 { - fn pk(&self) -> String { - format!("{}_{}", self.created_at, self.id) - } -} - #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] -#[native_model(id = 4, version = 4, from = AccountV3)] -#[native_db(primary_key(pk -> String))] -pub struct AccountV4 { - #[secondary_key(unique)] +pub struct Account { pub id: u64, pub imap: Option, pub enabled: bool, @@ -200,11 +97,16 @@ pub struct AccountV4 { pub auto_download_new_mailboxes: Option, } -impl AccountV4 { - fn pk(&self) -> String { - format!("{}_{}", self.created_at, self.id) +impl MemDbModel for Account { + fn collection() -> &'static str { + "accounts" } + fn key(&self) -> String { + self.id.to_string() + } +} +impl Account { pub fn new(user_id: u64, request: AccountCreateRequest) -> BichonResult { Ok(Self { id: id!(64), @@ -234,31 +136,8 @@ impl AccountV4 { }) } - pub async fn check_account_exists(account_id: u64) -> BichonResult { - let account = async_secondary_find_impl::( - DB_MANAGER.meta_db(), - AccountV4Key::id, - account_id, - ) - .await? - .ok_or_else(|| { - raise_error!( - format!("Account id='{account_id}' not found"), - ErrorCode::ResourceNotFound - ) - })?; - Ok(account) - } - - /// Fetches an `AccountEntity` by its `id`. - pub async fn async_get(account_id: u64) -> BichonResult { - let result: AccountModel = Self::async_find(account_id).await?.ok_or_else(|| { - raise_error!( - format!("Account with ID '{account_id}' not found"), - ErrorCode::ResourceNotFound - ) - })?; - Ok(result) + pub fn check_account_exists(account_id: u64) -> BichonResult { + Self::get(account_id) } pub fn get(account_id: u64) -> BichonResult { @@ -271,17 +150,9 @@ impl AccountV4 { Ok(result) } - pub async fn async_find(account_id: u64) -> BichonResult> { - async_secondary_find_impl::( - DB_MANAGER.meta_db(), - AccountV4Key::id, - account_id, - ) - .await - } - pub fn find(account_id: u64) -> BichonResult> { - secondary_find_impl::(DB_MANAGER.meta_db(), AccountV4Key::id, account_id) + let result = find_impl::(DB_MANAGER.db(), &account_id.to_string())?; + Ok(result) } pub async fn create_account( @@ -290,31 +161,36 @@ impl AccountV4 { ) -> BichonResult { let entity = request.create_entity(user_id)?; let cloned = entity.clone(); - with_transaction(DB_MANAGER.meta_db(), move |rw| { - let account_id = entity.id; - rw.insert::(entity) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let user = rw - .get() - .primary::(user_id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| { - raise_error!( - format!("User with id={} not found.", user_id), - ErrorCode::ResourceNotFound - ) - })?; - let mut updated = user.clone(); - updated - .account_access_map - .insert(account_id, DEFAULT_ACCOUNT_MANAGER_ROLE_ID); - updated.updated_at = utc_now!(); - rw.update(user, updated) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(()) - }) - .await?; + // Insert account into memdb + insert_impl(DB_MANAGER.db(), entity)?; + + // Update user's account_access_map + let user = UserModel::find(user_id)?.ok_or_else(|| { + raise_error!( + format!("User with id={} not found.", user_id), + ErrorCode::ResourceNotFound + ) + })?; + + let mut updated_map = user.account_access_map.clone(); + updated_map.insert(cloned.id, DEFAULT_ACCOUNT_MANAGER_ROLE_ID); + + UserModel::update( + user_id, + UserUpdateRequest { + username: None, + email: None, + password: None, + avatar_base64: None, + global_roles: None, + account_access_map: Some(updated_map), + acl: None, + description: None, + theme: None, + language: None, + }, + )?; if matches!(cloned.account_type, AccountType::IMAP) { DOWNLOAD_CONTROLLER @@ -324,27 +200,26 @@ impl AccountV4 { Ok(cloned) } - pub async fn update( + pub fn update( account_id: u64, request: AccountUpdateRequest, validate: bool, ) -> BichonResult<()> { - let account = AccountModel::async_get(account_id).await?; + let account = AccountModel::get(account_id)?; if validate { request.validate_update_request(&account)?; } update_impl( - DB_MANAGER.meta_db(), - move |_| Ok(account), - move |current| Self::apply_update_fields(current, request), - ) - .await?; + DB_MANAGER.db(), + &account_id.to_string(), + move |current: Account| Self::apply_update_fields(¤t, request), + )?; Ok(()) } pub async fn delete(account_id: u64) -> BichonResult<()> { - let account = Self::async_get(account_id).await?; + let account = Self::get(account_id)?; if let Err(error) = Self::cleanup_account_resources_sequential(&account).await { tracing::error!( "[CLEANUP_ACCOUNT_ERROR] Account {}: failed to cleanup resources: {:#?}", @@ -356,84 +231,81 @@ impl AccountV4 { Ok(()) } - async fn delete_account(account_id: u64) -> BichonResult<()> { - delete_impl(DB_MANAGER.meta_db(), move|rw|{ - rw.get().secondary::(AccountV4Key::id, account_id).map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(||raise_error!(format!("The account entity with id={account_id} that you want to delete was not found."), ErrorCode::ResourceNotFound)) - }).await + fn delete_account(account: &AccountModel) -> BichonResult<()> { + delete_impl::(DB_MANAGER.db(), &account.id.to_string()) } async fn cleanup_account_resources_sequential(account: &AccountModel) -> BichonResult<()> { if matches!(account.account_type, AccountType::IMAP) { SYNC_TASKS.stop(account.id).await?; - DownloadState::delete(account.id).await?; + DownloadState::delete(account.id)?; } - OAuth2AccessToken::try_delete(account.id).await?; - UserModel::cleanup_account(account.id).await?; - MailBox::clean(account.id).await?; + OAuth2AccessToken::try_delete(account.id)?; + UserModel::cleanup_account(account.id)?; + MailBox::clean(account.id)?; ENVELOPE_MANAGER .delete_account_envelopes(account.id) .await?; ATTACHMENT_MANAGER .delete_account_attachments(account.id) .await?; - Self::delete_account(account.id).await?; + Self::delete_account(account)?; info!("Sequential cleanup completed for account: {}", account.id); Ok(()) } - pub async fn update_download_folders( + pub fn update_download_folders( account_id: u64, download_folders: Vec, ) -> BichonResult<()> { - update_impl(DB_MANAGER.meta_db(), move |rw| { - rw.get().secondary::(AccountV4Key::id, account_id).map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| raise_error!(format!("When trying to update account download folders, the corresponding record was not found. account_id={}", account_id), ErrorCode::ResourceNotFound)) - }, |current|{ - let mut updated = current.clone(); - updated.download_folders = Some(download_folders); - Ok(updated) - }).await?; + update_impl( + DB_MANAGER.db(), + &account_id.to_string(), + move |current: Account| { + let mut updated = current.clone(); + updated.download_folders = Some(download_folders); + Ok(updated) + }, + )?; Ok(()) } - pub async fn update_known_folders( + pub fn update_known_folders( account_id: u64, known_folders: BTreeSet, ) -> BichonResult<()> { - update_impl(DB_MANAGER.meta_db(), move |rw| { - rw.get().secondary::(AccountV4Key::id, account_id).map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| raise_error!(format!("When trying to update account known_folders, the corresponding record was not found. account_id={}", account_id), ErrorCode::ResourceNotFound)) - }, |current|{ - let mut updated = current.clone(); - updated.known_folders = Some(known_folders); - Ok(updated) - }).await?; + update_impl( + DB_MANAGER.db(), + &account_id.to_string(), + move |current: Account| { + let mut updated = current.clone(); + updated.known_folders = Some(known_folders); + Ok(updated) + }, + )?; Ok(()) } - pub async fn update_capabilities( - account_id: u64, - capabilities: Vec, - ) -> BichonResult<()> { - update_impl(DB_MANAGER.meta_db(), move |rw| { - rw.get().secondary::(AccountV4Key::id, account_id).map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| raise_error!(format!("When trying to update account capabilities, the corresponding record was not found. account_id={}", account_id), ErrorCode::ResourceNotFound)) - }, |current|{ - let mut updated = current.clone(); - updated.capabilities = Some(capabilities); - Ok(updated) - }).await?; + pub fn update_capabilities(account_id: u64, capabilities: Vec) -> BichonResult<()> { + update_impl( + DB_MANAGER.db(), + &account_id.to_string(), + move |current: Account| { + let mut updated = current.clone(); + updated.capabilities = Some(capabilities); + Ok(updated) + }, + )?; Ok(()) } /// Retrieves a list of all `AccountEntity` instances. - pub async fn list_all() -> BichonResult> { - list_all_impl(DB_MANAGER.meta_db()).await + pub fn list_all() -> BichonResult> { + list_all_impl::(DB_MANAGER.db()) } - pub async fn find_by_email(email: &str) -> BichonResult> { - let all: Vec = list_all_impl(DB_MANAGER.meta_db()).await?; + pub fn find_by_email(email: &str) -> BichonResult> { + let all: Vec = list_all_impl::(DB_MANAGER.db())?; let target_email = email.trim().to_lowercase(); let first_match = all @@ -443,9 +315,8 @@ impl AccountV4 { Ok(first_match) } - pub async fn minimal_list(only_nosync: bool) -> BichonResult> { - let result = list_all_impl(DB_MANAGER.meta_db()) - .await? + pub fn minimal_list(only_nosync: bool) -> BichonResult> { + let result = list_all_impl::(DB_MANAGER.db())? .into_iter() .filter(|account: &AccountModel| { !only_nosync || matches!(account.account_type, AccountType::NoSync) @@ -458,19 +329,16 @@ impl AccountV4 { Ok(result) } - pub async fn count() -> BichonResult { - count_by_unique_secondary_key_impl::(DB_MANAGER.meta_db(), AccountV4Key::id) - .await + pub fn count() -> BichonResult { + count_impl::(DB_MANAGER.db()) } - pub async fn paginate_list( + pub fn paginate_list( page: Option, page_size: Option, desc: Option, ) -> BichonResult> { - paginate_query_primary_scan_all_impl(DB_MANAGER.meta_db(), page, page_size, desc) - .await - .map(DataPage::from) + paginate_impl::(DB_MANAGER.db(), page, page_size, desc).map(DataPage::from) } // This method applies the updates from the request to the old account entity @@ -571,158 +439,3 @@ impl AccountV4 { Ok(new) } } - -impl From for AccountV2 { - fn from(value: AccountV1) -> Self { - Self { - id: value.id, - imap: value.imap, - enabled: value.enabled, - email: value.email, - name: value.name, - capabilities: value.capabilities, - date_since: value.date_since, - folder_limit: value.folder_limit, - sync_folders: value.sync_folders, - account_type: value.account_type, - sync_interval_min: value.sync_interval_min, - known_folders: value.known_folders, - created_at: value.created_at, - updated_at: value.updated_at, - use_proxy: value.use_proxy, - use_dangerous: false, - pgp_key: None, - } - } -} - -impl From for AccountV1 { - fn from(value: AccountV2) -> Self { - Self { - id: value.id, - imap: value.imap, - enabled: value.enabled, - email: value.email, - name: value.name, - capabilities: value.capabilities, - date_since: value.date_since, - folder_limit: value.folder_limit, - sync_folders: value.sync_folders, - account_type: value.account_type, - sync_interval_min: value.sync_interval_min, - known_folders: value.known_folders, - created_at: value.created_at, - updated_at: value.updated_at, - use_proxy: value.use_proxy, - } - } -} - -impl From for AccountV2 { - fn from(value: AccountV3) -> Self { - Self { - id: value.id, - imap: value.imap, - enabled: value.enabled, - email: value.email, - name: value.name, - capabilities: value.capabilities, - date_since: value.date_since, - folder_limit: value.folder_limit, - sync_folders: value.sync_folders, - account_type: value.account_type, - sync_interval_min: value.sync_interval_min, - known_folders: value.known_folders, - created_at: value.created_at, - updated_at: value.updated_at, - use_proxy: value.use_proxy, - use_dangerous: value.use_dangerous, - pgp_key: value.pgp_key, - } - } -} - -impl From for AccountV3 { - fn from(value: AccountV2) -> Self { - Self { - id: value.id, - imap: value.imap, - enabled: value.enabled, - email: value.email, - name: value.name, - capabilities: value.capabilities, - date_since: value.date_since, - folder_limit: value.folder_limit, - sync_folders: value.sync_folders, - account_type: value.account_type, - sync_interval_min: value.sync_interval_min, - known_folders: value.known_folders, - created_at: value.created_at, - updated_at: value.updated_at, - created_by: DEFAULT_ADMIN_USER_ID, - use_proxy: value.use_proxy, - use_dangerous: value.use_dangerous, - pgp_key: value.pgp_key, - sync_batch_size: None, - date_before: None, - } - } -} - -impl From for AccountV3 { - fn from(value: AccountV4) -> Self { - Self { - id: value.id, - imap: value.imap, - enabled: value.enabled, - email: value.email, - name: value.login_name, - capabilities: value.capabilities, - date_since: value.date_since, - date_before: value.date_before, - folder_limit: value.folder_limit, - sync_folders: value.download_folders, - account_type: value.account_type, - sync_interval_min: value.download_interval_min, - sync_batch_size: value.download_batch_size, - known_folders: value.known_folders, - created_at: value.created_at, - updated_at: value.updated_at, - created_by: value.created_by, - use_proxy: value.use_proxy, - use_dangerous: value.use_dangerous, - pgp_key: value.pgp_key, - } - } -} - -impl From for AccountV4 { - fn from(value: AccountV3) -> Self { - Self { - id: value.id, - imap: value.imap, - enabled: value.enabled, - email: value.email, - account_name: None, - login_name: value.name, - capabilities: value.capabilities, - date_since: value.date_since, - date_before: value.date_before, - folder_limit: value.folder_limit, - download_folders: value.sync_folders, - account_type: value.account_type, - download_interval_min: value.sync_interval_min, - download_batch_size: value.sync_batch_size, - known_folders: value.known_folders, - created_at: value.created_at, - updated_at: value.updated_at, - created_by: value.created_by, - use_proxy: value.use_proxy, - use_dangerous: value.use_dangerous, - pgp_key: value.pgp_key, - imap_quota_window: None, - imap_quota_bytes: None, - auto_download_new_mailboxes: None, - } - } -} diff --git a/crates/core/src/account/old_state.rs b/crates/core/src/account/old_state.rs index 9bd5c13..4863765 100644 --- a/crates/core/src/account/old_state.rs +++ b/crates/core/src/account/old_state.rs @@ -16,8 +16,6 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use native_db::*; -use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -28,10 +26,7 @@ pub struct MailboxBatchProgress { } #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] -#[native_model(id = 2, version = 1)] -#[native_db] pub struct AccountRunningState { - #[primary_key] pub account_id: u64, pub last_incremental_sync_start: i64, pub last_incremental_sync_end: Option, diff --git a/crates/core/src/account/since.rs b/crates/core/src/account/since.rs index cc95ef9..8b018d3 100644 --- a/crates/core/src/account/since.rs +++ b/crates/core/src/account/since.rs @@ -260,16 +260,44 @@ mod test { use crate::account::since::{DateSince, RelativeDate, Unit}; #[test] - fn test1() { + fn fixed_date_valid() { let e = DateSince { fixed: Some("2014-09-12".to_string()), relative: None, }; + assert!(e.validate().is_ok()); + assert!(!e.since_date().unwrap().is_empty()); + } - e.validate().unwrap(); + #[test] + fn fixed_date_in_future_fails() { + let e = DateSince { + fixed: Some("2099-01-01".to_string()), + relative: None, + }; + assert!(e.validate().is_err()); + } - println!("{}", e.since_date().unwrap()); + #[test] + fn fixed_date_before_1970_fails() { + let e = DateSince { + fixed: Some("1960-01-01".to_string()), + relative: None, + }; + assert!(e.validate().is_err()); + } + #[test] + fn fixed_date_bad_format_fails() { + let e = DateSince { + fixed: Some("01-01-2020".to_string()), + relative: None, + }; + assert!(e.validate().is_err()); + } + + #[test] + fn relative_date_days_valid() { let e = DateSince { fixed: None, relative: Some(RelativeDate { @@ -277,9 +305,63 @@ mod test { value: 1, }), }; + assert!(e.validate().is_ok()); + } - e.validate().unwrap(); + #[test] + fn relative_date_months_valid() { + let e = DateSince { + fixed: None, + relative: Some(RelativeDate { + unit: Unit::Months, + value: 3, + }), + }; + assert!(e.validate().is_ok()); + } - println!("{}", e.since_date().unwrap()); + #[test] + fn relative_date_years_valid() { + let e = DateSince { + fixed: None, + relative: Some(RelativeDate { + unit: Unit::Years, + value: 1, + }), + }; + assert!(e.validate().is_ok()); + } + + #[test] + fn relative_date_zero_value_fails() { + let e = DateSince { + fixed: None, + relative: Some(RelativeDate { + unit: Unit::Days, + value: 0, + }), + }; + assert!(e.validate().is_err()); + } + + #[test] + fn both_fixed_and_relative_fails() { + let e = DateSince { + fixed: Some("2014-09-12".to_string()), + relative: Some(RelativeDate { + unit: Unit::Days, + value: 1, + }), + }; + assert!(e.validate().is_err()); + } + + #[test] + fn neither_fixed_nor_relative_fails() { + let e = DateSince { + fixed: None, + relative: None, + }; + assert!(e.validate().is_err()); } } diff --git a/crates/core/src/account/state.rs b/crates/core/src/account/state.rs index 9a7fb1e..ab3d53a 100644 --- a/crates/core/src/account/state.rs +++ b/crates/core/src/account/state.rs @@ -17,14 +17,10 @@ // along with this program. If not, see . use crate::{ - raise_error, utc_now, - { - database::{async_find_impl, delete_impl, manager::DB_MANAGER, update_impl, upsert_impl}, - error::{code::ErrorCode, BichonResult}, - }, + database::{delete_impl, find_impl, manager::DB_MANAGER, update_impl, upsert_impl, MemDbModel}, + error::BichonResult, + utc_now, }; -use native_db::*; -use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -82,10 +78,7 @@ pub struct DownloadSession { #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] -#[native_model(id = 3, version = 1)] -#[native_db] pub struct DownloadState { - #[primary_key] pub account_id: u64, pub active_session: Option, pub history: Vec, @@ -93,6 +86,15 @@ pub struct DownloadState { pub last_finished_at: Option, } +impl MemDbModel for DownloadState { + fn collection() -> &'static str { + "download_states" + } + fn key(&self) -> String { + self.account_id.to_string() + } +} + #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] pub struct AccountError { @@ -122,14 +124,14 @@ impl DownloadState { history: Default::default(), last_finished_at: Default::default(), }; - upsert_impl(DB_MANAGER.envelope_db(), state).await + upsert_impl(DB_MANAGER.db(), state) } - pub async fn get(account_id: u64) -> BichonResult> { - async_find_impl(DB_MANAGER.envelope_db(), account_id).await + pub fn get(account_id: u64) -> BichonResult> { + find_impl::(DB_MANAGER.db(), &account_id.to_string()) } - pub async fn start_new_session(account_id: u64, trigger: TriggerType) -> BichonResult<()> { + pub fn start_new_session(account_id: u64, trigger: TriggerType) -> BichonResult<()> { Self::update_state(account_id, move |current| { let mut updated = current.clone(); updated.last_trigger_at = utc_now!(); @@ -151,10 +153,9 @@ impl DownloadState { updated.active_session = Some(new_session); Ok(updated) }) - .await } - pub async fn update_session_status( + pub fn update_session_status( account_id: u64, status: DownloadStatus, message: Option, @@ -181,10 +182,9 @@ impl DownloadState { } Ok(updated) }) - .await } - pub async fn update_folder_progress( + pub fn update_folder_progress( account_id: u64, folder_name: String, planned: u64, @@ -213,10 +213,9 @@ impl DownloadState { } Ok(updated) }) - .await } - pub async fn init_folder_details(account_id: u64, folders: Vec) -> BichonResult<()> { + pub fn init_folder_details(account_id: u64, folders: Vec) -> BichonResult<()> { Self::update_state(account_id, move |state| { let mut updated = state.clone(); if let Some(ref mut session) = updated.active_session { @@ -235,10 +234,9 @@ impl DownloadState { } Ok(updated) }) - .await } - pub async fn append_session_error(account_id: u64, error: String) -> BichonResult<()> { + pub fn append_session_error(account_id: u64, error: String) -> BichonResult<()> { Self::update_state(account_id, move |current| { let mut updated = current.clone(); if let Some(ref mut session) = updated.active_session { @@ -254,53 +252,23 @@ impl DownloadState { } Ok(updated) }) - .await } - async fn update_state( + fn update_state( account_id: u64, - updater: impl FnOnce(&DownloadState) -> BichonResult + Send + 'static, + updater: impl FnOnce(DownloadState) -> BichonResult + Send + 'static, ) -> BichonResult<()> { - if Self::get(account_id).await?.is_some() { - update_impl( - DB_MANAGER.envelope_db(), - move |rw| { - rw.get() - .primary::(account_id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| { - raise_error!( - format!("Cannot find download info of account={}", account_id), - ErrorCode::ResourceNotFound - ) - }) - }, - updater, - ) - .await?; + if Self::get(account_id)?.is_some() { + update_impl(DB_MANAGER.db(), &account_id.to_string(), updater)?; } Ok(()) } - pub async fn delete(account_id: u64) -> BichonResult<()> { - if Self::get(account_id).await?.is_none() { + pub fn delete(account_id: u64) -> BichonResult<()> { + if Self::get(account_id)?.is_none() { return Ok(()); } - delete_impl(DB_MANAGER.envelope_db(), move |rw| { - rw.get() - .primary::(account_id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| { - raise_error!( - format!( - "DownloadState '{}' not found during deletion process.", - account_id - ), - ErrorCode::ResourceNotFound - ) - }) - }) - .await + delete_impl::(DB_MANAGER.db(), &account_id.to_string()) } } diff --git a/crates/core/src/admin/meta.rs b/crates/core/src/admin/meta.rs index 10c8d6b..b7f1439 100644 --- a/crates/core/src/admin/meta.rs +++ b/crates/core/src/admin/meta.rs @@ -16,74 +16,44 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{path::Path, rc::Rc}; +use std::path::Path; -use native_db::{Builder, Database}; +use memdb::{Durability, MemDb}; use crate::{ - account::migration::AccountV3, - database::META_MODELS, + database::MemDbModel, error::{code::ErrorCode, BichonResult}, raise_error, - token::{AccessTokenModel, AccessTokenModelKey, TokenType}, users::{UserModel, DEFAULT_ADMIN_USER_ID}, utils::encrypt::internal_encrypt_string, }; -use itertools::Itertools; -pub fn init_meta_database(path: impl AsRef) -> BichonResult>> { - let database = Builder::new() - .set_cache_size(134217728) - .create(&META_MODELS, path) - .map_err(|e| { - raise_error!( - format!("Failed to open database: {:?}", e), - ErrorCode::InternalError - ) - })?; - - Ok(Rc::new(database)) +pub fn open_database(path: impl AsRef) -> BichonResult { + MemDb::open_with(path, Durability::Full).map_err(|e| { + raise_error!( + format!("Failed to open database: {:?}", e), + ErrorCode::InternalError + ) + }) } -pub fn list_all_accounts(database: &Rc>) -> BichonResult> { - let r_transaction = database - .r_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let entities: Vec = r_transaction - .scan() - .primary() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .all() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .try_collect() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(entities) -} - -pub fn find_admin(database: &Rc>) -> BichonResult> { - let r_transaction = database - .r_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let entity: Option = r_transaction - .get() - .primary(DEFAULT_ADMIN_USER_ID) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - return Ok(entity); +pub fn find_admin(db: &MemDb) -> BichonResult> { + let key = DEFAULT_ADMIN_USER_ID.to_string(); + let coll = db.collection(UserModel::collection()); + coll.get(&key) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError)) } pub fn update_admin_password( - database: &Rc>, + db: &MemDb, password: String, encrypt_key: &str, ) -> BichonResult<()> { - let rw_transaction = database - .rw_transaction() + let key = DEFAULT_ADMIN_USER_ID.to_string(); + let coll = db.collection(UserModel::collection()); + let entity: UserModel = coll + .get_required(&key) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let entity: UserModel = rw_transaction - .get() - .primary(DEFAULT_ADMIN_USER_ID) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| raise_error!("admin is not found".into(), ErrorCode::InternalError))?; let mut updated = entity.clone(); updated.password = Some( @@ -91,53 +61,8 @@ pub fn update_admin_password( .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?, ); - rw_transaction - .update(entity, updated) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - rw_transaction - .commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - return Ok(()); -} - -pub fn reset_webui_token(database: &Rc>) -> BichonResult<()> { - let rw_transaction = database - .rw_transaction() + coll.upsert(&key, &updated) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let tokens: Vec = rw_transaction - .scan() - .secondary(AccessTokenModelKey::user_id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .start_with(DEFAULT_ADMIN_USER_ID) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .try_collect() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let webui_token = tokens - .into_iter() - .find(|t| t.token_type == TokenType::WebUI); - - let new_token = AccessTokenModel::new_webui_token(DEFAULT_ADMIN_USER_ID); - match webui_token { - Some(current) => { - rw_transaction - .remove(current) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - rw_transaction - .insert(new_token) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - } - None => { - rw_transaction - .insert(new_token) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - } - } - - rw_transaction - .commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; Ok(()) } diff --git a/crates/core/src/autoconfig/load.rs b/crates/core/src/autoconfig/load.rs index 56cba84..b02f8d8 100644 --- a/crates/core/src/autoconfig/load.rs +++ b/crates/core/src/autoconfig/load.rs @@ -16,23 +16,18 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . - use crate::autoconfig::entity::{MailServerConfig, ServerConfig}; use crate::error::code::ErrorCode; use crate::{ - { - account::entity::Encryption, autoconfig::CachedMailSettings, error::BichonResult, - }, raise_error, + {account::entity::Encryption, autoconfig::CachedMailSettings, error::BichonResult}, }; use autoconfig::config::{Server, ServerType}; use email_address::EmailAddress; use std::str::FromStr; use tracing::error; -pub async fn resolve_autoconfig( - email: impl AsRef, -) -> BichonResult> { +pub async fn resolve_autoconfig(email: impl AsRef) -> BichonResult> { let email = email.as_ref(); let email_address = EmailAddress::from_str(email).map_err(|error| { raise_error!( @@ -43,7 +38,7 @@ pub async fn resolve_autoconfig( let domain = email_address.domain(); // try read local cache first - if let Some(cached_entity) = CachedMailSettings::get(domain).await? { + if let Some(cached_entity) = CachedMailSettings::get(domain)? { return Ok(Some(cached_entity.config)); } @@ -66,12 +61,12 @@ pub async fn resolve_autoconfig( .incoming_servers() .into_iter() .find(|s| matches!(s.server_type(), ServerType::Imap)); - + let imap_server = match imap_server { Some(imap) => imap, None => return Ok(None), }; - + let get_encryption = |server: &Server| { server .security_type() @@ -109,6 +104,6 @@ pub async fn resolve_autoconfig( imap: imap_config, oauth2: config.oauth2().map(|f| f.into()), }; - CachedMailSettings::add(domain.into(), result.clone()).await?; + CachedMailSettings::add(domain.into(), result.clone())?; Ok(Some(result)) } diff --git a/crates/core/src/autoconfig/mod.rs b/crates/core/src/autoconfig/mod.rs index 8473f21..87d3417 100644 --- a/crates/core/src/autoconfig/mod.rs +++ b/crates/core/src/autoconfig/mod.rs @@ -16,16 +16,10 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . - use crate::database::manager::DB_MANAGER; -use crate::database::{delete_impl, async_find_impl, upsert_impl}; -use crate::error::code::ErrorCode; -use crate::raise_error; -use crate::{ - autoconfig::entity::MailServerConfig, error::BichonResult, utc_now, -}; -use native_db::*; -use native_model::{native_model, Model}; +use crate::database::{delete_impl, upsert_impl}; +use crate::database::{find_impl, MemDbModel}; +use crate::{autoconfig::entity::MailServerConfig, error::BichonResult, utc_now}; use serde::{Deserialize, Serialize}; pub mod entity; @@ -36,45 +30,39 @@ mod tests; const EXPIRE_TIME_MS: i64 = 30 * 24 * 60 * 60 * 1000; #[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] -#[native_model(id = 3, version = 1)] -#[native_db] pub struct CachedMailSettings { - #[primary_key] pub domain: String, pub config: MailServerConfig, pub created_at: i64, } +impl MemDbModel for CachedMailSettings { + fn collection() -> &'static str { + "autoconfig" + } + fn key(&self) -> String { + self.domain.clone() + } +} + impl CachedMailSettings { - pub async fn add(domain: String, config: MailServerConfig) -> BichonResult<()> { + pub fn add(domain: String, config: MailServerConfig) -> BichonResult<()> { Self { domain, config, created_at: utc_now!(), } .save() - .await } - async fn save(&self) -> BichonResult<()> { - upsert_impl(DB_MANAGER.meta_db(), self.to_owned()).await + fn save(&self) -> BichonResult<()> { + upsert_impl(DB_MANAGER.db(), self.to_owned()) } - pub async fn get(domain: &str) -> BichonResult> { - if let Some(found) = - async_find_impl::(DB_MANAGER.meta_db(), domain.to_string()).await? - { + pub fn get(domain: &str) -> BichonResult> { + if let Some(found) = find_impl::(DB_MANAGER.db(), domain)? { if (utc_now!() - found.created_at) > EXPIRE_TIME_MS { - let domain = domain.to_string(); - delete_impl(DB_MANAGER.meta_db(), |rw| { - rw.get() - .primary::(domain) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| { - raise_error!("auto config cache miss".into(), ErrorCode::InternalError) - }) - }) - .await?; + delete_impl::(DB_MANAGER.db(), domain)?; Ok(None) } else { Ok(Some(found)) diff --git a/crates/core/src/cache/imap/download/download_folders.rs b/crates/core/src/cache/imap/download/download_folders.rs index c6c1eaf..ab40cba 100644 --- a/crates/core/src/cache/imap/download/download_folders.rs +++ b/crates/core/src/cache/imap/download/download_folders.rs @@ -19,7 +19,7 @@ use std::collections::BTreeSet; use crate::{ - decode_mailbox_name, + decode_mailbox_name, raise_error, { account::migration::{AccountModel, AccountType}, cache::imap::mailbox::{AttributeEnum, MailBox}, @@ -27,7 +27,6 @@ use crate::{ imap::{executor::ImapExecutor, session::SessionStream}, mailbox::list::convert_names_to_mailboxes, }, - raise_error, }; use async_imap::{types::Name, Session}; use tracing::{debug, info, warn}; @@ -62,7 +61,7 @@ pub async fn get_download_folders( mailboxes.iter().map(|(m, _)| m.name.clone()).collect(), ) .await?; - let account = AccountModel::async_get(account.id).await?; + let account = AccountModel::get(account.id)?; let subscribed = &account.download_folders.unwrap_or_default(); let is_noselect = |mailbox: &MailBox| { mailbox @@ -110,7 +109,7 @@ pub async fn get_download_folders( .iter() .map(|n| decode_mailbox_name!(n.name().to_string())) .collect(); - AccountModel::update_download_folders(account.id, sync_folders).await?; + AccountModel::update_download_folders(account.id, sync_folders)?; } else { warn!( "Account {}: No subscribed mailboxes found. This is unexpected — IMAP server should at least provide INBOX.", @@ -131,7 +130,7 @@ pub async fn detect_mailbox_changes( ) -> BichonResult<()> { if account.known_folders.is_none() { // First time sync: just save without comparing - AccountModel::update_known_folders(account.id, all_names).await?; + AccountModel::update_known_folders(account.id, all_names)?; return Ok(()); } let known_folders = account.known_folders.clone().unwrap_or_default(); @@ -160,7 +159,7 @@ pub async fn detect_mailbox_changes( // Note: When all subscribed folders are deleted (remaining_sync_folders empty), // the system's default behavior is to automatically fall back to syncing // only the default folders (INBOX and Sent) in subsequent operations - AccountModel::update_download_folders(account.id, remaining_sync_folders).await?; + AccountModel::update_download_folders(account.id, remaining_sync_folders)?; } info!( @@ -179,7 +178,7 @@ pub async fn detect_mailbox_changes( // Update known folders only if there were changes if has_changes { - AccountModel::update_known_folders(account.id, all_names).await?; + AccountModel::update_known_folders(account.id, all_names)?; } Ok(()) } diff --git a/crates/core/src/cache/imap/download/download_type.rs b/crates/core/src/cache/imap/download/download_type.rs index b0fcc13..5327a27 100644 --- a/crates/core/src/cache/imap/download/download_type.rs +++ b/crates/core/src/cache/imap/download/download_type.rs @@ -38,7 +38,7 @@ pub async fn decide_next_download_task( account: &AccountModel, trigger_type: TriggerType, ) -> BichonResult { - let state = match DownloadState::get(account.id).await? { + let state = match DownloadState::get(account.id)? { None => { DownloadState::init(account.id).await?; return Ok(DownloadTask::FullFetch); @@ -56,7 +56,7 @@ pub async fn decide_next_download_task( }; if should_start { - DownloadState::start_new_session(account.id, trigger_type).await?; + DownloadState::start_new_session(account.id, trigger_type)?; Ok(DownloadTask::TraceFetch) } else { Ok(DownloadTask::Idle) diff --git a/crates/core/src/cache/imap/download/flow.rs b/crates/core/src/cache/imap/download/flow.rs index b0f2a7d..73e1a8e 100644 --- a/crates/core/src/cache/imap/download/flow.rs +++ b/crates/core/src/cache/imap/download/flow.rs @@ -17,6 +17,7 @@ // along with this program. If not, see . use crate::{ + raise_error, { account::{ migration::AccountModel, @@ -34,7 +35,6 @@ use crate::{ imap::executor::ImapExecutor, store::tantivy::envelope::ENVELOPE_MANAGER, }, - raise_error, }; use std::time::Instant; use tokio_util::sync::CancellationToken; @@ -67,9 +67,8 @@ pub async fn fetch_and_save_by_date( 0, FolderStatus::Failed, Some(err_msg.clone()), - ) - .await?; - DownloadState::append_session_error(account_id, err_msg).await?; + )?; + DownloadState::append_session_error(account_id, err_msg)?; return Err(e); } }; @@ -93,9 +92,8 @@ pub async fn fetch_and_save_by_date( 0, FolderStatus::Failed, Some(err_msg.clone()), - ) - .await?; - DownloadState::append_session_error(account_id, err_msg).await?; + )?; + DownloadState::append_session_error(account_id, err_msg)?; return Err(e); } }; @@ -109,8 +107,7 @@ pub async fn fetch_and_save_by_date( 0, FolderStatus::Success, None, - ) - .await?; + )?; return Ok(()); } @@ -145,8 +142,7 @@ pub async fn fetch_and_save_by_date( 0, FolderStatus::Pending, None, - ) - .await?; + )?; let mut current_processed = 0u64; let mut has_error_or_cancel = false; @@ -156,8 +152,7 @@ pub async fn fetch_and_save_by_date( account_id, DownloadStatus::Cancelled, Some("User stopped or system shutdown".to_string()), - ) - .await?; + )?; DownloadState::update_folder_progress( account_id, mailbox.name.clone(), @@ -165,8 +160,7 @@ pub async fn fetch_and_save_by_date( current_processed, FolderStatus::Cancelled, None, - ) - .await?; + )?; has_error_or_cancel = true; break; } @@ -189,12 +183,11 @@ pub async fn fetch_and_save_by_date( current_processed, FolderStatus::Downloading, None, - ) - .await?; + )?; } Err(e) => { let err_msg = format!("Batch {} failed: {:#?}", index, e); - DownloadState::append_session_error(account_id, err_msg.clone()).await?; + DownloadState::append_session_error(account_id, err_msg.clone())?; DownloadState::update_folder_progress( account_id, mailbox.name.clone(), @@ -202,8 +195,7 @@ pub async fn fetch_and_save_by_date( current_processed, FolderStatus::Failed, Some(err_msg), - ) - .await?; + )?; has_error_or_cancel = true; break; } @@ -217,8 +209,7 @@ pub async fn fetch_and_save_by_date( current_processed, FolderStatus::Success, None, - ) - .await?; + )?; } session.logout().await.ok(); Ok(()) @@ -243,9 +234,8 @@ pub async fn fetch_and_save_full_mailbox( 0, FolderStatus::Failed, Some(err_msg.clone()), - ) - .await?; - DownloadState::append_session_error(account_id, err_msg).await?; + )?; + DownloadState::append_session_error(account_id, err_msg)?; return Err(e); } }; @@ -261,10 +251,9 @@ pub async fn fetch_and_save_full_mailbox( 0, FolderStatus::Failed, Some(err_msg.clone()), - ) - .await?; + )?; - DownloadState::append_session_error(account_id, err_msg).await?; + DownloadState::append_session_error(account_id, err_msg)?; session.logout().await.ok(); return Err(raise_error!( format!("{:#?}", e), @@ -307,8 +296,7 @@ pub async fn fetch_and_save_full_mailbox( account_id, DownloadStatus::Cancelled, Some("User stopped or system shutdown".to_string()), - ) - .await?; + )?; DownloadState::update_folder_progress( account_id, mailbox.name.clone(), @@ -316,8 +304,7 @@ pub async fn fetch_and_save_full_mailbox( current_processed, FolderStatus::Cancelled, None, - ) - .await?; + )?; has_error_or_cancel = true; break; } @@ -344,12 +331,11 @@ pub async fn fetch_and_save_full_mailbox( current_processed, FolderStatus::Downloading, None, - ) - .await?; + )?; } Err(e) => { let err_msg = format!("Batch {} failed: {:#?}", page, e); - DownloadState::append_session_error(account_id, err_msg.clone()).await?; + DownloadState::append_session_error(account_id, err_msg.clone())?; DownloadState::update_folder_progress( account_id, mailbox.name.clone(), @@ -357,8 +343,7 @@ pub async fn fetch_and_save_full_mailbox( current_processed, FolderStatus::Failed, Some(err_msg), - ) - .await?; + )?; has_error_or_cancel = true; break; } @@ -373,8 +358,7 @@ pub async fn fetch_and_save_full_mailbox( current_processed, FolderStatus::Success, None, - ) - .await?; + )?; } session.logout().await.ok(); Ok(()) @@ -452,8 +436,7 @@ pub async fn reconcile_mailboxes( DownloadState::init_folder_details( account.id, remote_mailboxes.iter().map(|m| m.name.clone()).collect(), - ) - .await?; + )?; for (local_mailbox, remote_mailbox) in &existing_mailboxes { if token.is_cancelled() { @@ -461,8 +444,7 @@ pub async fn reconcile_mailboxes( account.id, DownloadStatus::Cancelled, Some("Received termination signal (User stop or System shutdown)".to_string()), - ) - .await?; + )?; break; } @@ -482,9 +464,8 @@ pub async fn reconcile_mailboxes( 0, FolderStatus::Failed, Some(err_msg.clone()), - ) - .await?; - DownloadState::append_session_error(account_id, err_msg).await?; + )?; + DownloadState::append_session_error(account_id, err_msg)?; continue; } info!( @@ -500,8 +481,7 @@ pub async fn reconcile_mailboxes( 0, FolderStatus::Downloading, Some("UID validity changed, rebuilding...".into()), - ) - .await?; + )?; match &account.date_since { Some(date_since) => { @@ -547,7 +527,7 @@ pub async fn reconcile_mailboxes( } //The metadata of this mailbox must only be updated after a successful synchronization; //otherwise, it may cause synchronization errors and result in missing emails in the local sync results. - MailBox::batch_upsert(&mailboxes_to_update).await?; + MailBox::batch_upsert(&mailboxes_to_update)?; } debug!( @@ -559,7 +539,7 @@ pub async fn reconcile_mailboxes( let missing_mailboxes = find_missing_mailboxes(local_mailboxes, remote_mailboxes); //Mail folders that are not locally need to be downloaded. if !missing_mailboxes.is_empty() { - MailBox::batch_insert(&missing_mailboxes).await?; + MailBox::batch_insert(&missing_mailboxes)?; let mut has_error = false; let mut last_err = None; @@ -569,8 +549,7 @@ pub async fn reconcile_mailboxes( account.id, DownloadStatus::Cancelled, Some("Received termination signal (User stop or System shutdown)".to_string()), - ) - .await?; + )?; break; } if mailbox.exists > 0 { @@ -650,9 +629,7 @@ async fn perform_incremental_sync( token: CancellationToken, ) -> BichonResult<()> { if remote_mailbox.exists > 0 { - let local_max_uid = ENVELOPE_MANAGER - .get_max_uid(account.id, local_mailbox.id) - .await?; + let local_max_uid = ENVELOPE_MANAGER.get_max_uid(account.id, local_mailbox.id)?; match local_max_uid { Some(max_uid) => { let mut session = ImapExecutor::create_connection(account.id).await?; diff --git a/crates/core/src/cache/imap/download/mod.rs b/crates/core/src/cache/imap/download/mod.rs index d68ef39..567b1ce 100644 --- a/crates/core/src/cache/imap/download/mod.rs +++ b/crates/core/src/cache/imap/download/mod.rs @@ -57,8 +57,7 @@ pub async fn process_imap_download( account_id, DownloadStatus::Failed, Some(format!("Failed to connect to IMAP server: {}", e)), - ) - .await?; + )?; return Err(e); } }; @@ -67,8 +66,11 @@ pub async fn process_imap_download( Err(err) => { let err_msg = format!("Failed to fetch mailboxes: {}", err); warn!(account_id = account.id, error = %err, "{}", err_msg); - DownloadState::update_session_status(account_id, DownloadStatus::Failed, Some(err_msg)) - .await?; + DownloadState::update_session_status( + account_id, + DownloadStatus::Failed, + Some(err_msg), + )?; return Ok(()); } }; @@ -101,34 +103,27 @@ pub async fn process_imap_download( }; match result { Ok(_) => { - DownloadState::update_session_status(account_id, DownloadStatus::Success, None) - .await?; + DownloadState::update_session_status(account_id, DownloadStatus::Success, None)?; } Err(e) => { DownloadState::update_session_status( account_id, DownloadStatus::Failed, Some(format!("Email Download interrupted: {:#?}", e)), - ) - .await?; + )?; } } return Ok(()); } - let local_mailboxes = MailBox::list_all(account_id).await?; + let local_mailboxes = MailBox::list_all(account_id)?; match reconcile_mailboxes(account, &remote_mailboxes, &local_mailboxes, token).await { - Ok(_) => { - DownloadState::update_session_status(account_id, DownloadStatus::Success, None).await? - } - Err(e) => { - DownloadState::update_session_status( - account_id, - DownloadStatus::Failed, - Some(format!("Email Download interrupted: {:#?}", e)), - ) - .await? - } + Ok(_) => DownloadState::update_session_status(account_id, DownloadStatus::Success, None)?, + Err(e) => DownloadState::update_session_status( + account_id, + DownloadStatus::Failed, + Some(format!("Email Download interrupted: {:#?}", e)), + )?, } let elapsed_time = start_time.elapsed().as_secs(); debug!( diff --git a/crates/core/src/cache/imap/download/rebuild.rs b/crates/core/src/cache/imap/download/rebuild.rs index c4042ae..4392e76 100644 --- a/crates/core/src/cache/imap/download/rebuild.rs +++ b/crates/core/src/cache/imap/download/rebuild.rs @@ -17,6 +17,7 @@ // along with this program. If not, see . use crate::{ + raise_error, { account::{ migration::AccountModel, @@ -34,7 +35,6 @@ use crate::{ error::{code::ErrorCode, BichonResult}, store::tantivy::envelope::ENVELOPE_MANAGER, }, - raise_error, }; use tokio_util::sync::CancellationToken; @@ -45,12 +45,11 @@ pub async fn rebuild_cache( remote_mailboxes: &[MailBox], token: CancellationToken, ) -> BichonResult<()> { - MailBox::batch_insert(remote_mailboxes).await?; + MailBox::batch_insert(remote_mailboxes)?; DownloadState::init_folder_details( account.id, remote_mailboxes.iter().map(|m| m.name.clone()).collect(), - ) - .await?; + )?; let mut has_error = false; let mut last_err = None; @@ -61,8 +60,7 @@ pub async fn rebuild_cache( account.id, DownloadStatus::Cancelled, Some("Received termination signal (User stop or System shutdown)".to_string()), - ) - .await?; + )?; break; } if mailbox.exists == 0 { @@ -77,8 +75,7 @@ pub async fn rebuild_cache( 0, FolderStatus::Success, None, - ) - .await?; + )?; continue; } let account = account.clone(); @@ -94,9 +91,9 @@ pub async fn rebuild_cache( continue; } }; - + match fetch_and_save_full_mailbox(&account, &mailbox, token.clone()).await { - Ok(_) => {}, + Ok(_) => {} Err(err) => { has_error = true; tracing::error!("Folder sync task failed: {:#?}", err); @@ -124,12 +121,11 @@ pub async fn rebuild_cache_by_date( direction: FetchDirection, token: CancellationToken, ) -> BichonResult<()> { - MailBox::batch_insert(remote_mailboxes).await?; + MailBox::batch_insert(remote_mailboxes)?; DownloadState::init_folder_details( account.id, remote_mailboxes.iter().map(|m| m.name.clone()).collect(), - ) - .await?; + )?; let mut has_error = false; let mut last_err = None; @@ -140,8 +136,7 @@ pub async fn rebuild_cache_by_date( account.id, DownloadStatus::Cancelled, Some("Received termination signal (User stop or System shutdown)".to_string()), - ) - .await?; + )?; break; } if mailbox.exists == 0 { @@ -157,8 +152,7 @@ pub async fn rebuild_cache_by_date( 0, FolderStatus::Success, None, - ) - .await?; + )?; continue; } let account = account.clone(); @@ -224,8 +218,7 @@ pub async fn rebuild_mailbox_cache( 0, FolderStatus::Success, None, - ) - .await?; + )?; return Ok(()); } @@ -257,8 +250,7 @@ pub async fn rebuild_mailbox_cache_by_date( 0, FolderStatus::Success, None, - ) - .await?; + )?; return Ok(()); } diff --git a/crates/core/src/cache/imap/mailbox.rs b/crates/core/src/cache/imap/mailbox.rs index d8f320f..b85e712 100644 --- a/crates/core/src/cache/imap/mailbox.rs +++ b/crates/core/src/cache/imap/mailbox.rs @@ -17,33 +17,24 @@ // along with this program. If not, see . use crate::{ - decode_mailbox_name, encode_mailbox_name, + decode_mailbox_name, encode_mailbox_name, raise_error, { database::{ - async_filter_by_secondary_key_impl, async_find_impl, batch_delete_impl, - batch_insert_impl, batch_upsert_impl, delete_impl, filter_by_secondary_key_impl, - find_impl, manager::DB_MANAGER, + batch_delete_impl, batch_insert_impl, batch_upsert_impl, delete_impl, filter_impl, + find_impl, manager::DB_MANAGER, MemDbModel, }, error::{code::ErrorCode, BichonResult}, }, - raise_error, }; use async_imap::types::{Name, NameAttribute}; -use itertools::Itertools; -use native_db::*; -use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] -#[native_model(id = 1, version = 1)] -#[native_db] pub struct MailBox { /// The unique identifier for the mailbox - #[primary_key] pub id: u64, /// The ID of the account associated with the mailbox - #[secondary_key] pub account_id: u64, /// The unique, decoded, human-readable name of the mailbox (e.g., "INBOX", "Sent Items"). /// This is the decoded name as presented to users, derived from the IMAP server's mailbox name @@ -67,23 +58,22 @@ pub struct MailBox { pub uid_validity: Option, } +impl MemDbModel for MailBox { + fn collection() -> &'static str { + "mailboxes" + } + fn key(&self) -> String { + self.id.to_string() + } +} + impl MailBox { pub fn encoded_name(&self) -> String { encode_mailbox_name!(&self.name) } - pub async fn async_get(id: u64) -> BichonResult { - let result = async_find_impl::(DB_MANAGER.envelope_db(), id).await?; - Ok(result.ok_or_else(|| { - raise_error!( - format!("mailbox {} not found", id), - ErrorCode::InternalError - ) - })?) - } - pub fn get(id: u64) -> BichonResult { - let result = find_impl::(DB_MANAGER.envelope_db(), id)?; + let result = find_impl::(DB_MANAGER.db(), &id.to_string())?; Ok(result.ok_or_else(|| { raise_error!( format!("mailbox {} not found", id), @@ -92,55 +82,34 @@ impl MailBox { })?) } - pub async fn delete(id: u64) -> BichonResult<()> { - delete_impl(DB_MANAGER.envelope_db(), move |rw| { - rw.get() - .primary::(id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| raise_error!("mailbox missing".into(), ErrorCode::InternalError)) - }) - .await + pub fn delete(id: u64) -> BichonResult<()> { + delete_impl::(DB_MANAGER.db(), &id.to_string()) } - pub async fn list_all(account_id: u64) -> BichonResult> { - async_filter_by_secondary_key_impl( - DB_MANAGER.envelope_db(), - MailBoxKey::account_id, - account_id, - ) - .await + pub fn list_all(account_id: u64) -> BichonResult> { + filter_impl::(DB_MANAGER.db(), move |m| m.account_id == account_id) } pub fn find_mailbox(account_id: u64, mailbox_id: u64) -> BichonResult> { - let all: Vec = filter_by_secondary_key_impl( - DB_MANAGER.envelope_db(), - MailBoxKey::account_id, - account_id, - )?; + let all = filter_impl::(DB_MANAGER.db(), move |m| m.account_id == account_id)?; Ok(all.into_iter().find(|m| m.id == mailbox_id)) } - pub async fn batch_insert(mailboxes: &[MailBox]) -> BichonResult<()> { - batch_insert_impl(DB_MANAGER.envelope_db(), mailboxes.to_vec()).await + pub fn batch_insert(mailboxes: &[MailBox]) -> BichonResult<()> { + batch_insert_impl(DB_MANAGER.db(), mailboxes.to_vec()) } - pub async fn batch_upsert(mailboxes: &[MailBox]) -> BichonResult<()> { - batch_upsert_impl(DB_MANAGER.envelope_db(), mailboxes.to_vec()).await + pub fn batch_upsert(mailboxes: &[MailBox]) -> BichonResult<()> { + batch_upsert_impl(DB_MANAGER.db(), mailboxes.to_vec()) } - pub async fn clean(account_id: u64) -> BichonResult<()> { - batch_delete_impl(DB_MANAGER.envelope_db(), move |rw| { - let mailboxes: Vec = rw - .scan() - .secondary::(MailBoxKey::account_id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .start_with(account_id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .try_collect() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(mailboxes) - }) - .await?; + pub fn clean(account_id: u64) -> BichonResult<()> { + let mailboxes = + filter_impl::(DB_MANAGER.db(), move |m| m.account_id == account_id)?; + let keys: Vec = mailboxes.iter().map(|m| m.id.to_string()).collect(); + if !keys.is_empty() { + batch_delete_impl::(DB_MANAGER.db(), keys)?; + } Ok(()) } } diff --git a/crates/core/src/cache/imap/mod.rs b/crates/core/src/cache/imap/mod.rs index 8ff116c..5a32b19 100644 --- a/crates/core/src/cache/imap/mod.rs +++ b/crates/core/src/cache/imap/mod.rs @@ -16,30 +16,14 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{ - collections::{HashMap, HashSet}, - sync::LazyLock, -}; +use std::collections::{HashMap, HashSet}; -use crate::{ - account::{old_state::AccountRunningState, state::DownloadState}, - database::ModelsAdapter, -}; use mailbox::MailBox; -use native_db::Models; pub mod download; pub mod mailbox; pub mod task; -pub static MAILBOX_MODELS: LazyLock = LazyLock::new(|| { - let mut adapter = ModelsAdapter::new(); - adapter.register_model::(); - adapter.register_model::(); - adapter.register_model::(); - adapter.models -}); - pub fn find_missing_mailboxes( local_mailboxes: &[MailBox], server_mailboxes: &[MailBox], diff --git a/crates/core/src/cache/imap/task.rs b/crates/core/src/cache/imap/task.rs index e9544e8..7d0ea08 100644 --- a/crates/core/src/cache/imap/task.rs +++ b/crates/core/src/cache/imap/task.rs @@ -99,7 +99,7 @@ impl AccountDownTask { SYNC_TASKS.set_busy(id, false).await; }); }); - let account = AccountModel::async_get(account_id).await.ok(); + let account = AccountModel::get(account_id).ok(); match account { Some(account) => { if !account.enabled { @@ -115,7 +115,7 @@ impl AccountDownTask { } else { if let Some(imap) = &account.imap { if let AuthType::OAuth2 = imap.auth.auth_type { - if OAuth2AccessToken::get(account.id).await?.is_none() { + if OAuth2AccessToken::get(account.id)?.is_none() { if utc_now!() % 300_000 == 0 { warn!("Account {}: download aborted. OAuth2 authorization not completed. Please visit the rustmailer admin page to authorize this account.", account_id); } @@ -133,8 +133,7 @@ impl AccountDownTask { DownloadState::append_session_error( account.id, format!("error in account download task: {:#?}", e), - ) - .await?; + )?; error!( "Failed to download mailbox data for '{}': {:?}", account_id, e @@ -228,7 +227,7 @@ impl AccountDownTask { if token_clone.is_cancelled() { return; } - let account = match AccountModel::async_get(account_id).await { + let account = match AccountModel::get(account_id) { Ok(acc) => acc, Err(e) => { error!("Failed to fetch account {}: {:?}", account_id, e); @@ -240,7 +239,7 @@ impl AccountDownTask { { error!("Manual download failed for {}: {:?}", account_id, e); let error_msg = format!("error in account download task: {:#?}", e); - let _ = DownloadState::append_session_error(account.id, error_msg).await; + let _ = DownloadState::append_session_error(account.id, error_msg); } }); { diff --git a/crates/core/src/common/auth.rs b/crates/core/src/common/auth.rs index 61b631f..2be0c00 100644 --- a/crates/core/src/common/auth.rs +++ b/crates/core/src/common/auth.rs @@ -16,12 +16,12 @@ pub struct ClientContext { } impl ClientContext { - pub async fn require_any_permission( + pub fn require_any_permission( &self, requirements: Vec<(Option, &str)>, ) -> BichonResult<()> { for (account_id, permission) in requirements { - if self.has_permission(account_id, permission).await { + if self.has_permission(account_id, permission) { return Ok(()); } } @@ -31,18 +31,18 @@ impl ClientContext { )) } - pub async fn check_has_permission( + pub fn check_has_permission( user: &UserModel, account_id: Option, permission: &str, ) -> bool { - if user.is_admin().await { + if user.is_admin() { return true; } let mut global_perms = HashSet::new(); for rid in &user.global_roles { - if let Some(role) = UserRole::find(*rid).await.ok().flatten() { + if let Some(role) = UserRole::find(*rid).ok().flatten() { global_perms.extend(role.permissions); } } @@ -53,7 +53,7 @@ impl ClientContext { if let Some(aid) = account_id { if let Some(role_id) = user.account_access_map.get(&aid) { - if let Some(role) = UserRole::find(*role_id).await.ok().flatten() { + if let Some(role) = UserRole::find(*role_id).ok().flatten() { if role.permissions.contains(&permission.to_string()) || Self::check_account_logic(&role.permissions, permission) { @@ -66,14 +66,14 @@ impl ClientContext { false } - pub async fn has_permission(&self, account_id: Option, permission: &str) -> bool { - if self.user.is_admin().await { + pub fn has_permission(&self, account_id: Option, permission: &str) -> bool { + if self.user.is_admin() { return true; } let mut global_perms = HashSet::new(); for rid in &self.user.global_roles { - if let Some(role) = UserRole::find(*rid).await.ok().flatten() { + if let Some(role) = UserRole::find(*rid).ok().flatten() { global_perms.extend(role.permissions); } } @@ -84,7 +84,7 @@ impl ClientContext { if let Some(aid) = account_id { if let Some(role_id) = self.user.account_access_map.get(&aid) { - if let Some(role) = UserRole::find(*role_id).await.ok().flatten() { + if let Some(role) = UserRole::find(*role_id).ok().flatten() { if role.permissions.contains(&permission.to_string()) || Self::check_account_logic(&role.permissions, permission) { @@ -126,12 +126,12 @@ impl ClientContext { } } - pub async fn require_permission( + pub fn require_permission( &self, account_id: Option, permission: &str, ) -> BichonResult<()> { - if self.has_permission(account_id, permission).await { + if self.has_permission(account_id, permission) { Ok(()) } else { Err(raise_error!( diff --git a/crates/core/src/common/paginated.rs b/crates/core/src/common/paginated.rs index 2f67e7f..32c4866 100644 --- a/crates/core/src/common/paginated.rs +++ b/crates/core/src/common/paginated.rs @@ -178,3 +178,83 @@ impl Paginated { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn paginate_vec_full_list_without_pagination() { + let items: Vec = (1..=10).collect(); + let result = paginate_vec(&items, None, None).unwrap(); + assert_eq!(result.items.len(), 10); + assert_eq!(result.total_items, 10); + assert_eq!(result.page, None); + assert_eq!(result.total_pages, None); + } + + #[test] + fn paginate_vec_first_page() { + let items: Vec = (1..=25).collect(); + let result = paginate_vec(&items, Some(1), Some(10)).unwrap(); + assert_eq!(result.items, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + assert_eq!(result.total_items, 25); + assert_eq!(result.total_pages, Some(3)); + assert_eq!(result.page, Some(1)); + } + + #[test] + fn paginate_vec_last_partial_page() { + let items: Vec = (1..=25).collect(); + let result = paginate_vec(&items, Some(3), Some(10)).unwrap(); + assert_eq!(result.items, vec![21, 22, 23, 24, 25]); + assert_eq!(result.total_items, 25); + assert_eq!(result.total_pages, Some(3)); + } + + #[test] + fn paginate_vec_page_beyond_range_returns_empty() { + let items: Vec = (1..=10).collect(); + let result = paginate_vec(&items, Some(5), Some(10)).unwrap(); + assert_eq!(result.items.len(), 0); + assert_eq!(result.total_items, 10); + } + + #[test] + fn paginate_vec_empty_list() { + let items: Vec = vec![]; + let result = paginate_vec(&items, Some(1), Some(10)).unwrap(); + assert_eq!(result.items.len(), 0); + assert_eq!(result.total_items, 0); + assert_eq!(result.total_pages, Some(0)); + } + + #[test] + fn paginate_vec_zero_page_returns_error() { + let items: Vec = (1..=10).collect(); + assert!(paginate_vec(&items, Some(0), Some(10)).is_err()); + } + + #[test] + fn paginate_vec_zero_page_size_returns_error() { + let items: Vec = (1..=10).collect(); + assert!(paginate_vec(&items, Some(1), Some(0)).is_err()); + } + + #[test] + fn paginate_vec_single_item() { + let items = vec![42]; + let result = paginate_vec(&items, Some(1), Some(10)).unwrap(); + assert_eq!(result.items, vec![42]); + assert_eq!(result.total_items, 1); + assert_eq!(result.total_pages, Some(1)); + } + + #[test] + fn paginate_vec_exact_page_boundary() { + let items: Vec = (1..=20).collect(); + let result = paginate_vec(&items, Some(2), Some(10)).unwrap(); + assert_eq!(result.items, vec![11, 12, 13, 14, 15, 16, 17, 18, 19, 20]); + assert_eq!(result.total_pages, Some(2)); + } +} diff --git a/crates/core/src/context/executors.rs b/crates/core/src/context/executors.rs index b9c64fc..7970a5d 100644 --- a/crates/core/src/context/executors.rs +++ b/crates/core/src/context/executors.rs @@ -50,7 +50,7 @@ impl BichonContext { } pub async fn start_account_downloader(&self) -> BichonResult<()> { - let accounts = AccountModel::list_all().await?; + let accounts = AccountModel::list_all()?; let active_accounts: Vec = accounts .into_iter() .filter(|a| a.enabled && matches!(a.account_type, AccountType::IMAP)) diff --git a/crates/core/src/dashboard/mod.rs b/crates/core/src/dashboard/mod.rs index f812c85..470f8cb 100644 --- a/crates/core/src/dashboard/mod.rs +++ b/crates/core/src/dashboard/mod.rs @@ -62,29 +62,21 @@ pub struct DashboardStats { impl DashboardStats { pub async fn get(context: ClientContext) -> BichonResult { - let has_all_accounts = context - .has_permission(None, Permission::ACCOUNT_MANAGE_ALL) - .await; - + let has_all_accounts = context.has_permission(None, Permission::ACCOUNT_MANAGE_ALL); let authorized_ids: Option> = if has_all_accounts { None } else { Some(context.user.account_access_map.keys().cloned().collect()) }; - let mut stat = ENVELOPE_MANAGER - .get_dashboard_stats(&authorized_ids) - .await?; + let mut stat = ENVELOPE_MANAGER.get_dashboard_stats(&authorized_ids)?; - stat.top_largest_emails = ENVELOPE_MANAGER - .top_10_largest_emails(&authorized_ids) - .await?; - stat.top_largest_attachments = ATTACHMENT_MANAGER - .top_10_largest_attachments(&authorized_ids) - .await?; + stat.top_largest_emails = ENVELOPE_MANAGER.top_10_largest_emails(&authorized_ids)?; + stat.top_largest_attachments = + ATTACHMENT_MANAGER.top_10_largest_attachments(&authorized_ids)?; stat.account_count = if has_all_accounts { - AccountModel::count().await? + AccountModel::count()? } else { authorized_ids.as_ref().map(|ids| ids.len()).unwrap_or(0) }; @@ -196,18 +188,11 @@ impl LargestAttachment { let size_bytes = value.as_u64().ok_or_else(|| { raise_error!("'size' field is not a u64".into(), ErrorCode::InternalError) })?; - let value = document.get_first(fields.f_name_exact).ok_or_else(|| { - raise_error!( - "'name_exact' field not found".into(), - ErrorCode::InternalError - ) - })?; - let name = value.as_str().map(|s| s.to_string()).ok_or_else(|| { - raise_error!( - "'name_exact' field is not a string".into(), - ErrorCode::InternalError - ) - })?; + let name = document + .get_first(fields.f_name_exact) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .unwrap_or_else(|| "Unknown".to_string()); let value = document.get_first(fields.f_id).ok_or_else(|| { raise_error!( diff --git a/crates/core/src/database/manager.rs b/crates/core/src/database/manager.rs index 1925ae9..b6b00bd 100644 --- a/crates/core/src/database/manager.rs +++ b/crates/core/src/database/manager.rs @@ -16,108 +16,33 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use crate::account::migration::AccountModel; -use crate::cache::imap::MAILBOX_MODELS; -use crate::error::{code::ErrorCode, BichonError}; -use crate::settings::cli::SETTINGS; use crate::settings::dir::DATA_DIR_MANAGER; -use crate::users::UserModel; -use crate::{database::META_MODELS, error::BichonResult}; -use crate::raise_error; -use native_db::{Builder, Database}; -use std::sync::{Arc, LazyLock}; -use tracing::info; +use memdb::{Durability, MemDb}; +use std::sync::LazyLock; +use std::time::Duration; pub static DB_MANAGER: LazyLock = LazyLock::new(DatabaseManager::new); -/// Metadata database instance pub struct DatabaseManager { - meta_db: Arc>, - /// Envelope database instance - envelope_db: Arc>, + db: MemDb, } impl DatabaseManager { fn new() -> Self { - let meta_db = Self::init_meta_database().expect("Failed to initialize metadata database"); - let envelope_db = - Self::init_evenlope_database().expect("Failed to initialize evenlope database"); - DatabaseManager { - meta_db, - envelope_db, - } + let db_path = &DATA_DIR_MANAGER.memdb_dir; + std::fs::create_dir_all(db_path).expect("Failed to create memdb data directory"); + + let db = MemDb::open_with(db_path, Durability::Full) + .expect("Failed to open memdb database"); + + // Start periodic snapshot worker (every 5 minutes) + db.start_snapshot_worker(Duration::from_secs(300)); + + DatabaseManager { db } } - /// Get a reference to the metadata database - pub fn meta_db(&self) -> &Arc> { - &self.meta_db - } - - pub fn envelope_db(&self) -> &Arc> { - &self.envelope_db - } - - /// Initialize metadata database with a fixed or configured file path - fn init_meta_database() -> BichonResult>> { - let mut database = Builder::new() - .set_cache_size( - SETTINGS - .bichon_metadata_cache_size - .unwrap_or(134217728) - .max(67108864), - ) //default 128MB - .create(&META_MODELS, DATA_DIR_MANAGER.meta_db.clone()) - .map_err(Self::handle_database_error)?; - - let rw = database - .rw_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - rw.migrate::() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - rw.migrate::() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - rw.commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - database - .compact() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(Arc::new(database)) - } - - fn init_evenlope_database() -> BichonResult>> { - info!( - "Initializing envelope database at: {:?}", - &DATA_DIR_MANAGER.mailbox_db - ); - - let mut database = Builder::new() - .set_cache_size( - SETTINGS - .bichon_envelope_cache_size - .unwrap_or(134217728) - .max(67108864), - ) //default 128MB - .create(&MAILBOX_MODELS, DATA_DIR_MANAGER.mailbox_db.clone()) - .map_err(Self::handle_database_error)?; - - let rw = database - .rw_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - rw.commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - database - .compact() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - Ok(Arc::new(database)) - } - - fn handle_database_error(error: native_db::db_type::Error) -> BichonError { - raise_error!( - format!("Failed to create database: {:?}", error), - ErrorCode::InternalError - ) + /// Get a reference to the MemDb instance. + pub fn db(&self) -> &MemDb { + &self.db } } diff --git a/crates/core/src/database/mod.rs b/crates/core/src/database/mod.rs index 12fb264..c7a6cc8 100644 --- a/crates/core/src/database/mod.rs +++ b/crates/core/src/database/mod.rs @@ -16,332 +16,157 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use crate::account::migration::{AccountV1, AccountV2, AccountV3, AccountV4}; -use crate::autoconfig::CachedMailSettings; use crate::common::paginated::Paginated; use crate::error::code::ErrorCode; use crate::error::BichonResult; -use crate::oauth2::entity::OAuth2; -use crate::oauth2::pending::OAuth2PendingEntity; -use crate::oauth2::token::OAuth2AccessToken; -use crate::settings::proxy::Proxy; -use crate::settings::system::SystemSetting; -use crate::token::AccessTokenModel; -use crate::users::role::UserRole; -use crate::users::{BichonUser, BichonUserV2}; use crate::raise_error; -use db_type::{KeyOptions, ToKeyDefinition}; -use itertools::Itertools; -use native_db::*; +use memdb::{MemDb, Transaction}; +use serde::de::DeserializeOwned; use serde::Serialize; -use std::sync::{Arc, LazyLock}; -use transaction::RwTransaction; pub mod manager; -pub static META_MODELS: LazyLock = LazyLock::new(|| { - let mut adapter = ModelsAdapter::new(); - adapter.register_metadata_models(); - adapter.models -}); - -pub struct ModelsAdapter { - pub models: Models, +/// Trait for models that can be stored in MemDb collections. +pub trait MemDbModel: Serialize + DeserializeOwned + Clone + Send + 'static { + /// The collection name this model is stored under. + fn collection() -> &'static str; + /// The primary key as a string for MemDb storage. + fn key(&self) -> String; } -impl ModelsAdapter { - pub fn new() -> Self { - ModelsAdapter { - models: Models::new(), - } +// ─── Insert ─────────────────────────────────────────────────────────────── + +pub fn insert_impl(db: &MemDb, item: M) -> BichonResult<()> { + let coll = db.collection(M::collection()); + let key = item.key(); + coll.insert(key, &item) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError)) +} + +pub fn batch_insert_impl(db: &MemDb, items: Vec) -> BichonResult<()> { + let txn = db.transaction(); + let mut txn = txn; + for item in &items { + txn = txn + .insert(M::collection(), item.key(), item) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; } + txn.commit() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError)) +} - pub fn register_model(&mut self) { - self.models.define::().expect("failed to define model "); - } +// ─── Upsert ──────────────────────────────────────────────────────────────── - pub fn register_metadata_models(&mut self) { - //Starting from version 0.2.0, `AccessToken` is deprecated/no longer used, but its ID must not be reused, otherwise it may cause model errors. - //self.register_model::(); - self.register_model::(); - self.register_model::(); - self.register_model::(); - self.register_model::(); - self.register_model::(); - self.register_model::(); - self.register_model::(); - self.register_model::(); - self.register_model::(); - self.register_model::(); - self.register_model::(); - self.register_model::(); - self.register_model::(); - self.register_model::(); +pub fn upsert_impl(db: &MemDb, item: M) -> BichonResult<()> { + let coll = db.collection(M::collection()); + coll.upsert(item.key(), &item) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError)) +} + +pub fn batch_upsert_impl(db: &MemDb, items: Vec) -> BichonResult<()> { + let txn = db.transaction(); + let mut txn = txn; + for item in &items { + txn = txn + .upsert(M::collection(), item.key(), item) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; } + txn.commit() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError)) } -pub async fn insert_impl( - database: &Arc>, - item: T, -) -> BichonResult<()> { - let db = database.clone(); - tokio::task::spawn_blocking(move || { - let rw_transaction = db - .rw_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - rw_transaction - .insert(item) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - rw_transaction - .commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(()) - }) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? +// ─── Find ────────────────────────────────────────────────────────────────── + +pub fn find_impl(db: &MemDb, key: &str) -> BichonResult> { + let coll = db.collection(M::collection()); + coll.get(key) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError)) } -pub async fn batch_insert_impl( - database: &Arc>, - batch: Vec, -) -> BichonResult<()> { - let db = database.clone(); - tokio::task::spawn_blocking(move || { - let rw_transaction = db - .rw_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - for item in batch { - rw_transaction - .insert(item) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - } - rw_transaction - .commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(()) - }) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? +// ─── Filter (replaces secondary key queries) ────────────────────────────── + +pub fn filter_impl(db: &MemDb, predicate: F) -> BichonResult> +where + M: MemDbModel, + F: Fn(&M) -> bool + Send + 'static, +{ + let coll = db.collection(M::collection()); + coll.filter(predicate) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError)) } -pub async fn batch_upsert_impl( - database: &Arc>, - batch: Vec, -) -> BichonResult<()> { - let db = database.clone(); - tokio::task::spawn_blocking(move || { - let rw_transaction = db - .rw_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - for item in batch { - rw_transaction - .upsert(item) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - } - rw_transaction - .commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(()) - }) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? -} +// ─── Update (read-modify-write under a single spawn_blocking) ───────────── -pub async fn upsert_impl( - database: &Arc>, - item: T, -) -> BichonResult<()> { - let db = database.clone(); - tokio::task::spawn_blocking(move || { - let rw_transaction = db - .rw_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - rw_transaction - .upsert(item) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - rw_transaction - .commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(()) - }) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? -} - -pub async fn update_impl( - database: &Arc>, - current: impl FnOnce(&RwTransaction) -> BichonResult + Send + 'static, - updated: impl FnOnce(&T) -> BichonResult + Send + 'static, -) -> BichonResult { - let db = database.clone(); - tokio::task::spawn_blocking(move || { - let rw = db - .rw_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let current_item = current(&rw)?; - let updated_item = updated(¤t_item)?; - rw.update(current_item, updated_item.clone()) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - rw.commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(updated_item) - }) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? -} - -pub async fn async_find_impl( - database: &Arc>, - key: impl ToKey + Send + 'static, -) -> BichonResult> { - let db = database.clone(); - tokio::task::spawn_blocking(move || { - let r_transaction = db - .r_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let entity: Option = r_transaction - .get() - .primary(key) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(entity) - }) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? -} - -pub fn find_impl( - database: &Arc>, - key: impl ToKey + Send + 'static, -) -> BichonResult> { - let db = database.clone(); - let r_transaction = db - .r_transaction() +pub fn update_impl( + db: &MemDb, + key: &str, + update_fn: impl FnOnce(M) -> BichonResult + Send + 'static, +) -> BichonResult { + let coll = db.collection(M::collection()); + let current: M = coll + .get_required(key) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let entity: Option = r_transaction - .get() - .primary(key) + let updated = update_fn(current)?; + coll.upsert(key, &updated) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(entity) + Ok(updated) } -pub async fn delete_impl( - database: &Arc>, - delete: impl FnOnce(&RwTransaction) -> BichonResult + Send + 'static, -) -> BichonResult<()> { - let db = database.clone(); - tokio::task::spawn_blocking(move || { - let rw_transaction = db - .rw_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let to_delete = delete(&rw_transaction)?; - rw_transaction - .remove::(to_delete) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - rw_transaction - .commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(()) - }) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? +// ─── Delete ──────────────────────────────────────────────────────────────── + +pub fn delete_impl(db: &MemDb, key: &str) -> BichonResult<()> { + let coll = db.collection(M::collection()); + let existed = coll + .delete(key) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + if !existed { + return Err(raise_error!( + format!("{} '{}' not found for deletion", M::collection(), key), + ErrorCode::ResourceNotFound + )); + } + Ok(()) } -pub async fn batch_delete_impl( - database: &Arc>, - delete: impl FnOnce(&RwTransaction) -> BichonResult> + Send + 'static, -) -> BichonResult { - let db = database.clone(); - tokio::task::spawn_blocking(move || { - let rw_transaction = db - .rw_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let to_delete = delete(&rw_transaction)?; - let delete_count = to_delete.len(); - for item in to_delete { - rw_transaction - .remove(item) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - } - rw_transaction - .commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(delete_count) - }) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? +pub fn batch_delete_impl(db: &MemDb, keys: Vec) -> BichonResult { + let txn = db.transaction(); + let mut txn = txn; + let mut count = 0usize; + for key in &keys { + txn = txn.delete(M::collection(), key.clone()); + count += 1; + } + txn.commit() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(count) } -pub async fn list_all_impl( - database: &Arc>, -) -> BichonResult> { - let db = database.clone(); - tokio::task::spawn_blocking(move || { - let r_transaction = db - .r_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let entities: Vec = r_transaction - .scan() - .primary() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .all() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .try_collect() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(entities) - }) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? +// ─── List / Count ────────────────────────────────────────────────────────── + +pub fn list_all_impl(db: &MemDb) -> BichonResult> { + let coll = db.collection(M::collection()); + coll.list_all() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError)) } -pub async fn with_transaction( - database: &Arc>, - f: impl FnOnce(&RwTransaction) -> BichonResult<()> + Send + 'static, -) -> BichonResult<()> { - let db: Arc> = database.clone(); - tokio::task::spawn_blocking(move || { - let rw_transaction = db - .rw_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - f(&rw_transaction)?; - rw_transaction - .commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(()) - }) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? +pub fn count_impl(db: &MemDb) -> BichonResult { + let coll = db.collection(M::collection()); + Ok(coll.count()) } -// For tables with a creation timestamp, place the creation time at the front of the primary key. -// This allows sorting by time, as the data is stored in dictionary order based on the primary key. -// If reverse sorting by time is needed, the iterator can be reversed. -pub async fn paginate_query_primary_scan_all_impl< - T: ToInput + Serialize + std::fmt::Debug + std::marker::Unpin + Send + Sync + 'static, ->( - database: &Arc>, +// ─── Paginate ────────────────────────────────────────────────────────────── + +pub fn paginate_impl( + db: &MemDb, page: Option, page_size: Option, desc: Option, -) -> BichonResult> { - let db = database.clone(); +) -> BichonResult> { + let coll = db.collection(M::collection()); + let total_items = coll.count() as u64; - tokio::task::spawn_blocking(move || { - let r_transaction = db - .r_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let total_items = r_transaction - .len() - .primary::() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - // Validate page and page_size - let (offset, total_pages) = if let (Some(p), Some(s)) = (page, page_size) { - if p == 0 || s == 0 { - return Err(raise_error!( - "'page' and 'page_size' must be greater than 0.".into(), - ErrorCode::InvalidParameter - )); - } + let (offset, total_pages) = match (page, page_size) { + (Some(p), Some(s)) if p > 0 && s > 0 => { let offset = (p - 1) * s; let total_pages = if total_items > 0 { (total_items as f64 / s as f64).ceil() as u64 @@ -349,158 +174,52 @@ pub async fn paginate_query_primary_scan_all_impl< 0 }; (Some(offset), Some(total_pages)) - } else { - (None, None) - }; - - // Handle empty result early - if let Some(offset) = offset { - if offset >= total_items { - return Ok(Paginated::new( - page, - page_size, - total_items, - total_pages, - vec![], - )); - } } + (Some(0), _) | (_, Some(0)) => { + return Err(raise_error!( + "'page' and 'page_size' must be greater than 0.".into(), + ErrorCode::InvalidParameter + )); + } + _ => (None, None), + }; - let scan = r_transaction - .scan() - .primary() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let iter = scan - .all() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - // Collect items based on the reverse flag and pagination - let items: Vec = match desc { - Some(true) => iter - .rev() - .skip(offset.unwrap_or(0) as usize) - .take(page_size.unwrap_or(total_items) as usize) - .try_collect() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?, - _ => iter - .skip(offset.unwrap_or(0) as usize) - .take(page_size.unwrap_or(total_items) as usize) - .try_collect() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?, - }; - - Ok(Paginated::new( - page, - page_size, - total_items, - total_pages, - items, - )) - }) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? -} - -pub async fn async_filter_by_secondary_key_impl( - database: &Arc>, - key_def: impl ToKeyDefinition + Send + 'static, - start_with: impl ToKey + Send + 'static, -) -> BichonResult> { - let db = database.clone(); - tokio::task::spawn_blocking(move || { - let r_transaction = db - .r_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let entities: Vec = r_transaction - .scan() - .secondary(key_def) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .start_with(start_with) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .try_collect() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(entities) - }) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? -} - -pub fn filter_by_secondary_key_impl( - database: &Arc>, - key_def: impl ToKeyDefinition + Send + 'static, - start_with: impl ToKey + Send + 'static, -) -> BichonResult> { - let db = database.clone(); - let r_transaction = db - .r_transaction() + let all: Vec = coll + .list_all() .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let entities: Vec = r_transaction - .scan() - .secondary(key_def) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .start_with(start_with) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .try_collect() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(entities) + + let items: Vec = match desc { + Some(true) => { + let iter: Vec = all.into_iter().rev().collect(); + let skip = offset.unwrap_or(0) as usize; + let take = page_size.unwrap_or(total_items) as usize; + iter.into_iter().skip(skip).take(take).collect() + } + _ => { + let skip = offset.unwrap_or(0) as usize; + let take = page_size.unwrap_or(total_items) as usize; + all.into_iter().skip(skip).take(take).collect() + } + }; + + Ok(Paginated::new( + page, + page_size, + total_items, + total_pages, + items, + )) } -pub async fn count_by_unique_secondary_key_impl( - database: &Arc>, - key_def: impl ToKeyDefinition + Send + 'static, -) -> BichonResult { - let db = database.clone(); - tokio::task::spawn_blocking(move || { - let r_transaction = db - .r_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let count = r_transaction - .scan() - .secondary::(key_def) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .all() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .count(); - Ok(count) - }) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? -} - -pub async fn async_secondary_find_impl( - database: &Arc>, - key_def: impl ToKeyDefinition + Send + 'static, - key: impl ToKey + Send + 'static, -) -> BichonResult> { - let db = database.clone(); - tokio::task::spawn_blocking(move || { - let r_transaction = db - .r_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let entities: Option = r_transaction - .get() - .secondary(key_def, key) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - Ok(entities) - }) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? -} - -pub fn secondary_find_impl( - database: &Arc>, - key_def: impl ToKeyDefinition + Send + 'static, - key: impl ToKey + Send + 'static, -) -> BichonResult> { - let db = database.clone(); - let r_transaction = db - .r_transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let entities: Option = r_transaction - .get() - .secondary(key_def, key) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(entities) +// ─── Transaction ─────────────────────────────────────────────────────────── + +/// Execute operations within a single atomic transaction (one WAL entry). +pub fn with_transaction( + db: &MemDb, + f: impl FnOnce(Transaction) -> BichonResult + Send + 'static, +) -> BichonResult<()> { + let txn = db.transaction(); + let txn = f(txn)?; + txn.commit() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError)) } diff --git a/crates/core/src/envelope/extractor.rs b/crates/core/src/envelope/extractor.rs index 1ee5416..8024870 100644 --- a/crates/core/src/envelope/extractor.rs +++ b/crates/core/src/envelope/extractor.rs @@ -433,13 +433,13 @@ pub async fn detach_and_store_attachments( attachment_infos } -pub async fn reattach_eml_content( +pub fn reattach_eml_content( account_id: u64, envelope_id: String, ) -> BichonResult<(Envelope, Bytes)> { let e = ENVELOPE_MANAGER .get_envelope_by_id(account_id, &envelope_id) - .await? + ? .ok_or_else(|| { raise_error!( format!( diff --git a/crates/core/src/envelope/utils.rs b/crates/core/src/envelope/utils.rs index 4623d14..4f91d4c 100644 --- a/crates/core/src/envelope/utils.rs +++ b/crates/core/src/envelope/utils.rs @@ -90,43 +90,86 @@ pub fn normalize_subject(raw_subject: Option<&str>) -> String { #[cfg(test)] mod tests { - use crate::envelope::utils::merge_contiguous_encoded_words; + use crate::envelope::utils::{merge_contiguous_encoded_words, normalize_subject}; + // ── merge_contiguous_encoded_words ────────────────────────────── - #[tokio::test] - async fn test3() { + #[test] + fn merge_basic_utf8_b() { let s = "Hello =?UTF-8?B?SGVsbG8=?= =?UTF-8?B?V29ybGQ=?= !!!"; assert_eq!( merge_contiguous_encoded_words(s), "Hello =?UTF-8?B?SGVsbG8=V29ybGQ=?= !!!" ); + } + #[test] + fn merge_three_blocks() { let s = "=?UTF-8?B?QQ==?= =?UTF-8?B?Qg==?= =?UTF-8?B?Qw==?="; assert_eq!( merge_contiguous_encoded_words(s), "=?UTF-8?B?QQ==Qg==Qw==?=" ); + } + #[test] + fn merge_noncontiguous_blocks() { let s = "=?UTF-8?B?QQ==?= =?UTF-8?B?Qg==?= test =?UTF-8?B?Qw==?= =?UTF-8?B?RA==?="; assert_eq!( merge_contiguous_encoded_words(s), "=?UTF-8?B?QQ==Qg==?= test =?UTF-8?B?Qw==RA==?=" ); + } + #[test] + fn reject_different_charsets() { let s = "=?UTF-8?B?QQ==?= =?GBK?B?Qg==?="; assert_eq!(merge_contiguous_encoded_words(s), s); + } + + #[test] + fn reject_different_encodings() { let s = "=?UTF-8?B?QQ==?= =?UTF-8?Q?Qg?="; assert_eq!(merge_contiguous_encoded_words(s), s); + } + #[test] + fn merge_case_insensitive_encoding() { let s = "=?UTF-8?b?QQ==?= =?UTF-8?B?Qg==?="; assert_eq!(merge_contiguous_encoded_words(s), "=?UTF-8?B?QQ==Qg==?="); + } + + #[test] + fn single_encoded_word_unchanged() { let s = "Hello =?UTF-8?B?SGVsbG8=?= !!!"; assert_eq!(merge_contiguous_encoded_words(s), s); + } + + #[test] + fn multiple_spaces_between_words() { let s = "=?UTF-8?B?QQ==?= =?UTF-8?B?Qg==?="; assert_eq!(merge_contiguous_encoded_words(s), "=?UTF-8?B?QQ==Qg==?="); + } + + #[test] + fn plain_subject_line() { let s = "Just a normal subject line"; assert_eq!(merge_contiguous_encoded_words(s), s); + } + + #[test] + fn merge_quoted_printable() { let s = "=?UTF-8?Q?Hello_?= =?UTF-8?Q?World?="; - assert_eq!(merge_contiguous_encoded_words(s), "=?UTF-8?Q?Hello_World?="); + assert_eq!( + merge_contiguous_encoded_words(s), + "=?UTF-8?Q?Hello_World?=" + ); + } + + // ── normalize_subject ─────────────────────────────────────────── + + #[test] + fn normalize_subject_none() { + assert_eq!(normalize_subject(None), ""); } } diff --git a/crates/core/src/imap/executor.rs b/crates/core/src/imap/executor.rs index 5ad2ef0..3d6fe11 100644 --- a/crates/core/src/imap/executor.rs +++ b/crates/core/src/imap/executor.rs @@ -18,13 +18,13 @@ use crate::account::migration::AccountModel; use crate::account::state::{DownloadState, FolderStatus}; -use crate::cache::imap::mailbox::MailBox; use crate::cache::imap::download::flow::{generate_uid_sequence_hashset, DEFAULT_BATCH_SIZE}; +use crate::cache::imap::mailbox::MailBox; use crate::envelope::extractor::extract_envelope_and_store_it; use crate::error::code::ErrorCode; use crate::imap::session::SessionStream; -use crate::{error::BichonResult, imap::manager::ImapConnectionManager}; use crate::raise_error; +use crate::{error::BichonResult, imap::manager::ImapConnectionManager}; use async_imap::types::Name; use async_imap::Session; use futures::TryStreamExt; @@ -106,9 +106,8 @@ impl ImapExecutor { 0, FolderStatus::Failed, Some(err_msg.clone()), - ) - .await?; - DownloadState::append_session_error(account.id, err_msg).await?; + )?; + DownloadState::append_session_error(account.id, err_msg)?; return Err(e); } }; @@ -126,8 +125,7 @@ impl ImapExecutor { 0, FolderStatus::Success, Some(msg), - ) - .await?; + )?; return Ok(()); } info!( @@ -166,12 +164,11 @@ impl ImapExecutor { current_processed, FolderStatus::Downloading, None, - ) - .await?; + )?; } Err(e) => { let err_msg = format!("Batch {} failed: {:#?}", index, e); - DownloadState::append_session_error(account.id, err_msg.clone()).await?; + DownloadState::append_session_error(account.id, err_msg.clone())?; DownloadState::update_folder_progress( account.id, mailbox.name.clone(), @@ -179,8 +176,7 @@ impl ImapExecutor { current_processed, FolderStatus::Failed, Some(err_msg), - ) - .await?; + )?; has_error_or_cancel = true; break; } @@ -195,8 +191,7 @@ impl ImapExecutor { current_processed, FolderStatus::Success, None, - ) - .await?; + )?; } Ok(()) diff --git a/crates/core/src/imap/manager.rs b/crates/core/src/imap/manager.rs index ac2f1cb..d32f517 100644 --- a/crates/core/src/imap/manager.rs +++ b/crates/core/src/imap/manager.rs @@ -20,9 +20,7 @@ use crate::account::entity::AuthType; use crate::account::migration::{AccountModel, AccountType}; use crate::error::code::ErrorCode; use crate::error::BichonResult; -use crate::imap::capabilities::{ - capability_to_string, check_capabilities, fetch_capabilities, -}; +use crate::imap::capabilities::{capability_to_string, check_capabilities, fetch_capabilities}; use crate::imap::client::Client; use crate::imap::oauth2::OAuth2; use crate::imap::session::SessionStream; @@ -73,7 +71,7 @@ impl ImapConnectionManager { }) } AuthType::OAuth2 => { - let record = OAuth2AccessToken::get(account.id).await?; + let record = OAuth2AccessToken::get(account.id)?; let access_token = record.and_then(|r| r.access_token).ok_or_else(|| { raise_error!( "Imap auth type is OAuth2, but OAuth2 authorization is not yet complete." @@ -85,7 +83,10 @@ impl ImapConnectionManager { .authenticate(OAuth2::new(login_name.clone(), access_token)) .await .map_err(|e| { - error!("IMAP OAuth2 auth failed for username '{}': {}", login_name, e); + error!( + "IMAP OAuth2 auth failed for username '{}': {}", + login_name, e + ); e }) } @@ -93,7 +94,7 @@ impl ImapConnectionManager { } pub async fn build(account_id: u64) -> BichonResult>> { - let account = AccountModel::async_get(account_id).await?; + let account = AccountModel::get(account_id)?; let client = match Self::create_client(&account).await { Ok(client) => client, Err(error) => { @@ -116,7 +117,7 @@ impl ImapConnectionManager { match fetch_capabilities(&mut session).await { Ok(capabilities) => { let to_save: Vec = capabilities.iter().map(capability_to_string).collect(); - AccountModel::update_capabilities(account_id, to_save).await?; + AccountModel::update_capabilities(account_id, to_save)?; if let Err(error) = check_capabilities(&capabilities) { error!("Failed to check IMAP capabilities: {:#?}", error); return Err(error); diff --git a/crates/core/src/import/mod.rs b/crates/core/src/import/mod.rs index 2f74a9d..0a6e897 100644 --- a/crates/core/src/import/mod.rs +++ b/crates/core/src/import/mod.rs @@ -67,7 +67,7 @@ pub struct ImportEmls; impl ImportEmls { pub async fn do_import(request: BatchEmlRequest) -> BichonResult { - let account = AccountModel::check_account_exists(request.account_id).await?; + let account = AccountModel::check_account_exists(request.account_id)?; if !account.enabled { return Err(raise_error!("The account is disabled and cannot be used for this operation.".into(), ErrorCode::InvalidParameter)); @@ -75,7 +75,7 @@ impl ImportEmls { let mailbox_id = match account.account_type { AccountType::IMAP => { - let all_mailboxes = MailBox::list_all(account.id).await?; + let all_mailboxes = MailBox::list_all(account.id)?; let mailbox = all_mailboxes.into_iter().find(|m| m.name == request.mail_folder); match mailbox { @@ -105,7 +105,7 @@ impl ImportEmls { }; let mailbox_id = mailbox.id; // Upsert the mailbox, creating it if it doesn't exist - MailBox::batch_upsert(&[mailbox]).await?; + MailBox::batch_upsert(&[mailbox])?; mailbox_id }, }; diff --git a/crates/core/src/mailbox/delete.rs b/crates/core/src/mailbox/delete.rs index 3bf9160..574323c 100644 --- a/crates/core/src/mailbox/delete.rs +++ b/crates/core/src/mailbox/delete.rs @@ -23,11 +23,11 @@ use crate::{ }; pub async fn delete_mailbox_impl(account_id: u64, mailbox_id: u64) -> BichonResult<()> { - let mailbox = MailBox::async_get(mailbox_id).await?; + let mailbox = MailBox::get(mailbox_id)?; let name = mailbox.name; let delimiter = mailbox.delimiter.unwrap_or("/".to_owned()); - let all_mailboxes = MailBox::list_all(account_id).await?; + let all_mailboxes = MailBox::list_all(account_id)?; let prefix = format!("{}{}", name, delimiter); let ids_to_delete: Vec = all_mailboxes @@ -41,7 +41,7 @@ pub async fn delete_mailbox_impl(account_id: u64, mailbox_id: u64) -> BichonResu } for id in &ids_to_delete { - MailBox::delete(*id).await?; + MailBox::delete(*id)?; } ENVELOPE_MANAGER diff --git a/crates/core/src/mailbox/list.rs b/crates/core/src/mailbox/list.rs index d3226c6..4a84b62 100644 --- a/crates/core/src/mailbox/list.rs +++ b/crates/core/src/mailbox/list.rs @@ -22,13 +22,13 @@ use crate::error::code::ErrorCode; use crate::error::BichonResult; use crate::imap::executor::ImapExecutor; use crate::imap::session::SessionStream; -use crate::utils::create_hash; use crate::raise_error; +use crate::utils::create_hash; use async_imap::types::Name; use async_imap::Session; pub async fn get_account_mailboxes(account_id: u64, remote: bool) -> BichonResult> { - let account = AccountModel::check_account_exists(account_id).await?; + let account = AccountModel::check_account_exists(account_id)?; if remote { if matches!(account.account_type, AccountType::IMAP) { request_imap_all_mailbox_list(account_id).await @@ -39,7 +39,7 @@ pub async fn get_account_mailboxes(account_id: u64, remote: bool) -> BichonResul )); } } else { - MailBox::list_all(account_id).await + MailBox::list_all(account_id) } } diff --git a/crates/core/src/message/append.rs b/crates/core/src/message/append.rs index cd59c81..491f586 100644 --- a/crates/core/src/message/append.rs +++ b/crates/core/src/message/append.rs @@ -1,12 +1,11 @@ use crate::{ - encode_mailbox_name, + encode_mailbox_name, raise_error, { account::migration::{AccountModel, AccountType}, envelope::extractor::reattach_eml_content, error::{code::ErrorCode, BichonResult}, imap::executor::ImapExecutor, }, - raise_error, }; //use poem_openapi::Object; use serde::{Deserialize, Serialize}; @@ -32,7 +31,7 @@ pub async fn restore_emails(account_id: u64, envelope_ids: Vec) -> Bicho )); } - let account = AccountModel::check_account_exists(account_id).await?; + let account = AccountModel::check_account_exists(account_id)?; if !matches!(account.account_type, AccountType::IMAP) { return Err(raise_error!( "Account type is not IMAP".into(), @@ -44,7 +43,7 @@ pub async fn restore_emails(account_id: u64, envelope_ids: Vec) -> Bicho let mut session = ImapExecutor::create_connection(account_id).await?; for envelope_id in envelope_ids { let result: BichonResult<()> = async { - let (envelope, eml) = reattach_eml_content(account_id, envelope_id.clone()).await?; + let (envelope, eml) = reattach_eml_content(account_id, envelope_id.clone())?; if let Some(mailbox_name) = envelope.mailbox_name { ImapExecutor::append( &mut session, diff --git a/crates/core/src/message/attachment.rs b/crates/core/src/message/attachment.rs index 28dd206..c42e5a5 100644 --- a/crates/core/src/message/attachment.rs +++ b/crates/core/src/message/attachment.rs @@ -1,13 +1,13 @@ use std::io::Cursor; use crate::{ + raise_error, { dashboard::Group, envelope::extractor::reattach_eml_content, error::{code::ErrorCode, BichonResult}, utils::compute_content_hash, }, - raise_error, }; use bytes::Bytes; use mail_parser::MessageParser; @@ -33,18 +33,15 @@ pub struct AttachmentMetadata { pub content_types: Vec, } -pub async fn retrieve_attachment_content( +pub fn retrieve_attachment_content( account_id: u64, envelope_id: String, content_hash: &str, ) -> BichonResult> { - let (_, eml) = reattach_eml_content(account_id, envelope_id).await?; - let message = MessageParser::default().parse(&eml).ok_or_else(|| { - raise_error!( - "Failed to parse EML".into(), - ErrorCode::InternalError - ) - })?; + let (_, eml) = reattach_eml_content(account_id, envelope_id)?; + let message = MessageParser::default() + .parse(&eml) + .ok_or_else(|| raise_error!("Failed to parse EML".into(), ErrorCode::InternalError))?; let attachment_content: &[u8] = message .attachments() @@ -59,13 +56,13 @@ pub async fn retrieve_attachment_content( Ok(Cursor::new(Bytes::copy_from_slice(attachment_content))) } -pub async fn retrieve_nested_attachment_content( +pub fn retrieve_nested_attachment_content( account_id: u64, envelope_id: String, content_hash: &str, nested_content_hash: &str, ) -> BichonResult> { - let (_, eml) = reattach_eml_content(account_id, envelope_id).await?; + let (_, eml) = reattach_eml_content(account_id, envelope_id)?; let parent_message = MessageParser::default().parse(&eml).ok_or_else(|| { raise_error!( "Failed to parse parent EML".into(), diff --git a/crates/core/src/message/content.rs b/crates/core/src/message/content.rs index 2c65594..0451959 100644 --- a/crates/core/src/message/content.rs +++ b/crates/core/src/message/content.rs @@ -16,11 +16,9 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use crate::base64_encode; use crate::account::migration::AccountModel; -use crate::envelope::extractor::{ - extract_envelope_from_nested_message, reattach_eml_content, -}; +use crate::base64_encode; +use crate::envelope::extractor::{extract_envelope_from_nested_message, reattach_eml_content}; use crate::error::code::ErrorCode; use crate::store::envelope::Envelope; use crate::utils::compute_content_hash; @@ -159,12 +157,12 @@ pub struct FullNestedMessageContent { pub envelope: Envelope, } -pub async fn retrieve_email_content( +pub fn retrieve_email_content( account_id: u64, envelope_id: String, ) -> BichonResult { - AccountModel::check_account_exists(account_id).await?; - let (envelope, eml) = reattach_eml_content(account_id, envelope_id).await?; + AccountModel::check_account_exists(account_id)?; + let (envelope, eml) = reattach_eml_content(account_id, envelope_id)?; let message = MessageParser::default().parse(&eml).ok_or_else(|| { raise_error!( "Failed to parse EML data — the message may be corrupted.".into(), @@ -232,12 +230,12 @@ pub async fn retrieve_email_content( }) } -pub async fn retrieve_nested_eml_content( +pub fn retrieve_nested_eml_content( account_id: u64, envelope_id: String, content_hash: &str, ) -> BichonResult { - let (_, eml) = reattach_eml_content(account_id, envelope_id).await?; + let (_, eml) = reattach_eml_content(account_id, envelope_id)?; let parent_message = MessageParser::default().parse(&eml).ok_or_else(|| { raise_error!( "Failed to parse parent EML".into(), diff --git a/crates/core/src/message/list.rs b/crates/core/src/message/list.rs index 47c4f04..6b4cf5f 100644 --- a/crates/core/src/message/list.rs +++ b/crates/core/src/message/list.rs @@ -17,17 +17,18 @@ // along with this program. If not, see . use crate::{ - account::migration::AccountModel, common::paginated::DataPage, error::BichonResult, store::{envelope::Envelope, tantivy::envelope::ENVELOPE_MANAGER} + account::migration::AccountModel, + common::paginated::DataPage, + error::BichonResult, + store::{envelope::Envelope, tantivy::envelope::ENVELOPE_MANAGER}, }; -pub async fn get_thread_messages( +pub fn get_thread_messages( account_id: u64, thread_id: &str, page: u64, page_size: u64, ) -> BichonResult> { - AccountModel::check_account_exists(account_id).await?; - ENVELOPE_MANAGER - .list_thread_envelopes(account_id, thread_id, page, page_size, true) - .await + AccountModel::check_account_exists(account_id)?; + ENVELOPE_MANAGER.list_thread_envelopes(account_id, thread_id, page, page_size, true) } diff --git a/crates/core/src/message/search.rs b/crates/core/src/message/search.rs index 40afdf5..00d6dde 100644 --- a/crates/core/src/message/search.rs +++ b/crates/core/src/message/search.rs @@ -94,21 +94,19 @@ impl EmailSearchRequest { } } -pub async fn search_messages_impl( +pub fn search_messages_impl( accounts: Option>, request: EmailSearchRequest, ) -> BichonResult> { request.validate()?; - ENVELOPE_MANAGER - .search( - accounts, - request.filter, - request.page, - request.page_size, - request.desc.unwrap_or(true), - request.sort_by.unwrap_or(SortBy::DATE), - ) - .await + ENVELOPE_MANAGER.search( + accounts, + request.filter, + request.page, + request.page_size, + request.desc.unwrap_or(true), + request.sort_by.unwrap_or(SortBy::DATE), + ) } #[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)] @@ -169,19 +167,17 @@ impl AttachmentSearchRequest { } } -pub async fn search_attachment_impl( +pub fn search_attachment_impl( accounts: Option>, request: AttachmentSearchRequest, ) -> BichonResult> { request.validate()?; - ATTACHMENT_MANAGER - .search( - accounts, - request.filter, - request.page, - request.page_size, - request.desc.unwrap_or(true), - request.sort_by.unwrap_or(SortBy::DATE), - ) - .await + ATTACHMENT_MANAGER.search( + accounts, + request.filter, + request.page, + request.page_size, + request.desc.unwrap_or(true), + request.sort_by.unwrap_or(SortBy::DATE), + ) } diff --git a/crates/core/src/oauth2/entity.rs b/crates/core/src/oauth2/entity.rs index 2b689d5..9b13a8c 100644 --- a/crates/core/src/oauth2/entity.rs +++ b/crates/core/src/oauth2/entity.rs @@ -19,27 +19,21 @@ use crate::{ common::paginated::DataPage, database::{ - async_secondary_find_impl, delete_impl, insert_impl, manager::DB_MANAGER, - paginate_query_primary_scan_all_impl, update_impl, + delete_impl, find_impl, insert_impl, manager::DB_MANAGER, paginate_impl, update_impl, + MemDbModel, }, encrypt, - error::{code::ErrorCode, BichonResult}, - id, raise_error, utc_now, + error::BichonResult, + id, utc_now, }; -use native_db::*; -use native_model::{native_model, Model}; -//use poem_openapi::Object; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; /// Represents the OAuth2 configuration for a client, including initialization and runtime values. #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] -#[native_model(id = 5, version = 1)] -#[native_db(primary_key(pk -> String))] pub struct OAuth2 { /// A unique identifier for the OAuth2 configuration. - #[secondary_key(unique)] pub id: u64, /// A description of what this configuration is used for. pub description: Option, @@ -71,11 +65,16 @@ pub struct OAuth2 { pub updated_at: i64, } -impl OAuth2 { - fn pk(&self) -> String { - format!("{}_{}", &self.created_at, &self.id) +impl MemDbModel for OAuth2 { + fn collection() -> &'static str { + "oauth2_configs" } + fn key(&self) -> String { + self.id.to_string() + } +} +impl OAuth2 { pub fn new(request: OAuth2CreateRequest) -> BichonResult { let request = request.encrypt()?; Ok(OAuth2 { @@ -116,59 +115,33 @@ impl OAuth2 { self.extra_params = None; } - pub async fn save(&self) -> BichonResult<()> { - insert_impl(DB_MANAGER.meta_db(), self.to_owned()).await?; + pub fn save(&self) -> BichonResult<()> { + insert_impl(DB_MANAGER.db(), self.to_owned())?; Ok(()) } - pub async fn paginate_list( + pub fn paginate_list( page: Option, page_size: Option, desc: Option, ) -> BichonResult> { - paginate_query_primary_scan_all_impl(DB_MANAGER.meta_db(), page, page_size, desc) - .await - .map(DataPage::from) + let paginated = paginate_impl::(DB_MANAGER.db(), page, page_size, desc)?; + Ok(DataPage::from(paginated)) } - pub async fn get(id: u64) -> BichonResult> { - async_secondary_find_impl(DB_MANAGER.meta_db(), OAuth2Key::id, id).await + pub fn get(id: u64) -> BichonResult> { + let results = find_impl::(DB_MANAGER.db(), &id.to_string())?; + Ok(results.into_iter().next()) } - pub async fn delete(id: u64) -> BichonResult<()> { - delete_impl(DB_MANAGER.meta_db(), move |rw| { - rw.get() - .secondary::(OAuth2Key::id, id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| { - raise_error!( - format!( - "The oauth2 entity with id={id} that you want to delete was not found." - ), - ErrorCode::ResourceNotFound - ) - }) - }) - .await + pub fn delete(id: u64) -> BichonResult<()> { + delete_impl::(DB_MANAGER.db(), &id.to_string()) } - pub async fn update(id: u64, request: OAuth2UpdateRequest) -> BichonResult<()> { - update_impl( - DB_MANAGER.meta_db(), - move |rw| { - rw.get() - .secondary::(OAuth2Key::id, id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| { - raise_error!( - format!("The oauth2 entity with id={id} that you want to modify was not found."), - ErrorCode::ResourceNotFound - ) - }) - }, - |current| apply_update(current, request), - ) - .await?; + pub fn update(id: u64, request: OAuth2UpdateRequest) -> BichonResult<()> { + update_impl(DB_MANAGER.db(), &id.to_string(), |current| { + apply_update(¤t, request) + })?; Ok(()) } diff --git a/crates/core/src/oauth2/flow.rs b/crates/core/src/oauth2/flow.rs index 9bc92c3..72f74a2 100644 --- a/crates/core/src/oauth2/flow.rs +++ b/crates/core/src/oauth2/flow.rs @@ -16,12 +16,9 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . - use crate::error::code::ErrorCode; use crate::error::BichonResult; -use crate::oauth2::{ - entity::OAuth2, pending::OAuth2PendingEntity, token::OAuth2AccessToken, -}; +use crate::oauth2::{entity::OAuth2, pending::OAuth2PendingEntity, token::OAuth2AccessToken}; use crate::settings::proxy::Proxy; use crate::{decrypt, encrypt, raise_error}; use oauth2::{ @@ -65,9 +62,9 @@ impl OAuth2Flow { Self { oauth2_id } } - pub async fn authorize_url(&self, account_id: u64) -> BichonResult { + pub fn authorize_url(&self, account_id: u64) -> BichonResult { // Fetch OAuth2 entity or return a custom error if not found - let entity = self.fetch_oauth2_entity().await?; + let entity = self.fetch_oauth2_entity()?; if !entity.enabled { return Err(raise_error!( @@ -107,8 +104,7 @@ impl OAuth2Flow { account_id, csrf_state.secret(), pkce_code_verifier.secret(), - ) - .await?; + )?; // Return the authorization URL Ok(authorize_url.to_string()) } @@ -119,9 +115,9 @@ impl OAuth2Flow { code_verifier: &str, code: &str, ) -> BichonResult<()> { - let entity = self.fetch_oauth2_entity().await?; + let entity = self.fetch_oauth2_entity()?; let client = self.build_oauth2_client(&entity)?; - let http_client = build_http_client(entity.use_proxy).await?; + let http_client = build_http_client(entity.use_proxy)?; let token_response = client .exchange_code(AuthorizationCode::new(code.to_owned())) @@ -142,13 +138,11 @@ impl OAuth2Flow { .secret() .to_owned(); - self.save_oauth2_entity(account_id, access_token, refresh_token) - .await?; - + self.save_oauth2_entity(account_id, access_token, refresh_token)?; Ok(()) } - async fn save_oauth2_entity( + fn save_oauth2_entity( &self, account_id: u64, access_token: String, @@ -156,10 +150,10 @@ impl OAuth2Flow { ) -> BichonResult<()> { let token = OAuth2AccessToken::create(account_id, self.oauth2_id, access_token, refresh_token)?; - token.save_or_update().await + token.save_or_update() } - async fn update_oauth2_entity( + fn update_oauth2_entity( &self, account_id: u64, access_token: String, @@ -170,20 +164,19 @@ impl OAuth2Flow { encrypt!(&access_token)?, encrypt!(&refresh_token)?, ) - .await } pub async fn refresh_access_token(&self, token: &OAuth2AccessToken) -> BichonResult<()> { - let entity = self.fetch_oauth2_entity().await?; + let entity = self.fetch_oauth2_entity()?; if !entity.enabled { - OAuth2AccessToken::delete_by_oauth2_id(token.oauth2_id).await?; + OAuth2AccessToken::delete_by_oauth2_id(token.oauth2_id)?; return Err(raise_error!( "OAuth2 authentication is disabled for this client".into(), ErrorCode::OAuth2ItemDisabled )); } let client = self.build_oauth2_client(&entity)?; - let http_client = build_http_client(entity.use_proxy).await?; + let http_client = build_http_client(entity.use_proxy)?; let refresh_token = token.refresh_token.clone().ok_or_else(|| { raise_error!( @@ -218,15 +211,13 @@ impl OAuth2Flow { .refresh_token() .map(|r| r.secret().to_owned()) .unwrap_or_else(|| refresh_token.clone()); - self.update_oauth2_entity(token.account_id, access_token, new_refresh_token) - .await?; - + self.update_oauth2_entity(token.account_id, access_token, new_refresh_token)?; Ok(()) } // Helper function to fetch the OAuth2 entity - async fn fetch_oauth2_entity(&self) -> BichonResult { - OAuth2::get(self.oauth2_id).await?.ok_or_else(|| { + fn fetch_oauth2_entity(&self) -> BichonResult { + OAuth2::get(self.oauth2_id)?.ok_or_else(|| { raise_error!( format!("OAuth2 entity with id '{}' not found", self.oauth2_id), ErrorCode::ResourceNotFound @@ -254,7 +245,7 @@ impl OAuth2Flow { } // Helper function to save the pending OAuth2 state - async fn save_pending_oauth2_state( + fn save_pending_oauth2_state( &self, account_id: u64, csrf_state: &str, @@ -267,14 +258,13 @@ impl OAuth2Flow { pkce_code_verifier.to_owned(), ) .save() - .await } } // Helper function to build the HTTP client -async fn build_http_client(use_proxy: Option) -> BichonResult { +fn build_http_client(use_proxy: Option) -> BichonResult { if let Some(proxy_id) = use_proxy { - let proxy = Proxy::get(proxy_id).await?; + let proxy = Proxy::get(proxy_id)?; return oauth2::reqwest::ClientBuilder::new() .redirect(oauth2::reqwest::redirect::Policy::none()) .proxy(reqwest::Proxy::all(&proxy.url).map_err(|e| { diff --git a/crates/core/src/oauth2/pending.rs b/crates/core/src/oauth2/pending.rs index 2473d4b..4b17344 100644 --- a/crates/core/src/oauth2/pending.rs +++ b/crates/core/src/oauth2/pending.rs @@ -16,33 +16,25 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . - use crate::{ - { - database::{ - batch_delete_impl, delete_impl, async_find_impl, insert_impl, manager::DB_MANAGER, - }, - error::{code::ErrorCode, BichonResult}, + database::{ + batch_delete_impl, delete_impl, find_impl, insert_impl, list_all_impl, manager::DB_MANAGER, + MemDbModel, }, - raise_error, utc_now, + error::BichonResult, + utc_now, }; -use itertools::Itertools; -use native_db::*; -use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; const EXPIRATION_DURATION_MS: i64 = 24 * 60 * 60 * 1000; #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] -#[native_model(id = 6, version = 1)] -#[native_db] pub struct OAuth2PendingEntity { /// Unique identifier for the OAuth2 request record pub oauth2_id: u64, pub account_id: u64, /// CSRF protection state parameter used to verify the integrity of the authorization request - #[primary_key] pub state: String, /// PKCE code verifier used in the authorization code exchange process to ensure security @@ -52,13 +44,17 @@ pub struct OAuth2PendingEntity { pub created_at: i64, } +impl MemDbModel for OAuth2PendingEntity { + fn collection() -> &'static str { + "oauth2_pending" + } + fn key(&self) -> String { + self.state.clone() + } +} + impl OAuth2PendingEntity { - pub fn new( - oauth2_id: u64, - account_id: u64, - state: String, - code_verifier: String, - ) -> Self { + pub fn new(oauth2_id: u64, account_id: u64, state: String, code_verifier: String) -> Self { Self { oauth2_id, account_id, @@ -68,66 +64,35 @@ impl OAuth2PendingEntity { } } - pub async fn save(&self) -> BichonResult<()> { - insert_impl(DB_MANAGER.meta_db(), self.to_owned()).await + pub fn save(&self) -> BichonResult<()> { + insert_impl(DB_MANAGER.db(), self.to_owned()) } - pub async fn delete(state: &str) -> BichonResult<()> { - let state = state.to_string(); - delete_impl(DB_MANAGER.meta_db(), move |rw| { - rw.get().primary::(state.clone()) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| raise_error!(format!( - "The oauth2 pending entity with state={state} that you want to delete was not found." - ), ErrorCode::ResourceNotFound)) - }).await + pub fn delete(state: &str) -> BichonResult<()> { + delete_impl::(DB_MANAGER.db(), state) } - pub async fn clean() -> BichonResult<()> { - batch_delete_impl(DB_MANAGER.meta_db(), |rw| { - let all: Vec = rw - .scan() - .primary() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .all() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .try_collect() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let now = utc_now!(); - let to_delete: Vec = all - .into_iter() - .filter(|e| now - e.created_at > EXPIRATION_DURATION_MS) - .collect(); - Ok(to_delete) - }) - .await?; + pub fn clean() -> BichonResult<()> { + let all = list_all_impl::(DB_MANAGER.db())?; + let now = utc_now!(); + let to_delete: Vec = all + .into_iter() + .filter(|e| now - e.created_at > EXPIRATION_DURATION_MS) + .map(|e| e.state) + .collect(); + if !to_delete.is_empty() { + batch_delete_impl::(DB_MANAGER.db(), to_delete)?; + } Ok(()) } - pub async fn get(state: &str) -> BichonResult> { - let entity = - async_find_impl::(DB_MANAGER.meta_db(), state.to_string()) - .await?; + pub fn get(state: &str) -> BichonResult> { + let entity = find_impl::(DB_MANAGER.db(), state)?; match entity { Some(entity) => { - let state = state.to_string(); if utc_now!() - entity.created_at > EXPIRATION_DURATION_MS { - delete_impl(DB_MANAGER.meta_db(), move |rw| { - rw.get() - .primary::(state) - .map_err(|e| { - raise_error!(format!("{:#?}", e), ErrorCode::InternalError) - })? - .ok_or_else(|| { - raise_error!( - "OAuth2 pending entity not found".into(), - ErrorCode::ResourceNotFound - ) - }) - }) - .await?; + delete_impl::(DB_MANAGER.db(), state)?; return Ok(None); } Ok(Some(entity)) diff --git a/crates/core/src/oauth2/refresh/mod.rs b/crates/core/src/oauth2/refresh/mod.rs index 8051281..c1624eb 100644 --- a/crates/core/src/oauth2/refresh/mod.rs +++ b/crates/core/src/oauth2/refresh/mod.rs @@ -38,7 +38,7 @@ impl BichonTask for OAuth2RefreshTask { debug!("Starting OAuth2 token refresh task"); // Try to retrieve all OAuth2 access tokens - match OAuth2AccessToken::list_all().await { + match OAuth2AccessToken::list_all() { Ok(all_tokens) => { let need_refresh: Vec = all_tokens .into_iter() diff --git a/crates/core/src/oauth2/task.rs b/crates/core/src/oauth2/task.rs index 1142e90..582f0df 100644 --- a/crates/core/src/oauth2/task.rs +++ b/crates/core/src/oauth2/task.rs @@ -34,7 +34,7 @@ impl BichonTask for OAuth2CleanTask { let task = move |_: Option| { Box::pin(async move { - OAuth2PendingEntity::clean().await?; + OAuth2PendingEntity::clean()?; Ok(()) }) }; diff --git a/crates/core/src/oauth2/token.rs b/crates/core/src/oauth2/token.rs index 237657e..2168be4 100644 --- a/crates/core/src/oauth2/token.rs +++ b/crates/core/src/oauth2/token.rs @@ -16,36 +16,26 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . - use crate::{ - decrypt, encrypt, - { - database::{ - async_find_impl, delete_impl, insert_impl, list_all_impl, manager::DB_MANAGER, - update_impl, upsert_impl, - }, - error::{code::ErrorCode, BichonResult}, - oauth2::entity::OAuth2, + database::{ + delete_impl, filter_impl, find_impl, insert_impl, list_all_impl, manager::DB_MANAGER, + update_impl, upsert_impl, MemDbModel, }, + decrypt, encrypt, + error::{code::ErrorCode, BichonResult}, + oauth2::entity::OAuth2, raise_error, utc_now, }; -use native_db::*; -use native_model::{native_model, Model}; -//use poem_openapi::Object; use serde::{Deserialize, Serialize}; pub const EXTERNAL_OAUTH_APP_ID: u64 = 0; #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] -#[native_model(id = 7, version = 1)] -#[native_db] pub struct OAuth2AccessToken { /// The ID of the account associated with this access token. - #[primary_key] pub account_id: u64, /// The id of the OAuth2 configuration associated with this access token. - #[secondary_key] pub oauth2_id: u64, /// The OAuth2 access token used to authenticate requests to the provider. pub access_token: Option, @@ -57,6 +47,15 @@ pub struct OAuth2AccessToken { pub updated_at: i64, } +impl MemDbModel for OAuth2AccessToken { + fn collection() -> &'static str { + "oauth2_tokens" + } + fn key(&self) -> String { + self.account_id.to_string() + } +} + impl OAuth2AccessToken { pub fn create( account_id: u64, @@ -74,14 +73,14 @@ impl OAuth2AccessToken { }) } - pub async fn upsert_external_oauth_token( + pub fn upsert_external_oauth_token( account_id: u64, request: ExternalOAuth2Request, ) -> BichonResult<()> { let now = utc_now!(); - request.validate().await?; + request.validate()?; - let current = Self::get(account_id).await?; + let current = Self::get(account_id)?; match current { Some(mut current) => { // Update existing record @@ -96,7 +95,7 @@ impl OAuth2AccessToken { } current.updated_at = now; - upsert_impl(DB_MANAGER.meta_db(), current).await?; + upsert_impl(DB_MANAGER.db(), current)?; } None => { // Insert new record @@ -116,20 +115,19 @@ impl OAuth2AccessToken { created_at: now, updated_at: now, }; - insert_impl(DB_MANAGER.meta_db(), entity).await?; + insert_impl(DB_MANAGER.db(), entity)?; } } Ok(()) } // This function may be called multiple times for one account, so we use upsert. - pub async fn save_or_update(&self) -> BichonResult<()> { - upsert_impl(DB_MANAGER.meta_db(), self.clone()).await + pub fn save_or_update(&self) -> BichonResult<()> { + upsert_impl(DB_MANAGER.db(), self.clone()) } - pub async fn get(account_id: u64) -> BichonResult> { - async_find_impl::(DB_MANAGER.meta_db(), account_id) - .await? + pub fn get(account_id: u64) -> BichonResult> { + find_impl::(DB_MANAGER.db(), &account_id.to_string())? .map(|mut token| { token.access_token = token.access_token.map(|t| decrypt!(&t)).transpose()?; token.refresh_token = token.refresh_token.map(|t| decrypt!(&t)).transpose()?; @@ -138,9 +136,8 @@ impl OAuth2AccessToken { .transpose() } - pub async fn list_all() -> BichonResult> { - list_all_impl::(DB_MANAGER.meta_db()) - .await? + pub fn list_all() -> BichonResult> { + list_all_impl::(DB_MANAGER.db())? .into_iter() .map(|mut token| { token.access_token = token.access_token.map(|t| decrypt!(&t)).transpose()?; @@ -150,48 +147,40 @@ impl OAuth2AccessToken { .collect() } - pub async fn try_delete(account_id: u64) -> BichonResult<()> { - if Self::get(account_id).await?.is_none() { + pub fn try_delete(account_id: u64) -> BichonResult<()> { + if Self::get(account_id)?.is_none() { return Ok(()); } - delete_impl(DB_MANAGER.meta_db(), move |rw|{ - rw.get().primary::(account_id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| raise_error!(format!( - "The oauth2 access token entity with account_id={account_id} that you want to delete was not found." - ),ErrorCode::ResourceNotFound)) - }).await + delete_impl::(DB_MANAGER.db(), &account_id.to_string()) } - pub async fn delete_by_oauth2_id(oauth2_id: u64) -> BichonResult<()> { - delete_impl(DB_MANAGER.meta_db(), move |rw|{ - rw.get().secondary::(OAuth2AccessTokenKey::oauth2_id, oauth2_id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| raise_error!(format!( - "The oauth2 access token entity with oauth2_id={oauth2_id} that you want to delete was not found." - ),ErrorCode::ResourceNotFound)) - }).await + pub fn delete_by_oauth2_id(oauth2_id: u64) -> BichonResult<()> { + let tokens = filter_impl::(DB_MANAGER.db(), move |t| { + t.oauth2_id == oauth2_id + })?; + if let Some(token) = tokens.first() { + delete_impl::(DB_MANAGER.db(), &token.account_id.to_string())?; + } + Ok(()) } - pub async fn set_access_token( + pub fn set_access_token( account_id: u64, access_token: String, refresh_token: String, ) -> BichonResult<()> { - update_impl(DB_MANAGER.meta_db(), move |rw|{ - rw.get().primary::(account_id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| raise_error!(format!( - "The oauth2 access token entity with account_id={account_id} that you want to modify was not found." - ),ErrorCode::ResourceNotFound)) - }, |current| { - let mut updated = current.clone(); - updated.access_token = Some(access_token); - updated.refresh_token = Some(refresh_token); - updated.updated_at = utc_now!(); - Ok(updated) - }).await?; + update_impl( + DB_MANAGER.db(), + &account_id.to_string(), + |current: OAuth2AccessToken| { + let mut updated = current.clone(); + updated.access_token = Some(access_token); + updated.refresh_token = Some(refresh_token); + updated.updated_at = utc_now!(); + Ok(updated) + }, + )?; Ok(()) } } @@ -213,7 +202,7 @@ impl ExternalOAuth2Request { /// Ensures mutual dependency between oauth2_id and refresh_token: /// - If `refresh_token` is provided, `oauth2_id` must also be present. /// - If `oauth2_id` is provided, `refresh_token` must also be present. - pub async fn validate(&self) -> BichonResult<()> { + pub fn validate(&self) -> BichonResult<()> { match (self.oauth2_id.is_some(), self.refresh_token.is_some()) { (true, false) => { return Err(raise_error!( @@ -232,7 +221,7 @@ impl ExternalOAuth2Request { // Validate that oauth2_id exists in the database if provided if let Some(oauth2_id) = self.oauth2_id { - let oauth2 = OAuth2::get(oauth2_id).await?; + let oauth2 = OAuth2::get(oauth2_id)?; if oauth2.is_none() { return Err(raise_error!( format!("OAuth2 configuration with id {} does not exist", oauth2_id), @@ -258,12 +247,12 @@ mod tests { "refresh_token".into(), ) .unwrap(); - token.save_or_update().await.unwrap(); - let token2 = OAuth2AccessToken::get(1000u64).await.unwrap().unwrap(); + token.save_or_update().unwrap(); + let token2 = OAuth2AccessToken::get(1000u64).unwrap().unwrap(); assert_eq!(token2.access_token, Some("access_token".into())); assert_eq!(token2.refresh_token, Some("refresh_token".into())); - let tokens = OAuth2AccessToken::list_all().await.unwrap(); + let tokens = OAuth2AccessToken::list_all().unwrap(); assert_eq!(tokens.len(), 1); let first = tokens.first().unwrap(); diff --git a/crates/core/src/settings/cli.rs b/crates/core/src/settings/cli.rs index e43c9f4..9a6a1c2 100644 --- a/crates/core/src/settings/cli.rs +++ b/crates/core/src/settings/cli.rs @@ -328,7 +328,14 @@ pub struct Settings { impl Settings { pub fn init() -> Self { - let s = Self::parse(); + // `cargo test` passes test-filter names and flags (e.g. --nocapture) + // as extra positional arguments. Try the full argv first; if clap + // rejects it, fall back to parsing with only the binary name so that + // the settings come entirely from environment variables. + let args: Vec = std::env::args().collect(); + let s = Self::try_parse_from(&args).unwrap_or_else(|_| { + Self::parse_from(std::iter::once(args[0].clone())) + }); if s.bichon_encrypt_password.is_none() && s.bichon_encrypt_password_file.is_none() { panic!( "One of --bichon_encrypt_password or --bichon_encrypt_password_file has to be set" diff --git a/crates/core/src/settings/dir.rs b/crates/core/src/settings/dir.rs index f973154..0578117 100644 --- a/crates/core/src/settings/dir.rs +++ b/crates/core/src/settings/dir.rs @@ -25,8 +25,7 @@ use crate::{ use std::path::PathBuf; use std::sync::LazyLock; -pub const META_FILE: &str = "meta.db"; -pub const MAILBOX_FILE: &str = "mailbox.db"; +const MEMDB_DIR: &str = "memdb"; const INDICES: &str = "bichon-indices"; const MAIL_METADATA: &str = "mail_metadata"; const ATTACHMENT_METADATA: &str = "attachment_metadata"; @@ -43,8 +42,7 @@ pub static DATA_DIR_MANAGER: LazyLock = #[derive(Debug)] pub struct DataDirManager { pub root_dir: PathBuf, - pub meta_db: PathBuf, - pub mailbox_db: PathBuf, + pub memdb_dir: PathBuf, pub temp_dir: PathBuf, pub tls_cert: PathBuf, pub tls_key: PathBuf, @@ -84,8 +82,7 @@ impl DataDirManager { Self { root_dir: root_dir.clone(), - meta_db: root_dir.join(META_FILE), - mailbox_db: root_dir.join(MAILBOX_FILE), + memdb_dir: root_dir.join(MEMDB_DIR), tls_key: root_dir.join(TLS_KEY), tls_cert: root_dir.join(TLS_CERT), log_dir: root_dir.join(LOG_DIR), diff --git a/crates/core/src/settings/proxy.rs b/crates/core/src/settings/proxy.rs index 49ee80d..2359240 100644 --- a/crates/core/src/settings/proxy.rs +++ b/crates/core/src/settings/proxy.rs @@ -16,31 +16,23 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use native_db::*; -use native_model::{native_model, Model}; //use poem_openapi::Object; use serde::{Deserialize, Serialize}; use crate::{ - id, - { - database::{ - async_find_impl, delete_impl, insert_impl, list_all_impl, manager::DB_MANAGER, - update_impl, - }, - error::{code::ErrorCode, BichonResult}, - utils::net::parse_proxy_addr, + database::{ + delete_impl, find_impl, insert_impl, list_all_impl, manager::DB_MANAGER, update_impl, + MemDbModel, }, - raise_error, utc_now, + error::{code::ErrorCode, BichonResult}, + id, raise_error, utc_now, + utils::net::parse_proxy_addr, }; #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] -#[native_model(id = 8, version = 1)] -#[native_db] pub struct Proxy { /// The unique identifier for this proxy configuration. - #[primary_key] pub id: u64, /// The proxy URL (e.g., socks5://127.0.0.1:1080) used to route network requests. @@ -53,6 +45,15 @@ pub struct Proxy { pub updated_at: i64, } +impl MemDbModel for Proxy { + fn collection() -> &'static str { + "proxies" + } + fn key(&self) -> String { + self.id.to_string() + } +} + impl Proxy { /// Create a new Proxy instance with the given URL and timestamps. pub fn new(url: String) -> Self { @@ -64,59 +65,37 @@ impl Proxy { } } - pub async fn get(id: u64) -> BichonResult { - async_find_impl(DB_MANAGER.meta_db(), id) - .await? - .ok_or_else(|| { - raise_error!( - format!("Proxy with id={} not found", id), - ErrorCode::ResourceNotFound - ) - }) - } - - pub async fn list_all() -> BichonResult> { - list_all_impl(DB_MANAGER.meta_db()).await - } - - pub async fn delete(id: u64) -> BichonResult<()> { - delete_impl(DB_MANAGER.meta_db(), move |rw| { - rw.get() - .primary::(id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| raise_error!("proxy missing".into(), ErrorCode::InternalError)) + pub fn get(id: u64) -> BichonResult { + let key = id.to_string(); + find_impl::(DB_MANAGER.db(), &key)?.ok_or_else(|| { + raise_error!( + format!("Proxy with id={} not found", id), + ErrorCode::ResourceNotFound + ) }) - .await } - pub async fn update(id: u64, url: String) -> BichonResult<()> { - update_impl( - DB_MANAGER.meta_db(), - move |rw| { - rw.get() - .primary::(id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| { - raise_error!( - format!("Proxy with id={} not found", id), - ErrorCode::ResourceNotFound - ) - }) - }, - move |current| { - let mut updated = current.clone(); - updated.url = url; - updated.updated_at = utc_now!(); - Ok(updated) - }, - ) - .await?; + pub fn list_all() -> BichonResult> { + list_all_impl::(DB_MANAGER.db()) + } + + pub fn delete(id: u64) -> BichonResult<()> { + delete_impl::(DB_MANAGER.db(), &id.to_string()) + } + + pub fn update(id: u64, url: String) -> BichonResult<()> { + update_impl(DB_MANAGER.db(), &id.to_string(), move |current: Proxy| { + let mut updated = current.clone(); + updated.url = url; + updated.updated_at = utc_now!(); + Ok(updated) + })?; Ok(()) } - pub async fn save(&self) -> BichonResult<()> { + pub fn save(&self) -> BichonResult<()> { self.validate()?; - insert_impl(DB_MANAGER.meta_db(), self.to_owned()).await + insert_impl(DB_MANAGER.db(), self.to_owned()) } /// Validate that the URL is a valid SOCKS5 proxy URL. diff --git a/crates/core/src/settings/system.rs b/crates/core/src/settings/system.rs index 115040f..f81d4bd 100644 --- a/crates/core/src/settings/system.rs +++ b/crates/core/src/settings/system.rs @@ -16,15 +16,10 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use native_db::*; -use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] -#[native_model(id = 2, version = 1)] -#[native_db] pub struct SystemSetting { - #[primary_key] pub key: String, pub value: String, pub created_at: i64, diff --git a/crates/core/src/store/blob.rs b/crates/core/src/store/blob.rs index e0f0a03..0fa2128 100644 --- a/crates/core/src/store/blob.rs +++ b/crates/core/src/store/blob.rs @@ -226,7 +226,7 @@ impl BlobManager { } } -pub async fn get_reader(account_id: u64, eid: String) -> BichonResult> { - let (_, data) = reattach_eml_content(account_id, eid).await?; +pub fn get_reader(account_id: u64, eid: String) -> BichonResult> { + let (_, data) = reattach_eml_content(account_id, eid)?; Ok(Cursor::new(data)) } diff --git a/crates/core/src/store/tantivy/attachment.rs b/crates/core/src/store/tantivy/attachment.rs index 29645ff..5129061 100644 --- a/crates/core/src/store/tantivy/attachment.rs +++ b/crates/core/src/store/tantivy/attachment.rs @@ -501,7 +501,7 @@ impl IndexManager { Ok(Box::new(BooleanQuery::new(subqueries))) } - pub async fn get_attachment_by_id( + pub fn get_attachment_by_id( &self, account_id: u64, id: &str, @@ -541,7 +541,7 @@ impl IndexManager { } } - pub async fn top_10_largest_attachments( + pub fn top_10_largest_attachments( &self, accounts: &Option>, ) -> BichonResult> { @@ -688,10 +688,7 @@ impl IndexManager { Ok(()) } - pub async fn get_all_tags( - &self, - accounts: Option>, - ) -> BichonResult> { + pub fn get_all_tags(&self, accounts: Option>) -> BichonResult> { let searcher = self.reader.searcher(); let query: Box = match accounts { @@ -798,7 +795,7 @@ impl IndexManager { Ok(()) } - pub async fn search( + pub fn search( &self, accounts: Option>, filter: AttachmentSearchFilter, @@ -890,10 +887,7 @@ impl IndexManager { Ok(self.reader.searcher()) } - pub async fn get_all_senders( - &self, - accounts: Option>, - ) -> BichonResult> { + pub fn get_all_senders(&self, accounts: Option>) -> BichonResult> { let searcher = self.create_searcher()?; let query: Box = match accounts { @@ -923,7 +917,7 @@ impl IndexManager { let doc: TantivyDocument = searcher .doc(doc_address) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let contacts = extract_senders(&doc).await?; + let contacts = extract_senders(&doc)?; for value in contacts { contacts_set.insert(value); } diff --git a/crates/core/src/store/tantivy/envelope.rs b/crates/core/src/store/tantivy/envelope.rs index 38eead0..2705da7 100644 --- a/crates/core/src/store/tantivy/envelope.rs +++ b/crates/core/src/store/tantivy/envelope.rs @@ -36,8 +36,8 @@ use crate::{ raise_error, settings::dir::DATA_DIR_MANAGER, store::{ - envelope::Envelope, blob::BLOB_MANAGER, + envelope::Envelope, tantivy::{ fatal_commit, fields::{ @@ -322,7 +322,7 @@ impl IndexManager { if let Some(ref body_val) = filter.body { let query_parser = QueryParser::for_index(&self.index, vec![f.f_body]); - + let q = query_parser .parse_query(body_val) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?; @@ -518,7 +518,7 @@ impl IndexManager { Box::new(boolean_query) } - pub async fn get_envelope_by_id( + pub fn get_envelope_by_id( &self, account_id: u64, envelope_id: &str, @@ -558,7 +558,7 @@ impl IndexManager { } } - pub async fn top_10_largest_emails( + pub fn top_10_largest_emails( &self, accounts: &Option>, ) -> BichonResult> { @@ -625,7 +625,7 @@ impl IndexManager { } } - pub async fn get_max_uid(&self, account_id: u64, mailbox_id: u64) -> BichonResult> { + pub fn get_max_uid(&self, account_id: u64, mailbox_id: u64) -> BichonResult> { let searcher = self.create_searcher()?; let query = self.mailbox_query(account_id, mailbox_id); @@ -646,7 +646,7 @@ impl IndexManager { Ok(Self::extract_max_uid(&agg_res)) } - pub async fn get_account_stats(&self, account_id: u64) -> BichonResult { + pub fn get_account_stats(&self, account_id: u64) -> BichonResult { let searcher = self.create_searcher()?; let query = self.account_query(account_id); @@ -705,7 +705,7 @@ impl IndexManager { pub async fn delete_account_envelopes(&self, account_id: u64) -> BichonResult<()> { let query = self.account_query(account_id); let (eml_content_hashes, attachments_content_hashes) = - self.collect_content_hashes(query).await?; + self.collect_content_hashes(query)?; let query = self.account_query(account_id); @@ -718,8 +718,7 @@ impl IndexManager { .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() { - self.cleanup_unused_content(eml_content_hashes, attachments_content_hashes) - .await?; + self.cleanup_unused_content(eml_content_hashes, attachments_content_hashes)?; } Ok(()) } @@ -738,7 +737,7 @@ impl IndexManager { for mailbox_id in &mailbox_ids { let query = self.mailbox_query(account_id, *mailbox_id); - let (eml_hashes, attachment_hashes) = self.collect_content_hashes(query).await?; + let (eml_hashes, attachment_hashes) = self.collect_content_hashes(query)?; eml_content_hashes.extend(eml_hashes); attachments_content_hashes.extend(attachment_hashes); } @@ -758,13 +757,12 @@ impl IndexManager { .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() { - self.cleanup_unused_content(eml_content_hashes, attachments_content_hashes) - .await?; + self.cleanup_unused_content(eml_content_hashes, attachments_content_hashes)?; } Ok(()) } - async fn collect_content_hashes( + fn collect_content_hashes( &self, query: Box, ) -> BichonResult<(HashSet, HashSet)> { @@ -802,7 +800,7 @@ impl IndexManager { Ok((eml_content_hashes, attachments_content_hashes)) } - async fn cleanup_unused_content( + fn cleanup_unused_content( &self, eml_content_hashes: HashSet, attachments_content_hashes: HashSet, @@ -862,7 +860,7 @@ impl IndexManager { for eid in unique_ids { let query = self.envelope_query(*account_id, eid); - let (eml_hashes, attachment_hashes) = self.collect_content_hashes(query).await?; + let (eml_hashes, attachment_hashes) = self.collect_content_hashes(query)?; eml_content_hashes.extend(eml_hashes); attachments_content_hashes.extend(attachment_hashes); } @@ -887,8 +885,7 @@ impl IndexManager { .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() { - self.cleanup_unused_content(eml_content_hashes, attachments_content_hashes) - .await?; + self.cleanup_unused_content(eml_content_hashes, attachments_content_hashes)?; } Ok(()) @@ -918,10 +915,7 @@ impl IndexManager { Ok(()) } - pub async fn get_all_tags( - &self, - accounts: Option>, - ) -> BichonResult> { + pub fn get_all_tags(&self, accounts: Option>) -> BichonResult> { let searcher = self.reader.searcher(); let query: Box = match accounts { @@ -945,7 +939,7 @@ impl IndexManager { Ok(all_facets) } - pub async fn get_all_contacts( + pub fn get_all_contacts( &self, accounts: Option>, ) -> BichonResult> { @@ -977,7 +971,7 @@ impl IndexManager { let doc: TantivyDocument = searcher .doc(doc_address) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let contacts = extract_contacts(&doc).await?; + let contacts = extract_contacts(&doc)?; for value in contacts { contacts_set.insert(value); } @@ -1067,7 +1061,7 @@ impl IndexManager { Ok(()) } - pub async fn search( + pub fn search( &self, accounts: Option>, filter: EmailSearchFilter, @@ -1159,7 +1153,7 @@ impl IndexManager { Ok(self.reader.searcher()) } - pub async fn num_messages_in_thread( + pub fn num_messages_in_thread( &self, searcher: &Searcher, account_id: u64, @@ -1207,7 +1201,7 @@ impl IndexManager { } } - pub async fn list_thread_envelopes( + pub fn list_thread_envelopes( &self, account_id: u64, thread_id: &str, @@ -1218,9 +1212,7 @@ impl IndexManager { assert!(page > 0, "Page number must be greater than 0"); assert!(page_size > 0, "Page size must be greater than 0"); let searcher = self.create_searcher()?; - let total = self - .num_messages_in_thread(&searcher, account_id, thread_id) - .await?; + let total = self.num_messages_in_thread(&searcher, account_id, thread_id)?; if total == 0 { return Ok(DataPage { current_page: Some(page), @@ -1271,7 +1263,7 @@ impl IndexManager { }) } - pub async fn get_dashboard_stats( + pub fn get_dashboard_stats( &self, accounts: &Option>, ) -> BichonResult { diff --git a/crates/core/src/store/tantivy/model.rs b/crates/core/src/store/tantivy/model.rs index bc9ce2d..f05fffa 100644 --- a/crates/core/src/store/tantivy/model.rs +++ b/crates/core/src/store/tantivy/model.rs @@ -280,7 +280,7 @@ fn extract_vec_string_field( Ok(value) } -pub async fn extract_contacts(doc: &TantivyDocument) -> BichonResult> { +pub fn extract_contacts(doc: &TantivyDocument) -> BichonResult> { let fields = SchemaTools::email_fields(); let mut all_contacts = HashSet::new(); @@ -305,7 +305,7 @@ pub async fn extract_contacts(doc: &TantivyDocument) -> BichonResult BichonResult> { +pub fn extract_senders(doc: &TantivyDocument) -> BichonResult> { let fields = SchemaTools::attachment_fields(); let mut senders = HashSet::new(); diff --git a/crates/core/src/token/mod.rs b/crates/core/src/token/mod.rs index 36199bf..ba6dd70 100644 --- a/crates/core/src/token/mod.rs +++ b/crates/core/src/token/mod.rs @@ -18,12 +18,11 @@ use std::collections::HashMap; -use super::error::code::ErrorCode; use crate::database::manager::DB_MANAGER; use crate::database::{ - async_filter_by_secondary_key_impl, async_find_impl, delete_impl, with_transaction, + MemDbModel, delete_impl, filter_impl, find_impl, insert_impl, list_all_impl, update_impl, with_transaction }; -use crate::database::{insert_impl, list_all_impl, update_impl}; +use crate::error::code::ErrorCode; use crate::raise_error; use crate::settings::cli::SETTINGS; use crate::token::view::AccessTokenResp; @@ -31,36 +30,12 @@ use crate::users::UserModel; use crate::{ error::BichonResult, generate_token, token::payload::AccessTokenCreateRequest, utc_now, }; -use native_db::*; -use native_model::{native_model, Model}; //use poem_openapi::{Enum, Object}; use serde::{Deserialize, Serialize}; pub mod payload; pub mod view; -// Starting from version 0.2.0, this model is deprecated/no longer used -// #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Object)] -// #[native_model(id = 1, version = 1)] -// #[native_db] -// pub struct AccessToken { -// /// The unique token string used for authentication -// #[primary_key] -// pub token: String, -// /// A set of account information associated with the token. -// pub accounts: BTreeSet, -// /// The timestamp (in milliseconds since epoch) when the token was created. -// pub created_at: i64, -// /// The timestamp (in milliseconds since epoch) when the token was last updated. -// pub updated_at: i64, -// /// An optional description of the token's purpose or usage. -// pub description: Option, -// /// The timestamp (in milliseconds since epoch) when the token was last used. -// pub last_access_at: i64, -// /// Optional access control settings -// pub acl: Option, -// } - #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))] pub enum TokenType { @@ -70,14 +45,10 @@ pub enum TokenType { #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] -#[native_model(id = 11, version = 1)] -#[native_db] pub struct AccessTokenModel { /// The ID of the user who owns this token - #[secondary_key] pub user_id: u64, /// The unique token string used for authentication - #[primary_key] pub token: String, /// An optional name of the token. pub name: Option, @@ -94,6 +65,15 @@ pub struct AccessTokenModel { pub last_access_at: i64, } +impl MemDbModel for AccessTokenModel { + fn collection() -> &'static str { + "tokens" + } + fn key(&self) -> String { + self.token.clone() + } +} + impl AccessTokenModel { pub fn new_api_token( token: String, @@ -127,74 +107,56 @@ impl AccessTokenModel { } } - pub async fn reset_webui_token(user_id: u64) -> BichonResult { - let old_token = Self::get_user_webui_token(user_id).await?; + pub fn reset_webui_token(user_id: u64) -> BichonResult { + let old_token = Self::get_user_webui_token(user_id)?; let new_token = Self::new_webui_token(user_id); let new_token_str = new_token.token.clone(); match old_token { Some(old) => { - with_transaction(DB_MANAGER.meta_db(), move |rw| { - rw.remove(old) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - rw.insert(new_token) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - Ok(()) - }) - .await?; + with_transaction(DB_MANAGER.db(), move |txn| { + let txn = txn.delete(AccessTokenModel::collection(), old.token.clone()); + txn.insert(AccessTokenModel::collection(), new_token.key(), &new_token) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError)) + })?; } None => { - insert_impl(DB_MANAGER.meta_db(), new_token).await?; + insert_impl(DB_MANAGER.db(), new_token)?; } } Ok(new_token_str) } - pub async fn get_user_webui_token(user_id: u64) -> BichonResult> { - let tokens = async_filter_by_secondary_key_impl::( - DB_MANAGER.meta_db(), - AccessTokenModelKey::user_id, - user_id, - ) - .await?; - + pub fn get_user_webui_token(user_id: u64) -> BichonResult> { + let tokens = + filter_impl::(DB_MANAGER.db(), move |t| t.user_id == user_id)?; Ok(tokens .into_iter() .find(|t| t.token_type == TokenType::WebUI)) } - pub async fn get_user_api_tokens(user_id: u64) -> BichonResult> { - let tokens = async_filter_by_secondary_key_impl::( - DB_MANAGER.meta_db(), - AccessTokenModelKey::user_id, - user_id, - ) - .await?; - + pub fn get_user_api_tokens(user_id: u64) -> BichonResult> { + let tokens = + filter_impl::(DB_MANAGER.db(), move |t| t.user_id == user_id)?; Ok(tokens .into_iter() .filter(|t| t.token_type == TokenType::Api) .collect()) } - pub async fn resolve_user_from_token(token: &str) -> BichonResult { - let token = token.to_string(); - let token_option = async_find_impl::(DB_MANAGER.meta_db(), token).await?; - let token = match token_option { - Some(token) => token, - None => { - return Err(raise_error!( + pub fn resolve_user_from_token(token: &str) -> BichonResult { + let token_str = token.to_string(); + let token_model = find_impl::(DB_MANAGER.db(), &token_str)? + .ok_or_else(|| { + raise_error!( "Invalid access token provided. Please check your credentials.".into(), ErrorCode::PermissionDenied - )) - } - }; + ) + })?; - if matches!(token.token_type, TokenType::WebUI) { - let life = utc_now!() - token.created_at; + if matches!(token_model.token_type, TokenType::WebUI) { + let life = utc_now!() - token_model.created_at; let max_life = SETTINGS.bichon_webui_token_expiration_hours * 60 * 60 * 1000; if life > (max_life as i64) { @@ -205,8 +167,8 @@ impl AccessTokenModel { } } - if matches!(token.token_type, TokenType::Api) { - if let Some(expire_at) = token.expire_at { + if matches!(token_model.token_type, TokenType::Api) { + if let Some(expire_at) = token_model.expire_at { if utc_now!() > expire_at { return Err(raise_error!( "Your API token has expired and is no longer valid.".into(), @@ -214,87 +176,57 @@ impl AccessTokenModel { )); } } - let token = token.token.clone(); - update_impl( - DB_MANAGER.meta_db(), - |rw| { - rw.get() - .primary::(token) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| { - raise_error!( - "The access token does not exist or has been reset.".into(), - ErrorCode::ResourceNotFound - ) - }) - }, - |current| { - let mut updated = current.clone(); - updated.last_access_at = utc_now!(); - Ok(updated) - }, - ) - .await?; + update_impl(DB_MANAGER.db(), &token_str, |current: AccessTokenModel| { + let mut updated = current.clone(); + updated.last_access_at = utc_now!(); + Ok(updated) + })?; } - let user = UserModel::find(token.user_id) - .await? + let user = UserModel::find(token_model.user_id) + ? .ok_or_else(|| raise_error!("The user associated with this access token does not exist or may have been deleted.".into(), ErrorCode::ResourceNotFound))?; Ok(user) } - pub async fn create_api_token( + pub fn create_api_token( user_id: u64, request: AccessTokenCreateRequest, ) -> BichonResult { // Validate request parameters first - request.validate().await?; + request.validate()?; let expire_at = request .expire_in .map(|hours| utc_now!() + (hours as i64) * 60 * 60 * 1000); let token = generate_token!(128); let access_token = AccessTokenModel::new_api_token(token.clone(), user_id, request.name, expire_at); - insert_impl(DB_MANAGER.meta_db(), access_token).await?; + insert_impl(DB_MANAGER.db(), access_token)?; Ok(token) } - pub async fn delete(token: &str) -> BichonResult<()> { - let token = token.to_string(); - delete_impl(DB_MANAGER.meta_db(), move |rw| { - rw.get() - .primary::(token.clone()) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| { - raise_error!( - format!("Token '{}' not found during deletion process.", token), - ErrorCode::ResourceNotFound - ) - }) + pub fn delete(token: &str) -> BichonResult<()> { + delete_impl::(DB_MANAGER.db(), token) + } + + pub fn get_token(token: &str) -> BichonResult { + find_impl::(DB_MANAGER.db(), token)?.ok_or_else(|| { + raise_error!( + format!("Access token '{}' not found", token), + ErrorCode::ResourceNotFound + ) }) - .await } - pub async fn get_token(token: &str) -> BichonResult { - async_find_impl(DB_MANAGER.meta_db(), token.to_string()) - .await? - .ok_or_else(|| { - raise_error!( - format!("Access token '{}' not found", token), - ErrorCode::ResourceNotFound - ) - }) - } + pub fn list_all_api_tokens() -> BichonResult> { + let users = UserModel::list_all()?; + let all = list_all_impl::(DB_MANAGER.db())?; - pub async fn list_all_api_tokens() -> BichonResult> { - let users = UserModel::list_all().await?; - let mut all = list_all_impl::(DB_MANAGER.meta_db()).await?; - - all.retain(|t| t.token_type == TokenType::Api); let user_map: HashMap = users.into_iter().map(|u| (u.id, u)).collect(); let resp = all .into_iter() + .filter(|t| t.token_type == TokenType::Api) .map(|token| { let user = user_map.get(&token.user_id); AccessTokenResp { diff --git a/crates/core/src/token/payload.rs b/crates/core/src/token/payload.rs index 1749b0b..a45260f 100644 --- a/crates/core/src/token/payload.rs +++ b/crates/core/src/token/payload.rs @@ -39,7 +39,7 @@ pub struct AccessTokenCreateRequest { } impl AccessTokenCreateRequest { - pub async fn validate(&self) -> BichonResult<()> { + pub fn validate(&self) -> BichonResult<()> { if let Some(expire_in) = self.expire_in { if expire_in == 0 { return Err(raise_error!( diff --git a/crates/core/src/users/manager.rs b/crates/core/src/users/manager.rs index 5df10f9..e65130e 100644 --- a/crates/core/src/users/manager.rs +++ b/crates/core/src/users/manager.rs @@ -26,7 +26,7 @@ pub struct UserManager; impl Initialize for UserManager { async fn initialize() -> BichonResult<()> { - UserRole::ensure_default_roles_exists().await?; - UserModel::ensure_default_admin_exists().await + UserRole::ensure_default_roles_exists()?; + UserModel::ensure_default_admin_exists() } } diff --git a/crates/core/src/users/minimal.rs b/crates/core/src/users/minimal.rs index b10508b..4968e02 100644 --- a/crates/core/src/users/minimal.rs +++ b/crates/core/src/users/minimal.rs @@ -34,8 +34,8 @@ pub struct MinimalUser { } impl MinimalUser { - pub async fn list_all() -> BichonResult> { - let all_users = list_all_impl::(DB_MANAGER.meta_db()).await?; + pub fn list_all() -> BichonResult> { + let all_users = list_all_impl::(DB_MANAGER.db())?; let minimal_list = all_users .into_iter() .map(|user| MinimalUser { diff --git a/crates/core/src/users/mod.rs b/crates/core/src/users/mod.rs index e523b35..9e6d994 100644 --- a/crates/core/src/users/mod.rs +++ b/crates/core/src/users/mod.rs @@ -17,27 +17,23 @@ // along with this program. If not, see . use crate::{ - decrypt, encrypt, generate_token, id, raise_error, utc_now, - { - database::{ - async_find_impl, async_secondary_find_impl, batch_delete_impl, delete_impl, - list_all_impl, manager::DB_MANAGER, update_impl, with_transaction, - }, - error::{code::ErrorCode, BichonResult}, - token::{AccessTokenModel, AccessTokenModelKey, TokenType}, - users::{ - acl::AccessControl, - payload::{UserCreateRequest, UserUpdateRequest}, - permissions::Permission, - role::{UserRole, DEFAULT_ADMIN_ROLE_ID}, - view::UserView, - }, + database::{ + delete_impl, filter_impl, find_impl, list_all_impl, manager::DB_MANAGER, update_impl, + with_transaction, MemDbModel, }, + decrypt, encrypt, + error::{code::ErrorCode, BichonResult}, + generate_token, id, raise_error, + token::{AccessTokenModel, TokenType}, + users::{ + acl::AccessControl, + payload::{UserCreateRequest, UserUpdateRequest}, + permissions::Permission, + role::{UserRole, DEFAULT_ADMIN_ROLE_ID}, + view::UserView, + }, + utc_now, }; -use itertools::Itertools; -use native_db::*; -use native_model::{native_model, Model}; -//use poem_openapi::Object; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet, HashSet}; use tracing::warn; @@ -64,46 +60,9 @@ pub struct LoginResult { pub const DEFAULT_ADMIN_USER_ID: u64 = 100000000000000; #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] -#[native_model(id = 10, version = 1)] -#[native_db] -pub struct BichonUser { - #[primary_key] - pub id: u64, - #[secondary_key(unique)] - pub username: String, - #[secondary_key(unique)] - pub email: String, - - pub password: Option, - - /// Scoped Access: Defines per-account permissions. - /// Example: - /// { account_id: 1, role_id: role_manager_id } -> Manager on Account 1 - /// { account_id: 2, role_id: role_viewer_id } -> Viewer on Account 2 - pub account_access_map: BTreeMap, - - pub description: Option, - - /// System Roles: Permissions that apply to the whole system - /// (e.g., system settings, creating new users). - pub global_roles: Vec, - - pub avatar: Option, - pub created_at: i64, - pub updated_at: i64, - /// Optional access control settings - pub acl: Option, -} - -#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] -#[native_model(id = 10, version = 2, from = BichonUser)] -#[native_db] pub struct BichonUserV2 { - #[primary_key] pub id: u64, - #[secondary_key(unique)] pub username: String, - #[secondary_key(unique)] pub email: String, pub password: Option, @@ -130,6 +89,15 @@ pub struct BichonUserV2 { pub language: Option, } +impl MemDbModel for BichonUserV2 { + fn collection() -> &'static str { + "users" + } + fn key(&self) -> String { + self.id.to_string() + } +} + impl BichonUserV2 { pub fn is_using_role(&self, role_id: u64) -> bool { if self.global_roles.contains(&role_id) { @@ -142,15 +110,15 @@ impl BichonUserV2 { false } - pub async fn list_all() -> BichonResult> { - Ok(list_all_impl::(DB_MANAGER.meta_db()).await?) + pub fn list_all() -> BichonResult> { + Ok(list_all_impl::(DB_MANAGER.db())?) } - async fn get_all_permissions(&self) -> HashSet { + fn get_all_permissions(&self) -> HashSet { let mut all_perms = HashSet::new(); for &role_id in &self.global_roles { - if let Ok(Some(role)) = UserRole::find(role_id).await { + if let Ok(Some(role)) = UserRole::find(role_id) { for perm in role.permissions { all_perms.insert(perm); } @@ -227,88 +195,77 @@ impl BichonUserV2 { } } - pub async fn is_admin(&self) -> bool { - self.get_all_permissions().await.contains(Permission::ROOT) + pub fn is_admin(&self) -> bool { + self.get_all_permissions().contains(Permission::ROOT) } - pub async fn ensure_default_admin_exists() -> BichonResult<()> { - with_transaction(DB_MANAGER.meta_db(), move |rw| { - let now = utc_now!(); + pub fn ensure_default_admin_exists() -> BichonResult<()> { + let now = utc_now!(); - // 1. Try to get the existing admin user - let admin = rw - .get() - .primary::(DEFAULT_ADMIN_USER_ID) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + // 1. Try to get the existing admin user + let admin = find_impl::(DB_MANAGER.db(), &DEFAULT_ADMIN_USER_ID.to_string())?; - if admin.is_none() { - // 2. Insert the BichonUser with the updated schema - rw.insert(UserModel { - id: DEFAULT_ADMIN_USER_ID, - username: "admin".into(), - email: "placeholder@example.com".into(), - password: Some(encrypt!("admin@bichon")?), + if admin.is_none() { + // 2. Insert the BichonUser with the updated schema + let user = UserModel { + id: DEFAULT_ADMIN_USER_ID, + username: "admin".into(), + email: "placeholder@example.com".into(), + password: Some(encrypt!("admin@bichon")?), - // Use global_roles as defined in our new schema - global_roles: vec![DEFAULT_ADMIN_ROLE_ID], + // Use global_roles as defined in our new schema + global_roles: vec![DEFAULT_ADMIN_ROLE_ID], - // Admin usually doesn't need specific scoped access - account_access_map: BTreeMap::new(), + // Admin usually doesn't need specific scoped access + account_access_map: BTreeMap::new(), - avatar: None, - created_at: now, - updated_at: now, - description: Some("System default administrator".into()), - acl: None, - theme: None, - language: None, - }) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + avatar: None, + created_at: now, + updated_at: now, + description: Some("System default administrator".into()), + acl: None, + theme: None, + language: None, + }; - // 3. Generate and insert an initial access token for the first-time setup - let access_token = AccessTokenModel { - token: generate_token!(128), - created_at: now, - updated_at: now, - last_access_at: Default::default(), - name: Some("Initial Setup Token".into()), - user_id: DEFAULT_ADMIN_USER_ID, - token_type: TokenType::WebUI, - expire_at: None, // Admin setup token usually persistent until changed - }; + // 3. Generate and insert an initial access token for the first-time setup + let access_token = AccessTokenModel { + token: generate_token!(128), + created_at: now, + updated_at: now, + last_access_at: Default::default(), + name: Some("Initial Setup Token".into()), + user_id: DEFAULT_ADMIN_USER_ID, + token_type: TokenType::WebUI, + expire_at: None, // Admin setup token usually persistent until changed + }; - rw.upsert(access_token) + with_transaction(DB_MANAGER.db(), move |txn| { + let txn = txn + .insert("users", DEFAULT_ADMIN_USER_ID.to_string(), &user) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .upsert("tokens", access_token.token.clone(), &access_token) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - } - - Ok(()) - }) - .await?; + Ok(txn) + })?; + } Ok(()) } - pub async fn authenticate_user( - username: String, - password: String, - ) -> BichonResult { - let user_option = async_secondary_find_impl::( - DB_MANAGER.meta_db(), - BichonUserV2Key::username, - username.clone(), - ) - .await?; - - let user = match user_option { + pub fn authenticate_user(username: String, password: String) -> BichonResult { + // Find by username + let username_for_first = username.clone(); + let users = filter_impl::(DB_MANAGER.db(), move |u| { + u.username == username_for_first + })?; + let user = match users.into_iter().next() { Some(u) => u, None => { - match async_secondary_find_impl::( - DB_MANAGER.meta_db(), - BichonUserV2Key::email, - username, - ) - .await? - { + // Fallback: find by email + let users = + filter_impl::(DB_MANAGER.db(), move |u| u.email == username)?; + match users.into_iter().next() { Some(u) => u, None => { return Ok(LoginResult { @@ -327,7 +284,7 @@ impl BichonUserV2 { Some(encrypted_password) => { let decrypted = decrypt!(encrypted_password)?; if password == decrypted { - let new_token = AccessTokenModel::reset_webui_token(user.id).await?; + let new_token = AccessTokenModel::reset_webui_token(user.id)?; Ok(LoginResult { success: true, error_message: None, @@ -358,7 +315,7 @@ impl BichonUserV2 { success: false, error_message: Some( format!( - "User '{}' has no password set. Please try logging in with an alternative method (e.g., OAuth/SSO).", + "User '{}' has no password set. Please try logging in with an alternative method (e.g., OAuth/SSO).", user.username ) ), @@ -370,20 +327,16 @@ impl BichonUserV2 { } } - pub async fn find(user_id: u64) -> BichonResult> { - async_find_impl(DB_MANAGER.meta_db(), user_id).await + pub fn find(user_id: u64) -> BichonResult> { + find_impl::(DB_MANAGER.db(), &user_id.to_string()) } - pub async fn check_username_conflict(username: &str) -> BichonResult<()> { - // Check username duplicate - if async_secondary_find_impl::( - DB_MANAGER.meta_db(), - BichonUserV2Key::username, - username.to_string(), - ) - .await? - .is_some() - { + pub fn check_username_conflict(username: &str) -> BichonResult<()> { + let username_clone = username.to_string(); + let users = + filter_impl::(DB_MANAGER.db(), move |u| u.username == username_clone)?; + + if users.into_iter().next().is_some() { return Err(raise_error!( format!("Username '{}' is already taken.", username).into(), ErrorCode::AlreadyExists @@ -393,16 +346,11 @@ impl BichonUserV2 { Ok(()) } - pub async fn check_email_conflict(email: &str) -> BichonResult<()> { - // Check email duplicate - if async_secondary_find_impl::( - DB_MANAGER.meta_db(), - BichonUserV2Key::email, - email.to_string(), - ) - .await? - .is_some() - { + pub fn check_email_conflict(email: &str) -> BichonResult<()> { + let email_clone = email.to_string(); + let users = filter_impl::(DB_MANAGER.db(), move |u| u.email == email_clone)?; + + if users.into_iter().next().is_some() { return Err(raise_error!( format!("Email '{}' is already registered.", email).into(), ErrorCode::AlreadyExists @@ -412,10 +360,10 @@ impl BichonUserV2 { Ok(()) } - pub async fn create(request: UserCreateRequest) -> BichonResult { - request.validate().await?; - Self::check_username_conflict(&request.username).await?; - Self::check_email_conflict(&request.email).await?; + pub fn create(request: UserCreateRequest) -> BichonResult { + request.validate()?; + Self::check_username_conflict(&request.username)?; + Self::check_email_conflict(&request.email)?; let password_hash = Some(encrypt!(&request.password)?); let now = utc_now!(); @@ -439,37 +387,31 @@ impl BichonUserV2 { let user_clone = user.clone(); // 4. Atomic transaction for User and Initial Token - with_transaction(DB_MANAGER.meta_db(), move |rw| { - let user_id = user.id; + let access_token = AccessTokenModel { + token: generate_token!(128), + created_at: now, + updated_at: now, + last_access_at: Default::default(), + name: Some("Default WebUI Token".into()), + user_id: user.id, + token_type: TokenType::WebUI, + expire_at: None, + }; - // Insert User - rw.insert(user) + with_transaction(DB_MANAGER.db(), move |txn| { + let txn = txn + .insert("users", user.key(), &user) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .insert("tokens", access_token.token.clone(), &access_token) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - // Create initial WebUI access token - let access_token = AccessTokenModel { - token: generate_token!(128), - created_at: now, - updated_at: now, - last_access_at: Default::default(), - name: Some("Default WebUI Token".into()), - user_id, - token_type: TokenType::WebUI, - expire_at: None, - }; - - rw.insert(access_token) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - Ok(()) - }) - .await?; + Ok(txn) + })?; Ok(user_clone) } //delete user， - pub async fn remove(id: u64) -> BichonResult<()> { + pub fn remove(id: u64) -> BichonResult<()> { if DEFAULT_ADMIN_USER_ID == id { return Err(raise_error!( format!("The default admin user (id={}) cannot be removed", id), @@ -477,36 +419,37 @@ impl BichonUserV2 { )); } - delete_impl(DB_MANAGER.meta_db(), move |rw| { - rw.get() - .primary::(id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| { - raise_error!( - format!("The User with id={id} that you want to delete was not found."), - ErrorCode::ResourceNotFound - ) - }) - }) - .await?; + delete_impl::(DB_MANAGER.db(), &id.to_string())?; + + // Find and delete tokens belonging to this user + let uid = id; + + let coll = DB_MANAGER.db().collection("tokens"); + let all_tokens: Vec = coll + .list_all() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let token_keys: Vec = all_tokens + .into_iter() + .filter(|t| t.user_id == uid) + .map(|t| t.token) + .collect(); + + if !token_keys.is_empty() { + with_transaction(DB_MANAGER.db(), move |txn| { + let mut txn = txn; + for key in token_keys { + txn = txn.delete("tokens", key); + } + Ok(txn) + })?; + } - batch_delete_impl(DB_MANAGER.meta_db(), move |rw| { - let tokens: Vec = rw - .scan() - .secondary::(AccessTokenModelKey::user_id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .start_with(id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .try_collect() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(tokens) - }) - .await?; Ok(()) } - pub async fn update(id: u64, request: UserUpdateRequest) -> BichonResult<()> { - let _ = &request.validate().await?; + pub fn update(id: u64, request: UserUpdateRequest) -> BichonResult<()> { + let _ = &request.validate()?; let password_changed = request.password.is_some(); let is_default_admin = id == DEFAULT_ADMIN_USER_ID; @@ -530,14 +473,12 @@ impl BichonUserV2 { } if let Some(username) = &request.username { - let user_option = async_secondary_find_impl::( - DB_MANAGER.meta_db(), - BichonUserV2Key::username, - username.to_string(), - ) - .await?; + let username_clone = username.clone(); + let users = filter_impl::(DB_MANAGER.db(), move |u| { + u.username == username_clone + })?; - if let Some(u) = user_option { + if let Some(u) = users.into_iter().next() { if u.id != id { return Err(raise_error!( format!("Username '{}' is already taken.", username).into(), @@ -548,14 +489,11 @@ impl BichonUserV2 { } if let Some(email) = &request.email { - let user_option = async_secondary_find_impl::( - DB_MANAGER.meta_db(), - BichonUserV2Key::email, - email.to_string(), - ) - .await?; + let email_clone = email.clone(); + let users = + filter_impl::(DB_MANAGER.db(), move |u| u.email == email_clone)?; - if let Some(u) = user_option { + if let Some(u) = users.into_iter().next() { if u.id != id { return Err(raise_error!( format!("Email '{}' is already registered.", email).into(), @@ -565,74 +503,59 @@ impl BichonUserV2 { } } - update_impl( - DB_MANAGER.meta_db(), - move |rw| { - rw.get() - .primary::(id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| { - raise_error!( - format!("User with id={} not found", id), - ErrorCode::ResourceNotFound - ) - }) - }, - move |current| { - let mut updated = current.clone(); - if let Some(username) = request.username { - updated.username = username; - } - if let Some(email) = request.email { - updated.email = email; - } - if let Some(desc) = request.description { - updated.description = Some(desc); - } - if let Some(password) = request.password { - updated.password = Some(encrypt!(&password)?); - } + update_impl::(DB_MANAGER.db(), &id.to_string(), move |current| { + let mut updated = current.clone(); + if let Some(username) = request.username { + updated.username = username; + } + if let Some(email) = request.email { + updated.email = email; + } + if let Some(desc) = request.description { + updated.description = Some(desc); + } + if let Some(password) = request.password { + updated.password = Some(encrypt!(&password)?); + } - if let Some(global_roles) = request.global_roles { - updated.global_roles = global_roles; - } + if let Some(global_roles) = request.global_roles { + updated.global_roles = global_roles; + } - if let Some(acl) = request.acl { - updated.acl = Some(acl); - } + if let Some(acl) = request.acl { + updated.acl = Some(acl); + } - if let Some(account_access_map) = request.account_access_map { - updated.account_access_map = account_access_map; - } + if let Some(account_access_map) = request.account_access_map { + updated.account_access_map = account_access_map; + } - if let Some(avatar_base64) = request.avatar_base64 { - updated.avatar = Some(avatar_base64); - } + if let Some(avatar_base64) = request.avatar_base64 { + updated.avatar = Some(avatar_base64); + } - if let Some(theme) = request.theme { - updated.theme = Some(theme); - } + if let Some(theme) = request.theme { + updated.theme = Some(theme); + } - if let Some(language) = request.language { - updated.language = Some(language); - } + if let Some(language) = request.language { + updated.language = Some(language); + } - updated.updated_at = utc_now!(); + updated.updated_at = utc_now!(); - Ok(updated) - }, - ) - .await?; + Ok(updated) + })?; if password_changed { - AccessTokenModel::reset_webui_token(id).await?; + AccessTokenModel::reset_webui_token(id)?; } Ok(()) } - async fn list_authorized_users(account_id: u64) -> BichonResult> { - let all = Self::list_all().await?; + fn list_authorized_users(account_id: u64) -> BichonResult> { + let all = Self::list_all()?; let result: Vec = all .into_iter() .filter(|e| e.account_access_map.contains_key(&account_id)) @@ -640,76 +563,24 @@ impl BichonUserV2 { Ok(result) } - pub async fn cleanup_account(account_id: u64) -> BichonResult<()> { - let users = Self::list_authorized_users(account_id).await?; + pub fn cleanup_account(account_id: u64) -> BichonResult<()> { + let users = Self::list_authorized_users(account_id)?; if users.is_empty() { return Ok(()); } - with_transaction(DB_MANAGER.meta_db(), move |rw| { - let now = utc_now!(); - for user in users { - let current = rw - .get() - .primary::(user.id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| { - raise_error!( - format!("User {} not found", user.id), - ErrorCode::ResourceNotFound - ) - })?; - + let now = utc_now!(); + for user in users { + let key = user.id.to_string(); + update_impl::(DB_MANAGER.db(), &key, move |current| { let mut updated = current.clone(); - if updated.account_access_map.remove(&account_id).is_some() { updated.updated_at = now; - rw.update(current, updated) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; } - } - Ok(()) - }) - .await?; + Ok(updated) + })?; + } Ok(()) } } - -impl From for BichonUser { - fn from(value: BichonUserV2) -> Self { - BichonUser { - id: value.id, - username: value.username, - email: value.email, - password: value.password, - account_access_map: value.account_access_map, - description: value.description, - global_roles: value.global_roles, - avatar: value.avatar, - created_at: value.created_at, - updated_at: value.updated_at, - acl: value.acl, - } - } -} - -impl From for BichonUserV2 { - fn from(value: BichonUser) -> Self { - BichonUserV2 { - id: value.id, - username: value.username, - email: value.email, - password: value.password, - account_access_map: value.account_access_map, - description: value.description, - global_roles: value.global_roles, - avatar: value.avatar, - created_at: value.created_at, - updated_at: value.updated_at, - acl: value.acl, - theme: None, - language: None, - } - } -} diff --git a/crates/core/src/users/payload.rs b/crates/core/src/users/payload.rs index ce09630..cea0cb4 100644 --- a/crates/core/src/users/payload.rs +++ b/crates/core/src/users/payload.rs @@ -95,7 +95,7 @@ pub struct RoleCreateRequest { } impl RoleCreateRequest { - pub async fn validate(&self) -> BichonResult<()> { + pub fn validate(&self) -> BichonResult<()> { let trimmed_name = self.name.trim(); if trimmed_name.is_empty() { return Err(raise_error!( @@ -148,7 +148,7 @@ pub struct RoleUpdateRequest { } impl RoleUpdateRequest { - pub async fn validate(&self) -> BichonResult<()> { + pub fn validate(&self) -> BichonResult<()> { // 1. Ensure at least one field is provided for the update if self.name.is_none() && self.description.is_none() && self.permissions.is_none() { return Err(raise_error!( @@ -216,7 +216,10 @@ impl RoleUpdateRequest { pub struct UserCreateRequest { pub username: String, - #[cfg_attr(feature = "web-api", oai(validator(custom = "crate::common::validator::EmailValidator")))] + #[cfg_attr( + feature = "web-api", + oai(validator(custom = "crate::common::validator::EmailValidator")) + )] pub email: String, pub password: String, @@ -236,7 +239,7 @@ pub struct UserCreateRequest { } impl UserCreateRequest { - pub async fn validate(&self) -> BichonResult<()> { + pub fn validate(&self) -> BichonResult<()> { let username_len = self.username.len(); // 1. Username constraints @@ -279,7 +282,7 @@ impl UserCreateRequest { validate_theme(&self.theme)?; validate_language(&self.language)?; - let all_roles = UserRole::list_all().await?; + let all_roles = UserRole::list_all()?; let role_type_map: HashMap = all_roles.into_iter().map(|r| (r.id, r.role_type)).collect(); @@ -302,7 +305,7 @@ impl UserCreateRequest { } for (aid, rid) in &self.account_access_map { - if AccountModel::async_find(*aid).await?.is_none() { + if AccountModel::find(*aid)?.is_none() { return Err(raise_error!( format!("Account {} not found", aid), ErrorCode::InvalidParameter @@ -353,7 +356,10 @@ impl UserCreateRequest { #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] pub struct UserUpdateRequest { pub username: Option, - #[cfg_attr(feature = "web-api", oai(validator(custom = "crate::common::validator::EmailValidator")))] + #[cfg_attr( + feature = "web-api", + oai(validator(custom = "crate::common::validator::EmailValidator")) + )] pub email: Option, pub password: Option, pub avatar_base64: Option, @@ -367,7 +373,7 @@ pub struct UserUpdateRequest { } impl UserUpdateRequest { - pub async fn validate(&self) -> BichonResult<()> { + pub fn validate(&self) -> BichonResult<()> { if let Some(username) = &self.username { let len = username.len(); if len < 3 || len > 32 { @@ -391,7 +397,7 @@ impl UserUpdateRequest { validate_theme(&self.theme)?; validate_language(&self.language)?; - let all_roles = UserRole::list_all().await?; + let all_roles = UserRole::list_all()?; let role_type_map: HashMap = all_roles.into_iter().map(|r| (r.id, r.role_type)).collect(); @@ -423,7 +429,7 @@ impl UserUpdateRequest { if let Some(account_access_map) = &self.account_access_map { for (aid, rid) in account_access_map { - if AccountModel::async_find(*aid).await?.is_none() { + if AccountModel::find(*aid)?.is_none() { return Err(raise_error!( format!("Account {} not found", aid), ErrorCode::InvalidParameter diff --git a/crates/core/src/users/role.rs b/crates/core/src/users/role.rs index f28e380..c395f41 100644 --- a/crates/core/src/users/role.rs +++ b/crates/core/src/users/role.rs @@ -21,17 +21,14 @@ use std::{ fmt::{self, Display}, }; -use native_db::*; -use native_model::{native_model, Model}; -//use poem_openapi::{Enum, Object}; use serde::{Deserialize, Serialize}; use crate::{ id, raise_error, utc_now, { database::{ - async_find_impl, delete_impl, insert_impl, list_all_impl, manager::DB_MANAGER, - update_impl, with_transaction, + delete_impl, find_impl, insert_impl, list_all_impl, manager::DB_MANAGER, update_impl, + with_transaction, MemDbModel, }, error::{code::ErrorCode, BichonResult}, users::{ @@ -184,10 +181,7 @@ pub enum RoleType { #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] -#[native_model(id = 9, version = 1)] -#[native_db] pub struct UserRole { - #[primary_key] pub id: u64, pub name: String, pub description: Option, @@ -198,8 +192,17 @@ pub struct UserRole { pub updated_at: i64, } +impl MemDbModel for UserRole { + fn collection() -> &'static str { + "roles" + } + fn key(&self) -> String { + self.id.to_string() + } +} + impl UserRole { - pub async fn ensure_default_roles_exists() -> BichonResult<()> { + pub fn ensure_default_roles_exists() -> BichonResult<()> { let builtin_roles = vec![ (BuiltinRole::Admin, DEFAULT_ADMIN_ROLE_ID, RoleType::Global), ( @@ -224,24 +227,19 @@ impl UserRole { ), ]; - with_transaction(DB_MANAGER.meta_db(), move |rw| { + with_transaction(DB_MANAGER.db(), move |txn| { + let mut txn = txn; let now = utc_now!(); - for (role, role_id, role_type) in builtin_roles { - let exists = rw - .get() - .primary::(role_id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .is_some(); - + let key = role_id.to_string(); + let exists = find_impl::(DB_MANAGER.db(), &key)?.is_some(); if !exists { let permissions: BTreeSet = role .get_permissions() .into_iter() .map(|s| s.to_string()) .collect(); - - rw.insert(UserRole { + let role_item = UserRole { id: role_id, name: role.to_string(), description: Some(role.description().to_string()), @@ -250,27 +248,28 @@ impl UserRole { updated_at: now, is_builtin: true, role_type, - }) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + }; + txn = txn + .insert(UserRole::collection(), role_item.key(), &role_item) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; } } - Ok(()) - }) - .await?; + Ok(txn) + })?; Ok(()) } - pub async fn list_all() -> BichonResult> { - list_all_impl(DB_MANAGER.meta_db()).await + pub fn list_all() -> BichonResult> { + list_all_impl::(DB_MANAGER.db()) } - pub async fn find(role_id: u64) -> BichonResult> { - async_find_impl(DB_MANAGER.meta_db(), role_id).await + pub fn find(role_id: u64) -> BichonResult> { + find_impl::(DB_MANAGER.db(), &role_id.to_string()) } - pub async fn create(request: RoleCreateRequest) -> BichonResult { - let _ = &request.validate().await?; + pub fn create(request: RoleCreateRequest) -> BichonResult { + let _ = &request.validate()?; let now = utc_now!(); let new_role = UserRole { id: id!(64), @@ -282,21 +281,21 @@ impl UserRole { is_builtin: false, role_type: request.role_type, }; - insert_impl(DB_MANAGER.meta_db(), new_role.clone()).await?; + insert_impl(DB_MANAGER.db(), new_role.clone())?; Ok(new_role) } - pub async fn update(id: u64, request: RoleUpdateRequest) -> BichonResult<()> { + pub fn update(id: u64, request: RoleUpdateRequest) -> BichonResult<()> { if is_builtin(id) && request.permissions.is_some() { return Err(raise_error!( "The permissions of a builtin role are immutable. Please create a custom role instead.".into(), ErrorCode::Forbidden )); } - let _ = &request.validate().await?; + let _ = &request.validate()?; if let Some(permissions) = &request.permissions { - let role = Self::find(id).await?.ok_or_else(|| { + let role = Self::find(id)?.ok_or_else(|| { raise_error!( format!("UserRole with id={} not found", id), ErrorCode::ResourceNotFound @@ -306,40 +305,27 @@ impl UserRole { } update_impl( - DB_MANAGER.meta_db(), - move |rw| { - rw.get() - .primary::(id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| { - raise_error!( - format!("UserRole with id={} not found", id), - ErrorCode::ResourceNotFound - ) - }) - }, - move |current| { + DB_MANAGER.db(), + &id.to_string(), + move |current: UserRole| { let mut updated = current.clone(); if let Some(name) = request.name { updated.name = name; } - if let Some(desc) = request.description { updated.description = Some(desc); } - if let Some(permissions) = request.permissions { updated.permissions = permissions; } updated.updated_at = utc_now!(); Ok(updated) }, - ) - .await?; + )?; Ok(()) } - pub async fn delete(id: u64) -> BichonResult<()> { + pub fn delete(id: u64) -> BichonResult<()> { if is_builtin(id) { return Err(raise_error!( format!("Cannot delete a default system role (ID: {}).", id), @@ -347,7 +333,7 @@ impl UserRole { )); } - let all_users = UserModel::list_all().await?; + let all_users = UserModel::list_all()?; let active_users: Vec = all_users .iter() .filter(|user| user.is_using_role(id)) @@ -365,17 +351,6 @@ impl UserRole { )); } - delete_impl(DB_MANAGER.meta_db(), move |rw| { - rw.get() - .primary::(id) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .ok_or_else(|| { - raise_error!( - format!("UserRole '{}' not found during deletion process.", id), - ErrorCode::ResourceNotFound - ) - }) - }) - .await + delete_impl::(DB_MANAGER.db(), &id.to_string()) } } diff --git a/crates/core/src/utils/encrypt.rs b/crates/core/src/utils/encrypt.rs index 5eb9533..63b48d4 100644 --- a/crates/core/src/utils/encrypt.rs +++ b/crates/core/src/utils/encrypt.rs @@ -136,12 +136,49 @@ mod tests { use super::*; #[test] - fn test_encrypt_decrypt() { - let password = "my_secure_passwasdasdasdasdasord"; - let plaintext = "Helloasdasdasdasdasd, World!"; + fn test_encrypt_decrypt_roundtrip() { + let password = "my_secure_password"; + let plaintext = "Hello, World!"; let encrypted = internal_encrypt_string(password, plaintext).unwrap(); - println!("{}", &encrypted); let decrypted = internal_decrypt_string(password, &encrypted).unwrap(); assert_eq!(decrypted, plaintext); } + + #[test] + fn test_wrong_password_fails() { + let encrypted = + internal_encrypt_string("correct_password", "secret").unwrap(); + assert!(internal_decrypt_string("wrong_password", &encrypted).is_err()); + } + + #[test] + fn test_empty_string() { + let encrypted = internal_encrypt_string("pw", "").unwrap(); + let decrypted = internal_decrypt_string("pw", &encrypted).unwrap(); + assert_eq!(decrypted, ""); + } + + #[test] + fn test_unicode_content() { + let plaintext = "你好世界 🌍 émoji test"; + let encrypted = internal_encrypt_string("pw", plaintext).unwrap(); + let decrypted = internal_decrypt_string("pw", &encrypted).unwrap(); + assert_eq!(decrypted, plaintext); + } + + #[test] + fn test_encryption_produces_different_ciphertexts() { + let p1 = internal_encrypt_string("pw", "data").unwrap(); + let p2 = internal_encrypt_string("pw", "data").unwrap(); + // Same plaintext should produce different ciphertexts (random salt+nonce) + assert_ne!(p1, p2); + } + + #[test] + fn test_decrypt_corrupted_data_fails() { + let mut encrypted = internal_encrypt_string("pw", "data").unwrap(); + // Corrupt the base64 data by modifying a character + encrypted.push('X'); + assert!(internal_decrypt_string("pw", &encrypted).is_err()); + } } diff --git a/crates/core/src/utils/html.rs b/crates/core/src/utils/html.rs index 05395dc..6080dc4 100644 --- a/crates/core/src/utils/html.rs +++ b/crates/core/src/utils/html.rs @@ -45,3 +45,47 @@ pub fn extract_text(html: String) -> String { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extract_text_from_plain_html() { + let html = "

Hello World

".to_string(); + let text = extract_text(html); + assert!(text.contains("Hello World")); + } + + #[test] + fn extract_text_strips_tags() { + let html = "

Title

Paragraph with bold text.

".to_string(); + let text = extract_text(html); + assert!(text.contains("Title")); + assert!(text.contains("Paragraph")); + assert!(text.contains("bold")); + assert!(!text.contains("

")); + assert!(!text.contains("")); + } + + #[test] + fn extract_text_empty_string() { + let html = "".to_string(); + let text = extract_text(html); + assert!(text.is_empty()); + } + + #[test] + fn extract_text_plain_text_passthrough() { + let html = "Just some plain text without any HTML tags.".to_string(); + let text = extract_text(html); + assert!(text.contains("plain text")); + } + + #[test] + fn extract_text_with_links() { + let html = "Click here".to_string(); + let text = extract_text(html); + assert!(text.contains("Click here")); + } +} diff --git a/crates/core/src/utils/mod.rs b/crates/core/src/utils/mod.rs index 3399ef2..35ef0a6 100644 --- a/crates/core/src/utils/mod.rs +++ b/crates/core/src/utils/mod.rs @@ -351,3 +351,150 @@ pub fn compute_content_hash(content: &[u8]) -> String { let hash = blake3::hash(content); hash.to_hex().to_string() } + +#[cfg(test)] +mod tests { + use super::*; + + // ── hash ────────────────────────────────────────────────────────── + + #[test] + fn hash_is_deterministic() { + let a = hash("hello world"); + let b = hash("hello world"); + assert_eq!(a, b); + } + + #[test] + fn hash_different_inputs_produce_different_outputs() { + let a = hash("hello"); + let b = hash("world"); + assert_ne!(a, b); + } + + #[test] + fn hash_empty_string() { + let h = hash(""); + assert!(h < (1u64 << 53)); + } + + #[test] + fn hex_hash_is_deterministic() { + let a = hex_hash("test"); + let b = hex_hash("test"); + assert_eq!(a, b); + assert_eq!(a.len(), 32); + } + + #[test] + fn hex_hash_different_inputs_produce_different_outputs() { + assert_ne!(hex_hash("a"), hex_hash("b")); + } + + // ── create_hash / create_hash2 ──────────────────────────────────── + + #[test] + fn create_hash_deterministic() { + let a = create_hash(1, "INBOX"); + let b = create_hash(1, "INBOX"); + assert_eq!(a, b); + } + + #[test] + fn create_hash_different_accounts_differ() { + assert_ne!(create_hash(1, "INBOX"), create_hash(2, "INBOX")); + } + + #[test] + fn create_hash_different_fields_differ() { + assert_ne!(create_hash(1, "INBOX"), create_hash(1, "Sent")); + } + + #[test] + fn create_hash2_deterministic() { + let a = create_hash2(1, 100, "INBOX"); + let b = create_hash2(1, 100, "INBOX"); + assert_eq!(a, b); + } + + #[test] + fn create_hash2_different_inputs_differ() { + assert_ne!(create_hash2(1, 100, "INBOX"), create_hash2(1, 200, "INBOX")); + assert_ne!(create_hash2(1, 100, "INBOX"), create_hash2(1, 100, "Sent")); + } + + // ── compute_content_hash ────────────────────────────────────────── + + #[test] + fn compute_content_hash_deterministic() { + let data = b"test content"; + let a = compute_content_hash(data); + let b = compute_content_hash(data); + assert_eq!(a, b); + assert_eq!(a.len(), 64); + } + + #[test] + fn compute_content_hash_different_content_differ() { + assert_ne!(compute_content_hash(b"a"), compute_content_hash(b"b")); + } + + // ── validate_email ──────────────────────────────────────────────── + + #[test] + fn validate_email_valid() { + assert!(validate_email("user@example.com").is_ok()); + assert!(validate_email("a@b.co").is_ok()); + assert!(validate_email("test.user+tag@domain.com").is_ok()); + } + + #[test] + fn validate_email_invalid() { + assert!(validate_email("not-an-email").is_err()); + assert!(validate_email("").is_err()); + assert!(validate_email("@domain.com").is_err()); + assert!(validate_email("user@").is_err()); + } + + // ── generate_token_impl ─────────────────────────────────────────── + + #[test] + fn generate_token_has_expected_length() { + let token = generate_token_impl(256); + // URL-safe base64 encodes 3 bytes → 4 chars, so length is roughly + // ceil(bit_strength / 24) * 4, but chars like /+=-_ are replaced + assert!(!token.is_empty()); + } + + #[test] + fn generate_token_does_not_contain_special_chars() { + for _ in 0..10 { + let token = generate_token_impl(256); + assert!(!token.contains('/')); + assert!(!token.contains('+')); + assert!(!token.contains('-')); + assert!(!token.contains('_')); + } + } + + #[test] + fn generate_token_is_random() { + let a = generate_token_impl(256); + let b = generate_token_impl(256); + assert_ne!(a, b); + } + + // ── decode_avatar_bytes ─────────────────────────────────────────── + + #[test] + fn decode_avatar_bytes_valid() { + // "avatar" in base64 = "YXZhdGFy" + let result = decode_avatar_bytes("YXZhdGFy").unwrap(); + assert_eq!(result, b"avatar"); + } + + #[test] + fn decode_avatar_bytes_invalid_base64() { + assert!(decode_avatar_bytes("!!!invalid!!!").is_err()); + } +} diff --git a/crates/core/src/utils/net.rs b/crates/core/src/utils/net.rs index 31f8543..7249d51 100644 --- a/crates/core/src/utils/net.rs +++ b/crates/core/src/utils/net.rs @@ -111,7 +111,7 @@ async fn connect_with_optional_proxy( ) -> BichonResult { // Try if proxy is enabled if let Some(proxy_id) = use_proxy { - let proxy = Proxy::get(proxy_id).await?; + let proxy = Proxy::get(proxy_id)?; let proxy = parse_proxy_addr(&proxy.url)?; return timeout(TIMEOUT, Socks5Stream::connect(proxy, address)) .await diff --git a/crates/memdb/Cargo.toml b/crates/memdb/Cargo.toml new file mode 100644 index 0000000..c0700e2 --- /dev/null +++ b/crates/memdb/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "memdb" +version = "0.1.0" +edition = "2021" + +[dependencies] +serde.workspace = true +serde_json.workspace = true +thiserror = "=2.0.18" +tokio.workspace = true + +[dev-dependencies] +rand = "=0.9.2" +tempfile = "=3.27.0" diff --git a/crates/memdb/src/db.rs b/crates/memdb/src/db.rs new file mode 100644 index 0000000..7a2443b --- /dev/null +++ b/crates/memdb/src/db.rs @@ -0,0 +1,598 @@ +use crate::error::{DbError, Result}; +use crate::query::{Page, Paginated}; +use crate::wal::{self, WalEntry, WalOp}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::BTreeMap; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +// ─── Durability ────────────────────────────────────────────────────────── + +/// Controls when WAL data is fsynced to disk. +#[derive(Clone, Debug)] +pub enum Durability { + /// fsync every write — safest, ~250 ops/s. + Full, + /// Buffer up to `max_ops` writes, then fsync once. + /// Call `MemDb::flush()` before shutdown to commit any remaining + /// buffered writes. + Batch { max_ops: usize }, + /// Never fsync — fastest (~10k+ ops/s), zero durability. + /// Useful for ephemeral caches or when `snapshot()` handles persistence. + Off, +} + +impl Default for Durability { + fn default() -> Self { + Durability::Full + } +} + +impl Durability { + /// Convenience: batch up to `max_ops` writes per fsync. + pub fn batch(max_ops: usize) -> Self { + assert!(max_ops > 0); + Durability::Batch { max_ops } + } +} + +// ─── In-memory state ───────────────────────────────────────────────────── + +/// Complete in-memory state, used for deserializing snapshot.json. +#[derive(Deserialize, Default)] +struct Snapshot { + /// The highest WAL seq covered by this snapshot. + last_seq: u64, + /// All collection data. + data: BTreeMap>, +} + +/// Borrowed snapshot for zero-copy serialization — avoids cloning the +/// entire dataset when writing a snapshot file. +#[derive(Serialize)] +struct SnapshotRef<'a> { + last_seq: u64, + data: &'a BTreeMap>, +} + +/// Runtime state. All writes are serialized under this lock. +struct Inner { + last_seq: u64, + data: BTreeMap>, + /// Open WAL file handle, reused across writes. + wal_file: Option, + /// WAL file path, used when snapshot truncation needs to reopen the handle. + wal_path: PathBuf, + snapshot_path: PathBuf, + durability: Durability, + /// Buffered WAL entries not yet flushed to disk (Batch mode). + pending: Vec, + /// When the first entry was added to the current batch. + pending_since: Option, +} + +impl Inner { + /// Execute a batch of ops under the lock: allocate seq → apply to memory + /// → write WAL (fsync behaviour depends on Durability). + fn commit(&mut self, ops: Vec) -> Result { + if ops.is_empty() { + return Ok(0); + } + self.last_seq += 1; + let seq = self.last_seq; + let entry = WalEntry { + seq, + ops, + }; + // Always apply to memory first — clients can read their own writes + // immediately regardless of durability mode. + for op in &entry.ops { + apply_op(&mut self.data, op.clone()); + } + // WAL path depends on Durability. + match self.durability { + Durability::Full => { + if let Some(ref mut f) = self.wal_file { + wal::write_entry(f, &entry)?; + wal::sync_wal(f)?; + } + } + Durability::Batch { .. } => { + self.push_pending(entry); + if self.pending.len() >= self.batch_threshold() { + self.flush_pending()?; + } + } + Durability::Off => { + if let Some(ref mut f) = self.wal_file { + wal::write_entry(f, &entry)?; + } + } + } + Ok(seq) + } + + fn batch_threshold(&self) -> usize { + match self.durability { + Durability::Batch { max_ops } => max_ops, + _ => 0, + } + } + + fn push_pending(&mut self, entry: WalEntry) { + if self.pending.is_empty() { + self.pending_since = Some(Instant::now()); + } + self.pending.push(entry); + } + + /// Write all buffered entries to WAL and fsync once. + fn flush_pending(&mut self) -> Result { + let count = self.pending.len(); + if count == 0 { + return Ok(0); + } + if let Some(ref mut f) = self.wal_file { + for entry in &self.pending { + wal::write_entry(f, entry)?; + } + wal::sync_wal(f)?; + } + self.pending.clear(); + self.pending_since = None; + Ok(count) + } + +} + +fn apply_op(data: &mut BTreeMap>, op: WalOp) { + match op { + WalOp::Insert { collection, key, value } => { + data.entry(collection).or_default().insert(key, value); + } + WalOp::Upsert { collection, key, value } => { + data.entry(collection).or_default().insert(key, value); + } + WalOp::Delete { collection, key } => { + if let Some(col) = data.get_mut(&collection) { + col.remove(&key); + } + } + } +} + +// ─── MemDb ─────────────────────────────────────────────────────────────── + +#[derive(Clone)] +pub struct MemDb { + /// All writes (WAL + memory) are serialized under this lock. + inner: Arc>, +} + +impl MemDb { + /// Open the database with `Durability::Full` (backward-compatible). + pub fn open(data_dir: impl AsRef) -> Result { + Self::open_with(data_dir, Durability::Full) + } + + /// Open the database with a specific durability policy. + pub fn open_with(data_dir: impl AsRef, durability: Durability) -> Result { + let dir = data_dir.as_ref(); + std::fs::create_dir_all(dir)?; + + let snapshot_path = dir.join("snapshot.json"); + let wal_path = dir.join("wal.jsonl"); + + // 1. Read snapshot. + let mut snapshot = if snapshot_path.exists() { + let bytes = std::fs::read(&snapshot_path)?; + serde_json::from_slice::(&bytes)? + } else { + Snapshot::default() + }; + + let after_seq = snapshot.last_seq; + + // 2. Replay WAL entries with seq > last_seq. + let entries = wal::read_after(&wal_path, after_seq)?; + let replayed = entries.len(); + for entry in entries { + for op in entry.ops { + apply_op(&mut snapshot.data, op); + } + snapshot.last_seq = snapshot.last_seq.max(entry.seq); + } + + if replayed > 0 { + eprintln!("[memdb] replayed {replayed} WAL entries after seq={after_seq}"); + } + + // 3. Open WAL file handle for subsequent writes. + let wal_file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&wal_path)?; + + Ok(Self { + inner: Arc::new(Mutex::new(Inner { + last_seq: snapshot.last_seq, + data: snapshot.data, + wal_file: Some(wal_file), + wal_path, + snapshot_path, + durability, + pending: vec![], + pending_since: None, + })), + }) + } + + /// Pure in-memory mode (for tests, no persistence). + pub fn in_memory() -> Self { + Self { + inner: Arc::new(Mutex::new(Inner { + last_seq: 0, + data: BTreeMap::new(), + wal_file: None, + wal_path: PathBuf::from("/dev/null"), + snapshot_path: PathBuf::from("/dev/null"), + durability: Durability::Off, + pending: vec![], + pending_since: None, + })), + } + } + + /// Flush any buffered WAL entries to disk. + /// Important in `Durability::Batch` mode before shutdown — without this + /// call the last buffered batch may be lost on crash. + pub fn flush(&self) -> Result { + self.inner.lock().unwrap().flush_pending() + } + + /// Return the number of buffered entries not yet flushed to disk. + pub fn pending_writes(&self) -> usize { + self.inner.lock().unwrap().pending.len() + } + + /// Trigger a manual snapshot: + /// 1. Flush pending WAL entries (so crash recovery sees them). + /// 2. Read (last_seq, data) atomically under the lock. + /// 3. Write snapshot file outside the lock (non-blocking for writers). + /// 4. Atomic rename ensures snapshot file is never partial. + /// 5. Re-lock and truncate the WAL only when no writes raced in between. + pub fn snapshot(&self) -> Result<()> { + // Flush pending so every committed write is in the WAL before we + // potentially truncate it. + self.flush()?; + + // Serialize inside the lock — borrows data directly (zero-copy), + // then write to disk outside the lock so writers aren't blocked. + let (bytes, last_seq, snapshot_path) = { + let inner = self.inner.lock().unwrap(); + let path = inner.snapshot_path.clone(); + if path == Path::new("/dev/null") { + return Ok(()); + } + let snap = SnapshotRef { + last_seq: inner.last_seq, + data: &inner.data, + }; + (serde_json::to_vec_pretty(&snap)?, inner.last_seq, path) + }; + + // Write snapshot outside lock so writers are not blocked. + let tmp = snapshot_path.with_extension("tmp"); + std::fs::write(&tmp, bytes)?; + std::fs::rename(&tmp, &snapshot_path)?; + + // Re-lock. Only truncate the WAL if no writes have committed since + // the snapshot was taken. + let mut inner = self.inner.lock().unwrap(); + if inner.last_seq == last_seq { + if let Some(f) = inner.wal_file.take() { + drop(f); + // Truncate the file to zero. + std::fs::OpenOptions::new() + .write(true) + .truncate(true) + .open(&inner.wal_path)?; + // Reopen in append mode for future writes. + inner.wal_file = Some( + std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&inner.wal_path)?, + ); + } + } + + eprintln!("[memdb] snapshot saved at seq={last_seq}"); + Ok(()) + } + + /// Start a background snapshot worker that fires at the given interval. + pub fn start_snapshot_worker(&self, interval: Duration) -> tokio::task::JoinHandle<()> { + let db = self.clone(); + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + ticker.tick().await; // skip immediate first tick + loop { + ticker.tick().await; + if let Err(e) = db.snapshot() { + eprintln!("[memdb] snapshot error: {e}"); + } + } + }) + } + + /// Start a background flush worker for Batch durability mode. + /// Guarantees that buffered writes are flushed at least every `interval`. + pub fn start_flush_worker(&self, interval: Duration) -> tokio::task::JoinHandle<()> { + let db = self.clone(); + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + ticker.tick().await; + loop { + ticker.tick().await; + if let Err(e) = db.flush() { + eprintln!("[memdb] flush error: {e}"); + } + } + }) + } + + /// Get a handle to the named collection. + pub fn collection(&self, name: &'static str) -> Collection { + Collection { + db: self.clone(), + name, + } + } + + /// Begin a cross-collection atomic transaction. + pub fn transaction(&self) -> Transaction { + Transaction { + db: self.clone(), + ops: vec![], + } + } + + fn commit(&self, ops: Vec) -> Result<()> { + self.inner.lock().unwrap().commit(ops)?; + Ok(()) + } +} + +// ─── Collection ────────────────────────────────────────────────────────── + +#[derive(Clone)] +pub struct Collection { + db: MemDb, + name: &'static str, +} + +impl Collection { + // ── Write operations ────────────────────────────────────────────────── + + /// Insert a record. Returns `DuplicateKey` if the key already exists. + pub fn insert(&self, key: impl Into, value: &T) -> Result<()> { + let key = key.into(); + let value = serde_json::to_value(value)?; + let mut inner = self.db.inner.lock().unwrap(); + if inner.data.get(self.name).and_then(|c| c.get(&key)).is_some() { + return Err(DbError::DuplicateKey(self.name.to_string(), key)); + } + inner.commit(vec![WalOp::Insert { + collection: self.name.to_string(), + key, + value, + }])?; + Ok(()) + } + + /// Insert or overwrite a record. + pub fn upsert(&self, key: impl Into, value: &T) -> Result<()> { + let key = key.into(); + let value = serde_json::to_value(value)?; + self.db.commit(vec![WalOp::Upsert { + collection: self.name.to_string(), + key, + value, + }]) + } + + /// Read-modify-write, performed atomically under the lock. + /// Returns the updated value. + pub fn update(&self, key: impl Into, f: impl FnOnce(T) -> T) -> Result + where + T: Serialize + for<'de> Deserialize<'de>, + { + let key = key.into(); + let mut inner = self.db.inner.lock().unwrap(); + let old_val = inner + .data + .get(self.name) + .and_then(|c| c.get(&key)) + .ok_or_else(|| DbError::NotFound(self.name.to_string(), key.clone()))? + .clone(); + let old: T = serde_json::from_value(old_val)?; + let updated = f(old); + let new_val = serde_json::to_value(&updated)?; + inner.commit(vec![WalOp::Upsert { + collection: self.name.to_string(), + key, + value: new_val, + }])?; + Ok(updated) + } + + /// Delete a record. Returns whether the record existed. + pub fn delete(&self, key: impl Into) -> Result { + let key = key.into(); + let mut inner = self.db.inner.lock().unwrap(); + let existed = inner.data.get(self.name).and_then(|c| c.get(&key)).is_some(); + if existed { + inner.commit(vec![WalOp::Delete { + collection: self.name.to_string(), + key, + }])?; + } + Ok(existed) + } + + // ── Read operations (in-memory, no WAL involvement) ─────────────────── + + /// Look up a record by primary key. + pub fn get Deserialize<'de>>(&self, key: &str) -> Result> { + let inner = self.db.inner.lock().unwrap(); + match inner.data.get(self.name).and_then(|c| c.get(key)) { + Some(v) => Ok(Some(serde_json::from_value(v.clone())?)), + None => Ok(None), + } + } + + /// Look up a record by primary key; return an error if not found. + pub fn get_required Deserialize<'de>>(&self, key: &str) -> Result { + self.get(key)? + .ok_or_else(|| DbError::NotFound(self.name.to_string(), key.to_string())) + } + + /// Full scan with a predicate filter. + pub fn filter(&self, predicate: F) -> Result> + where + T: for<'de> Deserialize<'de>, + F: Fn(&T) -> bool, + { + let inner = self.db.inner.lock().unwrap(); + let Some(col) = inner.data.get(self.name) else { + return Ok(vec![]); + }; + let mut results = vec![]; + for v in col.values() { + let item: T = serde_json::from_value(v.clone())?; + if predicate(&item) { + results.push(item); + } + } + Ok(results) + } + + /// List all records in key order. + pub fn list_all Deserialize<'de>>(&self) -> Result> { + let inner = self.db.inner.lock().unwrap(); + let Some(col) = inner.data.get(self.name) else { + return Ok(vec![]); + }; + col.values() + .map(|v| serde_json::from_value(v.clone()).map_err(DbError::from)) + .collect() + } + + /// Prefix scan — returns records whose key starts with the given prefix, + /// in key order. + pub fn scan_prefix Deserialize<'de>>(&self, prefix: &str) -> Result> { + let inner = self.db.inner.lock().unwrap(); + let Some(col) = inner.data.get(self.name) else { + return Ok(vec![]); + }; + col.range(prefix.to_string()..) + .take_while(|(k, _)| k.starts_with(prefix)) + .map(|(_, v)| serde_json::from_value(v.clone()).map_err(DbError::from)) + .collect() + } + + /// Paginated query in key order. + pub fn paginate Deserialize<'de>>( + &self, + page: &Page, + desc: bool, + ) -> Result> { + let inner = self.db.inner.lock().unwrap(); + let Some(col) = inner.data.get(self.name) else { + return Ok(Paginated::empty(page)); + }; + let total = col.len() as u64; + let offset = page.offset(); + let items: Vec = if desc { + col.values() + .rev() + .skip(offset) + .take(page.page_size as usize) + .map(|v| serde_json::from_value(v.clone()).map_err(DbError::from)) + .collect::>()? + } else { + col.values() + .skip(offset) + .take(page.page_size as usize) + .map(|v| serde_json::from_value(v.clone()).map_err(DbError::from)) + .collect::>()? + }; + Ok(Paginated::new(page, total, items)) + } + + /// Return the number of records in this collection. + pub fn count(&self) -> usize { + let inner = self.db.inner.lock().unwrap(); + inner.data.get(self.name).map(|c| c.len()).unwrap_or(0) + } + + /// Check whether a key exists. + pub fn exists(&self, key: &str) -> bool { + let inner = self.db.inner.lock().unwrap(); + inner.data.get(self.name).and_then(|c| c.get(key)).is_some() + } +} + +// ─── Transaction ───────────────────────────────────────────────────────── + +pub struct Transaction { + db: MemDb, + ops: Vec, +} + +impl Transaction { + pub fn insert( + mut self, + collection: &str, + key: impl Into, + value: &T, + ) -> Result { + self.ops.push(WalOp::Insert { + collection: collection.to_string(), + key: key.into(), + value: serde_json::to_value(value)?, + }); + Ok(self) + } + + pub fn upsert( + mut self, + collection: &str, + key: impl Into, + value: &T, + ) -> Result { + self.ops.push(WalOp::Upsert { + collection: collection.to_string(), + key: key.into(), + value: serde_json::to_value(value)?, + }); + Ok(self) + } + + pub fn delete(mut self, collection: &str, key: impl Into) -> Self { + self.ops.push(WalOp::Delete { + collection: collection.to_string(), + key: key.into(), + }); + self + } + + /// Atomically commit all buffered ops as a single WAL entry. + pub fn commit(self) -> Result<()> { + self.db.commit(self.ops) + } +} diff --git a/crates/memdb/src/error.rs b/crates/memdb/src/error.rs new file mode 100644 index 0000000..35a88bd --- /dev/null +++ b/crates/memdb/src/error.rs @@ -0,0 +1,18 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum DbError { + #[error("io error: {0}")] + Io(#[from] std::io::Error), + + #[error("serialization error: {0}")] + Serde(#[from] serde_json::Error), + + #[error("record not found: collection={0}, key={1}")] + NotFound(String, String), + + #[error("duplicate key: collection={0}, key={1}")] + DuplicateKey(String, String), +} + +pub type Result = std::result::Result; diff --git a/crates/memdb/src/lib.rs b/crates/memdb/src/lib.rs new file mode 100644 index 0000000..c5fff25 --- /dev/null +++ b/crates/memdb/src/lib.rs @@ -0,0 +1,8 @@ +pub mod db; +pub mod error; +pub mod query; +pub mod wal; + +pub use db::{Collection, Durability, MemDb, Transaction}; +pub use error::DbError; +pub use query::{Page, Paginated}; diff --git a/crates/memdb/src/query.rs b/crates/memdb/src/query.rs new file mode 100644 index 0000000..ecb56bc --- /dev/null +++ b/crates/memdb/src/query.rs @@ -0,0 +1,48 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug)] +pub struct Page { + pub page: u64, + pub page_size: u64, +} + +impl Page { + pub fn new(page: u64, page_size: u64) -> Self { + assert!(page >= 1 && page_size >= 1, "page and page_size must be >= 1"); + Self { page, page_size } + } + + pub fn offset(&self) -> usize { + ((self.page - 1) * self.page_size) as usize + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Paginated { + pub page: u64, + pub page_size: u64, + pub total: u64, + pub total_pages: u64, + pub items: Vec, +} + +impl Paginated { + pub fn new(page: &Page, total: u64, items: Vec) -> Self { + let total_pages = if total == 0 { + 0 + } else { + (total + page.page_size - 1) / page.page_size + }; + Self { + page: page.page, + page_size: page.page_size, + total, + total_pages, + items, + } + } + + pub fn empty(page: &Page) -> Self { + Self::new(page, 0, vec![]) + } +} diff --git a/crates/memdb/src/wal.rs b/crates/memdb/src/wal.rs new file mode 100644 index 0000000..89f0974 --- /dev/null +++ b/crates/memdb/src/wal.rs @@ -0,0 +1,74 @@ +use crate::error::Result; +use serde::{Deserialize, Serialize}; +use std::io::{BufRead, Write}; +use std::path::Path; + +/// A single WAL operation. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "op")] +pub enum WalOp { + Insert { + collection: String, + key: String, + value: serde_json::Value, + }, + Upsert { + collection: String, + key: String, + value: serde_json::Value, + }, + Delete { + collection: String, + key: String, + }, +} + +/// A WAL entry carrying a monotonically increasing sequence number. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WalEntry { + pub seq: u64, + /// One entry may carry multiple ops (transaction batch). + pub ops: Vec, +} + +/// Write a WAL entry line to the file (no fsync — caller decides when to +/// sync for durability). +pub fn write_entry(file: &mut std::fs::File, entry: &WalEntry) -> Result<()> { + let line = serde_json::to_string(entry)?; + writeln!(file, "{}", line)?; + Ok(()) +} + +/// Force buffered WAL data to disk. +pub fn sync_wal(file: &std::fs::File) -> Result<()> { + file.sync_data()?; + Ok(()) +} + +/// Read all valid WAL entries whose seq > `after_seq`. +/// Corrupted lines (e.g. partial write after power loss) are skipped +/// with a warning and do not prevent startup. +pub fn read_after(path: &Path, after_seq: u64) -> Result> { + if !path.exists() { + return Ok(vec![]); + } + let file = std::fs::File::open(path)?; + let reader = std::io::BufReader::new(file); + let mut entries = vec![]; + for line in reader.lines() { + let line = line?; + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + match serde_json::from_str::(trimmed) { + Ok(entry) if entry.seq > after_seq => entries.push(entry), + // seq <= after_seq — already covered by snapshot, skip. + Ok(_) => {} + Err(e) => { + eprintln!("[wal] skipping corrupted entry: {e}"); + } + } + } + Ok(entries) +} diff --git a/crates/memdb/tests/integration.rs b/crates/memdb/tests/integration.rs new file mode 100644 index 0000000..ef0d586 --- /dev/null +++ b/crates/memdb/tests/integration.rs @@ -0,0 +1,743 @@ +use memdb::{DbError, MemDb, Page}; +use serde::{Deserialize, Serialize}; +use tempfile::TempDir; + +// ─── Test models ───────────────────────────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +struct Account { + id: String, + email: String, + status: String, + #[serde(default)] + role: String, +} + +impl Account { + fn new(id: &str, email: &str, status: &str) -> Self { + Self { + id: id.into(), + email: email.into(), + status: status.into(), + role: "user".into(), + } + } +} + +fn open_tmp() -> (MemDb, TempDir) { + let dir = tempfile::tempdir().unwrap(); + let db = MemDb::open(dir.path()).unwrap(); + (db, dir) +} + +// ─── Basic CRUD ────────────────────────────────────────────────────────── + +#[test] +fn test_insert_and_get() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + let acc = Account::new("1", "a@x.com", "active"); + col.insert("1", &acc).unwrap(); + let found: Option = col.get("1").unwrap(); + assert_eq!(found, Some(acc)); +} + +#[test] +fn test_get_missing_returns_none() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + let found: Option = col.get("nope").unwrap(); + assert!(found.is_none()); +} + +#[test] +fn test_get_required_missing_returns_error() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + let err = col.get_required::("nope").unwrap_err(); + assert!(matches!(err, DbError::NotFound(_, _))); +} + +#[test] +fn test_insert_duplicate_returns_error() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + let acc = Account::new("1", "a@x.com", "active"); + col.insert("1", &acc).unwrap(); + let err = col.insert("1", &acc).unwrap_err(); + assert!(matches!(err, DbError::DuplicateKey(_, _))); +} + +#[test] +fn test_upsert_creates_and_overwrites() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + let acc = Account::new("1", "a@x.com", "active"); + col.upsert("1", &acc).unwrap(); + let found: Account = col.get_required("1").unwrap(); + assert_eq!(found.status, "active"); + + let updated = Account::new("1", "a@x.com", "disabled"); + col.upsert("1", &updated).unwrap(); + let found: Account = col.get_required("1").unwrap(); + assert_eq!(found.status, "disabled"); +} + +#[test] +fn test_update_modifies_record() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + col.insert("1", &Account::new("1", "a@x.com", "active")) + .unwrap(); + let result: Account = col + .update("1", |mut a: Account| { + a.status = "disabled".into(); + a + }) + .unwrap(); + assert_eq!(result.status, "disabled"); + let found: Account = col.get_required("1").unwrap(); + assert_eq!(found.status, "disabled"); +} + +#[test] +fn test_update_missing_returns_error() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + let err = col.update("nope", |a: Account| a).unwrap_err(); + assert!(matches!(err, DbError::NotFound(_, _))); +} + +#[test] +fn test_delete_existing() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + col.insert("1", &Account::new("1", "a@x.com", "active")) + .unwrap(); + let deleted = col.delete("1").unwrap(); + assert!(deleted); + assert!(!col.exists("1")); +} + +#[test] +fn test_delete_missing_returns_false() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + let deleted = col.delete("nope").unwrap(); + assert!(!deleted); +} + +#[test] +fn test_exists() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + assert!(!col.exists("1")); + col.insert("1", &Account::new("1", "a@x.com", "active")) + .unwrap(); + assert!(col.exists("1")); +} + +#[test] +fn test_count() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + assert_eq!(col.count(), 0); + col.insert("1", &Account::new("1", "a@x.com", "active")) + .unwrap(); + col.insert("2", &Account::new("2", "b@x.com", "active")) + .unwrap(); + assert_eq!(col.count(), 2); + col.delete("1").unwrap(); + assert_eq!(col.count(), 1); +} + +// ─── Queries ───────────────────────────────────────────────────────────── + +#[test] +fn test_list_all() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + col.insert("1", &Account::new("1", "a@x.com", "active")) + .unwrap(); + col.insert("2", &Account::new("2", "b@x.com", "active")) + .unwrap(); + let all: Vec = col.list_all().unwrap(); + assert_eq!(all.len(), 2); +} + +#[test] +fn test_list_all_empty_collection() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + let all: Vec = col.list_all().unwrap(); + assert!(all.is_empty()); +} + +#[test] +fn test_filter() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + col.insert("1", &Account::new("1", "a@x.com", "active")) + .unwrap(); + col.insert("2", &Account::new("2", "b@x.com", "disabled")) + .unwrap(); + col.insert("3", &Account::new("3", "c@x.com", "active")) + .unwrap(); + + let active: Vec = col.filter(|a: &Account| a.status == "active").unwrap(); + assert_eq!(active.len(), 2); + + let disabled: Vec = col.filter(|a: &Account| a.status == "disabled").unwrap(); + assert_eq!(disabled.len(), 1); +} + +#[test] +fn test_filter_empty_collection() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + let result: Vec = col.filter(|_: &Account| true).unwrap(); + assert!(result.is_empty()); +} + +#[test] +fn test_scan_prefix() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + col.insert("2024_001", &Account::new("1", "a@x.com", "active")) + .unwrap(); + col.insert("2024_002", &Account::new("2", "b@x.com", "active")) + .unwrap(); + col.insert("2025_001", &Account::new("3", "c@x.com", "active")) + .unwrap(); + + let result: Vec = col.scan_prefix("2024_").unwrap(); + assert_eq!(result.len(), 2); + + let result: Vec = col.scan_prefix("2025_").unwrap(); + assert_eq!(result.len(), 1); + + let result: Vec = col.scan_prefix("9999_").unwrap(); + assert!(result.is_empty()); +} + +#[test] +fn test_paginate_asc() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + for i in 1..=5 { + col.insert( + format!("{:03}", i), + &Account::new(&i.to_string(), &format!("{}@x.com", i), "active"), + ) + .unwrap(); + } + let page = col.paginate::(&Page::new(1, 2), false).unwrap(); + assert_eq!(page.total, 5); + assert_eq!(page.total_pages, 3); + assert_eq!(page.items.len(), 2); + assert_eq!(page.items[0].id, "1"); + assert_eq!(page.items[1].id, "2"); + + let page2 = col.paginate::(&Page::new(2, 2), false).unwrap(); + assert_eq!(page2.items.len(), 2); + assert_eq!(page2.items[0].id, "3"); + + let page3 = col.paginate::(&Page::new(3, 2), false).unwrap(); + assert_eq!(page3.items.len(), 1); + assert_eq!(page3.items[0].id, "5"); +} + +#[test] +fn test_paginate_desc() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + for i in 1..=5 { + col.insert( + format!("{:03}", i), + &Account::new(&i.to_string(), &format!("{}@x.com", i), "active"), + ) + .unwrap(); + } + let page = col.paginate::(&Page::new(1, 2), true).unwrap(); + assert_eq!(page.items[0].id, "5"); + assert_eq!(page.items[1].id, "4"); +} + +#[test] +fn test_paginate_out_of_range() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + col.insert("1", &Account::new("1", "a@x.com", "active")) + .unwrap(); + let page = col.paginate::(&Page::new(99, 10), false).unwrap(); + assert_eq!(page.total, 1); + assert!(page.items.is_empty()); +} + +#[test] +fn test_paginate_empty_collection() { + let db = MemDb::in_memory(); + let col = db.collection("accounts"); + let page = col.paginate::(&Page::new(1, 10), false).unwrap(); + assert_eq!(page.total, 0); + assert_eq!(page.total_pages, 0); + assert!(page.items.is_empty()); +} + +// ─── Transactions ──────────────────────────────────────────────────────── + +#[test] +fn test_transaction_commit() { + let db = MemDb::in_memory(); + let acc1 = Account::new("1", "a@x.com", "active"); + let acc2 = Account::new("2", "b@x.com", "active"); + + db.transaction() + .upsert("accounts", "1", &acc1) + .unwrap() + .upsert("accounts", "2", &acc2) + .unwrap() + .commit() + .unwrap(); + + let col = db.collection("accounts"); + assert!(col.exists("1")); + assert!(col.exists("2")); +} + +#[test] +fn test_transaction_delete_across_collections() { + let db = MemDb::in_memory(); + let acc = Account::new("1", "a@x.com", "active"); + db.collection("accounts").insert("1", &acc).unwrap(); + db.collection("logs").insert("log-1", &acc).unwrap(); + + db.transaction() + .delete("accounts", "1") + .delete("logs", "log-1") + .commit() + .unwrap(); + + assert!(!db.collection("accounts").exists("1")); + assert!(!db.collection("logs").exists("log-1")); +} + +// ─── Collection isolation ──────────────────────────────────────────────── + +#[test] +fn test_collections_are_isolated() { + let db = MemDb::in_memory(); + let acc = Account::new("1", "a@x.com", "active"); + db.collection("accounts").insert("1", &acc).unwrap(); + + // Same key in different collections should not interfere. + assert!(db.collection("accounts").exists("1")); + assert!(!db.collection("users").exists("1")); +} + +// ─── WAL + Snapshot persistence ────────────────────────────────────────── + +#[test] +fn test_persist_and_recover() { + let dir = tempfile::tempdir().unwrap(); + + // Write data. + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("accounts"); + col.insert("1", &Account::new("1", "a@x.com", "active")) + .unwrap(); + col.insert("2", &Account::new("2", "b@x.com", "disabled")) + .unwrap(); + } + + // Recover after restart. + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("accounts"); + assert_eq!(col.count(), 2); + let acc: Account = col.get_required("1").unwrap(); + assert_eq!(acc.email, "a@x.com"); + } +} + +#[test] +fn test_recover_after_snapshot() { + let dir = tempfile::tempdir().unwrap(); + + // Write + snapshot. + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("accounts"); + col.insert("1", &Account::new("1", "a@x.com", "active")) + .unwrap(); + db.snapshot().unwrap(); + } + + // Recover from snapshot. + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("accounts"); + assert_eq!(col.count(), 1); + assert!(col.exists("1")); + } +} + +#[test] +fn test_recover_snapshot_plus_wal() { + let dir = tempfile::tempdir().unwrap(); + + // Write 3 records, snapshot, then write 2 more. + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("accounts"); + col.insert("1", &Account::new("1", "a@x.com", "active")) + .unwrap(); + col.insert("2", &Account::new("2", "b@x.com", "active")) + .unwrap(); + col.insert("3", &Account::new("3", "c@x.com", "active")) + .unwrap(); + db.snapshot().unwrap(); // WAL truncated, last_seq=3 + // Writes after snapshot — go into fresh WAL. + col.insert("4", &Account::new("4", "d@x.com", "active")) + .unwrap(); + col.insert("5", &Account::new("5", "e@x.com", "active")) + .unwrap(); + } + + // Recover: snapshot(seq=3) + replay WAL(seq=4,5). + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("accounts"); + assert_eq!(col.count(), 5); + for i in 1..=5 { + assert!(col.exists(&i.to_string())); + } + } +} + +#[test] +fn test_snapshot_does_not_duplicate_on_recovery() { + let dir = tempfile::tempdir().unwrap(); + + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("accounts"); + col.insert("1", &Account::new("1", "a@x.com", "active")) + .unwrap(); + col.insert("2", &Account::new("2", "b@x.com", "active")) + .unwrap(); + db.snapshot().unwrap(); + // Multiple snapshots should not duplicate data. + db.snapshot().unwrap(); + } + + { + let db = MemDb::open(dir.path()).unwrap(); + assert_eq!(db.collection("accounts").count(), 2); + } +} + +#[test] +fn test_wal_seq_skips_already_snapshotted_entries() { + let dir = tempfile::tempdir().unwrap(); + + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("accounts"); + col.insert("1", &Account::new("1", "a@x.com", "active")) + .unwrap(); // seq=1 + col.insert("2", &Account::new("2", "b@x.com", "active")) + .unwrap(); // seq=2 + db.snapshot().unwrap(); // snapshot last_seq=2, WAL truncated + + // Write after snapshot. + col.insert("3", &Account::new("3", "c@x.com", "active")) + .unwrap(); // seq=3 + } + + // Verify WAL only contains seq=3. + let wal_path = dir.path().join("wal.jsonl"); + let entries = memdb::wal::read_after(&wal_path, 2).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].seq, 3); + + // Data is complete after recovery. + { + let db = MemDb::open(dir.path()).unwrap(); + assert_eq!(db.collection("accounts").count(), 3); + } +} + +#[test] +fn test_delete_persisted_across_restart() { + let dir = tempfile::tempdir().unwrap(); + + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("accounts"); + col.insert("1", &Account::new("1", "a@x.com", "active")) + .unwrap(); + col.insert("2", &Account::new("2", "b@x.com", "active")) + .unwrap(); + col.delete("1").unwrap(); + } + + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("accounts"); + assert!(!col.exists("1")); + assert!(col.exists("2")); + } +} + +#[test] +fn test_update_persisted_across_restart() { + let dir = tempfile::tempdir().unwrap(); + + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("accounts"); + col.insert("1", &Account::new("1", "a@x.com", "active")) + .unwrap(); + col.update("1", |mut a: Account| { + a.status = "disabled".into(); + a + }) + .unwrap(); + } + + { + let db = MemDb::open(dir.path()).unwrap(); + let acc: Account = db.collection("accounts").get_required("1").unwrap(); + assert_eq!(acc.status, "disabled"); + } +} + +#[test] +fn test_transaction_persisted_across_restart() { + let dir = tempfile::tempdir().unwrap(); + + { + let db = MemDb::open(dir.path()).unwrap(); + let acc1 = Account::new("1", "a@x.com", "active"); + let acc2 = Account::new("2", "b@x.com", "active"); + db.transaction() + .upsert("accounts", "1", &acc1) + .unwrap() + .upsert("accounts", "2", &acc2) + .unwrap() + .delete("accounts", "nonexistent") + .commit() + .unwrap(); + } + + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("accounts"); + assert_eq!(col.count(), 2); + assert!(col.exists("1")); + assert!(col.exists("2")); + } +} + +// ─── Schema evolution ──────────────────────────────────────────────────── + +#[test] +fn test_schema_add_field_with_default() { + let (db, dir) = open_tmp(); + + // Write with old model. + #[derive(Serialize, Deserialize)] + struct AccountV1 { + id: String, + email: String, + } + db.collection("accounts") + .insert( + "1", + &AccountV1 { + id: "1".into(), + email: "a@x.com".into(), + }, + ) + .unwrap(); + drop(db); + + // New model adds a field with #[serde(default)]. + #[derive(Serialize, Deserialize, Debug)] + struct AccountV2 { + id: String, + email: String, + #[serde(default)] + role: String, + } + let db = MemDb::open(dir.path()).unwrap(); + let acc: AccountV2 = db.collection("accounts").get_required("1").unwrap(); + assert_eq!(acc.email, "a@x.com"); + assert_eq!(acc.role, ""); // default fills empty string +} + +#[test] +fn test_schema_remove_field() { + let (db, dir) = open_tmp(); + + // Write with old model that has a legacy field. + #[derive(Serialize, Deserialize)] + struct AccountWithLegacy { + id: String, + email: String, + legacy_field: String, + } + db.collection("accounts") + .insert( + "1", + &AccountWithLegacy { + id: "1".into(), + email: "a@x.com".into(), + legacy_field: "old_value".into(), + }, + ) + .unwrap(); + drop(db); + + // New model drops legacy_field — serde ignores unknown fields by default. + #[derive(Serialize, Deserialize, Debug)] + struct AccountV2 { + id: String, + email: String, + } + let db = MemDb::open(dir.path()).unwrap(); + let acc: AccountV2 = db.collection("accounts").get_required("1").unwrap(); + assert_eq!(acc.email, "a@x.com"); +} + +#[test] +fn test_schema_rename_field_with_alias() { + let (db, dir) = open_tmp(); + + #[derive(Serialize, Deserialize)] + struct AccountOld { + id: String, + username: String, + } + db.collection("accounts") + .insert( + "1", + &AccountOld { + id: "1".into(), + username: "alice".into(), + }, + ) + .unwrap(); + drop(db); + + // Field renamed; alias keeps backward compatibility with old data. + #[derive(Serialize, Deserialize, Debug)] + struct AccountNew { + id: String, + #[serde(alias = "username")] + display_name: String, + } + let db = MemDb::open(dir.path()).unwrap(); + let acc: AccountNew = db.collection("accounts").get_required("1").unwrap(); + assert_eq!(acc.display_name, "alice"); +} + +// ─── Concurrency safety ────────────────────────────────────────────────── + +#[tokio::test] +async fn test_concurrent_writes_no_data_loss() { + let db = MemDb::in_memory(); + let db = std::sync::Arc::new(db); + let mut handles = vec![]; + + for i in 0..100 { + let db = db.clone(); + handles.push(tokio::spawn(async move { + let col = db.collection("accounts"); + let acc = Account::new(&i.to_string(), &format!("{}@x.com", i), "active"); + // spawn_blocking because the Mutex may block briefly. + tokio::task::spawn_blocking(move || col.upsert(i.to_string(), &acc)) + .await + .unwrap() + .unwrap(); + })); + } + + for h in handles { + h.await.unwrap(); + } + + assert_eq!(db.collection("accounts").count(), 100); +} + +#[tokio::test] +async fn test_concurrent_writes_wal_seq_monotonic() { + let dir = tempfile::tempdir().unwrap(); + let db = std::sync::Arc::new(MemDb::open(dir.path()).unwrap()); + let mut handles = vec![]; + + for i in 0..50 { + let db = db.clone(); + handles.push(tokio::spawn(async move { + let col = db.collection("accounts"); + let acc = Account::new(&i.to_string(), &format!("{}@x.com", i), "active"); + tokio::task::spawn_blocking(move || col.upsert(i.to_string(), &acc)) + .await + .unwrap() + .unwrap(); + })); + } + + for h in handles { + h.await.unwrap(); + } + + // Verify WAL seq is strictly monotonic. + let wal_path = dir.path().join("wal.jsonl"); + let entries = memdb::wal::read_after(&wal_path, 0).unwrap(); + assert_eq!(entries.len(), 50); + let mut last = 0u64; + for e in &entries { + assert!(e.seq > last, "seq not monotonic: {} <= {}", e.seq, last); + last = e.seq; + } +} + +#[tokio::test] +async fn test_concurrent_snapshot_and_writes() { + let dir = tempfile::tempdir().unwrap(); + let db = std::sync::Arc::new(MemDb::open(dir.path()).unwrap()); + + // Concurrent writes + snapshots. + let db_write = db.clone(); + let write_handle = tokio::spawn(async move { + for i in 0..100 { + let col = db_write.collection("accounts"); + let acc = Account::new(&i.to_string(), &format!("{}@x.com", i), "active"); + tokio::task::spawn_blocking(move || col.upsert(i.to_string(), &acc)) + .await + .unwrap() + .unwrap(); + } + }); + + let db_snap = db.clone(); + let snap_handle = tokio::spawn(async move { + for _ in 0..5 { + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + db_snap.snapshot().unwrap(); + } + }); + + write_handle.await.unwrap(); + snap_handle.await.unwrap(); + + // Final snapshot to ensure everything is on disk. + db.snapshot().unwrap(); + + // Data is complete after recovery. + let db2 = MemDb::open(dir.path()).unwrap(); + assert_eq!(db2.collection("accounts").count(), 100); +} diff --git a/crates/memdb/tests/stress.rs b/crates/memdb/tests/stress.rs new file mode 100644 index 0000000..31702c8 --- /dev/null +++ b/crates/memdb/tests/stress.rs @@ -0,0 +1,1602 @@ +use memdb::{Durability, MemDb, Page}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +// ─── Test models ───────────────────────────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +struct Record { + id: String, + name: String, + amount: u64, + tags: Vec, + metadata: String, +} + +impl Record { + fn new(id: usize) -> Self { + let tag_count = (id % 5) as usize + 1; + Self { + id: format!("rec_{:06}", id), + name: format!("record_{}", id), + amount: (id * 7 % 1_000_000) as u64 + 1, + tags: (0..tag_count) + .map(|t| format!("tag_{:02}", (id + t) % 20)) + .collect(), + metadata: format!("meta data blob for record {}", id), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Counter { + value: u64, + updates: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct SimpleVal { + val: u64, +} + +#[derive(Debug, Serialize, Deserialize)] +struct Blob { + id: usize, + data: Vec, +} + +// ─── Helpers ───────────────────────────────────────────────────────────── + +fn report(name: &str, count: u64, elapsed_ms: u64) { + let ops_per_sec = if elapsed_ms > 0 { + count * 1000 / elapsed_ms + } else { + count + }; + println!(" [{name}] {count} ops in {elapsed_ms}ms → {ops_per_sec} ops/sec"); +} + +// ─── 1. Bulk insert performance ────────────────────────────────────────── + +#[test] +fn stress_bulk_insert_performance() { + let db = MemDb::in_memory(); + let col = db.collection("records"); + let n = 10_000u64; + + let start = Instant::now(); + for i in 0..n { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + let elapsed = start.elapsed().as_millis() as u64; + assert_eq!(col.count(), n as usize); + report("bulk insert", n, elapsed); +} + +// ─── 2. Bulk read performance ──────────────────────────────────────────── + +#[test] +fn stress_bulk_read_performance() { + let db = MemDb::in_memory(); + let col = db.collection("records"); + let n = 10_000u64; + + for i in 0..n { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + + let start = Instant::now(); + for i in 0..n { + let _found: Option = col.get(&format!("{:06}", i)).unwrap(); + } + let elapsed = start.elapsed().as_millis() as u64; + report("bulk read (by key)", n, elapsed); +} + +// ─── 3. Bulk update performance ────────────────────────────────────────── + +#[test] +fn stress_bulk_update_performance() { + let db = MemDb::in_memory(); + let col = db.collection("records"); + let n = 5_000u64; + + for i in 0..n { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + + let start = Instant::now(); + for i in 0..n { + let key = format!("{:06}", i); + col.update(&key, |mut r: Record| { + r.amount += 1; + r + }) + .unwrap(); + } + let elapsed = start.elapsed().as_millis() as u64; + report("bulk update", n, elapsed); + + let r: Record = col.get_required("000000").unwrap(); + assert!(r.amount > 0); +} + +// ─── 4. Prefix scan performance ────────────────────────────────────────── + +#[test] +fn stress_prefix_scan_performance() { + let db = MemDb::in_memory(); + let col = db.collection("events"); + for day in 1..=100 { + for seq in 1..=200 { + let id = format!("2026_{:03}_{:05}", day, seq); + let rec = Record::new((day * 1000 + seq) as usize); + col.insert(id, &rec).unwrap(); + } + } + + let start = Instant::now(); + let results: Vec = col.scan_prefix("2026_050_").unwrap(); + let elapsed = start.elapsed().as_millis() as u64; + assert_eq!(results.len(), 200); + report("prefix scan (200 of 20k)", 200, elapsed); + + let start = Instant::now(); + let results: Vec = col.scan_prefix("9999_").unwrap(); + let empty_ms = start.elapsed().as_millis() as u64; + assert!(results.is_empty()); + println!(" [prefix scan empty] in {empty_ms}ms"); +} + +// ─── 5. Pagination performance ─────────────────────────────────────────── + +#[test] +fn stress_pagination_performance() { + let db = MemDb::in_memory(); + let col = db.collection("records"); + let n = 10_000u64; + + for i in 0..n { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + + let start = Instant::now(); + let page_size = 50u64; + let total_pages = (n + page_size - 1) / page_size; + let mut total_items = 0u64; + for p in 1..=total_pages { + let page = col + .paginate::(&Page::new(p, page_size), false) + .unwrap(); + total_items += page.items.len() as u64; + } + let elapsed = start.elapsed().as_millis() as u64; + assert_eq!(total_items, n); + report(&format!("paginate {} pages", total_pages), n, elapsed); +} + +// ─── 6. Filter performance ─────────────────────────────────────────────── + +#[test] +fn stress_filter_performance() { + let db = MemDb::in_memory(); + let col = db.collection("records"); + let n = 10_000u64; + + for i in 0..n { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + + let start = Instant::now(); + let filtered: Vec = col + .filter(|r: &Record| r.amount > 500_000 && r.tags.contains(&"tag_05".to_string())) + .unwrap(); + let elapsed = start.elapsed().as_millis() as u64; + println!( + " [filter] matched {}/{} in {}ms", + filtered.len(), + n, + elapsed + ); +} + +// ─── 7. Concurrent writes ──────────────────────────────────────────────── + +#[tokio::test] +async fn stress_concurrent_writes_no_data_loss() { + let db = Arc::new(MemDb::in_memory()); + let n_writers = 8u64; + let n_per_writer = 1_250u64; + let total = n_writers * n_per_writer; + + let start = Instant::now(); + let mut handles = vec![]; + for w in 0..n_writers { + let db = db.clone(); + handles.push(tokio::task::spawn_blocking(move || { + for i in 0..n_per_writer { + let id = format!("w{:02}_{:06}", w, i); + let rec = Record::new((w * 10000 + i) as usize); + db.collection("records").upsert(id, &rec).unwrap(); + } + })); + } + for h in handles { + h.await.unwrap(); + } + let elapsed = start.elapsed().as_millis() as u64; + + let count = db.collection("records").count() as u64; + assert_eq!(count, total); + report("concurrent writes (8×1250)", total, elapsed); +} + +// ─── 8. Concurrent reads and writes ────────────────────────────────────── + +#[tokio::test] +async fn stress_concurrent_read_write() { + let db = Arc::new(MemDb::in_memory()); + let n = 5_000u64; + + for i in 0..n { + db.collection("records") + .insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + + let start = Instant::now(); + + let db_w = db.clone(); + let write_handle = tokio::task::spawn_blocking(move || { + for i in n..n + 2_000 { + let id = format!("{:06}", i); + let rec = Record::new(i as usize); + db_w.collection("records").upsert(id, &rec).unwrap(); + } + }); + + let db_r = db.clone(); + let read_handle = tokio::task::spawn_blocking(move || { + for i in 0..2_000u64 { + let key = format!("{:06}", i % n); + let _: Option = db_r.collection("records").get(&key).unwrap(); + } + }); + + let (wr, rr) = tokio::join!(write_handle, read_handle); + wr.unwrap(); + rr.unwrap(); + let elapsed = start.elapsed().as_millis() as u64; + + let count = db.collection("records").count() as u64; + assert_eq!(count, n + 2_000); + report("concurrent r/w (2k+2k)", 4_000, elapsed); +} + +// ─── 9. Snapshot with concurrent writes (data integrity) ───────────────── + +#[tokio::test] +async fn stress_snapshot_with_concurrent_writes() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().to_path_buf(); + let n_writes = 500u64; + + { + let db = Arc::new(MemDb::open(&db_path).unwrap()); + + let db_w = db.clone(); + let write_handle = tokio::task::spawn_blocking(move || { + for i in 0..n_writes { + let rec = Record::new(i as usize); + db_w + .collection("records") + .upsert(format!("{:06}", i), &rec) + .unwrap(); + } + }); + + let db_s = db.clone(); + let snap_handle = tokio::spawn(async move { + for _ in 0..10 { + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + db_s.snapshot().unwrap(); + } + }); + + write_handle.await.unwrap(); + snap_handle.await.unwrap(); + db.snapshot().unwrap(); + } + + // Recover and verify. + { + let db = MemDb::open(&db_path).unwrap(); + let col = db.collection("records"); + let count = col.count() as u64; + println!(" [snapshot stress] wrote {n_writes}, recovered {count}"); + assert_eq!( + count, n_writes, + "data loss: wrote {n_writes} but recovered {count}" + ); + + for i in 0..n_writes { + let key = format!("{:06}", i); + assert!(col.exists(&key), "missing record: {key}"); + } + } +} + +// ─── 10. Large dataset recovery ───────────────────────────────────────── + +#[test] +fn stress_recover_large_dataset() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().to_path_buf(); + let n = 3_000u64; + + { + let db = MemDb::open(&db_path).unwrap(); + let col = db.collection("records"); + let start = Instant::now(); + for i in 0..n { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + let elapsed = start.elapsed().as_millis() as u64; + report("persistent insert", n, elapsed); + db.snapshot().unwrap(); + } + + { + let start = Instant::now(); + let db = MemDb::open(&db_path).unwrap(); + let elapsed = start.elapsed().as_millis() as u64; + println!(" [recovery] {n} records loaded in {elapsed}ms"); + + let col = db.collection("records"); + assert_eq!(col.count() as u64, n); + + for i in 0..n { + let key = format!("{:06}", i); + assert!(col.exists(&key), "missing after recovery: {key}"); + } + } +} + +// ─── 11. Transaction batch performance ─────────────────────────────────── + +#[test] +fn stress_transaction_batch_performance() { + let db = MemDb::in_memory(); + let batch_size = 100usize; + let batches = 100usize; + + let start = Instant::now(); + for b in 0..batches { + let mut txn = db.transaction(); + for i in 0..batch_size { + let id = format!("b{:03}_{:04}", b, i); + let rec = Record::new(b * batch_size + i); + txn = txn.upsert("records", &id, &rec).unwrap(); + } + txn.commit().unwrap(); + } + let elapsed = start.elapsed().as_millis() as u64; + let total = (batches * batch_size) as u64; + + let count = db.collection("records").count() as u64; + assert_eq!(count, total); + report("txn batch (100×100)", total, elapsed); +} + +// ─── 12. Cross-collection transactions ─────────────────────────────────── + +#[test] +fn stress_cross_collection_transaction() { + let db = MemDb::in_memory(); + let n = 1_000u64; + + let start = Instant::now(); + for i in 0..n { + let rec = Record::new(i as usize); + db.transaction() + .upsert("records", format!("{:06}", i), &rec) + .unwrap() + .upsert("audit_log", format!("log_{:06}", i), &rec) + .unwrap() + .commit() + .unwrap(); + } + let elapsed = start.elapsed().as_millis() as u64; + + assert_eq!(db.collection("records").count() as u64, n); + assert_eq!(db.collection("audit_log").count() as u64, n); + report("cross-collection txn", n, elapsed); +} + +// ─── 13. Update-heavy workload ─────────────────────────────────────────── + +#[test] +fn stress_update_heavy_workload() { + let db = MemDb::in_memory(); + let col = db.collection("counters"); + let n = 1_000u64; + + for i in 0..n { + col.insert( + format!("counter_{:04}", i), + &Counter { + value: 0, + updates: 0, + }, + ) + .unwrap(); + } + + let start = Instant::now(); + let rounds = 10u64; + for _ in 0..rounds { + for i in 0..n { + let key = format!("counter_{:04}", i); + let _ = col + .update::(&key, |mut c| { + c.value += 1; + c.updates += 1; + c + }) + .unwrap(); + } + } + let elapsed = start.elapsed().as_millis() as u64; + let total_ops = rounds * n; + + let all: Vec = col.list_all().unwrap(); + let total_value: u64 = all.iter().map(|c| c.value).sum(); + let total_updates: u64 = all.iter().map(|c| c.updates).sum(); + assert_eq!(total_value, total_updates); + report("update-heavy (10×1000)", total_ops, elapsed); + println!(" [update] total value={total_value}, total updates={total_updates}"); +} + +// ─── 14. Mass delete ───────────────────────────────────────────────────── + +#[test] +fn stress_mass_delete() { + let db = MemDb::in_memory(); + let col = db.collection("records"); + let n = 5_000u64; + + for i in 0..n { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + assert_eq!(col.count() as u64, n); + + let start = Instant::now(); + for i in 0..n / 2 { + let deleted = col.delete(format!("{:06}", i)).unwrap(); + assert!(deleted); + } + let elapsed = start.elapsed().as_millis() as u64; + assert_eq!(col.count() as u64, n / 2); + report("delete 2500 of 5000", n / 2, elapsed); +} + +// ─── 15. Pagination edge cases ─────────────────────────────────────────── + +#[test] +fn stress_pagination_edge_cases() { + let db = MemDb::in_memory(); + let col = db.collection("records"); + let n = 100u64; + + for i in 0..n { + col.insert(format!("{:06}", i), &SimpleVal { val: i }).unwrap(); + } + + // Single-item page. + let p1 = col + .paginate::(&Page::new(1, 1), false) + .unwrap(); + assert_eq!(p1.items.len(), 1); + assert_eq!(p1.total_pages, 100); + + // Out of range. + let p2 = col + .paginate::(&Page::new(999, 50), false) + .unwrap(); + assert!(p2.items.is_empty()); + assert_eq!(p2.total, 100); + + // Out of range (descending). + let p3 = col + .paginate::(&Page::new(999, 50), true) + .unwrap(); + assert!(p3.items.is_empty()); + + // First page descending. + let p4 = col + .paginate::(&Page::new(1, 3), true) + .unwrap(); + assert_eq!(p4.items.len(), 3); + + // Last page ascending. + let p5 = col + .paginate::(&Page::new(34, 3), false) + .unwrap(); + assert_eq!(p5.items.len(), 1); + assert_eq!(p5.total_pages, 34); +} + +// ─── 16. WAL seq monotonicity under load ───────────────────────────────── + +#[test] +fn stress_wal_seq_monotonic_under_load() { + let dir = tempfile::tempdir().unwrap(); + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("records"); + let n = 2_000; + + for i in 0..n { + col.upsert(format!("{:06}", i), &Record::new(i)).unwrap(); + } + + let wal_path = dir.path().join("wal.jsonl"); + let entries = memdb::wal::read_after(&wal_path, 0).unwrap(); + assert_eq!(entries.len(), n); + let mut last = 0u64; + for e in &entries { + assert!(e.seq > last, "seq not monotonic: {} <= {}", e.seq, last); + last = e.seq; + } + println!(" [wal seq] {n} entries, seq monotonic verified"); +} + +// ─── 17. Snapshot truncation safety ───────────────────────────────────── + +#[test] +fn stress_snapshot_truncate_safety() { + let dir = tempfile::tempdir().unwrap(); + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("records"); + + col.insert("a", &Record::new(1)).unwrap(); + db.snapshot().unwrap(); // seq=1 snapshotted, WAL truncated + + col.insert("b", &Record::new(2)).unwrap(); // seq=2 must survive + + assert_eq!(col.count(), 2); + + drop(db); + let db = MemDb::open(dir.path()).unwrap(); + assert_eq!( + db.collection("records").count(), + 2, + "data loss: record b missing after snapshot+truncate+recovery" + ); + println!(" [truncate safety] both records survived snapshot+truncate+recovery"); +} + +// ─── 18. Multi-collection isolation ────────────────────────────────────── + +#[test] +fn stress_multi_collection_isolation() { + let db = MemDb::in_memory(); + let n = 2_000; + + let collections = ["users", "orders", "products", "sessions", "audit_log"]; + for (i, col_name) in collections.iter().enumerate() { + let col = db.collection(col_name); + let base_id = i * 10_000; + for j in 0..n { + let id = format!("{:06}", base_id + j); + col.insert(&id, &Record::new(j)).unwrap(); + } + } + + for col_name in collections { + assert_eq!( + db.collection(col_name).count(), + n, + "collection {col_name} count mismatch" + ); + } + println!(" [isolation] 5×2000 records, all counts verified"); +} + +// ─── 19. Large value read/write ────────────────────────────────────────── + +#[test] +fn stress_large_value_read_write() { + let db = MemDb::in_memory(); + let col = db.collection("blobs"); + + let blob_size = 64 * 1024; // 64 KiB + let n = 100u64; + + let start = Instant::now(); + for i in 0..n { + let blob = Blob { + id: i as usize, + data: vec![(i % 256) as u8; blob_size], + }; + col.insert(format!("blob_{:04}", i), &blob).unwrap(); + } + let write_ms = start.elapsed().as_millis() as u64; + report(&format!("large value write ({}×64KB)", n), n, write_ms); + + let start = Instant::now(); + for i in 0..n { + let blob: Blob = col.get_required(&format!("blob_{:04}", i)).unwrap(); + assert_eq!(blob.id, i as usize); + assert_eq!(blob.data.len(), blob_size); + } + let read_ms = start.elapsed().as_millis() as u64; + report(&format!("large value read ({}×64KB)", n), n, read_ms); +} + +// ─── 20. Realistic scenario: chat messages ─────────────────────────────── + +#[test] +fn stress_realistic_chat_messages() { + let db = MemDb::in_memory(); + + #[derive(Serialize, Deserialize)] + struct Message { + room_id: String, + sender: String, + text: String, + ts: u64, + } + + let users = ["alice", "bob", "charlie", "diana", "eve"]; + let rooms = ["general", "random", "dev", "ops"]; + let n_messages = 5_000u64; + + let start = Instant::now(); + for i in 0..n_messages { + let room = rooms[i as usize % rooms.len()]; + let user = users[i as usize % users.len()]; + let msg = Message { + room_id: room.to_string(), + sender: user.to_string(), + text: format!("message number {} from {} in {}", i, user, room), + ts: 1700000000 + i, + }; + + // One collection per chat room. + let col = db.collection(room); + col.insert(format!("msg_{:06}", i), &msg).unwrap(); + } + let write_elapsed = start.elapsed().as_millis() as u64; + report("chat insert (5k msgs)", n_messages, write_elapsed); + + // Each room should have messages. + for room in rooms { + let col = db.collection(room); + assert!(col.count() > 0, "room {room} should have messages"); + } + + // Prefix-scan messages in one room. + let start = Instant::now(); + let general_msgs: Vec = + db.collection("general").scan_prefix("msg_000").unwrap(); + let scan_elapsed = start.elapsed().as_millis() as u64; + assert!(!general_msgs.is_empty()); + report("chat prefix scan", general_msgs.len() as u64, scan_elapsed); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// File-backed WAL stress tests +// These exercise real disk I/O — fsync, recovery, WAL growth, snapshot +// interleaving — at scale. Every test below uses MemDb::open(), not +// in_memory(). +// ═══════════════════════════════════════════════════════════════════════════ + +// ─── 21. Bulk persistent write (WAL + fsync cost) ──────────────────────── + +#[test] +fn wal_bulk_insert_fsync_cost() { + let dir = tempfile::tempdir().unwrap(); + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("records"); + let n = 3_000u64; + + let start = Instant::now(); + for i in 0..n { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + let elapsed = start.elapsed().as_millis() as u64; + + assert_eq!(col.count(), n as usize); + report("WAL insert (fsync per op)", n, elapsed); + + // Verify WAL file exists and has content. + let wal_path = dir.path().join("wal.jsonl"); + let wal_size = std::fs::metadata(&wal_path).unwrap().len(); + println!( + " [wal file] {} ops → {} KiB ({:.1} bytes/op)", + n, + wal_size / 1024, + wal_size as f64 / n as f64 + ); + + // Recover and verify. + drop(db); + let db = MemDb::open(dir.path()).unwrap(); + assert_eq!(db.collection("records").count(), n as usize); +} + +// ─── 22. WAL recovery from large dataset (no snapshot) ─────────────────── + +#[test] +fn wal_recover_pure_wal_no_snapshot() { + let dir = tempfile::tempdir().unwrap(); + let n = 5_000u64; + + // Write everything — never snapshot, so recovery must replay every entry. + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("records"); + let start = Instant::now(); + for i in 0..n { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + let elapsed = start.elapsed().as_millis() as u64; + report("WAL write (no snapshot)", n, elapsed); + } + + // Recover from pure WAL replay. + { + let start = Instant::now(); + let db = MemDb::open(dir.path()).unwrap(); + let elapsed = start.elapsed().as_millis() as u64; + report("WAL replay recovery", n, elapsed); + + let col = db.collection("records"); + assert_eq!(col.count(), n as usize); + + // Spot-check random keys across the whole range. + for i in (0..n).step_by(500) { + let key = format!("{:06}", i); + let rec: Record = col.get_required(&key).unwrap(); + assert_eq!(rec.id, format!("rec_{:06}", i as usize)); + } + } +} + +// ─── 23. Sustained write throughput over time ──────────────────────────── + +#[test] +fn wal_sustained_write_throughput() { + let dir = tempfile::tempdir().unwrap(); + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("records"); + let rounds = 5u64; + let per_round = 1_000u64; + + for r in 0..rounds { + let base = r * per_round; + let start = Instant::now(); + for i in 0..per_round { + let idx = base + i; + col.upsert(format!("{:06}", idx), &Record::new(idx as usize)) + .unwrap(); + } + let elapsed = start.elapsed().as_millis() as u64; + let wal_size = std::fs::metadata(dir.path().join("wal.jsonl")) + .unwrap() + .len(); + println!( + " [round {}] {} ops in {}ms → {} ops/sec | WAL {} KiB", + r + 1, + per_round, + elapsed, + per_round * 1000 / elapsed.max(1), + wal_size / 1024, + ); + } + + assert_eq!(col.count(), (rounds * per_round) as usize); +} + +// ─── 24. Concurrent persistent writes ──────────────────────────────────── + +#[tokio::test] +async fn wal_concurrent_persistent_writes() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().to_path_buf(); + let db = Arc::new(MemDb::open(&db_path).unwrap()); + let n_writers = 8u64; + let per_writer = 500u64; + let total = n_writers * per_writer; + + let start = Instant::now(); + let mut handles = vec![]; + for w in 0..n_writers { + let db = db.clone(); + handles.push(tokio::task::spawn_blocking(move || { + for i in 0..per_writer { + let id = format!("w{:02}_{:06}", w, i); + let rec = Record::new((w * 10000 + i) as usize); + db.collection("records").upsert(id, &rec).unwrap(); + } + })); + } + for h in handles { + h.await.unwrap(); + } + let elapsed = start.elapsed().as_millis() as u64; + + let count = db.collection("records").count() as u64; + assert_eq!(count, total); + report("WAL concurrent writes (8×500)", total, elapsed); + + // Verify strict seq ordering in WAL under concurrent load. + let wal_path = db_path.join("wal.jsonl"); + let entries = memdb::wal::read_after(&wal_path, 0).unwrap(); + assert_eq!(entries.len(), total as usize); + let mut last = 0u64; + for e in &entries { + assert!(e.seq > last, "seq not monotonic under concurrency"); + last = e.seq; + } + + // Recover. + drop(db); + let db = MemDb::open(&db_path).unwrap(); + assert_eq!(db.collection("records").count() as u64, total); +} + +// ─── 25. Snapshot interleaved with sustained writes ────────────────────── + +#[tokio::test] +async fn wal_snapshot_interleaved_heavy_writes() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().to_path_buf(); + let db = Arc::new(MemDb::open(&db_path).unwrap()); + let n = 2_000u64; + + let start = Instant::now(); + let db_w = db.clone(); + let write_handle = tokio::task::spawn_blocking(move || { + for i in 0..n { + let rec = Record::new(i as usize); + db_w + .collection("records") + .upsert(format!("{:06}", i), &rec) + .unwrap(); + } + }); + + let db_s = db.clone(); + let snap_handle = tokio::spawn(async move { + for _ in 0..20 { + tokio::time::sleep(std::time::Duration::from_millis(2)).await; + if let Err(e) = db_s.snapshot() { + eprintln!(" [snapshot] error: {e}"); + } + } + }); + + write_handle.await.unwrap(); + snap_handle.await.unwrap(); + db.snapshot().unwrap(); + let elapsed = start.elapsed().as_millis() as u64; + + let count = db.collection("records").count() as u64; + assert_eq!(count, n); + report("WAL snapshot+write interleaved", n, elapsed); + + // Recover and verify ALL records survived every snapshot+truncate cycle. + drop(db); + let db = MemDb::open(&db_path).unwrap(); + let col = db.collection("records"); + assert_eq!(col.count() as u64, n, "data loss during snapshot interleaving"); + for i in 0..n { + assert!(col.exists(&format!("{:06}", i)), "missing record {i}"); + } +} + +// ─── 26. Crash recovery simulation: kill without snapshot ──────────────── + +#[test] +fn wal_crash_recovery_no_snapshot() { + let dir = tempfile::tempdir().unwrap(); + let n = 2_000u64; + + // Simulate normal operation, then "crash" (just drop the db). + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("records"); + for i in 0..n { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + // No snapshot — crash! (db dropped without clean shutdown) + } + + // Recover — all data must be intact from WAL alone. + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("records"); + assert_eq!( + col.count() as u64, + n, + "crash recovery lost data (no snapshot)" + ); + // Verify a mix of early, middle, and late records. + for &i in &[0, 1, n / 2, n - 2, n - 1] { + let rec: Record = col.get_required(&format!("{:06}", i)).unwrap(); + assert_eq!(rec.name, format!("record_{}", i as usize)); + } + } +} + +// ─── 27. Mixed read/write with WAL ─────────────────────────────────────── + +#[tokio::test] +async fn wal_mixed_read_write_workload() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().to_path_buf(); + + // Pre-populate. + { + let db = MemDb::open(&db_path).unwrap(); + let col = db.collection("records"); + for i in 0..2_000u64 { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + } + + let db = Arc::new(MemDb::open(&db_path).unwrap()); + let start = Instant::now(); + + // Writer: append new records. + let db_w = db.clone(); + let write_handle = tokio::task::spawn_blocking(move || { + for i in 2_000..4_000u64 { + let rec = Record::new(i as usize); + db_w + .collection("records") + .upsert(format!("{:06}", i), &rec) + .unwrap(); + } + }); + + // Reader: random reads across existing range. + let db_r = db.clone(); + let read_handle = tokio::task::spawn_blocking(move || { + for i in 0..5_000u64 { + let key = format!("{:06}", i % 2500); + let _: Option = db_r.collection("records").get(&key).unwrap(); + } + }); + + // Snapshotter: periodic snapshots during the workload. + let db_s = db.clone(); + let snap_handle = tokio::spawn(async move { + for _ in 0..8 { + tokio::time::sleep(std::time::Duration::from_millis(3)).await; + let _ = db_s.snapshot(); + } + }); + + let (wr, rr, sr) = tokio::join!(write_handle, read_handle, snap_handle); + wr.unwrap(); + rr.unwrap(); + sr.unwrap(); + db.snapshot().unwrap(); + let elapsed = start.elapsed().as_millis() as u64; + + let count = db.collection("records").count() as u64; + assert_eq!(count, 4_000); + report("WAL mixed r/w/snapshot", 7_000, elapsed); + + // Final recovery check. + drop(db); + let db = MemDb::open(&db_path).unwrap(); + assert_eq!(db.collection("records").count() as u64, 4_000); +} + +// ─── 28. WAL behaviour: delete + insert same key ───────────────────────── + +#[test] +fn wal_delete_insert_same_key_replay() { + let dir = tempfile::tempdir().unwrap(); + + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("records"); + + col.insert("x", &Record::new(1)).unwrap(); // seq=1: insert + col.delete("x").unwrap(); // seq=2: delete + col.insert("x", &Record::new(3)).unwrap(); // seq=3: insert again + // Final state: key "x" exists with record_3 data. + let rec: Record = col.get_required("x").unwrap(); + assert_eq!(rec.name, "record_3"); + } + + // Recover and verify final state is preserved. + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("records"); + assert_eq!(col.count(), 1); + let rec: Record = col.get_required("x").unwrap(); + assert_eq!(rec.name, "record_3"); + } +} + +// ─── 29. WAL with cross-collection transactions ────────────────────────── + +#[test] +fn wal_cross_collection_txn_replay() { + let dir = tempfile::tempdir().unwrap(); + let n = 500u64; + + { + let db = MemDb::open(dir.path()).unwrap(); + for i in 0..n { + let rec = Record::new(i as usize); + db.transaction() + .upsert("alpha", format!("a_{:06}", i), &rec) + .unwrap() + .upsert("beta", format!("b_{:06}", i), &rec) + .unwrap() + .commit() + .unwrap(); + } + assert_eq!(db.collection("alpha").count(), n as usize); + assert_eq!(db.collection("beta").count(), n as usize); + } + + // Recover and verify both collections. + { + let db = MemDb::open(dir.path()).unwrap(); + assert_eq!(db.collection("alpha").count(), n as usize); + assert_eq!(db.collection("beta").count(), n as usize); + + // Spot-check: each collection's records should match. + let a: Record = db.collection("alpha").get_required("a_000123").unwrap(); + let b: Record = db.collection("beta").get_required("b_000123").unwrap(); + assert_eq!(a.name, "record_123"); + assert_eq!(b.name, "record_123"); + } +} + +// ─── 30. Snapshot then immediate crash — verify no data loss ───────────── + +#[test] +fn wal_snapshot_then_crash_recovery() { + let dir = tempfile::tempdir().unwrap(); + + // Phase 1: write batch A, snapshot, write batch B, snapshot, crash. + let snapshot_at: u64; + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("records"); + + // Batch A. + for i in 0..500u64 { + col.insert(format!("a_{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + db.snapshot().unwrap(); + + // Batch B. + for i in 0..500u64 { + col.insert(format!("b_{:06}", i), &Record::new(500 + i as usize)) + .unwrap(); + } + snapshot_at = col.count() as u64; // 1000 + db.snapshot().unwrap(); + + // Batch C — no snapshot after this (simulates crash). + for i in 0..500u64 { + col.insert(format!("c_{:06}", i), &Record::new(1000 + i as usize)) + .unwrap(); + } + } // crash + + // Phase 2: recover. Batches A+B must survive (snapshotted). + // Batch C must also survive (in WAL). + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("records"); + let count = col.count() as u64; + println!( + " [snapshot+cold crash] before crash={snapshot_at}, recovered={count}" + ); + assert_eq!(count, 1_500, "data loss across snapshot boundaries"); + + // Verify records from all three batches. + assert!(col.exists("a_000000")); + assert!(col.exists("a_000499")); + assert!(col.exists("b_000000")); + assert!(col.exists("b_000499")); + assert!(col.exists("c_000000")); + assert!(col.exists("c_000499")); + } +} + +// ─── 31. Large batch transaction with WAL ──────────────────────────────── + +#[test] +fn wal_large_transaction_batch() { + let dir = tempfile::tempdir().unwrap(); + let db = MemDb::open(dir.path()).unwrap(); + let n = 1_000u64; + + // Build one big transaction. + let start = Instant::now(); + let mut txn = db.transaction(); + for i in 0..n { + let rec = Record::new(i as usize); + txn = txn + .upsert("records", format!("{:06}", i), &rec) + .unwrap(); + } + txn.commit().unwrap(); + let elapsed = start.elapsed().as_millis() as u64; + + assert_eq!(db.collection("records").count(), n as usize); + report(&format!("WAL large txn ({} ops)", n), n, elapsed); + + // The entire transaction should be a single WAL entry. + let wal_path = dir.path().join("wal.jsonl"); + let entries = memdb::wal::read_after(&wal_path, 0).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].ops.len(), n as usize); + + // Recover and verify. + drop(db); + let db = MemDb::open(dir.path()).unwrap(); + assert_eq!(db.collection("records").count(), n as usize); +} + +// ─── 32. Update-heavy persistent workload ──────────────────────────────── + +#[test] +fn wal_update_heavy_persistent() { + let dir = tempfile::tempdir().unwrap(); + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("counters"); + let n = 500u64; + + // Seed counters. + for i in 0..n { + col.insert( + format!("c_{:04}", i), + &Counter { + value: 0, + updates: 0, + }, + ) + .unwrap(); + } + + let rounds = 5u64; + let start = Instant::now(); + for _ in 0..rounds { + for i in 0..n { + col.update::(&format!("c_{:04}", i), |mut c| { + c.value += 1; + c.updates += 1; + c + }) + .unwrap(); + } + } + let elapsed = start.elapsed().as_millis() as u64; + let total = rounds * n; + report("WAL update-heavy", total, elapsed); + + // Verify in-memory state. + let all: Vec = col.list_all().unwrap(); + let sum_v: u64 = all.iter().map(|c| c.value).sum(); + let sum_u: u64 = all.iter().map(|c| c.updates).sum(); + assert_eq!(sum_v, sum_u); + assert_eq!(sum_v, n * rounds); + + // Recover and verify again. + drop(db); + let db = MemDb::open(dir.path()).unwrap(); + let all: Vec = db.collection("counters").list_all().unwrap(); + assert_eq!(all.len(), n as usize); + let sum_v: u64 = all.iter().map(|c| c.value).sum(); + assert_eq!(sum_v, n * rounds); +} + +// ─── 33. WAL seq gaps do not affect recovery ───────────────────────────── + +#[test] +fn wal_seq_gaps_on_recovery() { + let dir = tempfile::tempdir().unwrap(); + + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("records"); + // Insert some records. + for i in 1..=5u64 { + col.insert(format!("k{}", i), &Record::new(i as usize)) + .unwrap(); + } + // Snapshot captures seq=5. + db.snapshot().unwrap(); + // More writes after snapshot — seq continues 6, 7, 8... + for i in 6..=10u64 { + col.insert(format!("k{}", i), &Record::new(i as usize)) + .unwrap(); + } + assert_eq!(col.count(), 10); + } + + // Recover. Seq 6-10 must replay from WAL on top of snapshot (seq=5). + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("records"); + assert_eq!(col.count(), 10); + for i in 1..=10u64 { + let rec: Record = col.get_required(&format!("k{}", i)).unwrap(); + assert_eq!(rec.name, format!("record_{}", i as usize)); + } + } +} + +// ─── 34. Stress: many small snapshots during continuous writes ─────────── + +#[tokio::test] +async fn wal_many_small_snapshots() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().to_path_buf(); + let db = Arc::new(MemDb::open(&db_path).unwrap()); + let n = 1_000u64; + + let start = Instant::now(); + let db_w = db.clone(); + let write_handle = tokio::task::spawn_blocking(move || { + for i in 0..n { + let rec = Record::new(i as usize); + db_w.collection("ticks") + .upsert(format!("{:06}", i), &rec) + .unwrap(); + } + }); + + let db_s = db.clone(); + let snap_handle = tokio::spawn(async move { + for _ in 0..50 { + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + let _ = db_s.snapshot(); + } + }); + + let (wr, sr) = tokio::join!(write_handle, snap_handle); + wr.unwrap(); + sr.unwrap(); + db.snapshot().unwrap(); + let elapsed = start.elapsed().as_millis() as u64; + + let count = db.collection("ticks").count() as u64; + assert_eq!(count, n); + report("WAL 50 snapshots + 1k writes", n, elapsed); + + // Recover. + drop(db); + let db = MemDb::open(&db_path).unwrap(); + assert_eq!(db.collection("ticks").count() as u64, n); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Durability mode comparison: Full vs Batch vs Off +// ═══════════════════════════════════════════════════════════════════════════ + +// ─── 35. Batch mode bulk insert performance ────────────────────────────── + +#[test] +fn durability_batch_bulk_insert_perf() { + let dir = tempfile::tempdir().unwrap(); + let db = MemDb::open_with(dir.path(), Durability::batch(100)).unwrap(); + let col = db.collection("records"); + let n = 3_000u64; + + let start = Instant::now(); + for i in 0..n { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + // Flush the final partial batch. + let _flushed = db.flush().unwrap(); + let elapsed = start.elapsed().as_millis() as u64; + + let wal_size = std::fs::metadata(dir.path().join("wal.jsonl")) + .unwrap() + .len(); + // 3000 ops / 100 batch = ~30 fsyncs (vs 3000 in Full mode) + let expected_syncs = (n + 99) / 100 + 1; // +1 for final flush + assert_eq!(col.count(), n as usize); + report("Batch-100 insert", n, elapsed); + println!( + " WAL {:.0} KiB, ~{expected_syncs} fsyncs (vs {n} in Full mode)", + wal_size as f64 / 1024.0, + ); + + // Recover — flush ensures everything is on disk. + drop(db); + let db = MemDb::open(dir.path()).unwrap(); + assert_eq!(db.collection("records").count(), n as usize); +} + +// ─── 36. Batch mode: concurrent writes ─────────────────────────────────── + +#[tokio::test] +async fn durability_batch_concurrent_writes() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().to_path_buf(); + let db = Arc::new(MemDb::open_with(&db_path, Durability::batch(50)).unwrap()); + let n_writers = 4u64; + let per_writer = 500u64; + let total = n_writers * per_writer; + + let start = Instant::now(); + let mut handles = vec![]; + for w in 0..n_writers { + let db = db.clone(); + handles.push(tokio::task::spawn_blocking(move || { + for i in 0..per_writer { + let id = format!("w{:02}_{:06}", w, i); + let rec = Record::new((w * 10000 + i) as usize); + db.collection("records").upsert(id, &rec).unwrap(); + } + })); + } + for h in handles { + h.await.unwrap(); + } + db.flush().unwrap(); + let elapsed = start.elapsed().as_millis() as u64; + + assert_eq!(db.collection("records").count() as u64, total); + report("Batch-50 concurrent (4×500)", total, elapsed); + + drop(db); + let db = MemDb::open(&db_path).unwrap(); + assert_eq!(db.collection("records").count() as u64, total); +} + +// ─── 37. Batch mode: crash recovery of batched writes ──────────────────── + +#[test] +fn durability_batch_crash_recovery() { + let dir = tempfile::tempdir().unwrap(); + + { + let db = MemDb::open_with(dir.path(), Durability::batch(20)).unwrap(); + let col = db.collection("records"); + + // Write 95 records — triggers 4 batches of 20 + 15 buffered. + for i in 0..95u64 { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + // Do NOT flush — last 15 are buffered, not yet on disk. + } // "crash" + + // Recover: 80 flushed records (4 batches × 20) should survive. + // The 15 buffered records are lost (expected behaviour). + { + let db = MemDb::open(dir.path()).unwrap(); + let col = db.collection("records"); + let count = col.count() as u64; + println!(" [batch crash] wrote 95, no flush, recovered {count}"); + assert!(count >= 80, "at least 4 batches should survive"); + assert!(count < 95, "unflushed records should be lost on crash"); + } +} + +// ─── 38. Batch mode: flush worker ensures eventual durability ──────────── + +#[tokio::test] +async fn durability_batch_flush_worker() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().to_path_buf(); + let db = Arc::new(MemDb::open_with(&db_path, Durability::batch(200)).unwrap()); + + // Start a flush worker that fires every 50ms. + let _flush_handle = db.start_flush_worker(Duration::from_millis(50)); + + let db_w = db.clone(); + let handle = tokio::task::spawn_blocking(move || { + for i in 0..500u64 { + db_w.collection("records") + .upsert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + }); + + // Meanwhile, the flush worker periodically commits buffered writes. + handle.await.unwrap(); + // Give the flush worker a moment to catch up. + tokio::time::sleep(Duration::from_millis(100)).await; + db.flush().unwrap(); + + assert_eq!(db.collection("records").count(), 500); + drop(db); + + // All records should survive because the flush worker (and final flush) + // pushed them to disk. + let db = MemDb::open(&db_path).unwrap(); + assert_eq!(db.collection("records").count(), 500); +} + +// ─── 39. Durability::Off mode — maximum throughput, zero fsync ─────────── + +#[test] +fn durability_off_bulk_insert_perf() { + let dir = tempfile::tempdir().unwrap(); + let db = MemDb::open_with(dir.path(), Durability::Off).unwrap(); + let col = db.collection("records"); + let n = 5_000u64; + + let start = Instant::now(); + for i in 0..n { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + let elapsed = start.elapsed().as_millis() as u64; + + assert_eq!(col.count(), n as usize); + report("Off (no fsync) insert", n, elapsed); + + // Data is written but never synced. On crash, recovery may lose data. + drop(db); + let db = MemDb::open(dir.path()).unwrap(); + // OS may have flushed some pages — count what survived. + let recovered = db.collection("records").count(); + println!(" [no fsync] wrote {n}, OS flushed {recovered} (may be 0 on crash)"); +} + +// ─── 40. Full vs Batch vs Off side-by-side comparison ──────────────────── + +#[test] +fn durability_full_vs_batch_vs_off() { + let n = 1_000u64; + + // Full. + { + let dir = tempfile::tempdir().unwrap(); + let db = MemDb::open_with(dir.path(), Durability::Full).unwrap(); + let col = db.collection("records"); + let start = Instant::now(); + for i in 0..n { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + let elapsed = start.elapsed().as_millis() as u64; + report("Full (fsync every op)", n, elapsed); + } + + // Batch 100. + { + let dir = tempfile::tempdir().unwrap(); + let db = MemDb::open_with(dir.path(), Durability::batch(100)).unwrap(); + let col = db.collection("records"); + let start = Instant::now(); + for i in 0..n { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + db.flush().unwrap(); + let elapsed = start.elapsed().as_millis() as u64; + report("Batch-100 (1 fsync/100)", n, elapsed); + } + + // Batch 10. + { + let dir = tempfile::tempdir().unwrap(); + let db = MemDb::open_with(dir.path(), Durability::batch(10)).unwrap(); + let col = db.collection("records"); + let start = Instant::now(); + for i in 0..n { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + db.flush().unwrap(); + let elapsed = start.elapsed().as_millis() as u64; + report("Batch-10 (1 fsync/10)", n, elapsed); + } + + // Off. + { + let dir = tempfile::tempdir().unwrap(); + let db = MemDb::open_with(dir.path(), Durability::Off).unwrap(); + let col = db.collection("records"); + let start = Instant::now(); + for i in 0..n { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + let elapsed = start.elapsed().as_millis() as u64; + report("Off (never fsync)", n, elapsed); + } +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Concurrent read scalability — how well does the Mutex hold up under +// pure read load at various thread counts? +// ═══════════════════════════════════════════════════════════════════════════ + +#[tokio::test] +async fn bench_concurrent_read_scaling() { + use rand::Rng; + + let db = Arc::new(MemDb::in_memory()); + let n_records = 50_000u64; + let reads_per_thread = 20_000u64; + + // Pre-populate. + { + let col = db.collection("records"); + for i in 0..n_records { + col.insert(format!("{:06}", i), &Record::new(i as usize)) + .unwrap(); + } + } + println!( + "\n dataset: {n_records} records, each thread does {reads_per_thread} random get() calls\n" + ); + + // Baseline: single-threaded. + let start = Instant::now(); + let mut rng = rand::rng(); + let col = db.collection("records"); + for _ in 0..reads_per_thread { + let key = format!("{:06}", rng.random_range(0..n_records)); + let _: Option = col.get(&key).unwrap(); + } + let single_ms = start.elapsed().as_millis() as u64; + let single_ops = reads_per_thread * 1000 / single_ms.max(1); + println!(" [1 thread ] {reads_per_thread} reads in {single_ms}ms → {single_ops} ops/sec"); + + // Multi-threaded: 2, 4, 8, 16 threads. + for &n_threads in &[2, 4, 8, 16] { + let start = Instant::now(); + let mut handles = vec![]; + for _t in 0..n_threads { + let db = db.clone(); + handles.push(tokio::task::spawn_blocking(move || { + let col = db.collection("records"); + let mut rng = rand::rng(); + let mut found = 0u64; + for _ in 0..reads_per_thread { + let key = format!("{:06}", rng.random_range(0..n_records)); + let _: Option = col.get(&key).unwrap(); + found += 1; + } + found + })); + } + + let mut total_found = 0u64; + for h in handles { + total_found += h.await.unwrap(); + } + let elapsed_ms = start.elapsed().as_millis() as u64; + let total_reads = n_threads * reads_per_thread; + let total_ops = total_reads * 1000 / elapsed_ms.max(1); + let speedup = total_ops as f64 / single_ops as f64; + println!( + " [{n_threads:>2} threads] {total_reads} reads in {elapsed_ms}ms → {total_ops} ops/sec (×{speedup:.2})" + ); + assert_eq!(total_found, total_reads); + } +} diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index e70e32e..171a0ce 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -30,3 +30,7 @@ tokio.workspace = true http.workspace = true urlencoding.workspace = true mimalloc.workspace = true + +[dev-dependencies] +poem = { version = "3.1.12", features = ["test"] } +tempfile = "3" diff --git a/crates/server/src/common/auth.rs b/crates/server/src/common/auth.rs index bf74e5d..3c2dcff 100644 --- a/crates/server/src/common/auth.rs +++ b/crates/server/src/common/auth.rs @@ -106,11 +106,9 @@ pub async fn extract_client_context(req: &Request) -> Result { })?; // Validate and update access token - let user = AccessTokenModel::resolve_user_from_token(&token) - .await - .map_err(|e| { - create_api_error_response(&format!("{:#?}", e), ErrorCode::PermissionDenied) - })?; + let user = AccessTokenModel::resolve_user_from_token(&token).map_err(|e| { + create_api_error_response(&format!("{:#?}", e), ErrorCode::PermissionDenied) + })?; return Ok(ClientContext { ip_addr: Some(ip_addr), diff --git a/crates/server/src/common/validator.rs b/crates/server/src/common/validator.rs index 2bc82a1..06e5fde 100644 --- a/crates/server/src/common/validator.rs +++ b/crates/server/src/common/validator.rs @@ -41,3 +41,33 @@ impl Validator for EmailValidator { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn valid_email_passes() { + let validator = EmailValidator; + assert!(validator.check(&"user@example.com".to_string())); + assert!(validator.check(&"a@b.co".to_string())); + assert!(validator.check(&"test.user+tag@domain.com".to_string())); + } + + #[test] + fn invalid_email_fails() { + let validator = EmailValidator; + assert!(!validator.check(&"not-an-email".to_string())); + assert!(!validator.check(&"".to_string())); + assert!(!validator.check(&"@domain.com".to_string())); + assert!(!validator.check(&"user@".to_string())); + } + + #[test] + fn display_message() { + assert_eq!( + EmailValidator.to_string(), + "Not a valid email address" + ); + } +} diff --git a/crates/server/src/error/code.rs b/crates/server/src/error/code.rs index a153d0b..a01ae41 100644 --- a/crates/server/src/error/code.rs +++ b/crates/server/src/error/code.rs @@ -53,3 +53,106 @@ impl IntoStatusCode for ErrorCode { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn invalid_parameter_is_bad_request() { + assert_eq!( + ErrorCode::InvalidParameter.status(), + StatusCode::BAD_REQUEST + ); + } + + #[test] + fn permission_denied_is_unauthorized() { + assert_eq!( + ErrorCode::PermissionDenied.status(), + StatusCode::UNAUTHORIZED + ); + } + + #[test] + fn forbidden_is_forbidden() { + assert_eq!(ErrorCode::Forbidden.status(), StatusCode::FORBIDDEN); + } + + #[test] + fn resource_not_found_is_not_found() { + assert_eq!( + ErrorCode::ResourceNotFound.status(), + StatusCode::NOT_FOUND + ); + } + + #[test] + fn internal_error_is_internal_server_error() { + assert_eq!( + ErrorCode::InternalError.status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + } + + #[test] + fn too_many_request_is_429() { + assert_eq!( + ErrorCode::TooManyRequest.status(), + StatusCode::TOO_MANY_REQUESTS + ); + } + + #[test] + fn already_exists_is_conflict() { + assert_eq!(ErrorCode::AlreadyExists.status(), StatusCode::CONFLICT); + } + + #[test] + fn method_not_allowed_is_405() { + assert_eq!( + ErrorCode::MethodNotAllowed.status(), + StatusCode::METHOD_NOT_ALLOWED + ); + } + + #[test] + fn every_error_code_maps_to_valid_status() { + // Ensure all variants produce a status code in the 4xx or 5xx range + let codes = [ + ErrorCode::InvalidParameter, + ErrorCode::MissingConfiguration, + ErrorCode::Incompatible, + ErrorCode::PermissionDenied, + ErrorCode::AccountDisabled, + ErrorCode::OAuth2ItemDisabled, + ErrorCode::Forbidden, + ErrorCode::ResourceNotFound, + ErrorCode::RequestTimeout, + ErrorCode::PayloadTooLarge, + ErrorCode::TooManyRequest, + ErrorCode::AlreadyExists, + ErrorCode::InternalError, + ErrorCode::AutoconfigFetchFailed, + ErrorCode::ImapCommandFailed, + ErrorCode::ImapUnexpectedResult, + ErrorCode::HttpResponseError, + ErrorCode::ImapAuthenticationFailed, + ErrorCode::MissingRefreshToken, + ErrorCode::NetworkError, + ErrorCode::ConnectionTimeout, + ErrorCode::ConnectionPoolTimeout, + ErrorCode::UnhandledPoemError, + ErrorCode::MethodNotAllowed, + ]; + for code in &codes { + let status = code.status(); + assert!( + status.is_client_error() || status.is_server_error(), + "{:?} should map to 4xx or 5xx, got {}", + code, + status + ); + } + } +} diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index cb1de30..922388f 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -139,3 +139,78 @@ async fn initialize() -> BichonResult<()> { LazyLock::force(&ATTACHMENT_MANAGER); Ok(()) } + +#[cfg(test)] +mod tests; + +#[cfg(test)] +mod api_tests { + use super::rest::api::create_openapi_service; + use poem::test::TestClient; + + #[tokio::test] + async fn openapi_spec_json_is_served() { + let api_service = create_openapi_service(); + let spec_endpoint = api_service.spec_endpoint(); + let cli = TestClient::new(spec_endpoint); + + let resp = cli.get("/").send().await; + resp.assert_status_is_ok(); + + let body = resp.json().await; + let obj = body.value().object(); + assert!(obj.get_opt("openapi").is_some(), "missing openapi version"); + assert!(obj.get_opt("info").is_some(), "missing info section"); + assert!(obj.get_opt("paths").is_some(), "missing paths section"); + } + + #[tokio::test] + async fn openapi_spec_yaml_is_served() { + let api_service = create_openapi_service(); + let spec_endpoint = api_service.spec_endpoint_yaml(); + let cli = TestClient::new(spec_endpoint); + + let resp = cli.get("/").send().await; + resp.assert_status_is_ok(); + } + + #[tokio::test] + async fn swagger_ui_is_served() { + let api_service = create_openapi_service(); + let swagger = api_service.swagger_ui(); + let cli = TestClient::new(swagger); + + let resp = cli.get("/").send().await; + resp.assert_status_is_ok(); + } + + #[tokio::test] + async fn openapi_spec_lists_all_tag_groups() { + let api_service = create_openapi_service(); + let spec_endpoint = api_service.spec_endpoint(); + let cli = TestClient::new(spec_endpoint); + + let resp = cli.get("/").send().await; + let body = resp.json().await; + let value = body.value(); + + let tag_names: Vec<&str> = value + .object() + .get("tags") + .array() + .iter() + .map(|v| v.object().get("name").string()) + .collect(); + + assert!(tag_names.contains(&"AccessToken"), "missing AccessToken tag"); + assert!(tag_names.contains(&"Attachment"), "missing Attachment tag"); + assert!(tag_names.contains(&"AutoConfig"), "missing AutoConfig tag"); + assert!(tag_names.contains(&"Account"), "missing Account tag"); + assert!(tag_names.contains(&"System"), "missing System tag"); + assert!(tag_names.contains(&"Mailbox"), "missing Mailbox tag"); + assert!(tag_names.contains(&"OAuth2"), "missing OAuth2 tag"); + assert!(tag_names.contains(&"Message"), "missing Message tag"); + assert!(tag_names.contains(&"Import"), "missing Import tag"); + assert!(tag_names.contains(&"Users"), "missing Users tag"); + } +} diff --git a/crates/server/src/rest/api/access_token.rs b/crates/server/src/rest/api/access_token.rs index 7c3ed6f..d8fd6fd 100644 --- a/crates/server/src/rest/api/access_token.rs +++ b/crates/server/src/rest/api/access_token.rs @@ -38,11 +38,9 @@ impl AccessTokenApi { &self, context: WrappedContext, ) -> ApiResult>> { - context - .require_permission(None, Permission::TOKEN_MANAGE) - .await?; + context.require_permission(None, Permission::TOKEN_MANAGE)?; - Ok(Json(AccessTokenModel::list_all_api_tokens().await?)) + Ok(Json(AccessTokenModel::list_all_api_tokens()?)) } /// Deletes a specific access token. @@ -58,14 +56,12 @@ impl AccessTokenApi { context: WrappedContext, ) -> ApiResult<()> { let token = token.0.trim(); - let token = AccessTokenModel::get_token(token).await?; + let token = AccessTokenModel::get_token(token)?; if context.user.id != token.user_id { - context - .require_permission(None, Permission::TOKEN_MANAGE) - .await?; + context.require_permission(None, Permission::TOKEN_MANAGE)?; } - Ok(AccessTokenModel::delete(&token.token).await?) + Ok(AccessTokenModel::delete(&token.token)?) } /// Creates a new api token. @@ -83,12 +79,10 @@ impl AccessTokenApi { let current_user_id = context.user.id; let target_user_id = payload.0.user_id.unwrap_or(current_user_id); if target_user_id != current_user_id { - context - .require_permission(None, Permission::USER_MANAGE) - .await?; + context.require_permission(None, Permission::USER_MANAGE)?; } - let token_string = AccessTokenModel::create_api_token(target_user_id, payload.0).await?; + let token_string = AccessTokenModel::create_api_token(target_user_id, payload.0)?; Ok(PlainText(token_string)) } } diff --git a/crates/server/src/rest/api/account.rs b/crates/server/src/rest/api/account.rs index c9b1960..414cc8e 100644 --- a/crates/server/src/rest/api/account.rs +++ b/crates/server/src/rest/api/account.rs @@ -56,10 +56,8 @@ impl AccountApi { context: WrappedContext, ) -> ApiResult> { let account_id = account_id.0; - context - .require_permission(Some(account_id), Permission::ACCOUNT_READ_DETAILS) - .await?; - Ok(Json(AccountModel::async_get(account_id).await?)) + context.require_permission(Some(account_id), Permission::ACCOUNT_READ_DETAILS)?; + Ok(Json(AccountModel::get(account_id)?)) } /// Delete an account by ID - WARNING: This permanently removes the account and all associated resources @@ -75,10 +73,9 @@ impl AccountApi { context: WrappedContext, ) -> ApiResult<()> { let account_id = account_id.0; - context - .require_permission(Some(account_id), Permission::ACCOUNT_MANAGE) - .await?; - Ok(AccountModel::delete(account_id).await?) + context.require_permission(Some(account_id), Permission::ACCOUNT_MANAGE)?; + AccountModel::delete(account_id).await?; + Ok(()) } /// Create a new account @@ -89,9 +86,7 @@ impl AccountApi { payload: Json, context: WrappedContext, ) -> ApiResult> { - context - .require_permission(None, Permission::ACCOUNT_CREATE) - .await?; + context.require_permission(None, Permission::ACCOUNT_CREATE)?; let account = AccountModel::create_account(context.user.id, payload.0).await?; Ok(Json(account)) } @@ -111,10 +106,8 @@ impl AccountApi { context: WrappedContext, ) -> ApiResult<()> { let account_id = account_id.0; - context - .require_permission(Some(account_id), Permission::ACCOUNT_MANAGE) - .await?; - Ok(AccountModel::update(account_id, payload.0, true).await?) + context.require_permission(Some(account_id), Permission::ACCOUNT_MANAGE)?; + Ok(AccountModel::update(account_id, payload.0, true)?) } /// List accounts with optional pagination parameters @@ -129,16 +122,15 @@ impl AccountApi { desc: Query>, context: WrappedContext, ) -> ApiResult>> { - let is_admin = context.user.is_admin().await; + let is_admin = context.user.is_admin(); let sort_desc = desc.0.unwrap_or(true); - let user_map: HashMap = UserModel::list_all() - .await? + let user_map: HashMap = UserModel::list_all()? .into_iter() .map(|u| (u.id, u)) .collect(); let page_data: DataPage = if is_admin { - AccountModel::paginate_list(page.0, page_size.0, desc.0).await? + AccountModel::paginate_list(page.0, page_size.0, desc.0)? } else { let authorized_ids: HashSet = context.user.account_access_map.keys().cloned().collect(); @@ -153,8 +145,7 @@ impl AccountApi { })); } - let mut accounts: Vec = AccountModel::list_all() - .await? + let mut accounts: Vec = AccountModel::list_all()? .into_iter() .filter(|acct| authorized_ids.contains(&acct.id)) .collect(); @@ -198,11 +189,9 @@ impl AccountApi { context: WrappedContext, ) -> ApiResult> { let account_id = account_id.0; - AccountModel::check_account_exists(account_id).await?; - context - .require_permission(Some(account_id), Permission::ACCOUNT_READ_DETAILS) - .await?; - let state = DownloadState::get(account_id).await?; + AccountModel::check_account_exists(account_id)?; + context.require_permission(Some(account_id), Permission::ACCOUNT_READ_DETAILS)?; + let state = DownloadState::get(account_id)?; let state = state.unwrap_or(DownloadState::empty(account_id)); Ok(Json(state)) } @@ -220,16 +209,14 @@ impl AccountApi { context: WrappedContext, ) -> ApiResult<()> { let account_id = account_id.0; - let account = AccountModel::check_account_exists(account_id).await?; + let account = AccountModel::check_account_exists(account_id)?; if !matches!(account.account_type, AccountType::IMAP) { return Err(raise_error!( format!("Manual download is not supported for '{:#?}' accounts. Only IMAP accounts are supported.", account.account_type), ErrorCode::InvalidParameter ))?; } - context - .require_permission(Some(account_id), Permission::ACCOUNT_MANAGE) - .await?; + context.require_permission(Some(account_id), Permission::ACCOUNT_MANAGE)?; SYNC_TASKS.start_manual_task(account_id).await?; Ok(()) } @@ -247,7 +234,7 @@ impl AccountApi { context: WrappedContext, ) -> ApiResult<()> { let account_id = account_id.0; - let account = AccountModel::check_account_exists(account_id).await?; + let account = AccountModel::check_account_exists(account_id)?; if !matches!(account.account_type, AccountType::IMAP) { return Err(raise_error!( @@ -255,9 +242,7 @@ impl AccountApi { ErrorCode::InvalidParameter ))?; } - context - .require_permission(Some(account_id), Permission::ACCOUNT_MANAGE) - .await?; + context.require_permission(Some(account_id), Permission::ACCOUNT_MANAGE)?; if !SYNC_TASKS.is_manual_running(account_id).await { return Err(raise_error!( @@ -282,11 +267,9 @@ impl AccountApi { context: WrappedContext, ) -> ApiResult> { let account_id = account_id.0; - AccountModel::check_account_exists(account_id).await?; - context - .require_permission(Some(account_id), Permission::ACCOUNT_READ_DETAILS) - .await?; - let state = ENVELOPE_MANAGER.get_account_stats(account_id).await?; + AccountModel::check_account_exists(account_id)?; + context.require_permission(Some(account_id), Permission::ACCOUNT_READ_DETAILS)?; + let state = ENVELOPE_MANAGER.get_account_stats(account_id)?; Ok(Json(state)) } @@ -304,10 +287,10 @@ impl AccountApi { only_nosync: Query>, context: WrappedContext, ) -> ApiResult>> { - let is_admin = context.user.is_admin().await; + let is_admin = context.user.is_admin(); let only_nosync = only_nosync.0.unwrap_or_default(); - let minimal_list = AccountModel::minimal_list(only_nosync).await?; + let minimal_list = AccountModel::minimal_list(only_nosync)?; if is_admin { return Ok(Json(minimal_list)); } @@ -323,8 +306,8 @@ impl AccountApi { req: Json, context: WrappedContext, ) -> ApiResult<()> { - req.validate_existence().await?; - req.0.do_assign(&context).await?; + req.validate_existence()?; + req.0.do_assign(&context)?; Ok(()) } } diff --git a/crates/server/src/rest/api/attachment.rs b/crates/server/src/rest/api/attachment.rs index 2459977..dfac50e 100644 --- a/crates/server/src/rest/api/attachment.rs +++ b/crates/server/src/rest/api/attachment.rs @@ -52,17 +52,13 @@ impl AttachmentApi { payload: Json, context: WrappedContext, ) -> ApiResult>> { - let authorized_ids: Option> = if context - .has_permission(None, Permission::DATA_READ_ALL) - .await - { - None - } else { - Some(context.user.account_access_map.keys().cloned().collect()) - }; - Ok(Json( - search_attachment_impl(authorized_ids, payload.0).await?, - )) + let authorized_ids: Option> = + if context.has_permission(None, Permission::DATA_READ_ALL) { + None + } else { + Some(context.user.account_access_map.keys().cloned().collect()) + }; + Ok(Json(search_attachment_impl(authorized_ids, payload.0)?)) } /// Retrieves the attachment (metadata) of a specific message. @@ -80,13 +76,10 @@ impl AttachmentApi { context: WrappedContext, ) -> ApiResult> { let account_id = account_id.0; - context - .require_permission(Some(account_id), Permission::DATA_READ) - .await?; + context.require_permission(Some(account_id), Permission::DATA_READ)?; let attachment_id = attachment_id.0; let a = ATTACHMENT_MANAGER - .get_attachment_by_id(account_id, &attachment_id) - .await? + .get_attachment_by_id(account_id, &attachment_id)? .ok_or_else(|| { raise_error!( format!( @@ -109,15 +102,13 @@ impl AttachmentApi { &self, context: WrappedContext, ) -> ApiResult>> { - let authorized_ids: Option> = if context - .has_permission(None, Permission::DATA_READ_ALL) - .await - { - None - } else { - Some(context.user.account_access_map.keys().cloned().collect()) - }; - Ok(Json(ATTACHMENT_MANAGER.get_all_tags(authorized_ids).await?)) + let authorized_ids: Option> = + if context.has_permission(None, Permission::DATA_READ_ALL) { + None + } else { + Some(context.user.account_access_map.keys().cloned().collect()) + }; + Ok(Json(ATTACHMENT_MANAGER.get_all_tags(authorized_ids)?)) } /// Adds or removes facet tags for multiple emails across accounts. @@ -137,9 +128,7 @@ impl AttachmentApi { } for account_id in req.updates.keys() { - context - .require_permission(Some(*account_id), Permission::DATA_MANAGE) - .await?; + context.require_permission(Some(*account_id), Permission::DATA_MANAGE)?; } ATTACHMENT_MANAGER.update_attachment_tags(req).await?; @@ -156,17 +145,13 @@ impl AttachmentApi { &self, context: WrappedContext, ) -> ApiResult>> { - let authorized_ids: Option> = if context - .has_permission(None, Permission::DATA_READ_ALL) - .await - { - None - } else { - Some(context.user.account_access_map.keys().cloned().collect()) - }; - Ok(Json( - ATTACHMENT_MANAGER.get_all_senders(authorized_ids).await?, - )) + let authorized_ids: Option> = + if context.has_permission(None, Permission::DATA_READ_ALL) { + None + } else { + Some(context.user.account_access_map.keys().cloned().collect()) + }; + Ok(Json(ATTACHMENT_MANAGER.get_all_senders(authorized_ids)?)) } /// Retrieves unique metadata for all attachments across authorized accounts. @@ -179,14 +164,12 @@ impl AttachmentApi { &self, context: WrappedContext, ) -> ApiResult> { - let authorized_ids: Option> = if context - .has_permission(None, Permission::DATA_READ_ALL) - .await - { - None - } else { - Some(context.user.account_access_map.keys().cloned().collect()) - }; + let authorized_ids: Option> = + if context.has_permission(None, Permission::DATA_READ_ALL) { + None + } else { + Some(context.user.account_access_map.keys().cloned().collect()) + }; Ok(Json( ATTACHMENT_MANAGER.collect_attachment_metadata(authorized_ids)?, )) diff --git a/crates/server/src/rest/api/auto_config.rs b/crates/server/src/rest/api/auto_config.rs index 5e430e5..a37e808 100644 --- a/crates/server/src/rest/api/auto_config.rs +++ b/crates/server/src/rest/api/auto_config.rs @@ -44,10 +44,7 @@ impl AutoConfigApi { email_address: Path, context: WrappedContext, ) -> ApiResult> { - context - .require_permission(None, Permission::ACCOUNT_CREATE) - .await?; - + context.require_permission(None, Permission::ACCOUNT_CREATE)?; let result = resolve_autoconfig(email_address.0.trim()) .await? .ok_or_else(|| { diff --git a/crates/server/src/rest/api/import.rs b/crates/server/src/rest/api/import.rs index 9067fa1..4be7e3b 100644 --- a/crates/server/src/rest/api/import.rs +++ b/crates/server/src/rest/api/import.rs @@ -44,9 +44,7 @@ impl ImportApi { payload: Json, context: WrappedContext, ) -> ApiResult> { - context - .require_permission(Some(payload.0.account_id), Permission::DATA_IMPORT_BATCH) - .await?; + context.require_permission(Some(payload.0.account_id), Permission::DATA_IMPORT_BATCH)?; Ok(Json(ImportEmls::do_import(payload.0).await?)) } } diff --git a/crates/server/src/rest/api/mailbox.rs b/crates/server/src/rest/api/mailbox.rs index b0375a5..d76a66f 100644 --- a/crates/server/src/rest/api/mailbox.rs +++ b/crates/server/src/rest/api/mailbox.rs @@ -53,9 +53,7 @@ impl MailBoxApi { context: WrappedContext, ) -> ApiResult>> { let account_id = account_id.0; - context - .require_permission(Some(account_id), Permission::ACCOUNT_READ_DETAILS) - .await?; + context.require_permission(Some(account_id), Permission::ACCOUNT_READ_DETAILS)?; let remote = remote.0.unwrap_or(false); Ok(Json(get_account_mailboxes(account_id, remote).await?)) } @@ -82,9 +80,7 @@ impl MailBoxApi { ) -> ApiResult<()> { let account_id = account_id.0; let mailbox_id = mailbox_id.0; - context - .require_permission(Some(account_id), Permission::DATA_DELETE) - .await?; + context.require_permission(Some(account_id), Permission::DATA_DELETE)?; Ok(delete_mailbox_impl(account_id, mailbox_id).await?) } } diff --git a/crates/server/src/rest/api/message.rs b/crates/server/src/rest/api/message.rs index e52b6cd..f426625 100644 --- a/crates/server/src/rest/api/message.rs +++ b/crates/server/src/rest/api/message.rs @@ -35,8 +35,8 @@ use bichon_core::message::search::{search_messages_impl, EmailSearchRequest}; use bichon_core::message::tags::TagCount; use bichon_core::message::tags::TagsRequest; use bichon_core::raise_error; -use bichon_core::store::envelope::Envelope; use bichon_core::store::blob::get_reader; +use bichon_core::store::envelope::Envelope; use bichon_core::store::tantivy::envelope::ENVELOPE_MANAGER; use bichon_core::store::tantivy::validate_facet; use bichon_core::users::permissions::Permission; @@ -65,9 +65,7 @@ impl MessageApi { ) -> ApiResult<()> { let request = payload.0; for account_id in request.keys() { - context - .require_permission(Some(*account_id), Permission::DATA_DELETE) - .await?; + context.require_permission(Some(*account_id), Permission::DATA_DELETE)?; } Ok(delete_messages_impl(request).await?) } @@ -84,15 +82,13 @@ impl MessageApi { payload: Json, context: WrappedContext, ) -> ApiResult>> { - let authorized_ids: Option> = if context - .has_permission(None, Permission::DATA_READ_ALL) - .await - { - None - } else { - Some(context.user.account_access_map.keys().cloned().collect()) - }; - Ok(Json(search_messages_impl(authorized_ids, payload.0).await?)) + let authorized_ids: Option> = + if context.has_permission(None, Permission::DATA_READ_ALL) { + None + } else { + Some(context.user.account_access_map.keys().cloned().collect()) + }; + Ok(Json(search_messages_impl(authorized_ids, payload.0)?)) } /// Retrieves all messages belonging to a specific thread. Requires `thread_id`, `page`, and `page_size` query parameters. @@ -115,12 +111,13 @@ impl MessageApi { ) -> ApiResult>> { let account_id = account_id.0; let thread_id = thread_id.0.trim(); - context - .require_permission(Some(account_id), Permission::DATA_READ) - .await?; - Ok(Json( - get_thread_messages(account_id, thread_id, page.0, page_size.0).await?, - )) + context.require_permission(Some(account_id), Permission::DATA_READ)?; + Ok(Json(get_thread_messages( + account_id, + thread_id, + page.0, + page_size.0, + )?)) } /// Fetches the content of a specific email. @@ -138,12 +135,8 @@ impl MessageApi { context: WrappedContext, ) -> ApiResult> { let account_id = account_id.0; - context - .require_permission(Some(account_id), Permission::DATA_READ) - .await?; - Ok(Json( - retrieve_email_content(account_id, envelope_id.0).await?, - )) + context.require_permission(Some(account_id), Permission::DATA_READ)?; + Ok(Json(retrieve_email_content(account_id, envelope_id.0)?)) } /// Retrieves the content of an email embedded as an attachment. @@ -162,13 +155,13 @@ impl MessageApi { context: WrappedContext, ) -> ApiResult> { let account_id = account_id.0; - context - .require_permission(Some(account_id), Permission::DATA_READ) - .await?; + context.require_permission(Some(account_id), Permission::DATA_READ)?; let content_hash = content_hash.0.trim(); - Ok(Json( - retrieve_nested_eml_content(account_id, envelope_id.0, content_hash).await?, - )) + Ok(Json(retrieve_nested_eml_content( + account_id, + envelope_id.0, + content_hash, + )?)) } /// Retrieves the envelope (metadata) of a specific message. @@ -186,13 +179,10 @@ impl MessageApi { context: WrappedContext, ) -> ApiResult> { let account_id = account_id.0; - context - .require_permission(Some(account_id), Permission::DATA_READ) - .await?; + context.require_permission(Some(account_id), Permission::DATA_READ)?; let envelope_id = envelope_id.0; let e = ENVELOPE_MANAGER - .get_envelope_by_id(account_id, &envelope_id) - .await? + .get_envelope_by_id(account_id, &envelope_id)? .ok_or_else(|| { raise_error!( format!( @@ -220,12 +210,10 @@ impl MessageApi { context: WrappedContext, ) -> ApiResult> { let account_id = account_id.0; - AccountModel::check_account_exists(account_id).await?; - context - .require_permission(Some(account_id), Permission::DATA_RAW_DOWNLOAD) - .await?; + AccountModel::check_account_exists(account_id)?; + context.require_permission(Some(account_id), Permission::DATA_RAW_DOWNLOAD)?; let envelope_id = envelope_id.0; - let reader = get_reader(account_id, envelope_id.clone()).await?; + let reader = get_reader(account_id, envelope_id.clone())?; let body = Body::from_async_read(reader); let attachment = Attachment::new(body) .attachment_type(AttachmentType::Attachment) @@ -247,9 +235,7 @@ impl MessageApi { context: WrappedContext, ) -> ApiResult<()> { let account_id = account_id.0; - context - .require_permission(Some(account_id), Permission::DATA_EXPORT_BATCH) - .await?; + context.require_permission(Some(account_id), Permission::DATA_EXPORT_BATCH)?; Ok(restore_emails(account_id, payload.0.envelope_ids).await?) } @@ -271,12 +257,10 @@ impl MessageApi { ) -> ApiResult> { let account_id = account_id.0; let envelope_id = envelope_id.0.trim().to_string(); - AccountModel::check_account_exists(account_id).await?; - context - .require_permission(Some(account_id), Permission::DATA_READ) - .await?; + AccountModel::check_account_exists(account_id)?; + context.require_permission(Some(account_id), Permission::DATA_READ)?; let content_hash = content_hash.0.trim(); - let reader = retrieve_attachment_content(account_id, envelope_id, content_hash).await?; + let reader = retrieve_attachment_content(account_id, envelope_id, content_hash)?; let body = Body::from_async_read(reader); let attachment = Attachment::new(body) .attachment_type(AttachmentType::Attachment) @@ -303,10 +287,8 @@ impl MessageApi { ) -> ApiResult> { let account_id = account_id.0; let envelope_id = envelope_id.0.trim().to_string(); - AccountModel::check_account_exists(account_id).await?; - context - .require_permission(Some(account_id), Permission::DATA_READ) - .await?; + AccountModel::check_account_exists(account_id)?; + context.require_permission(Some(account_id), Permission::DATA_READ)?; let content_hash = content_hash.0.trim(); let nested_content_hash = nested_content_hash.0.trim(); let reader = retrieve_nested_attachment_content( @@ -314,8 +296,7 @@ impl MessageApi { envelope_id, content_hash, nested_content_hash, - ) - .await?; + )?; let body = Body::from_async_read(reader); let attachment = Attachment::new(body) .attachment_type(AttachmentType::Attachment) @@ -326,15 +307,13 @@ impl MessageApi { /// Returns all facets in the index along with their document counts. #[oai(path = "/all-tags", method = "get", operation_id = "get_all_tags")] async fn get_all_tags(&self, context: WrappedContext) -> ApiResult>> { - let authorized_ids: Option> = if context - .has_permission(None, Permission::DATA_READ_ALL) - .await - { - None - } else { - Some(context.user.account_access_map.keys().cloned().collect()) - }; - Ok(Json(ENVELOPE_MANAGER.get_all_tags(authorized_ids).await?)) + let authorized_ids: Option> = + if context.has_permission(None, Permission::DATA_READ_ALL) { + None + } else { + Some(context.user.account_access_map.keys().cloned().collect()) + }; + Ok(Json(ENVELOPE_MANAGER.get_all_tags(authorized_ids)?)) } /// Adds or removes facet tags for multiple emails across accounts. @@ -354,9 +333,7 @@ impl MessageApi { } for account_id in req.updates.keys() { - context - .require_permission(Some(*account_id), Permission::DATA_MANAGE) - .await?; + context.require_permission(Some(*account_id), Permission::DATA_MANAGE)?; } ENVELOPE_MANAGER.update_envelope_tags(req).await?; @@ -370,16 +347,12 @@ impl MessageApi { operation_id = "get_all_contacts" )] async fn get_all_contacts(&self, context: WrappedContext) -> ApiResult>> { - let authorized_ids: Option> = if context - .has_permission(None, Permission::DATA_READ_ALL) - .await - { - None - } else { - Some(context.user.account_access_map.keys().cloned().collect()) - }; - Ok(Json( - ENVELOPE_MANAGER.get_all_contacts(authorized_ids).await?, - )) + let authorized_ids: Option> = + if context.has_permission(None, Permission::DATA_READ_ALL) { + None + } else { + Some(context.user.account_access_map.keys().cloned().collect()) + }; + Ok(Json(ENVELOPE_MANAGER.get_all_contacts(authorized_ids)?)) } } diff --git a/crates/server/src/rest/api/oauth2.rs b/crates/server/src/rest/api/oauth2.rs index 7e542c2..55c37f6 100644 --- a/crates/server/src/rest/api/oauth2.rs +++ b/crates/server/src/rest/api/oauth2.rs @@ -52,13 +52,13 @@ impl OAuth2Api { context: WrappedContext, ) -> ApiResult> { let id = id.0; - let mut oauth2 = OAuth2::get(id).await?.ok_or_else(|| { + let mut oauth2 = OAuth2::get(id)?.ok_or_else(|| { raise_error!( format!("OAuth2 configuration id='{id}' not found"), ErrorCode::ResourceNotFound ) })?; - if context.has_permission(None, Permission::ROOT).await { + if context.has_permission(None, Permission::ROOT) { return Ok(Json(oauth2)); } oauth2.scrub_sensitive_fields(); @@ -80,8 +80,8 @@ impl OAuth2Api { id: Path, context: WrappedContext, ) -> ApiResult<()> { - context.require_permission(None, Permission::ROOT).await?; - Ok(OAuth2::delete(id.0).await?) + context.require_permission(None, Permission::ROOT)?; + Ok(OAuth2::delete(id.0)?) } /// Creates a new OAuth2 configuration. @@ -99,9 +99,9 @@ impl OAuth2Api { request: Json, context: WrappedContext, ) -> ApiResult<()> { - context.require_permission(None, Permission::ROOT).await?; + context.require_permission(None, Permission::ROOT)?; let entity = OAuth2::new(request.0)?; - Ok(entity.save().await?) + Ok(entity.save()?) } /// Updates an existing OAuth2 configuration. @@ -121,8 +121,8 @@ impl OAuth2Api { payload: Json, context: WrappedContext, ) -> ApiResult<()> { - context.require_permission(None, Permission::ROOT).await?; - Ok(OAuth2::update(id.0, payload.0).await?) + context.require_permission(None, Permission::ROOT)?; + Ok(OAuth2::update(id.0, payload.0)?) } /// Lists OAuth2 configurations with pagination and sorting options. @@ -144,8 +144,8 @@ impl OAuth2Api { desc: Query>, context: WrappedContext, ) -> ApiResult>> { - let mut list = OAuth2::paginate_list(page.0, page_size.0, desc.0).await?; - if context.has_permission(None, Permission::ROOT).await { + let mut list = OAuth2::paginate_list(page.0, page_size.0, desc.0)?; + if context.has_permission(None, Permission::ROOT) { return Ok(Json(list)); } //Non-root users can only view masked data. @@ -172,15 +172,13 @@ impl OAuth2Api { context: WrappedContext, ) -> ApiResult> { let request = request.0; - context - .require_any_permission(vec![ - (None, Permission::ACCOUNT_CREATE), - (Some(request.account_id), Permission::ACCOUNT_MANAGE), - ]) - .await?; + context.require_any_permission(vec![ + (None, Permission::ACCOUNT_CREATE), + (Some(request.account_id), Permission::ACCOUNT_MANAGE), + ])?; let flow = OAuth2Flow::new(request.oauth2_id); - Ok(PlainText(flow.authorize_url(request.account_id).await?)) + Ok(PlainText(flow.authorize_url(request.account_id)?)) } /// Retrieves OAuth2 access tokens for a specified account. @@ -198,17 +196,13 @@ impl OAuth2Api { context: WrappedContext, ) -> ApiResult> { let account = account_id.0; - context - .require_permission(Some(account), Permission::ACCOUNT_MANAGE) - .await?; - Ok(Json(OAuth2AccessToken::get(account).await?.ok_or_else( - || { - raise_error!( - "OAuth2 access tokens not found".into(), - ErrorCode::ResourceNotFound - ) - }, - )?)) + context.require_permission(Some(account), Permission::ACCOUNT_MANAGE)?; + Ok(Json(OAuth2AccessToken::get(account)?.ok_or_else(|| { + raise_error!( + "OAuth2 access tokens not found".into(), + ErrorCode::ResourceNotFound + ) + })?)) } /// Configures an external OAuth2 token for a specified account. @@ -239,12 +233,10 @@ impl OAuth2Api { context: WrappedContext, ) -> ApiResult<()> { let account_id = account_id.0; - AccountModel::check_account_exists(account_id).await?; + AccountModel::check_account_exists(account_id)?; // Check account access permissions - context - .require_permission(Some(account_id), Permission::ACCOUNT_MANAGE) - .await?; - OAuth2AccessToken::upsert_external_oauth_token(account_id, request.0).await?; + context.require_permission(Some(account_id), Permission::ACCOUNT_MANAGE)?; + OAuth2AccessToken::upsert_external_oauth_token(account_id, request.0)?; Ok(()) } } diff --git a/crates/server/src/rest/api/system.rs b/crates/server/src/rest/api/system.rs index 3e8f87b..9a27bd9 100644 --- a/crates/server/src/rest/api/system.rs +++ b/crates/server/src/rest/api/system.rs @@ -75,7 +75,6 @@ impl SystemApi { async fn list_proxy(&self, _context: WrappedContext) -> ApiResult>> { //The proxy list is visible to all users. let proxies = Proxy::list_all() - .await .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; Ok(Json(proxies)) } @@ -88,8 +87,8 @@ impl SystemApi { id: Path, context: WrappedContext, ) -> ApiResult<()> { - context.require_permission(None, Permission::ROOT).await?; - Ok(Proxy::delete(id.0).await?) + context.require_permission(None, Permission::ROOT)?; + Ok(Proxy::delete(id.0)?) } /// Retrieve a specific proxy configuration by ID. Requires root permission. @@ -100,16 +99,16 @@ impl SystemApi { id: Path, context: WrappedContext, ) -> ApiResult> { - context.require_permission(None, Permission::ROOT).await?; - Ok(Json(Proxy::get(id.0).await?)) + context.require_permission(None, Permission::ROOT)?; + Ok(Json(Proxy::get(id.0)?)) } /// Create a new proxy configuration. Requires root permission. #[oai(path = "/proxy", method = "post", operation_id = "create_proxy")] async fn create_proxy(&self, url: PlainText, context: WrappedContext) -> ApiResult<()> { - context.require_permission(None, Permission::ROOT).await?; + context.require_permission(None, Permission::ROOT)?; let entity = Proxy::new(url.0); - Ok(entity.save().await?) + Ok(entity.save()?) } /// Update the URL of a specific proxy by ID. Requires root permission. @@ -120,8 +119,8 @@ impl SystemApi { url: PlainText, context: WrappedContext, ) -> ApiResult<()> { - context.require_permission(None, Permission::ROOT).await?; - Ok(Proxy::update(id.0, url.0).await?) + context.require_permission(None, Permission::ROOT)?; + Ok(Proxy::update(id.0, url.0)?) } /// Get system configurations. /// @@ -136,7 +135,7 @@ impl SystemApi { &self, context: WrappedContext, ) -> ApiResult> { - context.require_permission(None, Permission::ROOT).await?; + context.require_permission(None, Permission::ROOT)?; let config: SystemConfigurations = SystemConfigurations::from(&*SETTINGS); Ok(Json(config)) } diff --git a/crates/server/src/rest/api/users.rs b/crates/server/src/rest/api/users.rs index 17a320f..6441ba3 100644 --- a/crates/server/src/rest/api/users.rs +++ b/crates/server/src/rest/api/users.rs @@ -40,11 +40,8 @@ pub struct UsersApi; impl UsersApi { #[oai(path = "/list-roles", method = "get", operation_id = "list_roles")] async fn list_roles(&self, context: WrappedContext) -> ApiResult>> { - context - .require_permission(None, Permission::USER_MANAGE) - .await?; - - Ok(Json(UserRole::list_all().await?)) + context.require_permission(None, Permission::USER_MANAGE)?; + Ok(Json(UserRole::list_all()?)) } #[oai(path = "/roles/:id", method = "delete", operation_id = "remove_role")] @@ -55,10 +52,8 @@ impl UsersApi { context: WrappedContext, ) -> ApiResult<()> { let id = id.0; - context - .require_permission(None, Permission::USER_MANAGE) - .await?; - Ok(UserRole::delete(id).await?) + context.require_permission(None, Permission::USER_MANAGE)?; + Ok(UserRole::delete(id)?) } /// Create a new account @@ -69,10 +64,8 @@ impl UsersApi { payload: Json, context: WrappedContext, ) -> ApiResult> { - context - .require_permission(None, Permission::USER_MANAGE) - .await?; - let role = UserRole::create(payload.0).await?; + context.require_permission(None, Permission::USER_MANAGE)?; + let role = UserRole::create(payload.0)?; Ok(Json(role)) } @@ -87,20 +80,16 @@ impl UsersApi { context: WrappedContext, ) -> ApiResult<()> { let id = id.0; - context - .require_permission(None, Permission::USER_MANAGE) - .await?; - Ok(UserRole::update(id, payload.0).await?) + context.require_permission(None, Permission::USER_MANAGE)?; + Ok(UserRole::update(id, payload.0)?) } #[oai(path = "/list-users", method = "get", operation_id = "list_users")] async fn list_users(&self, context: WrappedContext) -> ApiResult>> { - context - .require_permission(None, Permission::USER_MANAGE) - .await?; - let roles = UserRole::list_all().await?; + context.require_permission(None, Permission::USER_MANAGE)?; + let roles = UserRole::list_all()?; let role_lookup: BTreeMap = roles.into_iter().map(|r| (r.id, r)).collect(); - let users = UserModel::list_all().await?; + let users = UserModel::list_all()?; let users = users.into_iter().map(|u| u.to_view(&role_lookup)).collect(); Ok(Json(users)) } @@ -116,13 +105,11 @@ impl UsersApi { context: WrappedContext, ) -> ApiResult>> { let target_user_id = id.0; - let tokens = AccessTokenModel::get_user_api_tokens(target_user_id).await?; + let tokens = AccessTokenModel::get_user_api_tokens(target_user_id)?; if context.user.id == target_user_id { return Ok(Json(tokens)); } - context - .require_permission(None, Permission::USER_MANAGE) - .await?; + context.require_permission(None, Permission::USER_MANAGE)?; Ok(Json(tokens)) } @@ -134,10 +121,8 @@ impl UsersApi { context: WrappedContext, ) -> ApiResult<()> { let id = id.0; - context - .require_permission(None, Permission::USER_MANAGE) - .await?; - Ok(UserModel::remove(id).await?) + context.require_permission(None, Permission::USER_MANAGE)?; + Ok(UserModel::remove(id)?) } #[oai(path = "/users", method = "post", operation_id = "create_user")] @@ -146,11 +131,9 @@ impl UsersApi { payload: Json, context: WrappedContext, ) -> ApiResult> { - context - .require_permission(None, Permission::USER_MANAGE) - .await?; - let user = UserModel::create(payload.0).await?; - let roles = UserRole::list_all().await?; + context.require_permission(None, Permission::USER_MANAGE)?; + let user = UserModel::create(payload.0)?; + let roles = UserRole::list_all()?; let role_lookup: BTreeMap = roles.into_iter().map(|r| (r.id, r)).collect(); Ok(Json(user.to_view(&role_lookup))) } @@ -165,19 +148,15 @@ impl UsersApi { let target_id = id.0; let current_user_id = context.user.id; if current_user_id != target_id { - context - .require_permission(None, Permission::USER_MANAGE) - .await?; + context.require_permission(None, Permission::USER_MANAGE)?; } let mut update_data = payload.0; - if current_user_id == target_id - && !context.has_permission(None, Permission::USER_MANAGE).await - { + if current_user_id == target_id && !context.has_permission(None, Permission::USER_MANAGE) { update_data.global_roles = None; update_data.account_access_map = None; update_data.acl = None; } - Ok(UserModel::update(target_id, update_data).await?) + Ok(UserModel::update(target_id, update_data)?) } #[oai( @@ -186,7 +165,7 @@ impl UsersApi { operation_id = "get_current_user" )] async fn get_current_user(&self, context: WrappedContext) -> ApiResult> { - let roles = UserRole::list_all().await?; + let roles = UserRole::list_all()?; let role_lookup: BTreeMap = roles.into_iter().map(|r| (r.id, r)).collect(); Ok(Json(context.0.user.to_view(&role_lookup))) } @@ -200,15 +179,12 @@ impl UsersApi { &self, context: WrappedContext, ) -> ApiResult>> { - let is_admin = context.user.is_admin().await; - let minimal_list = MinimalUser::list_all().await?; + let is_admin = context.user.is_admin(); + let minimal_list = MinimalUser::list_all()?; if is_admin { return Ok(Json(minimal_list)); } - context - .require_permission(None, Permission::USER_VIEW) - .await?; - + context.require_permission(None, Permission::USER_VIEW)?; Ok(Json(minimal_list)) } @@ -218,10 +194,8 @@ impl UsersApi { operation_id = "list_account_roles" )] async fn list_account_roles(&self, context: WrappedContext) -> ApiResult>> { - context - .require_permission(None, Permission::USER_VIEW) - .await?; - let all = UserRole::list_all().await?; + context.require_permission(None, Permission::USER_VIEW)?; + let all = UserRole::list_all()?; Ok(Json( all.into_iter() .filter(|r| matches!(r.role_type, RoleType::Account)) diff --git a/crates/server/src/rest/public/login.rs b/crates/server/src/rest/public/login.rs index 1369618..8d9e411 100644 --- a/crates/server/src/rest/public/login.rs +++ b/crates/server/src/rest/public/login.rs @@ -32,9 +32,9 @@ pub struct LoginPayload { /// Accepts a plain text password and returns the `root_token` /// on successful authentication. #[handler] -pub async fn login(payload: Json) -> Response { +pub fn login(payload: Json) -> Response { let payload = payload.0; - match UserModel::authenticate_user(payload.username, payload.password).await { + match UserModel::authenticate_user(payload.username, payload.password) { Ok(result) => match serde_json::to_string(&result) { Ok(json_string) => Response::builder() .status(http::StatusCode::OK) diff --git a/crates/server/src/rest/public/oauth2.rs b/crates/server/src/rest/public/oauth2.rs index bae454c..3578b65 100644 --- a/crates/server/src/rest/public/oauth2.rs +++ b/crates/server/src/rest/public/oauth2.rs @@ -16,7 +16,6 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . - use bichon_core::oauth2::{flow::OAuth2Flow, pending::OAuth2PendingEntity}; use poem::{ handler, @@ -56,7 +55,7 @@ pub async fn oauth2_callback( } }; - let pending = match OAuth2PendingEntity::get(state).await { + let pending = match OAuth2PendingEntity::get(state) { Ok(Some(pending)) => pending, _ => { let message = @@ -86,7 +85,7 @@ pub async fn oauth2_callback( .into_response()); } - if let Err(e) = OAuth2PendingEntity::delete(state).await { + if let Err(e) = OAuth2PendingEntity::delete(state) { error!("Failed to delete pending OAuth2 entity: {}", e); } diff --git a/crates/server/src/tests/access_token_tests.rs b/crates/server/src/tests/access_token_tests.rs new file mode 100644 index 0000000..7f58f23 --- /dev/null +++ b/crates/server/src/tests/access_token_tests.rs @@ -0,0 +1,58 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project + +use poem::test::TestClient; +use serde::Serialize; + +use super::{admin_token, build_api_route, setup}; + +fn api_client(route: impl poem::Endpoint) -> TestClient { + TestClient::new(route).default_header("X-Forwarded-For", "127.0.0.1") +} + +#[derive(Debug, Serialize)] +struct CreateTokenPayload { + name: String, +} + +#[tokio::test] +async fn access_token_crud() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + // Create a new API token + let create = CreateTokenPayload { + name: "Test API Token".into(), + }; + let resp = cli + .post("/api/v1/access-token") + .header("Authorization", &format!("Bearer {}", token)) + .body_json(&create) + .send() + .await; + resp.assert_status_is_ok(); + let new_token = resp.0.into_body().into_string().await.unwrap_or_default(); + assert!(!new_token.is_empty(), "token string should not be empty"); + + // Verify token now appears in the list + let resp = cli + .get("/api/v1/access-token-list") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); + let tokens: Vec = resp.json().await.value().deserialize(); + assert!(!tokens.is_empty(), "token list should not be empty after creation"); + + // Delete the NEW token (not the admin's WebUI token) + let resp = cli + .delete(&format!("/api/v1/access-token/{}", new_token)) + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); +} diff --git a/crates/server/src/tests/account_tests.rs b/crates/server/src/tests/account_tests.rs new file mode 100644 index 0000000..6f0491d --- /dev/null +++ b/crates/server/src/tests/account_tests.rs @@ -0,0 +1,202 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project + +use poem::test::TestClient; +use serde::{Deserialize, Serialize}; + +use super::{admin_token, build_api_route, setup}; + +fn api_client(route: impl poem::Endpoint) -> TestClient { + TestClient::new(route).default_header("X-Forwarded-For", "127.0.0.1") +} + +// ── Payloads ──────────────────────────────────────────────────────────────── + +#[derive(Debug, Serialize)] +struct CreateAccountPayload { + email: String, + enabled: bool, + account_type: String, + use_dangerous: bool, + #[serde(skip_serializing_if = "Option::is_none")] + account_name: Option, +} + +#[derive(Debug, Deserialize)] +struct AccountResp { + id: u64, + email: String, + enabled: bool, + account_name: Option, +} + +#[derive(Debug, Deserialize)] +struct DataPage { + items: Vec, + total_items: u64, +} + +#[derive(Debug, Serialize)] +struct UpdateAccountPayload { + enabled: Option, + account_name: Option, +} + +// ── Tests ─────────────────────────────────────────────────────────────────── + +#[tokio::test] +async fn account_crud() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + // ── Create ────────────────────────────────────────────────────────── + let create_payload = CreateAccountPayload { + email: "test-crud@example.com".into(), + enabled: false, + account_type: "NoSync".into(), + use_dangerous: false, + account_name: Some("CRUD Test Account".into()), + }; + + let resp = cli + .post("/api/v1/account") + .header("Authorization", &format!("Bearer {}", token)) + .body_json(&create_payload) + .send() + .await; + + resp.assert_status_is_ok(); + let account: AccountResp = resp.json().await.value().deserialize(); + assert_eq!(account.email, "test-crud@example.com"); + assert!(!account.enabled); + let account_id = account.id; + + // ── Read ──────────────────────────────────────────────────────────── + let resp = cli + .get(&format!("/api/v1/account/{}", account_id)) + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); + let account: AccountResp = resp.json().await.value().deserialize(); + assert_eq!(account.id, account_id); + + // ── List ──────────────────────────────────────────────────────────── + let resp = cli + .get("/api/v1/accounts") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); + + // ── Update ────────────────────────────────────────────────────────── + let update_payload = UpdateAccountPayload { + enabled: Some(true), + account_name: Some("Updated Name".into()), + }; + let resp = cli + .post(&format!("/api/v1/account/{}", account_id)) + .header("Authorization", &format!("Bearer {}", token)) + .body_json(&update_payload) + .send() + .await; + resp.assert_status_is_ok(); + + // ── Delete ────────────────────────────────────────────────────────── + let resp = cli + .delete(&format!("/api/v1/account/{}", account_id)) + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); + + // ── Verify deleted ────────────────────────────────────────────────── + let resp = cli + .get(&format!("/api/v1/account/{}", account_id)) + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + assert!(resp.0.status().is_client_error(), "should be 4xx after delete"); +} + +#[tokio::test] +async fn create_account_with_invalid_email_fails() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + let payload = CreateAccountPayload { + email: "not-an-email".into(), + enabled: false, + account_type: "NoSync".into(), + use_dangerous: false, + account_name: None, + }; + + let resp = cli + .post("/api/v1/account") + .header("Authorization", &format!("Bearer {}", token)) + .body_json(&payload) + .send() + .await; + assert!(resp.0.status().is_client_error(), "invalid email should fail"); +} + +#[tokio::test] +async fn create_account_with_empty_email_fails() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + let payload = CreateAccountPayload { + email: "".into(), + enabled: false, + account_type: "NoSync".into(), + use_dangerous: false, + account_name: None, + }; + + let resp = cli + .post("/api/v1/account") + .header("Authorization", &format!("Bearer {}", token)) + .body_json(&payload) + .send() + .await; + assert!(resp.0.status().is_client_error(), "empty email should fail"); +} + +#[tokio::test] +async fn get_nonexistent_account_returns_error() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + let resp = cli + .get("/api/v1/account/99999999") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + assert!(resp.0.status().is_client_error(), "nonexistent account should 4xx"); +} + +#[tokio::test] +async fn delete_nonexistent_account_returns_error() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + let resp = cli + .delete("/api/v1/account/99999999") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + assert!(resp.0.status().is_client_error(), "delete nonexistent should 4xx"); +} diff --git a/crates/server/src/tests/mod.rs b/crates/server/src/tests/mod.rs new file mode 100644 index 0000000..768b757 --- /dev/null +++ b/crates/server/src/tests/mod.rs @@ -0,0 +1,216 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project + +pub mod access_token_tests; +pub mod account_tests; +pub mod oauth2_tests; +pub mod proxy_tests; +pub mod role_tests; +pub mod system_tests; +pub mod user_tests; + +use std::{ + path::PathBuf, + sync::{ + LazyLock, + Mutex, + }, +}; + +use bichon_core::{ + common::signal::SignalManager, + context::{executors::BichonContext, Initialize}, + settings::{ + cli::SETTINGS, + dir::DataDirManager, + }, + store::{ + blob::BLOB_MANAGER, + tantivy::{attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER}, + }, + users::manager::UserManager, +}; +use poem::{EndpointExt, Route}; +use serde::{Deserialize, Serialize}; + +use crate::{ + common::{ + auth::ApiGuard, + error::ErrorCapture, + log::Tracing, + timeout::Timeout, + }, + rest::api::create_openapi_service, +}; + +static INIT: Mutex = Mutex::new(false); + +/// Initialize the test environment. Safe to call multiple times — only runs once. +pub async fn setup() { + let mut initialized = INIT.lock().unwrap(); + if *initialized { + drop(initialized); + return; + } + + let root = PathBuf::from(&SETTINGS.bichon_root_dir); + if root.exists() { + let _ = std::fs::remove_dir_all(&root); + } + + SignalManager::initialize().await.unwrap(); + DataDirManager::initialize().await.unwrap(); + UserManager::initialize().await.unwrap(); + BichonContext::initialize().await.unwrap(); + LazyLock::force(&BLOB_MANAGER); + LazyLock::force(&ENVELOPE_MANAGER); + LazyLock::force(&ATTACHMENT_MANAGER); + + *initialized = true; +} + +/// Build the full API route (same middleware stack as production). +pub fn build_api_route() -> impl poem::Endpoint { + let api_service = create_openapi_service(); + Route::new() + .nest_no_strip("/api/v1", api_service) + .with(ApiGuard) + .with(ErrorCapture) + .with(Timeout) + .with(Tracing) +} + +// ── Shared types ──────────────────────────────────────────────────────────── + +#[derive(Debug, Serialize)] +struct LoginPayload { + username: String, + password: String, +} + +#[derive(Debug, Deserialize)] +struct LoginResult { + success: bool, + #[allow(dead_code)] + error_message: Option, + access_token: Option, + #[allow(dead_code)] + theme: Option, + #[allow(dead_code)] + language: Option, +} + +// ── Helpers ───────────────────────────────────────────────────────────────── + +static TOKEN_CACHE: Mutex> = Mutex::new(None); + +/// Login as admin and return the access token. The token is cached so +/// multiple callers share the same token (each call to +/// `reset_webui_token` would invalidate the previous one). +pub async fn admin_token() -> String { + let mut cache = TOKEN_CACHE.lock().unwrap(); + if let Some(ref token) = *cache { + return token.clone(); + } + + let login_route = poem::Route::new() + .at("/api/login", poem::post(crate::rest::public::login::login)); + + let cli = poem::test::TestClient::new(login_route); + let resp = cli + .post("/api/login") + .body_json(&LoginPayload { + username: "admin".into(), + password: "admin@bichon".into(), + }) + .send() + .await; + + resp.assert_status_is_ok(); + let result: LoginResult = resp.json().await.value().deserialize(); + assert!(result.success, "Admin login failed"); + let token = result.access_token.expect("access_token should be present"); + *cache = Some(token.clone()); + token +} + +// ── Auth / Login Tests ───────────────────────────────────────────────────── + +#[tokio::test] +async fn login_with_wrong_password_fails() { + setup().await; + let login_route = poem::Route::new() + .at("/api/login", poem::post(crate::rest::public::login::login)); + + let cli = poem::test::TestClient::new(login_route); + let resp = cli + .post("/api/login") + .body_json(&LoginPayload { + username: "admin".into(), + password: "wrong-password".into(), + }) + .send() + .await; + + resp.assert_status_is_ok(); + let result: LoginResult = resp.json().await.value().deserialize(); + assert!(!result.success); +} + +#[tokio::test] +async fn login_with_nonexistent_user_fails() { + setup().await; + let login_route = poem::Route::new() + .at("/api/login", poem::post(crate::rest::public::login::login)); + + let cli = poem::test::TestClient::new(login_route); + let resp = cli + .post("/api/login") + .body_json(&LoginPayload { + username: "nonexistent".into(), + password: "whatever".into(), + }) + .send() + .await; + + resp.assert_status_is_ok(); + let result: LoginResult = resp.json().await.value().deserialize(); + assert!(!result.success); +} + +#[tokio::test] +async fn protected_endpoint_requires_auth() { + setup().await; + let route = build_api_route(); + let cli = poem::test::TestClient::new(route) + .default_header("X-Forwarded-For", "127.0.0.1"); + + // Without auth header — should fail (4xx) + let resp = cli.get("/api/v1/list-roles").send().await; + assert!( + resp.0.status().is_client_error(), + "expected 4xx for missing auth" + ); + + // With invalid token — should fail (4xx) + let resp = cli + .get("/api/v1/list-roles") + .header("Authorization", "Bearer invalid-token-here") + .send() + .await; + assert!( + resp.0.status().is_client_error(), + "expected 4xx for invalid token" + ); + + // With valid admin token — should succeed + let token = admin_token().await; + let resp = cli + .get("/api/v1/list-roles") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); +} diff --git a/crates/server/src/tests/oauth2_tests.rs b/crates/server/src/tests/oauth2_tests.rs new file mode 100644 index 0000000..ec81a95 --- /dev/null +++ b/crates/server/src/tests/oauth2_tests.rs @@ -0,0 +1,100 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project + +use poem::test::TestClient; +use serde::{Deserialize, Serialize}; + +use super::{admin_token, build_api_route, setup}; + +fn api_client(route: impl poem::Endpoint) -> TestClient { + TestClient::new(route).default_header("X-Forwarded-For", "127.0.0.1") +} + +#[derive(Debug, Serialize)] +struct CreateOAuth2Payload { + client_id: String, + client_secret: String, + auth_url: String, + token_url: String, + redirect_uri: String, + enabled: bool, +} + +#[derive(Debug, Deserialize)] +struct OAuth2Config { + id: u64, + client_id: String, + enabled: bool, +} + +#[derive(Debug, Serialize)] +struct UpdateOAuth2Payload { + enabled: Option, +} + +#[tokio::test] +async fn oauth2_crud() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + // Create + let create = CreateOAuth2Payload { + client_id: "test-client-id".into(), + client_secret: "test-client-secret".into(), + auth_url: "https://provider.example.com/auth".into(), + token_url: "https://provider.example.com/token".into(), + redirect_uri: "http://localhost/callback".into(), + enabled: false, + }; + let resp = cli + .post("/api/v1/oauth2") + .header("Authorization", &format!("Bearer {}", token)) + .body_json(&create) + .send() + .await; + resp.assert_status_is_ok(); + + // List + let resp = cli + .get("/api/v1/oauth2-list") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); + let list: serde_json::Value = resp.json().await.value().deserialize(); + let items = list["items"].as_array().expect("items array"); + assert!(!items.is_empty(), "should have at least one OAuth2 config"); + let id = items[0]["id"].as_u64().unwrap(); + + // Get by ID + let resp = cli + .get(&format!("/api/v1/oauth2/{}", id)) + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); + + // Update + let update = UpdateOAuth2Payload { + enabled: Some(true), + }; + let resp = cli + .post(&format!("/api/v1/oauth2/{}", id)) + .header("Authorization", &format!("Bearer {}", token)) + .body_json(&update) + .send() + .await; + resp.assert_status_is_ok(); + + // Delete + let resp = cli + .delete(&format!("/api/v1/oauth2/{}", id)) + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); +} diff --git a/crates/server/src/tests/proxy_tests.rs b/crates/server/src/tests/proxy_tests.rs new file mode 100644 index 0000000..2f88dce --- /dev/null +++ b/crates/server/src/tests/proxy_tests.rs @@ -0,0 +1,74 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project + +use poem::test::TestClient; +use serde::Deserialize; + +use super::{admin_token, build_api_route, setup}; + +fn api_client(route: impl poem::Endpoint) -> TestClient { + TestClient::new(route).default_header("X-Forwarded-For", "127.0.0.1") +} + +#[derive(Debug, Deserialize)] +struct Proxy { + id: u64, + url: String, +} + +#[tokio::test] +async fn proxy_crud() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + // Create proxy + let resp = cli + .post("/api/v1/proxy") + .header("Authorization", &format!("Bearer {}", token)) + .content_type("text/plain") + .body("socks5://127.0.0.1:1080") + .send() + .await; + resp.assert_status_is_ok(); + + // List proxies + let resp = cli + .get("/api/v1/list-proxy") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); + let proxies: Vec = resp.json().await.value().deserialize(); + assert!(!proxies.is_empty(), "should have at least one proxy"); + let proxy_id = proxies[0].id; + + // Get single proxy + let resp = cli + .get(&format!("/api/v1/proxy/{}", proxy_id)) + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); + + // Update proxy + let resp = cli + .post(&format!("/api/v1/proxy/{}", proxy_id)) + .header("Authorization", &format!("Bearer {}", token)) + .content_type("text/plain") + .body("socks5://192.168.1.1:1080") + .send() + .await; + resp.assert_status_is_ok(); + + // Delete proxy + let resp = cli + .delete(&format!("/api/v1/proxy/{}", proxy_id)) + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); +} diff --git a/crates/server/src/tests/role_tests.rs b/crates/server/src/tests/role_tests.rs new file mode 100644 index 0000000..681c4ff --- /dev/null +++ b/crates/server/src/tests/role_tests.rs @@ -0,0 +1,113 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project + +use poem::test::TestClient; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; + +use super::{admin_token, build_api_route, setup}; + +fn api_client(route: impl poem::Endpoint) -> TestClient { + TestClient::new(route).default_header("X-Forwarded-For", "127.0.0.1") +} + +#[derive(Debug, Deserialize)] +struct UserRole { + id: u64, + name: String, + is_builtin: bool, + permissions: BTreeSet, + role_type: String, +} + +#[derive(Debug, Serialize)] +struct CreateRolePayload { + name: String, + role_type: String, + permissions: BTreeSet, +} + +#[derive(Debug, Serialize)] +struct UpdateRolePayload { + name: Option, +} + +const ADMIN_ROLE_ID: u64 = 100_000_000_000_000; + +#[tokio::test] +async fn role_crud() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + // List roles (5 built-in roles exist) + let resp = cli + .get("/api/v1/list-roles") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); + let roles: Vec = resp.json().await.value().deserialize(); + assert!(roles.len() >= 5, "should have at least 5 built-in roles"); + assert!(roles.iter().any(|r| r.name == "admin")); + assert!(roles.iter().any(|r| r.name == "manager")); + assert!(roles.iter().any(|r| r.name == "member")); + + // Create custom role + let mut perms = BTreeSet::new(); + perms.insert("user:view".into()); + let create = CreateRolePayload { + name: "test-role".into(), + role_type: "Global".into(), + permissions: perms, + }; + let resp = cli + .post("/api/v1/roles") + .header("Authorization", &format!("Bearer {}", token)) + .body_json(&create) + .send() + .await; + resp.assert_status_is_ok(); + let role: UserRole = resp.json().await.value().deserialize(); + assert_eq!(role.name, "test-role"); + assert!(!role.is_builtin); + let role_id = role.id; + + // Update custom role + let update = UpdateRolePayload { + name: Some("test-role-updated".into()), + }; + let resp = cli + .post(&format!("/api/v1/roles/{}", role_id)) + .header("Authorization", &format!("Bearer {}", token)) + .body_json(&update) + .send() + .await; + resp.assert_status_is_ok(); + + // Delete custom role + let resp = cli + .delete(&format!("/api/v1/roles/{}", role_id)) + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); +} + +#[tokio::test] +async fn cannot_delete_builtin_role() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + let resp = cli + .delete(&format!("/api/v1/roles/{}", ADMIN_ROLE_ID)) + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + assert!(resp.0.status().is_client_error(), "deleting builtin role should fail"); +} diff --git a/crates/server/src/tests/system_tests.rs b/crates/server/src/tests/system_tests.rs new file mode 100644 index 0000000..21281ab --- /dev/null +++ b/crates/server/src/tests/system_tests.rs @@ -0,0 +1,123 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project + +use poem::test::TestClient; +use serde::Deserialize; + +use super::{admin_token, build_api_route, setup}; + +fn api_client(route: impl poem::Endpoint) -> TestClient { + TestClient::new(route).default_header("X-Forwarded-For", "127.0.0.1") +} + +#[derive(Debug, Deserialize)] +struct SystemConfig { + bichon_root_dir: String, + bichon_http_port: i32, + bichon_version: Option, +} + +#[derive(Debug, Deserialize)] +struct DashboardStats { + account_count: usize, + email_count: u64, + attachment_count: u64, + system_version: String, +} + +#[tokio::test] +async fn get_system_configurations() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + let resp = cli + .get("/api/v1/system-configurations") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); +} + +#[tokio::test] +async fn get_dashboard_stats_returns_data() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + let resp = cli + .get("/api/v1/dashboard-stats") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + + // Dashboard stats may return errors if tantivy schemas are empty, + // but should always produce a response (not a crash) + let status = resp.0.status(); + // Accept both success (200) and error (4xx/5xx) — just ensure it doesn't panic + assert!(status.as_u16() > 0, "should produce a valid HTTP response"); +} + +#[tokio::test] +async fn list_proxy_returns_array() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + let resp = cli + .get("/api/v1/list-proxy") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); +} + +#[tokio::test] +async fn list_roles_returns_builtins() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + let resp = cli + .get("/api/v1/list-roles") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); +} + +#[tokio::test] +async fn minimal_user_list_works() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + let resp = cli + .get("/api/v1/minimal-user-list") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); +} + +#[tokio::test] +async fn list_account_roles_works() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + let resp = cli + .get("/api/v1/list-account-roles") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); +} diff --git a/crates/server/src/tests/user_tests.rs b/crates/server/src/tests/user_tests.rs new file mode 100644 index 0000000..20dd2bf --- /dev/null +++ b/crates/server/src/tests/user_tests.rs @@ -0,0 +1,155 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project + +use poem::test::TestClient; +use serde::{Deserialize, Serialize}; + +use super::{admin_token, build_api_route, setup}; + +fn api_client(route: impl poem::Endpoint) -> TestClient { + TestClient::new(route).default_header("X-Forwarded-For", "127.0.0.1") +} + +#[derive(Debug, Serialize)] +struct CreateUserPayload { + username: String, + email: String, + password: String, + global_roles: Vec, + account_access_map: std::collections::BTreeMap, +} + +#[derive(Debug, Deserialize)] +struct UserView { + id: u64, + username: String, + email: String, +} + +#[derive(Debug, Serialize)] +struct UpdateUserPayload { + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, +} + +const ADMIN_ROLE_ID: u64 = 100_000_000_000_000; + +#[tokio::test] +async fn user_crud() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + // Create + let create_payload = CreateUserPayload { + username: "testuser1".into(), + email: "testuser1@example.com".into(), + password: "testpass123".into(), + global_roles: vec![ADMIN_ROLE_ID], + account_access_map: Default::default(), + }; + let resp = cli + .post("/api/v1/users") + .header("Authorization", &format!("Bearer {}", token)) + .body_json(&create_payload) + .send() + .await; + resp.assert_status_is_ok(); + let user: UserView = resp.json().await.value().deserialize(); + assert_eq!(user.username, "testuser1"); + let user_id = user.id; + + // List users + let resp = cli + .get("/api/v1/list-users") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); + + // Get current user + let resp = cli + .get("/api/v1/current-user") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); + + // Update + let update = UpdateUserPayload { + description: Some("Test description".into()), + }; + let resp = cli + .post(&format!("/api/v1/users/{}", user_id)) + .header("Authorization", &format!("Bearer {}", token)) + .body_json(&update) + .send() + .await; + resp.assert_status_is_ok(); + + // Delete + let resp = cli + .delete(&format!("/api/v1/users/{}", user_id)) + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + resp.assert_status_is_ok(); +} + +#[tokio::test] +async fn create_user_with_invalid_data_fails() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + // Username too short (min 3) + let payload = CreateUserPayload { + username: "ab".into(), + email: "valid@example.com".into(), + password: "testpass123".into(), + global_roles: vec![ADMIN_ROLE_ID], + account_access_map: Default::default(), + }; + let resp = cli + .post("/api/v1/users") + .header("Authorization", &format!("Bearer {}", token)) + .body_json(&payload) + .send() + .await; + assert!(resp.0.status().is_client_error(), "short username should fail"); + + // Password too short (min 8) + let payload = CreateUserPayload { + username: "validuser".into(), + email: "valid@example.com".into(), + password: "short".into(), + global_roles: vec![ADMIN_ROLE_ID], + account_access_map: Default::default(), + }; + let resp = cli + .post("/api/v1/users") + .header("Authorization", &format!("Bearer {}", token)) + .body_json(&payload) + .send() + .await; + assert!(resp.0.status().is_client_error(), "short password should fail"); +} + +#[tokio::test] +async fn cannot_delete_default_admin() { + setup().await; + let token = admin_token().await; + let route = build_api_route(); + let cli = api_client(route); + + let resp = cli + .delete("/api/v1/users/100000000000000") + .header("Authorization", &format!("Bearer {}", token)) + .send() + .await; + assert!(resp.0.status().is_client_error(), "deleting admin should fail"); +} diff --git a/crates/smtp/src/server.rs b/crates/smtp/src/server.rs index e6ca424..a953eb2 100644 --- a/crates/smtp/src/server.rs +++ b/crates/smtp/src/server.rs @@ -25,7 +25,7 @@ use bichon_core::cache::imap::mailbox::{Attribute, AttributeEnum}; use bichon_core::common::signal::SIGNAL_MANAGER; use bichon_core::envelope::extractor::extract_envelope_from_smtp; use bichon_core::error::BichonResult; -use bichon_core::settings::cli::{SETTINGS, SmtpEncryptionMode}; +use bichon_core::settings::cli::{SmtpEncryptionMode, SETTINGS}; use bichon_core::utils::create_hash; use bichon_core::{ account::migration::AccountModel, @@ -283,7 +283,7 @@ where let username = username.clone(); if let Ok(decoded) = BASE64_STANDARD.decode(trimmed) { let password = String::from_utf8_lossy(&decoded); - match AccessTokenModel::resolve_user_from_token(&password).await { + match AccessTokenModel::resolve_user_from_token(&password) { Ok(user) => { session.authenticated = true; session.user = Some(user); @@ -394,7 +394,7 @@ where } else { let addr = extract_address(&trimmed[8..]); //println!("DEBUG: SMTP RCPT TO extracted address -> '{}'", addr); - let account_result = AccountModel::find_by_email(addr.as_str()).await; + let account_result = AccountModel::find_by_email(addr.as_str()); match account_result { Ok(Some(account)) => { @@ -405,8 +405,7 @@ where user, Some(account.id), Permission::DATA_SMTP_INGEST, - ) - .await; + ); if !has_perm { tracing::warn!( @@ -528,7 +527,7 @@ async fn verify_plain_auth( let username = String::from_utf8_lossy(parts[1]); let password = String::from_utf8_lossy(parts[2]); - match AccessTokenModel::resolve_user_from_token(&password).await { + match AccessTokenModel::resolve_user_from_token(&password) { Ok(user) => { session.authenticated = true; session.user = Some(user); @@ -632,7 +631,7 @@ async fn parse_email(data: &[u8], session: &Session) -> BichonResult<()> { }; let mailbox_id = mailbox.id; - if let Err(e) = MailBox::batch_upsert(&[mailbox]).await { + if let Err(e) = MailBox::batch_upsert(&[mailbox]) { tracing::error!("SMTP: Failed to upsert mailbox for {}: {:?}", rcpt.email, e); return Err(e.into()); } diff --git a/web/src/features/dashboard/index.tsx b/web/src/features/dashboard/index.tsx index 9eed533..5335894 100644 --- a/web/src/features/dashboard/index.tsx +++ b/web/src/features/dashboard/index.tsx @@ -71,14 +71,14 @@ const MetricCardSkeleton = () => ( const EmptyChart = ({ title }: { title: string }) => (
-

{title}

+

{title}

); const EmptyTable = ({ title }: { title: string }) => (
-

{title}

+

{title}

);