diff --git a/crates/admin/src/meta.rs b/crates/admin/src/meta.rs index 45efed6..e23edb6 100644 --- a/crates/admin/src/meta.rs +++ b/crates/admin/src/meta.rs @@ -263,6 +263,8 @@ impl From for AccountModel { auto_download_new_mailboxes: None, download_schedule: None, deleting: false, + archive_rules: None, + extraction_rules: None, } } } diff --git a/crates/core/src/account/migration.rs b/crates/core/src/account/migration.rs index 170ce91..0344d57 100644 --- a/crates/core/src/account/migration.rs +++ b/crates/core/src/account/migration.rs @@ -64,6 +64,218 @@ pub enum QuotaWindow { Monthly, } +/// Include/exclude filter rule. +/// +/// - `include` non-empty: only values matching these patterns pass. +/// - `exclude` non-empty: values matching these patterns are rejected. +/// - Both empty: all values pass. +/// - Both set: include checked first, then exclude. +/// +/// Extension patterns use case-insensitive exact match; all others use regex. +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] +#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] +pub struct FilterRule { + #[serde(default)] + pub include: Vec, + #[serde(default)] + pub exclude: Vec, +} + +impl FilterRule { + pub fn is_empty(&self) -> bool { + self.include.is_empty() && self.exclude.is_empty() + } + + fn matches_exact(&self, value: &str) -> bool { + if !self.include.is_empty() + && !self.include.iter().any(|e| e.eq_ignore_ascii_case(value)) + { + return false; + } + if !self.exclude.is_empty() + && self.exclude.iter().any(|e| e.eq_ignore_ascii_case(value)) + { + return false; + } + true + } + + fn matches_regex(&self, value: &str) -> bool { + if !self.include.is_empty() && !matches_any_regex(&self.include, value) { + return false; + } + if !self.exclude.is_empty() && matches_any_regex(&self.exclude, value) { + return false; + } + true + } + + fn validate_regex(&self, field: &str) -> Result<(), String> { + validate_patterns(&self.include, &format!("{field}.include"))?; + validate_patterns(&self.exclude, &format!("{field}.exclude"))?; + Ok(()) + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] +#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] +pub struct ExtractionRules { + /// Type 0: Master switch. + #[serde(default)] + pub enabled: bool, + /// Type 1: File extensions (exact match, e.g. `{"include": ["pdf","docx"]}`). + #[serde(default)] + pub extensions: FilterRule, + /// Type 2: Folder patterns (regex, e.g. `{"include": ["^INBOX/Invoices"]}`). + #[serde(default)] + pub folders: FilterRule, + /// Type 3: Attachment filename patterns (regex). + #[serde(default)] + pub attachment_names: FilterRule, + /// Type 4: Sender patterns (regex). + #[serde(default)] + pub senders: FilterRule, +} + +impl ExtractionRules { + /// Returns `true` if the attachment should be extracted under these rules. + pub fn should_extract( + &self, + ext: &str, + folder: Option<&str>, + attachment_name: Option<&str>, + sender: Option<&str>, + ) -> bool { + if !self.enabled { + return false; + } + if !self.extensions.matches_exact(ext) { + return false; + } + if !self.folders.is_empty() { + if let Some(folder) = folder { + if !self.folders.matches_regex(folder) { + return false; + } + } + } + if !self.attachment_names.is_empty() { + if let Some(name) = attachment_name { + if !self.attachment_names.matches_regex(name) { + return false; + } + } + } + if !self.senders.is_empty() { + if let Some(sender) = sender { + if !self.senders.matches_regex(sender) { + return false; + } + } + } + true + } + + pub fn validate(&self) -> Result<(), String> { + self.folders.validate_regex("folders")?; + self.attachment_names.validate_regex("attachment_names")?; + self.senders.validate_regex("senders")?; + Ok(()) + } +} + +/// Archive filtering rules — skip unwanted emails before storage. +/// +/// Rule types: +/// 0 — Master switch +/// 1 — Sender filter (regex) +/// 2 — Subject filter (regex) +/// 3 — Skip emails larger than this (bytes) +/// 4 — Skip emails with spam headers (X-Spam-Flag, X-Spam) +/// +/// `None` = archive everything (backward compatible). +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] +pub struct ArchiveRules { + /// Type 0: Master switch. `false` = archive everything. + #[serde(default)] + pub enabled: bool, + /// Type 1: Sender filter (regex, include/exclude). + #[serde(default)] + pub senders: FilterRule, + /// Type 2: Subject filter (regex, include/exclude). + #[serde(default)] + pub subjects: FilterRule, + /// Type 3: Skip emails larger than this (bytes). `None` = no size limit. + #[serde(default)] + pub skip_larger_than: Option, + /// Type 4: Spam header names to check (e.g. `["X-Spam-Flag", "X-Spam"]`). + /// When the value is `yes` or `true` (case-insensitive), the email is skipped. + /// Empty = don't check. Common headers: `X-Spam-Flag` (SpamAssassin), + /// `X-Spam` (rspamd), `X-MS-Exchange-Organization-SCL` (Exchange). + #[serde(default)] + pub spam_headers: Vec, +} + +impl ArchiveRules { + /// Returns `true` if the email should be archived under these rules. + pub fn should_archive( + &self, + sender: Option<&str>, + subject: Option<&str>, + size: u32, + is_spam: bool, + ) -> bool { + if !self.enabled { + return true; + } + if !self.senders.is_empty() { + if let Some(sender) = sender { + if !self.senders.matches_regex(sender) { + return false; + } + } + } + if !self.subjects.is_empty() { + if let Some(subject) = subject { + if !self.subjects.matches_regex(subject) { + return false; + } + } + } + if let Some(limit) = self.skip_larger_than { + if size as u64 > limit { + return false; + } + } + if !self.spam_headers.is_empty() && is_spam { + return false; + } + true + } + + /// Validate all regex patterns are well-formed. + pub fn validate(&self) -> Result<(), String> { + self.senders.validate_regex("senders")?; + self.subjects.validate_regex("subjects")?; + Ok(()) + } +} + +fn matches_any_regex(patterns: &[String], value: &str) -> bool { + patterns + .iter() + .any(|p| regex::Regex::new(p).map(|re| re.is_match(value)).unwrap_or(false)) +} + +fn validate_patterns(patterns: &[String], field_name: &str) -> Result<(), String> { + for p in patterns { + regex::Regex::new(p) + .map_err(|e| format!("{} pattern '{}' is invalid regex: {}", field_name, p, e))?; + } + Ok(()) +} + #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] pub struct Account { @@ -99,6 +311,14 @@ pub struct Account { pub download_schedule: Option, #[serde(default)] pub deleting: bool, + /// Email-level filtering rules (Pro feature). + /// `None` = archive everything (backward compatible). + #[serde(default)] + pub archive_rules: Option, + /// Attachment text extraction rules (Pro feature). + /// `None` = extract everything (backward compatible). + #[serde(default)] + pub extraction_rules: Option, } impl MemDbModel for Account { @@ -139,6 +359,8 @@ impl Account { imap_quota_window: request.imap_quota_window, download_schedule: request.download_schedule, deleting: false, + archive_rules: request.archive_rules, + extraction_rules: request.extraction_rules, }) } @@ -477,7 +699,304 @@ impl Account { if request.clear_download_schedule == Some(true) { new.download_schedule = None; } + if request.extraction_rules.is_some() { + new.extraction_rules = request.extraction_rules; + } + if request.archive_rules.is_some() { + new.archive_rules = request.archive_rules; + } new.updated_at = utc_now!(); Ok(new) } } + +#[cfg(test)] +mod tests { + use super::*; + + // ── FilterRule ─────────────────────────────────────────────────── + + #[test] + fn filter_rule_include_only() { + let r = FilterRule { + include: vec![r"@ok\.com$".into()], + ..Default::default() + }; + assert!(r.matches_regex("bob@ok.com")); + assert!(!r.matches_regex("spam@bad.com")); + } + + #[test] + fn filter_rule_exclude_only() { + let r = FilterRule { + exclude: vec![r"@spam\.com$".into()], + ..Default::default() + }; + assert!(r.matches_regex("bob@ok.com")); + assert!(!r.matches_regex("bot@spam.com")); + } + + #[test] + fn filter_rule_include_then_exclude() { + let r = FilterRule { + include: vec![r"@company\.com$".into()], + exclude: vec![r"noreply@company\.com$".into()], + ..Default::default() + }; + assert!(r.matches_regex("bob@company.com")); + assert!(!r.matches_regex("noreply@company.com")); + assert!(!r.matches_regex("spam@other.com")); + } + + #[test] + fn filter_rule_exact_match() { + let r = FilterRule { + include: vec!["pdf".into(), "docx".into()], + exclude: vec!["xlsx".into()], + ..Default::default() + }; + assert!(r.matches_exact("pdf")); + assert!(r.matches_exact("docx")); + assert!(r.matches_exact("DOCX")); // case-insensitive + assert!(!r.matches_exact("xlsx")); + assert!(!r.matches_exact("txt")); + } + + // ── ExtractionRules ───────────────────────────────────────────── + + #[test] + fn extraction_rules_master_switch() { + let rules = ExtractionRules { + enabled: false, + ..Default::default() + }; + assert!(!rules.should_extract("pdf", None, None, None)); + } + + #[test] + fn extraction_rules_extension_include() { + let rules = ExtractionRules { + enabled: true, + extensions: FilterRule { + include: vec!["pdf".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.should_extract("pdf", None, None, None)); + assert!(!rules.should_extract("docx", None, None, None)); + } + + #[test] + fn extraction_rules_extension_exclude() { + let rules = ExtractionRules { + enabled: true, + extensions: FilterRule { + exclude: vec!["xlsx".into(), "pptx".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.should_extract("pdf", None, None, None)); + assert!(!rules.should_extract("xlsx", None, None, None)); + } + + #[test] + fn extraction_rules_folder_regex() { + let rules = ExtractionRules { + enabled: true, + folders: FilterRule { + include: vec![r"^INBOX/Invoices".into(), r"Contracts$".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.should_extract("pdf", Some("INBOX/Invoices"), None, None)); + assert!(rules.should_extract("pdf", Some("Finance/Contracts"), None, None)); + assert!(!rules.should_extract("pdf", Some("INBOX/Junk"), None, None)); + } + + #[test] + fn extraction_rules_attachment_name_regex() { + let rules = ExtractionRules { + enabled: true, + attachment_names: FilterRule { + include: vec![r"^invoice-.*\.pdf$".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.should_extract("pdf", None, Some("invoice-2024.pdf"), None)); + assert!(!rules.should_extract("pdf", None, Some("newsletter.pdf"), None)); + } + + #[test] + fn extraction_rules_sender_regex() { + let rules = ExtractionRules { + enabled: true, + senders: FilterRule { + exclude: vec![r"@noreply\.com$".into()], + ..Default::default() + }, + ..Default::default() + }; + // non-excluded sender passes + assert!(rules.should_extract("pdf", None, None, Some("bob@ok.com"))); + // excluded sender blocked + assert!(!rules.should_extract("pdf", None, None, Some("bot@noreply.com"))); + } + + #[test] + fn extraction_rules_empty_filters_pass_everything() { + let rules = ExtractionRules { + enabled: true, + ..Default::default() + }; + assert!(rules.should_extract( + "anything", + Some("any/folder"), + Some("any.pdf"), + Some("any@x.com") + )); + } + + // ── ArchiveRules ──────────────────────────────────────────────── + + #[test] + fn archive_rules_disabled_archives_everything() { + let rules = ArchiveRules { + enabled: false, + ..Default::default() + }; + assert!(rules.should_archive( + Some("spam@x.com"), + Some("BUY NOW"), + 999, + false + )); + } + + #[test] + fn archive_rules_sender_exclude() { + let rules = ArchiveRules { + enabled: true, + senders: FilterRule { + exclude: vec![r"@spam\.com$".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(!rules.should_archive(Some("bot@spam.com"), None, 100, false)); + assert!(rules.should_archive(Some("friend@ok.com"), None, 100, false)); + } + + #[test] + fn archive_rules_subject_exclude() { + let rules = ArchiveRules { + enabled: true, + subjects: FilterRule { + exclude: vec![r"(?i)unsubscribe|buy now|limited offer".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(!rules.should_archive(None, Some("UNSUBSCRIBE NOW"), 100, false)); + assert!(!rules.should_archive(None, Some("Limited Offer!!"), 100, false)); + assert!(rules.should_archive(None, Some("Meeting tomorrow"), 100, false)); + } + + #[test] + fn archive_rules_sender_include() { + // Only archive emails from specific senders + let rules = ArchiveRules { + enabled: true, + senders: FilterRule { + include: vec![r"@partner\.com$".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.should_archive(Some("bob@partner.com"), None, 100, false)); + assert!(!rules.should_archive(Some("spam@random.com"), None, 100, false)); + } + + #[test] + fn archive_rules_skip_larger_than() { + let rules = ArchiveRules { + enabled: true, + skip_larger_than: Some(50_000_000), + ..Default::default() + }; + assert!(rules.should_archive(None, None, 1_000_000, false)); + assert!(!rules.should_archive(None, None, 60_000_000, false)); + } + + #[test] + fn archive_rules_skip_spam_headers() { + let rules = ArchiveRules { + enabled: true, + spam_headers: vec!["X-Spam-Flag".into()], + ..Default::default() + }; + assert!(!rules.should_archive(None, None, 100, true)); + assert!(rules.should_archive(None, None, 100, false)); + } + + // ── Validation ────────────────────────────────────────────────── + + #[test] + fn validate_extraction_rules_valid() { + let rules = ExtractionRules { + folders: FilterRule { + include: vec![r"^INBOX/.*".into()], + ..Default::default() + }, + senders: FilterRule { + exclude: vec![r"@spam\.com$".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.validate().is_ok()); + } + + #[test] + fn validate_extraction_rules_invalid_regex() { + let rules = ExtractionRules { + folders: FilterRule { + include: vec!["***bad[".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.validate().is_err()); + } + + #[test] + fn validate_archive_rules_valid() { + let rules = ArchiveRules { + senders: FilterRule { + exclude: vec![r"@spam\.com$".into()], + ..Default::default() + }, + subjects: FilterRule { + include: vec![r"(?i)invoice".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.validate().is_ok()); + } + + #[test] + fn validate_archive_rules_invalid_regex() { + let rules = ArchiveRules { + senders: FilterRule { + include: vec!["[unclosed".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.validate().is_err()); + } +} diff --git a/crates/core/src/account/payload.rs b/crates/core/src/account/payload.rs index e796900..798a7aa 100644 --- a/crates/core/src/account/payload.rs +++ b/crates/core/src/account/payload.rs @@ -19,7 +19,7 @@ use std::str::FromStr; use crate::account::entity::ImapConfig; -use crate::account::migration::{AccountModel, AccountType, QuotaWindow}; +use crate::account::migration::{AccountModel, AccountType, ArchiveRules, ExtractionRules, QuotaWindow}; use crate::account::since::{DateSince, RelativeDate}; use crate::error::code::ErrorCode; use crate::error::BichonResult; @@ -56,6 +56,12 @@ pub struct AccountCreateRequest { pub imap_quota_window: Option, pub auto_download_new_mailboxes: Option, pub download_schedule: Option, + /// Email archive filtering rules (Pro feature). + /// `None` = archive everything (backward compatible). + pub archive_rules: Option, + /// Attachment text extraction rules (Pro feature). + /// `None` = extract everything (backward compatible). + pub extraction_rules: Option, } impl AccountCreateRequest { @@ -106,6 +112,16 @@ impl AccountCreateRequest { } AccountType::NoSync => {} } + if let Some(ref rules) = self.extraction_rules { + rules.validate().map_err(|e| { + raise_error!(format!("extraction_rules: {}", e), ErrorCode::InvalidParameter) + })?; + } + if let Some(ref rules) = self.archive_rules { + rules.validate().map_err(|e| { + raise_error!(format!("archive_rules: {}", e), ErrorCode::InvalidParameter) + })?; + } Ok(AccountModel::new(user_id, self)?) } @@ -180,6 +196,12 @@ pub struct AccountUpdateRequest { pub auto_download_new_mailboxes: Option, pub download_schedule: Option, pub clear_download_schedule: Option, + /// Email archive filtering rules (Pro feature). + /// `None` = no change. Use `Some(ArchiveRules { .. })` to set. + pub archive_rules: Option, + /// Attachment text extraction rules (Pro feature). + /// `None` = no change. Use `Some(ExtractionRules { .. })` to set. + pub extraction_rules: Option, } impl AccountUpdateRequest { @@ -235,6 +257,16 @@ impl AccountUpdateRequest { validate_cron_expression(schedule)?; } } + if let Some(ref rules) = self.extraction_rules { + rules.validate().map_err(|e| { + raise_error!(format!("extraction_rules: {}", e), ErrorCode::InvalidParameter) + })?; + } + if let Some(ref rules) = self.archive_rules { + rules.validate().map_err(|e| { + raise_error!(format!("archive_rules: {}", e), ErrorCode::InvalidParameter) + })?; + } Ok(()) } } diff --git a/crates/core/src/envelope/extractor.rs b/crates/core/src/envelope/extractor.rs index db21626..c412c7d 100644 --- a/crates/core/src/envelope/extractor.rs +++ b/crates/core/src/envelope/extractor.rs @@ -16,6 +16,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +use crate::account::migration::AccountModel; use crate::cache::imap::mailbox::MailBox; use crate::common::AddrVec; use crate::envelope::meta::parse_bichon_metadata; @@ -112,6 +113,34 @@ async fn extract_envelope_core( ) })?; + if let Ok(account) = AccountModel::get(account_id) { + if let Some(ref rules) = account.archive_rules { + let sender = message.from().and_then(|addr| { + AddrVec::from(addr).0.into_iter().next().and_then(|a| a.address) + }); + let subject = message.subject().map(|s| s.to_string()); + + let is_spam = !rules.spam_headers.is_empty() + && rules.spam_headers.iter().any(|h| { + message + .header_raw(h.clone()) + .map(|v| matches!(v.trim().to_lowercase().as_str(), "yes" | "true")) + .unwrap_or(false) + }); + + if !rules.should_archive(sender.as_deref(), subject.as_deref(), size, is_spam) { + tracing::debug!( + account_id, + uid, + sender = sender.as_deref().unwrap_or("?"), + subject = subject.as_deref().unwrap_or("?"), + "Email filtered out by archive rules" + ); + return Ok(()); + } + } + } + let preview_limit = 100; let text = if let Some(text) = message.body_text(0).map(|cow| cow.into_owned()) { text @@ -173,7 +202,7 @@ async fn extract_envelope_core( .and_then(|add| add.address) .unwrap_or_else(|| "unknown".to_string()); let attachment_count = message.attachment_count(); - let attachments = detach_and_store_attachments(body, &message, &email_content_hash).await; + let attachments = detach_and_store_attachments(body, &message, &email_content_hash, account_id, mailbox_id).await; let envelope_id = Uuid::new_v4().to_string(); let now = utc_now!(); @@ -399,7 +428,27 @@ pub async fn detach_and_store_attachments( original_body: &[u8], message: &Message<'_>, eml_content_hash: &str, + account_id: u64, + mailbox_id: u64, ) -> Vec { + let rules = if account_id > 0 { + AccountModel::get(account_id) + .ok() + .and_then(|a| a.extraction_rules) + } else { + None + }; + + let mailbox_name = match rules.as_ref().map(|r| !r.folders.is_empty()) { + Some(true) => MailBox::get(mailbox_id).ok().map(|mb| mb.name), + _ => None, + }; + + let sender = message + .from() + .and_then(|addr| AddrVec::from(addr).0.into_iter().next()) + .and_then(|add| add.address); + let mut stripped_eml = original_body.to_vec(); let mut attachment_infos = Vec::new(); // Step 1: Collect and sort attachment ranges in reverse to maintain offset integrity @@ -468,19 +517,30 @@ pub async fn detach_and_store_attachments( }) .unwrap_or_else(|| "application/octet-stream".to_string()); let has_cid = att.content_id().is_some(); - let ext = att - .attachment_name() + let att_name = att.attachment_name().map(|n| n.to_string()); + let ext = att_name + .as_deref() .and_then(|n| { - std::path::Path::new(&n) + std::path::Path::new(n) .extension() .and_then(|e| e.to_str()) .map(|s| s.to_ascii_lowercase()) }) .unwrap_or_default(); + let should_extract = rules.as_ref().map_or(true, |r| { + r.should_extract( + &ext, + mailbox_name.as_deref(), + att_name.as_deref(), + sender.as_deref(), + ) + }); + if !inline || !has_cid { let decoded_len = att.contents().len(); - if decoded_len <= crate::ext::text_extractor::MAX_EXTRACT_BYTES + if should_extract + && decoded_len <= crate::ext::text_extractor::MAX_EXTRACT_BYTES && crate::ext::text_extractor::should_try_extract(&file_type, &ext) { text_candidates.push(TextCandidate { @@ -731,7 +791,7 @@ async fn recover_message_blob(envelope: &Envelope) -> BichonResult { ErrorCode::InternalError ) })?; - detach_and_store_attachments(&raw_body, &message, &fetched_hash).await; + detach_and_store_attachments(&raw_body, &message, &fetched_hash, envelope.account_id, envelope.mailbox_id).await; Ok(Bytes::from(raw_body)) } @@ -828,6 +888,8 @@ mod test { truncated, &message, "test_content_hash", + 0, + 0, ) .await;