From f583c3413c2c0482c331b564a3bf59bc8370a81a Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 5 May 2026 03:49:20 +0800 Subject: [PATCH] feat: use stemmer for multilingual token matching --- Cargo.lock | 36 +++++-- Cargo.toml | 8 +- crates/core/Cargo.toml | 4 +- crates/core/src/store/tantivy/attachment.rs | 71 +++++++------ crates/core/src/store/tantivy/envelope.rs | 70 ++++++++----- crates/core/src/store/tantivy/fields.rs | 12 +++ crates/core/src/store/tantivy/filter.rs | 73 ++++++++++++++ crates/core/src/store/tantivy/mod.rs | 2 + crates/core/src/store/tantivy/model.rs | 21 +++- crates/core/src/store/tantivy/schema.rs | 106 +++++++++++++------- crates/core/src/store/tantivy/tokenizers.rs | 72 +++++++++++++ crates/server/src/main.rs | 2 + 12 files changed, 367 insertions(+), 110 deletions(-) create mode 100644 crates/core/src/store/tantivy/filter.rs create mode 100644 crates/core/src/store/tantivy/tokenizers.rs diff --git a/Cargo.lock b/Cargo.lock index 0ada5af..6fb36f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -505,6 +505,7 @@ dependencies = [ "chrono", "clap", "dashmap", + "deunicode", "email_address", "encoding_rs", "fjall", @@ -513,7 +514,7 @@ dependencies = [ "html2text", "itertools", "itoa", - "lru 0.17.0", + "lru 0.18.0", "mail-parser", "mail-send", "murmur3", @@ -546,6 +547,7 @@ dependencies = [ "utf7-imap", "uuid", "webpki-roots", + "whichlang", ] [[package]] @@ -1331,6 +1333,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "deunicode" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04" + [[package]] name = "dialoguer" version = "0.12.0" @@ -2719,9 +2727,9 @@ checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "libmimalloc-sys" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc89deee4af0429081d2a518c0431ae068222a5a262a3bc6ff4d8535ec2e02fe" +checksum = "2d1eacfa31c33ec25e873c136ba5669f00f9866d0688bea7be4d3f7e43067df6" dependencies = [ "cc", ] @@ -2795,9 +2803,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.17.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0b564323a0fb6d54b864f625ae139de9612e27edb944dda37c109f05aac531" +checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9" dependencies = [ "hashbrown 0.17.0", ] @@ -2879,9 +2887,9 @@ dependencies = [ [[package]] name = "mail-parser" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f82a3d6522697593ba4c683e0a6ee5a40fee93bc1a525e3cc6eeb3da11fd8897" +checksum = "d8a2420e9ce11c2b0583ca97ddff7ab2398c8a613154e9b72e3bafdbf767f1d7" dependencies = [ "encoding_rs", "hashify", @@ -2966,9 +2974,9 @@ dependencies = [ [[package]] name = "mimalloc" -version = "0.1.49" +version = "0.1.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aca3c01a711f395b4257b81674c0e90e8dd1f1e62c4b7db45f684cc7a4fcb18a" +checksum = "b3627c4272df786b9260cabaa46aec1d59c93ede723d4c3ef646c503816b0640" dependencies = [ "libmimalloc-sys", ] @@ -4274,9 +4282,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.39" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "log", @@ -6097,6 +6105,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whichlang" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9aa3ad29c3d08283ac6b769e3ec15ad1ddb88af7d2e9bc402c574973b937e7" + [[package]] name = "widestring" version = "1.2.1" diff --git a/Cargo.toml b/Cargo.toml index f0abf28..44d763f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ edition = "2021" [workspace.dependencies] chrono = "0.4.44" clap = { version = "4.6.1", features = ["derive", "env"] } -mimalloc = "0.1.49" +mimalloc = "0.1.50" native_db = "0.8.2" itertools = "0.14.0" native_model = "0.4.20" @@ -38,7 +38,7 @@ regex = "1.12.3" email_address = "0.2.9" futures = "0.3.32" utf7-imap = "0.3.2" -mail-parser = { version = '0.11.2', features = ["serde"] } +mail-parser = { version = '0.11.3', features = ["serde"] } # mail-send = "0.5.2" tokio-rustls = { version = "0.26.4", default-features = false, features = [ "ring", @@ -52,12 +52,12 @@ num_cpus = "1.17.0" rand = "0.10.1" encoding_rs = "0.8.35" webpki-roots = "1.0.7" -rustls = { version = "0.23.39", default-features = false, features = ["ring"] } +rustls = { version = "0.23.40", default-features = false, features = ["ring"] } rustls-pki-types = "1.14.1" tokio-io-timeout = "1.2.1" semver = "1.0.28" governor = "0.10.4" -lru = "0.17.0" +lru = "0.18.0" mime_guess = "2.0.5" hex = "0.4.3" time = { version = "0.3.47", features = [ diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 63508ef..fd891a5 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -48,7 +48,7 @@ async-imap = { git = "https://github.com/rustmailer/async-imap.git", branch = "m "runtime-tokio", "compress", ] } -tantivy = { version = "0.26.0", features = ["zstd-compression"] } +tantivy = { version = "0.26.1", features = ["zstd-compression"] } webpki-roots.workspace = true rustls.workspace = true rustls-pki-types.workspace = true @@ -68,3 +68,5 @@ uuid.workspace = true fjall.workspace = true tracing-log.workspace = true tokio-util.workspace = true +whichlang = "0.1.1" +deunicode = "1.6.2" diff --git a/crates/core/src/store/tantivy/attachment.rs b/crates/core/src/store/tantivy/attachment.rs index 0b6d967..c2a23ef 100644 --- a/crates/core/src/store/tantivy/attachment.rs +++ b/crates/core/src/store/tantivy/attachment.rs @@ -25,19 +25,26 @@ use std::{ }; use crate::{ - common::{paginated::DataPage, signal::SIGNAL_MANAGER}, dashboard::{Group, LargestAttachment}, error::{BichonResult, code::ErrorCode}, message::{ - attachment::AttachmentMetadata, - search::{AttachmentSearchFilter, SortBy}, - tags::{TagAction, TagCount, TagsRequest}, - }, raise_error, settings::dir::DATA_DIR_MANAGER, store::tantivy::{ - fatal_commit, - fields::{ - F_ATTACHMENT_CATEGORY, F_ATTACHMENT_CONTENT_TYPE, F_ATTACHMENT_EXT, F_DATE, F_SIZE, - F_TAGS, - }, - model::{AttachmentModel, extract_senders}, - schema::SchemaTools, - } + common::{paginated::DataPage, signal::SIGNAL_MANAGER}, + dashboard::{Group, LargestAttachment}, + error::{code::ErrorCode, BichonResult}, + message::{ + attachment::AttachmentMetadata, + search::{AttachmentSearchFilter, SortBy}, + tags::{TagAction, TagCount, TagsRequest}, + }, + raise_error, + settings::dir::DATA_DIR_MANAGER, + store::tantivy::{ + fatal_commit, + fields::{ + F_ATTACHMENT_CATEGORY, F_ATTACHMENT_CONTENT_TYPE, F_ATTACHMENT_EXT, F_DATE, F_SIZE, + F_TAGS, + }, + model::{extract_senders, AttachmentModel}, + schema::SchemaTools, + tokenizers::EuroTokenizer, + }, }; use serde_json::json; @@ -67,7 +74,6 @@ pub struct IndexManager { index_writer: Arc>, sender: mpsc::Sender, reader: IndexReader, - query_parser: QueryParser, handle: Mutex>>, } @@ -80,6 +86,7 @@ impl IndexManager { } pub fn new() -> Self { let index = Self::open_or_create_index(&DATA_DIR_MANAGER.attachment_dir); + index.tokenizers().register("euro", EuroTokenizer::new()); let mut merge_policy = LogMergePolicy::default(); merge_policy.set_min_num_segments(25); merge_policy.set_min_layer_size(10_000); @@ -101,9 +108,6 @@ impl IndexManager { &DATA_DIR_MANAGER.envelope_dir, e ) }); - let mut query_parser = - QueryParser::for_index(&index, SchemaTools::attachment_default_fields()); - query_parser.set_conjunction_by_default(); let (sender, mut receiver) = mpsc::channel::(100); @@ -186,7 +190,6 @@ impl IndexManager { index_writer, sender, reader, - query_parser, handle: Mutex::new(Some(handler)), } } @@ -267,7 +270,6 @@ impl IndexManager { &self, accounts: Option>, filter: AttachmentSearchFilter, - parser: QueryParser, ) -> BichonResult> { let f = SchemaTools::attachment_fields(); let mut subqueries: Vec<(Occur, Box)> = Vec::new(); @@ -296,16 +298,21 @@ impl IndexManager { } if let Some(ref text) = filter.text { - let query = parser + let query_parser = + QueryParser::for_index(&self.index, SchemaTools::attachment_default_fields()); + + let query = query_parser .parse_query(text) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?; subqueries.push((Occur::Must, Box::new(query))); } if let Some(ref subject_val) = filter.subject { - let term = Term::from_field_text(f.f_subject, subject_val); - let query = TermQuery::new(term, IndexRecordOption::Basic); - subqueries.push((Occur::Must, Box::new(query))); + let query_parser = QueryParser::for_index(&self.index, vec![f.f_subject]); + let q = query_parser + .parse_query(subject_val) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?; + subqueries.push((Occur::Must, q)); } if let Some(ref tags) = filter.tags { @@ -329,9 +336,11 @@ impl IndexManager { } if let Some(from_query) = &filter.from { - let term = Term::from_field_text(f.f_from, from_query); - let query = TermQuery::new(term, IndexRecordOption::Basic); - subqueries.push((Occur::Must, Box::new(query))); + let query_parser = QueryParser::for_index(&self.index, vec![f.f_from_text]); + let q = query_parser + .parse_query(from_query) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?; + subqueries.push((Occur::Must, q)) } if let Some(content_hash) = &filter.content_hash { @@ -349,9 +358,11 @@ impl IndexManager { if let Some(ref name) = filter.attachment_name { let query_parser = QueryParser::for_index(&self.index, vec![f.f_name_text, f.f_name_exact]); - if let Ok(q) = query_parser.parse_query(name) { - subqueries.push((Occur::Must, q)); - } + + let q = query_parser + .parse_query(name) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?; + subqueries.push((Occur::Must, q)); } if let Some(ref extension) = filter.attachment_extension { @@ -774,7 +785,7 @@ impl IndexManager { ) -> BichonResult> { assert!(page > 0, "Page number must be greater than 0"); assert!(page_size > 0, "Page size must be greater than 0"); - let query = self.filter_query(accounts, filter, self.query_parser.clone())?; + let query = self.filter_query(accounts, filter)?; let searcher = self.create_searcher()?; let total = searcher .search(&query, &Count) diff --git a/crates/core/src/store/tantivy/envelope.rs b/crates/core/src/store/tantivy/envelope.rs index b1efe3f..f77c810 100644 --- a/crates/core/src/store/tantivy/envelope.rs +++ b/crates/core/src/store/tantivy/envelope.rs @@ -46,6 +46,7 @@ use crate::{ }, model::{extract_contacts, EnvelopeWithAttachments}, schema::SchemaTools, + tokenizers::EuroTokenizer, }, }, utc_now, @@ -67,7 +68,7 @@ use tantivy::{ schema::{IndexRecordOption, Value}, DocAddress, Index, IndexReader, IndexWriter, Order, TantivyDocument, Term, }; -use tantivy::{query::RegexQuery, schema::Facet, Searcher}; +use tantivy::{schema::Facet, Searcher}; use tokio::{ sync::{mpsc, Mutex}, task::{self, JoinHandle}, @@ -81,7 +82,6 @@ pub struct IndexManager { index_writer: Arc>, sender: mpsc::Sender, reader: IndexReader, - query_parser: QueryParser, handle: Mutex>>, } @@ -94,6 +94,7 @@ impl IndexManager { } pub fn new() -> Self { let index = Self::open_or_create_index(&DATA_DIR_MANAGER.envelope_dir); + index.tokenizers().register("euro", EuroTokenizer::new()); let mut merge_policy = LogMergePolicy::default(); merge_policy.set_min_num_segments(25); merge_policy.set_min_layer_size(10_000); @@ -115,8 +116,6 @@ impl IndexManager { &DATA_DIR_MANAGER.envelope_dir, e ) }); - let mut query_parser = QueryParser::for_index(&index, SchemaTools::email_default_fields()); - query_parser.set_conjunction_by_default(); let (sender, mut receiver) = mpsc::channel::(100); @@ -199,7 +198,6 @@ impl IndexManager { index_writer, sender, reader, - query_parser, handle: Mutex::new(Some(handler)), } } @@ -277,7 +275,6 @@ impl IndexManager { &self, accounts: Option>, filter: EmailSearchFilter, - parser: QueryParser, ) -> BichonResult> { let f = SchemaTools::email_fields(); let mut subqueries: Vec<(Occur, Box)> = Vec::new(); @@ -306,7 +303,10 @@ impl IndexManager { } if let Some(ref text) = filter.text { - let query = parser + let query_parser = + QueryParser::for_index(&self.index, SchemaTools::email_default_fields()); + + let query = query_parser .parse_query(text) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?; subqueries.push((Occur::Must, Box::new(query))); @@ -314,16 +314,20 @@ impl IndexManager { if let Some(ref subject_val) = filter.subject { let query_parser = QueryParser::for_index(&self.index, vec![f.f_subject]); - if let Ok(q) = query_parser.parse_query(subject_val) { - subqueries.push((Occur::Must, q)); - } + let q = query_parser + .parse_query(subject_val) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?; + println!("{:#?}", &q); + subqueries.push((Occur::Must, q)); } if let Some(ref body_val) = filter.body { let query_parser = QueryParser::for_index(&self.index, vec![f.f_body]); - if let Ok(q) = query_parser.parse_query(body_val) { - subqueries.push((Occur::Must, q)); - } + + let q = query_parser + .parse_query(body_val) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?; + subqueries.push((Occur::Must, q)); } if let Some(ref tags) = filter.tags { @@ -347,15 +351,17 @@ impl IndexManager { } for (field, opt_value) in [ - (f.f_from, &filter.from), - (f.f_to, &filter.to), - (f.f_cc, &filter.cc), - (f.f_bcc, &filter.bcc), + (f.f_from_text, &filter.from), + (f.f_to_text, &filter.to), + (f.f_cc_text, &filter.cc), + (f.f_bcc_text, &filter.bcc), ] { if let Some(ref v) = opt_value { - if let Ok(query) = RegexQuery::from_pattern(v.as_str(), field) { - subqueries.push((Occur::Must, Box::new(query))); - } + let query_parser = QueryParser::for_index(&self.index, vec![field]); + let q = query_parser + .parse_query(v) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?; + subqueries.push((Occur::Must, q)); } } @@ -378,12 +384,24 @@ impl IndexManager { if let Some(ref name) = filter.attachment_name { if name.contains('.') { let term = Term::from_field_text(f.f_attachment_name_exact, name); - let query = TermQuery::new(term, IndexRecordOption::Basic); - subqueries.push((Occur::Should, Box::new(query))); - } + let exact_query = TermQuery::new(term, IndexRecordOption::Basic); - let query_parser = QueryParser::for_index(&self.index, vec![f.f_attachment_name_text]); - if let Ok(q) = query_parser.parse_query(name) { + let query_parser = + QueryParser::for_index(&self.index, vec![f.f_attachment_name_text]); + let q: Box = query_parser + .parse_query(name) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?; + let query = BooleanQuery::new(vec![ + (Occur::Should, Box::new(exact_query)), + (Occur::Should, Box::new(q)), + ]); + subqueries.push((Occur::Must, Box::new(query))); + } else { + let query_parser = + QueryParser::for_index(&self.index, vec![f.f_attachment_name_text]); + let q = query_parser + .parse_query(name) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?; subqueries.push((Occur::Must, q)); } } @@ -1061,7 +1079,7 @@ impl IndexManager { ) -> BichonResult> { assert!(page > 0, "Page number must be greater than 0"); assert!(page_size > 0, "Page size must be greater than 0"); - let query = self.filter_query(accounts, filter, self.query_parser.clone())?; + let query = self.filter_query(accounts, filter)?; let searcher = self.create_searcher()?; let total = searcher .search(&query, &Count) diff --git a/crates/core/src/store/tantivy/fields.rs b/crates/core/src/store/tantivy/fields.rs index e2f4e04..c13b33b 100644 --- a/crates/core/src/store/tantivy/fields.rs +++ b/crates/core/src/store/tantivy/fields.rs @@ -27,10 +27,17 @@ pub const F_SUBJECT: &str = "subject"; pub const F_BODY: &str = "body"; pub const F_PREVIEW: &str = "preview"; pub const F_CONTENT_HASH: &str = "content_hash"; + pub const F_FROM: &str = "from"; pub const F_TO: &str = "to"; pub const F_CC: &str = "cc"; pub const F_BCC: &str = "bcc"; + +pub const F_FROM_TEXT: &str = "from_text"; +pub const F_TO_TEXT: &str = "to_text"; +pub const F_CC_TEXT: &str = "cc_text"; +pub const F_BCC_TEXT: &str = "bcc_text"; + pub const F_DATE: &str = "date"; pub const F_INTERNAL_DATE: &str = "internal_date"; pub const F_INGEST_AT: &str = "ingest_at"; @@ -73,6 +80,10 @@ pub struct EmailFields { pub f_to: Field, pub f_cc: Field, pub f_bcc: Field, + pub f_from_text: Field, + pub f_to_text: Field, + pub f_cc_text: Field, + pub f_bcc_text: Field, pub f_date: Field, pub f_internal_date: Field, pub f_ingest_at: Field, @@ -97,6 +108,7 @@ pub struct AttachmentFields { pub f_account_id: Field, pub f_mailbox_id: Field, pub f_from: Field, + pub f_from_text: Field, pub f_subject: Field, pub f_content_hash: Field, pub f_text: Field, diff --git a/crates/core/src/store/tantivy/filter.rs b/crates/core/src/store/tantivy/filter.rs new file mode 100644 index 0000000..d4fe979 --- /dev/null +++ b/crates/core/src/store/tantivy/filter.rs @@ -0,0 +1,73 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use deunicode::deunicode; +use tantivy::tokenizer::{Token, TokenFilter, TokenStream, Tokenizer}; + +#[derive(Clone, Debug, Default)] +pub struct DeunicodeFilter; + +impl TokenFilter for DeunicodeFilter { + type Tokenizer = DeunicodeFilterWrapper; + + fn transform(self, tokenizer: T) -> Self::Tokenizer { + DeunicodeFilterWrapper { inner: tokenizer } + } +} + +#[derive(Clone, Debug)] +pub struct DeunicodeFilterWrapper { + inner: T, +} + +impl Tokenizer for DeunicodeFilterWrapper { + type TokenStream<'a> = DeunicodeTokenStream>; + + fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { + DeunicodeTokenStream { + inner: self.inner.token_stream(text), + } + } +} + +pub struct DeunicodeTokenStream { + inner: S, +} + +impl TokenStream for DeunicodeTokenStream { + fn advance(&mut self) -> bool { + if !self.inner.advance() { + return false; + } + let token: &mut Token = self.inner.token_mut(); + // Only allocate a new String when the text actually contains + // non-ASCII characters to avoid unnecessary overhead. + if !token.text.is_ascii() { + token.text = deunicode(&token.text); + } + true + } + + fn token(&self) -> &Token { + self.inner.token() + } + + fn token_mut(&mut self) -> &mut Token { + self.inner.token_mut() + } +} diff --git a/crates/core/src/store/tantivy/mod.rs b/crates/core/src/store/tantivy/mod.rs index d500bfe..32a7814 100644 --- a/crates/core/src/store/tantivy/mod.rs +++ b/crates/core/src/store/tantivy/mod.rs @@ -26,8 +26,10 @@ use crate::{ pub mod attachment; pub mod envelope; pub mod fields; +pub mod filter; pub mod model; pub mod schema; +pub mod tokenizers; pub fn fatal_commit(writer: &mut IndexWriter) { const MAX_RETRIES: usize = 3; diff --git a/crates/core/src/store/tantivy/model.rs b/crates/core/src/store/tantivy/model.rs index 473fa0e..bc9ce2d 100644 --- a/crates/core/src/store/tantivy/model.rs +++ b/crates/core/src/store/tantivy/model.rs @@ -67,16 +67,20 @@ impl EnvelopeWithAttachments { doc.add_text(fields.f_preview, &self.envelope.preview); doc.add_text(fields.f_content_hash, &self.envelope.content_hash); doc.add_text(fields.f_from, &self.envelope.from); + doc.add_text(fields.f_from_text, &self.envelope.from); doc.add_text(fields.f_body, body_text); for to in &self.envelope.to { doc.add_text(fields.f_to, to); + doc.add_text(fields.f_to_text, to); } for cc in &self.envelope.cc { doc.add_text(fields.f_cc, cc); + doc.add_text(fields.f_cc_text, cc); } for bcc in &self.envelope.bcc { doc.add_text(fields.f_bcc, bcc); + doc.add_text(fields.f_bcc_text, bcc); } doc.add_i64(fields.f_date, self.envelope.date); @@ -353,9 +357,10 @@ impl AttachmentModel { doc.add_text(f.f_envelope_id, self.envelope_id); doc.add_u64(f.f_account_id, self.account_id); doc.add_u64(f.f_mailbox_id, self.mailbox_id); - doc.add_text(f.f_subject, self.subject); + doc.add_text(f.f_subject, &self.subject); doc.add_text(f.f_content_hash, self.content_hash); - doc.add_text(f.f_from, self.from); + doc.add_text(f.f_from, &self.from); + doc.add_text(f.f_from_text, &self.from); doc.add_i64(f.f_date, self.date); doc.add_i64(f.f_ingest_at, self.ingest_at); doc.add_u64(f.f_size, self.size); @@ -385,6 +390,18 @@ impl AttachmentModel { doc.add_text(f.f_name_exact, name); } + if let Some(tags) = &self.tags { + for tag in tags { + doc.add_facet(f.f_tags, tag); + } + } + + if let Some(tags) = &self.auto_tags { + for tag in tags { + doc.add_facet(f.f_auto_tags, tag); + } + } + doc } diff --git a/crates/core/src/store/tantivy/schema.rs b/crates/core/src/store/tantivy/schema.rs index 0d0b114..430b969 100644 --- a/crates/core/src/store/tantivy/schema.rs +++ b/crates/core/src/store/tantivy/schema.rs @@ -17,17 +17,19 @@ // along with this program. If not, see . use std::sync::{Arc, LazyLock}; -use tantivy::schema::{FacetOptions, Field, INDEXED}; -use tantivy::schema::{Schema, FAST, STORED, STRING, TEXT}; +use tantivy::schema::{ + FacetOptions, Field, IndexRecordOption, TextFieldIndexing, TextOptions, INDEXED, +}; +use tantivy::schema::{Schema, FAST, STORED, STRING}; use crate::store::tantivy::fields::{ AttachmentFields, EmailFields, F_ACCOUNT_ID, F_ATTACHMENTS, F_ATTACHMENT_CATEGORY, F_ATTACHMENT_CONTENT_HASH, F_ATTACHMENT_CONTENT_TYPE, F_ATTACHMENT_COUNT, F_ATTACHMENT_EXT, - F_ATTACHMENT_NAME_EXACT, F_ATTACHMENT_NAME_TEXT, F_AUTO_TAGS, F_BCC, F_BODY, F_CC, - F_CONTENT_HASH, F_DATE, F_ENVELOPE_ID, F_FROM, F_HAS_TEXT, F_ID, F_INGEST_AT, F_INTERNAL_DATE, - F_IS_INDEXED, F_IS_MESSAGE, F_IS_OCR, F_MAILBOX_ID, F_MESSAGE_ID, F_NAME_EXACT, F_NAME_TEXT, - F_PAGE_COUNT, F_PREVIEW, F_REGULAR_ATTACHMENT_COUNT, F_SHARD_ID, F_SIZE, F_SUBJECT, F_TAGS, - F_TEXT, F_THREAD_ID, F_TO, F_UID, + F_ATTACHMENT_NAME_EXACT, F_ATTACHMENT_NAME_TEXT, F_AUTO_TAGS, F_BCC, F_BCC_TEXT, F_BODY, F_CC, + F_CC_TEXT, F_CONTENT_HASH, F_DATE, F_ENVELOPE_ID, F_FROM, F_FROM_TEXT, F_HAS_TEXT, F_ID, + F_INGEST_AT, F_INTERNAL_DATE, F_IS_INDEXED, F_IS_MESSAGE, F_IS_OCR, F_MAILBOX_ID, F_MESSAGE_ID, + F_NAME_EXACT, F_NAME_TEXT, F_PAGE_COUNT, F_PREVIEW, F_REGULAR_ATTACHMENT_COUNT, F_SHARD_ID, + F_SIZE, F_SUBJECT, F_TAGS, F_TEXT, F_THREAD_ID, F_TO, F_TO_TEXT, F_UID, }; static EMAIL_FIELDS: LazyLock> = LazyLock::new(|| { @@ -58,29 +60,10 @@ impl SchemaTools { fields.f_subject, fields.f_body, fields.f_attachment_name_text, - fields.f_attachment_name_exact, - fields.f_from, - fields.f_to, - ] - } - - pub fn attachment_schema() -> Schema { - let (schema, _) = Self::create_attachment_schema(); - schema - } - - pub fn attachment_fields() -> &'static AttachmentFields { - &ATTACHMENT_FIELDS - } - - pub fn attachment_default_fields() -> Vec { - let fields = Self::attachment_fields(); - vec![ - fields.f_subject, - fields.f_text, - fields.f_name_exact, - fields.f_name_text, - fields.f_from, + fields.f_from_text, + fields.f_to_text, + fields.f_cc_text, + fields.f_bcc_text, ] } @@ -91,14 +74,21 @@ impl SchemaTools { let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST); let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST); let f_uid = builder.add_u64_field(F_UID, INDEXED | STORED | FAST); - let f_subject = builder.add_text_field(F_SUBJECT, TEXT | STORED); - let f_body = builder.add_text_field(F_BODY, TEXT); + let f_subject = builder.add_text_field(F_SUBJECT, Self::text_store("euro")); + let f_body = builder.add_text_field(F_BODY, Self::text_no_store("euro")); let f_preview = builder.add_text_field(F_PREVIEW, STORED); let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST); + let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST); let f_to = builder.add_text_field(F_TO, STRING | STORED); let f_cc = builder.add_text_field(F_CC, STRING | STORED); let f_bcc = builder.add_text_field(F_BCC, STRING | STORED); + + let f_from_text = builder.add_text_field(F_FROM_TEXT, Self::text_no_store("euro")); + let f_to_text = builder.add_text_field(F_TO_TEXT, Self::text_no_store("euro")); + let f_cc_text = builder.add_text_field(F_CC_TEXT, Self::text_no_store("euro")); + let f_bcc_text = builder.add_text_field(F_BCC_TEXT, Self::text_no_store("euro")); + let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST); let f_internal_date = builder.add_i64_field(F_INTERNAL_DATE, INDEXED | STORED | FAST); let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST); @@ -107,7 +97,8 @@ impl SchemaTools { let f_attachment_count = builder.add_u64_field(F_ATTACHMENT_COUNT, INDEXED | STORED | FAST); let f_regular_attachment_count = builder.add_u64_field(F_REGULAR_ATTACHMENT_COUNT, INDEXED | STORED | FAST); - let f_attachment_name_text = builder.add_text_field(F_ATTACHMENT_NAME_TEXT, TEXT); + let f_attachment_name_text = + builder.add_text_field(F_ATTACHMENT_NAME_TEXT, Self::text_no_store("euro")); let f_attachment_name_exact = builder.add_text_field(F_ATTACHMENT_NAME_EXACT, STRING); let f_attachments = builder.add_text_field(F_ATTACHMENTS, STORED); let f_attachment_content_hash = @@ -133,6 +124,10 @@ impl SchemaTools { f_to, f_cc, f_bcc, + f_from_text, + f_to_text, + f_cc_text, + f_bcc_text, f_date, f_internal_date, f_ingest_at, @@ -153,15 +148,35 @@ impl SchemaTools { (builder.build(), fields) } + pub fn attachment_schema() -> Schema { + let (schema, _) = Self::create_attachment_schema(); + schema + } + + pub fn attachment_fields() -> &'static AttachmentFields { + &ATTACHMENT_FIELDS + } + + pub fn attachment_default_fields() -> Vec { + let fields = Self::attachment_fields(); + vec![ + fields.f_subject, + fields.f_text, + fields.f_name_text, + fields.f_from_text, + ] + } + pub fn create_attachment_schema() -> (Schema, AttachmentFields) { let mut builder = Schema::builder(); let f_id = builder.add_text_field(F_ID, STRING | STORED | FAST); let f_envelope_id = builder.add_text_field(F_ENVELOPE_ID, STRING | STORED | FAST); let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST); let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST); - let f_subject = builder.add_text_field(F_SUBJECT, TEXT | STORED); + let f_subject = builder.add_text_field(F_SUBJECT, Self::text_store("euro")); let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST); let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST); + let f_from_text = builder.add_text_field(F_FROM_TEXT, Self::text_no_store("euro")); let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST); let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST); let f_size = builder.add_u64_field(F_SIZE, INDEXED | STORED | FAST); @@ -170,13 +185,13 @@ impl SchemaTools { let f_content_type = builder.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST); let f_shard_id = builder.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST); - let f_text = builder.add_text_field(F_TEXT, TEXT); + let f_text = builder.add_text_field(F_TEXT, Self::text_no_store("euro")); let f_has_text = builder.add_bool_field(F_HAS_TEXT, INDEXED | STORED | FAST); let f_is_ocr = builder.add_bool_field(F_IS_OCR, INDEXED | STORED | FAST); let f_page_count = builder.add_u64_field(F_PAGE_COUNT, INDEXED | STORED | FAST); let f_is_indexed = builder.add_bool_field(F_IS_INDEXED, INDEXED | STORED | FAST); let f_is_message = builder.add_bool_field(F_IS_MESSAGE, INDEXED | STORED | FAST); - let f_name_text = builder.add_text_field(F_NAME_TEXT, TEXT); + let f_name_text = builder.add_text_field(F_NAME_TEXT, Self::text_no_store("euro")); let f_name_exact = builder.add_text_field(F_NAME_EXACT, STRING | STORED); let f_tags = builder.add_facet_field(F_TAGS, FacetOptions::default().set_stored()); let f_auto_tags = @@ -189,6 +204,7 @@ impl SchemaTools { f_subject, f_content_hash, f_from, + f_from_text, f_date, f_ingest_at, f_size, @@ -209,4 +225,22 @@ impl SchemaTools { }; (builder.build(), fields) } + + fn text_no_store(tokenizer: &str) -> TextOptions { + TextOptions::default().set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer(tokenizer) + .set_index_option(IndexRecordOption::WithFreqsAndPositions), + ) + } + + fn text_store(tokenizer: &str) -> TextOptions { + TextOptions::default() + .set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer(tokenizer) + .set_index_option(IndexRecordOption::WithFreqsAndPositions), + ) + .set_stored() + } } diff --git a/crates/core/src/store/tantivy/tokenizers.rs b/crates/core/src/store/tantivy/tokenizers.rs new file mode 100644 index 0000000..511bf18 --- /dev/null +++ b/crates/core/src/store/tantivy/tokenizers.rs @@ -0,0 +1,72 @@ +// +// Copyright (c) 2025 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use tantivy::tokenizer::*; +use whichlang::{detect_language, Lang}; + +use crate::store::tantivy::filter::DeunicodeFilter; + +#[derive(Clone)] +pub struct EuroTokenizer { + en: TextAnalyzer, + fr: TextAnalyzer, + de: TextAnalyzer, + es: TextAnalyzer, + nl: TextAnalyzer, + default: TextAnalyzer, +} + +impl EuroTokenizer { + pub fn new() -> Self { + fn build(lang: Language) -> TextAnalyzer { + TextAnalyzer::builder(SimpleTokenizer::default()) + .filter(RemoveLongFilter::limit(40)) + .filter(LowerCaser) + .filter(DeunicodeFilter) + .filter(Stemmer::new(lang)) + .build() + } + + Self { + en: build(Language::English), + fr: build(Language::French), + de: build(Language::German), + es: build(Language::Spanish), + nl: build(Language::Dutch), + default: TextAnalyzer::builder(SimpleTokenizer::default()) + .filter(RemoveLongFilter::limit(40)) + .filter(LowerCaser) + .build(), + } + } +} + +impl Tokenizer for EuroTokenizer { + type TokenStream<'a> = BoxTokenStream<'a>; + + fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { + match detect_language(text) { + Lang::Eng => self.en.token_stream(text), + Lang::Fra => self.fr.token_stream(text), + Lang::Deu => self.de.token_stream(text), + Lang::Spa => self.es.token_stream(text), + Lang::Nld => self.nl.token_stream(text), + _ => self.default.token_stream(text), // fallback + } + } +} diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 0295905..ab5cd72 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -117,5 +117,7 @@ async fn initialize() -> BichonResult<()> { BichonTls::initialize().await?; BichonContext::initialize().await?; LazyLock::force(&BLOB_MANAGER); + LazyLock::force(&ENVELOPE_MANAGER); + LazyLock::force(&ATTACHMENT_MANAGER); Ok(()) }