8 Commits
1.5.2 ... 1.5.3

Author SHA1 Message Date
rustmailer
6fcc9e0e8e bump to v1.5.3 2026-06-23 17:18:46 +08:00
rustmailer
220aa268c1 feat(imap): handle UIDVALIDITY changes via Message-ID comparison instead of full rebuild 2026-06-23 10:46:36 +08:00
rustmailer
e3fd9d2f29 fix: purge DedupCache entries on account/mailbox/envelope removal 2026-06-22 17:11:20 +08:00
rustmailer
41c3b84e65 feat(ui): persist account table sorting to localStorage 2026-06-21 12:50:21 +08:00
rustmailer
89557700ae Merge branch 'main' of https://github.com/rustmailer/bichon 2026-06-11 09:23:00 +08:00
rustmailer
2f5de48c6a fix(smtp): reject journaling attempts to non-local accounts 2026-06-11 09:22:57 +08:00
rustmailer
debb119d3d Merge pull request #298 from shadowdao/fix/smtp-inbox-uidvalidity-clobber-297
fix(smtp): don't clobber the IMAP-owned INBOX uid_validity on journal ingest (#297)
2026-06-11 09:10:01 +08:00
Josh
ce3f8944a3 fix(smtp): don't clobber the IMAP-owned INBOX uid_validity on journal ingest
When SMTP journaling ingests a message, parse_email() upserts the account's
INBOX MailBox row so the journaled envelope has a row to attach to. But it
built the row with uid_validity/highest_uid/uid_next = None and called
batch_upsert, which replaces the WHOLE row. The INBOX row id
(create_hash(account_id, "INBOX")) is the same id the IMAP sync maintains,
so every journaled delivery reset the IMAP-maintained uid_validity to None.

On the next reconcile, local_mailbox.uid_validity != Some(remote) is then
true, so the mailbox is treated as invalid and wiped + rebuilt. For a large,
UID-sparse INBOX whose rebuild gets interrupted, the local copy is silently
lost and never restored (the incremental fetch resumes past all existing
UIDs). See #297 for the full diagnosis and DB evidence.

Fix: only create the INBOX row when it does not already exist; otherwise
leave the IMAP-owned row untouched. The journaling path only needs the row
to exist so the envelope can attach to it.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-10 11:36:38 -07:00
12 changed files with 2418 additions and 177 deletions

10
Cargo.lock generated
View File

@@ -293,7 +293,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bichon-admin"
version = "1.5.2"
version = "1.5.3"
dependencies = [
"bichon-core",
"console",
@@ -311,7 +311,7 @@ dependencies = [
[[package]]
name = "bichon-cli"
version = "1.5.2"
version = "1.5.3"
dependencies = [
"base64 0.22.1",
"bichon-core",
@@ -337,7 +337,7 @@ dependencies = [
[[package]]
name = "bichon-core"
version = "1.5.2"
version = "1.5.3"
dependencies = [
"async-imap",
"base64 0.22.1",
@@ -396,7 +396,7 @@ dependencies = [
[[package]]
name = "bichon-server"
version = "1.5.2"
version = "1.5.3"
dependencies = [
"bichon-core",
"bichon-smtp",
@@ -420,7 +420,7 @@ dependencies = [
[[package]]
name = "bichon-smtp"
version = "1.5.2"
version = "1.5.3"
dependencies = [
"base64 0.22.1",
"bichon-core",

View File

@@ -12,7 +12,7 @@ members = [
resolver = "2"
[workspace.package]
version = "1.5.2"
version = "1.5.3"
edition = "2021"
[workspace.dependencies]

File diff suppressed because it is too large Load Diff

View File

@@ -34,6 +34,25 @@ use std::ops::DerefMut;
use tokio::io::BufWriter;
use tracing::debug;
/// Classify an `io::Error` (from TLS stream I/O) for IMAP connection errors.
/// `UnexpectedEof` is treated as a network error because many servers skip
/// the TLS `close_notify` alert, causing rustls to emit this error when the
/// TCP connection is dropped normally.
fn classify_io_error(e: &std::io::Error) -> ErrorCode {
use std::io::ErrorKind;
matches!(
e.kind(),
ErrorKind::BrokenPipe
| ErrorKind::ConnectionReset
| ErrorKind::ConnectionAborted
| ErrorKind::TimedOut
| ErrorKind::UnexpectedEof
| ErrorKind::NotConnected
)
.then_some(ErrorCode::NetworkError)
.unwrap_or(ErrorCode::ImapCommandFailed)
}
#[derive(Debug)]
pub(crate) struct Client {
inner: ImapClient<Box<dyn SessionStream>>,
@@ -141,7 +160,7 @@ impl Client {
let _greeting = client
.read_response()
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
.map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))?
.ok_or_else(|| {
raise_error!(
"Failed to read IMAP greeting — this usually indicates an incorrect encryption setting (SSL vs. STARTTLS). Your current setting is SSL.".into(),
@@ -171,7 +190,7 @@ impl Client {
let _greeting = client
.read_response()
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
.map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))?
.ok_or_else(|| {
raise_error!(
"failed to read greeting".into(),
@@ -202,7 +221,7 @@ impl Client {
let _greeting = client
.read_response()
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
.map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))?
.ok_or_else(|| {
raise_error!(
"Failed to read IMAP greeting — this usually indicates an incorrect encryption setting (SSL vs. STARTTLS). Your current setting is STARTTLS.".into(),

View File

@@ -27,7 +27,7 @@ use crate::{error::BichonResult, imap::manager::ImapConnectionManager};
use async_imap::types::Name;
use async_imap::Session;
use futures::TryStreamExt;
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use tokio_util::sync::CancellationToken;
use tracing::info;
@@ -260,7 +260,9 @@ impl ImapExecutor {
let mut count = 0u64;
let mut skipped = 0u64;
let mut max_uid: Option<u32> = None;
let size_limit = account.max_email_size_bytes.unwrap_or(DEFAULT_MAX_EMAIL_SIZE);
let size_limit = account
.max_email_size_bytes
.unwrap_or(DEFAULT_MAX_EMAIL_SIZE);
while let Some(fetch) = stream
.try_next()
.await
@@ -363,14 +365,14 @@ impl ImapExecutor {
let mut size_stream = session
.fetch(sequence_set.as_str(), SIZE_ONLY_FETCH)
.await
.map_err(|e| {
raise_error!(format!("{:#?}", e), classify_imap_error(&e))
})?;
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?;
let mut uids: Vec<u32> = Vec::new();
while let Some(fetch) = size_stream.try_next().await.map_err(|e| {
raise_error!(format!("{:#?}", e), classify_imap_error(&e))
})? {
while let Some(fetch) = size_stream
.try_next()
.await
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?
{
let uid = fetch.uid.unwrap_or(0);
let msg_size = fetch.size.unwrap_or(0) as u64;
if msg_size == 0 || msg_size <= limit {
@@ -437,14 +439,14 @@ impl ImapExecutor {
let mut size_stream = session
.uid_fetch(uid_set, SIZE_ONLY_FETCH)
.await
.map_err(|e| {
raise_error!(format!("{:#?}", e), classify_imap_error(&e))
})?;
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?;
let mut uids: Vec<u32> = Vec::new();
while let Some(fetch) = size_stream.try_next().await.map_err(|e| {
raise_error!(format!("{:#?}", e), classify_imap_error(&e))
})? {
while let Some(fetch) = size_stream
.try_next()
.await
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?
{
let uid = fetch.uid.unwrap_or(0);
let msg_size = fetch.size.unwrap_or(0) as u64;
if msg_size == 0 || msg_size <= limit {
@@ -550,6 +552,37 @@ impl ImapExecutor {
) -> BichonResult<Session<Box<dyn SessionStream>>> {
ImapConnectionManager::build(account_id).await
}
/// Fetch UID → Message-ID mapping without downloading bodies.
/// `uid_set` is an IMAP sequence-set string (e.g. "1:100" or "1,3,5").
pub async fn fetch_uid_metadata(
session: &mut Session<Box<dyn SessionStream>>,
uid_set: &str,
token: CancellationToken,
) -> BichonResult<HashMap<u32, Option<String>>> {
let mut stream = session
.uid_fetch(uid_set, "(UID BODY.PEEK[HEADER])")
.await
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?;
let mut result = HashMap::new();
while let Some(fetch) = stream
.try_next()
.await
.map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?
{
if token.is_cancelled() {
return Err(raise_error!(
"Stream cancelled".into(),
ErrorCode::InternalError
));
}
let uid = fetch.uid.unwrap_or(0);
let msg_id = fetch.header().and_then(parse_message_id_header);
result.insert(uid, msg_id);
}
Ok(result)
}
}
pub const DEFAULT_BATCH_SIZE: u32 = 30;
@@ -613,6 +646,27 @@ pub fn generate_uid_sequence_hashset(
result
}
fn parse_message_id_header(header_bytes: &[u8]) -> Option<String> {
let header = std::str::from_utf8(header_bytes).ok()?;
for line in header.lines() {
if let Some(value) = line
.strip_prefix("Message-ID:")
.or_else(|| line.strip_prefix("Message-Id:"))
.or_else(|| line.strip_prefix("Message-id:"))
{
// mail_parser strips angle brackets, so we must do the same
// to ensure comparisons against the Tantivy index match.
let trimmed = value.trim();
let stripped = trimmed.strip_prefix('<').unwrap_or(trimmed);
let stripped = stripped.strip_suffix('>').unwrap_or(stripped);
if !stripped.is_empty() {
return Some(stripped.to_string());
}
}
}
None
}
#[cfg(test)]
mod test {
use super::*;
@@ -668,4 +722,80 @@ mod test {
assert_eq!(batches[2].0, "5");
assert_eq!(batches[2].1, 1);
}
// ── parse_message_id_header ─────────────────────────────────────
#[test]
fn parse_standard_message_id() {
let header = b"Message-ID: <abc123@example.com>\r\n";
assert_eq!(
parse_message_id_header(header),
Some("abc123@example.com".into())
);
}
#[test]
fn parse_message_id_lowercase() {
let header = b"Message-Id: <foo@bar.com>\r\n";
assert_eq!(
parse_message_id_header(header),
Some("foo@bar.com".into())
);
}
#[test]
fn parse_message_id_extra_whitespace() {
let header = b"Message-ID: <spaces@test.com> \r\n";
assert_eq!(
parse_message_id_header(header),
Some("spaces@test.com".into())
);
}
#[test]
fn parse_empty_message_id_returns_none() {
let header = b"Message-ID: <>\r\n";
assert_eq!(parse_message_id_header(header), None);
}
#[test]
fn parse_missing_header_returns_none() {
let header = b"X-Custom: something\r\n";
assert_eq!(parse_message_id_header(header), None);
}
#[test]
fn parse_empty_body_returns_none() {
assert_eq!(parse_message_id_header(b""), None);
}
#[test]
fn parse_message_id_in_full_header() {
// The Message-ID line is in the middle, not at the start.
let header = b"From: sender@example.com\r\n\
Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\
Subject: test\r\n\
Message-ID: <mid@example.com>\r\n\
To: recipient@example.com\r\n\r\n";
assert_eq!(
parse_message_id_header(header),
Some("mid@example.com".into())
);
}
#[test]
fn parse_message_id_only_in_full_header() {
// Only a few headers, Message-ID is among them.
let header = b"From: a@b.com\r\nMessage-ID: <x@y.com>\r\n\r\n";
assert_eq!(parse_message_id_header(header), Some("x@y.com".into()));
}
#[test]
fn parse_message_id_no_brackets_still_works() {
let header = b"Message-ID: plain@example.com\r\n";
assert_eq!(
parse_message_id_header(header),
Some("plain@example.com".into())
);
}
}

View File

@@ -95,16 +95,40 @@ impl ImapConnectionManager {
pub async fn build(account_id: u64) -> BichonResult<Session<Box<dyn SessionStream>>> {
let account = AccountModel::get(account_id)?;
let client = match Self::create_client(&account).await {
Ok(client) => client,
Err(error) => {
error!(
"Failed to create IMAP {}'s client: {:#?}",
&account.email, error
);
return Err(error);
let account_email = account.email.clone();
let mut client = None;
for attempt in 0..3u32 {
match Self::create_client(&account).await {
Ok(c) => {
client = Some(c);
break;
}
Err(error) if error.code() == ErrorCode::NetworkError && attempt < 2 => {
warn!(
"IMAP connection attempt {}/3 to {} failed (network error), retrying...",
attempt + 1,
account_email
);
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
continue;
}
Err(error) => {
error!(
"Failed to create IMAP {}'s client: {:#?}",
account_email, error
);
return Err(error);
}
}
};
}
let client = client.ok_or_else(|| {
raise_error!(
format!("Failed to create IMAP {}'s client after 3 attempts", account_email),
ErrorCode::NetworkError
)
})?;
let mut session = match Self::authenticate(client, &account).await {
Ok(session) => session,

View File

@@ -0,0 +1,538 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//! A minimal scriptable IMAP server for integration testing.
//!
//! Each instance listens on a random localhost port and responds to a
//! pre-configured script of (expected_command, response) pairs. Commands
//! are matched by substring — the first matching pattern wins.
//!
//! # Example
//! ```ignore
//! let server = MockImapServer::new()
//! .greeting("* OK ready\r\n")
//! .respond("LOGIN", "A0 OK logged in\r\n")
//! .respond("CAPABILITY", "* CAPABILITY IMAP4rev1\r\nA0 OK done\r\n")
//! .respond("STATUS", "* STATUS INBOX (MESSAGES 10 UIDVALIDITY 42)\r\nA0 OK\r\n")
//! .respond("LOGOUT", "* BYE\r\nA0 OK\r\n")
//! .start()
//! .await;
//!
//! let (host, port) = server.addr();
//! // connect to host:port with Encryption::None
//! ```
use std::net::SocketAddr;
use std::sync::Arc;
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::net::{TcpListener, TcpStream};
type Response = Vec<u8>;
pub struct MockImapServer {
greeting: Vec<u8>,
script: Vec<(String, Response)>,
}
impl MockImapServer {
pub fn new() -> Self {
Self {
greeting: b"* OK Mock IMAP server ready\r\n".to_vec(),
script: Vec::new(),
}
}
/// Set the greeting banner sent immediately after connection.
pub fn greeting(mut self, banner: impl Into<Vec<u8>>) -> Self {
self.greeting = banner.into();
self
}
/// Add a script step: when a client command *contains* `pattern` (case-insensitive),
/// respond with `response`. Steps are checked in insertion order.
pub fn respond(mut self, pattern: impl Into<String>, response: impl Into<Vec<u8>>) -> Self {
self.script.push((pattern.into(), response.into()));
self
}
/// Start the server on a random port. Returns a handle whose `addr()` gives
/// the `(host, port)` to connect to.
pub async fn start(self) -> MockImapServerHandle {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
let addr = listener.local_addr().expect("local_addr");
let server = Arc::new(self);
tokio::spawn(async move {
loop {
match listener.accept().await {
Ok((stream, _)) => {
let srv = server.clone();
tokio::spawn(async move {
srv.handle_connection(stream).await;
});
}
Err(_) => break,
}
}
});
MockImapServerHandle { addr }
}
async fn handle_connection(&self, mut stream: TcpStream) {
let (reader, mut writer) = stream.split();
let mut reader = BufReader::new(reader);
// Send greeting
if writer.write_all(&self.greeting).await.is_err() {
return;
}
let mut line = String::new();
loop {
line.clear();
match reader.read_line(&mut line).await {
Ok(0) => break, // EOF
Ok(_) => {}
Err(_) => break,
}
let tag = extract_tag(&line).unwrap_or("A0");
let matched = self.find_match(&line);
if let Some(response) = matched {
let substituted = substitute_tag(response, tag);
if writer.write_all(&substituted).await.is_err() {
break;
}
} else {
// Default: send tagged OK for commands we don't handle
let fallback = format!("{tag} OK done\r\n");
if writer.write_all(fallback.as_bytes()).await.is_err() {
break;
}
}
}
}
fn find_match(&self, line: &str) -> Option<&[u8]> {
let line_lower = line.to_lowercase();
for (pattern, response) in &self.script {
if line_lower.contains(&pattern.to_lowercase()) {
return Some(response);
}
}
None
}
}
impl Default for MockImapServer {
fn default() -> Self {
Self::new()
}
}
/// Handle to a running mock IMAP server. The server stops when this handle
/// is dropped.
pub struct MockImapServerHandle {
addr: SocketAddr,
}
impl MockImapServerHandle {
pub fn host(&self) -> String {
self.addr.ip().to_string()
}
pub fn port(&self) -> u16 {
self.addr.port()
}
}
fn extract_tag(line: &str) -> Option<&str> {
line.split_whitespace().next()
}
/// Replace `{TAG}` placeholders in `response` with `tag`.
fn substitute_tag(response: &[u8], tag: &str) -> Vec<u8> {
let placeholder = b"{TAG}";
if response.is_empty() || !contains_slice(response, placeholder) {
return response.to_vec();
}
let tag_bytes = tag.as_bytes();
let mut result = Vec::with_capacity(response.len());
let mut pos = 0;
while let Some(idx) = find_slice(&response[pos..], placeholder) {
result.extend_from_slice(&response[pos..pos + idx]);
result.extend_from_slice(tag_bytes);
pos += idx + placeholder.len();
}
result.extend_from_slice(&response[pos..]);
result
}
fn contains_slice(haystack: &[u8], needle: &[u8]) -> bool {
haystack.windows(needle.len()).any(|w| w == needle)
}
fn find_slice(haystack: &[u8], needle: &[u8]) -> Option<usize> {
haystack
.windows(needle.len())
.position(|w| w == needle)
}
// ============================================================
// Pre-built response helpers
// ============================================================
/// Build a tagged OK response.
pub fn ok(tag: impl AsRef<str>, msg: impl AsRef<str>) -> Vec<u8> {
format!("{} OK {}\r\n", tag.as_ref(), msg.as_ref()).into_bytes()
}
/// Build a STATUS response line.
pub fn status_response(
mailbox: &str,
messages: u32,
unseen: u32,
uid_next: u32,
uid_validity: Option<u32>,
) -> Vec<u8> {
let uv = uid_validity
.map(|v| format!(" UIDVALIDITY {v}"))
.unwrap_or_default();
let text = format!(
"* STATUS \"{mailbox}\" (MESSAGES {messages} UNSEEN {unseen} UIDNEXT {uid_next}{uv})\r\n"
);
// Clients expect a tagged response after the untagged STATUS line.
// We produce a generic OK that works for any tag.
let mut out = text.into_bytes();
out.extend_from_slice(b"{TAG} OK STATUS completed\r\n");
out
}
/// Build an EXAMINE response with mailbox data.
pub fn examine_response(
_mailbox: &str,
exists: u32,
uid_validity: u32,
uid_next: u32,
) -> Vec<u8> {
format!(
"* FLAGS (\\Seen \\Answered \\Flagged \\Deleted \\Draft)\r\n\
* OK [PERMANENTFLAGS ()]\r\n\
* {exists} EXISTS\r\n\
* 0 RECENT\r\n\
* OK [UIDVALIDITY {uid_validity}]\r\n\
* OK [UIDNEXT {uid_next}]\r\n\
* OK [HIGHESTMODSEQ 1]\r\n\
{{TAG}} OK [READ-ONLY] EXAMINE completed\r\n"
)
.into_bytes()
}
/// Build a UID SEARCH response for the given UID list.
pub fn uid_search_response(uids: &[u32]) -> Vec<u8> {
let uid_str = uids
.iter()
.map(|u| u.to_string())
.collect::<Vec<_>>()
.join(" ");
format!("* SEARCH {uid_str}\r\n{{TAG}} OK SEARCH completed\r\n").into_bytes()
}
/// Build a UID FETCH response returning full headers (for BODY[HEADER]).
/// Each entry: (uid, message_id)
pub fn uid_fetch_metadata_response(entries: &[(u32, &str)]) -> Vec<u8> {
let mut out = Vec::new();
for (uid, msg_id) in entries {
// Build a minimal header that contains the Message-ID line.
let header_data = format!(
"From: sender@example.com\r\n\
To: recipient@example.com\r\n\
Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\
Subject: test\r\n\
Message-ID: {msg_id}\r\n\r\n"
);
let header_len = header_data.len();
let line = format!(
"* {uid} FETCH (UID {uid} BODY[HEADER] {{{header_len}}}\r\n\
{header_data}\
)\r\n",
);
out.extend_from_slice(line.as_bytes());
}
out.extend_from_slice(b"{TAG} OK FETCH completed\r\n");
out
}
/// Build a UID FETCH RFC822 response with a full email body.
pub fn uid_fetch_rfc822_response(uid: u32, eml: &[u8]) -> Vec<u8> {
let header = format!(
"* {uid} FETCH (UID {uid} RFC822 {{{len}}}\r\n",
len = eml.len()
);
let mut out = header.into_bytes();
out.extend_from_slice(eml);
out.extend_from_slice(b")\r\n{TAG} OK FETCH completed\r\n");
out
}
/// A minimal RFC822 email fixture for testing.
pub fn minimal_eml(subject: &str, message_id: &str) -> Vec<u8> {
format!(
"From: sender@example.com\r\n\
To: recipient@example.com\r\n\
Subject: {subject}\r\n\
Message-ID: <{message_id}>\r\n\
Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\
MIME-Version: 1.0\r\n\
Content-Type: text/plain; charset=utf-8\r\n\
\r\n\
This is a test email: {subject}.\r\n"
)
.into_bytes()
}
// ============================================================
// Self-tests for the mock server itself
// ============================================================
#[cfg(test)]
mod tests {
use super::*;
async fn connect_and_read_greeting(host: &str, port: u16) -> String {
let mut stream = TcpStream::connect((host, port)).await.unwrap();
let (reader, _writer) = stream.split();
let mut reader = BufReader::new(reader);
let mut line = String::new();
reader.read_line(&mut line).await.unwrap();
line
}
async fn send_and_recv(host: &str, port: u16, cmd: &str) -> String {
let mut stream = TcpStream::connect((host, port)).await.unwrap();
let (reader, mut writer) = stream.split();
let mut reader = BufReader::new(reader);
// Read greeting
let mut line = String::new();
reader.read_line(&mut line).await.unwrap();
// Send command
writer.write_all(cmd.as_bytes()).await.unwrap();
writer.write_all(b"\r\n").await.unwrap();
// Read response (may be multi-line; read until tagged response)
let mut out = String::new();
loop {
line.clear();
reader.read_line(&mut line).await.unwrap();
out.push_str(&line);
if line.starts_with("A0") || line.starts_with("A1") {
break;
}
}
out
}
#[tokio::test]
async fn test_mock_greeting() {
let handle = MockImapServer::new().start().await;
let greeting = connect_and_read_greeting(&handle.host(), handle.port()).await;
assert!(greeting.starts_with("* OK"));
}
#[tokio::test]
async fn test_mock_scripted_response() {
let handle = MockImapServer::new()
.respond(
"LOGIN",
"A0 OK LOGIN completed\r\n",
)
.start()
.await;
let resp = send_and_recv(&handle.host(), handle.port(), "A0 LOGIN u p").await;
assert!(resp.contains("LOGIN completed"));
}
#[tokio::test]
async fn test_mock_fallback_on_unmatched() {
let handle = MockImapServer::new().start().await;
// Send a command that has no scripted response
let resp = send_and_recv(&handle.host(), handle.port(), "A0 NOOP").await;
assert!(resp.contains("OK done"), "unmatched command should get fallback OK");
}
#[tokio::test]
async fn test_status_response_helper() {
let resp = status_response("INBOX", 10, 2, 11, Some(42));
let text = String::from_utf8(resp).unwrap();
assert!(text.contains("MESSAGES 10"));
assert!(text.contains("UNSEEN 2"));
assert!(text.contains("UIDNEXT 11"));
assert!(text.contains("UIDVALIDITY 42"));
}
#[tokio::test]
async fn test_status_response_without_uidvalidity() {
let resp = status_response("INBOX", 10, 2, 11, None);
let text = String::from_utf8(resp).unwrap();
assert!(!text.contains("UIDVALIDITY"));
assert!(text.contains("MESSAGES 10"));
}
#[tokio::test]
async fn test_examine_response() {
let resp = examine_response("INBOX", 10, 42, 11);
let text = String::from_utf8(resp).unwrap();
assert!(text.contains("UIDVALIDITY 42"));
assert!(text.contains("10 EXISTS"));
}
#[tokio::test]
async fn test_uid_search_response() {
let resp = uid_search_response(&[1, 3, 5]);
let text = String::from_utf8(resp).unwrap();
assert!(text.contains("SEARCH 1 3 5"));
}
#[tokio::test]
async fn test_uid_fetch_metadata_response() {
let resp = uid_fetch_metadata_response(&[(1, "msg-a@x.com"), (2, "msg-b@x.com")]);
let text = String::from_utf8(resp).unwrap();
assert!(text.contains("Message-ID: msg-a@x.com"));
assert!(text.contains("Message-ID: msg-b@x.com"));
}
#[tokio::test]
async fn test_multiple_commands_in_sequence() {
let handle = MockImapServer::new()
.respond("LOGIN", "A0 OK LOGIN\r\n")
.respond("STATUS", status_response("INBOX", 5, 1, 6, Some(99)))
.respond("LOGOUT", "* BYE\r\nA0 OK\r\n")
.start()
.await;
let mut stream = TcpStream::connect((handle.host(), handle.port()))
.await
.unwrap();
let (reader, mut writer) = stream.split();
let mut reader = BufReader::new(reader);
// Read greeting
let mut buf = String::new();
reader.read_line(&mut buf).await.unwrap();
// LOGIN
writer.write_all(b"A0 LOGIN u p\r\n").await.unwrap();
buf.clear();
reader.read_line(&mut buf).await.unwrap();
assert!(buf.contains("LOGIN"));
// STATUS
writer
.write_all(b"A0 STATUS INBOX (MESSAGES UNSEEN UIDNEXT UIDVALIDITY)\r\n")
.await
.unwrap();
buf.clear();
// Read multi-line STATUS response (untagged line + tagged OK)
loop {
reader.read_line(&mut buf).await.unwrap();
if buf.contains("UIDVALIDITY 99") {
// Consume the tagged OK line that follows
buf.clear();
reader.read_line(&mut buf).await.unwrap();
break;
}
}
// LOGOUT
writer.write_all(b"A0 LOGOUT\r\n").await.unwrap();
buf.clear();
reader.read_line(&mut buf).await.unwrap();
assert!(buf.contains("BYE"));
}
#[tokio::test]
async fn test_tag_substitution_in_response() {
// Use {TAG} placeholder in the response and verify it gets the
// client's actual tag ("A5") substituted in.
let handle = MockImapServer::new()
.respond("LOGIN", "{TAG} OK LOGIN succeeded\r\n")
.start()
.await;
let mut stream = TcpStream::connect((handle.host(), handle.port()))
.await
.unwrap();
let (reader, mut writer) = stream.split();
let mut reader = BufReader::new(reader);
// Read greeting
let mut buf = String::new();
reader.read_line(&mut buf).await.unwrap();
// Send LOGIN with non-standard tag
writer.write_all(b"A5 LOGIN u p\r\n").await.unwrap();
buf.clear();
reader.read_line(&mut buf).await.unwrap();
assert!(
buf.contains("A5 OK LOGIN succeeded"),
"expected 'A5 OK LOGIN succeeded', got '{buf}'"
);
}
#[tokio::test]
async fn test_tag_substitution_multiple_placeholders() {
let handle = MockImapServer::new()
.respond("NOOP", "* 0 RECENT\r\n{TAG} OK NOOP done\r\n")
.start()
.await;
let mut stream = TcpStream::connect((handle.host(), handle.port()))
.await
.unwrap();
let (reader, mut writer) = stream.split();
let mut reader = BufReader::new(reader);
// Read greeting
let mut buf = String::new();
reader.read_line(&mut buf).await.unwrap();
// Send with tag "B99"
writer.write_all(b"B99 NOOP\r\n").await.unwrap();
// Read all lines
let mut all = String::new();
loop {
buf.clear();
reader.read_line(&mut buf).await.unwrap();
all.push_str(&buf);
if buf.starts_with("B99") {
break;
}
}
assert!(all.contains("* 0 RECENT\r\n"));
assert!(all.contains("B99 OK NOOP done\r\n"));
}
}

View File

@@ -26,3 +26,5 @@ pub mod session;
pub mod stats;
#[cfg(test)]
mod tests;
#[cfg(test)]
pub mod mock_server;

View File

@@ -172,6 +172,52 @@ impl DedupCache {
POPULATE_WINDOW_MS / (24 * 60 * 60 * 1000),
);
}
/// Remove all entries for a specific account.
pub fn remove_by_account(&self, account_id: u64) {
let mut entries = self.entries.lock().unwrap();
let before = entries.len();
entries.retain(|(aid, _, _), _| *aid != account_id);
let removed = before - entries.len();
if removed > 0 {
tracing::info!(
"DedupCache: removed {} entries for account {}",
removed,
account_id
);
}
}
/// Remove all entries for a specific mailbox (across all accounts).
pub fn remove_by_mailbox(&self, mailbox_id: u64) {
let mut entries = self.entries.lock().unwrap();
let before = entries.len();
entries.retain(|(_, mid, _), _| *mid != mailbox_id);
let removed = before - entries.len();
if removed > 0 {
tracing::info!(
"DedupCache: removed {} entries for mailbox {}",
removed,
mailbox_id
);
}
}
/// Remove a specific triple (most precise removal).
pub fn remove(&self, account_id: u64, mailbox_id: u64, hash: &str) {
let mut entries = self.entries.lock().unwrap();
if entries
.remove(&(account_id, mailbox_id, hash.to_string()))
.is_some()
{
tracing::debug!(
"DedupCache: removed specific entry ({}, {}, {})",
account_id,
mailbox_id,
hash
);
}
}
}
// ── Tests ─────────────────────────────────────────────────────────────────────
@@ -366,7 +412,11 @@ mod tests {
for segment_reader in searcher.segment_readers() {
let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap();
let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap();
let hash_col = segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap().unwrap();
let hash_col = segment_reader
.fast_fields()
.str(F_CONTENT_HASH)
.unwrap()
.unwrap();
let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap();
let max_doc = segment_reader.max_doc();
@@ -381,7 +431,11 @@ mod tests {
let account_id = account_col.values.get_val(doc_id);
let mailbox_id = mailbox_col.values.get_val(doc_id);
let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0);
let hash_ord = hash_col
.ords()
.values_for_doc(doc_id as u32)
.next()
.unwrap_or(0);
let mut hash_buf = String::new();
hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap();
@@ -419,7 +473,11 @@ mod tests {
for segment_reader in searcher.segment_readers() {
let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap();
let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap();
let hash_col = segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap().unwrap();
let hash_col = segment_reader
.fast_fields()
.str(F_CONTENT_HASH)
.unwrap()
.unwrap();
let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap();
let max_doc = segment_reader.max_doc();
@@ -434,7 +492,11 @@ mod tests {
let account_id = account_col.values.get_val(doc_id);
let mailbox_id = mailbox_col.values.get_val(doc_id);
let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0);
let hash_ord = hash_col
.ords()
.values_for_doc(doc_id as u32)
.next()
.unwrap_or(0);
let mut hash_buf = String::new();
hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap();
@@ -474,7 +536,11 @@ mod tests {
for segment_reader in searcher.segment_readers() {
let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap();
let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap();
let hash_col = segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap().unwrap();
let hash_col = segment_reader
.fast_fields()
.str(F_CONTENT_HASH)
.unwrap()
.unwrap();
let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap();
let max_doc = segment_reader.max_doc();
@@ -489,7 +555,11 @@ mod tests {
let account_id = account_col.values.get_val(doc_id);
let mailbox_id = mailbox_col.values.get_val(doc_id);
let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0);
let hash_ord = hash_col
.ords()
.values_for_doc(doc_id as u32)
.next()
.unwrap_or(0);
let mut hash_buf = String::new();
hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap();
@@ -501,4 +571,71 @@ mod tests {
assert!(cache.contains(1, 10, "hash-keep"));
assert!(!cache.contains(1, 10, "hash-delete"));
}
// ── removal methods ─────────────────────────────────────────────────────
#[test]
fn remove_by_account_works() {
let cache = DedupCache::new_for_test();
cache.insert(1, 10, "hash-a1");
cache.insert(1, 20, "hash-a2");
cache.insert(2, 10, "hash-b1");
cache.insert(2, 30, "hash-b2");
cache.insert(1, 10, "hash-a3");
assert_eq!(cache.entries.lock().unwrap().len(), 5);
cache.remove_by_account(1);
let entries = cache.entries.lock().unwrap();
assert_eq!(entries.len(), 2);
assert!(!entries.contains_key(&(1, 10, "hash-a1".to_string())));
assert!(!entries.contains_key(&(1, 20, "hash-a2".to_string())));
assert!(!entries.contains_key(&(1, 10, "hash-a3".to_string())));
assert!(entries.contains_key(&(2, 10, "hash-b1".to_string())));
assert!(entries.contains_key(&(2, 30, "hash-b2".to_string())));
}
#[test]
fn remove_by_mailbox_works() {
let cache = DedupCache::new_for_test();
cache.insert(1, 10, "hash-1");
cache.insert(1, 20, "hash-2");
cache.insert(2, 10, "hash-3");
cache.insert(3, 20, "hash-4");
cache.insert(1, 10, "hash-5");
cache.remove_by_mailbox(10);
let entries = cache.entries.lock().unwrap();
assert_eq!(entries.len(), 2);
assert!(entries.contains_key(&(1, 20, "hash-2".to_string())));
assert!(entries.contains_key(&(3, 20, "hash-4".to_string())));
assert!(!entries.contains_key(&(1, 10, "hash-1".to_string())));
assert!(!entries.contains_key(&(2, 10, "hash-3".to_string())));
}
#[test]
fn remove_specific_triple_works() {
let cache = DedupCache::new_for_test();
cache.insert(1, 10, "hash-aaa");
cache.insert(1, 10, "hash-bbb");
cache.insert(2, 20, "hash-aaa");
assert!(cache.contains(1, 10, "hash-aaa"));
assert!(cache.contains(1, 10, "hash-bbb"));
assert!(cache.contains(2, 20, "hash-aaa"));
cache.remove(1, 10, "hash-aaa");
assert!(!cache.contains(1, 10, "hash-aaa"));
assert!(cache.contains(1, 10, "hash-bbb"));
assert!(cache.contains(2, 20, "hash-aaa"));
}
}

View File

@@ -40,6 +40,7 @@ use crate::{
envelope::Envelope,
tantivy::{
attachment::ATTACHMENT_MANAGER,
dedup_cache::DEDUP_CACHE,
fatal_commit,
fields::{
F_ACCOUNT_ID, F_DATE, F_FROM, F_ID, F_INGEST_AT, F_INTERNAL_DATE,
@@ -50,8 +51,8 @@ use crate::{
tokenizers::EuroTokenizer,
},
},
utils::html::extract_text,
utc_now,
utils::html::extract_text,
};
use chrono::Utc;
@@ -277,6 +278,80 @@ impl IndexManager {
Box::new(boolean_query)
}
/// Return all Message-IDs stored in Tantivy for a given mailbox.
/// Prefer `mailbox_contains_message_id` for existence checks on large
/// mailboxes — this method loads everything into a HashSet.
pub fn get_message_ids_for_mailbox(
&self,
account_id: u64,
mailbox_id: u64,
) -> BichonResult<HashSet<String>> {
let query = self.mailbox_query(account_id, mailbox_id);
let fields = SchemaTools::email_fields();
let searcher = self.create_searcher()?;
let docs = searcher
.search(&query, &DocSetCollector)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
let mut result = HashSet::new();
for doc_address in docs {
let doc = searcher
.doc::<TantivyDocument>(doc_address)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
if let Some(v) = doc.get_first(fields.f_message_id) {
if let Some(s) = v.as_str() {
if !s.is_empty() {
result.insert(s.to_string());
}
}
}
}
Ok(result)
}
/// Check whether a specific Message-ID exists in a mailbox.
/// Uses a TermQuery — O(1) per call, no allocation proportional to
/// mailbox size. Suitable for large mailboxes where
/// `get_message_ids_for_mailbox` would allocate too much memory.
pub fn mailbox_contains_message_id(
&self,
account_id: u64,
mailbox_id: u64,
message_id: &str,
) -> BichonResult<bool> {
let fields = SchemaTools::email_fields();
let query = BooleanQuery::new(vec![
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_u64(fields.f_account_id, account_id),
IndexRecordOption::Basic,
)),
),
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_u64(fields.f_mailbox_id, mailbox_id),
IndexRecordOption::Basic,
)),
),
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_text(fields.f_message_id, message_id),
IndexRecordOption::Basic,
)),
),
]);
let searcher = self.create_searcher()?;
let count = searcher
.search(&query, &Count)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
Ok(count > 0)
}
fn envelope_query(&self, account_id: u64, eid: &str) -> Box<dyn Query> {
let account_id_query = TermQuery::new(
Term::from_field_u64(SchemaTools::email_fields().f_account_id, account_id),
@@ -792,6 +867,8 @@ impl IndexManager {
attachments_content_hashes,
)?;
}
DEDUP_CACHE.remove_by_account(account_id);
Ok(())
}
@@ -815,8 +892,8 @@ impl IndexManager {
}
let mut queries: Vec<Box<dyn Query>> = Vec::with_capacity(mailbox_ids.len());
for mailbox_id in mailbox_ids {
queries.push(self.mailbox_query(account_id, mailbox_id));
for mailbox_id in &mailbox_ids {
queries.push(self.mailbox_query(account_id, *mailbox_id));
}
let mut writer = self.index_writer.lock().await;
for query in queries {
@@ -835,6 +912,11 @@ impl IndexManager {
attachments_content_hashes,
)?;
}
for mailbox_id in mailbox_ids {
DEDUP_CACHE.remove_by_mailbox(mailbox_id);
}
Ok(())
}
@@ -842,6 +924,21 @@ impl IndexManager {
&self,
query: Box<dyn Query>,
) -> BichonResult<(HashSet<String>, HashSet<String>)> {
let (eml_with_mailbox, attachments_content_hashes) =
self.collect_content_hashes_with_mailbox(query)?;
let eml_content_hashes = eml_with_mailbox
.into_iter()
.map(|(hash, _mailbox_id)| hash)
.collect();
Ok((eml_content_hashes, attachments_content_hashes))
}
fn collect_content_hashes_with_mailbox(
&self,
query: Box<dyn Query>,
) -> BichonResult<(HashSet<(String, u64)>, HashSet<String>)> {
let mut eml_content_hashes = HashSet::new();
let mut attachments_content_hashes = HashSet::new();
@@ -857,10 +954,14 @@ impl IndexManager {
.doc::<TantivyDocument>(doc_address)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
let mailbox_id = doc.get_first(fields.f_mailbox_id).and_then(|v| v.as_u64());
// Extract content_hash
if let Some(content_hash_value) = doc.get_first(fields.f_content_hash) {
if let Some(str) = content_hash_value.as_str() {
eml_content_hashes.insert(str.to_string());
if let (Some(hash_str), Some(mailbox_id)) =
(content_hash_value.as_str(), mailbox_id)
{
eml_content_hashes.insert((hash_str.to_string(), mailbox_id));
}
}
@@ -933,7 +1034,7 @@ impl IndexManager {
return Ok(());
}
let mut eml_content_hashes: HashSet<String> = HashSet::new();
let mut eml_content_hash_triples: HashSet<(u64, u64, String)> = HashSet::new();
let mut attachments_content_hashes: HashSet<String> = HashSet::new();
for (account_id, envelope_ids) in &deletes {
@@ -944,8 +1045,14 @@ impl IndexManager {
for eid in unique_ids {
let query = self.envelope_query(*account_id, eid);
let (eml_hashes, attachment_hashes) = self.collect_content_hashes(query)?;
eml_content_hashes.extend(eml_hashes);
let (eml_hashes_with_mailbox, attachment_hashes) =
self.collect_content_hashes_with_mailbox(query)?;
eml_content_hash_triples.extend(
eml_hashes_with_mailbox
.into_iter()
.map(|(hash, mailbox_id)| (*account_id, mailbox_id, hash)),
);
attachments_content_hashes.extend(attachment_hashes);
}
}
@@ -968,7 +1075,12 @@ impl IndexManager {
.commit()
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() {
if !eml_content_hash_triples.is_empty() || !attachments_content_hashes.is_empty() {
let eml_content_hashes: HashSet<String> = eml_content_hash_triples
.iter()
.map(|(_, _, hash)| hash.clone())
.collect();
self.cleanup_unused_content(
&mut writer,
eml_content_hashes,
@@ -976,6 +1088,10 @@ impl IndexManager {
)?;
}
for (aid, mid, hash) in eml_content_hash_triples {
DEDUP_CACHE.remove(aid, mid, &hash);
}
Ok(())
}
@@ -1157,8 +1273,7 @@ impl IndexManager {
// f_attachments JSON blob.
if let Some(attrs_val) = old_doc.get_first(f.f_attachments) {
if let Some(json_str) = attrs_val.as_str() {
if let Ok(parsed) =
serde_json::from_str::<serde_json::Value>(json_str)
if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(json_str)
{
if let Some(arr) = parsed.as_array() {
for att in arr {
@@ -1179,14 +1294,8 @@ impl IndexManager {
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
{
new_doc.add_text(
f.f_attachment_name_text,
filename,
);
new_doc.add_text(
f.f_attachment_name_exact,
filename,
);
new_doc.add_text(f.f_attachment_name_text, filename);
new_doc.add_text(f.f_attachment_name_exact, filename);
}
}
}
@@ -1200,22 +1309,18 @@ impl IndexManager {
if let Some(content_hash) = hash_val.as_str() {
match BLOB_MANAGER.get_email(content_hash) {
Ok(Some(eml_bytes)) => {
if let Some(message) =
MessageParser::new().parse(&eml_bytes)
{
if let Some(message) = MessageParser::new().parse(&eml_bytes) {
let text = message
.body_text(0)
.map(|cow| cow.into_owned())
.or_else(|| {
message.body_html(0).map(|cow| {
extract_text(cow.into_owned())
})
message
.body_html(0)
.map(|cow| extract_text(cow.into_owned()))
})
.unwrap_or_default();
let body_text = text
.split_whitespace()
.collect::<Vec<_>>()
.join(" ");
let body_text =
text.split_whitespace().collect::<Vec<_>>().join(" ");
if !body_text.is_empty() {
new_doc.add_text(f.f_body, &body_text);
}
@@ -1304,7 +1409,7 @@ impl IndexManager {
let mailbox_docs: Vec<DocAddress>;
match sort_by {
SortBy::DATE => {
SortBy::DATE => {
let date_docs: Vec<(Option<i64>, DocAddress)> = searcher
.search(
&query,
@@ -1801,10 +1906,8 @@ mod tests {
if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(json_str) {
if let Some(arr) = parsed.as_array() {
for att in arr {
let is_inline = att
.get("inline")
.and_then(|v| v.as_bool())
.unwrap_or(false);
let is_inline =
att.get("inline").and_then(|v| v.as_bool()).unwrap_or(false);
let has_cid = att
.get("content_id")
.and_then(|v| v.as_str())
@@ -1841,8 +1944,7 @@ mod tests {
.map(|cow| extract_text(cow.into_owned()))
})
.unwrap_or_default();
let body_text =
text.split_whitespace().collect::<Vec<_>>().join(" ");
let body_text = text.split_whitespace().collect::<Vec<_>>().join(" ");
if !body_text.is_empty() {
new_doc.add_text(f.f_body, &body_text);
}
@@ -1911,8 +2013,7 @@ mod tests {
let mut writer2 = index
.writer_with_num_threads(1, 15_000_000)
.expect("writer2");
writer2
.delete_term(Term::from_field_text(f.f_id, "test-eid-001"));
writer2.delete_term(Term::from_field_text(f.f_id, "test-eid-001"));
writer2.add_document(new_doc).unwrap();
writer2.commit().unwrap();
@@ -1922,16 +2023,14 @@ mod tests {
let searcher = reader.searcher();
// Body text (tokenized via "euro")
let body_parser =
QueryParser::for_index(&index, vec![f.f_body]);
let body_parser = QueryParser::for_index(&index, vec![f.f_body]);
let body_hits = searcher
.search(&body_parser.parse_query("quick brown fox").unwrap(), &Count)
.unwrap();
assert_eq!(body_hits, 1, "body text should survive tag update");
// from_text (tokenized via "euro")
let from_parser =
QueryParser::for_index(&index, vec![f.f_from_text]);
let from_parser = QueryParser::for_index(&index, vec![f.f_from_text]);
let from_hits = searcher
.search(
&from_parser.parse_query("alice@example.com").unwrap(),
@@ -1943,10 +2042,7 @@ mod tests {
// to_text
let to_parser = QueryParser::for_index(&index, vec![f.f_to_text]);
let to_hits = searcher
.search(
&to_parser.parse_query("bob@example.com").unwrap(),
&Count,
)
.search(&to_parser.parse_query("bob@example.com").unwrap(), &Count)
.unwrap();
assert_eq!(to_hits, 1, "to_text should survive tag update");
@@ -1969,10 +2065,7 @@ mod tests {
let tags_hits = searcher
.search(
&TermQuery::new(
Term::from_facet(
f.f_tags,
&Facet::from_text("/important").unwrap(),
),
Term::from_facet(f.f_tags, &Facet::from_text("/important").unwrap()),
IndexRecordOption::Basic,
),
&Count,
@@ -1984,19 +2077,13 @@ mod tests {
let old_tag_hits = searcher
.search(
&TermQuery::new(
Term::from_facet(
f.f_tags,
&Facet::from_text("/unread").unwrap(),
),
Term::from_facet(f.f_tags, &Facet::from_text("/unread").unwrap()),
IndexRecordOption::Basic,
),
&Count,
)
.unwrap();
assert_eq!(
old_tag_hits, 0,
"old tag /unread should have been removed"
);
assert_eq!(old_tag_hits, 0, "old tag /unread should have been removed");
}
#[test]
@@ -2056,4 +2143,362 @@ mod tests {
"body should be absent when EML is missing"
);
}
// ── get_message_ids_for_mailbox ─────────────────────────────────
#[test]
fn get_message_ids_returns_stored_ids() {
let f = SchemaTools::email_fields();
let index = Index::create_in_ram(SchemaTools::email_schema());
index.tokenizers().register("euro", EuroTokenizer::new());
// Insert two docs for mailbox 10, one for mailbox 20
{
let mut writer = index
.writer_with_num_threads(1, 15_000_000)
.expect("writer");
let mut doc1 = TantivyDocument::new();
doc1.add_u64(f.f_account_id, 1);
doc1.add_u64(f.f_mailbox_id, 10);
doc1.add_text(f.f_message_id, "<msg-a@test>");
doc1.add_text(f.f_id, "id-a");
doc1.add_u64(f.f_uid, 1);
doc1.add_text(f.f_content_hash, "hash-a");
writer.add_document(doc1).unwrap();
let mut doc2 = TantivyDocument::new();
doc2.add_u64(f.f_account_id, 1);
doc2.add_u64(f.f_mailbox_id, 10);
doc2.add_text(f.f_message_id, "<msg-b@test>");
doc2.add_text(f.f_id, "id-b");
doc2.add_u64(f.f_uid, 2);
doc2.add_text(f.f_content_hash, "hash-b");
writer.add_document(doc2).unwrap();
let mut doc3 = TantivyDocument::new();
doc3.add_u64(f.f_account_id, 1);
doc3.add_u64(f.f_mailbox_id, 20);
doc3.add_text(f.f_message_id, "<msg-c@test>");
doc3.add_text(f.f_id, "id-c");
doc3.add_u64(f.f_uid, 3);
doc3.add_text(f.f_content_hash, "hash-c");
writer.add_document(doc3).unwrap();
writer.commit().unwrap();
}
let reader = index.reader().unwrap();
reader.reload().unwrap();
let searcher = reader.searcher();
// We can't easily call ENVELOPE_MANAGER.get_message_ids_for_mailbox
// because it reads from ENVELOPE_MANAGER's own index, not our in-memory one.
// Instead, test the query pattern directly.
let query: Box<dyn Query> = {
let account_query = TermQuery::new(
Term::from_field_u64(f.f_account_id, 1),
IndexRecordOption::Basic,
);
let mailbox_query = TermQuery::new(
Term::from_field_u64(f.f_mailbox_id, 10),
IndexRecordOption::Basic,
);
Box::new(BooleanQuery::new(vec![
(Occur::Must, Box::new(account_query)),
(Occur::Must, Box::new(mailbox_query)),
]))
};
let docs = searcher
.search(&query, &DocSetCollector)
.unwrap();
let mut ids: Vec<String> = Vec::new();
for addr in docs {
let doc: TantivyDocument = searcher.doc(addr).unwrap();
if let Some(v) = doc.get_first(f.f_message_id) {
if let Some(s) = v.as_str() {
ids.push(s.to_string());
}
}
}
ids.sort();
assert_eq!(ids, vec!["<msg-a@test>", "<msg-b@test>"]);
}
#[test]
fn get_message_ids_empty_mailbox_returns_empty() {
let f = SchemaTools::email_fields();
let index = Index::create_in_ram(SchemaTools::email_schema());
index.tokenizers().register("euro", EuroTokenizer::new());
{
let mut writer = index
.writer_with_num_threads(1, 15_000_000)
.expect("writer");
// Doc for a different mailbox
let mut doc = TantivyDocument::new();
doc.add_u64(f.f_account_id, 1);
doc.add_u64(f.f_mailbox_id, 99);
doc.add_text(f.f_message_id, "<other@test>");
doc.add_text(f.f_id, "id-other");
doc.add_u64(f.f_uid, 1);
doc.add_text(f.f_content_hash, "hash-other");
writer.add_document(doc).unwrap();
writer.commit().unwrap();
}
let reader = index.reader().unwrap();
reader.reload().unwrap();
let searcher = reader.searcher();
let query: Box<dyn Query> = {
let account_query = TermQuery::new(
Term::from_field_u64(f.f_account_id, 1),
IndexRecordOption::Basic,
);
let mailbox_query = TermQuery::new(
Term::from_field_u64(f.f_mailbox_id, 10),
IndexRecordOption::Basic,
);
Box::new(BooleanQuery::new(vec![
(Occur::Must, Box::new(account_query)),
(Occur::Must, Box::new(mailbox_query)),
]))
};
let docs = searcher.search(&query, &DocSetCollector).unwrap();
assert!(docs.is_empty());
}
// ── mailbox_contains_message_id ───────────────────────────────
#[test]
fn mailbox_contains_message_id_finds_existing() {
let f = SchemaTools::email_fields();
let index = Index::create_in_ram(SchemaTools::email_schema());
index.tokenizers().register("euro", EuroTokenizer::new());
{
let mut writer = index
.writer_with_num_threads(1, 15_000_000)
.expect("writer");
let mut doc = TantivyDocument::new();
doc.add_u64(f.f_account_id, 1);
doc.add_u64(f.f_mailbox_id, 10);
doc.add_text(f.f_message_id, "abc@example.com");
doc.add_text(f.f_id, "id-1");
doc.add_u64(f.f_uid, 1);
doc.add_text(f.f_content_hash, "hash-1");
writer.add_document(doc).unwrap();
writer.commit().unwrap();
}
// We test the query pattern directly (can't call ENVELOPE_MANAGER
// which uses a different index).
let reader = index.reader().unwrap();
reader.reload().unwrap();
let searcher = reader.searcher();
let query = BooleanQuery::new(vec![
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_u64(f.f_account_id, 1),
IndexRecordOption::Basic,
)),
),
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_u64(f.f_mailbox_id, 10),
IndexRecordOption::Basic,
)),
),
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_text(f.f_message_id, "abc@example.com"),
IndexRecordOption::Basic,
)),
),
]);
let count = searcher.search(&query, &Count).unwrap();
assert_eq!(count, 1);
}
#[test]
fn mailbox_contains_message_id_returns_zero_for_missing() {
let f = SchemaTools::email_fields();
let index = Index::create_in_ram(SchemaTools::email_schema());
index.tokenizers().register("euro", EuroTokenizer::new());
{
let mut writer = index
.writer_with_num_threads(1, 15_000_000)
.expect("writer");
let mut doc = TantivyDocument::new();
doc.add_u64(f.f_account_id, 1);
doc.add_u64(f.f_mailbox_id, 10);
doc.add_text(f.f_message_id, "existing@example.com");
doc.add_text(f.f_id, "id-1");
doc.add_u64(f.f_uid, 1);
doc.add_text(f.f_content_hash, "hash-1");
writer.add_document(doc).unwrap();
writer.commit().unwrap();
}
let reader = index.reader().unwrap();
reader.reload().unwrap();
let searcher = reader.searcher();
let query = BooleanQuery::new(vec![
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_u64(f.f_account_id, 1),
IndexRecordOption::Basic,
)),
),
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_u64(f.f_mailbox_id, 10),
IndexRecordOption::Basic,
)),
),
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_text(f.f_message_id, "nonexistent@example.com"),
IndexRecordOption::Basic,
)),
),
]);
let count = searcher.search(&query, &Count).unwrap();
assert_eq!(count, 0);
}
#[test]
fn mailbox_contains_message_id_respects_mailbox_boundary() {
let f = SchemaTools::email_fields();
let index = Index::create_in_ram(SchemaTools::email_schema());
index.tokenizers().register("euro", EuroTokenizer::new());
{
let mut writer = index
.writer_with_num_threads(1, 15_000_000)
.expect("writer");
// Same Message-ID in mailbox 10
let mut doc1 = TantivyDocument::new();
doc1.add_u64(f.f_account_id, 1);
doc1.add_u64(f.f_mailbox_id, 10);
doc1.add_text(f.f_message_id, "shared@example.com");
doc1.add_text(f.f_id, "id-1");
doc1.add_u64(f.f_uid, 1);
doc1.add_text(f.f_content_hash, "hash-1");
writer.add_document(doc1).unwrap();
// Same Message-ID in mailbox 20 (different mailbox)
let mut doc2 = TantivyDocument::new();
doc2.add_u64(f.f_account_id, 1);
doc2.add_u64(f.f_mailbox_id, 20);
doc2.add_text(f.f_message_id, "shared@example.com");
doc2.add_text(f.f_id, "id-2");
doc2.add_u64(f.f_uid, 2);
doc2.add_text(f.f_content_hash, "hash-2");
writer.add_document(doc2).unwrap();
writer.commit().unwrap();
}
let reader = index.reader().unwrap();
reader.reload().unwrap();
let searcher = reader.searcher();
// Query mailbox 10: should find 1
let q10 = BooleanQuery::new(vec![
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_u64(f.f_account_id, 1),
IndexRecordOption::Basic,
)),
),
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_u64(f.f_mailbox_id, 10),
IndexRecordOption::Basic,
)),
),
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_text(f.f_message_id, "shared@example.com"),
IndexRecordOption::Basic,
)),
),
]);
assert_eq!(searcher.search(&q10, &Count).unwrap(), 1);
// Query mailbox 20: should find 1
let q20 = BooleanQuery::new(vec![
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_u64(f.f_account_id, 1),
IndexRecordOption::Basic,
)),
),
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_u64(f.f_mailbox_id, 20),
IndexRecordOption::Basic,
)),
),
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_text(f.f_message_id, "shared@example.com"),
IndexRecordOption::Basic,
)),
),
]);
assert_eq!(searcher.search(&q20, &Count).unwrap(), 1);
// Query mailbox 99 (no docs): should find 0
let q99 = BooleanQuery::new(vec![
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_u64(f.f_account_id, 1),
IndexRecordOption::Basic,
)),
),
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_u64(f.f_mailbox_id, 99),
IndexRecordOption::Basic,
)),
),
(
Occur::Must,
Box::new(TermQuery::new(
Term::from_field_text(f.f_message_id, "shared@example.com"),
IndexRecordOption::Basic,
)),
),
]);
assert_eq!(searcher.search(&q99, &Count).unwrap(), 0);
}
}

View File

@@ -21,6 +21,7 @@ use std::net::SocketAddr;
use std::time::Duration;
use base64::{prelude::BASE64_STANDARD, Engine as _};
use bichon_core::account::migration::AccountType;
use bichon_core::cache::imap::mailbox::{Attribute, AttributeEnum};
use bichon_core::common::signal::SIGNAL_MANAGER;
use bichon_core::envelope::extractor::extract_envelope_from_smtp;
@@ -429,8 +430,20 @@ where
}
if is_allowed {
session.rcpt_to.push(account);
stream.write_all(b"250 OK\r\n").await?;
if !matches!(account.account_type, AccountType::NoSync) {
tracing::warn!(
"SMTP: Rejected journaling attempt to IMAP account <{}>",
addr
);
let err = format!(
"550 5.7.1 <{}>: Not a Bichon local account, journaling is not supported\r\n",
account.email
);
stream.write_all(err.as_bytes()).await?;
} else {
session.rcpt_to.push(account);
stream.write_all(b"250 OK\r\n").await?;
}
}
}
Ok(None) => {
@@ -615,26 +628,40 @@ async fn parse_email(data: &[u8], session: &Session) -> BichonResult<()> {
return Ok(());
}
};
let mailbox = MailBox {
id: create_hash(rcpt.id, "INBOX"),
account_id: rcpt.id,
name: "INBOX".into(),
delimiter: Some("/".to_string()),
attributes: vec![Attribute {
attr: AttributeEnum::Extension,
extension: Some("CreatedByBichon".into()),
}],
exists: 0,
unseen: None,
uid_next: None,
uid_validity: None,
highest_uid: None,
};
let mailbox_id = mailbox.id;
let mailbox_id = create_hash(rcpt.id, "INBOX");
if let Err(e) = MailBox::batch_upsert(&[mailbox]) {
tracing::error!("SMTP: Failed to upsert mailbox for {}: {:?}", rcpt.email, e);
return Err(e.into());
// The INBOX row is owned by the IMAP sync, which maintains `uid_validity`,
// `highest_uid` and `uid_next` on it. `batch_upsert` replaces the *whole*
// row, so blindly upserting here (with those fields = None) clobbers the
// IMAP-maintained state back to None. The next reconcile then sees
// `uid_validity` change from Some -> None, treats the mailbox as invalid,
// and wipes + rebuilds it — silently losing the local copy of a large
// mailbox when that rebuild is interrupted (see #297).
//
// We only need the row to *exist* so the journaled envelope can attach to
// it, so create it only when it is missing and otherwise leave the
// IMAP-owned row untouched.
if MailBox::find_mailbox(rcpt.id, mailbox_id)?.is_none() {
let mailbox = MailBox {
id: mailbox_id,
account_id: rcpt.id,
name: "INBOX".into(),
delimiter: Some("/".to_string()),
attributes: vec![Attribute {
attr: AttributeEnum::Extension,
extension: Some("CreatedByBichon".into()),
}],
exists: 0,
unseen: None,
uid_next: None,
uid_validity: None,
highest_uid: None,
};
if let Err(e) = MailBox::batch_upsert(&[mailbox]) {
tracing::error!("SMTP: Failed to upsert mailbox for {}: {:?}", rcpt.email, e);
return Err(e.into());
}
}
extract_envelope_from_smtp(data, rcpt.id, mailbox_id)

View File

@@ -17,7 +17,7 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
import { useState } from 'react'
import { useEffect, useRef, useState } from 'react'
import {
ColumnDef,
ColumnFiltersState,
@@ -64,7 +64,19 @@ export function AccountTable({ columns, data }: DataTableProps) {
const [rowSelection, setRowSelection] = useState({})
const [columnVisibility, setColumnVisibility] = useState<VisibilityState>({})
const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([])
const [sorting, setSorting] = useState<SortingState>([])
const [sorting, setSorting] = useState<SortingState>(() => {
const saved = localStorage.getItem('bichon_accounts_sorting');
return saved ? JSON.parse(saved) : [];
})
// Persist sorting state to localStorage
const prevSortingRef = useRef(sorting);
useEffect(() => {
if (prevSortingRef.current !== sorting) {
localStorage.setItem('bichon_accounts_sorting', JSON.stringify(sorting));
prevSortingRef.current = sorting;
}
}, [sorting]);
const table = useReactTable({
data,