diff --git a/crates/core/src/cache/imap/download/download_folders.rs b/crates/core/src/cache/imap/download/download_folders.rs index ab40cba..4d8d0de 100644 --- a/crates/core/src/cache/imap/download/download_folders.rs +++ b/crates/core/src/cache/imap/download/download_folders.rs @@ -23,6 +23,7 @@ use crate::{ { account::migration::{AccountModel, AccountType}, cache::imap::mailbox::{AttributeEnum, MailBox}, + cache::imap::mailbox_cache, error::{code::ErrorCode, BichonResult}, imap::{executor::ImapExecutor, session::SessionStream}, mailbox::list::convert_names_to_mailboxes, @@ -179,6 +180,7 @@ pub async fn detect_mailbox_changes( // Update known folders only if there were changes if has_changes { AccountModel::update_known_folders(account.id, all_names)?; + mailbox_cache::invalidate(account.id).await; } Ok(()) } diff --git a/crates/core/src/cache/imap/mailbox_cache.rs b/crates/core/src/cache/imap/mailbox_cache.rs new file mode 100644 index 0000000..5adfc62 --- /dev/null +++ b/crates/core/src/cache/imap/mailbox_cache.rs @@ -0,0 +1,114 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use crate::cache::imap::mailbox::MailBox; +use crate::utc_now; +use lru::LruCache; +use std::collections::HashMap; +use std::num::NonZeroUsize; +use std::sync::LazyLock; +use tokio::sync::Mutex; + +struct CacheEntry { + mailboxes: Vec, + fetched_at: i64, +} + +static CACHE: LazyLock>> = LazyLock::new(|| { + Mutex::new(LruCache::new(NonZeroUsize::new(64).unwrap())) +}); + +const TTL_MS: i64 = 10 * 60 * 1000; // 10 minutes + +pub async fn get(account_id: u64) -> Option> { + let mut guard = CACHE.lock().await; + if let Some(entry) = guard.get(&account_id) { + if utc_now!() - entry.fetched_at < TTL_MS { + return Some(entry.mailboxes.clone()); + } + guard.pop(&account_id); + } + None +} + +pub async fn set(account_id: u64, mailboxes: Vec) { + let mut guard = CACHE.lock().await; + guard.put( + account_id, + CacheEntry { + mailboxes, + fetched_at: utc_now!(), + }, + ); +} + +pub async fn invalidate(account_id: u64) { + let mut guard = CACHE.lock().await; + guard.pop(&account_id); +} + +// Background fetch state tracking +#[derive(Clone, Debug)] +pub enum FetchStatus { + Fetching { examined: usize, total: usize }, + Ready, + Error(String), +} + +static FETCH_STATES: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +pub async fn fetch_status(account_id: u64) -> Option { + FETCH_STATES.lock().await.get(&account_id).cloned() +} + +pub async fn set_fetching(account_id: u64) { + FETCH_STATES.lock().await.insert( + account_id, + FetchStatus::Fetching { + examined: 0, + total: 0, + }, + ); +} + +pub async fn update_fetch_progress(account_id: u64, examined: usize, total: usize) { + let mut guard = FETCH_STATES.lock().await; + guard.insert( + account_id, + FetchStatus::Fetching { examined, total }, + ); +} + +pub async fn set_fetch_ready(account_id: u64) { + FETCH_STATES + .lock() + .await + .insert(account_id, FetchStatus::Ready); +} + +pub async fn set_fetch_error(account_id: u64, error: String) { + FETCH_STATES + .lock() + .await + .insert(account_id, FetchStatus::Error(error)); +} + +pub async fn clear_fetch_state(account_id: u64) { + FETCH_STATES.lock().await.remove(&account_id); +} diff --git a/crates/core/src/cache/imap/mod.rs b/crates/core/src/cache/imap/mod.rs index 5a32b19..a3b3c7a 100644 --- a/crates/core/src/cache/imap/mod.rs +++ b/crates/core/src/cache/imap/mod.rs @@ -22,6 +22,7 @@ use mailbox::MailBox; pub mod download; pub mod mailbox; +pub mod mailbox_cache; pub mod task; pub fn find_missing_mailboxes( diff --git a/crates/core/src/mailbox/list.rs b/crates/core/src/mailbox/list.rs index 4a84b62..667e220 100644 --- a/crates/core/src/mailbox/list.rs +++ b/crates/core/src/mailbox/list.rs @@ -18,6 +18,7 @@ use crate::account::migration::{AccountModel, AccountType}; use crate::cache::imap::mailbox::{Attribute, AttributeEnum, MailBox}; +use crate::cache::imap::mailbox_cache::{self, FetchStatus}; use crate::error::code::ErrorCode; use crate::error::BichonResult; use crate::imap::executor::ImapExecutor; @@ -26,12 +27,28 @@ use crate::raise_error; use crate::utils::create_hash; use async_imap::types::Name; use async_imap::Session; +use serde::{Deserialize, Serialize}; -pub async fn get_account_mailboxes(account_id: u64, remote: bool) -> BichonResult> { +#[derive(Clone, Debug, Serialize, Deserialize)] +#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] +pub struct MailboxListResponse { + pub mailboxes: Vec, + /// "ready" | "fetching" | "error" + pub status: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + pub examined: Option, + pub total: Option, +} + +pub async fn get_account_mailboxes( + account_id: u64, + remote: bool, +) -> BichonResult { let account = AccountModel::check_account_exists(account_id)?; if remote { if matches!(account.account_type, AccountType::IMAP) { - request_imap_all_mailbox_list(account_id).await + return Ok(remote_mailboxes(account_id).await); } else { return Err(raise_error!( "The 'remote' option can only be used with IMAP accounts.".into(), @@ -39,10 +56,124 @@ pub async fn get_account_mailboxes(account_id: u64, remote: bool) -> BichonResul )); } } else { - MailBox::list_all(account_id) + let mailboxes = MailBox::list_all(account_id)?; + return Ok(MailboxListResponse { + mailboxes, + status: "ready".into(), + error: None, + examined: None, + total: None, + }); } } +fn make_pending_response(status: &FetchStatus, error: Option) -> MailboxListResponse { + let (examined, total) = match status { + FetchStatus::Fetching { examined, total } => (Some(*examined), Some(*total)), + _ => (None, None), + }; + MailboxListResponse { + mailboxes: vec![], + status: match status { + FetchStatus::Ready => "ready".into(), + FetchStatus::Fetching { .. } => "fetching".into(), + FetchStatus::Error(_) => "error".into(), + }, + error, + examined, + total, + } +} + +async fn remote_mailboxes(account_id: u64) -> MailboxListResponse { + // Cache hit + if let Some(cached) = mailbox_cache::get(account_id).await { + return MailboxListResponse { + mailboxes: cached, + status: "ready".into(), + error: None, + examined: None, + total: None, + }; + } + + match mailbox_cache::fetch_status(account_id).await { + Some(status @ FetchStatus::Fetching { .. }) => { + return make_pending_response(&status, None); + } + Some(FetchStatus::Error(err)) => { + mailbox_cache::clear_fetch_state(account_id).await; + return MailboxListResponse { + mailboxes: vec![], + status: "error".into(), + error: Some(err), + examined: None, + total: None, + }; + } + _ => {} + } + + // No cache, no fetch in progress — start background fetch + mailbox_cache::set_fetching(account_id).await; + spawn_fetch_task(account_id); + MailboxListResponse { + mailboxes: vec![], + status: "fetching".into(), + error: None, + examined: Some(0), + total: Some(0), + } +} + +fn spawn_fetch_task(account_id: u64) { + tokio::spawn(async move { + match fetch_remote_with_progress(account_id).await { + Ok(mailboxes) => { + mailbox_cache::set(account_id, mailboxes).await; + mailbox_cache::set_fetch_ready(account_id).await; + } + Err(e) => { + mailbox_cache::set_fetch_error(account_id, format!("{:#?}", e)).await; + } + } + }); +} + +async fn fetch_remote_with_progress(account_id: u64) -> BichonResult> { + let mut session = ImapExecutor::create_connection(account_id).await?; + let names = ImapExecutor::list_all_mailboxes(&mut session).await?; + let total = names.len(); + mailbox_cache::update_fetch_progress(account_id, 0, total).await; + + let mut mailboxes = Vec::new(); + for (i, name) in names.iter().enumerate() { + let mailbox_name = name.name().to_string(); + let mut mailbox: MailBox = name.into(); + + if contains_no_select(&mailbox.attributes) { + continue; + } + + mailbox.account_id = account_id; + mailbox.id = create_hash(account_id, &mailbox.name); + let mx = session + .examine(mailbox_name.as_str()) + .await + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + mailbox.exists = mx.exists; + mailbox.unseen = mx.unseen; + mailbox.uid_next = mx.uid_next; + mailbox.uid_validity = mx.uid_validity; + + mailboxes.push(mailbox); + mailbox_cache::update_fetch_progress(account_id, i + 1, total).await; + } + + session.logout().await.ok(); + Ok(mailboxes) +} + pub async fn request_imap_all_mailbox_list(account_id: u64) -> BichonResult> { let mut session = ImapExecutor::create_connection(account_id).await?; let names = ImapExecutor::list_all_mailboxes(&mut session).await?; diff --git a/crates/memdb/src/db.rs b/crates/memdb/src/db.rs index 7a2443b..d0db542 100644 --- a/crates/memdb/src/db.rs +++ b/crates/memdb/src/db.rs @@ -310,7 +310,7 @@ impl MemDb { } } - eprintln!("[memdb] snapshot saved at seq={last_seq}"); + //eprintln!("[memdb] snapshot saved at seq={last_seq}"); Ok(()) } diff --git a/crates/server/src/rest/api/mailbox.rs b/crates/server/src/rest/api/mailbox.rs index d76a66f..d0b1eb2 100644 --- a/crates/server/src/rest/api/mailbox.rs +++ b/crates/server/src/rest/api/mailbox.rs @@ -19,9 +19,8 @@ use crate::common::auth::WrappedContext; use crate::rest::api::ApiTags; use crate::rest::ApiResult; -use bichon_core::cache::imap::mailbox::MailBox; use bichon_core::mailbox::delete::delete_mailbox_impl; -use bichon_core::mailbox::list::get_account_mailboxes; +use bichon_core::mailbox::list::{get_account_mailboxes, MailboxListResponse}; use bichon_core::users::permissions::Permission; use poem_openapi::param::{Path, Query}; use poem_openapi::payload::Json; @@ -51,7 +50,7 @@ impl MailBoxApi { account_id: Path, remote: Query>, context: WrappedContext, - ) -> ApiResult>> { + ) -> ApiResult> { let account_id = account_id.0; context.require_permission(Some(account_id), Permission::ACCOUNT_READ_DETAILS)?; let remote = remote.0.unwrap_or(false); diff --git a/web/src/api/mailbox/api.ts b/web/src/api/mailbox/api.ts index 19def31..a49f824 100644 --- a/web/src/api/mailbox/api.ts +++ b/web/src/api/mailbox/api.ts @@ -32,8 +32,16 @@ export interface MailboxData { unseen: number | null; } +export interface MailboxListResponse { + mailboxes: MailboxData[]; + status: "ready" | "fetching" | "error"; + error?: string | null; + examined?: number | null; + total?: number | null; +} + export const list_mailboxes = async (accountId: number, remote: boolean) => { - const response = await axiosInstance.get(`api/v1/list-mailboxes/${accountId}?remote=${remote}`); + const response = await axiosInstance.get(`api/v1/list-mailboxes/${accountId}?remote=${remote}`); return response.data; }; diff --git a/web/src/features/accounts/components/download-folders.tsx b/web/src/features/accounts/components/download-folders.tsx index c71dcfd..c3e98f3 100644 --- a/web/src/features/accounts/components/download-folders.tsx +++ b/web/src/features/accounts/components/download-folders.tsx @@ -161,6 +161,7 @@ export function DownloadFoldersDialog({ currentRow, open, onOpenChange }: Props) const [treeData, setTreeData] = useState([]); const [isLoading, setIsLoading] = useState(false); const [error, setError] = useState(undefined); + const [fetchProgress, setFetchProgress] = useState<{ examined: number; total: number } | null>(null); const queryClient = useQueryClient(); const { t } = useTranslation() const { theme } = useTheme() @@ -168,42 +169,65 @@ export function DownloadFoldersDialog({ currentRow, open, onOpenChange }: Props) useEffect(() => { if (!open) return; let cancelled = false; - const fetchMailboxes = async () => { - setIsLoading(true); - try { - const data = await list_mailboxes(currentRow.id, true); - if (!cancelled) { - setMailboxes(data); - const allIds = data.map(mailbox => String(mailbox.id)); - setAllIds(allIds); + let pollingTimer: ReturnType | null = null; - const tree = buildTree(data); - setTreeData(tree); - const itemsWithChildren = getParentIds(tree); - setItemsWithChildren(itemsWithChildren); - setExpandedItems(itemsWithChildren); - const download_folders = data - .filter(mailbox => currentRow.download_folders.includes(mailbox.name)) - .map(mailbox => mailbox.id.toString()); - setSelectedItems(download_folders); + const processMailboxes = (data: MailboxData[]) => { + setMailboxes(data); + const allIds = data.map(mailbox => String(mailbox.id)); + setAllIds(allIds); + const tree = buildTree(data); + setTreeData(tree); + const itemsWithChildren = getParentIds(tree); + setItemsWithChildren(itemsWithChildren); + setExpandedItems(itemsWithChildren); + const download_folders = data + .filter(mailbox => currentRow.download_folders.includes(mailbox.name)) + .map(mailbox => mailbox.id.toString()); + setSelectedItems(download_folders); + }; + + const fetchMailboxes = async () => { + try { + const response = await list_mailboxes(currentRow.id, true); + if (cancelled) return; + + if (response.status === "ready") { + processMailboxes(response.mailboxes); setError(undefined); + setIsLoading(false); + } else if (response.status === "fetching") { + setIsLoading(true); + setError(undefined); + if (response.examined != null && response.total != null && response.total > 0) { + setFetchProgress({ examined: response.examined, total: response.total }); + } + pollingTimer = setTimeout(fetchMailboxes, 2000); + } else if (response.status === "error") { + setIsLoading(false); + setError(response.error || "Unknown error"); } } catch (err: any) { - if (axios.isAxiosError(err)) { - const resData = err.response?.data; - if (resData) { - setError(`Error ${resData.code || ''}: ${resData.message || ''}`); + if (!cancelled) { + if (axios.isAxiosError(err)) { + const resData = err.response?.data; + if (resData) { + setError(`Error ${resData.code || ''}: ${resData.message || ''}`); + } else { + setError(err.message); + } } else { - setError(err.message); + setError(err.message || String(err)); } + setIsLoading(false); } - } finally { - if (!cancelled) setIsLoading(false); } }; + + setIsLoading(true); fetchMailboxes(); return () => { cancelled = true; + if (pollingTimer) clearTimeout(pollingTimer); }; }, [currentRow, open]); @@ -455,12 +479,16 @@ export function DownloadFoldersDialog({ currentRow, open, onOpenChange }: Props) - + {isLoading && (
- {t('accounts.folderSync.loadingMailboxFolders')} + + {fetchProgress && fetchProgress.total > 0 + ? `${t('accounts.folderSync.loadingMailboxFolders')} (${fetchProgress.examined}/${fetchProgress.total})` + : t('accounts.folderSync.loadingMailboxFolders')} +
diff --git a/web/src/features/attachment/mailbox-popover.tsx b/web/src/features/attachment/mailbox-popover.tsx index 646d140..6fe9968 100644 --- a/web/src/features/attachment/mailbox-popover.tsx +++ b/web/src/features/attachment/mailbox-popover.tsx @@ -181,7 +181,7 @@ export function MailboxPopover() { const { data: activeMailboxes = [], isLoading: activeIsLoading } = useQuery({ queryKey: ['search-mailboxes', activeAccountId], - queryFn: () => list_mailboxes(activeAccountId!, false), + queryFn: async () => (await list_mailboxes(activeAccountId!, false)).mailboxes, enabled: !!activeAccountId, }); diff --git a/web/src/features/search/mailbox-popover.tsx b/web/src/features/search/mailbox-popover.tsx index 3349419..183f7a0 100644 --- a/web/src/features/search/mailbox-popover.tsx +++ b/web/src/features/search/mailbox-popover.tsx @@ -181,7 +181,7 @@ export function MailboxPopover() { const { data: activeMailboxes = [], isLoading: activeIsLoading } = useQuery({ queryKey: ['search-mailboxes', activeAccountId], - queryFn: () => list_mailboxes(activeAccountId!, false), + queryFn: async () => (await list_mailboxes(activeAccountId!, false)).mailboxes, enabled: !!activeAccountId, });