6 Commits
0.3.6 ... 0.3.7

Author SHA1 Message Date
rustmailer
579822762f chore: remove bb8 pool for IMAP; create a new session per operation to avoid stale connections 2026-01-28 20:41:17 +08:00
rustmailer
d63b1e0d7c fix: batch size validation 2026-01-28 20:39:47 +08:00
rustmailer
a0d8d069c0 bump versions 2026-01-28 13:20:45 +08:00
rustmailer
bdbbc04832 chore: adjust IMAP connection timeout configuration 2026-01-28 13:20:09 +08:00
rustmailer
fed28c3eca fix: add tolerant HTML-to-text extraction (#141) 2026-01-28 13:19:33 +08:00
rustmailer
01dba4f71b fix: switch from PUID/PGID env vars to Docker --user for permissions 2026-01-28 13:15:06 +08:00
44 changed files with 386 additions and 537 deletions

186
Cargo.lock generated
View File

@@ -430,27 +430,14 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bb8"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "457d7ed3f888dfd2c7af56d4975cade43c622f74bdcddfed6d4352f57acc6310"
dependencies = [
"futures-util",
"parking_lot",
"portable-atomic",
"tokio",
]
[[package]]
name = "bichon"
version = "0.3.6"
version = "0.3.7"
dependencies = [
"ahash",
"async-imap",
"autoconfig",
"base64 0.22.1",
"bb8",
"bytes 1.11.0",
"cacache",
"chrono",
@@ -501,7 +488,7 @@ dependencies = [
"sysinfo",
"tantivy",
"tempfile",
"time 0.3.45",
"time 0.3.46",
"timeago",
"tokio",
"tokio-io-timeout",
@@ -756,7 +743,7 @@ dependencies = [
"num-traits",
"serde",
"wasm-bindgen",
"windows-link 0.2.1",
"windows-link",
]
[[package]]
@@ -1622,7 +1609,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8"
dependencies = [
"rustix",
"windows-link 0.2.1",
"windows-link",
]
[[package]]
@@ -1843,7 +1830,7 @@ dependencies = [
"rustls",
"rustls-pki-types",
"thiserror 2.0.17",
"time 0.3.45",
"time 0.3.46",
"tinyvec",
"tokio",
"tokio-rustls",
@@ -2108,7 +2095,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
"windows-core 0.62.2",
"windows-core",
]
[[package]]
@@ -2903,9 +2890,9 @@ dependencies = [
[[package]]
name = "num-conv"
version = "0.1.0"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050"
[[package]]
name = "num-traits"
@@ -3086,7 +3073,7 @@ dependencies = [
"libc",
"redox_syscall",
"smallvec",
"windows-link 0.2.1",
"windows-link",
]
[[package]]
@@ -3715,7 +3702,7 @@ dependencies = [
"cfg-if",
"libc",
"rustix",
"windows 0.62.2",
"windows",
]
[[package]]
@@ -4532,16 +4519,16 @@ dependencies = [
[[package]]
name = "sysinfo"
version = "0.37.2"
version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f"
checksum = "fe840c5b1afe259a5657392a4dbb74473a14c8db999c3ec2f4ae812e028a94da"
dependencies = [
"libc",
"memchr",
"ntapi",
"objc2-core-foundation",
"objc2-io-kit",
"windows 0.61.3",
"windows",
]
[[package]]
@@ -4600,7 +4587,7 @@ dependencies = [
"tantivy-tokenizer-api",
"tempfile",
"thiserror 2.0.17",
"time 0.3.45",
"time 0.3.46",
"uuid",
"winapi",
"zstd",
@@ -4641,7 +4628,7 @@ dependencies = [
"byteorder",
"ownedbytes",
"serde",
"time 0.3.45",
"time 0.3.46",
]
[[package]]
@@ -4789,9 +4776,9 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.45"
version = "0.3.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd"
checksum = "9da98b7d9b7dad93488a84b8248efc35352b0b2657397d4167e7ad67e5d535e5"
dependencies = [
"deranged",
"itoa",
@@ -4801,14 +4788,14 @@ dependencies = [
"powerfmt",
"serde_core",
"time-core",
"time-macros 0.2.25",
"time-macros 0.2.26",
]
[[package]]
name = "time-core"
version = "0.1.7"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca"
checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"
[[package]]
name = "time-macros"
@@ -4822,9 +4809,9 @@ dependencies = [
[[package]]
name = "time-macros"
version = "0.2.25"
version = "0.2.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd"
checksum = "78cc610bac2dcee56805c99642447d4c5dbde4d01f752ffea0199aee1f601dc4"
dependencies = [
"num-conv",
"time-core",
@@ -5078,7 +5065,7 @@ checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf"
dependencies = [
"crossbeam-channel",
"thiserror 2.0.17",
"time 0.3.45",
"time 0.3.46",
"tracing-subscriber",
]
@@ -5579,38 +5566,16 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
version = "0.61.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
dependencies = [
"windows-collections 0.2.0",
"windows-core 0.61.2",
"windows-future 0.2.1",
"windows-link 0.1.3",
"windows-numerics 0.2.0",
]
[[package]]
name = "windows"
version = "0.62.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580"
dependencies = [
"windows-collections 0.3.2",
"windows-core 0.62.2",
"windows-future 0.3.2",
"windows-numerics 0.3.1",
]
[[package]]
name = "windows-collections"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8"
dependencies = [
"windows-core 0.61.2",
"windows-collections",
"windows-core",
"windows-future",
"windows-numerics",
]
[[package]]
@@ -5619,20 +5584,7 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610"
dependencies = [
"windows-core 0.62.2",
]
[[package]]
name = "windows-core"
version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
dependencies = [
"windows-implement",
"windows-interface",
"windows-link 0.1.3",
"windows-result 0.3.4",
"windows-strings 0.4.2",
"windows-core",
]
[[package]]
@@ -5643,20 +5595,9 @@ checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
dependencies = [
"windows-implement",
"windows-interface",
"windows-link 0.2.1",
"windows-result 0.4.1",
"windows-strings 0.5.1",
]
[[package]]
name = "windows-future"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
dependencies = [
"windows-core 0.61.2",
"windows-link 0.1.3",
"windows-threading 0.1.0",
"windows-link",
"windows-result",
"windows-strings",
]
[[package]]
@@ -5665,9 +5606,9 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb"
dependencies = [
"windows-core 0.62.2",
"windows-link 0.2.1",
"windows-threading 0.2.1",
"windows-core",
"windows-link",
"windows-threading",
]
[[package]]
@@ -5692,45 +5633,20 @@ dependencies = [
"syn 2.0.111",
]
[[package]]
name = "windows-link"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"
[[package]]
name = "windows-link"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-numerics"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1"
dependencies = [
"windows-core 0.61.2",
"windows-link 0.1.3",
]
[[package]]
name = "windows-numerics"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26"
dependencies = [
"windows-core 0.62.2",
"windows-link 0.2.1",
]
[[package]]
name = "windows-result"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
dependencies = [
"windows-link 0.1.3",
"windows-core",
"windows-link",
]
[[package]]
@@ -5739,16 +5655,7 @@ version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
dependencies = [
"windows-link 0.2.1",
]
[[package]]
name = "windows-strings"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"
dependencies = [
"windows-link 0.1.3",
"windows-link",
]
[[package]]
@@ -5757,7 +5664,7 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
dependencies = [
"windows-link 0.2.1",
"windows-link",
]
[[package]]
@@ -5802,7 +5709,7 @@ version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
dependencies = [
"windows-link 0.2.1",
"windows-link",
]
[[package]]
@@ -5842,7 +5749,7 @@ version = "0.53.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
dependencies = [
"windows-link 0.2.1",
"windows-link",
"windows_aarch64_gnullvm 0.53.1",
"windows_aarch64_msvc 0.53.1",
"windows_i686_gnu 0.53.1",
@@ -5853,22 +5760,13 @@ dependencies = [
"windows_x86_64_msvc 0.53.1",
]
[[package]]
name = "windows-threading"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6"
dependencies = [
"windows-link 0.1.3",
]
[[package]]
name = "windows-threading"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37"
dependencies = [
"windows-link 0.2.1",
"windows-link",
]
[[package]]
@@ -6190,7 +6088,7 @@ dependencies = [
"pbkdf2",
"ppmd-rust",
"sha1 0.10.6",
"time 0.3.45",
"time 0.3.46",
"zeroize",
"zopfli",
"zstd",

View File

@@ -1,6 +1,6 @@
[package]
name = "bichon"
version = "0.3.6"
version = "0.3.7"
edition = "2021"
[[bin]]
@@ -77,7 +77,7 @@ timeago = "0.5.0"
ahash = "0.8.12"
oauth2 = { version = "5.0.0", features = ["reqwest-blocking"] }
url = { version = "2.5.8", features = ["serde"] }
sysinfo = "0.37.2"
sysinfo = "0.38.0"
num_cpus = "1.17.0"
cacache = { version = "13.1.0", default-features = false, features = [
"tokio-runtime",
@@ -93,13 +93,12 @@ webpki-roots = "1.0.5"
rustls = { version = "0.23.36", default-features = false, features = ["ring"] }
rustls-pki-types = "1.14.0"
tokio-io-timeout = "1.2.1"
bb8 = "0.9.1"
semver = "1.0.27"
governor = "0.10.4"
lru = "0.16.3"
mime_guess = "2.0.5"
hex = "0.4.3"
time = { version = "0.3.45", features = [
time = { version = "0.3.46", features = [
"formatting",
"parsing",
"local-offset",

View File

@@ -122,36 +122,64 @@ docker pull rustmailer/bichon:latest
# Create data directory
mkdir -p ./bichon-data
# Optional: Set PUID and PGID to match your host user for proper file permissions
# Find your user ID with: id $USER
# This prevents permission issues when using NFS mounts or shared volumes
# Run container
docker run -d \
--name bichon \
-p 15630:15630 \
-v $(pwd)/bichon-data:/data \
-e PUID=1000 \
-e PGID=1000 \
--user 1000:1000 \
-e BICHON_LOG_LEVEL=info \
-e BICHON_ROOT_DIR=/data \
rustmailer/bichon:latest
```
# Optional: For custom storage configuration with separate volumes
### Optional: Custom storage layout
```bash
docker run -d \
--name bichon \
-p 15630:15630 \
-v $(pwd)/bichon-data:/data \
-v $(pwd)/envelope:/envelope \
-v $(pwd)/eml:/eml \
-e PUID=1000 \
-e PGID=1000 \
--user 1000:1000 \
-e BICHON_ROOT_DIR=/data \
-e BICHON_INDEX_DIR=/envelope \
-e BICHON_DATA_DIR=/eml \
rustmailer/bichon:latest
```
### Recommended docker-compose example
```bash
services:
bichon:
image: rustmailer/bichon:latest
container_name: bichon
ports:
- "15630:15630"
volumes:
- ./bichon-data:/data
user: "1000:1000"
environment:
BICHON_ROOT_DIR: /data
BICHON_LOG_LEVEL: info
```
### User and permissions
`PUID` and `PGID` are no longer used to create users or groups inside the container.
Please use Dockers native `--user` option (or `user:` in docker-compose) to specify the UID and GID:
```bash
docker run --user 1000:1000 ...
```
This ensures container file permissions match the ownership of host-mounted directories.
## CORS Configuration (Important for Browser Access)
Starting from **v0.1.4**, Bichon changes how `BICHON_CORS_ORIGINS` works:

View File

@@ -25,9 +25,6 @@ RUN chmod +x /opt/bichon/bichon
RUN chmod +x /usr/local/bin/bichonctl
RUN chmod +x /usr/local/bin/bichon-admin
# Copy and setup entrypoint script for PUID/PGID support
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
RUN chmod +x /usr/local/bin/entrypoint.sh
# Install ca-certificates to ensure HTTPS certificate verification works correctly
RUN apt update && apt install -y ca-certificates curl && rm -rf /var/lib/apt/lists/*
@@ -43,6 +40,4 @@ WORKDIR /data
HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \
CMD curl -fs http://localhost:15630/api/status || exit 1
# Entrypoint with PUID/PGID support
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
CMD ["/opt/bichon/bichon"]

View File

@@ -1,54 +0,0 @@
#!/bin/bash
# Entrypoint script for Bichon Docker container
# Handles PUID/PGID environment variables for proper user permissions
set -e
# If not running as root, do nothing
if [ "$(id -u)" != "0" ]; then
echo "Running as non-root user ($(id -u)), skipping PUID/PGID handling"
exec "$@"
fi
# Function to create user and switch to it
switch_user() {
local puid="$1"
local pgid="$2"
local USER_NAME
local GROUP_NAME
# group
if getent group "$pgid" >/dev/null 2>&1; then
GROUP_NAME=$(getent group "$pgid" | cut -d: -f1)
else
groupadd -g "$pgid" bichon
GROUP_NAME=bichon
fi
# user
if getent passwd "$puid" >/dev/null 2>&1; then
USER_NAME=$(getent passwd "$puid" | cut -d: -f1)
else
useradd -u "$puid" -g "$GROUP_NAME" -s /bin/bash -d /data bichon
USER_NAME=bichon
fi
chown -R "$puid:$pgid" /data
chown -R "$puid:$pgid" /opt/bichon
[ -d /envelope ] && chown -R "$puid:$pgid" /envelope
[ -d /eml ] && chown -R "$puid:$pgid" /eml
exec runuser -u "$USER_NAME" -- "$@"
}
# Check if PUID and PGID are set
if [ -n "$PUID" ] && [ -n "$PGID" ]; then
echo "Switching to user with PUID=$PUID, PGID=$PGID"
switch_user "$PUID" "$PGID" "$@"
else
echo "No PUID/PGID specified, running as root"
exec "$@"
fi

View File

@@ -20,7 +20,7 @@ use bichon::{
bichon_version,
modules::{
common::rustls::RustMailerTls,
context::{executors::EmailClientExecutors, Initialize},
context::{executors::BichonContext, Initialize},
error::BichonResult,
logger,
rest::start_http_server,
@@ -71,7 +71,7 @@ async fn initialize() -> BichonResult<()> {
DataDirManager::initialize().await?;
UserManager::initialize().await?;
RustMailerTls::initialize().await?;
EmailClientExecutors::initialize().await?;
BichonContext::initialize().await?;
PeriodicTasks::start_background_tasks();
Ok(())
}

View File

@@ -47,7 +47,6 @@ use crate::modules::account::payload::AccountUpdateRequest;
use crate::modules::account::payload::MinimalAccount;
use crate::modules::cache::imap::task::SYNC_TASKS;
use crate::modules::context::controller::SYNC_CONTROLLER;
use crate::modules::context::executors::MAIL_CONTEXT;
use crate::modules::database::count_by_unique_secondary_key_impl;
use crate::modules::database::delete_impl;
use crate::modules::database::manager::DB_MANAGER;
@@ -297,7 +296,7 @@ impl AccountV3 {
if matches!(account.account_type, AccountType::IMAP) {
SYNC_TASKS.stop(account.id).await?;
AccountRunningState::delete(account.id).await?;
MAIL_CONTEXT.clean_account(account.id).await?;
//BICHON_CONTEXT.clean_account(account.id).await?;
}
OAuth2AccessToken::try_delete(account.id).await?;
UserModel::cleanup_account(account.id).await?;

View File

@@ -39,7 +39,7 @@ pub struct AccountCreateRequest {
pub folder_limit: Option<u32>,
#[oai(validator(minimum(value = "10")))]
pub sync_interval_min: Option<i64>,
#[oai(validator(minimum(value = "30"), maximum(value = "200")))]
#[oai(validator(minimum(value = "10"), maximum(value = "200")))]
pub sync_batch_size: Option<u32>,
pub use_proxy: Option<u64>,
pub use_dangerous: bool,
@@ -145,7 +145,7 @@ pub struct AccountUpdateRequest {
/// Incremental sync interval (seconds)
#[oai(validator(minimum(value = "10")))]
pub sync_interval_min: Option<i64>,
#[oai(validator(minimum(value = "30"), maximum(value = "200")))]
#[oai(validator(minimum(value = "10"), maximum(value = "200")))]
pub sync_batch_size: Option<u32>,
/// Optional proxy ID for establishing the connection to external APIs (e.g., Gmail, Outlook).
/// - If `None` or not provided, the client will connect directly to the API server.

View File

@@ -30,8 +30,8 @@ use crate::{
},
SEMAPHORE,
},
context::executors::MAIL_CONTEXT,
error::{code::ErrorCode, BichonError, BichonResult},
imap::executor::ImapExecutor,
indexer::manager::ENVELOPE_INDEX_MANAGER,
},
raise_error,
@@ -55,16 +55,15 @@ pub async fn fetch_and_save_by_date(
direction: FetchDirection,
) -> BichonResult<usize> {
let account_id = account.id;
let executor = MAIL_CONTEXT.imap(account_id).await?;
let mut session = ImapExecutor::create_connection(account_id).await?;
let search_criteria = match direction {
FetchDirection::Since => format!("SINCE {date}"),
FetchDirection::Before => format!("BEFORE {date}"),
};
let uid_list = executor
.uid_search(&mailbox.encoded_name(), &search_criteria)
.await?;
let uid_list =
ImapExecutor::uid_search(&mut session, &mailbox.encoded_name(), &search_criteria).await?;
let len = uid_list.len();
if len == 0 {
@@ -109,12 +108,18 @@ pub async fn fetch_and_save_by_date(
(index + 1) as u32,
)
.await?;
let executor = MAIL_CONTEXT.imap(account_id).await?;
// Fetch metadata for the current batch of UIDs
executor
.uid_batch_retrieve_emails(account_id, mailbox.id, &batch, &mailbox.encoded_name())
.await?;
ImapExecutor::uid_batch_retrieve_emails(
&mut session,
account_id,
mailbox.id,
&batch,
&mailbox.encoded_name(),
)
.await?;
}
session.logout().await.ok();
Ok(len)
}
@@ -153,26 +158,29 @@ pub async fn fetch_and_save_full_mailbox(
"Starting full mailbox sync for '{}', total={}, limit={:?}, batches={}, desc={}",
mailbox.name, total, folder_limit, total_batches, desc
);
let mut session = ImapExecutor::create_connection(account_id).await?;
for page in 1..=total_batches {
AccountRunningState::set_current_sync_batch_number(account_id, mailbox.name.clone(), page)
.await?;
let executor = MAIL_CONTEXT.imap(account_id).await?;
let count = executor
.batch_retrieve_emails(
account_id,
mailbox_id,
page as u64,
page_size as u64,
&mailbox.encoded_name(),
desc,
)
.await?;
let count = ImapExecutor::batch_retrieve_emails(
&mut session,
account_id,
mailbox_id,
page as u64,
page_size as u64,
&mailbox.encoded_name(),
desc,
)
.await?;
inserted_count += count;
info!(
"Batch insertion completed for mailbox: {}, current page: {}, inserted count: {}",
&mailbox.name, page, count
);
}
session.logout().await.ok();
Ok(inserted_count)
}
@@ -423,16 +431,22 @@ async fn perform_incremental_sync(
.await?;
match local_max_uid {
Some(max_uid) => {
let executor = MAIL_CONTEXT.imap(account.id).await?;
let mut session = ImapExecutor::create_connection(account.id).await?;
let before_date = account
.date_before
.as_ref()
.map(|r| r.calculate_date())
.transpose()?;
executor
.fetch_new_mail(account, local_mailbox, max_uid + 1, before_date.as_deref())
.await?;
ImapExecutor::fetch_new_mail(
&mut session,
account,
local_mailbox,
max_uid + 1,
before_date.as_deref(),
)
.await?;
session.logout().await.ok();
}
None => {
info!(

View File

@@ -25,6 +25,7 @@ use crate::{
},
cache::imap::{mailbox::MailBox, sync::flow::FetchDirection},
error::BichonResult,
imap::executor::ImapExecutor,
},
utc_now,
};
@@ -33,7 +34,7 @@ use rebuild::{rebuild_cache, rebuild_cache_by_date};
use std::time::Instant;
use sync_folders::get_sync_folders;
use sync_type::{determine_sync_type, SyncType};
use tracing::debug;
use tracing::{debug, warn};
pub mod flow;
pub mod rebuild;
@@ -48,7 +49,19 @@ pub async fn execute_imap_sync(account: &AccountModel) -> BichonResult<()> {
if matches!(sync_type, SyncType::SkipSync) {
return Ok(());
}
let remote_mailboxes = get_sync_folders(account).await?;
let mut session = ImapExecutor::create_connection(account_id).await?;
let remote_mailboxes = match get_sync_folders(account, &mut session).await {
Ok(mailboxes) => mailboxes,
Err(err) => {
warn!(
account_id = account.id,
error = %err,
"Failed to get sync folders, logging out and skipping this account"
);
return Ok(());
}
};
session.logout().await.ok();
if matches!(sync_type, SyncType::InitialSync) {
AccountRunningState::add(account.id).await?;
// AccountRunningState::set_initial_sync_start(account_id).await?;

View File

@@ -121,7 +121,6 @@ pub async fn rebuild_cache_by_date(
MailBox::batch_insert(remote_mailboxes).await?;
let mut handles = Vec::new();
let local_semaphore = Arc::new(Semaphore::new(DEFAULT_MAX_CONCURRENT_PER_ACCOUNT));
for mailbox in remote_mailboxes {
@@ -164,7 +163,6 @@ pub async fn rebuild_cache_by_date(
tokio::spawn(async move {
let _global_permit = global_permit;
let _local_permit = local_permit;
fetch_and_save_by_date(&account, date.as_str(), &mailbox, direction).await
});
handles.push(handle);

View File

@@ -16,7 +16,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::collections::BTreeSet;
use crate::{
@@ -24,19 +23,21 @@ use crate::{
modules::{
account::migration::{AccountModel, AccountType},
cache::imap::mailbox::{AttributeEnum, MailBox},
context::executors::MAIL_CONTEXT,
error::{code::ErrorCode, BichonResult},
imap::{executor::ImapExecutor, session::SessionStream},
mailbox::list::convert_names_to_mailboxes,
},
raise_error,
};
use async_imap::types::Name;
use async_imap::{types::Name, Session};
use tracing::{debug, info, warn};
pub async fn get_sync_folders(account: &AccountModel) -> BichonResult<Vec<MailBox>> {
pub async fn get_sync_folders(
account: &AccountModel,
session: &mut Session<Box<dyn SessionStream>>,
) -> BichonResult<Vec<MailBox>> {
assert_eq!(account.account_type, AccountType::IMAP);
let executor = MAIL_CONTEXT.imap(account.id).await?;
let names = executor.list_all_mailboxes().await?;
let names = ImapExecutor::list_all_mailboxes(session).await?;
if names.is_empty() {
warn!(
"Account {}: No mailboxes returned from IMAP server.",
@@ -121,7 +122,7 @@ pub async fn get_sync_folders(account: &AccountModel) -> BichonResult<Vec<MailBo
), ErrorCode::ImapUnexpectedResult));
}
}
convert_names_to_mailboxes(account.id, matched_mailboxes).await
convert_names_to_mailboxes(account.id, session, matched_mailboxes).await
}
pub async fn detect_mailbox_changes(

View File

@@ -55,7 +55,7 @@ impl<E: Endpoint> Endpoint for TimeoutEndpoint<E> {
async fn call(&self, req: Request) -> Result<Self::Output> {
let timeout = extract_timeout(&req);
let seconds = timeout.unwrap_or(30).min(600);
let seconds = timeout.unwrap_or(60).min(600);
match tokio::time::timeout(Duration::from_secs(seconds), self.ep.call(req)).await {
Ok(Ok(response)) => Ok(response), // If the request completes successfully
Ok(Err(e)) => Err(e), // If the request returns an error

View File

@@ -18,73 +18,37 @@
use crate::modules::account::migration::AccountType;
use crate::modules::context::Initialize;
use crate::modules::error::code::ErrorCode;
use crate::raise_error;
use crate::{
modules::{
account::migration::AccountModel,
context::controller::SYNC_CONTROLLER,
error::BichonResult,
imap::{executor::ImapExecutor, pool::build_imap_pool},
account::migration::AccountModel, context::controller::SYNC_CONTROLLER, error::BichonResult,
},
utc_now,
};
use dashmap::DashMap;
use std::sync::{Arc, LazyLock};
use std::sync::LazyLock;
use tracing::info;
pub static MAIL_CONTEXT: LazyLock<EmailClientExecutors> = LazyLock::new(EmailClientExecutors::new);
pub static BICHON_CONTEXT: LazyLock<BichonContext> = LazyLock::new(BichonContext::new);
pub struct EmailClientExecutors {
pub struct BichonContext {
start_at: i64,
imap: DashMap<u64, Arc<ImapExecutor>>,
}
impl Initialize for EmailClientExecutors {
impl Initialize for BichonContext {
async fn initialize() -> BichonResult<()> {
MAIL_CONTEXT.start_account_syncers().await
BICHON_CONTEXT.start_account_syncers().await
}
}
impl EmailClientExecutors {
impl BichonContext {
pub fn new() -> Self {
Self {
start_at: utc_now!(),
imap: DashMap::new(),
}
}
pub fn uptime_ms(&self) -> i64 {
utc_now!() - self.start_at
}
pub async fn imap(&self, account_id: u64) -> BichonResult<Arc<ImapExecutor>> {
if let Some(executor) = self.imap.get(&account_id) {
return Ok(executor.value().clone());
}
let pool = build_imap_pool(account_id).await?;
let new_executor = Arc::new(ImapExecutor::new(account_id, pool));
match self.imap.try_entry(account_id) {
Some(dashmap::mapref::entry::Entry::Occupied(entry)) => Ok(entry.get().clone()),
Some(dashmap::mapref::entry::Entry::Vacant(entry)) => {
entry.insert(new_executor.clone());
Ok(new_executor)
}
None => Err(raise_error!(
"DashMap locked".into(),
ErrorCode::InternalError
)),
}
}
pub async fn clean_account(&self, account_id: u64) -> BichonResult<()> {
if self.imap.remove(&account_id).is_some() {
info!(account_id, "Closed IMAP pool for account");
}
Ok(())
}
pub async fn start_account_syncers(&self) -> BichonResult<()> {
let accounts = AccountModel::list_all().await?;
let active_accounts: Vec<AccountModel> = accounts

View File

@@ -17,7 +17,7 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::modules::context::executors::MAIL_CONTEXT;
use crate::modules::context::executors::BICHON_CONTEXT;
use chrono::Local;
use poem_openapi::Object;
use serde::Deserialize;
@@ -40,9 +40,9 @@ pub struct BichonStatus {
impl BichonStatus {
pub fn get() -> Self {
Self {
uptime_ms: MAIL_CONTEXT.uptime_ms(),
uptime_ms: BICHON_CONTEXT.uptime_ms(),
timeago: Formatter::new()
.convert(Duration::from_millis(MAIL_CONTEXT.uptime_ms() as u64)),
.convert(Duration::from_millis(BICHON_CONTEXT.uptime_ms() as u64)),
timezone: Local::now().offset().to_string(),
version: env!("CARGO_PKG_VERSION").into(),
}

View File

@@ -21,6 +21,7 @@ use crate::modules::envelope::utils::normalize_subject;
use crate::modules::error::code::ErrorCode;
use crate::modules::error::BichonResult;
use crate::modules::utils::create_hash;
use crate::modules::utils::html::extract_text;
use crate::{calculate_hash, raise_error, utc_now};
use crate::{id, modules::indexer::envelope::Envelope};
use async_imap::types::Fetch;
@@ -48,10 +49,7 @@ pub fn extract_envelope(fetch: &Fetch, account_id: u64, mailbox_id: u64) -> Bich
let text = if let Some(text) = message.body_text(0).map(|cow| cow.into_owned()) {
text
} else if let Some(html) = message.body_html(0).map(|cow| cow.into_owned()) {
html2text::config::plain()
.allow_width_overflow()
.string_from_read(html.as_bytes(), 100)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
extract_text(html)
} else {
String::new()
};
@@ -150,10 +148,7 @@ pub fn extract_envelope_from_eml(
let text = if let Some(text) = message.body_text(0).map(|cow| cow.into_owned()) {
text
} else if let Some(html) = message.body_html(0).map(|cow| cow.into_owned()) {
html2text::config::plain()
.allow_width_overflow()
.string_from_read(html.as_bytes(), 100)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
extract_text(html)
} else {
String::new()
};

View File

@@ -22,12 +22,13 @@ use crate::modules::cache::imap::mailbox::MailBox;
use crate::modules::cache::imap::sync::flow::{generate_uid_sequence_hashset, DEFAULT_BATCH_SIZE};
use crate::modules::envelope::extractor::extract_envelope;
use crate::modules::error::code::ErrorCode;
use crate::modules::imap::session::SessionStream;
use crate::modules::indexer::manager::{EML_INDEX_MANAGER, ENVELOPE_INDEX_MANAGER};
use crate::modules::indexer::schema::SchemaTools;
use crate::modules::{error::BichonResult, imap::manager::ImapConnectionManager};
use crate::raise_error;
use async_imap::types::{Mailbox, Name};
use bb8::{Pool, RunError};
use async_imap::types::Name;
use async_imap::Session;
use futures::TryStreamExt;
use std::collections::HashSet;
use tantivy::doc;
@@ -35,18 +36,12 @@ use tracing::info;
const BODY_FETCH_COMMAND: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[])";
pub struct ImapExecutor {
account_id: u64,
pool: Pool<ImapConnectionManager>,
}
pub struct ImapExecutor;
impl ImapExecutor {
pub fn new(account_id: u64, pool: Pool<ImapConnectionManager>) -> Self {
Self { account_id, pool }
}
pub async fn list_all_mailboxes(&self) -> BichonResult<Vec<Name>> {
let mut session = self.get_connection().await?;
pub async fn list_all_mailboxes(
session: &mut Session<Box<dyn SessionStream>>,
) -> BichonResult<Vec<Name>> {
let list = session
.list(Some(""), Some("*"))
.await
@@ -58,16 +53,11 @@ impl ImapExecutor {
Ok(result)
}
pub async fn examine_mailbox(&self, mailbox_name: &str) -> BichonResult<Mailbox> {
let mut session = self.get_connection().await?;
session
.examine(mailbox_name)
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))
}
pub async fn uid_search(&self, mailbox_name: &str, query: &str) -> BichonResult<HashSet<u32>> {
let mut session = self.get_connection().await?;
pub async fn uid_search(
session: &mut Session<Box<dyn SessionStream>>,
mailbox_name: &str,
query: &str,
) -> BichonResult<HashSet<u32>> {
session
.examine(mailbox_name)
.await
@@ -80,13 +70,12 @@ impl ImapExecutor {
}
pub async fn append(
&self,
session: &mut Session<Box<dyn SessionStream>>,
mailbox_name: impl AsRef<str>,
flags: Option<&str>,
internaldate: Option<&str>,
content: impl AsRef<[u8]>,
) -> BichonResult<()> {
let mut session = self.get_connection().await?;
session
.append(mailbox_name, flags, internaldate, content)
.await
@@ -94,7 +83,7 @@ impl ImapExecutor {
}
pub async fn fetch_new_mail(
&self,
session: &mut Session<Box<dyn SessionStream>>,
account: &AccountModel,
mailbox: &MailBox,
start_uid: u64,
@@ -107,7 +96,7 @@ impl ImapExecutor {
None => format!("UID {start_uid}:*"),
};
let uid_list = self.uid_search(&mailbox.encoded_name(), &query).await?;
let uid_list = Self::uid_search(session, &mailbox.encoded_name(), &query).await?;
let len = uid_list.len();
if len == 0 {
@@ -145,14 +134,20 @@ impl ImapExecutor {
)
.await?;
}
self.uid_batch_retrieve_emails(account.id, mailbox.id, &batch, &mailbox.encoded_name())
.await?;
Self::uid_batch_retrieve_emails(
session,
account.id,
mailbox.id,
&batch,
&mailbox.encoded_name(),
)
.await?;
}
Ok(())
}
pub async fn batch_retrieve_emails(
&self,
session: &mut Session<Box<dyn SessionStream>>,
account_id: u64,
mailbox_id: u64,
page: u64,
@@ -163,7 +158,6 @@ impl ImapExecutor {
assert!(page > 0, "Page number must be greater than 0");
assert!(page_size > 0, "Page size must be greater than 0");
let mut session = self.get_connection().await?;
let total = session
.examine(encoded_mailbox_name)
.await
@@ -226,13 +220,12 @@ impl ImapExecutor {
}
pub async fn uid_batch_retrieve_emails(
&self,
session: &mut Session<Box<dyn SessionStream>>,
account_id: u64,
mailbox_id: u64,
uid_set: &str,
encoded_mailbox_name: &str,
) -> BichonResult<()> {
let mut session = self.get_connection().await?;
session
.examine(encoded_mailbox_name)
.await
@@ -260,40 +253,46 @@ impl ImapExecutor {
Ok(())
}
async fn get_connection(
&self,
) -> BichonResult<bb8::PooledConnection<'_, ImapConnectionManager>> {
match self.pool.get().await {
Ok(connection) => Ok(connection),
Err(e) => match e {
RunError::User(e) => Err(e),
RunError::TimedOut => {
let state = self.pool.state();
tracing::warn!(
"{}: connections={}, idle={}, \
get_started={}, get_direct={}, get_waited={}, get_timed_out={}, \
wait_time_ms={}, created={}, closed_broken={}, closed_invalid={}, \
closed_lifetime={}, closed_idle={}",
self.account_id,
state.connections,
state.idle_connections,
state.statistics.get_started,
state.statistics.get_direct,
state.statistics.get_waited,
state.statistics.get_timed_out,
state.statistics.get_wait_time.as_millis(),
state.statistics.connections_created,
state.statistics.connections_closed_broken,
state.statistics.connections_closed_invalid,
state.statistics.connections_closed_max_lifetime,
state.statistics.connections_closed_idle_timeout,
);
return Err(raise_error!(
"Timed out while attempting to acquire a connection from the pool".into(),
ErrorCode::ConnectionPoolTimeout
));
}
},
}
// async fn get_connection(
// &self,
// ) -> BichonResult<bb8::PooledConnection<'_, ImapConnectionManager>> {
// match self.pool.get().await {
// Ok(connection) => Ok(connection),
// Err(e) => match e {
// RunError::User(e) => Err(e),
// RunError::TimedOut => {
// let state = self.pool.state();
// tracing::warn!(
// "{}: connections={}, idle={}, \
// get_started={}, get_direct={}, get_waited={}, get_timed_out={}, \
// wait_time_ms={}, created={}, closed_broken={}, closed_invalid={}, \
// closed_lifetime={}, closed_idle={}",
// self.account_id,
// state.connections,
// state.idle_connections,
// state.statistics.get_started,
// state.statistics.get_direct,
// state.statistics.get_waited,
// state.statistics.get_timed_out,
// state.statistics.get_wait_time.as_millis(),
// state.statistics.connections_created,
// state.statistics.connections_closed_broken,
// state.statistics.connections_closed_invalid,
// state.statistics.connections_closed_max_lifetime,
// state.statistics.connections_closed_idle_timeout,
// );
// return Err(raise_error!(
// "Timed out while attempting to acquire a connection from the pool".into(),
// ErrorCode::ConnectionPoolTimeout
// ));
// }
// },
// }
// }
pub async fn create_connection(
account_id: u64,
) -> BichonResult<Session<Box<dyn SessionStream>>> {
ImapConnectionManager::build(account_id).await
}
}

View File

@@ -32,22 +32,10 @@ use crate::{bichon_version, decrypt, raise_error};
use async_imap::Session;
use tracing::error;
#[derive(Debug)]
pub struct ImapConnectionManager {
pub account_id: u64,
}
pub struct ImapConnectionManager;
impl ImapConnectionManager {
pub fn new(account_id: u64) -> Self {
Self { account_id }
}
pub async fn fetch_account(&self) -> BichonResult<AccountModel> {
// Fetch the account entity in non-test environment
AccountModel::get(self.account_id).await
}
async fn create_client(&self, account: &AccountModel) -> BichonResult<Client> {
async fn create_client(account: &AccountModel) -> BichonResult<Client> {
assert_eq!(account.account_type, AccountType::IMAP);
let imap = account.imap.as_ref().unwrap();
Client::connection(
@@ -61,7 +49,6 @@ impl ImapConnectionManager {
}
async fn authenticate(
&self,
client: Client,
account: &AccountModel,
) -> BichonResult<Session<Box<dyn SessionStream>>> {
@@ -87,7 +74,7 @@ impl ImapConnectionManager {
})
}
AuthType::OAuth2 => {
let record = OAuth2AccessToken::get(self.account_id).await?;
let record = OAuth2AccessToken::get(account.id).await?;
let access_token = record.and_then(|r| r.access_token).ok_or_else(|| {
raise_error!(
"Imap auth type is OAuth2, but OAuth2 authorization is not yet complete."
@@ -106,9 +93,9 @@ impl ImapConnectionManager {
}
}
pub async fn build(&self) -> BichonResult<Session<Box<dyn SessionStream>>> {
let account = self.fetch_account().await?;
let client = match self.create_client(&account).await {
pub async fn build(account_id: u64) -> BichonResult<Session<Box<dyn SessionStream>>> {
let account = AccountModel::get(account_id).await?;
let client = match Self::create_client(&account).await {
Ok(client) => client,
Err(error) => {
error!(
@@ -117,7 +104,7 @@ impl ImapConnectionManager {
);
STATUS_DISPATCHER
.append_error(
self.account_id,
account_id,
format!("imap client connect error: {:#?}", error),
)
.await;
@@ -125,14 +112,13 @@ impl ImapConnectionManager {
}
};
let mut session = match self.authenticate(client, &account).await {
let mut session = match Self::authenticate(client, &account).await {
Ok(session) => session,
Err(error) => {
error!("Failed to authenticate IMAP session: {:#?}", error);
STATUS_DISPATCHER
.append_error(
self.account_id,
account_id,
format!("imap client authenticate error: {:#?}", error),
)
.await;
@@ -143,12 +129,12 @@ impl ImapConnectionManager {
match fetch_capabilities(&mut session).await {
Ok(capabilities) => {
let to_save: Vec<String> = capabilities.iter().map(capability_to_string).collect();
AccountModel::update_capabilities(self.account_id, to_save).await?;
AccountModel::update_capabilities(account_id, to_save).await?;
if let Err(error) = check_capabilities(&capabilities) {
error!("Failed to check IMAP capabilities: {:#?}", error);
STATUS_DISPATCHER
.append_error(
self.account_id,
account_id,
format!("imap client check capabilities error: {:#?}", error),
)
.await;
@@ -172,7 +158,7 @@ impl ImapConnectionManager {
error!("Failed to fetch IMAP capabilities: {:#?}", error);
STATUS_DISPATCHER
.append_error(
self.account_id,
account_id,
format!("imap client fetch capabilities error: {:#?}", error),
)
.await;

View File

@@ -22,7 +22,6 @@ pub mod client;
pub mod executor;
pub mod manager;
pub mod oauth2;
pub mod pool;
pub mod session;
pub mod stats;
#[cfg(test)]

View File

@@ -1,59 +0,0 @@
//
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::modules::error::code::ErrorCode;
use crate::modules::error::{BichonError, BichonResult};
use crate::modules::imap::{manager::ImapConnectionManager, session::SessionStream};
use crate::raise_error;
use async_imap::Session;
use bb8::Pool;
use std::time::Duration;
impl bb8::ManageConnection for ImapConnectionManager {
type Connection = Session<Box<dyn SessionStream>>;
type Error = BichonError;
async fn connect(&self) -> BichonResult<Self::Connection> {
self.build().await
}
// call this function before using the connection
async fn is_valid(&self, conn: &mut Self::Connection) -> BichonResult<()> {
conn.noop()
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))
}
fn has_broken(&self, _: &mut Self::Connection) -> bool {
false
}
}
pub async fn build_imap_pool(account_id: u64) -> BichonResult<Pool<ImapConnectionManager>> {
let manager = ImapConnectionManager::new(account_id);
let pool = Pool::builder()
.connection_timeout(Duration::from_secs(30))
//.idle_timeout(Duration::from_secs(120))
.retry_connection(true)
.max_size(10)
.test_on_check_out(true)
.build(manager)
.await?;
Ok(pool)
}

View File

@@ -18,12 +18,14 @@
use crate::modules::account::migration::{AccountModel, AccountType};
use crate::modules::cache::imap::mailbox::{Attribute, AttributeEnum, MailBox};
use crate::modules::context::executors::MAIL_CONTEXT;
use crate::modules::error::code::ErrorCode;
use crate::modules::error::{BichonError, BichonResult};
use crate::modules::error::BichonResult;
use crate::modules::imap::executor::ImapExecutor;
use crate::modules::imap::session::SessionStream;
use crate::modules::utils::create_hash;
use crate::raise_error;
use async_imap::types::Name;
use async_imap::Session;
pub async fn get_account_mailboxes(account_id: u64, remote: bool) -> BichonResult<Vec<MailBox>> {
let account = AccountModel::check_account_exists(account_id).await?;
@@ -42,9 +44,11 @@ pub async fn get_account_mailboxes(account_id: u64, remote: bool) -> BichonResul
}
pub async fn request_imap_all_mailbox_list(account_id: u64) -> BichonResult<Vec<MailBox>> {
let executor = MAIL_CONTEXT.imap(account_id).await?;
let names = executor.list_all_mailboxes().await?;
convert_names_to_mailboxes(account_id, names.iter()).await
let mut session = ImapExecutor::create_connection(account_id).await?;
let names = ImapExecutor::list_all_mailboxes(&mut session).await?;
let result = convert_names_to_mailboxes(account_id, &mut session, names.iter()).await?;
session.logout().await.ok();
Ok(result)
}
fn contains_no_select(attributes: &[Attribute]) -> bool {
@@ -55,50 +59,31 @@ fn contains_no_select(attributes: &[Attribute]) -> bool {
pub async fn convert_names_to_mailboxes(
account_id: u64,
session: &mut Session<Box<dyn SessionStream>>,
names: impl IntoIterator<Item = &Name>,
) -> BichonResult<Vec<MailBox>> {
// Preallocate enough space in the vector to avoid multiple reallocations
let mut tasks = Vec::new();
let mut mailboxes = Vec::new();
for name in names.into_iter() {
// Convert the name into a MailBox structure
for name in names {
let mailbox_name = name.name().to_string();
let mut mailbox: MailBox = name.into();
tracing::debug!(
raw = &mailbox_name,
decoded = &mailbox.name,
"mailbox name comparison"
);
if contains_no_select(&mailbox.attributes) {
continue;
}
mailbox.account_id = account_id;
mailbox.id = create_hash(account_id, &mailbox.name);
let task: tokio::task::JoinHandle<Result<MailBox, BichonError>> =
tokio::spawn(async move {
let executor = MAIL_CONTEXT.imap(account_id).await?;
let mx = executor.examine_mailbox(mailbox_name.as_str()).await?;
// Update the mailbox status information
mailbox.exists = mx.exists; // Number of messages in the mailbox
mailbox.unseen = mx.unseen; // Number of unseen messages
mailbox.uid_next = mx.uid_next; // Next unique identifier to be assigned
mailbox.uid_validity = mx.uid_validity; // Validity of the UIDs
Ok(mailbox)
});
tasks.push(task);
}
let mx = session
.examine(mailbox_name.as_str())
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?;
mailbox.exists = mx.exists;
mailbox.unseen = mx.unseen;
mailbox.uid_next = mx.uid_next;
mailbox.uid_validity = mx.uid_validity;
let mut mailboxes = Vec::new();
for task in tasks {
match task.await {
Ok(Ok(mailbox)) => mailboxes.push(mailbox),
Ok(Err(err)) => return Err(err), // Handle mailbox-level errors
Err(e) => return Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError)), // Handle task-level panics or errors
}
mailboxes.push(mailbox);
}
Ok(mailboxes)

View File

@@ -2,8 +2,8 @@ use crate::{
encode_mailbox_name,
modules::{
account::migration::{AccountModel, AccountType},
context::executors::MAIL_CONTEXT,
error::{code::ErrorCode, BichonResult},
imap::executor::ImapExecutor,
indexer::manager::{EML_INDEX_MANAGER, ENVELOPE_INDEX_MANAGER},
},
raise_error,
@@ -38,10 +38,9 @@ pub async fn restore_emails(account_id: u64, message_ids: Vec<u64>) -> BichonRes
ErrorCode::Incompatible
));
}
let executor = MAIL_CONTEXT.imap(account.id).await?;
let mut failed = Vec::new();
let mut session = ImapExecutor::create_connection(account_id).await?;
for message_id in message_ids {
let result: BichonResult<()> = async {
let envelope = ENVELOPE_INDEX_MANAGER
@@ -71,9 +70,14 @@ pub async fn restore_emails(account_id: u64, message_ids: Vec<u64>) -> BichonRes
})?;
if let Some(mailbox_name) = envelope.mailbox_name {
executor
.append(encode_mailbox_name!(&mailbox_name), None, None, &eml)
.await?;
ImapExecutor::append(
&mut session,
encode_mailbox_name!(&mailbox_name),
None,
None,
&eml,
)
.await?;
}
Ok(())
@@ -100,5 +104,7 @@ pub async fn restore_emails(account_id: u64, message_ids: Vec<u64>) -> BichonRes
);
}
session.logout().await.ok();
Ok(())
}

28
src/modules/utils/html.rs Normal file
View File

@@ -0,0 +1,28 @@
use std::panic;
use tracing::error;
pub fn extract_text(html: String) -> String {
let result = panic::catch_unwind(|| {
html2text::config::plain()
.allow_width_overflow()
.string_from_read(html.as_bytes(), 100)
});
match result {
Ok(Ok(text)) => text,
Ok(Err(err)) => {
error!("html2text error: {}", err);
html
}
Err(err) => {
if let Some(s) = err.downcast_ref::<&str>() {
error!("html2text panic: {}", s);
} else if let Some(s) = err.downcast_ref::<String>() {
error!("html2text panic: {}", s);
} else {
error!("html2text panic: unknown error");
}
html
}
}
}

View File

@@ -26,6 +26,7 @@ use rand::{rng, Rng};
use super::error::code::ErrorCode;
pub mod encrypt;
pub mod html;
pub mod net;
pub mod rate_limit;
pub mod shutdown;

View File

@@ -30,7 +30,7 @@ use tokio_io_timeout::TimeoutStream;
use tokio_socks::tcp::Socks5Stream;
use tracing::error;
pub(crate) const TIMEOUT: Duration = Duration::from_secs(60);
pub(crate) const TIMEOUT: Duration = Duration::from_secs(30);
pub(crate) async fn establish_tcp_connection_with_timeout(
address: SocketAddr,
@@ -39,17 +39,17 @@ pub(crate) async fn establish_tcp_connection_with_timeout(
// Establish the TCP connection with a timeout
let tcp_stream = connect_with_optional_proxy(use_proxy, address).await?;
// Disable Nagle's algorithm for more efficient network communication
tcp_stream
.set_nodelay(true)
.map_err(|e| raise_error!(e.to_string(), ErrorCode::NetworkError))?;
// // Disable Nagle's algorithm for more efficient network communication
// tcp_stream
// .set_nodelay(true)
// .map_err(|e| raise_error!(e.to_string(), ErrorCode::NetworkError))?;
// Wrap the TCP stream in a TimeoutStream for timeout management
let mut timeout_stream = TimeoutStream::new(tcp_stream);
// Set read and write timeouts
timeout_stream.set_write_timeout(Some(TIMEOUT));
timeout_stream.set_read_timeout(Some(TIMEOUT));
timeout_stream.set_write_timeout(Some(Duration::from_secs(15)));
timeout_stream.set_read_timeout(Some(Duration::from_secs(30)));
// Return the timeout-wrapped TCP stream as a Pin
Ok(Box::pin(timeout_stream))

View File

@@ -132,10 +132,10 @@ const getAccountSchema = (isEdit: boolean, t: (key: string) => string) =>
.optional(),
sync_interval_min: z.number({ invalid_type_error: t('validation.incrementalSyncMustBeNumber') }).int().min(10, { message: t('validation.incrementalSyncMustBeAtLeast10') }),
sync_batch_size: z
.number({ invalid_type_error: t('validation.incrementalSyncMustBeNumber') })
.number({ invalid_type_error: t('validation.singleRequestBatchSizeMustBeNumber') })
.int()
.min(30, { message: t('validation.incrementalSyncMustBeAtLeast10') })
.max(200, { message: t('validation.incrementalSyncMustBeAtLeast10') }),
.min(10, { message: t('validation.singleRequestBatchSizeTooSmall') })
.max(200, { message: t('validation.singleRequestBatchSizeTooLarge') }),
});
type Step = {
@@ -180,7 +180,7 @@ const defaultValues: Account = {
date_before: undefined,
folder_limit: undefined,
sync_interval_min: 10,
sync_batch_size: 50,
sync_batch_size: 30,
};
const emptyImap: ImapConfig = {

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "يجب أن يكون حد المجلد 100 على الأقل",
"incrementalSyncMustBeNumber": "يجب أن يكون فاصل المزامنة التزايدية رقمًا",
"incrementalSyncMustBeAtLeast10": "يجب أن يكون فاصل المزامنة التزايدية 10 دقائق على الأقل",
"pleaseEnterUsernameOrEmail": "الرجاء إدخال اسم المستخدم أو البريد الإلكتروني الخاص بك"
"pleaseEnterUsernameOrEmail": "الرجاء إدخال اسم المستخدم أو البريد الإلكتروني الخاص بك",
"singleRequestBatchSizeMustBeNumber": "يجب أن يكون حجم الدُفعة رقماً",
"singleRequestBatchSizeTooSmall": "يجب أن يكون حجم الدُفعة على الأقل 10",
"singleRequestBatchSizeTooLarge": "يجب أن يكون حجم الدُفعة على الأكثر 200"
},
"table": {
"noResults": "لم يتم العثور على نتائج",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "Mappegrænse skal være mindst 100",
"incrementalSyncMustBeNumber": "Inkrementelt synkroniseringsinterval skal være et tal",
"incrementalSyncMustBeAtLeast10": "Inkrementelt synkroniseringsinterval skal være mindst 10 minutter",
"pleaseEnterUsernameOrEmail": "Indtast dit brugernavn eller din e-mailadresse"
"pleaseEnterUsernameOrEmail": "Indtast dit brugernavn eller din e-mailadresse",
"singleRequestBatchSizeMustBeNumber": "Batchstørrelse skal være et tal",
"singleRequestBatchSizeTooSmall": "Batchstørrelse skal være mindst 10",
"singleRequestBatchSizeTooLarge": "Batchstørrelse skal være højst 200"
},
"table": {
"noResults": "Ingen resultater fundet",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "Das Ordnerlimit muss mindestens 100 betragen",
"incrementalSyncMustBeNumber": "Das inkrementelle Synchronisierungsintervall muss eine Zahl sein",
"incrementalSyncMustBeAtLeast10": "Das inkrementelle Synchronisierungsintervall muss mindestens 10 Minuten betragen",
"pleaseEnterUsernameOrEmail": "Bitte geben Sie Ihren Benutzernamen oder Ihre E-Mail-Adresse ein"
"pleaseEnterUsernameOrEmail": "Bitte geben Sie Ihren Benutzernamen oder Ihre E-Mail-Adresse ein",
"singleRequestBatchSizeMustBeNumber": "Die Stapelgröße muss eine Zahl sein",
"singleRequestBatchSizeTooSmall": "Die Stapelgröße muss mindestens 10 sein",
"singleRequestBatchSizeTooLarge": "Die Stapelgröße darf höchstens 200 sein"
},
"table": {
"noResults": "Keine Ergebnisse gefunden",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "Folder limit must be at least 100",
"incrementalSyncMustBeNumber": "Incremental sync interval must be a number",
"incrementalSyncMustBeAtLeast10": "Incremental sync interval must be at least 10 minutes",
"pleaseEnterUsernameOrEmail": "Please enter your username or email"
"pleaseEnterUsernameOrEmail": "Please enter your username or email",
"singleRequestBatchSizeMustBeNumber": "Batch size must be a number",
"singleRequestBatchSizeTooSmall": "Batch size must be at least 10",
"singleRequestBatchSizeTooLarge": "Batch size must be at most 200"
},
"table": {
"noResults": "No results found",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "El límite de carpetas debe ser de al menos 100",
"incrementalSyncMustBeNumber": "El intervalo de sincronización incremental debe ser un número",
"incrementalSyncMustBeAtLeast10": "El intervalo de sincronización incremental debe ser de al menos 10 minutos",
"pleaseEnterUsernameOrEmail": "Por favor, introduzca su nombre de usuario o correo electrónico"
"pleaseEnterUsernameOrEmail": "Por favor, introduzca su nombre de usuario o correo electrónico",
"singleRequestBatchSizeMustBeNumber": "El tamaño del lote debe ser un número",
"singleRequestBatchSizeTooSmall": "El tamaño del lote debe ser al menos 10",
"singleRequestBatchSizeTooLarge": "El tamaño del lote debe ser como máximo 200"
},
"table": {
"noResults": "No se encontraron resultados",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "Kansiorajan on oltava vähintään 100",
"incrementalSyncMustBeNumber": "Lisäävän synkronoinnin välin on oltava numero",
"incrementalSyncMustBeAtLeast10": "Lisäävän synkronoinnin välin on oltava vähintään 10 minuuttia",
"pleaseEnterUsernameOrEmail": "Ole hyvä ja anna käyttäjänimesi tai sähköpostiosoitteesi"
"pleaseEnterUsernameOrEmail": "Ole hyvä ja anna käyttäjänimesi tai sähköpostiosoitteesi",
"singleRequestBatchSizeMustBeNumber": "Eräkoko tulee olla numero",
"singleRequestBatchSizeTooSmall": "Eräkoko tulee olla vähintään 10",
"singleRequestBatchSizeTooLarge": "Eräkoko tulee olla enintään 200"
},
"table": {
"noResults": "Ei tuloksia löydy",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "La limite de dossiers doit être d'au moins 100",
"incrementalSyncMustBeNumber": "L'intervalle de synchronisation incrémentielle doit être un nombre",
"incrementalSyncMustBeAtLeast10": "L'intervalle de synchronisation incrémentielle doit être d'au moins 10 minutes",
"pleaseEnterUsernameOrEmail": "Veuillez entrer votre nom d'utilisateur ou votre courriel"
"pleaseEnterUsernameOrEmail": "Veuillez entrer votre nom d'utilisateur ou votre courriel",
"singleRequestBatchSizeMustBeNumber": "La taille du lot doit être un nombre",
"singleRequestBatchSizeTooSmall": "La taille du lot doit être au moins 10",
"singleRequestBatchSizeTooLarge": "La taille du lot doit être au plus 200"
},
"table": {
"noResults": "Aucun résultat trouvé",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "Il limite di cartelle deve essere almeno 100",
"incrementalSyncMustBeNumber": "L'intervallo di sincronizzazione incrementale deve essere un numero",
"incrementalSyncMustBeAtLeast10": "L'intervallo di sincronizzazione incrementale deve essere di almeno 10 minuti",
"pleaseEnterUsernameOrEmail": "Inserisci il tuo nome utente o email"
"pleaseEnterUsernameOrEmail": "Inserisci il tuo nome utente o email",
"singleRequestBatchSizeMustBeNumber": "La dimensione del batch deve essere un numero",
"singleRequestBatchSizeTooSmall": "La dimensione del batch deve essere almeno 10",
"singleRequestBatchSizeTooLarge": "La dimensione del batch deve essere al massimo 200"
},
"table": {
"noResults": "Nessun risultato trovato",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "フォルダーの制限は100以上である必要があります",
"incrementalSyncMustBeNumber": "増分同期間隔は数値である必要があります",
"incrementalSyncMustBeAtLeast10": "増分同期間隔は10分以上である必要があります",
"pleaseEnterUsernameOrEmail": "ユーザー名またはメールアドレスを入力してください"
"pleaseEnterUsernameOrEmail": "ユーザー名またはメールアドレスを入力してください",
"singleRequestBatchSizeMustBeNumber": "バッチサイズは数値でなければなりません",
"singleRequestBatchSizeTooSmall": "バッチサイズは最低でも10でなければなりません",
"singleRequestBatchSizeTooLarge": "バッチサイズは最大でも200でなければなりません"
},
"table": {
"noResults": "結果が見つかりませんでした",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "폴더 제한은 최소 100 이상이어야 합니다",
"incrementalSyncMustBeNumber": "증분 동기화 간격은 숫자여야 합니다",
"incrementalSyncMustBeAtLeast10": "증분 동기화 간격은 최소 10분 이상이어야 합니다",
"pleaseEnterUsernameOrEmail": "사용자 이름 또는 이메일을 입력하세요"
"pleaseEnterUsernameOrEmail": "사용자 이름 또는 이메일을 입력하세요",
"singleRequestBatchSizeMustBeNumber": "배치 크기는 숫자여야 합니다",
"singleRequestBatchSizeTooSmall": "배치 크기는 최소 10이어야 합니다",
"singleRequestBatchSizeTooLarge": "배치 크기는 최대 200이어야 합니다"
},
"table": {
"noResults": "검색 결과 없음",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "Mappenlimiet moet ten minste 100 zijn",
"incrementalSyncMustBeNumber": "Incrementaal sync interval moet een nummer zijn",
"incrementalSyncMustBeAtLeast10": "Incrementaal sync interval moet ten minste 10 minuten zijn",
"pleaseEnterUsernameOrEmail": "Voer alstublieft uw gebruikersnaam of e-mailadres in"
"pleaseEnterUsernameOrEmail": "Voer alstublieft uw gebruikersnaam of e-mailadres in",
"singleRequestBatchSizeMustBeNumber": "Batchgrootte moet een nummer zijn",
"singleRequestBatchSizeTooSmall": "Batchgrootte moet ten minste 10 zijn",
"singleRequestBatchSizeTooLarge": "Batchgrootte moet hoogstens 200 zijn"
},
"table": {
"noResults": "Geen resultaten gevonden",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "Mappegrense må være minst 100",
"incrementalSyncMustBeNumber": "Intervall for inkrementell synkronisering må være et tall",
"incrementalSyncMustBeAtLeast10": "Intervall for inkrementell synkronisering må være minst 10 minutter",
"pleaseEnterUsernameOrEmail": "Skriv inn brukernavn eller e-postadresse"
"pleaseEnterUsernameOrEmail": "Skriv inn brukernavn eller e-postadresse",
"singleRequestBatchSizeMustBeNumber": "Batchstørrelse må være et tall",
"singleRequestBatchSizeTooSmall": "Batchstørrelse må være minst 10",
"singleRequestBatchSizeTooLarge": "Batchstørrelse må være maksimalt 200"
},
"table": {
"noResults": "Ingen resultater funnet",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "Limit folderów musi być liczbą nie mniejszą niż 100",
"incrementalSyncMustBeNumber": "Przyrostowy interwał synchronizacji musi być liczbą",
"incrementalSyncMustBeAtLeast10": "Przyrostowy interwał synchronizacji musi wynosić co najmniej 10 minut",
"pleaseEnterUsernameOrEmail": "Proszę podać nazwę użytkownika lub adres e-mail"
"pleaseEnterUsernameOrEmail": "Proszę podać nazwę użytkownika lub adres e-mail",
"singleRequestBatchSizeMustBeNumber": "Rozmiar partii musi być liczbą",
"singleRequestBatchSizeTooSmall": "Rozmiar partii musi wynosić co najmniej 10",
"singleRequestBatchSizeTooLarge": "Rozmiar partii może wynosić maksymalnie 200"
},
"table": {
"noResults": "Brak wyników",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "O Limite de Pasta deve ser pelo menos 100",
"incrementalSyncMustBeNumber": "O Intervalo de Sincronização Incremental deve ser um número",
"incrementalSyncMustBeAtLeast10": "O Intervalo de Sincronização Incremental deve ser de pelo menos 10 minutos",
"pleaseEnterUsernameOrEmail": "Por favor, digite seu nome de usuário ou e-mail"
"pleaseEnterUsernameOrEmail": "Por favor, digite seu nome de usuário ou e-mail",
"singleRequestBatchSizeMustBeNumber": "O tamanho do lote deve ser um número",
"singleRequestBatchSizeTooSmall": "O tamanho do lote deve ser pelo menos 10",
"singleRequestBatchSizeTooLarge": "O tamanho do lote deve ser no máximo 200"
},
"table": {
"noResults": "Nenhum resultado encontrado",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "Лимит папки должен быть не менее 100",
"incrementalSyncMustBeNumber": "Интервал инкрементальной синхронизации должен быть числом",
"incrementalSyncMustBeAtLeast10": "Интервал инкрементальной синхронизации должен быть не менее 10 минут",
"pleaseEnterUsernameOrEmail": "Пожалуйста, введите ваше имя пользователя или адрес электронной почты"
"pleaseEnterUsernameOrEmail": "Пожалуйста, введите ваше имя пользователя или адрес электронной почты",
"singleRequestBatchSizeMustBeNumber": "Размер пакета должен быть числом",
"singleRequestBatchSizeTooSmall": "Размер пакета должен быть не менее 10",
"singleRequestBatchSizeTooLarge": "Размер пакета должен быть не более 200"
},
"table": {
"noResults": "Результатов не найдено",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "Mappgräns måste vara minst 100",
"incrementalSyncMustBeNumber": "Intervall för inkrementell synkronisering måste vara ett tal",
"incrementalSyncMustBeAtLeast10": "Intervall för inkrementell synkronisering måste vara minst 10 minuter",
"pleaseEnterUsernameOrEmail": "Vänligen ange ditt användarnamn eller din e-postadress"
"pleaseEnterUsernameOrEmail": "Vänligen ange ditt användarnamn eller din e-postadress",
"singleRequestBatchSizeMustBeNumber": "Batchstorlek måste vara ett tal",
"singleRequestBatchSizeTooSmall": "Batchstorlek måste vara minst 10",
"singleRequestBatchSizeTooLarge": "Batchstorlek måste vara högst 200"
},
"table": {
"noResults": "Inga resultat hittades",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "資料夾限制必須至少為 100",
"incrementalSyncMustBeNumber": "增量同步間隔必須是數字",
"incrementalSyncMustBeAtLeast10": "增量同步間隔必須至少為 10 分鐘",
"pleaseEnterUsernameOrEmail": "請輸入您的使用者名稱或電子郵件"
"pleaseEnterUsernameOrEmail": "請輸入您的使用者名稱或電子郵件",
"singleRequestBatchSizeMustBeNumber": "批次大小必須是數字",
"singleRequestBatchSizeTooSmall": "批次大小必須至少為10",
"singleRequestBatchSizeTooLarge": "批次大小必須最多為200"
},
"table": {
"noResults": "查無結果",

View File

@@ -938,7 +938,10 @@
"folderLimitMustBeAtLeast100": "文件夹限制必须至少为 100",
"incrementalSyncMustBeNumber": "增量同步间隔必须是数字",
"incrementalSyncMustBeAtLeast10": "增量同步间隔必须至少为 10 分钟",
"pleaseEnterUsernameOrEmail": "请输入您的用户名或邮箱"
"pleaseEnterUsernameOrEmail": "请输入您的用户名或邮箱",
"singleRequestBatchSizeMustBeNumber": "批大小必须是数字",
"singleRequestBatchSizeTooSmall": "批大小必须至少为10",
"singleRequestBatchSizeTooLarge": "批大小必须最多为200"
},
"table": {
"noResults": "未找到结果",