From 16578fb8e29548da3905f9b770ecbcffa1a89f44 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Sat, 27 Dec 2025 03:38:53 +0800 Subject: [PATCH] fix: stitch adjacent RFC2047 words to prevent byte-split artifacts #79 --- src/modules/envelope/extractor.rs | 18 +++- src/modules/envelope/mod.rs | 2 +- src/modules/envelope/utils.rs | 114 ++++++++++++++++++++ src/modules/imap/tests.rs | 75 ++++++++++++- web/src/features/users/api-tokens/index.tsx | 1 - 5 files changed, 203 insertions(+), 7 deletions(-) create mode 100644 src/modules/envelope/utils.rs diff --git a/src/modules/envelope/extractor.rs b/src/modules/envelope/extractor.rs index 23f1a1a..59b565a 100644 --- a/src/modules/envelope/extractor.rs +++ b/src/modules/envelope/extractor.rs @@ -17,13 +17,14 @@ // along with this program. If not, see . use crate::modules::common::AddrVec; +use crate::modules::envelope::utils::normalize_subject; use crate::modules::error::code::ErrorCode; use crate::modules::error::BichonResult; use crate::modules::utils::create_hash; use crate::{calculate_hash, raise_error, utc_now}; use crate::{id, modules::indexer::envelope::Envelope}; use async_imap::types::Fetch; -use mail_parser::{Message, MessageParser, MimeHeaders}; +use mail_parser::{HeaderName, Message, MessageParser, MimeHeaders}; pub fn extract_envelope(fetch: &Fetch, account_id: u64, mailbox_id: u64) -> BichonResult { let internal_date = fetch @@ -62,7 +63,13 @@ pub fn extract_envelope(fetch: &Fetch, account_id: u64, mailbox_id: u64) -> Bich let in_reply_to = message.in_reply_to().as_text().map(String::from); let references = extract_references(&message); let thread_id = compute_thread_id(in_reply_to, references, &message_id); - let subject = message.subject().map(String::from).unwrap_or("".into()); + + let mut subject = message.subject().map(String::from).unwrap_or_default(); + + if subject.contains('\u{FFFD}') { + subject = normalize_subject(message.header_raw(HeaderName::Subject)); + } + let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0); let bcc: Option> = message.bcc().map(|addr| { AddrVec::from(addr) @@ -150,7 +157,12 @@ pub fn extract_envelope_from_eml( let in_reply_to = message.in_reply_to().as_text().map(String::from); let references = extract_references(&message); let thread_id = compute_thread_id(in_reply_to, references, &message_id); - let subject = message.subject().map(String::from).unwrap_or("".into()); + + let mut subject = message.subject().map(String::from).unwrap_or_default(); + if subject.contains('\u{FFFD}') { + subject = normalize_subject(message.header_raw(HeaderName::Subject)); + } + let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0); let bcc: Option> = message.bcc().map(|addr| { AddrVec::from(addr) diff --git a/src/modules/envelope/mod.rs b/src/modules/envelope/mod.rs index 872957e..87f2ec9 100644 --- a/src/modules/envelope/mod.rs +++ b/src/modules/envelope/mod.rs @@ -16,5 +16,5 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . - pub mod extractor; +pub mod utils; diff --git a/src/modules/envelope/utils.rs b/src/modules/envelope/utils.rs new file mode 100644 index 0000000..1fe281d --- /dev/null +++ b/src/modules/envelope/utils.rs @@ -0,0 +1,114 @@ +use mail_parser::parsers::MessageStream; +use regex::{Captures, Regex}; + +fn merge_contiguous_encoded_words(input: &str) -> String { + let block_re = + Regex::new(r"(?:=\?[^?]+\?[bBqQ]\?[^?]+\?=)(?:\s+(?:=\?[^?]+\?[bBqQ]\?[^?]+\?=))+") + .unwrap(); + + let word_re = Regex::new(r"=\?([^?]+)\?([bBqQ])\?([^?]+)\?=").unwrap(); + + block_re + .replace_all(input, |caps: &Captures| { + let whole = caps.get(0).unwrap().as_str(); + + let mut charset: Option = None; + let mut encoding: Option = None; + let mut combined = String::new(); + let mut ok = true; + + for cap in word_re.captures_iter(whole) { + let cs = &cap[1]; + let enc = cap[2].to_ascii_uppercase(); + let text = &cap[3]; + + if let Some(ref c) = charset { + if c != cs { + ok = false; + break; + } + } else { + charset = Some(cs.to_string()); + } + + if let Some(ref e) = encoding { + if e != &enc { + ok = false; + break; + } + } else { + encoding = Some(enc); + } + + combined.push_str(text); + } + + if ok { + format!( + "=?{}?{}?{}?=", + charset.unwrap(), + encoding.unwrap(), + combined + ) + } else { + whole.to_string() + } + }) + .to_string() +} + +pub fn normalize_subject(raw_subject: Option<&str>) -> String { + let subject = match raw_subject { + Some(subject) => merge_contiguous_encoded_words(subject), + None => return String::new(), + }; + + MessageStream::new(subject.as_bytes()) + .parse_unstructured() + .as_text() + .map(String::from) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use crate::modules::envelope::utils::merge_contiguous_encoded_words; + + + #[tokio::test] + async fn test3() { + let s = "Hello =?UTF-8?B?SGVsbG8=?= =?UTF-8?B?V29ybGQ=?= !!!"; + assert_eq!( + merge_contiguous_encoded_words(s), + "Hello =?UTF-8?B?SGVsbG8=V29ybGQ=?= !!!" + ); + + let s = "=?UTF-8?B?QQ==?= =?UTF-8?B?Qg==?= =?UTF-8?B?Qw==?="; + assert_eq!( + merge_contiguous_encoded_words(s), + "=?UTF-8?B?QQ==Qg==Qw==?=" + ); + + let s = "=?UTF-8?B?QQ==?= =?UTF-8?B?Qg==?= test =?UTF-8?B?Qw==?= =?UTF-8?B?RA==?="; + assert_eq!( + merge_contiguous_encoded_words(s), + "=?UTF-8?B?QQ==Qg==?= test =?UTF-8?B?Qw==RA==?=" + ); + + let s = "=?UTF-8?B?QQ==?= =?GBK?B?Qg==?="; + assert_eq!(merge_contiguous_encoded_words(s), s); + let s = "=?UTF-8?B?QQ==?= =?UTF-8?Q?Qg?="; + assert_eq!(merge_contiguous_encoded_words(s), s); + + let s = "=?UTF-8?b?QQ==?= =?UTF-8?B?Qg==?="; + assert_eq!(merge_contiguous_encoded_words(s), "=?UTF-8?B?QQ==Qg==?="); + let s = "Hello =?UTF-8?B?SGVsbG8=?= !!!"; + assert_eq!(merge_contiguous_encoded_words(s), s); + let s = "=?UTF-8?B?QQ==?= =?UTF-8?B?Qg==?="; + assert_eq!(merge_contiguous_encoded_words(s), "=?UTF-8?B?QQ==Qg==?="); + let s = "Just a normal subject line"; + assert_eq!(merge_contiguous_encoded_words(s), s); + let s = "=?UTF-8?Q?Hello_?= =?UTF-8?Q?World?="; + assert_eq!(merge_contiguous_encoded_words(s), "=?UTF-8?Q?Hello_World?="); + } +} diff --git a/src/modules/imap/tests.rs b/src/modules/imap/tests.rs index f403cb7..67b390d 100644 --- a/src/modules/imap/tests.rs +++ b/src/modules/imap/tests.rs @@ -16,11 +16,13 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use mail_parser::MessageParser; +use mail_parser::{parsers::MessageStream, HeaderName, MessageParser}; use crate::{ base64_encode_url_safe, - modules::{account::entity::Encryption, imap::client::Client}, + modules::{ + account::entity::Encryption, envelope::utils::normalize_subject, imap::client::Client, + }, }; #[tokio::test] @@ -48,3 +50,72 @@ async fn test1() { println!("{}", part.is_multipart()); } } + +#[tokio::test] +async fn test2() { + const MESSAGE: &str = r#"From: Art Vandelay (Vandelay Industries) +To: "Colleagues": "James Smythe" ; Friends: + jane@example.com, =?UTF-8?Q?John_Sm=C3=AEth?= ; +Date: Sat, 20 Nov 2021 14:22:01 -0800 +Subject: =?utf-8?B?SnVzdCAxNSBkYXlzIGxlZnQgdG8gdmlzaXQgTkFSTklBISDinYTvuI/wn462?= +Content-Type: multipart/mixed; boundary="festivus"; + +--festivus +Content-Type: text/html; charset="us-ascii" +Content-Transfer-Encoding: base64 + +PGh0bWw+PHA+SSB3YXMgdGhpbmtpbmcgYWJvdXQgcXVpdHRpbmcgdGhlICZsZHF1bztle +HBvcnRpbmcmcmRxdW87IHRvIGZvY3VzIGp1c3Qgb24gdGhlICZsZHF1bztpbXBvcnRpbm +cmcmRxdW87LDwvcD48cD5idXQgdGhlbiBJIHRob3VnaHQsIHdoeSBub3QgZG8gYm90aD8 +gJiN4MjYzQTs8L3A+PC9odG1sPg== +--festivus +Content-Type: message/rfc822 + +From: "Cosmo Kramer" +Subject: Exporting my book about coffee tables +Content-Type: multipart/mixed; boundary="giddyup"; + +--giddyup +Content-Type: text/plain; charset="utf-16" +Content-Transfer-Encoding: quoted-printable + +=FF=FE=0C!5=D8"=DD5=D8)=DD5=D8-=DD =005=D8*=DD5=D8"=DD =005=D8"= +=DD5=D85=DD5=D8-=DD5=D8,=DD5=D8/=DD5=D81=DD =005=D8*=DD5=D86=DD = +=005=D8=1F=DD5=D8,=DD5=D8,=DD5=D8(=DD =005=D8-=DD5=D8)=DD5=D8"= +=DD5=D8=1E=DD5=D80=DD5=D8"=DD!=00 +--giddyup +Content-Type: image/gif; name*1="about "; name*0="Book "; + name*2*=utf-8''%e2%98%95 tables.gif +Content-Transfer-Encoding: Base64 +Content-Disposition: attachment + +R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7 +--giddyup-- +--festivus-- +"#; + + let message = MessageParser::default().parse(MESSAGE).unwrap(); + let raw_subject = message.header_raw("Subject").unwrap().as_bytes(); + + let data = MessageStream::new(raw_subject) + .parse_unstructured() + .unwrap_text() + .to_string(); + + println!("{}", data); + // RFC2047 support for encoded text in message readers + println!("{}", message.subject().unwrap()); +} + +#[tokio::test] +async fn test44() { + let path = r"C:\Users\polly\Downloads\test222.eml"; + let input = std::fs::read(path).unwrap(); + let message = MessageParser::default().parse(&input).unwrap(); + let subject = message.subject().unwrap(); + println!("Subject: {}", subject); + if subject.contains('\u{FFFD}') { + let subject = normalize_subject(message.header_raw(HeaderName::Subject)); + println!("Subject: {}", subject); + } +} diff --git a/web/src/features/users/api-tokens/index.tsx b/web/src/features/users/api-tokens/index.tsx index b9d918b..8aef357 100644 --- a/web/src/features/users/api-tokens/index.tsx +++ b/web/src/features/users/api-tokens/index.tsx @@ -18,7 +18,6 @@ import { useState } from 'react' import useDialogState from '@/hooks/use-dialog-state' -import { Button } from '@/components/ui/button' import { getColumns } from './components/columns' import { ApiTokenDeleteDialog } from './components/delete-dialog' import { ApiTokensTable } from './components/table'