From 427f7248d2005416102566f439e924b93efdbcbc Mon Sep 17 00:00:00 2001
From: fama <720793+fama@users.noreply.github.com>
Date: Wed, 3 Jun 2026 05:06:08 +0000
Subject: [PATCH] fix: open NewIndexWriter once across all migration segments

Previously, do_migrate_segment created a fresh NewIndexWriter (and
therefore a new Fjall Database) on every call, meaning the Fjall
database at bichon-storage/ was opened and closed once per segment.

This caused the migration to fail mid-way through (observed at segment
9/16) with:

  Storage(InvalidTag(("ChecksumType", 171)))

Root cause: after segment N writes email blobs via Fjall's ingestion
API (start_ingestion / write / finish), those SSTables and KV-separated
blob files are flushed to disk and the Database is dropped. When segment
N+1 calls Database::builder(storage_dir).open(), Fjall must discover and
catalog all on-disk files produced by the previous segments. During that
discovery it reads SSTable or blob-file block headers and encounters a
ChecksumType discriminant byte (171 / 0xAB) that lsm-tree 3.1.4 does
not recognise, causing the fatal error.

The first N segments succeed because the cumulative set of ingested
SSTables stays small enough that Fjall does not need to read the
offending headers during reopen. Once enough data has accumulated the
reopen triggers a manifest or compaction read that exposes the mismatch.

Fix: open NewIndexWriter once, before the segment loop, and pass a
&mut reference into each do_migrate_segment call. finish_writers() is
called a single time after all segments complete. The Fjall Database
stays open for the entire migration and is never closed and reopened,
eliminating the incompatible-reopen path entirely.
---
 crates/admin/src/migrate.rs    | 23 +++++++++++++++++++++--
 crates/core/src/migrate/mod.rs |  5 +----
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/crates/admin/src/migrate.rs b/crates/admin/src/migrate.rs
index 4de0896..0005e5f 100644
--- a/crates/admin/src/migrate.rs
+++ b/crates/admin/src/migrate.rs
@@ -2,7 +2,7 @@ use std::path::{Path, PathBuf};
 
 use bichon_core::migrate::{
     count_eml_segments, do_migrate_segment, is_tantivy_index_dir,
-    store::{LegacyDirs, NewDirs},
+    store::{LegacyDirs, NewDirs, NewIndexWriter},
 };
 use console::style;
 use dialoguer::{theme::ColorfulTheme, Confirm, Input};
@@ -326,6 +326,18 @@ pub fn handle_migration(theme: &ColorfulTheme) {
             .progress_chars("#>-"),
     );
 
+    let mut writer = match NewIndexWriter::open(NewDirs::new(
+        new_index_path.clone(),
+        new_data_path.clone(),
+    )) {
+        Ok(w) => w,
+        Err(e) => {
+            pb.finish_with_message(format!("{}", style("Migration failed.").red()));
+            eprintln!("\n{} {:?}", style("✘").red().bold(), e);
+            return;
+        }
+    };
+
     let mut grand_total_migrated: usize = 0;
     let mut grand_total_skipped: usize = 0;
 
@@ -337,7 +349,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
         match do_migrate_segment(
             batch_size,
             legacy,
-            NewDirs::new(new_index_path.clone(), new_data_path.clone()),
+            &mut writer,
             seg_idx,
             |msg| {
                 if let Some(data) = msg.strip_prefix("TOTAL:") {
@@ -407,6 +419,13 @@ pub fn handle_migration(theme: &ColorfulTheme) {
         pb.set_position((seg_idx + 1) as u64);
     }
 
+    pb.set_message(style("Finalizing indexes...").dim().to_string());
+    if let Err(e) = writer.finish_writers() {
+        pb.finish_with_message(format!("{}", style("Migration failed.").red()));
+        eprintln!("\n{} {:?}", style("✘").red().bold(), e);
+        return;
+    }
+
     pb.finish_with_message(format!(
         "Migration finished. Total: {}, Skipped: {}",
         grand_total_migrated, grand_total_skipped
diff --git a/crates/core/src/migrate/mod.rs b/crates/core/src/migrate/mod.rs
index 02f15aa..80255cd 100644
--- a/crates/core/src/migrate/mod.rs
+++ b/crates/core/src/migrate/mod.rs
@@ -121,7 +121,7 @@ fn is_dir_not_empty(path: &PathBuf) -> std::io::Result<bool> {
 pub fn do_migrate_segment<F>(
     batch_size: u32,
     legacy: LegacyDirs,
-    new_dirs: NewDirs,
+    writer: &mut NewIndexWriter,
     segment_index: usize,
     mut on_progress: F,
 ) -> BichonResult<()>
@@ -226,8 +226,6 @@ where
     drop(envelope_index);
 
     // ── Phase 2: process EML docs, streaming one at a time ─────────────
-    let mut writer = NewIndexWriter::open(new_dirs)?;
-
     let mut total_migrated = 0usize;
     let mut total_skipped = 0usize;
 
@@ -308,7 +306,6 @@ where
         chunk_start = chunk_end;
     }
 
-    writer.finish_writers()?;
     on_progress(&format!("DONE:{}:{}", total_migrated, total_skipped));
     Ok(())
 }