diff --git a/pumpkin-inventory/src/screen_handler.rs b/pumpkin-inventory/src/screen_handler.rs
index 4e8b1a165..5e76f3594 100644
--- a/pumpkin-inventory/src/screen_handler.rs
+++ b/pumpkin-inventory/src/screen_handler.rs
@@ -1223,6 +1223,9 @@ pub trait ScreenHandler: Send + Sync {
             } else if action_type == SlotActionType::Swap && (0..9).contains(&button)
                 || button == 40
             {
+                if slot_index < 0 {
+                    return;
+                }
                 let mut button_stack = player
                     .get_inventory()
                     .get_stack(button as usize)