majjo/Pumpkin
1
0
Fork 0
mirror of https://github.com/Pumpkin-MC/Pumpkin.git synced 2026-08-30 20:14:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improved chunk deserializing performance

Browse Source 
doing `let _ = io::copy(&mut self.reader.by_ref().take(count), &mut io::sink())` was just crazy slow when skipping bytes in NBT, we now instead use `seek`

The Palette was also changed and does not use a HashMap anymore
This commit is contained in:
Alexander Medvedev
2025-08-15 14:57:43 +02:00
parent ea2678a235
commit fbbade23dc
10 changed files with 241 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
pumpkin-nbt/src/compound.rs
View File

@@ -5,7 +5,7 @@ use crate::deserializer::NbtReadHelper;
use crate::serializer::WriteAdaptor;
use crate::tag::NbtTag;
use crate::{END_ID, Error, Nbt, get_nbt_string};
use std::io::{ErrorKind, Read, Write};
use std::io::{ErrorKind, Read, Seek, Write};
use std::vec::IntoIter;
#[derive(Clone, Debug, Default, PartialEq, PartialOrd)]
@@ -20,7 +20,7 @@ impl NbtCompound {
}
}
pub fn skip_content<R: Read>(reader: &mut NbtReadHelper<R>) -> Result<(), Error> {
pub fn skip_content<R: Read + Seek>(reader: &mut NbtReadHelper<R>) -> Result<(), Error> {
loop {
let tag_id = match reader.get_u8_be() {
Ok(id) => id,
@@ -43,7 +43,7 @@ impl NbtCompound {
}
let len = reader.get_u16_be()?;
reader.skip_bytes(len as u64)?;
reader.skip_bytes(len as i64)?;
NbtTag::skip_data(reader, tag_id)?;
}
@@ -51,7 +51,7 @@ impl NbtCompound {
Ok(())
}
pub fn deserialize_content<R: Read>(
pub fn deserialize_content<R: Read + Seek>(
reader: &mut NbtReadHelper<R>,
) -> Result<NbtCompound, Error> {
let mut compound = NbtCompound::new();

91
pumpkin-nbt/src/deserializer.rs
View File

@@ -1,3 +1,5 @@
use std::io::{Seek, SeekFrom};
use crate::*;
use io::Read;
use serde::de::{self, DeserializeSeed, IntoDeserializer, MapAccess, SeqAccess, Visitor};
@@ -6,11 +8,11 @@ use serde::{Deserialize, forward_to_deserialize_any};
pub type Result<T> = std::result::Result<T, Error>;
#[derive(Debug)]
pub struct NbtReadHelper<R: Read> {
pub struct NbtReadHelper<R: Read + Seek> {
reader: R,
}
impl<R: Read> NbtReadHelper<R> {
impl<R: Read + Seek> NbtReadHelper<R> {
pub fn new(r: R) -> Self {
Self { reader: r }
}
@@ -29,9 +31,11 @@ macro_rules! define_get_number_be {
};
}
impl<R: Read> NbtReadHelper<R> {
pub fn skip_bytes(&mut self, count: u64) -> Result<()> {
let _ = io::copy(&mut self.reader.by_ref().take(count), &mut io::sink())
impl<R: Read + Seek> NbtReadHelper<R> {
pub fn skip_bytes(&mut self, count: i64) -> Result<()> {
self.reader
.by_ref()
.seek(SeekFrom::Current(count))
.map_err(Error::Incomplete)?;
Ok(())
}
@@ -58,41 +62,38 @@ impl<R: Read> NbtReadHelper<R> {
}
#[derive(Debug)]
pub struct Deserializer<R: Read> {
pub struct Deserializer<R: Read + Seek> {
input: NbtReadHelper<R>,
tag_to_deserialize_stack: Vec<u8>,
tag_to_deserialize_stack: Option<u8>,
// Yes, this breaks with recursion. Just an attempt at a sanity check
in_list: bool,
is_named: bool,
// For debugging
key_stack: Vec<String>,
}
impl<R: Read> Deserializer<R> {
impl<R: Read + Seek> Deserializer<R> {
pub fn new(input: R, is_named: bool) -> Self {
Deserializer {
input: NbtReadHelper { reader: input },
tag_to_deserialize_stack: Vec::new(),
tag_to_deserialize_stack: None,
in_list: false,
is_named,
key_stack: Vec::new(),
}
}
}
/// Deserializes struct using Serde Deserializer from normal NBT
pub fn from_bytes<'a, T: Deserialize<'a>>(r: impl Read) -> Result<T> {
pub fn from_bytes<'a, T: Deserialize<'a>>(r: impl Read + Seek) -> Result<T> {
let mut deserializer = Deserializer::new(r, true);
T::deserialize(&mut deserializer)
}
/// Deserializes struct using Serde Deserializer from network NBT
pub fn from_bytes_unnamed<'a, T: Deserialize<'a>>(r: impl Read) -> Result<T> {
pub fn from_bytes_unnamed<'a, T: Deserialize<'a>>(r: impl Read + Seek) -> Result<T> {
let mut deserializer = Deserializer::new(r, false);
T::deserialize(&mut deserializer)
}
impl<'de, R: Read> de::Deserializer<'de> for &mut Deserializer<R> {
impl<'de, R: Read + Seek> de::Deserializer<'de> for &mut Deserializer<R> {
type Error = Error;
forward_to_deserialize_any! {
@@ -101,7 +102,7 @@ impl<'de, R: Read> de::Deserializer<'de> for &mut Deserializer<R> {
}
fn deserialize_ignored_any<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> {
let Some(tag) = self.tag_to_deserialize_stack.pop() else {
let Some(tag) = self.tag_to_deserialize_stack else {
return Err(Error::SerdeError("Ignoring nothing!".to_string()));
};
@@ -110,7 +111,7 @@ impl<'de, R: Read> de::Deserializer<'de> for &mut Deserializer<R> {
}
fn deserialize_any<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> {
let Some(tag_to_deserialize) = self.tag_to_deserialize_stack.pop() else {
let Some(tag_to_deserialize) = self.tag_to_deserialize_stack else {
return Err(Error::SerdeError(
"The top level must be a component (e.g. a struct)".to_string(),
));
@@ -188,25 +189,13 @@ impl<'de, R: Read> de::Deserializer<'de> for &mut Deserializer<R> {
}
fn deserialize_bool<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> {
if let Some(tag_id) = self.tag_to_deserialize_stack.last() {
if *tag_id == BYTE_ID {
let value = self.input.get_u8_be()?;
if value != 0 {
visitor.visit_bool(true)
} else {
visitor.visit_bool(false)
}
} else {
Err(Error::UnsupportedType(format!(
"Non-byte bool (found type {tag_id})"
)))
if self.tag_to_deserialize_stack.unwrap() == BYTE_ID {
let value = self.input.get_u8_be()?;
if value != 0 {
return visitor.visit_bool(true);
}
} else {
Err(Error::SerdeError(
"Wanted to deserialize a bool, but there was no type hint on the stack!"
.to_string(),
))
}
visitor.visit_bool(false)
}
fn deserialize_enum<V: Visitor<'de>>(
@@ -225,14 +214,10 @@ impl<'de, R: Read> de::Deserializer<'de> for &mut Deserializer<R> {
}
fn deserialize_map<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> {
if let Some(tag_id) = self.tag_to_deserialize_stack.pop() {
if let Some(tag_id) = self.tag_to_deserialize_stack {
if tag_id != COMPOUND_ID {
return Err(Error::SerdeError(format!(
"Trying to deserialize a map without a compound ID ({} with id {})",
self.key_stack
.last()
.cloned()
.unwrap_or_else(|| "compound root".to_string()),
"Trying to deserialize a map without a compound ID (id {})",
tag_id
)));
}
@@ -243,8 +228,9 @@ impl<'de, R: Read> de::Deserializer<'de> for &mut Deserializer<R> {
}
if self.is_named {
// Consume struct name
let _ = get_nbt_string(&mut self.input)?;
// Consume struct name, similar to get_nbt_string but without cesu8::from_java_cesu8
let length = self.input.get_u16_be()? as usize;
let _ = self.input.read_boxed_slice(length)?;
}
}
@@ -271,16 +257,16 @@ impl<'de, R: Read> de::Deserializer<'de> for &mut Deserializer<R> {
}
}
struct CompoundAccess<'a, R: Read> {
struct CompoundAccess<'a, R: Read + Seek> {
de: &'a mut Deserializer<R>,
}
impl<'de, R: Read> MapAccess<'de> for CompoundAccess<'_, R> {
impl<'de, R: Read + Seek> MapAccess<'de> for CompoundAccess<'_, R> {
type Error = Error;
fn next_key_seed<K: DeserializeSeed<'de>>(&mut self, seed: K) -> Result<Option<K::Value>> {
let tag = self.de.input.get_u8_be()?;
self.de.tag_to_deserialize_stack.push(tag);
self.de.tag_to_deserialize_stack = Some(tag);
if tag == END_ID {
return Ok(None);
@@ -290,22 +276,19 @@ impl<'de, R: Read> MapAccess<'de> for CompoundAccess<'_, R> {
}
fn next_value_seed<V: DeserializeSeed<'de>>(&mut self, seed: V) -> Result<V::Value> {
let result = seed.deserialize(&mut *self.de);
self.de.key_stack.pop();
result
seed.deserialize(&mut *self.de)
}
}
struct MapKey<'a, R: Read> {
struct MapKey<'a, R: Read + Seek> {
de: &'a mut Deserializer<R>,
}
impl<'de, R: Read> de::Deserializer<'de> for MapKey<'_, R> {
impl<'de, R: Read + Seek> de::Deserializer<'de> for MapKey<'_, R> {
type Error = Error;
fn deserialize_any<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value> {
let key = get_nbt_string(&mut self.de.input)?;
self.de.key_stack.push(key.clone());
visitor.visit_string(key)
}
@@ -315,13 +298,13 @@ impl<'de, R: Read> de::Deserializer<'de> for MapKey<'_, R> {
}
}
struct ListAccess<'a, R: Read> {
struct ListAccess<'a, R: Read + Seek> {
de: &'a mut Deserializer<R>,
remaining_values: usize,
list_type: u8,
}
impl<'de, R: Read> SeqAccess<'de> for ListAccess<'_, R> {
impl<'de, R: Read + Seek> SeqAccess<'de> for ListAccess<'_, R> {
type Error = Error;
fn size_hint(&self) -> Option<usize> {
@@ -334,7 +317,7 @@ impl<'de, R: Read> SeqAccess<'de> for ListAccess<'_, R> {
}
self.remaining_values -= 1;
self.de.tag_to_deserialize_stack.push(self.list_type);
self.de.tag_to_deserialize_stack = Some(self.list_type);
self.de.in_list = true;
let result = seed.deserialize(&mut *self.de).map(Some);
self.de.in_list = false;

24
pumpkin-nbt/src/lib.rs
View File

@@ -1,6 +1,6 @@
use std::{
fmt::Display,
io::{self, Read, Write},
io::{self, Read, Seek, Write},
ops::Deref,
};
@@ -83,7 +83,7 @@ impl Nbt {
}
}
pub fn read<R: Read>(reader: &mut NbtReadHelper<R>) -> Result<Nbt, Error> {
pub fn read<R: Read + Seek>(reader: &mut NbtReadHelper<R>) -> Result<Nbt, Error> {
let tag_type_id = reader.get_u8_be()?;
if tag_type_id != COMPOUND_ID {
@@ -97,7 +97,7 @@ impl Nbt {
}
/// Reads an NBT tag that doesn't contain the name of the root `Compound`.
pub fn read_unnamed<R: Read>(reader: &mut NbtReadHelper<R>) -> Result<Nbt, Error> {
pub fn read_unnamed<R: Read + Seek>(reader: &mut NbtReadHelper<R>) -> Result<Nbt, Error> {
let tag_type_id = reader.get_u8_be()?;
if tag_type_id != COMPOUND_ID {
@@ -174,7 +174,7 @@ impl AsMut<NbtCompound> for Nbt {
}
}
pub fn get_nbt_string<R: Read>(bytes: &mut NbtReadHelper<R>) -> Result<String, Error> {
pub fn get_nbt_string<R: Read + Seek>(bytes: &mut NbtReadHelper<R>) -> Result<String, Error> {
let len = bytes.get_u16_be()? as usize;
let string_bytes = bytes.read_boxed_slice(len)?;
let string = cesu8::from_java_cesu8(&string_bytes).map_err(|_| Error::Cesu8DecodingError)?;
@@ -205,6 +205,8 @@ impl_array!(nbt_byte_array, NBT_BYTE_ARRAY_TAG);
#[cfg(test)]
mod test {
use std::io::Cursor;
use crate::Error;
use crate::deserializer::from_bytes;
use crate::nbt_byte_array;
@@ -238,7 +240,7 @@ mod test {
let mut bytes = Vec::new();
to_bytes_unnamed(&test, &mut bytes).unwrap();
let recreated_struct: Test = from_bytes_unnamed(&bytes[..]).unwrap();
let recreated_struct: Test = from_bytes_unnamed(Cursor::new(bytes)).unwrap();
assert_eq!(test, recreated_struct);
}
@@ -263,7 +265,7 @@ mod test {
let mut bytes = Vec::new();
to_bytes_unnamed(&test, &mut bytes).unwrap();
let recreated_struct: TestArray = from_bytes_unnamed(&bytes[..]).unwrap();
let recreated_struct: TestArray = from_bytes_unnamed(Cursor::new(bytes)).unwrap();
assert_eq!(test, recreated_struct);
}
@@ -282,7 +284,7 @@ mod test {
let mut bytes = Vec::new();
to_bytes_named(&test, name, &mut bytes).unwrap();
let recreated_struct: Test = from_bytes(&bytes[..]).unwrap();
let recreated_struct: Test = from_bytes(Cursor::new(bytes)).unwrap();
assert_eq!(test, recreated_struct);
}
@@ -298,7 +300,7 @@ mod test {
let mut bytes = Vec::new();
to_bytes_named(&test, name, &mut bytes).unwrap();
let recreated_struct: TestArray = from_bytes(&bytes[..]).unwrap();
let recreated_struct: TestArray = from_bytes(Cursor::new(bytes)).unwrap();
assert_eq!(test, recreated_struct);
}
@@ -358,7 +360,7 @@ mod test {
let mut bytes = Vec::new();
to_bytes_unnamed(&list_compound, &mut bytes).unwrap();
let recreated_struct: TestList = from_bytes_unnamed(&bytes[..]).unwrap();
let recreated_struct: TestList = from_bytes_unnamed(Cursor::new(bytes)).unwrap();
assert_eq!(list_compound, recreated_struct);
}
@@ -396,7 +398,7 @@ mod test {
let mut bytes = Vec::new();
to_bytes_named(&list_compound, "a".to_string(), &mut bytes).unwrap();
let recreated_struct: TestList = from_bytes(&bytes[..]).unwrap();
let recreated_struct: TestList = from_bytes(Cursor::new(bytes)).unwrap();
assert_eq!(list_compound, recreated_struct);
}
@@ -511,7 +513,7 @@ mod test {
let mut bytes = Vec::new();
to_bytes(&value, &mut bytes).unwrap();
let reconstructed = from_bytes(&bytes[..]).unwrap();
let reconstructed = from_bytes(Cursor::new(bytes)).unwrap();
assert_eq!(value, reconstructed);
}

16
pumpkin-nbt/src/nbt_compress.rs
View File

@@ -1,7 +1,7 @@
use crate::deserializer::NbtReadHelper;
use crate::{Error, Nbt, NbtCompound, deserializer, serializer};
use flate2::{Compression, read::GzDecoder, write::GzEncoder};
use std::io::{Read, Write};
use std::io::{Cursor, Read, Seek, Write};
/// Reads a GZipped NBT compound tag from any reader.
///
@@ -12,10 +12,12 @@ use std::io::{Read, Write};
/// # Returns
///
/// A Result containing either the parsed NbtCompound or an Error
pub fn read_gzip_compound_tag(input: impl Read) -> Result<NbtCompound, Error> {
pub fn read_gzip_compound_tag(input: impl Read + Seek) -> Result<NbtCompound, Error> {
// Create a GZip decoder and directly chain it to the NBT reader
let decoder = GzDecoder::new(input);
let mut reader = NbtReadHelper::new(decoder);
let mut decoder = GzDecoder::new(input);
let mut buf = Vec::new();
decoder.read_to_end(&mut buf).unwrap();
let mut reader = NbtReadHelper::new(Cursor::new(buf));
// Read the NBT data directly from the decoder stream
let nbt = Nbt::read(&mut reader)?;
@@ -67,8 +69,10 @@ pub fn write_gzip_compound_tag_to_bytes(compound: &NbtCompound) -> Result<Vec<u8
/// A Result containing either the deserialized type or an Error
pub fn from_gzip_bytes<'a, T: serde::Deserialize<'a>, R: Read>(input: R) -> Result<T, Error> {
// Create a GZip decoder and directly use it for deserialization
let decoder = GzDecoder::new(input);
deserializer::from_bytes(decoder)
let mut decoder = GzDecoder::new(input);
let mut buf = Vec::new();
decoder.read_to_end(&mut buf).unwrap();
deserializer::from_bytes(Cursor::new(buf))
}
/// Writes a Rust type as GZipped NBT to any writer.

17
pumpkin-nbt/src/tag.rs
View File

@@ -108,12 +108,15 @@ impl NbtTag {
Ok(())
}
pub fn deserialize<R: Read>(reader: &mut NbtReadHelper<R>) -> Result<NbtTag, Error> {
pub fn deserialize<R: Read + Seek>(reader: &mut NbtReadHelper<R>) -> Result<NbtTag, Error> {
let tag_id = reader.get_u8_be()?;
Self::deserialize_data(reader, tag_id)
}
pub fn skip_data<R: Read>(reader: &mut NbtReadHelper<R>, tag_id: u8) -> Result<(), Error> {
pub fn skip_data<R: Read + Seek>(
reader: &mut NbtReadHelper<R>,
tag_id: u8,
) -> Result<(), Error> {
match tag_id {
END_ID => Ok(()),
BYTE_ID => reader.skip_bytes(1),
@@ -127,11 +130,11 @@ impl NbtTag {
if len < 0 {
return Err(Error::NegativeLength(len));
}
reader.skip_bytes(len as u64)
reader.skip_bytes(len as i64)
}
STRING_ID => {
let len = reader.get_u16_be()?;
reader.skip_bytes(len as u64)
reader.skip_bytes(len as i64)
}
LIST_ID => {
let tag_type_id = reader.get_u8_be()?;
@@ -153,7 +156,7 @@ impl NbtTag {
return Err(Error::NegativeLength(len));
}
reader.skip_bytes(len as u64 * 4)
reader.skip_bytes(len as i64 * 4)
}
LONG_ARRAY_ID => {
let len = reader.get_i32_be()?;
@@ -161,13 +164,13 @@ impl NbtTag {
return Err(Error::NegativeLength(len));
}
reader.skip_bytes(len as u64 * 8)
reader.skip_bytes(len as i64 * 8)
}
_ => Err(Error::UnknownTagId(tag_id)),
}
}
pub fn deserialize_data<R: Read>(
pub fn deserialize_data<R: Read + Seek>(
reader: &mut NbtReadHelper<R>,
tag_id: u8,
) -> Result<NbtTag, Error> {

8
pumpkin-world/src/chunk/format/mod.rs
View File

@@ -1,4 +1,4 @@
use std::{collections::HashMap, path::PathBuf};
use std::{collections::HashMap, io::Cursor, path::PathBuf};
use async_trait::async_trait;
use bytes::Bytes;
@@ -80,7 +80,7 @@ impl ChunkData {
position: Vector2<i32>,
) -> Result<Self, ChunkParsingError> {
// TODO: Implement chunk stages?
if from_bytes::<ChunkStatusWrapper>(chunk_data)
if from_bytes::<ChunkStatusWrapper>(Cursor::new(chunk_data))
.map_err(ChunkParsingError::FailedReadStatus)?
.status
!= ChunkStatus::Full
@@ -88,7 +88,7 @@ impl ChunkData {
return Err(ChunkParsingError::ChunkNotGenerated);
}
let chunk_data = from_bytes::<ChunkNbt>(chunk_data)
let chunk_data = from_bytes::<ChunkNbt>(Cursor::new(chunk_data))
.map_err(|e| ChunkParsingError::ErrorDeserializingChunk(e.to_string()))?;
if chunk_data.light_correct {
@@ -298,7 +298,7 @@ impl ChunkEntityData {
chunk_data: &[u8],
position: Vector2<i32>,
) -> Result<Self, ChunkParsingError> {
let chunk_entity_data = pumpkin_nbt::from_bytes::<EntityNbt>(chunk_data)
let chunk_entity_data = pumpkin_nbt::from_bytes::<EntityNbt>(Cursor::new(chunk_data))
.map_err(|e| ChunkParsingError::ErrorDeserializingChunk(e.to_string()))?;
if chunk_entity_data.position[0] != position.x

240
pumpkin-world/src/chunk/palette.rs
View File

@@ -1,8 +1,4 @@
use std::{
cmp::Ordering,
collections::{HashMap, hash_map::Entry},
hash::Hash,
};
use std::{collections::HashMap, hash::Hash};
use pumpkin_data::{Block, BlockState, chunk::Biome};
use pumpkin_util::encompassing_bits;
@@ -17,7 +13,8 @@ type AbstractCube<T, const DIM: usize> = [[[T; DIM]; DIM]; DIM];
#[derive(Debug, Clone)]
pub struct HeterogeneousPaletteData<V: Hash + Eq + Copy, const DIM: usize> {
cube: Box<AbstractCube<V, DIM>>,
counts: HashMap<V, u16>,
palette: Vec<V>,
counts: Vec<u16>,
}
impl<V: Hash + Eq + Copy, const DIM: usize> HeterogeneousPaletteData<V, DIM> {
@@ -36,19 +33,26 @@ impl<V: Hash + Eq + Copy, const DIM: usize> HeterogeneousPaletteData<V, DIM> {
debug_assert!(z < DIM);
let original = self.cube[y][z][x];
if let Entry::Occupied(mut entry) = self.counts.entry(original) {
let count = entry.get_mut();
*count -= 1;
if *count == 0 {
let _ = entry.remove();
}
let original_index = self.palette.iter().position(|v| v == &original).unwrap();
self.counts[original_index] -= 1;
if self.counts[original_index] == 0 {
// Remove from palette and counts Vecs if the count hits zero.
self.palette.swap_remove(original_index);
self.counts.swap_remove(original_index);
}
// Set the new value in the cube
self.cube[y][z][x] = value;
self.counts
.entry(value)
.and_modify(|count| *count += 1)
.or_insert(1);
// Find or add the new value to the palette.
if let Some(new_index) = self.palette.iter().position(|v| v == &value) {
self.counts[new_index] += 1;
} else {
self.palette.push(value);
self.counts.push(1);
}
original
}
}
@@ -66,19 +70,31 @@ impl<V: Hash + Eq + Copy + Default, const DIM: usize> PalettedContainer<V, DIM>
pub const VOLUME: usize = DIM * DIM * DIM;
fn from_cube(cube: Box<AbstractCube<V, DIM>>) -> Self {
let counts =
cube.as_flattened()
.as_flattened()
.iter()
.fold(HashMap::new(), |mut acc, key| {
acc.entry(*key).and_modify(|count| *count += 1).or_insert(1);
acc
});
let mut palette: Vec<V> = Vec::new();
let mut counts: Vec<u16> = Vec::new();
if counts.len() == 1 {
Self::Homogeneous(*counts.keys().next().unwrap())
// Iterate over the flattened cube to populate the palette and counts
for val in cube.as_flattened().as_flattened().iter() {
if let Some(index) = palette.iter().position(|v| v == val) {
// Value already exists, increment its count
counts[index] += 1;
} else {
// New value, add it to the palette and start its count
palette.push(*val);
counts.push(1);
}
}
if palette.len() == 1 {
// Fast path: the cube is homogeneous, so we can store just one value
Self::Homogeneous(palette[0])
} else {
Self::Heterogeneous(Box::new(HeterogeneousPaletteData { cube, counts }))
// Heterogeneous cube, store the full data
Self::Heterogeneous(Box::new(HeterogeneousPaletteData {
cube,
palette,
counts,
}))
}
}
@@ -96,7 +112,8 @@ impl<V: Hash + Eq + Copy + Default, const DIM: usize> PalettedContainer<V, DIM>
debug_assert!(bits_per_entry >= encompassing_bits(data.counts.len()));
debug_assert!(bits_per_entry <= 15);
let palette: Box<[V]> = data.counts.keys().copied().collect();
let palette: Box<[V]> = data.palette.iter().copied().collect();
let key_to_index_map: HashMap<V, usize> = palette
.iter()
.enumerate()
@@ -128,62 +145,79 @@ impl<V: Hash + Eq + Copy + Default, const DIM: usize> PalettedContainer<V, DIM>
}
pub fn from_palette_and_packed_data(
palette: &[V],
palette_slice: &[V],
packed_data: &[i64],
minimum_bits_per_entry: u8,
) -> Self {
if palette.is_empty() {
if palette_slice.is_empty() {
log::warn!("No palette data! Defaulting...");
Self::Homogeneous(V::default())
} else if palette.len() == 1 {
Self::Homogeneous(palette[0])
} else {
let bits_per_key = encompassing_bits(palette.len()).max(minimum_bits_per_entry);
let index_mask = (1 << bits_per_key) - 1;
let keys_per_i64 = 64 / bits_per_key;
return Self::Homogeneous(V::default());
}
let expected_i64_count = Self::VOLUME.div_ceil(keys_per_i64 as usize);
if palette_slice.len() == 1 {
return Self::Homogeneous(palette_slice[0]);
}
match packed_data.len().cmp(&expected_i64_count) {
Ordering::Greater => {
// Handled by the zip
log::warn!("Filled the section but there is still more data! Ignoring...");
}
Ordering::Less => {
// Handled by the array initialization and zip
log::warn!(
"Ran out of packed indices, but did not fill the section ({} vs {} for {}). Defaulting...",
packed_data.len() * keys_per_i64 as usize,
Self::VOLUME,
palette.len(),
);
}
// This is what we want!
Ordering::Equal => {}
let bits_per_key = encompassing_bits(palette_slice.len()).max(minimum_bits_per_entry);
let index_mask = (1 << bits_per_key) - 1;
let keys_per_i64 = 64 / bits_per_key;
let mut decompressed_values = Vec::with_capacity(Self::VOLUME);
// We already have the palette from the input `palette_slice`.
// The counts will be created in the next step.
let mut packed_data_iter = packed_data.iter();
let mut current_packed_word = *packed_data_iter.next().unwrap_or(&0);
for i in 0..Self::VOLUME {
let bit_index_in_word = i % keys_per_i64 as usize;
if bit_index_in_word == 0 && i > 0 {
current_packed_word = *packed_data_iter.next().unwrap_or(&0);
}
// TODO: Can we do this all with an `array::from_fn` or something?
let mut cube = Box::new([[[V::default(); DIM]; DIM]; DIM]);
cube.as_flattened_mut()
.as_flattened_mut()
.chunks_mut(keys_per_i64 as usize)
.zip(packed_data)
.for_each(|(values, packed)| {
values.iter_mut().enumerate().for_each(|(index, value)| {
let lookup_index =
(*packed as u64 >> (index as u64 * bits_per_key as u64)) & index_mask;
let lookup_index = (current_packed_word as u64
>> (bit_index_in_word as u64 * bits_per_key as u64))
& index_mask;
if let Some(v) = palette.get(lookup_index as usize) {
*value = *v;
} else {
// The cube is already initialized to the default
log::warn!("Lookup index out of bounds! Defaulting...");
}
});
let value = palette_slice
.get(lookup_index as usize)
.copied()
.unwrap_or_else(|| {
log::warn!("Lookup index out of bounds! Defaulting...");
V::default()
});
Self::from_cube(cube)
decompressed_values.push(value);
}
// Now, with all decompressed values, build the counts.
let mut counts = vec![0; palette_slice.len()];
for &value in &decompressed_values {
// This is the key optimization: find the index in the palette Vec
// and increment the corresponding count.
if let Some(index) = palette_slice.iter().position(|v| v == &value) {
counts[index] += 1;
} else {
// This case should ideally not happen if the palette is complete.
log::warn!("Decompressed value not found in palette!");
}
}
let mut cube = Box::new([[[V::default(); DIM]; DIM]; DIM]);
cube.as_flattened_mut()
.as_flattened_mut()
.copy_from_slice(&decompressed_values);
let palette_vec: Vec<V> = palette_slice.to_vec();
Self::Heterogeneous(Box::new(HeterogeneousPaletteData {
cube,
palette: palette_vec,
counts,
}))
}
pub fn get(&self, x: usize, y: usize, z: usize) -> V {
@@ -211,7 +245,7 @@ impl<V: Hash + Eq + Copy + Default, const DIM: usize> PalettedContainer<V, DIM>
Self::Heterogeneous(data) => {
let original = data.set(x, y, z, value);
if data.counts.len() == 1 {
*self = Self::Homogeneous(*data.counts.keys().next().unwrap());
*self = Self::Homogeneous(data.palette[0]);
}
original
}
@@ -384,51 +418,48 @@ impl BlockPalette {
packed_data: Box::new([]),
},
Self::Heterogeneous(data) => {
let bits_per_entry = encompassing_bits(data.counts.len());
let palette: Box<[u16]> = data.counts.keys().copied().collect();
let key_to_index_map: HashMap<_, usize> = palette
let bits_per_entry = encompassing_bits(data.palette.len());
let key_to_index_map: HashMap<_, usize> = data
.palette
.iter()
.enumerate()
.map(|(index, key)| (*key, index))
.collect();
let blocks_per_word = 32 / bits_per_entry;
let expected_word_count = Self::VOLUME.div_ceil(blocks_per_word as usize);
let mut packed_data = Vec::with_capacity(expected_word_count);
// Direktes Verarbeiten in der Reihenfolge [x][y][z] ohne Kopie
let mut packed_data = Vec::new();
let mut current_word = 0;
let mut current_index = 0;
let mut current_word: u32 = 0;
let mut current_index_in_word = 0;
for x in 0..16 {
for y in 0..16 {
for z in 0..16 {
let key = data.cube[z][y][x]; // Zugriff in [x][y][z]-Reihenfolge
let key_index = key_to_index_map.get(&key).unwrap();
debug_assert!((1 << bits_per_entry) > *key_index);
for key in data.cube.as_flattened().as_flattened().iter() {
let key_index = key_to_index_map.get(key).unwrap();
debug_assert!((1 << bits_per_entry) > *key_index);
let packed_offset_index = (*key_index as u32)
<< (bits_per_entry as u32 * current_index as u32);
current_word |= packed_offset_index;
current_word |=
(*key_index as u32) << (bits_per_entry as u32 * current_index_in_word);
current_index_in_word += 1;
current_index += 1;
if current_index == blocks_per_word {
packed_data.push(current_word);
current_word = 0;
current_index = 0;
}
}
if current_index_in_word == blocks_per_word as u32 {
packed_data.push(current_word);
current_word = 0;
current_index_in_word = 0;
}
}
if current_index > 0 {
// Push any remaining bits if the volume isn't a multiple of blocks_per_word
if current_index_in_word > 0 {
packed_data.push(current_word);
}
BeNetworkSerialization {
bits_per_entry,
palette: NetworkPalette::Indirect(
palette
.into_iter()
.map(BlockState::to_be_network_id)
data.palette
.iter()
.map(|&id| BlockState::to_be_network_id(id))
.collect(),
),
packed_data: packed_data.into_boxed_slice(),
@@ -447,13 +478,14 @@ impl BlockPalette {
}
}
Self::Heterogeneous(data) => data
.counts
.palette
.iter()
.map(|(registry_id, count)| {
.zip(data.counts.iter())
.filter_map(|(registry_id, count)| {
if !BlockState::from_id(*registry_id).is_air() {
*count
Some(*count)
} else {
0
None
}
})
.sum(),

11
pumpkin-world/src/generation/biome.rs
View File

@@ -1,4 +1,4 @@
use pumpkin_util::math::{floor_mod, square, vector3::Vector3};
use pumpkin_util::math::{square, vector3::Vector3};
use super::biome_coords;
@@ -134,7 +134,14 @@ fn score_permutation(
#[inline]
fn scale_mix(l: i64) -> f64 {
let d = floor_mod(l >> 24, 1024i32 as i64) as i32 as f64 / 1024.0;
const RECIPROCAL_1024: f64 = 1.0 / 1024.0;
// Use a bitwise AND for a faster modulus by a power of two
let scaled_l = (l >> 24) & 1023;
// Use a multiplication instead of a division
let d = scaled_l as f64 * RECIPROCAL_1024;
(d - 0.5) * 0.9
}

23
pumpkin-world/src/generation/surface/mod.rs
View File

@@ -1,6 +1,3 @@
use std::{cell::RefCell, num::NonZeroUsize};
use lru::LruCache;
use pumpkin_data::chunk::Biome;
use pumpkin_util::{
math::{lerp2, vector2::Vector2, vector3::Vector3, vertical_surface_type::VerticalSurfaceType},
@@ -398,7 +395,7 @@ pub struct VerticalGradientMaterialCondition {
true_at_and_below: YOffset,
false_at_and_above: YOffset,
#[serde(skip)]
random_deriver: ThreadLocal<RefCell<LruCache<usize, RandomDeriver>>>,
random_deriver: ThreadLocal<RandomDeriver>,
}
impl VerticalGradientMaterialCondition {
@@ -406,21 +403,9 @@ impl VerticalGradientMaterialCondition {
let true_at = self.true_at_and_below.get_y(context.min_y, context.height);
let false_at = self.false_at_and_above.get_y(context.min_y, context.height);
let context_pointer: *const RandomDeriver = context.random_deriver;
let key = context_pointer.addr();
let mut cache = self
.random_deriver
.get_or(|| {
let cache_size = NonZeroUsize::new(128).unwrap();
let cache = LruCache::new(cache_size);
RefCell::new(cache)
})
.borrow_mut();
let splitter = cache.get_or_insert(key, || {
context
.random_deriver
let splitter = self.random_deriver.get_or(|| {
let random_deriver_clone = context.random_deriver;
random_deriver_clone
.split_string(&self.random_name)
.next_splitter()
});

22
pumpkin-world/src/world_info/anvil.rs
View File

@@ -1,6 +1,6 @@
use std::{
fs::OpenOptions,
io::Read,
io::{Cursor, Read},
path::Path,
time::{SystemTime, UNIX_EPOCH},
};
@@ -38,7 +38,7 @@ fn check_file_data_version(raw_nbt: &[u8]) -> Result<(), WorldInfoError> {
data: LevelData,
}
let info: LevelDat = pumpkin_nbt::from_bytes(raw_nbt)
let info: LevelDat = pumpkin_nbt::from_bytes(Cursor::new(raw_nbt))
.map_err(|e|{
log::error!("The level.dat file does not have a data version! This means it is either corrupt or very old (read unsupported)");
WorldInfoError::DeserializationError(e.to_string())})?;
@@ -65,7 +65,7 @@ fn check_file_level_version(raw_nbt: &[u8]) -> Result<(), WorldInfoError> {
data: LevelData,
}
let info: LevelDat = pumpkin_nbt::from_bytes(raw_nbt)
let info: LevelDat = pumpkin_nbt::from_bytes(Cursor::new(raw_nbt))
.map_err(|e|{
log::error!("The level.dat file does not have a level version! This means it is either corrupt or very old (read unsupported)");
WorldInfoError::DeserializationError(e.to_string())})?;
@@ -91,7 +91,7 @@ impl WorldInfoReader for AnvilLevelInfo {
check_file_data_version(&buf)?;
check_file_level_version(&buf)?;
let info = pumpkin_nbt::from_bytes::<LevelDat>(&buf[..])
let info = pumpkin_nbt::from_bytes::<LevelDat>(Cursor::new(buf))
.map_err(|e| WorldInfoError::DeserializationError(e.to_string()))?;
// TODO: check version
@@ -143,7 +143,11 @@ pub struct LevelDat {
#[cfg(test)]
mod test {
use std::{fs, sync::LazyLock};
use std::{
fs,
io::{Cursor, Read},
sync::LazyLock,
};
use flate2::read::GzDecoder;
use pumpkin_data::game_rules::GameRuleRegistry;
@@ -281,8 +285,10 @@ mod test {
let raw_compressed_nbt = fs::read("assets/level_1_21_4.dat").unwrap();
assert!(!raw_compressed_nbt.is_empty());
let decoder = GzDecoder::new(&raw_compressed_nbt[..]);
let level_dat: LevelDat = from_bytes(decoder).expect("Failed to decode from file");
let mut decoder = GzDecoder::new(&raw_compressed_nbt[..]);
let mut buf = Vec::new();
decoder.read_to_end(&mut buf).unwrap();
let level_dat: LevelDat = from_bytes(Cursor::new(buf)).expect("Failed to decode from file");
assert_eq!(level_dat, *LEVEL_DAT);
}
@@ -295,7 +301,7 @@ mod test {
assert!(!serialized.is_empty());
let level_dat_again: LevelDat =
from_bytes(&serialized[..]).expect("Failed to decode from bytes");
from_bytes(Cursor::new(serialized)).expect("Failed to decode from bytes");
assert_eq!(level_dat_again, *LEVEL_DAT);
}