majjo/Pumpkin
1
0
Fork 0
mirror of https://github.com/Pumpkin-MC/Pumpkin.git synced 2026-08-30 20:14:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Anvil data Abstraction

Browse Source 
this will help adding entity saving
This commit is contained in:
Alexander Medvedev
2025-06-23 22:46:25 +02:00
parent 0df97ad3aa
commit 8755908761
6 changed files with 257 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

193
pumpkin-world/src/chunk/format/anvil.rs
View File

@@ -1,16 +1,14 @@
use async_trait::async_trait;
use bytes::*;
use flate2::read::{GzDecoder, GzEncoder, ZlibDecoder, ZlibEncoder};
use futures::future::join_all;
use itertools::Itertools;
use lz4_java_wrc::Context;
use pumpkin_config::advanced_config;
use pumpkin_data::{Block, chunk::ChunkStatus};
use pumpkin_nbt::{compound::NbtCompound, serializer::to_bytes};
use pumpkin_util::math::vector2::Vector2;
use std::{
collections::HashSet,
io::{Read, SeekFrom, Write},
marker::PhantomData,
path::{Path, PathBuf},
time::{SystemTime, UNIX_EPOCH},
};
@@ -19,17 +17,12 @@ use tokio::{
sync::Mutex,
};
use crate::{
chunk::{
ChunkData, ChunkParsingError, ChunkReadingError, ChunkSerializingError, ChunkWritingError,
CompressionError,
io::{ChunkSerializer, LoadedData},
},
generation::section_coords,
use crate::chunk::{
ChunkParsingError, ChunkReadingError, ChunkSerializingError, ChunkWritingError,
CompressionError,
io::{ChunkSerializer, Dirtiable, LoadedData},
};
use super::{ChunkNbt, ChunkSectionNBT, LightContainer, SerializedScheduledTick};
/// The side size of a region in chunks (one region is 32x32 chunks)
pub const REGION_SIZE: usize = 32;
@@ -45,7 +38,7 @@ pub const CHUNK_COUNT: usize = REGION_SIZE * REGION_SIZE;
const SECTOR_BYTES: usize = 4096;
// 1.21.6
const WORLD_DATA_VERSION: i32 = 4435;
pub const WORLD_DATA_VERSION: i32 = 4435;
#[derive(Clone, Default)]
pub struct AnvilChunkFormat;
@@ -118,10 +111,12 @@ struct AnvilChunkMetadata {
file_sector_offset: u32,
}
pub struct AnvilChunkFile {
pub struct AnvilChunkFile<S: SingleChunkDataSerializer> {
chunks_data: [Option<AnvilChunkMetadata>; CHUNK_COUNT],
end_sector: u32,
write_action: Mutex<WriteAction>,
_dummy: PhantomData<S>,
}
impl Compression {
@@ -303,26 +298,30 @@ impl AnvilChunkData {
Ok(())
}
fn to_chunk(&self, pos: Vector2<i32>) -> Result<ChunkData, ChunkReadingError> {
let chunk = if let Some(compression) = self.compression {
fn to_chunk<S>(&self, pos: Vector2<i32>) -> Result<S, ChunkReadingError>
where
S: SingleChunkDataSerializer,
{
if let Some(compression) = self.compression {
let decompress_bytes = compression
.decompress_data(&self.compressed_data)
.map_err(ChunkReadingError::Compression)?;
ChunkData::from_bytes(&decompress_bytes, pos)
S::from_bytes(decompress_bytes.into(), pos)
} else {
ChunkData::from_bytes(&self.compressed_data, pos)
S::from_bytes(self.compressed_data.clone(), pos)
}
.map_err(ChunkReadingError::ParsingError)?;
Ok(chunk)
}
async fn from_chunk(
chunk: &ChunkData,
async fn from_chunk<S>(
chunk: &S,
compression: Option<Compression>,
) -> Result<Self, ChunkWritingError> {
let raw_bytes = chunk_to_bytes(chunk)
) -> Result<Self, ChunkWritingError>
where
S: SingleChunkDataSerializer,
{
let raw_bytes = chunk
.to_bytes()
.await
.map_err(|err| ChunkWritingError::ChunkSerializingError(err.to_string()))?;
@@ -341,7 +340,7 @@ impl AnvilChunkData {
}
}
impl AnvilChunkFile {
impl<S: SingleChunkDataSerializer> AnvilChunkFile<S> {
pub const fn get_region_coords(at: &Vector2<i32>) -> (i32, i32) {
// Divide by 32 for the region coordinates
(at.x >> SUBREGION_BITS, at.z >> SUBREGION_BITS)
@@ -506,20 +505,28 @@ impl AnvilChunkFile {
}
}
impl Default for AnvilChunkFile {
impl<S: SingleChunkDataSerializer> Default for AnvilChunkFile<S> {
fn default() -> Self {
Self {
chunks_data: [const { None }; CHUNK_COUNT],
write_action: Mutex::new(WriteAction::Pass),
// Two sectors for offset + timestamp
end_sector: 2,
_dummy: Default::default(),
}
}
}
#[async_trait]
impl ChunkSerializer for AnvilChunkFile {
type Data = ChunkData;
pub trait SingleChunkDataSerializer: Send + Sync + Sized + Dirtiable {
async fn to_bytes(&self) -> Result<Bytes, ChunkSerializingError>;
fn from_bytes(bytes: Bytes, pos: Vector2<i32>) -> Result<Self, ChunkReadingError>;
fn position(&self) -> &Vector2<i32>;
}
#[async_trait]
impl<S: SingleChunkDataSerializer> ChunkSerializer for AnvilChunkFile<S> {
type Data = S;
type WriteBackend = PathBuf;
fn should_write(&self, is_watched: bool) -> bool {
@@ -611,13 +618,13 @@ impl ChunkSerializer for AnvilChunkFile {
Ok(chunk_file)
}
async fn update_chunk(&mut self, chunk: &ChunkData) -> Result<(), ChunkWritingError> {
async fn update_chunk(&mut self, chunk: &Self::Data) -> Result<(), ChunkWritingError> {
let epoch = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs() as u32;
let index = AnvilChunkFile::get_chunk_index(&chunk.position);
let index = AnvilChunkFile::<S>::get_chunk_index(chunk.position());
// Default to the compression type read from the file
let compression_type = self.chunks_data[index]
.as_ref()
@@ -785,12 +792,12 @@ impl ChunkSerializer for AnvilChunkFile {
async fn get_chunks(
&self,
chunks: &[Vector2<i32>],
stream: tokio::sync::mpsc::Sender<LoadedData<ChunkData, ChunkReadingError>>,
stream: tokio::sync::mpsc::Sender<LoadedData<Self::Data, ChunkReadingError>>,
) {
// Don't par iter here so we can prevent backpressure with the await in the async
// runtime
for chunk in chunks.iter().cloned() {
let index = AnvilChunkFile::get_chunk_index(&chunk);
let index = AnvilChunkFile::<S>::get_chunk_index(&chunk);
let is_ok = match &self.chunks_data[index] {
None => stream.send(LoadedData::Missing(chunk)).await.is_ok(),
Some(chunk_metadata) => {
@@ -812,94 +819,6 @@ impl ChunkSerializer for AnvilChunkFile {
}
}
pub async fn chunk_to_bytes(chunk_data: &ChunkData) -> Result<Vec<u8>, ChunkSerializingError> {
let sections: Vec<_> = (0..chunk_data.section.sections.len() + 2)
.map(|i| {
let has_blocks = i >= 1 && i - 1 < chunk_data.section.sections.len();
let section = has_blocks.then(|| &chunk_data.section.sections[i - 1]);
ChunkSectionNBT {
y: (i as i8) - 1i8
+ section_coords::block_to_section(chunk_data.section.min_y) as i8,
block_states: section.map(|section| section.block_states.to_disk_nbt()),
biomes: section.map(|section| section.biomes.to_disk_nbt()),
block_light: match chunk_data.light_engine.block_light[i].clone() {
LightContainer::Empty(_) => None,
LightContainer::Full(data) => Some(data),
},
sky_light: match chunk_data.light_engine.sky_light[i].clone() {
LightContainer::Empty(_) => None,
LightContainer::Full(data) => Some(data),
},
}
})
.filter(|nbt| {
nbt.block_states.is_some()
|| nbt.biomes.is_some()
|| nbt.block_light.is_some()
|| nbt.sky_light.is_some()
})
.collect();
let nbt = ChunkNbt {
data_version: WORLD_DATA_VERSION,
x_pos: chunk_data.position.x,
z_pos: chunk_data.position.z,
min_y_section: section_coords::block_to_section(chunk_data.section.min_y),
status: ChunkStatus::Full,
heightmaps: chunk_data.heightmap.clone(),
sections,
block_ticks: {
chunk_data
.block_ticks
.iter()
.map(|tick| SerializedScheduledTick {
x: tick.block_pos.0.x,
y: tick.block_pos.0.y,
z: tick.block_pos.0.z,
delay: tick.delay as i32,
priority: tick.priority as i32,
target_block: format!(
"minecraft:{}",
Block::from_id(tick.target_block_id).unwrap().name
),
})
.collect()
},
fluid_ticks: {
chunk_data
.fluid_ticks
.iter()
.map(|tick| SerializedScheduledTick {
x: tick.block_pos.0.x,
y: tick.block_pos.0.y,
z: tick.block_pos.0.z,
delay: tick.delay as i32,
priority: tick.priority as i32,
target_block: format!(
"minecraft:{}",
Block::from_id(tick.target_block_id).unwrap().name
),
})
.collect()
},
block_entities: join_all(chunk_data.block_entities.values().map(
|block_entity| async move {
let mut nbt = NbtCompound::new();
block_entity.1.write_internal(&mut nbt).await;
nbt
},
))
.await,
// we have not implemented light engine
light_correct: false,
};
let mut result = Vec::new();
to_bytes(&nbt, &mut result).map_err(ChunkSerializingError::ErrorSerializingChunk)?;
Ok(result)
}
#[cfg(test)]
mod tests {
use async_trait::async_trait;
@@ -913,9 +832,10 @@ mod tests {
use temp_dir::TempDir;
use tokio::sync::RwLock;
use crate::chunk::format::anvil::AnvilChunkFile;
use crate::chunk::io::chunk_file_manager::ChunkFileManager;
use crate::chunk::io::{ChunkIO, LoadedData};
use crate::chunk::ChunkData;
use crate::chunk::format::anvil::{AnvilChunkFile, SingleChunkDataSerializer};
use crate::chunk::io::file_manager::{ChunkFileManager, PathFromLevelFolder};
use crate::chunk::io::{FileIO, LoadedData};
use crate::dimension::Dimension;
use crate::generation::{Seed, get_world_gen};
use crate::level::{Level, LevelFolder, SyncChunk};
@@ -936,11 +856,14 @@ mod tests {
}
}
async fn get_chunks(
saver: &ChunkFileManager<AnvilChunkFile>,
async fn get_chunks<S>(
saver: &ChunkFileManager<AnvilChunkFile<S>>,
folder: &LevelFolder,
chunks: &[(Vector2<i32>, SyncChunk)],
) -> Box<[SyncChunk]> {
) -> Box<[Arc<RwLock<S>>]>
where
S: SingleChunkDataSerializer + PathFromLevelFolder + 'static,
{
let mut read_chunks = Vec::new();
let (send, mut recv) = tokio::sync::mpsc::channel(1);
@@ -971,7 +894,7 @@ mod tests {
#[tokio::test(flavor = "multi_thread")]
async fn not_existing() {
let region_path = PathBuf::from("not_existing");
let chunk_saver = ChunkFileManager::<AnvilChunkFile>::default();
let chunk_saver = ChunkFileManager::<AnvilChunkFile<ChunkData>>::default();
let mut chunks = Vec::new();
let (send, mut recv) = tokio::sync::mpsc::channel(1);
@@ -1011,7 +934,7 @@ mod tests {
region_folder: temp_dir.path().join("region"),
};
fs::create_dir(&level_folder.region_folder).expect("couldn't create region folder");
let chunk_saver = ChunkFileManager::<AnvilChunkFile>::default();
let chunk_saver = ChunkFileManager::<AnvilChunkFile<ChunkData>>::default();
let block_registry = Arc::new(BlockRegistry);
// Generate chunks
@@ -1040,7 +963,7 @@ mod tests {
.expect("Failed to write chunk");
// Create a new manager to ensure nothing is cached
let chunk_saver = ChunkFileManager::<AnvilChunkFile>::default();
let chunk_saver = ChunkFileManager::<AnvilChunkFile<ChunkData>>::default();
let read_chunks = get_chunks(&chunk_saver, &level_folder, &chunks).await;
for (_, chunk) in &chunks {
@@ -1098,7 +1021,7 @@ mod tests {
.expect("Failed to write chunk");
// Create a new manager to ensure nothing is cached
let chunk_saver = ChunkFileManager::<AnvilChunkFile>::default();
let chunk_saver = ChunkFileManager::<AnvilChunkFile<ChunkData>>::default();
let read_chunks = get_chunks(&chunk_saver, &level_folder, &chunks).await;
for (_, chunk) in &chunks {
@@ -1171,7 +1094,7 @@ mod tests {
.expect("Failed to write chunk");
// Create a new manager to ensure nothing is cached
let chunk_saver = ChunkFileManager::<AnvilChunkFile>::default();
let chunk_saver = ChunkFileManager::<AnvilChunkFile<ChunkData>>::default();
let read_chunks = get_chunks(&chunk_saver, &level_folder, &chunks).await;
for (_, chunk) in &chunks {
@@ -1232,7 +1155,7 @@ mod tests {
.expect("Failed to write chunk");
// Create a new manager to ensure nothing is cached
let chunk_saver = ChunkFileManager::<AnvilChunkFile>::default();
let chunk_saver = ChunkFileManager::<AnvilChunkFile<ChunkData>>::default();
let read_chunks = get_chunks(&chunk_saver, &level_folder, &chunks).await;
for (_, chunk) in &chunks {
@@ -1288,7 +1211,7 @@ mod tests {
region_folder: temp_dir.path().join("region"),
};
fs::create_dir(&level_folder.region_folder).expect("couldn't create region folder");
let chunk_saver = ChunkFileManager::<AnvilChunkFile>::default();
let chunk_saver = ChunkFileManager::<AnvilChunkFile<ChunkData>>::default();
let block_registry = Arc::new(BlockRegistry);
// Generate chunks
@@ -1322,7 +1245,7 @@ mod tests {
.expect("Failed to write chunk");
// Create a new manager to ensure nothing is cached
let chunk_saver = ChunkFileManager::<AnvilChunkFile>::default();
let chunk_saver = ChunkFileManager::<AnvilChunkFile<ChunkData>>::default();
let read_chunks = get_chunks(&chunk_saver, &level_folder, &chunks).await;
for (_, chunk) in &chunks {

44
pumpkin-world/src/chunk/format/linear.rs
View File

@@ -1,8 +1,9 @@
use std::io::ErrorKind;
use std::marker::PhantomData;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
use crate::chunk::format::anvil::AnvilChunkFile;
use crate::chunk::format::anvil::{AnvilChunkFile, SingleChunkDataSerializer};
use crate::chunk::io::{ChunkSerializer, LoadedData};
use crate::chunk::{ChunkData, ChunkReadingError, ChunkWritingError};
use async_trait::async_trait;
@@ -12,7 +13,7 @@ use pumpkin_config::advanced_config;
use pumpkin_util::math::vector2::Vector2;
use tokio::io::{AsyncWriteExt, BufWriter};
use super::anvil::{CHUNK_COUNT, chunk_to_bytes};
use super::anvil::CHUNK_COUNT;
/// The signature of the linear file format
/// used as a header and footer described in https://gist.github.com/Aaron2550/5701519671253d4c6190bde6706f9f98
@@ -52,9 +53,11 @@ struct LinearFileHeader {
/// (16..24 Bytes) A hash of the region file (unused).
region_hash: u64,
}
pub struct LinearFile {
pub struct LinearFile<S: SingleChunkDataSerializer> {
chunks_headers: [LinearChunkHeader; CHUNK_COUNT],
chunks_data: [Option<Bytes>; CHUNK_COUNT],
_dummy: PhantomData<S>,
}
impl LinearChunkHeader {
@@ -134,9 +137,9 @@ impl LinearFileHeader {
}
}
impl LinearFile {
impl<S: SingleChunkDataSerializer> LinearFile<S> {
const fn get_chunk_index(at: &Vector2<i32>) -> usize {
AnvilChunkFile::get_chunk_index(at)
AnvilChunkFile::<S>::get_chunk_index(at)
}
fn check_signature(bytes: &[u8]) -> Result<(), ChunkReadingError> {
@@ -149,17 +152,18 @@ impl LinearFile {
}
}
impl Default for LinearFile {
impl<S: SingleChunkDataSerializer> Default for LinearFile<S> {
fn default() -> Self {
LinearFile {
chunks_headers: [LinearChunkHeader::default(); CHUNK_COUNT],
chunks_data: [const { None }; CHUNK_COUNT],
_dummy: Default::default(),
}
}
}
#[async_trait]
impl ChunkSerializer for LinearFile {
impl<S: SingleChunkDataSerializer> ChunkSerializer for LinearFile<S> {
type Data = ChunkData;
type WriteBackend = PathBuf;
@@ -168,7 +172,7 @@ impl ChunkSerializer for LinearFile {
}
fn get_chunk_key(chunk: &Vector2<i32>) -> String {
let (region_x, region_z) = AnvilChunkFile::get_region_coords(chunk);
let (region_x, region_z) = AnvilChunkFile::<S>::get_region_coords(chunk);
format!("./r.{region_x}.{region_z}.linear")
}
@@ -312,15 +316,16 @@ impl ChunkSerializer for LinearFile {
Ok(LinearFile {
chunks_headers: chunk_headers,
chunks_data: chunks,
_dummy: Default::default(),
})
}
async fn update_chunk(&mut self, chunk: &ChunkData) -> Result<(), ChunkWritingError> {
let index = LinearFile::get_chunk_index(&chunk.position);
let chunk_raw: Bytes = chunk_to_bytes(chunk)
let index = LinearFile::<S>::get_chunk_index(chunk.position());
let chunk_raw: Bytes = chunk
.to_bytes()
.await
.map_err(|err| ChunkWritingError::ChunkSerializingError(err.to_string()))?
.into();
.map_err(|err| ChunkWritingError::ChunkSerializingError(err.to_string()))?;
let header = &mut self.chunks_headers[index];
header.size = chunk_raw.len() as u32;
@@ -343,11 +348,13 @@ impl ChunkSerializer for LinearFile {
// Don't par iter here so we can prevent backpressure with the await in the async
// runtime
for chunk in chunks.iter().cloned() {
let index = LinearFile::get_chunk_index(&chunk);
let index = LinearFile::<S>::get_chunk_index(&chunk);
let linear_chunk_data = &self.chunks_data[index];
let result = if let Some(data) = linear_chunk_data {
match ChunkData::from_bytes(data, chunk).map_err(ChunkReadingError::ParsingError) {
match ChunkData::internal_from_bytes(data, chunk)
.map_err(ChunkReadingError::ParsingError)
{
Ok(chunk) => LoadedData::Loaded(chunk),
Err(err) => LoadedData::Error((chunk, err)),
}
@@ -376,9 +383,10 @@ mod tests {
use temp_dir::TempDir;
use tokio::sync::RwLock;
use crate::chunk::ChunkData;
use crate::chunk::format::linear::LinearFile;
use crate::chunk::io::chunk_file_manager::ChunkFileManager;
use crate::chunk::io::{ChunkIO, LoadedData};
use crate::chunk::io::file_manager::ChunkFileManager;
use crate::chunk::io::{FileIO, LoadedData};
use crate::dimension::Dimension;
use crate::generation::{Seed, get_world_gen};
use crate::level::{Level, LevelFolder};
@@ -402,7 +410,7 @@ mod tests {
#[tokio::test(flavor = "multi_thread")]
async fn not_existing() {
let region_path = PathBuf::from("not_existing");
let chunk_saver = ChunkFileManager::<LinearFile>::default();
let chunk_saver = ChunkFileManager::<LinearFile<ChunkData>>::default();
let mut chunks = Vec::new();
let (send, mut recv) = tokio::sync::mpsc::channel(1);
@@ -437,7 +445,7 @@ mod tests {
region_folder: temp_dir.path().join("region"),
};
fs::create_dir(&level_folder.region_folder).expect("couldn't create region folder");
let chunk_saver = ChunkFileManager::<LinearFile>::default();
let chunk_saver = ChunkFileManager::<LinearFile<ChunkData>>::default();
let block_registry = Arc::new(BlockRegistry);
// Generate chunks
let mut chunks = vec![];

139
pumpkin-world/src/chunk/format/mod.rs
View File

@@ -1,9 +1,21 @@
use std::collections::HashMap;
use std::{collections::HashMap, path::PathBuf};
use async_trait::async_trait;
use bytes::Bytes;
use futures::future::join_all;
use pumpkin_data::{Block, chunk::ChunkStatus};
use pumpkin_nbt::{compound::NbtCompound, from_bytes, nbt_long_array};
use crate::{block::entities::block_entity_from_nbt, generation::section_coords};
use crate::{
block::entities::block_entity_from_nbt,
chunk::{
ChunkReadingError, ChunkSerializingError,
format::anvil::{SingleChunkDataSerializer, WORLD_DATA_VERSION},
io::{Dirtiable, file_manager::PathFromLevelFolder},
},
generation::section_coords,
level::LevelFolder,
};
use pumpkin_util::math::{position::BlockPos, vector2::Vector2};
use serde::{Deserialize, Serialize};
@@ -25,8 +37,45 @@ pub struct ChunkStatusWrapper {
status: ChunkStatus,
}
#[async_trait]
impl SingleChunkDataSerializer for ChunkData {
#[inline]
fn from_bytes(bytes: Bytes, pos: Vector2<i32>) -> Result<Self, ChunkReadingError> {
Self::internal_from_bytes(&bytes, pos).map_err(ChunkReadingError::ParsingError)
}
#[inline]
async fn to_bytes(&self) -> Result<Bytes, ChunkSerializingError> {
self.internal_to_bytes().await
}
#[inline]
fn position(&self) -> &Vector2<i32> {
&self.position
}
}
impl PathFromLevelFolder for ChunkData {
#[inline]
fn file_path(folder: &LevelFolder, file_name: &str) -> PathBuf {
folder.region_folder.join(file_name)
}
}
impl Dirtiable for ChunkData {
#[inline]
fn mark_dirty(&mut self, flag: bool) {
self.dirty = flag;
}
#[inline]
fn is_dirty(&self) -> bool {
self.dirty
}
}
impl ChunkData {
pub fn from_bytes(
pub fn internal_from_bytes(
chunk_data: &[u8],
position: Vector2<i32>,
) -> Result<Self, ChunkParsingError> {
@@ -181,6 +230,90 @@ impl ChunkData {
light_engine,
})
}
async fn internal_to_bytes(&self) -> Result<Bytes, ChunkSerializingError> {
let sections: Vec<_> = (0..self.section.sections.len() + 2)
.map(|i| {
let has_blocks = i >= 1 && i - 1 < self.section.sections.len();
let section = has_blocks.then(|| &self.section.sections[i - 1]);
ChunkSectionNBT {
y: (i as i8) - 1i8 + section_coords::block_to_section(self.section.min_y) as i8,
block_states: section.map(|section| section.block_states.to_disk_nbt()),
biomes: section.map(|section| section.biomes.to_disk_nbt()),
block_light: match self.light_engine.block_light[i].clone() {
LightContainer::Empty(_) => None,
LightContainer::Full(data) => Some(data),
},
sky_light: match self.light_engine.sky_light[i].clone() {
LightContainer::Empty(_) => None,
LightContainer::Full(data) => Some(data),
},
}
})
.filter(|nbt| {
nbt.block_states.is_some()
|| nbt.biomes.is_some()
|| nbt.block_light.is_some()
|| nbt.sky_light.is_some()
})
.collect();
let nbt = ChunkNbt {
data_version: WORLD_DATA_VERSION,
x_pos: self.position.x,
z_pos: self.position.z,
min_y_section: section_coords::block_to_section(self.section.min_y),
status: ChunkStatus::Full,
heightmaps: self.heightmap.clone(),
sections,
block_ticks: {
self.block_ticks
.iter()
.map(|tick| SerializedScheduledTick {
x: tick.block_pos.0.x,
y: tick.block_pos.0.y,
z: tick.block_pos.0.z,
delay: tick.delay as i32,
priority: tick.priority as i32,
target_block: format!(
"minecraft:{}",
Block::from_id(tick.target_block_id).unwrap().name
),
})
.collect()
},
fluid_ticks: {
self.fluid_ticks
.iter()
.map(|tick| SerializedScheduledTick {
x: tick.block_pos.0.x,
y: tick.block_pos.0.y,
z: tick.block_pos.0.z,
delay: tick.delay as i32,
priority: tick.priority as i32,
target_block: format!(
"minecraft:{}",
Block::from_id(tick.target_block_id).unwrap().name
),
})
.collect()
},
block_entities: join_all(self.block_entities.values().map(|block_entity| async move {
let mut nbt = NbtCompound::new();
block_entity.1.write_internal(&mut nbt).await;
nbt
}))
.await,
// we have not implemented light engine
light_correct: false,
};
let mut result = Vec::new();
pumpkin_nbt::to_bytes(&nbt, &mut result)
.map_err(ChunkSerializingError::ErrorSerializingChunk)?;
Ok(result.into())
}
}
#[derive(Serialize, Deserialize, Debug)]

45
pumpkin-world/src/chunk/io/chunk_file_manager.rs → pumpkin-world/src/chunk/io/file_manager.rs
View File

@@ -18,11 +18,11 @@ use tokio::{
};
use crate::{
chunk::{ChunkData, ChunkReadingError, ChunkWritingError},
level::{LevelFolder, SyncChunk},
chunk::{ChunkReadingError, ChunkWritingError, io::Dirtiable},
level::LevelFolder,
};
use super::{ChunkIO, ChunkSerializer, LoadedData};
use super::{ChunkSerializer, FileIO, LoadedData};
/// A simple implementation of the ChunkSerializer trait
/// that load and save the data from a file in the disk
@@ -43,12 +43,16 @@ pub struct ChunkFileManager<S: ChunkSerializer<WriteBackend = PathBuf>> {
watchers: RwLock<BTreeMap<PathBuf, usize>>,
}
pub(crate) trait PathFromLevelFolder {
fn file_path(folder: &LevelFolder, file_name: &str) -> PathBuf;
}
struct ChunkSerializerLazyLoader<S: ChunkSerializer<WriteBackend = PathBuf>> {
path: PathBuf,
internal: OnceCell<Arc<RwLock<S>>>,
}
impl<S: ChunkSerializer<Data = ChunkData, WriteBackend = PathBuf>> ChunkSerializerLazyLoader<S> {
impl<S: ChunkSerializer<WriteBackend = PathBuf>> ChunkSerializerLazyLoader<S> {
fn new(path: PathBuf) -> Self {
Self {
path,
@@ -125,12 +129,6 @@ impl<S: ChunkSerializer<WriteBackend = PathBuf>> Default for ChunkFileManager<S>
}
impl<S: ChunkSerializer<WriteBackend = PathBuf>> ChunkFileManager<S> {
fn map_key(folder: &LevelFolder, file_name: &str) -> PathBuf {
folder.region_folder.join(file_name)
}
}
impl<S: ChunkSerializer<Data = ChunkData, WriteBackend = PathBuf>> ChunkFileManager<S> {
async fn get_serializer(&self, path: &Path) -> Result<Arc<RwLock<S>>, ChunkReadingError> {
// We get the entry from the DashMap and try to insert a new lock if it doesn't exist
// using dead-lock safe methods like `or_try_insert_with`
@@ -152,18 +150,19 @@ impl<S: ChunkSerializer<Data = ChunkData, WriteBackend = PathBuf>> ChunkFileMana
}
#[async_trait]
impl<S> ChunkIO for ChunkFileManager<S>
impl<P, S> FileIO for ChunkFileManager<S>
where
S: ChunkSerializer<Data = ChunkData, WriteBackend = PathBuf>,
P: PathFromLevelFolder + Send + Sync + Sized + Dirtiable + 'static,
S: ChunkSerializer<Data = P, WriteBackend = PathBuf>,
{
type Data = SyncChunk;
type Data = Arc<RwLock<S::Data>>;
async fn watch_chunks(&self, folder: &LevelFolder, chunks: &[Vector2<i32>]) {
// It is intentional that regions are watched multiple times (once per chunk)
let mut watchers = self.watchers.write().await;
for chunk in chunks {
let key = S::get_chunk_key(chunk);
let map_key = Self::map_key(folder, &key);
let map_key = P::file_path(folder, &key);
match watchers.entry(map_key) {
std::collections::btree_map::Entry::Vacant(vacant) => {
let _ = vacant.insert(1);
@@ -179,7 +178,7 @@ where
let mut watchers = self.watchers.write().await;
for chunk in chunks {
let key = S::get_chunk_key(chunk);
let map_key = Self::map_key(folder, &key);
let map_key = P::file_path(folder, &key);
match watchers.entry(map_key) {
std::collections::btree_map::Entry::Vacant(_vacant) => {}
std::collections::btree_map::Entry::Occupied(mut occupied) => {
@@ -200,7 +199,7 @@ where
&self,
folder: &LevelFolder,
chunk_coords: &[Vector2<i32>],
stream: tokio::sync::mpsc::Sender<LoadedData<SyncChunk, ChunkReadingError>>,
stream: tokio::sync::mpsc::Sender<LoadedData<Self::Data, ChunkReadingError>>,
) {
let mut regions_chunks: BTreeMap<String, Vec<Vector2<i32>>> = BTreeMap::new();
@@ -216,7 +215,7 @@ where
// we use a Sync Closure with an Async Block to execute the tasks concurrently
// Also improves File Cache utilizations.
let region_read_tasks = regions_chunks.into_iter().map(async |(file_name, chunks)| {
let path = Self::map_key(folder, &file_name);
let path = P::file_path(folder, &file_name);
let chunk_serializer = match self.get_serializer(&path).await {
Ok(chunk_serializer) => chunk_serializer,
Err(ChunkReadingError::ChunkNotExist) => {
@@ -229,7 +228,7 @@ where
};
// Intermediate channel for wrapping the data with the Arc<RwLock>
let (send, mut recv) = mpsc::channel::<LoadedData<ChunkData, ChunkReadingError>>(1);
let (send, mut recv) = mpsc::channel::<LoadedData<S::Data, ChunkReadingError>>(1);
let intermediary = async {
while let Some(data) = recv.recv().await {
@@ -254,9 +253,9 @@ where
async fn save_chunks(
&self,
folder: &LevelFolder,
chunks_data: Vec<(Vector2<i32>, SyncChunk)>,
chunks_data: Vec<(Vector2<i32>, Self::Data)>,
) -> Result<(), ChunkWritingError> {
let mut regions_chunks: BTreeMap<String, Vec<SyncChunk>> = BTreeMap::new();
let mut regions_chunks: BTreeMap<String, Vec<Self::Data>> = BTreeMap::new();
for (at, chunk) in chunks_data {
let key = S::get_chunk_key(&at);
@@ -276,7 +275,7 @@ where
let tasks = regions_chunks
.into_iter()
.map(async |(file_name, chunk_locks)| {
let path = Self::map_key(folder, &file_name);
let path = P::file_path(folder, &file_name);
log::trace!("Updating data for file {path:?}");
let chunk_serializer = match self.get_serializer(&path).await {
@@ -297,10 +296,10 @@ where
let mut serializer = chunk_serializer.write().await;
for chunk_lock in chunk_locks {
let mut chunk = chunk_lock.write().await;
let chunk_is_dirty = chunk.dirty || chunk.block_entities.values().any(|block_entity| block_entity.1.is_dirty());
let chunk_is_dirty = chunk.is_dirty();
// Edge case: this chunk is loaded while we were saving, mark it as cleaned since we are
// updating what we will write here
chunk.dirty = false;
chunk.mark_dirty(false);
// It is important that we keep the lock after we mark the chunk as clean so no one else
// can modify it
let chunk = chunk.downgrade();

11
pumpkin-world/src/chunk/io/mod.rs
View File

@@ -7,7 +7,7 @@ use pumpkin_util::math::vector2::Vector2;
use super::{ChunkReadingError, ChunkWritingError};
use crate::level::LevelFolder;
pub mod chunk_file_manager;
pub mod file_manager;
/// The result of loading a chunk data.
///
@@ -37,6 +37,11 @@ impl<D: Send, E: error::Error> LoadedData<D, E> {
}
}
pub trait Dirtiable {
fn is_dirty(&self) -> bool;
fn mark_dirty(&mut self, flag: bool);
}
/// Trait to handle the IO of chunks
/// for loading and saving chunks data
/// can be implemented for different types of IO
@@ -45,7 +50,7 @@ impl<D: Send, E: error::Error> LoadedData<D, E> {
/// The `R` type is the type of the data that will be loaded/saved
/// like ChunkData or EntityData
#[async_trait]
pub trait ChunkIO
pub trait FileIO
where
Self: Send + Sync,
{
@@ -87,7 +92,7 @@ where
/// like ChunkData or EntityData
#[async_trait]
pub trait ChunkSerializer: Send + Sync + Default {
type Data: Send + Sync + Sized;
type Data: Send + Sync + Sized + Dirtiable;
type WriteBackend;
/// Get the key for the chunk (like the file name)

13
pumpkin-world/src/level.rs
View File

@@ -28,7 +28,7 @@ use crate::{
chunk::{
ChunkData, ChunkParsingError, ChunkReadingError, ScheduledTick, TickPriority,
format::{anvil::AnvilChunkFile, linear::LinearFile},
io::{ChunkIO, LoadedData, chunk_file_manager::ChunkFileManager},
io::{FileIO, LoadedData, file_manager::ChunkFileManager},
},
dimension::Dimension,
generation::{Seed, get_world_gen, implementation::WorldGenerator},
@@ -59,7 +59,7 @@ pub struct Level {
loaded_chunks: Arc<DashMap<Vector2<i32>, SyncChunk>>,
chunk_watchers: Arc<DashMap<Vector2<i32>, usize>>,
chunk_saver: Arc<dyn ChunkIO<Data = SyncChunk>>,
chunk_saver: Arc<dyn FileIO<Data = SyncChunk>>,
world_gen: Arc<dyn WorldGenerator>,
block_ticks: Arc<Mutex<Vec<ScheduledTick>>>,
@@ -99,10 +99,11 @@ impl Level {
let seed = Seed(seed as u64);
let world_gen = get_world_gen(seed, dimension).into();
let chunk_saver: Arc<dyn ChunkIO<Data = SyncChunk>> = match advanced_config().chunk.format {
//ChunkFormat::Anvil => (Arc::new(AnvilChunkFormat), Arc::new(AnvilChunkFormat)),
ChunkFormat::Linear => Arc::new(ChunkFileManager::<LinearFile>::default()),
ChunkFormat::Anvil => Arc::new(ChunkFileManager::<AnvilChunkFile>::default()),
let chunk_saver: Arc<dyn FileIO<Data = SyncChunk>> = match advanced_config().chunk.format {
ChunkFormat::Linear => Arc::new(ChunkFileManager::<LinearFile<ChunkData>>::default()),
ChunkFormat::Anvil => {
Arc::new(ChunkFileManager::<AnvilChunkFile<ChunkData>>::default())
}
};
Self {