From dcb42f19703521875b853d615a87980b28fd4a4d Mon Sep 17 00:00:00 2001 From: adman Date: Sun, 19 Apr 2026 11:04:37 +0200 Subject: [PATCH] chunk serialization --- data/biomes.go | 23 ++++++-- level/packet.go | 128 ++++++++++++++++++++++++++++++++++++++++++- level/packet_test.go | 62 +++++++++++++++++++++ util/must.go | 8 +++ util/slices.go | 51 ++++++++++++++++- 5 files changed, 262 insertions(+), 10 deletions(-) create mode 100644 level/packet_test.go create mode 100644 util/must.go diff --git a/data/biomes.go b/data/biomes.go index 2e5cc40..2cae67a 100644 --- a/data/biomes.go +++ b/data/biomes.go @@ -14,7 +14,7 @@ package data // }, type Biome struct { - Id int + Id int32 Name string Category string Temperature float64 @@ -24,22 +24,33 @@ type Biome struct { Color int } -var BiomesCache = make(map[string][]Biome) +var BiomesCache = make(map[string][]*Biome) -func BiomesForVersion(v string) (ret []Biome) { +func BiomesForVersion(v string) (ret []*Biome) { var ok bool if ret, ok = BiomesCache[v]; ok { return } - var b []Biome + var b []*Biome must(LoadVersionedJson(v, "biomes", &b)) BiomesCache[v] = b return b } -func LookupBiomeById(version string, id int32) (biome Biome, ok bool) { +func LookupBiomeById(version string, id int32) (biome *Biome, ok bool) { for _, b := range BiomesForVersion(version) { - if b.Id == int(id) { + if b.Id == id { + biome = b + ok = true + return + } + } + return +} + +func LookupBiomeByName(version string, name string) (biome *Biome, ok bool) { + for _, b := range BiomesForVersion(version) { + if b.Name == name { biome = b ok = true return diff --git a/level/packet.go b/level/packet.go index 3870358..9b0f96e 100644 --- a/level/packet.go +++ b/level/packet.go @@ -2,17 +2,21 @@ package level import ( "encoding/binary" + "errors" "io" "math" "github.com/admin-else/strom/proto_base" + "github.com/admin-else/strom/util" ) const ( - BlocksPerChunkSection = 4096 - BiomesPerChunkSection = 64 + BlocksPerChunkSection = 16 * 16 * 16 + BiomesPerChunkSection = 4 * 4 * 4 ) +var InvalidPalletIndexErr = errors.New("invalid palette index") + func UnpackArrayPalette(r io.Reader, bitsPerEntry uint8, numberOfEntries int) (data []int32, err error) { palletLen, err := proto_base.DecodeVarInt(r) if err != nil { @@ -28,13 +32,39 @@ func UnpackArrayPalette(r io.Reader, bitsPerEntry uint8, numberOfEntries int) (d pallet = append(pallet, entry) } data, err = UnpackLongData(r, bitsPerEntry, numberOfEntries) + for i, b := range data { - data[i] = pallet[b] // maybe check bounds? + if b < 0 || b >= palletLen { + err = InvalidPalletIndexErr + return + } + data[i] = pallet[b] } return } +func PackArrayPallet(unique map[int32]struct{}, data []int32, w io.Writer) (err error) { + pallet := util.SetToSliceOrdered(unique) // this may improve performance in combination with zlib i dont know tho + err = proto_base.EncodeVarInt(w, int32(len(pallet))) + if err != nil { + return + } + var palletMap = make(map[int32]int32) + for i, entry := range pallet { + palletMap[entry] = int32(i) + err = proto_base.EncodeVarInt(w, entry) + if err != nil { + return + } + } + var palletedData = make([]int32, len(data)) + for i, entry := range data { + palletedData[i] = palletMap[entry] + } + return PackLongData(palletedData, w) +} + func UnpackLongData(r io.Reader, bitsPerEntry uint8, numberOfEntries int) (data []int32, err error) { entriesPerLong := int(64 / bitsPerEntry) numberOfLongs := int(math.Ceil(float64(numberOfEntries) / float64(entriesPerLong))) @@ -52,6 +82,16 @@ func UnpackLongData(r io.Reader, bitsPerEntry uint8, numberOfEntries int) (data return } +func PackLongData(data []int32, w io.Writer) (err error) { + unique := util.GetUniqueSlice(data) + bpe := int32(math.Ceil(math.Log2(float64(len(unique))))) + dataL := DataToLongs(data, bpe) + for _, entry := range dataL { + err = binary.Write(w, binary.BigEndian, entry) + } + return +} + func UnpackSingleValuePalette(r io.Reader, numberOfEntries int) (data []int32, err error) { entry, err := proto_base.DecodeVarInt(r) if err != nil { @@ -75,6 +115,22 @@ func LongsToData(data []uint64, n, bpe int32) (ret []int32) { return } +func DataToLongs(data []int32, bpe int32) (ret []uint64) { + elementsPerLong := 64 / bpe + retLen := int32(math.Ceil(float64(len(data)) / float64(elementsPerLong))) + ret = make([]uint64, retLen) + for i := range retLen { + for j := range elementsPerLong { + dataI := i*elementsPerLong + j + if dataI >= int32(len(data)) { + break + } + ret[i] |= uint64(data[dataI]) << (uint32(j) * uint32(bpe)) + } + } + return +} + func GetEntryFromLongs(data []uint64, i, bpe int32) (ret int32) { mask := (uint64(1) << uint32(bpe)) - 1 elementsPerLong := 64 / bpe @@ -103,6 +159,21 @@ func UnpackBlockData(r io.Reader) (blocks []int32, err error) { return } +func PackBlockData(blocks []int32, w io.Writer) (err error) { + unique := util.GetUniqueSlice(blocks) + bpe := int32(math.Ceil(math.Log2(float64(len(unique))))) + + switch bpe { + case 0: + err = proto_base.EncodeVarInt(w, blocks[0]) + case 1, 2, 3: + err = PackArrayPallet(unique, blocks, w) + default: + err = PackLongData(blocks, w) + } + return +} + func UnpackBiomeData(r io.Reader) (biomes []int32, err error) { // net.minecraft.world.chunk.PaletteProvider#forBiomes var bitsPerEntry uint8 @@ -124,6 +195,21 @@ func UnpackBiomeData(r io.Reader) (biomes []int32, err error) { return } +func PackBiomeData(biomes []int32, w io.Writer) (err error) { + unique := util.GetUniqueSlice(biomes) + bpe := int32(math.Ceil(math.Log2(float64(len(unique))))) + + switch bpe { + case 0: + err = proto_base.EncodeVarInt(w, biomes[0]) + case 1, 2, 3: + err = PackArrayPallet(unique, biomes, w) + default: + err = PackLongData(biomes, w) + } + return +} + func UnpackSection(r io.Reader) (blocks, biomes []int32, err error) { var blockCount int16 err = binary.Read(r, binary.BigEndian, &blockCount) @@ -150,3 +236,39 @@ func UnpackNSections(r io.Reader, n int) (blocks, biomes [][]int32, err error) { } return } + +var LenMustMatchErr = errors.New("length must match") +var BiomeMust64LongErr = errors.New("biome section must be 64 long") +var BlockMust4096LongErr = errors.New("block section must be 4096 long") + +func PackSection(blocks, biomes []int32, w io.Writer) (err error) { + if len(blocks) != BlocksPerChunkSection { + return BlockMust4096LongErr + } + if len(biomes) != BiomesPerChunkSection { + return BiomeMust64LongErr + } + err = proto_base.EncodeVarInt(w, BlocksPerChunkSection-int32(util.CountSlice(blocks, 0))) + if err != nil { + return + } + err = PackBlockData(blocks, w) + if err != nil { + return + } + err = PackBiomeData(biomes, w) + return +} + +func PackSections(blocks, biomes [][]int32, w io.Writer) (err error) { + if len(blocks) != len(biomes) { + return LenMustMatchErr + } + for i, blocksSection := range blocks { + err = PackSection(blocksSection, biomes[i], w) + if err != nil { + return + } + } + return +} diff --git a/level/packet_test.go b/level/packet_test.go new file mode 100644 index 0000000..317a964 --- /dev/null +++ b/level/packet_test.go @@ -0,0 +1,62 @@ +package level_test + +import ( + "bytes" + "slices" + "testing" + + "github.com/admin-else/strom/data" + "github.com/admin-else/strom/level" + "github.com/admin-else/strom/util" +) + +var RedMushroomBlock = util.MustOk(data.LookupBlockByName("1.21.11", "red_mushroom_block")) +var TheVoid = util.MustOk(data.LookupBiomeByName("1.21.11", "the_void")) + +func TestPackSections(t *testing.T) { + blocks := util.MakeSingleValuedSlice(RedMushroomBlock.DefaultState, level.BlocksPerChunkSection) + biomes := util.MakeSingleValuedSlice(TheVoid.Id, level.BiomesPerChunkSection) + + b := bytes.NewBuffer(nil) + err := level.PackSection(blocks, biomes, b) + if err != nil { + t.Fatal(err) + return + } +} + +func TestDataLongsTest(t *testing.T) { + var bpes = []int32{1, 2, 3, 4, 5, 10, 31} + var testValues = [][]int32{{1}, {1, 2}, {1, 2, 3}, {1, 2, 3, 4}, {1, 2, 3, 4, 5}, {999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999}} + + for _, bpe := range bpes { + maxValue := int32((1 << bpe) - 1) + t.Logf("BPE: %v, Maxvalue: %v", bpe, maxValue) + for _, values := range testValues { + t.Logf("Data: %v", values) + + skip := false + for _, v := range values { + if v >= maxValue { + skip = true + break + } + } + if skip { + t.Log("Skipping because of overflow") + continue + } + + longs := level.DataToLongs(values, bpe) + + t.Logf("Longs: %v", longs) + + dataNew := level.LongsToData(longs, int32(len(values)), bpe) + + if !slices.Equal(values, dataNew) { + t.Error("Data is not equal") + } + t.Log("Data conversion successful") + } + } +} diff --git a/util/must.go b/util/must.go new file mode 100644 index 0000000..12e2053 --- /dev/null +++ b/util/must.go @@ -0,0 +1,8 @@ +package util + +func MustOk[T any](v T, ok bool) T { + if !ok { + panic("not ok") + } + return v +} diff --git a/util/slices.go b/util/slices.go index ea45328..582ce4e 100644 --- a/util/slices.go +++ b/util/slices.go @@ -1,8 +1,57 @@ package util -import "math/rand/v2" +import ( + "cmp" + "math/rand/v2" + "slices" +) func ShuffleSlice[T any](s []T) (ret []T) { rand.Shuffle(len(s), func(i, j int) { s[i], s[j] = s[j], s[i] }) return s } + +func CountSlice[T comparable](s []T, o T) (ret int) { + for _, v := range s { + if v == o { + ret++ + } + } + return +} + +func GetUniqueSlice[T comparable](s []T) (unique map[T]struct{}) { + unique = make(map[T]struct{}) + for _, v := range s { + if _, ok := unique[v]; !ok { + unique[v] = struct{}{} + } + } + return +} + +func CountUniqueSlice[T comparable](s []T) (ret int) { + return len(GetUniqueSlice(s)) +} + +func MakeSingleValuedSlice[T any](v T, n int) (ret []T) { + ret = make([]T, n) + for i := range ret { + ret[i] = v + } + return +} + +func SetToSlice[T comparable](set map[T]struct{}) (slice []T) { + slice = make([]T, 0, len(set)) + for k := range set { + slice = append(slice, k) + } + return +} + +func SetToSliceOrdered[T cmp.Ordered](set map[T]struct{}) (slice []T) { + slice = SetToSlice(set) + slices.Sort(slice) + return +}