anygatecloud/strom
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chunk serialization fixes and fuzzies

Browse Source
This commit is contained in:
adman
2026-04-19 16:10:09 +02:00
parent dcb42f1970
commit c2777f2ca9
7 changed files with 161 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
level/packet.go
+88 -33
View File
@@ -6,6 +6,7 @@ import (
"io"
"math"
"github.com/admin-else/strom/data"
"github.com/admin-else/strom/proto_base"
"github.com/admin-else/strom/util"
)
@@ -44,7 +45,7 @@ func UnpackArrayPalette(r io.Reader, bitsPerEntry uint8, numberOfEntries int) (d
return
}
func PackArrayPallet(unique map[int32]struct{}, data []int32, w io.Writer) (err error) {
func PackArrayPallet(unique map[int32]struct{}, bpe uint8, data []int32, w io.Writer) (err error) {
pallet := util.SetToSliceOrdered(unique) // this may improve performance in combination with zlib i dont know tho
err = proto_base.EncodeVarInt(w, int32(len(pallet)))
if err != nil {
@@ -62,10 +63,17 @@ func PackArrayPallet(unique map[int32]struct{}, data []int32, w io.Writer) (err
for i, entry := range data {
palletedData[i] = palletMap[entry]
}
return PackLongData(palletedData, w)
return PackLongData(palletedData, bpe, w)
}
var BadBitsPerEntryErr = errors.New("bad bits per entry")
func UnpackLongData(r io.Reader, bitsPerEntry uint8, numberOfEntries int) (data []int32, err error) {
if bitsPerEntry < 1 || bitsPerEntry > 31 {
err = BadBitsPerEntryErr
return
}
entriesPerLong := int(64 / bitsPerEntry)
numberOfLongs := int(math.Ceil(float64(numberOfEntries) / float64(entriesPerLong)))
var dataL []uint64
@@ -78,14 +86,15 @@ func UnpackLongData(r io.Reader, bitsPerEntry uint8, numberOfEntries int) (data
dataL = append(dataL, entry)
}
data = LongsToData(dataL, int32(numberOfEntries), int32(bitsPerEntry))
data = LongsToData(dataL, int32(numberOfEntries), bitsPerEntry)
return
}
func PackLongData(data []int32, w io.Writer) (err error) {
unique := util.GetUniqueSlice(data)
bpe := int32(math.Ceil(math.Log2(float64(len(unique)))))
dataL := DataToLongs(data, bpe)
func PackLongData(data []int32, bpe uint8, w io.Writer) (err error) {
dataL, err := DataToLongs(data, bpe)
if err != nil {
return
}
for _, entry := range dataL {
err = binary.Write(w, binary.BigEndian, entry)
}
@@ -104,19 +113,23 @@ func UnpackSingleValuePalette(r io.Reader, numberOfEntries int) (data []int32, e
return
}
func LongsToData(data []uint64, n, bpe int32) (ret []int32) {
func LongsToData(data []uint64, n int32, bpe uint8) (ret []int32) {
mask := (uint64(1) << uint32(bpe)) - 1
elementsPerLong := 64 / bpe
elementsPerLong := int32(64 / bpe)
ret = make([]int32, n)
for i := range n {
ret[i] = int32((data[i/elementsPerLong] >> ((i % elementsPerLong) * bpe)) & mask)
ret[i] = int32((data[i/elementsPerLong] >> ((i % elementsPerLong) * int32(bpe))) & mask)
}
return
}
func DataToLongs(data []int32, bpe int32) (ret []uint64) {
elementsPerLong := 64 / bpe
var BpeTooSmallErr = errors.New("bits per entry too small")
func DataToLongs(data []int32, bpe uint8) (ret []uint64, err error) {
maxValue := uint64((1 << bpe) - 1)
elementsPerLong := 64 / int32(bpe)
retLen := int32(math.Ceil(float64(len(data)) / float64(elementsPerLong)))
ret = make([]uint64, retLen)
for i := range retLen {
@@ -125,7 +138,12 @@ func DataToLongs(data []int32, bpe int32) (ret []uint64) {
if dataI >= int32(len(data)) {
break
}
ret[i] |= uint64(data[dataI]) << (uint32(j) * uint32(bpe))
toInsert := uint64(data[dataI])
if toInsert > maxValue {
err = BpeTooSmallErr
return
}
ret[i] |= toInsert << (uint32(j) * uint32(bpe))
}
}
return
@@ -159,22 +177,38 @@ func UnpackBlockData(r io.Reader) (blocks []int32, err error) {
return
}
func PackBlockData(blocks []int32, w io.Writer) (err error) {
func PackBlockData(blocks []int32, version string, w io.Writer) (err error) {
unique := util.GetUniqueSlice(blocks)
bpe := int32(math.Ceil(math.Log2(float64(len(unique)))))
bpe := uint8(math.Ceil(math.Log2(float64(len(unique)))))
if bpe > 8 {
bpe = uint8(math.Ceil(math.Log2(float64(len(data.BlocksForVersion(version))))))
}
err = binary.Write(w, binary.BigEndian, bpe)
if err != nil {
return
}
switch bpe {
case 0:
err = proto_base.EncodeVarInt(w, blocks[0])
case 1, 2, 3:
err = PackArrayPallet(unique, blocks, w)
case 1, 2, 3, 4, 5, 6, 7, 8:
err = PackArrayPallet(unique, bpe, blocks, w)
default:
err = PackLongData(blocks, w)
// matches net.minecraft.world.chunk.PaletteProvider
err = PackLongData(blocks, bpe, w)
}
// This should also maybe get a maxblockid check but my fuzzer does not trigger a fail here so its probably fine
// also that would eat alot of performace
return
}
func UnpackBiomeData(r io.Reader) (biomes []int32, err error) {
var BadBiomeIdErr = errors.New("bad biome id")
func UnpackBiomeData(r io.Reader, version string) (biomes []int32, err error) {
// net.minecraft.world.chunk.PaletteProvider#forBiomes
var bitsPerEntry uint8
err = binary.Read(r, binary.BigEndian, &bitsPerEntry)
@@ -189,28 +223,47 @@ func UnpackBiomeData(r io.Reader) (biomes []int32, err error) {
default:
biomes, err = UnpackLongData(r, bitsPerEntry, BiomesPerChunkSection)
}
maxBiome := int32(len(data.BiomesForVersion(version)))
for _, b := range biomes {
if b < 0 || b >= maxBiome {
err = BadBiomeIdErr
return
}
}
if err != nil {
return
}
return
}
func PackBiomeData(biomes []int32, w io.Writer) (err error) {
func PackBiomeData(biomes []int32, version string, w io.Writer) (err error) {
unique := util.GetUniqueSlice(biomes)
bpe := int32(math.Ceil(math.Log2(float64(len(unique)))))
bpe := uint8(math.Ceil(math.Log2(float64(len(unique)))))
if bpe > 3 {
bpe = uint8(math.Ceil(math.Log2(float64(len(data.BiomesForVersion(version))))))
}
err = binary.Write(w, binary.BigEndian, bpe)
if err != nil {
return
}
switch bpe {
case 0:
err = proto_base.EncodeVarInt(w, biomes[0])
case 1, 2, 3:
err = PackArrayPallet(unique, biomes, w)
err = PackArrayPallet(unique, bpe, biomes, w)
default:
err = PackLongData(biomes, w)
// matches net.minecraft.world.chunk.PaletteProvider
err = PackLongData(biomes, bpe, w)
}
return
}
func UnpackSection(r io.Reader) (blocks, biomes []int32, err error) {
func UnpackSection(r io.Reader, version string) (blocks, biomes []int32, err error) {
var blockCount int16
err = binary.Read(r, binary.BigEndian, &blockCount)
if err != nil {
@@ -220,14 +273,14 @@ func UnpackSection(r io.Reader) (blocks, biomes []int32, err error) {
if err != nil {
return
}
biomes, err = UnpackBiomeData(r)
biomes, err = UnpackBiomeData(r, version)
return
}
func UnpackNSections(r io.Reader, n int) (blocks, biomes [][]int32, err error) {
func UnpackNSections(r io.Reader, n int) (blocks, biomes [][]int32, version string, err error) {
for range n {
var blocksSection, biomesSections []int32
blocksSection, biomesSections, err = UnpackSection(r)
blocksSection, biomesSections, err = UnpackSection(r, version)
if err != nil {
return
}
@@ -241,31 +294,33 @@ var LenMustMatchErr = errors.New("length must match")
var BiomeMust64LongErr = errors.New("biome section must be 64 long")
var BlockMust4096LongErr = errors.New("block section must be 4096 long")
func PackSection(blocks, biomes []int32, w io.Writer) (err error) {
func PackSection(blocks, biomes []int32, version string, w io.Writer) (err error) {
if len(blocks) != BlocksPerChunkSection {
return BlockMust4096LongErr
}
if len(biomes) != BiomesPerChunkSection {
return BiomeMust64LongErr
}
err = proto_base.EncodeVarInt(w, BlocksPerChunkSection-int32(util.CountSlice(blocks, 0)))
numBlocks := BlocksPerChunkSection - int32(util.CountSlice(blocks, 0))
err = binary.Write(w, binary.BigEndian, int16(numBlocks))
if err != nil {
return
}
err = PackBlockData(blocks, w)
err = PackBlockData(blocks, version, w)
if err != nil {
return
}
err = PackBiomeData(biomes, w)
err = PackBiomeData(biomes, version, w)
return
}
func PackSections(blocks, biomes [][]int32, w io.Writer) (err error) {
func PackSections(blocks, biomes [][]int32, version string, w io.Writer) (err error) {
if len(blocks) != len(biomes) {
return LenMustMatchErr
}
for i, blocksSection := range blocks {
err = PackSection(blocksSection, biomes[i], w)
err = PackSection(blocksSection, biomes[i], version, w)
if err != nil {
return
}
level/packet_test.go
+63 -20
View File
@@ -2,6 +2,7 @@ package level_test
import (
"bytes"
"math"
"slices"
"testing"
@@ -10,53 +11,95 @@ import (
"github.com/admin-else/strom/util"
)
var RedMushroomBlock = util.MustOk(data.LookupBlockByName("1.21.11", "red_mushroom_block"))
var TheVoid = util.MustOk(data.LookupBiomeByName("1.21.11", "the_void"))
var TestVersion = "1.21.11"
var RedMushroomBlock = util.MustOk(data.LookupBlockByName(TestVersion, "red_mushroom_block"))
var TheVoid = util.MustOk(data.LookupBiomeByName(TestVersion, "the_void"))
func TestPackSections(t *testing.T) {
blocks := util.MakeSingleValuedSlice(RedMushroomBlock.DefaultState, level.BlocksPerChunkSection)
biomes := util.MakeSingleValuedSlice(TheVoid.Id, level.BiomesPerChunkSection)
b := bytes.NewBuffer(nil)
err := level.PackSection(blocks, biomes, b)
err := level.PackSection(blocks, biomes, TestVersion, b)
if err != nil {
t.Fatal(err)
return
}
}
func FuzzPackSection(f *testing.F) {
corpusBlocks := util.MakeSingleValuedSlice(int32(1), level.BlocksPerChunkSection)
corpusBiomes := util.MakeSingleValuedSlice(int32(1), level.BiomesPerChunkSection)
b := bytes.NewBuffer(nil)
_ = level.PackSection(corpusBlocks, corpusBiomes, TestVersion, b)
f.Add(b.Bytes())
f.Fuzz(func(t *testing.T, raw []byte) {
r := bytes.NewReader(raw)
t.Logf("Raw: %v", raw)
blocks, biomes, err := level.UnpackSection(r, TestVersion)
if err != nil {
t.Skipf("unpacking failed: %v", err)
return
}
t.Logf("Blocks: %#v, Biomes: %#v", blocks, biomes)
buf := bytes.NewBuffer(nil)
err = level.PackSection(blocks, biomes, TestVersion, buf)
if err != nil {
t.Fatal(err)
return
}
t.Logf("Packed: %v", buf.Bytes())
newBlocks, newBimoes, err := level.UnpackSection(buf, TestVersion)
if err != nil {
t.Fatal(err)
return
}
if !slices.Equal(blocks, newBlocks) {
t.Error("Blocks are not equal")
}
if !slices.Equal(biomes, newBimoes) {
t.Error("Biomes are not equal")
}
})
}
func TestDataLongsTest(t *testing.T) {
var bpes = []int32{1, 2, 3, 4, 5, 10, 31}
var bpes = []uint8{1, 2, 3, 4, 5, 10, 31}
var testValues = [][]int32{{1}, {1, 2}, {1, 2, 3}, {1, 2, 3, 4}, {1, 2, 3, 4, 5}, {999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999}}
for _, bpe := range bpes {
maxValue := int32((1 << bpe) - 1)
t.Logf("BPE: %v, Maxvalue: %v", bpe, maxValue)
t.Logf("BPE: %v", bpe)
for _, values := range testValues {
t.Logf("Data: %v", values)
skip := false
for _, v := range values {
if v >= maxValue {
skip = true
break
}
}
if skip {
t.Log("Skipping because of overflow")
longs, err := level.DataToLongs(values, bpe)
if err != nil {
t.Logf("Error: %v", err)
continue
}
longs := level.DataToLongs(values, bpe)
t.Logf("Longs: %v", longs)
dataNew := level.LongsToData(longs, int32(len(values)), bpe)
t.Logf("Data new: %v", dataNew)
if !slices.Equal(values, dataNew) {
t.Error("Data is not equal")
}
} else {
t.Log("Data conversion successful")
}
}
}
}
func TestWeirdBiomesData(t *testing.T) {
biomes := []int32{48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 67, 66, 65, 57, 56, 55, 50, 49}
var b = bytes.NewBuffer(nil)
unique := util.GetUniqueSlice(biomes)
bpe := uint8(math.Ceil(math.Log2(float64(len(unique)))))
err := level.PackLongData(biomes, bpe, b)
_ = err
}
level/testdata/fuzz/FuzzPackSection/1d71329b7e8473da
Vendored
+2
View File
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("00\x000\x000")
level/testdata/fuzz/FuzzPackSection/720a2176d4a6d755
Vendored
+2
View File
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("00A")
level/testdata/fuzz/FuzzPackSection/746414e054d7661b
Vendored
+2
View File
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("00\x000\b0000000000000000000000000000000000000000000000000000000012789ABC")
level/testdata/fuzz/FuzzPackSection/7fa8e415d92ee53a
Vendored
+2
View File
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("00\x000\b000000000\x9d0000000000000000000000000000000000000000000000012789AB")
level/testdata/fuzz/FuzzPackSection/c6a489835f1c78cc
Vendored
+2
View File
@@ -0,0 +1,2 @@
go test fuzz v1
[]byte("00\x00\x00\x000")