chunk serialization fixes and fuzzies

level/packet.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"math"
 
+	"github.com/admin-else/strom/data"
 	"github.com/admin-else/strom/proto_base"
 	"github.com/admin-else/strom/util"
 )
@@ -44,7 +45,7 @@ func UnpackArrayPalette(r io.Reader, bitsPerEntry uint8, numberOfEntries int) (d
 	return
 }
 
-func PackArrayPallet(unique map[int32]struct{}, data []int32, w io.Writer) (err error) {
+func PackArrayPallet(unique map[int32]struct{}, bpe uint8, data []int32, w io.Writer) (err error) {
 	pallet := util.SetToSliceOrdered(unique) // this may improve performance in combination with zlib i dont know tho
 	err = proto_base.EncodeVarInt(w, int32(len(pallet)))
 	if err != nil {
@@ -62,10 +63,17 @@ func PackArrayPallet(unique map[int32]struct{}, data []int32, w io.Writer) (err
 	for i, entry := range data {
 		palletedData[i] = palletMap[entry]
 	}
-	return PackLongData(palletedData, w)
+	return PackLongData(palletedData, bpe, w)
 }
 
+var BadBitsPerEntryErr = errors.New("bad bits per entry")
+
 func UnpackLongData(r io.Reader, bitsPerEntry uint8, numberOfEntries int) (data []int32, err error) {
+	if bitsPerEntry < 1 || bitsPerEntry > 31 {
+		err = BadBitsPerEntryErr
+		return
+	}
+
 	entriesPerLong := int(64 / bitsPerEntry)
 	numberOfLongs := int(math.Ceil(float64(numberOfEntries) / float64(entriesPerLong)))
 	var dataL []uint64
@@ -78,14 +86,15 @@ func UnpackLongData(r io.Reader, bitsPerEntry uint8, numberOfEntries int) (data
 		dataL = append(dataL, entry)
 	}
 
-	data = LongsToData(dataL, int32(numberOfEntries), int32(bitsPerEntry))
+	data = LongsToData(dataL, int32(numberOfEntries), bitsPerEntry)
 	return
 }
 
-func PackLongData(data []int32, w io.Writer) (err error) {
+func PackLongData(data []int32, bpe uint8, w io.Writer) (err error) {
-	unique := util.GetUniqueSlice(data)
+	dataL, err := DataToLongs(data, bpe)
-	bpe := int32(math.Ceil(math.Log2(float64(len(unique)))))
+	if err != nil {
-	dataL := DataToLongs(data, bpe)
+		return
+	}
 	for _, entry := range dataL {
 		err = binary.Write(w, binary.BigEndian, entry)
 	}
@@ -104,19 +113,23 @@ func UnpackSingleValuePalette(r io.Reader, numberOfEntries int) (data []int32, e
 	return
 }
 
-func LongsToData(data []uint64, n, bpe int32) (ret []int32) {
+func LongsToData(data []uint64, n int32, bpe uint8) (ret []int32) {
 	mask := (uint64(1) << uint32(bpe)) - 1
-	elementsPerLong := 64 / bpe
+	elementsPerLong := int32(64 / bpe)
 
 	ret = make([]int32, n)
 	for i := range n {
-		ret[i] = int32((data[i/elementsPerLong] >> ((i % elementsPerLong) * bpe)) & mask)
+		ret[i] = int32((data[i/elementsPerLong] >> ((i % elementsPerLong) * int32(bpe))) & mask)
 	}
 	return
 }
 
-func DataToLongs(data []int32, bpe int32) (ret []uint64) {
+var BpeTooSmallErr = errors.New("bits per entry too small")
-	elementsPerLong := 64 / bpe
+
+func DataToLongs(data []int32, bpe uint8) (ret []uint64, err error) {
+	maxValue := uint64((1 << bpe) - 1)
+
+	elementsPerLong := 64 / int32(bpe)
 	retLen := int32(math.Ceil(float64(len(data)) / float64(elementsPerLong)))
 	ret = make([]uint64, retLen)
 	for i := range retLen {
@@ -125,7 +138,12 @@ func DataToLongs(data []int32, bpe int32) (ret []uint64) {
 			if dataI >= int32(len(data)) {
 				break
 			}
-			ret[i] |= uint64(data[dataI]) << (uint32(j) * uint32(bpe))
+			toInsert := uint64(data[dataI])
+			if toInsert > maxValue {
+				err = BpeTooSmallErr
+				return
+			}
+			ret[i] |= toInsert << (uint32(j) * uint32(bpe))
 		}
 	}
 	return
@@ -159,22 +177,38 @@ func UnpackBlockData(r io.Reader) (blocks []int32, err error) {
 	return
 }
 
-func PackBlockData(blocks []int32, w io.Writer) (err error) {
+func PackBlockData(blocks []int32, version string, w io.Writer) (err error) {
 	unique := util.GetUniqueSlice(blocks)
-	bpe := int32(math.Ceil(math.Log2(float64(len(unique)))))
+	bpe := uint8(math.Ceil(math.Log2(float64(len(unique)))))
+
+	if bpe > 8 {
+		bpe = uint8(math.Ceil(math.Log2(float64(len(data.BlocksForVersion(version))))))
+	}
+
+	err = binary.Write(w, binary.BigEndian, bpe)
+	if err != nil {
+		return
+	}
 
 	switch bpe {
 	case 0:
 		err = proto_base.EncodeVarInt(w, blocks[0])
-	case 1, 2, 3:
+	case 1, 2, 3, 4, 5, 6, 7, 8:
-		err = PackArrayPallet(unique, blocks, w)
+		err = PackArrayPallet(unique, bpe, blocks, w)
 	default:
-		err = PackLongData(blocks, w)
+		// matches net.minecraft.world.chunk.PaletteProvider
+		err = PackLongData(blocks, bpe, w)
 	}
+
+	// This should also maybe get a maxblockid check but my fuzzer does not trigger a fail here so its probably fine
+	// also that would eat alot of performace
+
 	return
 }
 
-func UnpackBiomeData(r io.Reader) (biomes []int32, err error) {
+var BadBiomeIdErr = errors.New("bad biome id")
+
+func UnpackBiomeData(r io.Reader, version string) (biomes []int32, err error) {
 	// net.minecraft.world.chunk.PaletteProvider#forBiomes
 	var bitsPerEntry uint8
 	err = binary.Read(r, binary.BigEndian, &bitsPerEntry)
@@ -189,28 +223,47 @@ func UnpackBiomeData(r io.Reader) (biomes []int32, err error) {
 	default:
 		biomes, err = UnpackLongData(r, bitsPerEntry, BiomesPerChunkSection)
 	}
+
+	maxBiome := int32(len(data.BiomesForVersion(version)))
+	for _, b := range biomes {
+		if b < 0 || b >= maxBiome {
+			err = BadBiomeIdErr
+			return
+		}
+	}
+
 	if err != nil {
 		return
 	}
 	return
 }
 
-func PackBiomeData(biomes []int32, w io.Writer) (err error) {
+func PackBiomeData(biomes []int32, version string, w io.Writer) (err error) {
 	unique := util.GetUniqueSlice(biomes)
-	bpe := int32(math.Ceil(math.Log2(float64(len(unique)))))
+	bpe := uint8(math.Ceil(math.Log2(float64(len(unique)))))
+
+	if bpe > 3 {
+		bpe = uint8(math.Ceil(math.Log2(float64(len(data.BiomesForVersion(version))))))
+	}
+
+	err = binary.Write(w, binary.BigEndian, bpe)
+	if err != nil {
+		return
+	}
 
 	switch bpe {
 	case 0:
 		err = proto_base.EncodeVarInt(w, biomes[0])
 	case 1, 2, 3:
-		err = PackArrayPallet(unique, biomes, w)
+		err = PackArrayPallet(unique, bpe, biomes, w)
 	default:
-		err = PackLongData(biomes, w)
+		// matches net.minecraft.world.chunk.PaletteProvider
+		err = PackLongData(biomes, bpe, w)
 	}
 	return
 }
 
-func UnpackSection(r io.Reader) (blocks, biomes []int32, err error) {
+func UnpackSection(r io.Reader, version string) (blocks, biomes []int32, err error) {
 	var blockCount int16
 	err = binary.Read(r, binary.BigEndian, &blockCount)
 	if err != nil {
@@ -220,14 +273,14 @@ func UnpackSection(r io.Reader) (blocks, biomes []int32, err error) {
 	if err != nil {
 		return
 	}
-	biomes, err = UnpackBiomeData(r)
+	biomes, err = UnpackBiomeData(r, version)
 	return
 }
 
-func UnpackNSections(r io.Reader, n int) (blocks, biomes [][]int32, err error) {
+func UnpackNSections(r io.Reader, n int) (blocks, biomes [][]int32, version string, err error) {
 	for range n {
 		var blocksSection, biomesSections []int32
-		blocksSection, biomesSections, err = UnpackSection(r)
+		blocksSection, biomesSections, err = UnpackSection(r, version)
 		if err != nil {
 			return
 		}
@@ -241,31 +294,33 @@ var LenMustMatchErr = errors.New("length must match")
 var BiomeMust64LongErr = errors.New("biome section must be 64 long")
 var BlockMust4096LongErr = errors.New("block section must be 4096 long")
 
-func PackSection(blocks, biomes []int32, w io.Writer) (err error) {
+func PackSection(blocks, biomes []int32, version string, w io.Writer) (err error) {
 	if len(blocks) != BlocksPerChunkSection {
 		return BlockMust4096LongErr
 	}
 	if len(biomes) != BiomesPerChunkSection {
 		return BiomeMust64LongErr
 	}
-	err = proto_base.EncodeVarInt(w, BlocksPerChunkSection-int32(util.CountSlice(blocks, 0)))
+	numBlocks := BlocksPerChunkSection - int32(util.CountSlice(blocks, 0))
+
+	err = binary.Write(w, binary.BigEndian, int16(numBlocks))
 	if err != nil {
 		return
 	}
-	err = PackBlockData(blocks, w)
+	err = PackBlockData(blocks, version, w)
 	if err != nil {
 		return
 	}
-	err = PackBiomeData(biomes, w)
+	err = PackBiomeData(biomes, version, w)
 	return
 }
 
-func PackSections(blocks, biomes [][]int32, w io.Writer) (err error) {
+func PackSections(blocks, biomes [][]int32, version string, w io.Writer) (err error) {
 	if len(blocks) != len(biomes) {
 		return LenMustMatchErr
 	}
 	for i, blocksSection := range blocks {
-		err = PackSection(blocksSection, biomes[i], w)
+		err = PackSection(blocksSection, biomes[i], version, w)
 		if err != nil {
 			return
 		}

level/packet_test.go

@@ -2,6 +2,7 @@ package level_test
 
 import (
 	"bytes"
+	"math"
 	"slices"
 	"testing"
 
@@ -10,53 +11,95 @@ import (
 	"github.com/admin-else/strom/util"
 )
 
-var RedMushroomBlock = util.MustOk(data.LookupBlockByName("1.21.11", "red_mushroom_block"))
+var TestVersion = "1.21.11"
-var TheVoid = util.MustOk(data.LookupBiomeByName("1.21.11", "the_void"))
+var RedMushroomBlock = util.MustOk(data.LookupBlockByName(TestVersion, "red_mushroom_block"))
+var TheVoid = util.MustOk(data.LookupBiomeByName(TestVersion, "the_void"))
 
 func TestPackSections(t *testing.T) {
 	blocks := util.MakeSingleValuedSlice(RedMushroomBlock.DefaultState, level.BlocksPerChunkSection)
 	biomes := util.MakeSingleValuedSlice(TheVoid.Id, level.BiomesPerChunkSection)
 
 	b := bytes.NewBuffer(nil)
-	err := level.PackSection(blocks, biomes, b)
+	err := level.PackSection(blocks, biomes, TestVersion, b)
 	if err != nil {
 		t.Fatal(err)
 		return
 	}
 }
 
+func FuzzPackSection(f *testing.F) {
+	corpusBlocks := util.MakeSingleValuedSlice(int32(1), level.BlocksPerChunkSection)
+	corpusBiomes := util.MakeSingleValuedSlice(int32(1), level.BiomesPerChunkSection)
+	b := bytes.NewBuffer(nil)
+	_ = level.PackSection(corpusBlocks, corpusBiomes, TestVersion, b)
+	f.Add(b.Bytes())
+
+	f.Fuzz(func(t *testing.T, raw []byte) {
+		r := bytes.NewReader(raw)
+		t.Logf("Raw: %v", raw)
+		blocks, biomes, err := level.UnpackSection(r, TestVersion)
+		if err != nil {
+			t.Skipf("unpacking failed: %v", err)
+			return
+		}
+		t.Logf("Blocks: %#v, Biomes: %#v", blocks, biomes)
+
+		buf := bytes.NewBuffer(nil)
+		err = level.PackSection(blocks, biomes, TestVersion, buf)
+		if err != nil {
+			t.Fatal(err)
+			return
+		}
+		t.Logf("Packed: %v", buf.Bytes())
+		newBlocks, newBimoes, err := level.UnpackSection(buf, TestVersion)
+		if err != nil {
+			t.Fatal(err)
+			return
+		}
+
+		if !slices.Equal(blocks, newBlocks) {
+			t.Error("Blocks are not equal")
+		}
+		if !slices.Equal(biomes, newBimoes) {
+			t.Error("Biomes are not equal")
+		}
+	})
+}
+
 func TestDataLongsTest(t *testing.T) {
-	var bpes = []int32{1, 2, 3, 4, 5, 10, 31}
+	var bpes = []uint8{1, 2, 3, 4, 5, 10, 31}
 	var testValues = [][]int32{{1}, {1, 2}, {1, 2, 3}, {1, 2, 3, 4}, {1, 2, 3, 4, 5}, {999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999}}
 
 	for _, bpe := range bpes {
-		maxValue := int32((1 << bpe) - 1)
+		t.Logf("BPE: %v", bpe)
-		t.Logf("BPE: %v, Maxvalue: %v", bpe, maxValue)
 		for _, values := range testValues {
 			t.Logf("Data: %v", values)
 
-			skip := false
+			longs, err := level.DataToLongs(values, bpe)
-			for _, v := range values {
+			if err != nil {
-				if v >= maxValue {
+				t.Logf("Error: %v", err)
-					skip = true
-					break
-				}
-			}
-			if skip {
-				t.Log("Skipping because of overflow")
 				continue
 			}
-
-			longs := level.DataToLongs(values, bpe)
-
 			t.Logf("Longs: %v", longs)
 
 			dataNew := level.LongsToData(longs, int32(len(values)), bpe)
-
+			t.Logf("Data new: %v", dataNew)
 			if !slices.Equal(values, dataNew) {
 				t.Error("Data is not equal")
+			} else {
+				t.Log("Data conversion successful")
 			}
-			t.Log("Data conversion successful")
 		}
 	}
 }
+
+func TestWeirdBiomesData(t *testing.T) {
+	biomes := []int32{48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 67, 66, 65, 57, 56, 55, 50, 49}
+
+	var b = bytes.NewBuffer(nil)
+
+	unique := util.GetUniqueSlice(biomes)
+	bpe := uint8(math.Ceil(math.Log2(float64(len(unique)))))
+	err := level.PackLongData(biomes, bpe, b)
+	_ = err
+}

level/testdata/fuzz/FuzzPackSection/1d71329b7e8473da

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("00\x000\x000")

level/testdata/fuzz/FuzzPackSection/720a2176d4a6d755

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("00A")

level/testdata/fuzz/FuzzPackSection/746414e054d7661b

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("00\x000\b0000000000000000000000000000000000000000000000000000000012789ABC")

level/testdata/fuzz/FuzzPackSection/7fa8e415d92ee53a

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("00\x000\b000000000\x9d0000000000000000000000000000000000000000000000012789AB")

level/testdata/fuzz/FuzzPackSection/c6a489835f1c78cc

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("00\x00\x00\x000")