chunk serialization

data/biomes.go

@@ -14,7 +14,7 @@ package data
 //	},
 
 type Biome struct {
-	Id               int
+	Id               int32
 	Name             string
 	Category         string
 	Temperature      float64
@@ -24,22 +24,33 @@ type Biome struct {
 	Color            int
 }
 
-var BiomesCache = make(map[string][]Biome)
+var BiomesCache = make(map[string][]*Biome)
 
-func BiomesForVersion(v string) (ret []Biome) {
+func BiomesForVersion(v string) (ret []*Biome) {
 	var ok bool
 	if ret, ok = BiomesCache[v]; ok {
 		return
 	}
-	var b []Biome
+	var b []*Biome
 	must(LoadVersionedJson(v, "biomes", &b))
 	BiomesCache[v] = b
 	return b
 }
 
-func LookupBiomeById(version string, id int32) (biome Biome, ok bool) {
+func LookupBiomeById(version string, id int32) (biome *Biome, ok bool) {
 	for _, b := range BiomesForVersion(version) {
-		if b.Id == int(id) {
+		if b.Id == id {
+			biome = b
+			ok = true
+			return
+		}
+	}
+	return
+}
+
+func LookupBiomeByName(version string, name string) (biome *Biome, ok bool) {
+	for _, b := range BiomesForVersion(version) {
+		if b.Name == name {
 			biome = b
 			ok = true
 			return

level/packet.go

@@ -2,17 +2,21 @@ package level
 
 import (
 	"encoding/binary"
+	"errors"
 	"io"
 	"math"
 
 	"github.com/admin-else/strom/proto_base"
+	"github.com/admin-else/strom/util"
 )
 
 const (
-	BlocksPerChunkSection = 4096
+	BlocksPerChunkSection = 16 * 16 * 16
-	BiomesPerChunkSection = 64
+	BiomesPerChunkSection = 4 * 4 * 4
 )
 
+var InvalidPalletIndexErr = errors.New("invalid palette index")
+
 func UnpackArrayPalette(r io.Reader, bitsPerEntry uint8, numberOfEntries int) (data []int32, err error) {
 	palletLen, err := proto_base.DecodeVarInt(r)
 	if err != nil {
@@ -28,13 +32,39 @@ func UnpackArrayPalette(r io.Reader, bitsPerEntry uint8, numberOfEntries int) (d
 		pallet = append(pallet, entry)
 	}
 	data, err = UnpackLongData(r, bitsPerEntry, numberOfEntries)
+
 	for i, b := range data {
-		data[i] = pallet[b] // maybe check bounds?
+		if b < 0 || b >= palletLen {
+			err = InvalidPalletIndexErr
+			return
+		}
+		data[i] = pallet[b]
 	}
 
 	return
 }
 
+func PackArrayPallet(unique map[int32]struct{}, data []int32, w io.Writer) (err error) {
+	pallet := util.SetToSliceOrdered(unique) // this may improve performance in combination with zlib i dont know tho
+	err = proto_base.EncodeVarInt(w, int32(len(pallet)))
+	if err != nil {
+		return
+	}
+	var palletMap = make(map[int32]int32)
+	for i, entry := range pallet {
+		palletMap[entry] = int32(i)
+		err = proto_base.EncodeVarInt(w, entry)
+		if err != nil {
+			return
+		}
+	}
+	var palletedData = make([]int32, len(data))
+	for i, entry := range data {
+		palletedData[i] = palletMap[entry]
+	}
+	return PackLongData(palletedData, w)
+}
+
 func UnpackLongData(r io.Reader, bitsPerEntry uint8, numberOfEntries int) (data []int32, err error) {
 	entriesPerLong := int(64 / bitsPerEntry)
 	numberOfLongs := int(math.Ceil(float64(numberOfEntries) / float64(entriesPerLong)))
@@ -52,6 +82,16 @@ func UnpackLongData(r io.Reader, bitsPerEntry uint8, numberOfEntries int) (data
 	return
 }
 
+func PackLongData(data []int32, w io.Writer) (err error) {
+	unique := util.GetUniqueSlice(data)
+	bpe := int32(math.Ceil(math.Log2(float64(len(unique)))))
+	dataL := DataToLongs(data, bpe)
+	for _, entry := range dataL {
+		err = binary.Write(w, binary.BigEndian, entry)
+	}
+	return
+}
+
 func UnpackSingleValuePalette(r io.Reader, numberOfEntries int) (data []int32, err error) {
 	entry, err := proto_base.DecodeVarInt(r)
 	if err != nil {
@@ -75,6 +115,22 @@ func LongsToData(data []uint64, n, bpe int32) (ret []int32) {
 	return
 }
 
+func DataToLongs(data []int32, bpe int32) (ret []uint64) {
+	elementsPerLong := 64 / bpe
+	retLen := int32(math.Ceil(float64(len(data)) / float64(elementsPerLong)))
+	ret = make([]uint64, retLen)
+	for i := range retLen {
+		for j := range elementsPerLong {
+			dataI := i*elementsPerLong + j
+			if dataI >= int32(len(data)) {
+				break
+			}
+			ret[i] |= uint64(data[dataI]) << (uint32(j) * uint32(bpe))
+		}
+	}
+	return
+}
+
 func GetEntryFromLongs(data []uint64, i, bpe int32) (ret int32) {
 	mask := (uint64(1) << uint32(bpe)) - 1
 	elementsPerLong := 64 / bpe
@@ -103,6 +159,21 @@ func UnpackBlockData(r io.Reader) (blocks []int32, err error) {
 	return
 }
 
+func PackBlockData(blocks []int32, w io.Writer) (err error) {
+	unique := util.GetUniqueSlice(blocks)
+	bpe := int32(math.Ceil(math.Log2(float64(len(unique)))))
+
+	switch bpe {
+	case 0:
+		err = proto_base.EncodeVarInt(w, blocks[0])
+	case 1, 2, 3:
+		err = PackArrayPallet(unique, blocks, w)
+	default:
+		err = PackLongData(blocks, w)
+	}
+	return
+}
+
 func UnpackBiomeData(r io.Reader) (biomes []int32, err error) {
 	// net.minecraft.world.chunk.PaletteProvider#forBiomes
 	var bitsPerEntry uint8
@@ -124,6 +195,21 @@ func UnpackBiomeData(r io.Reader) (biomes []int32, err error) {
 	return
 }
 
+func PackBiomeData(biomes []int32, w io.Writer) (err error) {
+	unique := util.GetUniqueSlice(biomes)
+	bpe := int32(math.Ceil(math.Log2(float64(len(unique)))))
+
+	switch bpe {
+	case 0:
+		err = proto_base.EncodeVarInt(w, biomes[0])
+	case 1, 2, 3:
+		err = PackArrayPallet(unique, biomes, w)
+	default:
+		err = PackLongData(biomes, w)
+	}
+	return
+}
+
 func UnpackSection(r io.Reader) (blocks, biomes []int32, err error) {
 	var blockCount int16
 	err = binary.Read(r, binary.BigEndian, &blockCount)
@@ -150,3 +236,39 @@ func UnpackNSections(r io.Reader, n int) (blocks, biomes [][]int32, err error) {
 	}
 	return
 }
+
+var LenMustMatchErr = errors.New("length must match")
+var BiomeMust64LongErr = errors.New("biome section must be 64 long")
+var BlockMust4096LongErr = errors.New("block section must be 4096 long")
+
+func PackSection(blocks, biomes []int32, w io.Writer) (err error) {
+	if len(blocks) != BlocksPerChunkSection {
+		return BlockMust4096LongErr
+	}
+	if len(biomes) != BiomesPerChunkSection {
+		return BiomeMust64LongErr
+	}
+	err = proto_base.EncodeVarInt(w, BlocksPerChunkSection-int32(util.CountSlice(blocks, 0)))
+	if err != nil {
+		return
+	}
+	err = PackBlockData(blocks, w)
+	if err != nil {
+		return
+	}
+	err = PackBiomeData(biomes, w)
+	return
+}
+
+func PackSections(blocks, biomes [][]int32, w io.Writer) (err error) {
+	if len(blocks) != len(biomes) {
+		return LenMustMatchErr
+	}
+	for i, blocksSection := range blocks {
+		err = PackSection(blocksSection, biomes[i], w)
+		if err != nil {
+			return
+		}
+	}
+	return
+}

level/packet_test.go

@@ -0,0 +1,62 @@
+package level_test
+
+import (
+	"bytes"
+	"slices"
+	"testing"
+
+	"github.com/admin-else/strom/data"
+	"github.com/admin-else/strom/level"
+	"github.com/admin-else/strom/util"
+)
+
+var RedMushroomBlock = util.MustOk(data.LookupBlockByName("1.21.11", "red_mushroom_block"))
+var TheVoid = util.MustOk(data.LookupBiomeByName("1.21.11", "the_void"))
+
+func TestPackSections(t *testing.T) {
+	blocks := util.MakeSingleValuedSlice(RedMushroomBlock.DefaultState, level.BlocksPerChunkSection)
+	biomes := util.MakeSingleValuedSlice(TheVoid.Id, level.BiomesPerChunkSection)
+
+	b := bytes.NewBuffer(nil)
+	err := level.PackSection(blocks, biomes, b)
+	if err != nil {
+		t.Fatal(err)
+		return
+	}
+}
+
+func TestDataLongsTest(t *testing.T) {
+	var bpes = []int32{1, 2, 3, 4, 5, 10, 31}
+	var testValues = [][]int32{{1}, {1, 2}, {1, 2, 3}, {1, 2, 3, 4}, {1, 2, 3, 4, 5}, {999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999}}
+
+	for _, bpe := range bpes {
+		maxValue := int32((1 << bpe) - 1)
+		t.Logf("BPE: %v, Maxvalue: %v", bpe, maxValue)
+		for _, values := range testValues {
+			t.Logf("Data: %v", values)
+
+			skip := false
+			for _, v := range values {
+				if v >= maxValue {
+					skip = true
+					break
+				}
+			}
+			if skip {
+				t.Log("Skipping because of overflow")
+				continue
+			}
+
+			longs := level.DataToLongs(values, bpe)
+
+			t.Logf("Longs: %v", longs)
+
+			dataNew := level.LongsToData(longs, int32(len(values)), bpe)
+
+			if !slices.Equal(values, dataNew) {
+				t.Error("Data is not equal")
+			}
+			t.Log("Data conversion successful")
+		}
+	}
+}

util/must.go

@@ -0,0 +1,8 @@
+package util
+
+func MustOk[T any](v T, ok bool) T {
+	if !ok {
+		panic("not ok")
+	}
+	return v
+}

util/slices.go

@@ -1,8 +1,57 @@
 package util
 
-import "math/rand/v2"
+import (
+	"cmp"
+	"math/rand/v2"
+	"slices"
+)
 
 func ShuffleSlice[T any](s []T) (ret []T) {
 	rand.Shuffle(len(s), func(i, j int) { s[i], s[j] = s[j], s[i] })
 	return s
 }
+
+func CountSlice[T comparable](s []T, o T) (ret int) {
+	for _, v := range s {
+		if v == o {
+			ret++
+		}
+	}
+	return
+}
+
+func GetUniqueSlice[T comparable](s []T) (unique map[T]struct{}) {
+	unique = make(map[T]struct{})
+	for _, v := range s {
+		if _, ok := unique[v]; !ok {
+			unique[v] = struct{}{}
+		}
+	}
+	return
+}
+
+func CountUniqueSlice[T comparable](s []T) (ret int) {
+	return len(GetUniqueSlice(s))
+}
+
+func MakeSingleValuedSlice[T any](v T, n int) (ret []T) {
+	ret = make([]T, n)
+	for i := range ret {
+		ret[i] = v
+	}
+	return
+}
+
+func SetToSlice[T comparable](set map[T]struct{}) (slice []T) {
+	slice = make([]T, 0, len(set))
+	for k := range set {
+		slice = append(slice, k)
+	}
+	return
+}
+
+func SetToSliceOrdered[T cmp.Ordered](set map[T]struct{}) (slice []T) {
+	slice = SetToSlice(set)
+	slices.Sort(slice)
+	return
+}