Merge pull request #1 from admin-else/feat/mca2packet

start work on mca to packet conversion

.github/workflows/go.yml

@@ -19,7 +19,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.25'
+        go-version: '1.26.2'
 
     - name: Build
       run: go build -v ./...

go.mod

@@ -1,6 +1,6 @@
 module github.com/admin-else/strom
 
-go 1.25
+go 1.26.2
 
 require (
 	github.com/go-viper/mapstructure/v2 v2.4.0

level/anvil/chunk.go

@@ -22,6 +22,7 @@ const McPrefix = "minecraft:"
 func (c Chunk) ToStorage(version string) (toChunk *level.Chunk, err error) {
 	f := level.MakeBlockFormat(version)
 	toChunk = new(level.Chunk)
+	toChunk.Version = version
 
 	toChunk.Sections = make([]level.Section, len(c.Sections))
 	for i, s := range c.Sections {
@@ -33,15 +34,24 @@ func (c Chunk) ToStorage(version string) (toChunk *level.Chunk, err error) {
 			}
 		}
 		// this is probably fine
-		uintSlice := unsafe.Slice((*uint64)(unsafe.Pointer(unsafe.SliceData(s.BlockStates.Data))), len(s.BlockStates.Data))
 
-		toChunk.Sections[i].Blocks, err = f.Import(uintSlice, util.BpeByNum(float64(len(s.BlockStates.Palette))), pallete)
+		if s.BlockStates.Data != nil {
-		if err != nil {
+			uintSlice := unsafe.Slice((*uint64)(unsafe.Pointer(unsafe.SliceData(s.BlockStates.Data))), len(s.BlockStates.Data))
-			return
+			bpe := util.BpeByNum(float64(len(s.BlockStates.Palette)))
+			toChunk.Sections[i].Blocks, err = f.Import(uintSlice, bpe, pallete)
+			if err != nil {
+				return
+			}
+			toChunk.Sections[i].BlockCount = int16(level.BlocksPerChunkSection - level.CountStorage(toChunk.Sections[i].Blocks, 0))
+		} else {
+			toChunk.Sections[i].Blocks, err = f.FullWith(0) //TODO PALLETE index 0 if avial
+			if err != nil {
+				return
+			}
+			toChunk.Sections[i].BlockCount = int16(f.Len)
 		}
-		toChunk.Sections[i].BlockCount = int16(level.CountStorage(toChunk.Sections[i].Blocks, 0))
 	}
-
+	f = level.MakeBiomeFormat(version)
 	for i, s := range c.Sections {
 		pallete := make([]int32, len(s.Biomes.Palette))
 		for j, p := range s.Biomes.Palette {
@@ -53,12 +63,21 @@ func (c Chunk) ToStorage(version string) (toChunk *level.Chunk, err error) {
 			}
 			pallete[j] = b.Id
 		}
-		// this is probably fine
-		uintSlice := unsafe.Slice((*uint64)(unsafe.Pointer(unsafe.SliceData(s.Biomes.Data))), len(s.Biomes.Data))
 
-		toChunk.Sections[i].Biomes, err = f.Import(uintSlice, util.BpeByNum(float64(len(pallete))), pallete)
+		if s.Biomes.Data != nil {
-		if err != nil {
+
-			return
+			// this is probably fine
+			uintSlice := unsafe.Slice((*uint64)(unsafe.Pointer(unsafe.SliceData(s.Biomes.Data))), len(s.Biomes.Data))
+			bpe := util.BpeByNum(float64(len(pallete)))
+			toChunk.Sections[i].Biomes, err = f.Import(uintSlice, bpe, pallete)
+			if err != nil {
+				return
+			}
+		} else {
+			toChunk.Sections[i].Biomes, err = f.FullWith(0) //TODO PALLETE index 0 if avial
+			if err != nil {
+				return
+			}
 		}
 	}
 	return
@@ -72,9 +91,9 @@ type Section struct {
 			Name       string
 			Properties map[string]string `nbt:"properties,omitempty"`
 		}
-	}
+	} `nbt:"block_states,omitempty"`
 	Biomes struct {
 		Data    []int64 `nbt:"data,omitempty"`
 		Palette []string
-	}
+	} `nbt:"biomes,omitempty"`
 }

level/anvil/level_test.go

@@ -1,11 +1,13 @@
 package anvil_test
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"os"
 	"testing"
 
+	"github.com/admin-else/strom/level"
 	"github.com/admin-else/strom/level/anvil"
 	"github.com/admin-else/strom/util"
 )
@@ -44,11 +46,8 @@ func TestRegionFile(t *testing.T) {
 	fmt.Println("Chunk loaded successfully")
 }
 
-func TestConverToPacket(t *testing.T) {
+func loadPacketToChunk(b []byte) (chunk *level.Chunk, err error) {
-	f, err := os.Open("./level/anvil/testdata/r.0.0.mca")
+	buffer := bytes.NewBuffer(b)
-	if err != nil {
+	chunk, err = level.ReadChunkFromChunkPacketData(buffer, "1.21.11", 384)
-		return
+	return
-	}
-	defer f.Close()
-
 }

level/chunk.go

@@ -1,15 +1,18 @@
 package level
 
-import "io"
+import (
+	"io"
+	"log/slog"
+)
 
 type Chunk struct {
 	Sections []Section
-	version  string
+	Version  string
 }
 
-func ReadChunkFromChunkPacketData(r io.Reader, version string, worldHeight int) (err error) {
+func ReadChunkFromChunkPacketData(r io.Reader, version string, worldHeight int) (c *Chunk, err error) {
-	c := &Chunk{}
+	c = &Chunk{}
-	c.version = version
+	c.Version = version
 	nSections := worldHeight / ChunkWidth
 
 	c.Sections = make([]Section, nSections)
@@ -31,3 +34,20 @@ func (c *Chunk) WriteChunkData(w io.Writer) (err error) {
 	}
 	return
 }
+
+func (c *Chunk) Equals(other *Chunk) bool {
+	if c.Version != other.Version {
+		slog.Debug("Chunk versions don't match: ", "version 1", c.Version, "version2", other.Version)
+		return false
+	}
+	if len(c.Sections) != len(other.Sections) {
+		slog.Debug("Chunk section len does not match: ", "chunk1 len", len(c.Sections), "chunk 2 len", len(other.Sections))
+		return false
+	}
+	for i, s := range c.Sections {
+		if !SectionEquals(s, other.Sections[i]) {
+			return false
+		}
+	}
+	return true
+}

level/litematic/main.go

@@ -3,7 +3,6 @@ package litematic
 import (
 	"errors"
 	"io"
-	"math"
 	"os"
 	"strings"
 	"unsafe"
@@ -11,13 +10,31 @@ import (
 	"github.com/admin-else/strom/data"
 	"github.com/admin-else/strom/level"
 	"github.com/admin-else/strom/nbt"
+	"github.com/admin-else/strom/util"
 	"github.com/go-viper/mapstructure/v2"
 )
 
+var OutOfBoundsErr = errors.New("out of bounds")
+
+const mcNamespaced = "minecraft:"
+
 type Position struct {
 	X, Y, Z int32
 }
 
+func (p Position) FixNegatives() Position {
+	if p.X < 0 {
+		p.X *= -1
+	}
+	if p.Y < 0 {
+		p.Y *= -1
+	}
+	if p.Z < 0 {
+		p.Z *= -1
+	}
+	return p
+}
+
 type MetaData struct {
 	Author        string
 	Description   string
@@ -40,12 +57,11 @@ type Region struct {
 	Size              Position
 	BlockStatePalette []BlockState
 	BlockStates       []int64
+	*level.Storage
 }
 
-func (r *Region) GetBlockAt(x, y, z int32) (name string, properties map[string]string, err error) {
+func (s *Structure) GetBlockAt(region string, x, y, z int32) (stateId int32, err error) {
-
+	r := s.Regions[region]
-	// this is probably fine
-	uintSlice := unsafe.Slice((*uint64)(unsafe.Pointer(unsafe.SliceData(r.BlockStates))), len(r.BlockStates))
 	if r.Size.X < 0 {
 		r.Size.X *= -1
 	}
@@ -61,15 +77,7 @@ func (r *Region) GetBlockAt(x, y, z int32) (name string, properties map[string]s
 		return
 	}
 	i := x + z*r.Size.X + y*r.Size.X*r.Size.Z
-	bpe := int32(math.Ceil(math.Log2(float64(len(r.BlockStatePalette)))))
+	stateId, err = r.Get(i)
-	i = level.GetEntryFromLongs(uintSlice, i, bpe)
-	if i >= int32(len(r.BlockStatePalette)) {
-		err = BlockStateNotInPalletErr
-		return
-	}
-	block := r.BlockStatePalette[i]
-	name = block.Name
-	properties = block.Properties
 	return
 }
 
@@ -83,7 +91,7 @@ type File struct {
 
 type Structure struct {
 	File
-	BlockStatePalletRealized map[string][]int32
+	Version string
 }
 
 func LoadFromPath(path string) (s *Structure, err error) {
@@ -114,57 +122,32 @@ func Load(v any) (s *Structure, err error) {
 	if err != nil {
 		return
 	}
-	s.BlockStatePalletRealized = make(map[string][]int32)
+	s.Version = ret.MinecraftVersion
-	for k, r := range s.Regions {
+	for k, region := range s.Regions {
-		s.BlockStatePalletRealized[k] = make([]int32, len(r.BlockStatePalette))
+		palette := make([]int32, len(region.BlockStatePalette))
-		for i, b := range r.BlockStatePalette {
+		for i, b := range region.BlockStatePalette {
-			var stateId int32
+			var paletteEntry int32
-			stateId, err = data.StateIdFromBlocKAndStateMap(ret.MinecraftVersion, strings.TrimPrefix(b.Name, "minecraft:"), b.Properties)
+			paletteEntry, err = data.StateIdFromBlocKAndStateMap(s.Version, strings.TrimPrefix(b.Name, mcNamespaced), b.Properties)
 			if err != nil {
 				return
 			}
-			s.BlockStatePalletRealized[k][i] = stateId
+			palette[i] = paletteEntry
 		}
+
+		sizeFixed := region.Size.FixNegatives()
+		l := sizeFixed.X * sizeFixed.Y * sizeFixed.Z
+		bpe := util.BpeByNum(float64(len(region.BlockStatePalette)))
+		uintSlice := unsafe.Slice((*uint64)(unsafe.Pointer(unsafe.SliceData(region.BlockStates))), len(region.BlockStates))
+		region.Storage, err = level.StorageFormat{
+			AvailableBpes: []uint8{bpe},
+			BiggestDirect: false,
+			Len:           l,
+		}.Import(uintSlice, bpe, palette)
+		if err != nil {
+			return
+		}
+		s.Regions[k] = region
 	}
-	return
-}
 
-var BlockStateNotInPalletErr = errors.New("block state not in pallet")
-var OutOfBoundsErr = errors.New("out of bounds")
-var RegionNotInFileErr = errors.New("region not in file")
-
-func (s *Structure) GetBlockAt(name string, x, y, z int32) (blockState int32, err error) {
-	r, ok := s.Regions[name]
-	if !ok {
-		err = RegionNotInFileErr
-		return
-	}
-	uintSlice := make([]uint64, len(r.BlockStates))
-	for i, v := range r.BlockStates {
-		uintSlice[i] = uint64(v)
-	}
-	if r.Size.X < 0 {
-		r.Size.X *= -1
-	}
-	if r.Size.Y < 0 {
-		r.Size.Y *= -1
-	}
-	if r.Size.Z < 0 {
-		r.Size.Z *= -1
-	}
-
-	if x >= r.Size.X || y >= r.Size.Y || z >= r.Size.Z {
-		err = OutOfBoundsErr
-		return
-	}
-	i := x + z*r.Size.X + y*r.Size.X*r.Size.Z
-	bpe := int32(math.Ceil(math.Log2(float64(len(r.BlockStatePalette)))))
-	i = level.GetEntryFromLongs(uintSlice, i, bpe)
-	mapSlice := s.BlockStatePalletRealized[name]
-	if i >= int32(len(mapSlice)) {
-		err = BlockStateNotInPalletErr
-		return
-	}
-	blockState = mapSlice[i]
 	return
 }

level/packet.go

@@ -1,328 +0,0 @@
-package level
-
-import (
-	"encoding/binary"
-	"errors"
-	"io"
-	"math"
-
-	"github.com/admin-else/strom/data"
-	"github.com/admin-else/strom/proto_base"
-	"github.com/admin-else/strom/util"
-)
-
-var InvalidPalletIndexErr = errors.New("invalid palette index")
-
-func UnpackArrayPalette(r io.Reader, bitsPerEntry uint8, numberOfEntries int) (data []int32, err error) {
-	palletLen, err := proto_base.DecodeVarInt(r)
-	if err != nil {
-		return
-	}
-	var pallet []int32 // remember we can't use make because this is user-controlled data
-	for range palletLen {
-		var entry int32
-		entry, err = proto_base.DecodeVarInt(r)
-		if err != nil {
-			return
-		}
-		pallet = append(pallet, entry)
-	}
-	data, err = UnpackLongData(r, bitsPerEntry, numberOfEntries)
-
-	for i, b := range data {
-		if b < 0 || b >= palletLen {
-			err = InvalidPalletIndexErr
-			return
-		}
-		data[i] = pallet[b]
-	}
-
-	return
-}
-
-func PackArrayPallet(unique map[int32]struct{}, bpe uint8, data []int32, w io.Writer) (err error) {
-	pallet := util.SetToSliceOrdered(unique) // this may improve performance in combination with zlib i dont know tho
-	err = proto_base.EncodeVarInt(w, int32(len(pallet)))
-	if err != nil {
-		return
-	}
-	var palletMap = make(map[int32]int32)
-	for i, entry := range pallet {
-		palletMap[entry] = int32(i)
-		err = proto_base.EncodeVarInt(w, entry)
-		if err != nil {
-			return
-		}
-	}
-	palletedData := make([]int32, len(data))
-	for i, entry := range data {
-		palletedData[i] = palletMap[entry]
-	}
-	return PackLongData(palletedData, bpe, w)
-}
-
-var BadBitsPerEntryErr = errors.New("bad bits per entry")
-
-func UnpackLongData(r io.Reader, bitsPerEntry uint8, numberOfEntries int) (data []int32, err error) {
-	if bitsPerEntry < 1 || bitsPerEntry > 31 {
-		err = BadBitsPerEntryErr
-		return
-	}
-
-	entriesPerLong := int(64 / bitsPerEntry)
-	numberOfLongs := int(math.Ceil(float64(numberOfEntries) / float64(entriesPerLong)))
-	var dataL []uint64
-	for range numberOfLongs {
-		var entry uint64
-		err = binary.Read(r, binary.BigEndian, &entry)
-		if err != nil {
-			return
-		}
-		dataL = append(dataL, entry)
-	}
-
-	data = LongsToData(dataL, int32(numberOfEntries), bitsPerEntry)
-	return
-}
-
-func PackLongData(data []int32, bpe uint8, w io.Writer) (err error) {
-	dataL, err := DataToLongs(data, bpe)
-	if err != nil {
-		return
-	}
-	for _, entry := range dataL {
-		err = binary.Write(w, binary.BigEndian, entry)
-	}
-	return
-}
-
-func UnpackSingleValuePalette(r io.Reader, numberOfEntries int) (data []int32, err error) {
-	entry, err := proto_base.DecodeVarInt(r)
-	if err != nil {
-		return
-	}
-	data = make([]int32, numberOfEntries)
-	for i := range data {
-		data[i] = entry
-	}
-	return
-}
-
-func LongsToData(data []uint64, n int32, bpe uint8) (ret []int32) {
-	mask := (uint64(1) << uint32(bpe)) - 1
-	elementsPerLong := int32(64 / bpe)
-
-	ret = make([]int32, n)
-	for i := range n {
-		ret[i] = int32((data[i/elementsPerLong] >> ((i % elementsPerLong) * int32(bpe))) & mask)
-	}
-	return
-}
-
-var BpeTooSmallErr = errors.New("bits per entry too small")
-
-func DataToLongs(data []int32, bpe uint8) (ret []uint64, err error) {
-	maxValue := uint64((1 << bpe) - 1)
-
-	elementsPerLong := 64 / int32(bpe)
-	retLen := int32(math.Ceil(float64(len(data)) / float64(elementsPerLong)))
-	ret = make([]uint64, retLen)
-	for i := range retLen {
-		for j := range elementsPerLong {
-			dataI := i*elementsPerLong + j
-			if dataI >= int32(len(data)) {
-				break
-			}
-			toInsert := uint64(data[dataI])
-			if toInsert > maxValue {
-				err = BpeTooSmallErr
-				return
-			}
-			ret[i] |= toInsert << (uint32(j) * uint32(bpe))
-		}
-	}
-	return
-}
-
-func GetEntryFromLongs(data []uint64, i, bpe int32) (ret int32) {
-	mask := (uint64(1) << uint32(bpe)) - 1
-	elementsPerLong := 64 / bpe
-	ret = int32((data[i/elementsPerLong] >> ((i % elementsPerLong) * bpe)) & mask)
-	return
-}
-
-// UnpackBlockData minecraft does this at net.minecraft.world.chunk.PaletteProvider
-func UnpackBlockData(r io.Reader) (blocks []int32, err error) {
-	var bitsPerEntry uint8
-	err = binary.Read(r, binary.BigEndian, &bitsPerEntry)
-	if err != nil {
-		return
-	}
-	switch bitsPerEntry {
-	case 0:
-		blocks, err = UnpackSingleValuePalette(r, BlocksPerChunkSection)
-	case 1, 2, 3, 4:
-		blocks, err = UnpackArrayPalette(r, 4, BlocksPerChunkSection)
-	case 5, 6, 7, 8:
-		blocks, err = UnpackArrayPalette(r, bitsPerEntry, BlocksPerChunkSection)
-	default:
-		blocks, err = UnpackLongData(r, bitsPerEntry, BlocksPerChunkSection)
-	}
-	if err != nil {
-		return
-	}
-	return
-}
-
-func PackBlockData(blocks []int32, version string, w io.Writer) (err error) {
-	unique := util.GetUniqueSlice(blocks)
-	bpe := util.BpeByNum(float64(len(unique)))
-
-	if bpe > 8 {
-		bpe = util.BpeByNum(float64(len(data.BlocksForVersion(version))))
-	}
-
-	err = binary.Write(w, binary.BigEndian, bpe)
-	if err != nil {
-		return
-	}
-
-	switch bpe {
-	case 0:
-		err = proto_base.EncodeVarInt(w, blocks[0])
-	case 1, 2, 3, 4:
-		err = PackArrayPallet(unique, 4, blocks, w) // MOJANK
-	case 5, 6, 7, 8:
-		err = PackArrayPallet(unique, bpe, blocks, w)
-	default:
-		// matches net.minecraft.world.chunk.PaletteProvider
-		err = PackLongData(blocks, bpe, w)
-	}
-
-	// This should also maybe get a maxblockid check but my fuzzer does not trigger a fail here so its probably fine
-	// also that would eat alot of performace
-
-	return
-}
-
-var BadBiomeIdErr = errors.New("bad biome id")
-
-func UnpackBiomeData(r io.Reader, version string) (biomes []int32, err error) {
-	// net.minecraft.world.chunk.PaletteProvider#forBiomes
-	var bitsPerEntry uint8
-	err = binary.Read(r, binary.BigEndian, &bitsPerEntry)
-	if err != nil {
-		return
-	}
-	switch bitsPerEntry {
-	case 0:
-		biomes, err = UnpackSingleValuePalette(r, BiomesPerChunkSection)
-	case 1, 2, 3:
-		biomes, err = UnpackArrayPalette(r, bitsPerEntry, BiomesPerChunkSection)
-	default:
-		biomes, err = UnpackLongData(r, bitsPerEntry, BiomesPerChunkSection)
-	}
-
-	maxBiome := int32(len(data.BiomesForVersion(version)))
-	for _, b := range biomes {
-		if b < 0 || b >= maxBiome {
-			err = BadBiomeIdErr
-			return
-		}
-	}
-
-	if err != nil {
-		return
-	}
-	return
-}
-
-func PackBiomeData(biomes []int32, version string, w io.Writer) (err error) {
-	unique := util.GetUniqueSlice(biomes)
-	bpe := uint8(math.Ceil(math.Log2(float64(len(unique)))))
-
-	if bpe > 3 {
-		bpe = uint8(math.Ceil(math.Log2(float64(len(data.BiomesForVersion(version))))))
-	}
-
-	err = binary.Write(w, binary.BigEndian, bpe)
-	if err != nil {
-		return
-	}
-
-	switch bpe {
-	case 0:
-		err = proto_base.EncodeVarInt(w, biomes[0])
-	case 1, 2, 3:
-		err = PackArrayPallet(unique, bpe, biomes, w)
-	default:
-		// matches net.minecraft.world.chunk.PaletteProvider
-		err = PackLongData(biomes, bpe, w)
-	}
-	return
-}
-
-func UnpackSection(r io.Reader, version string) (blocks, biomes []int32, err error) {
-	var blockCount int16
-	err = binary.Read(r, binary.BigEndian, &blockCount)
-	if err != nil {
-		return
-	}
-	blocks, err = UnpackBlockData(r)
-	if err != nil {
-		return
-	}
-	biomes, err = UnpackBiomeData(r, version)
-	return
-}
-
-func UnpackNSections(r io.Reader, n int) (blocks, biomes [][]int32, version string, err error) {
-	for range n {
-		var blocksSection, biomesSections []int32
-		blocksSection, biomesSections, err = UnpackSection(r, version)
-		if err != nil {
-			return
-		}
-		blocks = append(blocks, blocksSection)
-		biomes = append(biomes, biomesSections)
-	}
-	return
-}
-
-var LenMustMatchErr = errors.New("length must match")
-var BiomeMust64LongErr = errors.New("biome section must be 64 long")
-var BlockMust4096LongErr = errors.New("block section must be 4096 long")
-
-func PackSection(blocks, biomes []int32, version string, w io.Writer) (err error) {
-	if len(blocks) != BlocksPerChunkSection {
-		return BlockMust4096LongErr
-	}
-	if len(biomes) != BiomesPerChunkSection {
-		return BiomeMust64LongErr
-	}
-	numBlocks := BlocksPerChunkSection - int32(util.CountSlice(blocks, 0))
-
-	err = binary.Write(w, binary.BigEndian, int16(numBlocks))
-	if err != nil {
-		return
-	}
-	err = PackBlockData(blocks, version, w)
-	if err != nil {
-		return
-	}
-	err = PackBiomeData(biomes, version, w)
-	return
-}
-
-func PackSections(blocks, biomes [][]int32, version string, w io.Writer) (err error) {
-	if len(blocks) != len(biomes) {
-		return LenMustMatchErr
-	}
-	for i, blocksSection := range blocks {
-		err = PackSection(blocksSection, biomes[i], version, w)
-		if err != nil {
-			return
-		}
-	}
-	return
-}

level/packet_roundtrip_test.go

@@ -0,0 +1,34 @@
+package level_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/admin-else/strom/level"
+)
+
+func FuzzReadChunkFromChunkPacketData(f *testing.F) {
+	f.Fuzz(func(t *testing.T, raw []byte) {
+		chunk, err := level.ReadChunkFromChunkPacketData(bytes.NewReader(raw), "1.21.11", 384)
+		if err != nil {
+			t.Skipf("decode failed: %v", err)
+			return
+		}
+
+		b := bytes.NewBuffer(nil)
+		err = chunk.WriteChunkData(b)
+		if err != nil {
+			t.Errorf("Failed to write chunk data: %v", err)
+			return
+		}
+		chunk2, err := level.ReadChunkFromChunkPacketData(bytes.NewReader(b.Bytes()), "1.21.11", 384)
+		if err != nil {
+			t.Errorf("Failed to decode chunk data: %v", err)
+			return
+		}
+		if !chunk.Equals(chunk2) {
+			t.Errorf("Chunks are not equal")
+			return
+		}
+	})
+}

level/section.go

@@ -4,6 +4,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"io"
+	"log/slog"
 	"math"
 
 	"github.com/admin-else/strom/data"
@@ -41,9 +42,10 @@ func MakeBiomeFormat(version string) StorageFormat {
 func MakeBlockFormat(version string) StorageFormat {
 	directBpe := uint8(math.Ceil(math.Log2(float64(len(data.BlocksForVersion(version))))))
 	return StorageFormat{
-		AvailableBpes: []uint8{0, 1, 2, 3, directBpe},
+		RedirectingBpes: []uint8{1, 2, 3},
-		BiggestDirect: true,
+		AvailableBpes:   []uint8{0, 4, 5, 6, 7, 8, directBpe},
-		Len:           BiomesPerChunkSection,
+		BiggestDirect:   true,
+		Len:             BlocksPerChunkSection,
 	}
 }
 
@@ -73,12 +75,22 @@ func ReadSectionStorage(r io.Reader, format StorageFormat) (s *Storage, err erro
 			err = BadPaletteLenErr
 			return
 		}
-		for range pallette {
+		for range paletteLen {
 			var v int32
 			v, err = proto_base.DecodeVarInt(r)
 			if err != nil {
 				return
 			}
+			if slog.Default().Enabled(nil, slog.LevelDebug) {
+				b, _ := data.LookupBlockByStateId("1.21.11", v)
+				if b == nil {
+					slog.Debug("unknown palette block in chunk packet", "id", v)
+				} else {
+					slog.Debug("palette block in chunk packet", "id", v, "name", b.Name)
+				}
+
+			}
+
 			pallette = append(pallette, v)
 		}
 	}
@@ -136,3 +148,16 @@ func SectionEncodePacket(w io.Writer, s Section) (err error) {
 	err = WriteSectionStorage(w, s.Biomes)
 	return
 }
+
+func SectionEquals(a, b Section) bool {
+	if a.BlockCount != b.BlockCount {
+		return false
+	}
+	if !a.Blocks.Equals(b.Blocks) {
+		return false
+	}
+	if !a.Biomes.Equals(b.Biomes) {
+		return false
+	}
+	return true
+}

level/section_test.go

@@ -1,43 +0,0 @@
-package level_test
-
-import (
-	"bytes"
-	"testing"
-
-	"github.com/admin-else/strom/level"
-)
-
-func FuzzSectionDeEnCodePacket(f *testing.F) {
-	f.Fuzz(func(t *testing.T, raw []byte) {
-		r := bytes.NewReader(raw)
-		s1, err := level.SectionDecodePacket(r, TestVersion)
-		if err != nil {
-			t.Skipf("decode failed: %v", err)
-			return
-		}
-
-		buf := bytes.NewBuffer(nil)
-		err = level.SectionEncodePacket(buf, s1)
-		if err != nil {
-			t.Fatal(err)
-			return
-		}
-
-		s2, err := level.SectionDecodePacket(buf, TestVersion)
-		if err != nil {
-			t.Fatal(err)
-			return
-		}
-
-		if s1.BlockCount != s2.BlockCount {
-			t.Error("BlockCount mismatch")
-		}
-		if level.CompareStorage(s1.Blocks, s2.Blocks) {
-			t.Error("Blocks mismatch")
-		}
-		if level.CompareStorage(s1.Biomes, s2.Biomes) {
-			t.Error("Biomes mismatch")
-		}
-
-	})
-}

level/storage.go

@@ -6,15 +6,17 @@ import (
 	"io"
 	"math"
 	"slices"
+
+	"github.com/admin-else/strom/util"
 )
 
 type StorageFormat struct {
-	AvailableBpes []uint8
+	RedirectingBpes []uint8 // block bpe 1 2 3 redirect to 4
-	BiggestDirect bool
+	AvailableBpes   []uint8
-	Len           int32
+	BiggestDirect   bool
+	Len             int32
 }
 
-var BpeNotAvailableErr = errors.New("bpe not available")
 var OutOfBoundsErr = errors.New("out of bounds")
 var AvailableBpeMustBeLenOneErr = errors.New("available bpe must be 1")
 var NoAvailableBpeErr = errors.New("no available bpe")
@@ -34,24 +36,16 @@ func (r StorageFormat) Import(data []uint64, bpe uint8, palette []int32) (s *Sto
 	s.resize(bpe)
 
 	if !slices.Contains(r.AvailableBpes, bpe) {
-		var newBpe uint8
+		if !slices.Contains(r.RedirectingBpes, bpe) {
-		for _, newBpe = range r.AvailableBpes {
+			err = NoAvailableBpeErr
-			if bpe < newBpe {
+			return
-				continue
-			}
 		}
-		var tmps *Storage
+		var found bool
-		tmps, err = StorageFormat{
+		bpe, found = util.GetNextLargestNumberSlice(bpe, r.AvailableBpes)
-			AvailableBpes: []uint8{bpe},
+		if !found {
-			BiggestDirect: false,
+			err = NoAvailableBpeErr
-			Len:           r.Len,
-		}.ImportDataDirect(data)
-		if err != nil {
 			return
 		}
-		tmps.resize(newBpe)
-		data = tmps.data
-		bpe = newBpe
 	}
 	if bpe == 0 && len(palette) != 1 {
 		err = Bpe0PalletMustBeLenOneErr
@@ -73,10 +67,6 @@ func (r StorageFormat) ImportDataDirect(data []uint64) (s *Storage, err error) {
 }
 
 func (r StorageFormat) ImportFromReader(reader io.Reader, bpe uint8, palette []int32) (s *Storage, err error) {
-	if !slices.Contains(r.AvailableBpes, bpe) {
-		err = BpeNotAvailableErr
-		return
-	}
 	var longs []uint64
 	if bpe == 0 {
 		return r.Import(longs, bpe, palette)
@@ -94,6 +84,10 @@ func (r StorageFormat) ImportFromReader(reader io.Reader, bpe uint8, palette []i
 	return r.Import(longs, bpe, palette)
 }
 
+func (r StorageFormat) FullWith(v int32) (s *Storage, err error) {
+	return r.Import(nil, 0, []int32{v})
+}
+
 type Storage struct {
 	format  StorageFormat
 	data    []uint64
@@ -242,14 +236,14 @@ func (r *Storage) Export() (data []uint64, bpe uint8, palette []int32, format St
 	return r.data, r.bpe, r.palette, r.format
 }
 
-func CompareStorage(a, b *Storage) bool {
+func (r *Storage) Equals(other *Storage) bool {
-	if a.format.Len != b.format.Len {
+	if r.format.Len != other.format.Len {
 		return false
 	}
-	for i := range a.format.Len {
+	for i := range r.format.Len {
-		av, _ := a.Get(i)
+		av, _ := r.Get(i)
-		bv, _ := b.Get(i)
+		bv, _ := other.Get(i)
-		if av == bv {
+		if av != bv {
 			return false
 		}
 	}

level/testdata/fuzz/FuzzReadChunkFromChunkPacketData/e8072e41f4d36fcd

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("00\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x020000000000000000000\x0200000000000000000\x0200000000000000000")

proto_generator/main.go

@@ -198,7 +198,7 @@ func FixPointerReceivers(e ast.Expr) (td ast.Expr, ret ast.Expr) {
 func (g *Generator) GenerateTypes(prefix string, types Types) (err error) {
 	g.CurrentlyGeneratingTypes = types
 	g.CurrentlyGeneratingTypesPrefix = prefix
-	for _, k := range OrderedKeys(types.Types) {
+	for _, k := range util.OrderedKeys(types.Types) {
 		v := types.Types[k]
 		g.Depth = 0
 		e, err2 := g.VisitType(v)
@@ -267,11 +267,11 @@ func (g *Generator) GenerateProtocol(protocol Protocol, version string) (err err
 	}
 
 	// Move common packet to their own areas to make packets more identifiable
-	for _, k := range OrderedKeys(protocol.Types.Types) {
+	for _, k := range util.OrderedKeys(protocol.Types.Types) {
 		v := protocol.Types.Types[k]
 		if strings.Contains(k, "packet") {
 			delete(protocol.Types.Types, k)
-			for _, prefix := range OrderedKeys(prefixTypeMap) {
+			for _, prefix := range util.OrderedKeys(prefixTypeMap) {
 				types := prefixTypeMap[prefix]
 				if prefix == "" {
 					continue
@@ -281,18 +281,18 @@ func (g *Generator) GenerateProtocol(protocol Protocol, version string) (err err
 		}
 	}
 
-	for _, prefix := range OrderedKeys(prefixTypeMap) {
+	for _, prefix := range util.OrderedKeys(prefixTypeMap) {
 		types := prefixTypeMap[prefix]
-		for _, k := range OrderedKeys(types.Types.Types) {
+		for _, k := range util.OrderedKeys(types.Types.Types) {
 			typeData := types.Types.Types[k]
 			if k != "packet" {
 				continue
 			}
 			packetIds := typeData.([]any)[1].([]any)[0].(map[string]any)["type"].([]any)[1].(map[string]any)["mappings"].(map[string]any)
-			packetIdsStrings := AssertAndConvertMapValues[string](packetIds)
+			packetIdsStrings := util.AssertAndConvertMapValues[string](packetIds)
-			packetIdsRev := ReverseMap(packetIdsStrings)
+			packetIdsRev := util.ReverseMap(packetIdsStrings)
 			v := typeData.([]any)[1].([]any)[1].(map[string]any)["type"].([]any)[1].(map[string]any)["fields"].(map[string]any)
-			for _, k2 := range OrderedKeys(v) {
+			for _, k2 := range util.OrderedKeys(v) {
 				v2 := v[k2].(string)
 				typeName := prefix + util.CamelCase(v2)
 				g.packetInfos = append(g.packetInfos, PacketInfo{
@@ -309,7 +309,7 @@ func (g *Generator) GenerateProtocol(protocol Protocol, version string) (err err
 		}
 	}
 
-	for _, prefix := range OrderedKeys(prefixTypeMap) {
+	for _, prefix := range util.OrderedKeys(prefixTypeMap) {
 		types := prefixTypeMap[prefix]
 		err = g.GenerateTypes(prefix, types.Types)
 		if err != nil {

proto_generator/native_compare_to.go

@@ -21,7 +21,7 @@ func (g *Generator) ParseCompareTo(compareTo string) (e ast.Expr, cet CaseExprTy
 	}
 	parts = parts[downPrefixCount:]
 	startingPoint := g.ContainerStack[len(g.ContainerStack)-downPrefixCount-1]
-	e, cet, err = g.VisitCompareTo(parts, startingPoint.VarToSet, CombineNamAndData("container", startingPoint.Data))
+	e, cet, err = g.VisitCompareTo(parts, startingPoint.VarToSet, util.CombineNamAndData("container", startingPoint.Data))
 	//	return ParseCompareToLegacy(compareTo, varToSet)
 	return
 }
@@ -46,7 +46,7 @@ func ContainerCompareTo(g *Generator, parts []string, inExpr ast.Expr, dataRaw a
 		return
 	}
 
-	name, parts, ok := PopFront(parts)
+	name, parts, ok := util.PopFront(parts)
 	if !ok {
 		err = errors.New("expected name part")
 		return
@@ -70,7 +70,7 @@ func BitfieldCompareTo(_ *Generator, parts []string, inExpr ast.Expr, dataRaw an
 	if err != nil {
 		return
 	}
-	fieldName, parts, found := PopFront(parts)
+	fieldName, parts, found := util.PopFront(parts)
 	if !found {
 		err = errors.New("expected name part")
 		return
@@ -98,7 +98,7 @@ func BitflagsCompareTo(_ *Generator, parts []string, inExpr ast.Expr, dataRaw an
 	if err != nil {
 		return
 	}
-	name, parts, ok := PopFront(parts)
+	name, parts, ok := util.PopFront(parts)
 	if !ok {
 		err = errors.New("expected name part")
 		return

proto_generator/native_decoder.go

@@ -249,7 +249,7 @@ func SwitchDecoder(g *Generator, varToSet ast.Expr, dataRaw any, name string) (s
 	if err != nil {
 		return
 	}
-	for _, fName := range OrderedKeys(data.Fields) {
+	for _, fName := range util.OrderedKeys(data.Fields) {
 		fType := data.Fields[fName]
 		e := MultiTypeFix(fName, &cet)
 		if e == nil {
@@ -324,7 +324,7 @@ func MapperDecoder(g *Generator, varToSet ast.Expr, dataRaw any, name string) (s
 	mName := name + "Map"
 
 	expressions := Exprs()
-	for _, k := range OrderedKeys(data.Mappings) {
+	for _, k := range util.OrderedKeys(data.Mappings) {
 		v := data.Mappings[k]
 		expressions = append(expressions, KeyValueExpr(NumLitStr(k), StrLit(v)))
 	}

proto_generator/native_encoder.go

@@ -201,7 +201,7 @@ func SwitchEncoder(g *Generator, varToSet ast.Expr, dataRaw any, name string) (s
 	if err != nil {
 		return
 	}
-	for _, fName := range OrderedKeys(data.Fields) {
+	for _, fName := range util.OrderedKeys(data.Fields) {
 		fType := data.Fields[fName]
 		e := MultiTypeFix(fName, &cet)
 		if e == nil {
@@ -275,7 +275,7 @@ func MapperEncoder(g *Generator, varToSet ast.Expr, dataRaw any, name string) (s
 	mName := name + "ReverseMap"
 
 	exprs := Exprs()
-	for _, k := range OrderedKeys(data.Mappings) {
+	for _, k := range util.OrderedKeys(data.Mappings) {
 		v := data.Mappings[k]
 		exprs = append(exprs, KeyValueExpr(StrLit(v), NumLitStr(k)))
 	}

util/math.go

@@ -2,10 +2,6 @@ package util
 
 import "math"
 
-func FloorDiv(a, b int32) int32 {
-	return (a - ((a%b + b) % b)) / b
-}
-
 func BpeByNum(x float64) uint8 {
 	return uint8(math.Ceil(math.Log2(x)))
 }

util/slices.go

@@ -1,87 +1,20 @@
 package util
 
-import (
-	"cmp"
-	"math/rand/v2"
-	"slices"
-)
-
-func ShuffleSlice[T any](s []T) (ret []T) {
-	rand.Shuffle(len(s), func(i, j int) { s[i], s[j] = s[j], s[i] })
-	return s
-}
-
-func CountSlice[T comparable](s []T, o T) (ret int) {
-	for _, v := range s {
-		if v == o {
-			ret++
-		}
-	}
-	return
-}
-
-func GetUniqueSlice[T comparable](s []T) (unique map[T]struct{}) {
-	unique = make(map[T]struct{})
-	for _, v := range s {
-		if _, ok := unique[v]; !ok {
-			unique[v] = struct{}{}
-		}
-	}
-	return
-}
-
-func CountUniqueSlice[T comparable](s []T) (ret int) {
-	return len(GetUniqueSlice(s))
-}
-
-func MakeSingleValuedSlice[T any](v T, n int) (ret []T) {
-	ret = make([]T, n)
-	for i := range ret {
-		ret[i] = v
-	}
-	return
-}
-
-func SetToSlice[T comparable](set map[T]struct{}) (slice []T) {
-	slice = make([]T, 0, len(set))
-	for k := range set {
-		slice = append(slice, k)
-	}
-	return
-}
-
-func SetToSliceOrdered[T cmp.Ordered](set map[T]struct{}) (slice []T) {
-	slice = SetToSlice(set)
-	slices.Sort(slice)
-	return
-}
-
-func MinSlice[T cmp.Ordered](s []T) (ret T) {
-	ret = s[0]
-	for _, v := range s[1:] {
-		if v < ret {
-			ret = v
-		}
-	}
-	return
-}
-
 type Numeric interface {
 	~int | ~int8 | ~int16 | ~int32 | ~int64 |
 		~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr |
 		~float32 | ~float64
 }
 
-func SliceRange[T Numeric](s, e T) (ret []T) {
-	if s > e {
-		return
-	}
-	for v := s; v < e; v++ {
-		ret = append(ret, v)
-	}
-	return
-}
-
 func IsLastElement[T comparable](s []T, e T) bool {
 	return len(s) > 0 && s[len(s)-1] == e
 }
+
+func GetNextLargestNumberSlice[T Numeric](a T, s []T) (ret T, found bool) {
+	for _, v := range s {
+		if v > a {
+			return v, true
+		}
+	}
+	return
+}

util/utils.go

@@ -1,4 +1,4 @@
-package main
+package util
 
 import (
 	"cmp"