storage

level/anvil/chunk.go

@@ -0,0 +1,24 @@
+package anvil
+
+type Chunk struct {
+	DataVersion      int32
+	Heightmaps       map[string][]int64
+	InhabitedTime    int64
+	XPos, YPos, ZPos int32
+	Sections         []Section
+}
+
+type Section struct {
+	Y           int8
+	BlockStates struct {
+		Data    []int64
+		Palette []struct {
+			Name       string
+			Properties map[string]string `nbt:"properties,omitempty"`
+		}
+	}
+	Biomes []struct {
+		Data    []int64
+		Palette []string
+	}
+}

level/anvil/level_test.go

@@ -41,4 +41,6 @@ func TestRegionFile(t *testing.T) {
 	}
 
 	fmt.Println(string(util.MustT(json.MarshalIndent(n, "", "  "))))
+	fmt.Println("Chunk loaded successfully")
+
 }

level/packet.go

@@ -60,7 +60,7 @@ func PackArrayPallet(unique map[int32]struct{}, bpe uint8, data []int32, w io.Wr
 			return
 		}
 	}
-	var palletedData = make([]int32, len(data))
+	palletedData := make([]int32, len(data))
 	for i, entry := range data {
 		palletedData[i] = palletMap[entry]
 	}

level/storage.go

@@ -0,0 +1,112 @@
+package level
+
+import (
+	"errors"
+	"math"
+	"slices"
+)
+
+type Storage[T comparable] struct {
+	Palette                    []T
+	Data                       []uint64
+	PossibleBitsPerEntryValues []uint8 // These should be sorted
+	BitsPerEntry               uint8
+	Len                        int
+	MaxAtCurrentBpe            int
+	ElementsPerLong            int
+	Mask                       uint64
+}
+
+var CannotGrowPalletErr = errors.New("cannot grow pallet")
+
+func NewStorage[T comparable](len int, possibleBpe []uint8) (s *Storage[T]) {
+	s = new(Storage[T])
+	s.Len = len
+	s.PossibleBitsPerEntryValues = possibleBpe
+	s.Resize(possibleBpe[0])
+	return
+}
+
+func (s *Storage[T]) Resize(bpe uint8) {
+	if bpe == 0 {
+		s.Data = nil
+		s.BitsPerEntry = 0
+		s.ElementsPerLong = 0
+		s.Mask = 0
+		s.MaxAtCurrentBpe = 1
+		return
+	}
+	elementsPerLong := int(64 / int(bpe))
+	numberOfLongs := int(math.Ceil(float64(s.Len) / float64(elementsPerLong)))
+	newData := make([]uint64, numberOfLongs)
+	if s.BitsPerEntry == 0 {
+		s.Data = newData
+		s.BitsPerEntry = bpe
+		s.ElementsPerLong = elementsPerLong
+		s.Mask = (uint64(1) << uint32(bpe)) - 1
+		s.MaxAtCurrentBpe = 1 << s.BitsPerEntry
+		return
+	}
+	for i := range s.Len {
+		v := s.GetLow(i)
+		newData[i/elementsPerLong] |= uint64(v) << ((i % elementsPerLong) * int(bpe))
+	}
+	s.Data = newData
+	s.BitsPerEntry = bpe
+	s.ElementsPerLong = elementsPerLong
+	s.Mask = (uint64(1) << uint32(bpe)) - 1
+	s.MaxAtCurrentBpe = 1 << s.BitsPerEntry
+	return
+}
+
+func (s *Storage[T]) GrowPallet(v T) (index int, err error) {
+	index = len(s.Palette)
+	if index+1 > s.MaxAtCurrentBpe {
+		bpei := slices.Index(s.PossibleBitsPerEntryValues, s.BitsPerEntry)
+		if bpei == len(s.PossibleBitsPerEntryValues)-1 {
+			err = CannotGrowPalletErr
+			return
+		}
+		newBpe := s.PossibleBitsPerEntryValues[bpei+1]
+		s.Resize(newBpe)
+	}
+	s.Palette = append(s.Palette, v)
+	return
+}
+
+func (s *Storage[T]) Set(i int, v T) (err error) {
+	pi := slices.Index(s.Palette, v)
+	if pi != -1 {
+		if s.BitsPerEntry == 0 {
+			return
+		}
+		s.SetLow(i, uint32(pi))
+		return
+	}
+	pi, err = s.GrowPallet(v)
+	if err != nil {
+		return
+	}
+	if s.BitsPerEntry == 0 {
+		return
+	}
+	s.SetLow(i, uint32(pi))
+	return
+}
+
+func (s *Storage[T]) Get(i int) (v T) {
+	if s.BitsPerEntry == 0 {
+		return s.Palette[0]
+	}
+	return s.Palette[s.GetLow(i)]
+}
+
+func (s *Storage[T]) GetLow(i int) (j uint32) {
+	j = uint32((s.Data[i/s.ElementsPerLong] >> ((i % s.ElementsPerLong) * int(s.BitsPerEntry))) & s.Mask)
+	return
+}
+
+func (s *Storage[T]) SetLow(i int, v uint32) {
+	s.Data[i/s.ElementsPerLong] &= ^(s.Mask << ((i % s.ElementsPerLong) * int(s.BitsPerEntry)))
+	s.Data[i/s.ElementsPerLong] |= uint64(v) << ((i % s.ElementsPerLong) * int(s.BitsPerEntry))
+}

level/storage_test.go

@@ -0,0 +1,135 @@
+package level_test
+
+import (
+	"testing"
+
+	"github.com/admin-else/strom/level"
+)
+
+var possibleBpe = []uint8{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
+
+func FuzzStorage(f *testing.F) {
+	f.Add([]byte{4, 0, 1})
+	f.Add([]byte{4, 0, 1, 0, 2, 0, 1})
+	f.Add([]byte{8, 0, 10, 1, 20, 2, 30, 3, 40, 4, 50})
+	f.Add([]byte{4, 0, 5, 1, 10, 2, 15, 3, 20, 0, 25})
+	f.Add([]byte{2, 0, 1, 1, 2, 0, 3, 1, 4})
+	f.Add([]byte{8, 0, 42, 1, 42, 2, 42, 3, 42})
+
+	f.Fuzz(func(t *testing.T, data []byte) {
+		if len(data) < 1 {
+			t.Skip()
+		}
+
+		length := int(data[0])%64 + 1
+		s := level.NewStorage[int32](length, possibleBpe)
+
+		ref := make([]int32, length)
+		refSet := make([]bool, length)
+
+		for i := 1; i+1 < len(data); i += 2 {
+			idx := int(data[i]) % length
+			val := int32(data[i+1])
+
+			err := s.Set(idx, val)
+			if err == level.CannotGrowPalletErr {
+				continue
+			}
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			ref[idx] = val
+			refSet[idx] = true
+		}
+
+		for i := range length {
+			if !refSet[i] {
+				continue
+			}
+			got := s.Get(i)
+			if got != ref[i] {
+				t.Errorf("Get(%d) = %d, want %d", i, got, ref[i])
+			}
+		}
+	})
+}
+
+func TestSetOverwrite(t *testing.T) {
+	s := level.NewStorage[int32](8, []uint8{2, 4})
+
+	s.Set(0, 1)
+	s.Set(0, 2)
+	s.Set(0, 1)
+
+	got := s.Get(0)
+	if got != 1 {
+		t.Errorf("Get(0) = %d, want 1 (|= does not clear old bits)", got)
+	}
+}
+
+func TestResizePreservesData(t *testing.T) {
+	s := level.NewStorage[int32](8, []uint8{2, 4})
+
+	s.Set(0, 10)
+	s.Set(1, 20)
+	s.Set(2, 30)
+	s.Set(3, 40)
+	s.Set(4, 50)
+
+	for i, want := range []int32{10, 20, 30, 40} {
+		got := s.Get(i)
+		if got != want {
+			t.Errorf("Get(%d) = %d, want %d (Resize repacks with old BPE)", i, got, want)
+		}
+	}
+}
+
+func TestBpeZeroSingleValue(t *testing.T) {
+	s := level.NewStorage[int32](64, possibleBpe)
+	s.Palette = append(s.Palette, 42)
+
+	for i := range 64 {
+		if s.Get(i) != 42 {
+			t.Errorf("Get(%d) = %d, want 42 (BPE 0 should always return first palette entry)", i, s.Get(i))
+		}
+	}
+
+	s.Set(30, 42)
+	s.Set(0, 42)
+
+	for i := range 64 {
+		if s.Get(i) != 42 {
+			t.Errorf("Get(%d) = %d, want 42 after Set same value", i, s.Get(i))
+		}
+	}
+}
+
+func TestBpeZeroGrowPallet(t *testing.T) {
+	s := level.NewStorage[int32](64, possibleBpe)
+	s.Palette = append(s.Palette, 10)
+
+	s.Set(0, 10)
+	s.Set(1, 20)
+
+	if s.Get(0) != 10 {
+		t.Errorf("Get(0) = %d, want 10 after grow from BPE 0", s.Get(0))
+	}
+	if s.Get(1) != 20 {
+		t.Errorf("Get(1) = %d, want 20 after grow from BPE 0", s.Get(1))
+	}
+}
+
+func TestResizeElementsPerLong(t *testing.T) {
+	s := level.NewStorage[int32](8, []uint8{2, 4})
+
+	s.Set(0, 10)
+	s.Set(1, 20)
+	s.Set(2, 30)
+	s.Set(3, 40)
+	s.Set(4, 50)
+
+	if s.ElementsPerLong != 16 {
+		t.Errorf("ElementsPerLong = %d, want 16 (Resize stores old ElementsPerLong)", s.ElementsPerLong)
+	}
+}

util/slices.go

@@ -55,3 +55,13 @@ func SetToSliceOrdered[T cmp.Ordered](set map[T]struct{}) (slice []T) {
 	slices.Sort(slice)
 	return
 }
+
+func MinSlice[T cmp.Ordered](s []T) (ret T) {
+	ret = s[0]
+	for _, v := range s[1:] {
+		if v < ret {
+			ret = v
+		}
+	}
+	return
+}