packos

packos is a binary packing protocol with offset-indexed framing, designed for fast composition, random access, and reliable for RPC or BadgerDB workloads with small blobs(<8kb)(the size limitation maybe lifted to make it suitable for all purpose) It supports: All packing paths emit canonical headers, preserve offset domains, and support recursive structures. PAOSP ensures schema validation, GC neutrality, and reproducible output across variants.

• Canonical encoding Emits offset-indexed binary frames with consistent headers and type tags.
• Sequential decoding Supports single-pass traversal with offset tracking and teardown-safe accessors.
• Random access decoding Enables direct lookup via offset domains without full unpacking.
• Recursive structure support Handles nested maps, slices, and tagged frames with offset provenance.
• Simple schema validation Validates type tags, offsets, and structure boundaries during decode.
• GC neutrality Avoids retained slices and ensures allocation discipline across packing paths.
• Reproducible output Guarantees stable binary layout across runs and variants.
• General-purpose container support Supports arbitrarily large maps, slices, and tagged frames without size limits.
• Generate big and nested complex structures above 8kb limit

Encoding Example

put := packos.NewPutAccessFromPool()
put.AddInt16(42)                 // 2 bytes
put.AddBool(true)                // 1 byte
put.AddString("go")              // 2-byte length + 2 bytes
put.AddBytes([]byte{0xAA, 0xBB}) // 2-byte length + 2 bytes

actual := put.Pack()
ReleasePutAccess(put)

expected := []byte{
	// Headers (5 × 2 bytes)
	0x51, 0x00, // header[0]: absolute offset=10, TypeInt16
	0x15, 0x00, // header[1]: delta=2, TypeBool
	0x1E, 0x00, // header[2]: delta=3, TypeString ("go")
	0x2E, 0x00, // header[3]: delta=5, TypeString (bytes)
	0x38, 0x00, // header[4]: delta=7, TypeEnd

	// Payload (7 bytes)
	0x2A, 0x00, // int16(42)
	0x01,       // bool(true)
	0x67, 0x6F, // "go"
	0xAA, 0xBB, // bytes
}

actual := Pack(
	PackInt16(12345),
	PackMapSorted{
		"meta": PackMapSorted{
			"user": PackByteArray([]byte("alice")),
			"role": PackByteArray([]byte("admin")),
		},
		"name": PackString("gopher"),
	},
)

expected := []byte{
	// Outer Header Block
	0x31, 0x00, // offset = 6, TypeInt16
	0x17, 0x00, // delta = 2, TypeMap
	0xB0, 0x01, // delta = 54, TypeEnd

	// Outer Payload
	0x39, 0x30, // int16(12345)

	// inner1 Header Block
	0x56, 0x00, // "meta"
	0x27, 0x00, // map
	0x06, 0x01, // "name"
	0x26, 0x01, // "gopher"
	0x50, 0x01, // TypeEnd

	// inner1 Payload
	'm', 'e', 't', 'a',

	// inner1.1 Header Block
	0x56, 0x00, // "role"
	0x26, 0x00, // "admin"
	0x4E, 0x00, // "user"
	0x6E, 0x00, // "alice"
	0x90, 0x00, // TypeEnd

	// inner1.1 Payload
	'r', 'o', 'l', 'e',
	'a', 'd', 'm', 'i', 'n',
	'u', 's', 'e', 'r',
	'a', 'l', 'i', 'c', 'e',

	// Remaining inner1 Payload
	'n', 'a', 'm', 'e',
	'g', 'o', 'p', 'h', 'e', 'r',	
}

actual := Pack(
	PackTuple(
		PackInt32(2025),
		PackBool(false),
		PackString("az"),
	),
	PackTuple(
		PackInt16(7),
		PackBool(true),
		PackString("go"),
	),
)
expected := []byte{
	// Outer headers (3 × 2 bytes)
	0x34, 0x00, // header[0]: absolute offset = 6,  type=4 → Tuple   @ offset 0 → payload @ offset 6
	0x7C, 0x00, // header[1]: delta = 15,           type=4 → Tuple   @ offset 2 → payload @ offset 21 (6 + 15)
	0xE0, 0x00, // header[2]: delta = 28,           type=0 → End     @ offset 4 → marks end @ offset 34 (6 + 28)
	// Tuple 1 headers (4 × 2 bytes)                                 @ offset 6
	0x41, 0x00, // header[0]: absolute offset = 8,  type=1 → Int32   @ offset 6  → inner_offset 8
	0x25, 0x00, // header[1]: delta = 4,            type=5 → Bool    @ offset 8  → inner_offset 12 (8 + 4)
	0x2E, 0x00, // header[2]: delta = 5,            type=6 → String  @ offset 10 → inner_offset 13 (8 + 5)
	0x38, 0x00, // header[3]: delta = 7,            type=0 → End     @ offset 12 → inner_offset 15 (8 + 7)
	// Tuple 1 payload (7 bytes)
	0xE9, 0x07, 0x00, 0x00, // int32(2025)                           @ offset 14 → inner_offset 8
	0x00,       // bool(false)                                       @ offset 18 → inner_offset 12
	0x61, 0x7A, // "az"                                              @ offset 19 → inner_offset 13
	// Tuple 2 headers (4 × 2 bytes)                                 @ offset 21
	0x41, 0x00, // header[0]: absolute offset = 8,  type=1 → Int16   @ offset 21 → inner_offset 8
	0x15, 0x00, // header[1]: delta = 2,            type=5 → Bool    @ offset 23 → inner_offset 10 (8 + 2)
	0x1E, 0x00, // header[2]: delta = 3,            type=6 → String  @ offset 25 → inner_offset 11 (8 + 3)
	0x28, 0x00, // header[3]: delta = 5,            type=0 → End     @ offset 27 → inner_offset 13 (8 + 5)
	// Tuple 2 payload (5 bytes)
	0x07, 0x00, // int16(7)                                          @ offset 29 → inner_offset 8
	0x01,       // bool(true)                                        @ offset 31 → inner_offset 10
	0x67, 0x6F, // "go"                                              @ offset 32 → inner_offset 11
	//                                              final byte       @ offset 34 → inner_offset 13
}

Scheme examples

actual := pack.Pack(
	pack.PackInt16(12345),
	pack.PackFloat32(3.14),
	pack.PackInt64(9876543210),
	pack.PackBool(true),
	pack.PackMapSorted{
		"meta": pack.PackMapSorted{
			"user": pack.PackByteArray([]byte("alice")),
			"role": pack.PackByteArray([]byte("admin")),
		},
		"name": pack.PackString("gopher"),
	},
)

chain := SChain(
	SInt16,
	SFloat32,
	SInt64,
	SBool,
	//will match in written order
	SMap(
		SStringExact("meta"), // key
		SMap(                     // → value
			SStringExact("role"), // key
			SBytes(len("admin")), // → value
			SStringExact("user"),
			SBytes(len("alice")),
		),
		SStringExact("name"),   // key
		SStringLen(len("gopher")), // → value
	),
)

err := ValidateBuffer(actual, chain)

actual := pack.Pack(
	pack.PackInt16(12345),
	pack.PackFloat32(3.14),
	pack.PackInt64(9876543210),
	pack.PackBool(true),
	pack.PackMapSorted{
		"meta": pack.PackMapSorted{
			"user": pack.PackByteArray([]byte("alice")), // valid
			"role": pack.PackString("admin"),            // valid
			"age":  pack.PackInt32(30),                  // valid
		},
		"name": pack.PackString("gopher"), // valid
	},
)

chain := SChain(
	SInt16,
	SFloat32,
	SInt64,
	SBool,
	SMap(
		SString.Match("meta"),
		SMapUnordered(map[string]Scheme{
			"age":  SInt32.Range(18, 99),               // declared first
			"user": SBytes(len("alice")),               // declared second
			"role": SString.Pattern(`^(admin|guest)$`), // declared third
		}),
		SString.Match("name"),
		SString.WithWidth(len("gopher")),
	),
)

err := ValidateBuffer(actual, chain)

import . "github.com/BranchAndLink/packos/packable"

/////////////////////////////////////////////////////


actual := Pack(
	PackString("2025-09-10"),        // date
	PackInt32(42),                   // range
	PackString("[email protected]"), // email
	PackString("prefix-hello"),      // prefix
	PackString("world-suffix"),      // suffix
)

chain := SChain(
	SString.Pattern(`^\d{4}-\d{2}-\d{2}$`),                              // date pattern YYYY-MM-DD
	SInt32.Range(1, 100),                                                // int range
	SString.Pattern(`^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$`), // email
	SString.Prefix("prefix-"),                                           // prefix match
	SString.Suffix("-suffix"),                                           // suffix match
)

err := ValidateBuffer(actual, chain)