refactor: rework decimal integration to conform to other types

Author: RaphDal

README.md

@@ -191,12 +191,11 @@ ingestion workload.
 ## Decimal columns
 
 QuestDB server version 9.2.0 and newer supports decimal columns with arbitrary precision and scale.
-The Go client converts supported decimal values to QuestDB's text/binary wire format automatically, pass any of the following to `DecimalColumn`:
+The Go client converts supported decimal values to QuestDB's text/binary wire format automatically:
 
-- `questdb.ScaledDecimal`, including helpers like `questdb.NewDecimalFromInt64` and `questdb.NewDecimal`.
-- Types implementing `questdb.DecimalMarshaler`.
-- `github.com/shopspring/decimal.Decimal` values or pointers.
-- `nil` or `questdb.NullDecimal()` to send a `NULL`.
+- `DecimalColumnScaled`: `questdb.ScaledDecimal`, including helpers like `questdb.NewDecimalFromInt64` and `questdb.NewDecimal`.
+- `DecimalColumnShopspring`: `github.com/shopspring/decimal.Decimal` values or pointers.
+- `DecimalColumnString`: `string` literals representing decimal values (validated at runtime).
 
 ```go
 price := qdb.NewDecimalFromInt64(12345, 2) // 123.45 with scale 2
@@ -205,17 +204,17 @@ commission := qdb.NewDecimal(big.NewInt(-750), 4) // -0.0750 with scale 4
 err = sender.
     Table("trades").
     Symbol("symbol", "ETH-USD").
-    DecimalColumn("price", price).
-    DecimalColumn("commission", commission).
+    DecimalColumnScaled("price", price).
+    DecimalColumnScaled("commission", commission).
     AtNow(ctx)
 ```
 
-To emit textual decimals, pass a validated string literal (without the trailing `d`—the client adds it):
+To emit textual decimals, pass a validated string literal:
 
 ```go
 err = sender.
     Table("quotes").
-    DecimalColumn("mid", "1.23456").
+    DecimalColumnString("mid", "1.23456").
     AtNow(ctx)
 ```
 

buffer.go

@@ -573,52 +573,70 @@ func (b *buffer) Float64Column(name string, val float64) *buffer {
 	return b
 }
 
-func (b *buffer) DecimalColumn(name string, val any) *buffer {
+func (b *buffer) DecimalColumnScaled(name string, val ScaledDecimal) *buffer {
 	if !b.prepareForField() {
 		return b
 	}
-	b.lastErr = b.writeColumnName(name)
-	if b.lastErr != nil {
+	return b.decimalColumnScaled(name, val)
+}
+
+func (b *buffer) decimalColumnScaled(name string, val ScaledDecimal) *buffer {
+	if err := val.ensureValidScale(); err != nil {
+		b.lastErr = err
 		return b
 	}
-	if str, ok := val.(string); ok {
-		if err := validateDecimalText(str); err != nil {
-			b.lastErr = err
-			return b
-		}
-		b.WriteByte('=')
-		b.WriteString(str)
-		b.WriteByte('d')
-		b.hasFields = true
+	if val.IsNull() {
+		// Don't write null decimals
+		return b
+	}
+	b.lastErr = b.writeColumnName(name)
+	if b.lastErr != nil {
 		return b
 	}
+	b.WriteByte('=')
+	b.WriteByte('=')
+	b.WriteByte(decimalBinaryTypeCode)
+	b.WriteByte((uint8)(val.scale))
+	b.WriteByte(32 - val.offset)
+	b.Write(val.unscaled[val.offset:])
+	b.hasFields = true
+	return b
+}
 
-	dec, err := normalizeDecimalValue(val)
-	if err != nil {
-		b.lastErr = err
+func (b *buffer) DecimalColumnString(name string, val string) *buffer {
+	if !b.prepareForField() {
 		return b
 	}
-	scale, payload, err := dec.toBinary()
-	if err != nil {
+	if err := validateDecimalText(val); err != nil {
 		b.lastErr = err
 		return b
 	}
-	if len(payload) == 0 {
-		// Don't write null decimals
+	b.lastErr = b.writeColumnName(name)
+	if b.lastErr != nil {
 		return b
 	}
 	b.WriteByte('=')
-	b.WriteByte('=')
-	b.WriteByte(decimalBinaryTypeCode)
-	b.WriteByte(scale)
-	b.WriteByte(byte(len(payload)))
-	if len(payload) > 0 {
-		b.Write(payload)
-	}
+	b.WriteString(val)
+	b.WriteByte('d')
 	b.hasFields = true
 	return b
 }
 
+func (b *buffer) DecimalColumnShopspring(name string, val ShopspringDecimal) *buffer {
+	if !b.prepareForField() {
+		return b
+	}
+	if val == nil {
+		return b
+	}
+	dec, err := convertShopspringDecimal(val)
+	if err != nil {
+		b.lastErr = err
+		return b
+	}
+	return b.decimalColumnScaled(name, dec)
+}
+
 func (b *buffer) Float64ColumnBinary(name string, val float64) *buffer {
 	if !b.prepareForField() {
 		return b

buffer_test.go

@@ -494,11 +494,14 @@ func TestFloat64ColumnBinary(t *testing.T) {
 	}
 }
 
-func TestDecimalColumnText(t *testing.T) {
+func TestDecimalColumnScaled(t *testing.T) {
+	negative, err := qdb.NewDecimal(big.NewInt(-12345), 3)
+	assert.NoError(t, err)
+
 	prefix := []byte(testTable + " price==")
 	testCases := []struct {
 		name     string
-		value    any
+		value    qdb.ScaledDecimal
 		expected []byte
 	}{
 		{
@@ -508,37 +511,140 @@ func TestDecimalColumnText(t *testing.T) {
 		},
 		{
 			name:     "negative",
-			value:    qdb.NewDecimal(big.NewInt(-12345), 3),
+			value:    negative,
 			expected: append(prefix, 0x17, 0x03, 0x02, 0xCF, 0xC7, 0x0A),
 		},
 		{
 			name:     "zero with scale",
 			value:    qdb.NewDecimalFromInt64(0, 4),
 			expected: append(prefix, 0x17, 0x04, 0x01, 0x0, 0x0A),
 		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			buf := newTestBuffer()
+			err := buf.Table(testTable).DecimalColumnScaled("price", tc.value).At(time.Time{}, false)
+			assert.NoError(t, err)
+			assert.Equal(t, tc.expected, buf.Messages())
+		})
+	}
+}
+
+func TestDecimalColumnScaledTrimmingAndPadding(t *testing.T) {
+	prefix := []byte(testTable + " price==")
+
+	testCases := []struct {
+		name          string
+		value         qdb.ScaledDecimal
+		expectedBytes []byte
+	}{
+		{
+			name:          "127 boundary",
+			value:         qdb.NewDecimalFromInt64(127, 0),
+			expectedBytes: []byte{0x17, 0x00, 0x01, 0x7F},
+		},
+		{
+			name:          "128 sign extension",
+			value:         qdb.NewDecimalFromInt64(128, 0),
+			expectedBytes: []byte{0x17, 0x00, 0x02, 0x00, 0x80},
+		},
+		{
+			name:          "255 sign extension",
+			value:         qdb.NewDecimalFromInt64(255, 0),
+			expectedBytes: []byte{0x17, 0x00, 0x02, 0x00, 0xFF},
+		},
+		{
+			name:          "32768 sign extension",
+			value:         qdb.NewDecimalFromInt64(32768, 0),
+			expectedBytes: []byte{0x17, 0x00, 0x03, 0x00, 0x80, 0x00},
+		},
 		{
-			name:     "null decimal",
-			value:    qdb.NullDecimal(),
-			expected: append(prefix, 0x17, 0x0, 0x0, 0x0A),
+			name:          "-1",
+			value:         qdb.NewDecimalFromInt64(-1, 0),
+			expectedBytes: []byte{0x17, 0x00, 0x01, 0xFF},
 		},
 		{
-			name:     "shopspring compatible",
-			value:    fakeShopspringDecimal{coeff: big.NewInt(123456), exp: -4},
-			expected: append(prefix, 0x17, 0x04, 0x03, 0x01, 0xE2, 0x40, 0x0A),
+			name:          "-2",
+			value:         qdb.NewDecimalFromInt64(-2, 0),
+			expectedBytes: []byte{0x17, 0x00, 0x01, 0xFE},
 		},
 		{
-			name:     "nil pointer treated as null",
-			value:    (*fakeShopspringDecimal)(nil),
-			expected: append(prefix, 0x17, 0x0, 0x0, 0x0A),
+			name:          "-127",
+			value:         qdb.NewDecimalFromInt64(-127, 0),
+			expectedBytes: []byte{0x17, 0x00, 0x01, 0x81},
+		},
+		{
+			name:          "-128",
+			value:         qdb.NewDecimalFromInt64(-128, 0),
+			expectedBytes: []byte{0x17, 0x00, 0x01, 0x80},
+		},
+		{
+			name:          "-129",
+			value:         qdb.NewDecimalFromInt64(-129, 0),
+			expectedBytes: []byte{0x17, 0x00, 0x02, 0xFF, 0x7F},
+		},
+		{
+			name:          "-256 sign extension",
+			value:         qdb.NewDecimalFromInt64(-256, 0),
+			expectedBytes: []byte{0x17, 0x00, 0x02, 0xFF, 0x00},
 		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			buf := newTestBuffer()
-			err := buf.Table(testTable).DecimalColumn("price", tc.value).At(time.Time{}, false)
+
+			err := buf.Table(testTable).DecimalColumnScaled("price", tc.value).At(time.Time{}, false)
 			assert.NoError(t, err)
-			assert.Equal(t, tc.expected, buf.Messages())
+
+			expected := append(append([]byte{}, prefix...), tc.expectedBytes...)
+			expected = append(expected, '\n')
+			assert.Equal(t, expected, buf.Messages())
+		})
+	}
+}
+
+func TestDecimalColumnShopspring(t *testing.T) {
+	prefix := []byte(testTable + " price==")
+
+	testCases := []struct {
+		name          string
+		value         fakeShopspringDecimal
+		expectedBytes []byte
+	}{
+		{
+			name:          "negative exponent scales value",
+			value:         fakeShopspringDecimal{coeff: big.NewInt(12345), exp: -2},
+			expectedBytes: []byte{0x17, 0x02, 0x02, 0x30, 0x39},
+		},
+		{
+			name:          "positive exponent multiplies coefficient",
+			value:         fakeShopspringDecimal{coeff: big.NewInt(123), exp: 2},
+			expectedBytes: []byte{0x17, 0x00, 0x02, 0x30, 0x0C},
+		},
+		{
+			name:          "positive value sign extension",
+			value:         fakeShopspringDecimal{coeff: big.NewInt(128), exp: 0},
+			expectedBytes: []byte{0x17, 0x00, 0x02, 0x00, 0x80},
+		},
+		{
+			name:          "negative value sign extension",
+			value:         fakeShopspringDecimal{coeff: big.NewInt(-12345), exp: -3},
+			expectedBytes: []byte{0x17, 0x03, 0x02, 0xCF, 0xC7},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			buf := newTestBuffer()
+
+			err := buf.Table(testTable).DecimalColumnShopspring("price", tc.value).At(time.Time{}, false)
+			assert.NoError(t, err)
+
+			expected := append(append([]byte{}, prefix...), tc.expectedBytes...)
+			expected = append(expected, '\n')
+			assert.Equal(t, expected, buf.Messages())
 		})
 	}
 }
@@ -561,7 +667,7 @@ func TestDecimalColumnStringValidation(t *testing.T) {
 		for _, tc := range testCases {
 			t.Run(tc.name, func(t *testing.T) {
 				buf := newTestBuffer()
-				err := buf.Table(testTable).DecimalColumn("price", tc.value).At(time.Time{}, false)
+				err := buf.Table(testTable).DecimalColumnString("price", tc.value).At(time.Time{}, false)
 				assert.NoError(t, err)
 				expected := []byte(testTable + " price=" + tc.expected + "\n")
 				assert.Equal(t, expected, buf.Messages())
@@ -585,7 +691,7 @@ func TestDecimalColumnStringValidation(t *testing.T) {
 		for _, tc := range testCases {
 			t.Run(tc.name, func(t *testing.T) {
 				buf := newTestBuffer()
-				err := buf.Table(testTable).DecimalColumn("price", tc.value).At(time.Time{}, false)
+				err := buf.Table(testTable).DecimalColumnString("price", tc.value).At(time.Time{}, false)
 				assert.Error(t, err)
 				assert.Contains(t, err.Error(), "decimal")
 				assert.Empty(t, buf.Messages())
@@ -598,30 +704,20 @@ func TestDecimalColumnErrors(t *testing.T) {
 	t.Run("invalid scale", func(t *testing.T) {
 		buf := newTestBuffer()
 		dec := qdb.NewDecimalFromInt64(1, 100)
-		err := buf.Table(testTable).DecimalColumn("price", dec).At(time.Time{}, false)
+		err := buf.Table(testTable).DecimalColumnScaled("price", dec).At(time.Time{}, false)
 		assert.ErrorContains(t, err, "decimal scale")
 		assert.Empty(t, buf.Messages())
 	})
 
 	t.Run("overflow", func(t *testing.T) {
-		buf := newTestBuffer()
 		bigVal := new(big.Int).Lsh(big.NewInt(1), 2100)
-		dec := qdb.NewDecimal(bigVal, 0)
-		err := buf.Table(testTable).DecimalColumn("price", dec).At(time.Time{}, false)
-		assert.ErrorContains(t, err, "exceeds 127-bytes limit")
-		assert.Empty(t, buf.Messages())
-	})
-
-	t.Run("unsupported type", func(t *testing.T) {
-		buf := newTestBuffer()
-		err := buf.Table(testTable).DecimalColumn("price", struct{}{}).At(time.Time{}, false)
-		assert.ErrorContains(t, err, "unsupported decimal column value type")
-		assert.Empty(t, buf.Messages())
+		_, err := qdb.NewDecimal(bigVal, 0)
+		assert.ErrorContains(t, err, "exceeds 32 bytes")
 	})
 
 	t.Run("no column", func(t *testing.T) {
 		buf := newTestBuffer()
-		err := buf.Table(testTable).DecimalColumn("price", qdb.NullDecimal()).At(time.Time{}, false)
+		err := buf.Table(testTable).DecimalColumnShopspring("price", nil).At(time.Time{}, false)
 		assert.ErrorContains(t, err, "no symbols or columns were provided: invalid message")
 		assert.Empty(t, buf.Messages())
 	})