matrixorigin
diff --git a/‎go.mod‎
Lines changed: 1 addition & 1 deletion b/‎go.mod‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎go.sum‎
Lines changed: 0 additions & 2 deletions b/‎go.sum‎
Lines changed: 0 additions & 2 deletions
diff --git a/‎pkg/common/const.go‎
Lines changed: 48 additions & 0 deletions b/‎pkg/common/const.go‎
Lines changed: 48 additions & 0 deletions
diff --git a/‎pkg/container/types/packer.go‎
Lines changed: 10 additions & 2 deletions b/‎pkg/container/types/packer.go‎
Lines changed: 10 additions & 2 deletions
diff --git a/‎pkg/fileservice/local_fs.go‎
Lines changed: 69 additions & 1 deletion b/‎pkg/fileservice/local_fs.go‎
Lines changed: 69 additions & 1 deletion
diff --git a/‎pkg/fileservice/local_fs_test.go‎
Lines changed: 57 additions & 0 deletions b/‎pkg/fileservice/local_fs_test.go‎
Lines changed: 57 additions & 0 deletions
diff --git a/‎pkg/frontend/compiler_context.go‎
Lines changed: 1 addition & 1 deletion b/‎pkg/frontend/compiler_context.go‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎pkg/frontend/variables.go‎
Lines changed: 9 additions & 0 deletions b/‎pkg/frontend/variables.go‎
Lines changed: 9 additions & 0 deletions
diff --git a/‎pkg/monlp/llm/mockllm.go‎
Lines changed: 1 addition & 1 deletion b/‎pkg/monlp/llm/mockllm.go‎
Lines changed: 1 addition & 1 deletion
@@ -24,7 +24,6 @@ require (
 	github.com/buger/jsonparser v1.1.1
 	github.com/bytedance/sonic v1.14.1
 	github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5
-	github.com/cespare/xxhash v1.1.0
 	github.com/cespare/xxhash/v2 v2.3.0
 	github.com/cockroachdb/errors v1.9.1
 	github.com/colinmarc/hdfs/v2 v2.4.0
@@ -110,6 +109,7 @@ require (
 	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/bytedance/gopkg v0.1.3 // indirect
 	github.com/bytedance/sonic/loader v0.3.0 // indirect
+	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cilium/ebpf v0.9.1 // indirect
 	github.com/clbanning/mxj v1.8.4 // indirect
 	github.com/cloudwego/base64x v0.1.6 // indirect
 
@@ -41,7 +41,6 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8=
 github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
-github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
 github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY=
@@ -771,7 +770,6 @@ github.com/smartystreets/assertions v1.13.1/go.mod h1:cXr/IwVfSo/RbCSPhoAPv73p3h
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
 github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 
@@ -0,0 +1,48 @@
+// Copyright 2024 Matrix Origin
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import "fmt"
+
+const (
+	KiB = 1024
+	MiB = 1024 * KiB
+	GiB = 1024 * MiB
+	TiB = 1024 * GiB
+	PiB = 1024 * TiB
+
+	THOUSAND    = 1000
+	MILLION     = 1000 * THOUSAND
+	BILLION     = 1000 * MILLION
+	TRILLION    = 1000 * BILLION
+	QUADRILLION = 1000 * TRILLION
+)
+
+func ConvertBytesToHumanReadable(bytes int64) string {
+	num := float64(bytes)
+	if bytes < KiB {
+		return fmt.Sprintf("%d bytes", bytes)
+	}
+	if bytes < MiB {
+		return fmt.Sprintf("%.2f KiB", num/KiB)
+	}
+	if bytes < GiB {
+		return fmt.Sprintf("%.2f MiB", num/MiB)
+	}
+	if bytes < TiB {
+		return fmt.Sprintf("%.2f GiB", num/GiB)
+	}
+	return fmt.Sprintf("%.2f TiB", num/TiB)
+}
@@ -40,7 +40,11 @@ var packerAllocator = malloc.NewShardedAllocator(
 )
 
 func NewPacker() *Packer {
-	bs, dec, err := packerAllocator.Allocate(4096, malloc.NoClear)
+	return NewPackerWithSize(4096)
+}
+
+func NewPackerWithSize(size uint64) *Packer {
+	bs, dec, err := packerAllocator.Allocate(size, malloc.NoClear)
 	if err != nil {
 		panic(err)
 	}
@@ -51,9 +55,13 @@ func NewPacker() *Packer {
 }
 
 func NewPackerArray(length int) []*Packer {
+	return NewPackerArrayWithSize(length, 4096)
+}
+
+func NewPackerArrayWithSize(length int, size uint64) []*Packer {
 	ret := make([]*Packer, 0, length)
 	for i := 0; i < length; i++ {
-		ret = append(ret, NewPacker())
+		ret = append(ret, NewPackerWithSize(size))
 	}
 	return ret
 }
 
@@ -18,7 +18,6 @@ import (
 	"bytes"
 	"context"
 	"errors"
-	"github.com/matrixorigin/matrixone/pkg/common/malloc"
 	"io"
 	"io/fs"
 	"iter"
@@ -31,6 +30,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/matrixorigin/matrixone/pkg/common/malloc"
+
 	"go.uber.org/zap"
 
 	"github.com/matrixorigin/matrixone/pkg/common/moerr"
@@ -1218,3 +1219,70 @@ func entryIsDir(path string, name string, entry fs.FileInfo) (bool, error) {
 	}
 	return false, nil
 }
+
+// open for read and write
+func (l *LocalFS) OpenFile(ctx context.Context, filePath string) (*os.File, error) {
+	err := ctx.Err()
+	if err != nil {
+		return nil, err
+	}
+
+	path, err := ParsePathAtService(filePath, l.name)
+	if err != nil {
+		return nil, err
+	}
+	nativePath := l.toNativeFilePath(path.File)
+	return os.OpenFile(nativePath, os.O_RDWR, 0644)
+}
+
+// create or truncate.
+func (l *LocalFS) CreateFile(ctx context.Context, filePath string) (*os.File, error) {
+	err := ctx.Err()
+	if err != nil {
+		return nil, err
+	}
+
+	path, err := ParsePathAtService(filePath, l.name)
+	if err != nil {
+		return nil, err
+	}
+	nativePath := l.toNativeFilePath(path.File)
+	return os.Create(nativePath)
+}
+
+// remove file
+func (l *LocalFS) RemoveFile(ctx context.Context, filePath string) error {
+	err := ctx.Err()
+	if err != nil {
+		return err
+	}
+
+	path, err := ParsePathAtService(filePath, l.name)
+	if err != nil {
+		return err
+	}
+	nativePath := l.toNativeFilePath(path.File)
+	return os.Remove(nativePath)
+}
+
+// open/create then immediately remove.   the opend file is good for read/write.
+func (l *LocalFS) CreateAndRemoveFile(ctx context.Context, filePath string) (*os.File, error) {
+	err := ctx.Err()
+	if err != nil {
+		return nil, err
+	}
+
+	path, err := ParsePathAtService(filePath, l.name)
+	if err != nil {
+		return nil, err
+	}
+	nativePath := l.toNativeFilePath(path.File)
+	f, err := os.Create(nativePath)
+	if err != nil {
+		return nil, err
+	}
+
+	// do not check error for this one
+	os.Remove(nativePath)
+	return f, nil
+}
@@ -278,3 +278,60 @@ func TestLocalFSEmptyRootPath(t *testing.T) {
 	assert.Nil(t, err)
 	assert.NotNil(t, fs)
 }
+
+func TestLocalFSOpenFile(t *testing.T) {
+	ctx := context.Background()
+	var counter perfcounter.CounterSet
+	ctx = perfcounter.WithCounterSet(ctx, &counter)
+	const (
+		n       = 32
+		dataLen = 32
+	)
+
+	// new fs
+	fs, err := NewLocalFS(
+		ctx,
+		"foo",
+		t.TempDir(),
+		CacheConfig{
+			DiskPath:                  ptrTo(t.TempDir()),
+			DiskCapacity:              ptrTo[toml.ByteSize](dataLen * n / 32),
+			enableDiskCacheForLocalFS: true,
+		},
+		nil,
+	)
+	assert.Nil(t, err)
+
+	f, err := fs.CreateFile(ctx, "aaa.txt")
+	assert.Nil(t, err)
+
+	f.Write([]byte("0123456789"))
+	f.Close()
+
+	f, err = fs.OpenFile(ctx, "aaa.txt")
+	assert.Nil(t, err)
+
+	buf := make([]byte, 10)
+	f.Read(buf)
+	assert.Equal(t, []byte("0123456789"), buf)
+	f.Close()
+
+	err = fs.RemoveFile(ctx, "aaa.txt")
+	assert.Nil(t, err)
+
+	// removed, open should fail.
+	f, err = fs.OpenFile(ctx, "aaa.txt")
+	assert.NotNil(t, err)
+
+	f2, err := fs.CreateAndRemoveFile(ctx, "aaa2.txt")
+	assert.Nil(t, err)
+	f2.Write([]byte("0123456789"))
+	nn, err := f2.ReadAt(buf, 0)
+	assert.Nil(t, err)
+	assert.Equal(t, 10, nn)
+	assert.Equal(t, []byte("0123456789"), buf)
+	f2.Close()
+
+	f2, err = fs.OpenFile(ctx, "aaa2.txt")
+	assert.NotNil(t, err)
+}
@@ -77,7 +77,7 @@ func (tcc *TxnCompilerContext) GetLowerCaseTableNames() int64 {
 	defer tcc.mu.Unlock()
 	val, err := tcc.execCtx.ses.GetSessionSysVar("lower_case_table_names")
 	if err != nil || val == nil {
-		val = int64(1)
+		return 1
 	}
 	return val.(int64)
 }
 
@@ -24,6 +24,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/matrixorigin/matrixone/pkg/common"
 	"github.com/matrixorigin/matrixone/pkg/common/moerr"
 	"github.com/matrixorigin/matrixone/pkg/config"
 	"github.com/matrixorigin/matrixone/pkg/container/types"
@@ -3801,6 +3802,14 @@ var gSysVarsDefs = map[string]SystemVariable{
 		Type:              InitSystemVariableStringType("mo_table_stats.reset_update_time"),
 		Default:           "",
 	},
+	"agg_spill_mem": {
+		Name:              "agg_spill_mem",
+		Scope:             ScopeBoth,
+		Dynamic:           true,
+		SetVarHintApplies: false,
+		Type:              InitSystemVariableIntType("agg_spill_mem", 0, common.TiB, false),
+		Default:           int64(common.GiB),
+	},
 }
 
 func updateTimeZone(ctx context.Context, sess *Session, sv *SystemVariables, name string, val interface{}) error {
 
@@ -17,7 +17,7 @@ package llm
 import (
 	"context"
 
-	"github.com/cespare/xxhash"
+	"github.com/cespare/xxhash/v2"
 	"github.com/matrixorigin/matrixone/pkg/common/moerr"
 )
Original file line number	Diff line number	Diff line change
`@@ -40,7 +40,11 @@ var packerAllocator = malloc.NewShardedAllocator(`
`40`	`40`	`)`
`41`	`41`
`42`	`42`	`func NewPacker() *Packer {`
`43`		`- bs, dec, err := packerAllocator.Allocate(4096, malloc.NoClear)`
	`43`	`+ return NewPackerWithSize(4096)`
	`44`	`+}`
	`45`	`+`
	`46`	`+func NewPackerWithSize(size uint64) *Packer {`
	`47`	`+ bs, dec, err := packerAllocator.Allocate(size, malloc.NoClear)`
`44`	`48`	`if err != nil {`
`45`	`49`	`panic(err)`
`46`	`50`	`}`
`@@ -51,9 +55,13 @@ func NewPacker() *Packer {`
`51`	`55`	`}`
`52`	`56`
`53`	`57`	`func NewPackerArray(length int) []*Packer {`
	`58`	`+ return NewPackerArrayWithSize(length, 4096)`
	`59`	`+}`
	`60`	`+`
	`61`	`+func NewPackerArrayWithSize(length int, size uint64) []*Packer {`
`54`	`62`	`ret := make([]*Packer, 0, length)`
`55`	`63`	`for i := 0; i < length; i++ {`
`56`		`- ret = append(ret, NewPacker())`
	`64`	`+ ret = append(ret, NewPackerWithSize(size))`
`57`	`65`	`}`
`58`	`66`	`return ret`
`59`	`67`	`}`
Original file line number	Diff line number	Diff line change
`@@ -77,7 +77,7 @@ func (tcc *TxnCompilerContext) GetLowerCaseTableNames() int64 {`
`77`	`77`	`defer tcc.mu.Unlock()`
`78`	`78`	`val, err := tcc.execCtx.ses.GetSessionSysVar("lower_case_table_names")`
`79`	`79`	`if err != nil \|\| val == nil {`
`80`		`- val = int64(1)`
	`80`	`+ return 1`
`81`	`81`	`}`
`82`	`82`	`return val.(int64)`
`83`	`83`	`}`
Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,7 @@ package llm`
`17`	`17`	`import (`
`18`	`18`	`"context"`
`19`	`19`
`20`		`- "github.com/cespare/xxhash"`
	`20`	`+ "github.com/cespare/xxhash/v2"`
`21`	`21`	`"github.com/matrixorigin/matrixone/pkg/common/moerr"`
`22`	`22`	`)`
`23`	`23`