Skip to content

Commit fd70e1f

Browse files
Merge pull request #254 from codecrafters-io/aof-2
AOF (Stages 3-5)
2 parents 87c69cf + 5f60fb9 commit fd70e1f

37 files changed

+1976
-37
lines changed

go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@ go 1.24.0
55
toolchain go1.24.2
66

77
require (
8+
al.essio.dev/pkg/shellescape v1.6.0
89
github.com/codecrafters-io/tester-utils v0.4.15
910
github.com/dustin/go-humanize v1.0.1
1011
github.com/fatih/color v1.18.0
1112
github.com/hdt3213/rdb v1.2.0
13+
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
1214
github.com/stretchr/testify v1.10.0
1315
github.com/tidwall/pretty v1.2.1
1416
gopkg.in/yaml.v3 v3.0.1

go.sum

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
al.essio.dev/pkg/shellescape v1.6.0 h1:NxFcEqzFSEVCGN2yq7Huv/9hyCEGVa/TncnOOBBeXHA=
2+
al.essio.dev/pkg/shellescape v1.6.0/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
13
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
24
github.com/bytedance/sonic v1.12.1/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk=
35
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
@@ -17,8 +19,12 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
1719
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
1820
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
1921
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
22+
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
23+
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
2024
github.com/hdt3213/rdb v1.2.0 h1:wJgSW3A0Q28k/RuSKg7shvSj6+F9YsRAviMUDOuTSyI=
2125
github.com/hdt3213/rdb v1.2.0/go.mod h1:p2O7ep2/CDdaZt4gywZevL6Vdjash4+imZ0wpinogm8=
26+
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
27+
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
2228
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
2329
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
2430
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=

internal/aof_directory_creator.go

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
package internal
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"path/filepath"
7+
"strings"
8+
9+
encoder "github.com/codecrafters-io/redis-tester/internal/resp/encoder"
10+
value "github.com/codecrafters-io/redis-tester/internal/resp/value"
11+
"github.com/codecrafters-io/tester-utils/logger"
12+
"github.com/codecrafters-io/tester-utils/test_case_harness"
13+
"github.com/dustin/go-humanize/english"
14+
)
15+
16+
// AofDirectoryCreator is used to create an append-only directory
17+
// Redis uses the same name for mannifest and the append-only file
18+
// Eg. foo.manifest and foo.1.incr.aof
19+
// But since this is used for testing user's code, to ensure that the users actually read from the
20+
// manifest file and parse the append-only file name, AppendFilenameFlag and
21+
// AppendOnlyFilenameInManifest can be specified separately
22+
type AofDirectoryCreator struct {
23+
DataDirectory string // directory inside which Aof directory is created
24+
AppendDirName string // Value of appendonlydir flag
25+
AppendFileNameInFlag string // Value of appendfilename (Used for manifest file name)
26+
AppendOnlyFileNameInManifest string // Value appendfilename to be used inside manifest
27+
CommandsInsideAppendOnlyFile [][]string // slice of commands to be written to append-only file
28+
}
29+
30+
func (a *AofDirectoryCreator) Create(logger *logger.Logger) error {
31+
a.verifyMemberValues()
32+
33+
appendDirPath := filepath.Join(a.DataDirectory, a.AppendDirName)
34+
manifestFileName := a.AppendFileNameInFlag + ".manifest"
35+
manifestFilePath := filepath.Join(appendDirPath, manifestFileName)
36+
actualAppendFileName := fmt.Sprintf("%s.1.incr.aof", a.AppendOnlyFileNameInManifest)
37+
actualAppendFilePath := filepath.Join(appendDirPath, actualAppendFileName)
38+
manifestFileEntry := fmt.Sprintf("file %s seq 1 type i", actualAppendFileName)
39+
40+
if err := a.createAppendOnlyDirectory(logger, appendDirPath, manifestFileName, actualAppendFileName); err != nil {
41+
return err
42+
}
43+
44+
if err := a.createAppendOnlyFile(logger, actualAppendFilePath); err != nil {
45+
return err
46+
}
47+
48+
if err := a.createManifestFile(logger, manifestFilePath, manifestFileEntry); err != nil {
49+
return err
50+
}
51+
52+
return nil
53+
}
54+
55+
func (a *AofDirectoryCreator) createAppendOnlyDirectory(logger *logger.Logger, appendDirPath, manifestFileName, actualAppendFileName string) error {
56+
logger.Infof("Creating append-only directory %q:", a.AppendDirName)
57+
58+
logger.WithAdditionalSecondaryPrefix(a.AppendDirName, func() {
59+
logger.Infof(" - %s", manifestFileName)
60+
logger.Infof(" - %s", actualAppendFileName)
61+
})
62+
63+
if err := os.MkdirAll(appendDirPath, 0755); err != nil {
64+
return fmt.Errorf("Failed to create append-only directory %s: %w", appendDirPath, err)
65+
}
66+
67+
return nil
68+
}
69+
70+
func (a *AofDirectoryCreator) createAppendOnlyFile(logger *logger.Logger, actualAppendFilePath string) error {
71+
actualAppendFileName := filepath.Base(actualAppendFilePath)
72+
73+
if len(a.CommandsInsideAppendOnlyFile) > 0 {
74+
logger.Infof(
75+
"Writing %s to append-only file %q",
76+
english.Plural(len(a.CommandsInsideAppendOnlyFile), "command", "commands"),
77+
actualAppendFileName,
78+
)
79+
} else {
80+
logger.Infof("Creating empty append-only file %s", actualAppendFileName)
81+
}
82+
83+
var aofFileContents []byte
84+
85+
for _, command := range a.CommandsInsideAppendOnlyFile {
86+
commandRespBytes := a.encodeCommandAsRESPBytes(command)
87+
aofFileContents = append(aofFileContents, commandRespBytes...)
88+
89+
// Display the command as if it would be displayed using the quoted "%q" directive
90+
// But remove the surrounding quotes
91+
comandRespBytesFormatted := strings.Trim(
92+
fmt.Sprintf("%q", commandRespBytes),
93+
"\"",
94+
)
95+
96+
logger.WithAdditionalSecondaryPrefix(actualAppendFileName, func() {
97+
logger.Infof("%s", comandRespBytesFormatted)
98+
})
99+
}
100+
101+
if err := os.WriteFile(actualAppendFilePath, aofFileContents, 0o644); err != nil {
102+
return fmt.Errorf("Failed to create append-only file %s: %w", actualAppendFilePath, err)
103+
}
104+
105+
return nil
106+
}
107+
108+
func (a *AofDirectoryCreator) createManifestFile(logger *logger.Logger, manifestFilePath, manifestFileEntry string) error {
109+
manifestFileName := filepath.Base(manifestFilePath)
110+
111+
logger.Infof("Creating manifest file %q", manifestFileName)
112+
113+
logger.WithAdditionalSecondaryPrefix(manifestFileName, func() {
114+
logger.Infof("%s", manifestFileEntry)
115+
})
116+
117+
manifestFileRawBytes := manifestFileEntry + "\n"
118+
if err := os.WriteFile(manifestFilePath, []byte(manifestFileRawBytes), 0o644); err != nil {
119+
return fmt.Errorf("Failed to create manifest file %s: %w", manifestFilePath, err)
120+
}
121+
return nil
122+
}
123+
124+
func (a *AofDirectoryCreator) Cleanup(stageHarness *test_case_harness.TestCaseHarness) error {
125+
return os.RemoveAll(filepath.Join(a.DataDirectory, a.AppendDirName))
126+
}
127+
128+
func (a *AofDirectoryCreator) verifyMemberValues() {
129+
if a.DataDirectory == "" {
130+
panic("Codecrafters Internal Error - DataDirectory cannot be empty in AofDirectoryCreator")
131+
}
132+
133+
if a.AppendDirName == "" {
134+
panic("Codecrafters Internal Error - AppendDirName cannot be empty in AofDirectoryCreator")
135+
}
136+
137+
if a.AppendFileNameInFlag == "" {
138+
panic("Codecrafters Internal Error - AppendFileName cannot be empty in AofDirectoryCreator")
139+
}
140+
141+
if a.AppendOnlyFileNameInManifest == "" {
142+
panic("Codecrafters Internal Error - AppendOnlyFileNameInManifest cannot be empty in AofDirectoryCreator")
143+
}
144+
145+
for i, cmd := range a.CommandsInsideAppendOnlyFile {
146+
if len(cmd) == 0 {
147+
panic(
148+
fmt.Sprintf(
149+
"Codecrafters Internal Error - CommandsInsideAppendOnlyFile[%d] is empty in AofDirectoryCreator",
150+
i,
151+
),
152+
)
153+
}
154+
}
155+
}
156+
157+
// encodeCommandAsRESPBytes encodes a given command as RESP bytes to be written to the append-only file
158+
func (a *AofDirectoryCreator) encodeCommandAsRESPBytes(command []string) []byte {
159+
return encoder.Encode(value.NewStringArrayValue(command))
160+
}
Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
package filesystem_asserter
2+
3+
import (
4+
"sync"
5+
"time"
6+
7+
"github.com/codecrafters-io/redis-tester/internal/filesystem_assertion"
8+
"github.com/codecrafters-io/tester-utils/logger"
9+
)
10+
11+
type FilesystemAsserter struct {
12+
Timeout time.Duration
13+
assertions []filesystem_assertion.FilesystemAssertion
14+
}
15+
16+
func NewFilesystemAsserter(assertions []filesystem_assertion.FilesystemAssertion) *FilesystemAsserter {
17+
return &FilesystemAsserter{
18+
// Default timeout for FS asserter
19+
Timeout: 2 * time.Second,
20+
assertions: assertions,
21+
}
22+
}
23+
24+
// RunAssertions runs all assertions concurrently.
25+
// Each assertion is run until either it returns no error, or timeout expires
26+
// After either all assertions have returned no error, or timeout expires,
27+
// The accumulated success logs (of assertions which passed) are logged
28+
// If there are any errors, the first error from the a.assertions slice is returned to preserve order
29+
func (a *FilesystemAsserter) RunAssertions(logger *logger.Logger) error {
30+
if a.Timeout == 0 {
31+
panic("Codecrafters Internal Error - FilesystemAsserter: Timeout cannot be 0")
32+
}
33+
34+
outcomes := a.runAssertionsConcurrently(a.assertions)
35+
a.logAssertionResultLogs(logger, outcomes)
36+
return a.firstAssertionErrorInOrder(outcomes)
37+
}
38+
39+
// runAssertionsConcurrently runs each assertion in its own goroutine and fills outcomes by index.
40+
func (a *FilesystemAsserter) runAssertionsConcurrently(assertions []filesystem_assertion.FilesystemAssertion) []filesystem_assertion.FilesystemAssertionResult {
41+
outcomes := make([]filesystem_assertion.FilesystemAssertionResult, len(assertions))
42+
var waitGroup sync.WaitGroup
43+
44+
for assertionIndex, assertion := range assertions {
45+
waitGroup.Add(1)
46+
go func(idx int, assertion filesystem_assertion.FilesystemAssertion) {
47+
defer waitGroup.Done()
48+
outcomes[idx] = a.runAssertionUntilSuccessOrTimeout(assertion)
49+
}(assertionIndex, assertion)
50+
}
51+
52+
waitGroup.Wait()
53+
return outcomes
54+
}
55+
56+
// runAssertionUntilSuccessOrTimeout retries filesystem_assertion.FilesystemAssertion.Run until nil error is returned or deadline is reached.
57+
func (a *FilesystemAsserter) runAssertionUntilSuccessOrTimeout(assertion filesystem_assertion.FilesystemAssertion) filesystem_assertion.FilesystemAssertionResult {
58+
var lastResult filesystem_assertion.FilesystemAssertionResult
59+
deadline := time.Now().Add(a.Timeout)
60+
61+
for {
62+
result := assertion.Run()
63+
lastResult = result
64+
65+
if result.Err == nil {
66+
return result
67+
}
68+
69+
if time.Now().After(deadline) {
70+
return lastResult
71+
}
72+
73+
// Sleep 10ms instead of 1ms because fs operations can take longer
74+
time.Sleep(10 * time.Millisecond)
75+
}
76+
}
77+
78+
// logAssertionResultLogs logs non-empty success messages in slice order.
79+
func (a *FilesystemAsserter) logAssertionResultLogs(logger *logger.Logger, outcomes []filesystem_assertion.FilesystemAssertionResult) {
80+
for _, outcome := range outcomes {
81+
// If error is nil, log the success log of that outcome
82+
if outcome.Err == nil {
83+
for _, log := range outcome.Logs {
84+
log.LogMessageUsingLogger(logger)
85+
}
86+
}
87+
}
88+
89+
for _, outcome := range outcomes {
90+
if outcome.Err != nil {
91+
for _, log := range outcome.Logs {
92+
log.LogMessageUsingLogger(logger)
93+
}
94+
}
95+
}
96+
}
97+
98+
// firstAssertionErrorInOrder returns the first non-nil error in outcomes slice order, or nil if all passed.
99+
func (a *FilesystemAsserter) firstAssertionErrorInOrder(outcomes []filesystem_assertion.FilesystemAssertionResult) error {
100+
for _, outcome := range outcomes {
101+
if outcome.Err != nil {
102+
return outcome.Err
103+
}
104+
}
105+
return nil
106+
}

0 commit comments

Comments
 (0)