Skip to content

crl-release-25.4: db: delete obsolete files before scheduling flushes, compactions #5448

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: crl-release-25.4
Choose a base branch
from
Open

crl-release-25.4: db: delete obsolete files before scheduling flushes, compactions #5448

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 9 additions & 16 deletions obsolete_files.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,23 +356,16 @@ func (d *DB) getDeletionPacerInfo() deletionPacerInfo {
return pacerInfo
}

// scanObsoleteFiles scans the filesystem for files that are no longer needed
// and adds those to the internal lists of obsolete files. Note that the files
// are not actually deleted by this method. A subsequent call to
// deleteObsoleteFiles must be performed. Must be not be called concurrently
// with compactions and flushes. db.mu must be held when calling this function.
// scanObsoleteFiles compares the provided directory listing to the set of
// known, in-use files to find files no longer needed and adds those to the
// internal lists of obsolete files. Note that the files are not actually
// deleted by this method. A subsequent call to deleteObsoleteFiles must be
// performed. Must be not be called concurrently with compactions and flushes
// and will panic if any are in-progress. db.mu must be held when calling this
// function.
func (d *DB) scanObsoleteFiles(list []string, flushableIngests []*ingestedFlushable) {
// Disable automatic compactions temporarily to avoid concurrent compactions /
// flushes from interfering. The original value is restored on completion.
disabledPrev := d.opts.DisableAutomaticCompactions
defer func() {
d.opts.DisableAutomaticCompactions = disabledPrev
}()
d.opts.DisableAutomaticCompactions = true

// Wait for any ongoing compaction to complete before continuing.
for d.mu.compact.compactingCount > 0 || d.mu.compact.downloadingCount > 0 || d.mu.compact.flushing {
d.mu.compact.cond.Wait()
if d.mu.compact.compactingCount > 0 || d.mu.compact.downloadingCount > 0 || d.mu.compact.flushing {
panic(errors.AssertionFailedf("compaction or flush in progress"))
}

liveFileNums := make(map[base.DiskFileNum]struct{})
Expand Down
85 changes: 39 additions & 46 deletions open.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -538,6 +538,45 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
}
d.mu.versions.visibleSeqNum.Store(d.mu.versions.logSeqNum.Load())

if !d.opts.ReadOnly {
// Write the current options to disk.
d.optionsFileNum = d.mu.versions.getNextDiskFileNum()
tmpPath := base.MakeFilepath(opts.FS, dirname, base.FileTypeTemp, d.optionsFileNum)
optionsPath := base.MakeFilepath(opts.FS, dirname, base.FileTypeOptions, d.optionsFileNum)

// Write them to a temporary file first, in case we crash before
// we're done. A corrupt options file prevents opening the
// database.
optionsFile, err := opts.FS.Create(tmpPath, vfs.WriteCategoryUnspecified)
if err != nil {
return nil, err
}
serializedOpts := []byte(opts.String())
if _, err := optionsFile.Write(serializedOpts); err != nil {
return nil, errors.CombineErrors(err, optionsFile.Close())
}
d.optionsFileSize = uint64(len(serializedOpts))
if err := optionsFile.Sync(); err != nil {
return nil, errors.CombineErrors(err, optionsFile.Close())
}
if err := optionsFile.Close(); err != nil {
return nil, err
}
// Atomically rename to the OPTIONS-XXXXXX path. This rename is
// guaranteed to be atomic because the destination path does not
// exist.
if err := opts.FS.Rename(tmpPath, optionsPath); err != nil {
return nil, err
}
if err := d.dataDir.Sync(); err != nil {
return nil, err
}

// Delete any obsolete files.
d.scanObsoleteFiles(ls, flushableIngests)
d.deleteObsoleteFiles(jobID)
}

// Register with the CompactionScheduler before calling
// d.maybeScheduleFlush, since completion of the flush can trigger
// compactions.
Expand All @@ -560,7 +599,6 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
d.mu.mem.queue[len(d.mu.mem.queue)-1].logNum = newLogNum
}
d.updateReadStateLocked(d.opts.DebugCheck)

if !d.opts.ReadOnly {
// If the Options specify a format major version higher than the
// loaded database's, upgrade it. If this is a new database, this
Expand All @@ -580,52 +618,7 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
return nil, err
}
}

// Write the current options to disk.
d.optionsFileNum = d.mu.versions.getNextDiskFileNum()
tmpPath := base.MakeFilepath(opts.FS, dirname, base.FileTypeTemp, d.optionsFileNum)
optionsPath := base.MakeFilepath(opts.FS, dirname, base.FileTypeOptions, d.optionsFileNum)

// Write them to a temporary file first, in case we crash before
// we're done. A corrupt options file prevents opening the
// database.
optionsFile, err := opts.FS.Create(tmpPath, vfs.WriteCategoryUnspecified)
if err != nil {
return nil, err
}
serializedOpts := []byte(opts.String())
if _, err := optionsFile.Write(serializedOpts); err != nil {
return nil, errors.CombineErrors(err, optionsFile.Close())
}
d.optionsFileSize = uint64(len(serializedOpts))
if err := optionsFile.Sync(); err != nil {
return nil, errors.CombineErrors(err, optionsFile.Close())
}
if err := optionsFile.Close(); err != nil {
return nil, err
}
// Atomically rename to the OPTIONS-XXXXXX path. This rename is
// guaranteed to be atomic because the destination path does not
// exist.
if err := opts.FS.Rename(tmpPath, optionsPath); err != nil {
return nil, err
}
if err := d.dataDir.Sync(); err != nil {
return nil, err
}
}

if !d.opts.ReadOnly {
// Get a fresh list of files, in case some of the earlier flushes/compactions
// have deleted some files.
ls, err := opts.FS.List(dirname)
if err != nil {
return nil, err
}
d.scanObsoleteFiles(ls, flushableIngests)
d.deleteObsoleteFiles(jobID)
}
// Else, nothing is obsolete.

d.mu.tableStats.cond.L = &d.mu.Mutex
d.mu.tableValidation.cond.L = &d.mu.Mutex
Expand Down
4 changes: 2 additions & 2 deletions open_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -499,10 +499,10 @@ func TestOpenAlreadyLocked(t *testing.T) {
func TestNewDBFilenames(t *testing.T) {
versions := map[FormatMajorVersion][]string{
internalFormatNewest: {
"000002.log",
"000003.log",
"LOCK",
"MANIFEST-000001",
"OPTIONS-000003",
"OPTIONS-000002",
"marker.format-version.000015.028",
"marker.manifest.000001.MANIFEST-000001",
},
Expand Down
20 changes: 10 additions & 10 deletions replay/testdata/corpus/findManifestStart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,10 @@ open
list-files build
----
build:
000002.log
000003.log
LOCK
MANIFEST-000001
OPTIONS-000003
OPTIONS-000002
marker.format-version.000001.013
marker.manifest.000001.MANIFEST-000001

Expand All @@ -56,12 +56,12 @@ flush
list-files build
----
build:
000002.log
000003.log
000004.log
000005.sst
LOCK
MANIFEST-000001
OPTIONS-000003
OPTIONS-000002
marker.format-version.000001.013
marker.manifest.000001.MANIFEST-000001

Expand All @@ -76,20 +76,20 @@ list-files build
----
build:
000005.sst
000008.log
000009.log
LOCK
MANIFEST-000001
MANIFEST-000007
OPTIONS-000009
MANIFEST-000008
OPTIONS-000007
marker.format-version.000001.013
marker.manifest.000002.MANIFEST-000007
marker.manifest.000002.MANIFEST-000008

delete-all build/MANIFEST-000007
----

find-manifest-start build
----
index: 0, offset: 0, error: nil
index: 1, offset: 87, error: nil

make-file build manifest 7
bf13d7161a00010114636f636b726f6163685f636f6d70617261746f7203
Expand Down Expand Up @@ -122,4 +122,4 @@ created

find-manifest-start build
----
index: 1, offset: 739, error: nil
index: 2, offset: 87, error: nil
8 changes: 4 additions & 4 deletions replay/testdata/corpus/high_read_amp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ open
list-files build
----
build:
000002.log
000003.log
LOCK
MANIFEST-000001
OPTIONS-000003
OPTIONS-000002
marker.format-version.000001.013
marker.manifest.000001.MANIFEST-000001

Expand Down Expand Up @@ -75,7 +75,7 @@ build:
LOCK
MANIFEST-000001
MANIFEST-000010
OPTIONS-000003
OPTIONS-000002
marker.format-version.000001.013
marker.manifest.000002.MANIFEST-000010

Expand All @@ -91,7 +91,7 @@ high_read_amp/checkpoint:
000008.log
000009.sst
MANIFEST-000010
OPTIONS-000003
OPTIONS-000002
marker.format-version.000001.013
marker.manifest.000001.MANIFEST-000010

Expand Down
10 changes: 5 additions & 5 deletions replay/testdata/corpus/simple
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ open
list-files build
----
build:
000002.log
000003.log
LOCK
MANIFEST-000001
OPTIONS-000003
OPTIONS-000002
marker.format-version.000001.013
marker.manifest.000001.MANIFEST-000001

Expand All @@ -23,12 +23,12 @@ flush
list-files build
----
build:
000002.log
000003.log
000004.log
000005.sst
LOCK
MANIFEST-000001
OPTIONS-000003
OPTIONS-000002
marker.format-version.000001.013
marker.manifest.000001.MANIFEST-000001

Expand All @@ -47,7 +47,7 @@ simple/checkpoint:
000004.log
000005.sst
MANIFEST-000001
OPTIONS-000003
OPTIONS-000002
marker.format-version.000001.013
marker.manifest.000001.MANIFEST-000001

Expand Down
12 changes: 6 additions & 6 deletions replay/testdata/corpus/simple_ingest
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ open-ingest-excise
list-files build
----
build:
000002.log
000003.log
LOCK
MANIFEST-000001
OPTIONS-000003
OPTIONS-000002
marker.format-version.000012.025
marker.manifest.000001.MANIFEST-000001

Expand All @@ -34,12 +34,12 @@ ingest-and-excised
list-files build
----
build:
000002.log
000003.log
000004.sst
000005.sst
LOCK
MANIFEST-000001
OPTIONS-000003
OPTIONS-000002
marker.format-version.000012.025
marker.manifest.000001.MANIFEST-000001

Expand All @@ -55,11 +55,11 @@ simple_ingest:
list-files simple_ingest/checkpoint
----
simple_ingest/checkpoint:
000002.log
000003.log
000004.sst
000005.sst
MANIFEST-000001
OPTIONS-000003
OPTIONS-000002
marker.format-version.000001.025
marker.manifest.000001.MANIFEST-000001

Expand Down
8 changes: 4 additions & 4 deletions replay/testdata/corpus/simple_val_sep
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ open-val-sep
list-files build
----
build:
000002.log
000003.log
LOCK
MANIFEST-000001
OPTIONS-000003
OPTIONS-000002
marker.format-version.000011.024
marker.manifest.000001.MANIFEST-000001

Expand Down Expand Up @@ -51,7 +51,7 @@ build:
LOCK
MANIFEST-000010
MANIFEST-000013
OPTIONS-000003
OPTIONS-000002
marker.format-version.000011.024
marker.manifest.000003.MANIFEST-000013

Expand All @@ -74,7 +74,7 @@ simple_val_sep/checkpoint:
000011.log
000012.sst
MANIFEST-000013
OPTIONS-000003
OPTIONS-000002
marker.format-version.000001.024
marker.manifest.000001.MANIFEST-000013

Expand Down
6 changes: 3 additions & 3 deletions replay/testdata/replay
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ tree
508 000007.sst
0 LOCK
133 MANIFEST-000001
2769 OPTIONS-000003
2769 OPTIONS-000002
0 marker.format-version.000001.013
0 marker.manifest.000001.MANIFEST-000001
simple/
Expand All @@ -21,11 +21,11 @@ tree
11 000004.log
480 000005.sst
85 MANIFEST-000001
2769 OPTIONS-000003
2769 OPTIONS-000002
0 marker.format-version.000001.013
0 marker.manifest.000001.MANIFEST-000001

cat build/OPTIONS-000003
cat build/OPTIONS-000002
----
----
[Version]
Expand Down
Loading