diff --git a/.gitignore b/.gitignore index 22e416ab6..b6c703ee1 100644 --- a/.gitignore +++ b/.gitignore @@ -224,3 +224,5 @@ FakesAssemblies/ # Uncomment if necessary however generally it will be regenerated when needed #!**/packages/repositories.config /LiteDB.BadJsonTest +/LiteDB/Properties/PublishProfiles/FolderProfile.pubxml +/LiteDB/README.md diff --git a/LiteDB.Tests/Issues/Issue2504_Tests.cs b/LiteDB.Tests/Issues/Issue2504_Tests.cs new file mode 100644 index 000000000..6240db931 --- /dev/null +++ b/LiteDB.Tests/Issues/Issue2504_Tests.cs @@ -0,0 +1,112 @@ +using System; +using System.IO; +using System.Linq; + +using Xunit; + +namespace LiteDB.Tests.Issues +{ + public class Person + { + [BsonId] + public int Id { get; set; } + public string Name { get; set; } + + public Person(int id, string name) + { + Id = id; + Name = name; + } + } + + public class Issue2504_Tests + { + private static string CreateCorruptedDatabase() + { + using var memoryStream = new MemoryStream(); + using (var db = new LiteDatabase(memoryStream)) + { + var col1 = db.GetCollection("col1"); + col1.Insert(new Person(1, "Alpha")); + var col2 = db.GetCollection("col2"); + col2.Insert(new Person(2, "Beta")); + db.DropCollection("col2"); + } + + // 2) Zmień typ wszystkich pustych stron na Data (4) + var bytes = memoryStream.ToArray(); + const int pageSize = 8192; + const int PAGE_TYPE_OFFSET = 4; + const byte PAGE_TYPE_EMPTY = 0; + const byte PAGE_TYPE_DATA = 4; + + for (int offset = 0; offset + pageSize <= bytes.Length; offset += pageSize) + { + if (bytes[offset + PAGE_TYPE_OFFSET] == PAGE_TYPE_EMPTY) + { + bytes[offset + PAGE_TYPE_OFFSET] = PAGE_TYPE_DATA; + } + } + + var tempPath = Path.Combine(Path.GetTempPath(), $"LiteDB_Issue2504_{Guid.NewGuid():N}.db"); + File.WriteAllBytes(tempPath, bytes); + return tempPath; + } + + [Fact] + public void AutoRebuild_Disabled_ShouldThrow() + { + var dbPath = CreateCorruptedDatabase(); + var backupPath = dbPath + "-backup"; + + try + { + using var db = new LiteDatabase(dbPath); + var col1 = db.GetCollection("col1"); + var bulk = Enumerable.Range(3, 5_000).Select(i => new Person(i, "Gamma")); + var ex = Record.Exception(() => col1.InsertBulk(bulk)); + Assert.NotNull(ex); + Assert.False(File.Exists(backupPath)); + } + finally + { + if (File.Exists(backupPath)) File.Delete(backupPath); + if (File.Exists(dbPath)) File.Delete(dbPath); + } + } + + [Fact] + public void AutoRebuild_Enabled_ShouldRecover() + { + string dbPath = CreateCorruptedDatabase(); + string backupPath = dbPath + "-backup"; + try + { + using (var db = new LiteDatabase($"Filename={dbPath};AutoRebuild=true")) + { + var col1 = db.GetCollection("col1"); + + var bulk = Enumerable.Range(3, 500).Select(i => new Person(i, "Gamma")); + col1.InsertBulk(bulk); + + var allDocs = col1.Query().ToList(); + Assert.Contains(allDocs, x => x.Name == "Alpha"); + Assert.True(allDocs.Count >= 2); + Assert.False(db.CollectionExists("col2")); + if (db.CollectionExists("_rebuild_errors")) + { + var rebuildErrors = db.GetCollection("_rebuild_errors"); + Assert.True(rebuildErrors.Count() > 0, "Rebuild errors should be logged due to corruption"); + } + } + Assert.True(File.Exists(backupPath), "Backup should exist when AutoRebuild has executed"); + + } + finally + { + if (File.Exists(backupPath)) File.Delete(backupPath); + if (File.Exists(dbPath)) File.Delete(dbPath); + } + } + } +} diff --git a/LiteDB.Tests/Issues/Issue2525_Tests.cs b/LiteDB.Tests/Issues/Issue2525_Tests.cs new file mode 100644 index 000000000..b52ebd02b --- /dev/null +++ b/LiteDB.Tests/Issues/Issue2525_Tests.cs @@ -0,0 +1,157 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using LiteDB; +using LiteDB.Engine; +using Xunit; + +namespace LiteDB.Tests.Issues +{ + public class Issue2525_Tests + { + private static string NewTempDbPath() + => Path.Combine(Path.GetTempPath(), $"LiteDB_FindAllLoop_{Guid.NewGuid():N}.db"); + + private static LiteEngine NewEngine(string path) + => new LiteEngine(new EngineSettings { Filename = path }); + + private static void InsertPeopleWithEngine(LiteEngine engine, string collection, IEnumerable<(int id, string name)> rows) + { + var docs = rows.Select(r => new BsonDocument { ["_id"] = r.id, ["name"] = r.name }); + engine.Insert(collection, docs, BsonAutoId.Int32); + } + + private static void CreateIndexSelfLoop(LiteEngine engine, string collection, string indexName) + { + engine.BeginTrans(); + var monitorField = typeof(LiteEngine).GetField("_monitor", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + var monitor = (TransactionMonitor)monitorField.GetValue(engine); + + var tx = monitor.GetTransaction(create: false, queryOnly: false, out _); + Assert.NotNull(tx); + + var snapshot = tx.CreateSnapshot(LockMode.Write, collection, addIfNotExists: false); + Assert.NotNull(snapshot); + Assert.NotNull(snapshot.CollectionPage); + + CollectionIndex ci = indexName == "_id" + ? snapshot.CollectionPage.PK + : snapshot.CollectionPage.GetCollectionIndex(indexName); + + Assert.NotNull(ci); + + var headPage = snapshot.GetPage(ci.Head.PageID); + var headNode = headPage.GetIndexNode(ci.Head.Index); + var firstAddr = headNode.Next[0]; + Assert.False(firstAddr.IsEmpty); + var firstPage = snapshot.GetPage(firstAddr.PageID); + var firstNode = firstPage.GetIndexNode(firstAddr.Index); + firstNode.SetNext(0, firstNode.Position); + tx.Commit(); + } + + [Fact] + public void PK_Loop_Should_Throw_On_EnsureIndex() + { + var path = NewTempDbPath(); + try + { + using (var engine = NewEngine(path)) + { + InsertPeopleWithEngine(engine, "col", new[] + { + (1, "a"), + (2, "b"), + (3, "c") + }); + } + + using (var engine = NewEngine(path)) + { + CreateIndexSelfLoop(engine, "col", "_id"); + } + + using (var db = new LiteDatabase(path)) + { + var col = db.GetCollection("col"); + var ex = Record.Exception(() => + { + col.EnsureIndex("name"); // albo Find(Query.All("name")), zależnie od testu + }); + + Assert.NotNull(ex); + Assert.Contains("Detected loop in FindAll", ex.Message, StringComparison.OrdinalIgnoreCase); + + } + } + finally + { + if (File.Exists(path)) File.Delete(path); + } + } + + [Fact] + public void Secondary_Index_Loop_Should_Throw_On_Query_Then_Rebuild_Fixes() + { + var path = NewTempDbPath(); + var backup = path + "-backup"; + try + { + using (var engine = NewEngine(path)) + { + InsertPeopleWithEngine(engine, "col", new[] + { + (1, "a"), + (2, "b"), + (3, "c") + }); + } + + using (var db = new LiteDatabase(path)) + { + var col = db.GetCollection("col"); + col.EnsureIndex("name"); + } + + using (var engine = NewEngine(path)) + { + CreateIndexSelfLoop(engine, "col", "name"); + } + + using (var db = new LiteDatabase(path)) + { + var ex = Record.Exception(() => + { + var _ = db.GetCollection("col").Query().OrderBy("name").ToList(); + }); + + Assert.NotNull(ex); + Assert.Contains("Detected loop in FindAll", ex.Message, StringComparison.OrdinalIgnoreCase); + + } + + using (var db = new LiteDatabase(path)) + { + db.Rebuild(); + } + + using (var db = new LiteDatabase(path)) + { + var col = db.GetCollection("col"); + var list = col.Find(Query.All("name")).ToList(); + + Assert.Equal(3, list.Count); + Assert.Equal(new[] { "a", "b", "c" }, list.Select(x => x["name"].AsString).OrderBy(x => x).ToArray()); + } + + if (File.Exists(backup)) File.Delete(backup); + } + finally + { + if (File.Exists(path)) File.Delete(path); + if (File.Exists(backup)) File.Delete(backup); + } + } + } +} diff --git a/LiteDB/Client/Structures/ConnectionString.cs b/LiteDB/Client/Structures/ConnectionString.cs index 945649764..ee4cdcd4b 100644 --- a/LiteDB/Client/Structures/ConnectionString.cs +++ b/LiteDB/Client/Structures/ConnectionString.cs @@ -1,7 +1,9 @@ using LiteDB.Engine; + using System; using System.Collections.Generic; using System.Globalization; + using static LiteDB.Constants; namespace LiteDB @@ -85,7 +87,7 @@ public ConnectionString(string connectionString) this.Password = _values.GetValue("password", this.Password); - if(this.Password == string.Empty) + if (this.Password == string.Empty) { this.Password = null; } @@ -96,6 +98,10 @@ public ConnectionString(string connectionString) this.Collation = _values.ContainsKey("collation") ? new Collation(_values.GetValue("collation")) : this.Collation; this.Upgrade = _values.GetValue("upgrade", this.Upgrade); + + if (_values.TryGetValue("autorebuild", out var v) && !_values.ContainsKey("auto-rebuild")) + _values["auto-rebuild"] = v; + this.AutoRebuild = _values.GetValue("auto-rebuild", this.AutoRebuild); } diff --git a/LiteDB/Engine/Disk/DiskService.cs b/LiteDB/Engine/Disk/DiskService.cs index 73e7910b5..d02d40872 100644 --- a/LiteDB/Engine/Disk/DiskService.cs +++ b/LiteDB/Engine/Disk/DiskService.cs @@ -3,6 +3,9 @@ using System.Collections.Generic; using System.IO; using System.Threading; + +using LiteDB.Utils; + using static LiteDB.Constants; namespace LiteDB.Engine @@ -54,8 +57,15 @@ public DiskService( if (isNew) { LOG($"creating new database: '{Path.GetFileName(_dataFactory.Name)}'", "DISK"); - - this.Initialize(_dataPool.Writer.Value, settings.Collation, settings.InitialSize); + try + { + this.Initialize(_dataPool.Writer.Value, settings.Collation, settings.InitialSize); + } + catch (Exception ex) + { + LOG($"Error while initializing DiskService: {ex.Message}", "ERROR"); + throw; + } } // if not readonly, force open writable datafile @@ -340,14 +350,14 @@ public void Dispose() // can change file size var delete = _logFactory.Exists() && _logPool.Writer.Value.Length == 0; - // dispose Stream pools - _dataPool.Dispose(); - _logPool.Dispose(); + var tc = new TryCatch(); + tc.Catch(_dataPool.Dispose); + tc.Catch(_logPool.Dispose); - if (delete) _logFactory.Delete(); + if (delete) tc.Catch(_logFactory.Delete); // other disposes - _cache.Dispose(); + tc.Catch(_cache.Dispose); } } } diff --git a/LiteDB/Engine/Engine/LiteEngine.AutoRebuild.cs b/LiteDB/Engine/Engine/LiteEngine.AutoRebuild.cs new file mode 100644 index 000000000..3d94b69ae --- /dev/null +++ b/LiteDB/Engine/Engine/LiteEngine.AutoRebuild.cs @@ -0,0 +1,43 @@ +using System; + +namespace LiteDB.Engine +{ + public partial class LiteEngine + { + private DateTime _lastAutoRebuildUtc = DateTime.MinValue; + private static readonly TimeSpan _autoRebuildCooldown = TimeSpan.FromMinutes(5); + + private bool TryAutoRebuild(Exception ex, bool viaOpen = false) + { + try + { + if (!_settings.AutoRebuild) return false; + if (!IsStructuralCorruption(ex)) return false; + if (_autoRebuildInProgress) return false; + if (DateTime.UtcNow - _lastAutoRebuildUtc < _autoRebuildCooldown) return false; + + _autoRebuildInProgress = true; + _lastAutoRebuildUtc = DateTime.UtcNow; + + try + { + if (viaOpen) + AutoRebuildAndReopenViaOpenPath(); + else + AutoRebuildAndReopen(); + + return true; + } + finally + { + _autoRebuildInProgress = false; + } + } + catch + { + // nie wyciekaj wyjątków na zewnątrz – decyzja o rethrow zostaje w wyższym poziomie + return false; + } + } + } +} diff --git a/LiteDB/Engine/Engine/Rebuild.cs b/LiteDB/Engine/Engine/Rebuild.cs index 37036bad6..6e5a1d148 100644 --- a/LiteDB/Engine/Engine/Rebuild.cs +++ b/LiteDB/Engine/Engine/Rebuild.cs @@ -10,34 +10,38 @@ namespace LiteDB.Engine { public partial class LiteEngine { - /// - /// Implement a full rebuild database. Engine will be closed and re-created in another instance. - /// A backup copy will be created with -backup extention. All data will be readed and re created in another database - /// After run, will re-open database - /// public long Rebuild(RebuildOptions options) { - if (string.IsNullOrEmpty(_settings.Filename)) return 0; // works only with os file + if (string.IsNullOrEmpty(_settings.Filename)) return 0; - this.Close(); - - // run build service - var rebuilder = new RebuildService(_settings); + lock (_exclusiveRebuildGate) + { + var dataFile = _settings.Filename; + try + { + var collation = _header?.Pragmas?.Collation ?? options?.Collation ?? Collation.Default; + var password = options?.Password ?? _settings.Password; + var effective = options ?? new RebuildOptions(); + if (effective.Collation == null) effective.Collation = collation; + if (effective.Password == null) effective.Password = password; - // return how many bytes of diference from original/rebuild version - var diff = rebuilder.Rebuild(options); + this.Close(); - // re-open engine - this.Open(); + var diff = new RebuildService(_settings).Rebuild(effective); - _state.Disposed = false; + this.Open(); + _state.Disposed = false; - return diff; + return diff; + } + finally + { + try { CleanupOrphanTempFiles(dataFile); } catch { /* ignore */ } + } + } } - /// - /// Implement a full rebuild database. A backup copy will be created with -backup extention. All data will be readed and re created in another database - /// + public long Rebuild() { var collation = new Collation(this.Pragma(Pragmas.COLLATION)); @@ -46,52 +50,78 @@ public long Rebuild() return this.Rebuild(new RebuildOptions { Password = password, Collation = collation }); } - /// - /// Fill current database with data inside file reader - run inside a transacion - /// internal void RebuildContent(IFileReader reader) { - // begin transaction and get TransactionID - var transaction = _monitor.GetTransaction(true, false, out _); + var maxCount = GetSourceMaxItemsCount(_settings); + RebuildContent(reader, maxCount); + } + + private static uint GetSourceMaxItemsCount(EngineSettings settings) + { + var dataBytes = new FileInfo(settings.Filename).Length; + var logFile = FileHelper.GetLogFile(settings.Filename); + var logBytes = File.Exists(logFile) ? new FileInfo(logFile).Length : 0; + return (uint)(((dataBytes + logBytes) / PAGE_SIZE + 10) * byte.MaxValue); + } + + internal void RebuildContent(IFileReader reader, uint maxItemsCount) + { + var transaction = _monitor.GetTransaction(create: true, queryOnly: false, out _); try { foreach (var collection in reader.GetCollections()) { - // get snapshot, indexer and data services - var snapshot = transaction.CreateSnapshot(LockMode.Write, collection, true); - var indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT); - var data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT); + var snapshot = transaction.CreateSnapshot(LockMode.Write, collection, addIfNotExists: true); - // get all documents from current collection - var docs = reader.GetDocuments(collection); + var indexer = new IndexService(snapshot, _header.Pragmas.Collation, maxItemsCount); + var data = new DataService(snapshot, maxItemsCount); - // insert one-by-one - foreach (var doc in docs) + foreach (var doc in reader.GetDocuments(collection)) { transaction.Safepoint(); + InsertDocument(snapshot, doc, BsonAutoId.ObjectId, indexer, data); + } - this.InsertDocument(snapshot, doc, BsonAutoId.ObjectId, indexer, data); + if (!RebuildHelpers.ValidatePkNoCycle(indexer, snapshot.CollectionPage.PK, out var pkCount, maxItemsCount)) + { + throw new LiteException(0, $"Detected loop in PK index for collection '{collection}'."); } - // first create all user indexes (exclude _id index) - foreach (var index in reader.GetIndexes(collection)) + foreach (var idx in reader.GetIndexes(collection)) { - this.EnsureIndex(collection, - index.Name, - BsonExpression.Create(index.Expression), - index.Unique); + try + { + EnsureIndex(collection, + idx.Name, + BsonExpression.Create(idx.Expression), + idx.Unique); + } + catch (LiteException ex) when (ex.Message.IndexOf("Detected loop in FindAll", StringComparison.OrdinalIgnoreCase) >= 0) + { + try { DropIndex(collection, idx.Name); } catch { /* best effort */ } + + var expr = BsonExpression.Create(idx.Expression); + + RebuildHelpers.EnsureIndexFromDataScan( + snapshot, + idx.Name, + expr, + idx.Unique, + indexer, + data, + transaction.Safepoint + ); + } } } transaction.Commit(); - _monitor.ReleaseTransaction(transaction); } catch (Exception ex) { - this.Close(ex); - + Close(ex); throw; } } diff --git a/LiteDB/Engine/Engine/RebuildHelpers.cs b/LiteDB/Engine/Engine/RebuildHelpers.cs new file mode 100644 index 000000000..bdf94d643 --- /dev/null +++ b/LiteDB/Engine/Engine/RebuildHelpers.cs @@ -0,0 +1,113 @@ +using System; +using System.Collections.Generic; + +using static LiteDB.Constants; + +namespace LiteDB.Engine +{ + internal static class RebuildHelpers + { + internal readonly struct ScannedDoc + { + public ScannedDoc(PageAddress block, BsonDocument doc) + { + Block = block; + Doc = doc; + } + + public PageAddress Block { get; } + public BsonDocument Doc { get; } + } + + public static IEnumerable ScanDocuments(Snapshot snapshot, DataService data, uint maxItemsCount) + { + if (snapshot.CollectionPage == null) yield break; + + var counter = 0u; + + foreach (var page in snapshot.EnumerateDataPages()) + { + foreach (var addr in page.GetBlocks()) + { + ENSURE(counter++ < maxItemsCount, "Detected loop in ScanDocuments({0})", snapshot.CollectionName); + + using (var reader = new BufferReader(data.Read(addr))) + { + var result = reader.ReadDocument(); + if (result.Fail) throw result.Exception; + + yield return new ScannedDoc(addr, result.Value); + } + } + } + } + + public static void EnsureIndexFromDataScan( + Snapshot snapshot, + string indexName, + BsonExpression expression, + bool unique, + IndexService indexer, + DataService data, + Action safepoint) + { + var index = indexer.CreateIndex(indexName, expression.Source, unique); + var pk = snapshot.CollectionPage.PK; + var collation = indexer.Collation; + + foreach (var item in ScanDocuments(snapshot, data, uint.MaxValue)) + { + var doc = item.Doc; + var id = doc["_id"]; + + if (id.IsNull || id.IsMinValue || id.IsMaxValue) continue; + var pkNode = indexer.Find(pk, id, false, Query.Ascending); + if (pkNode == null) continue; + + IndexNode last = null; + IndexNode first = null; + + var keys = expression.GetIndexKeys(doc, collation); + + foreach (var key in keys) + { + var node = indexer.AddNode(index, key, item.Block, last); + if (first == null) first = node; + last = node; + } + + if (first != null) + { + last.SetNextNode(pkNode.NextNode); + pkNode.SetNextNode(first.Position); + } + + safepoint?.Invoke(); + } + } + + public static bool ValidatePkNoCycle(IndexService indexer, CollectionIndex pk, out int count, uint maxItemsCount) + { + count = 0; + + var seen = new HashSet(); + + try + { + foreach (var node in indexer.FindAll(pk, Query.Ascending)) + { + if (!seen.Add(node.Position)) return false; + + count++; + if (count > maxItemsCount) return false; + } + + return true; + } + catch (LiteException) + { + return false; + } + } + } +} diff --git a/LiteDB/Engine/Engine/Transaction.cs b/LiteDB/Engine/Engine/Transaction.cs index 4db9f57b3..1724363f6 100644 --- a/LiteDB/Engine/Engine/Transaction.cs +++ b/LiteDB/Engine/Engine/Transaction.cs @@ -1,7 +1,6 @@ -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Threading; +// plik: Transaction.cs +using System; + using static LiteDB.Constants; namespace LiteDB.Engine @@ -74,34 +73,45 @@ public bool Rollback() } /// - /// Create (or reuse) a transaction an add try/catch block. Commit transaction if is new transaction + /// Create (or reuse) a transaction and add try/catch block. Commit transaction if is new transaction /// private T AutoTransaction(Func fn) { _state.Validate(); - var transaction = _monitor.GetTransaction(true, false, out var isNew); + bool retried = false; - try + while (true) { - var result = fn(transaction); - - // if this transaction was auto-created for this operation, commit & dispose now - if (isNew) - this.CommitAndReleaseTransaction(transaction); + var transaction = _monitor.GetTransaction(true, false, out var isNew); - return result; - } - catch(Exception ex) - { - if (_state.Handle(ex)) + try { - transaction.Rollback(); + var result = fn(transaction); + + if (isNew) + this.CommitAndReleaseTransaction(transaction); - _monitor.ReleaseTransaction(transaction); + return result; } + catch (Exception ex) + { + bool isExplicit = false; + try { isExplicit = transaction?.ExplicitTransaction == true; } catch { } + + try { transaction.Rollback(); } catch { } + try { _monitor.ReleaseTransaction(transaction); } catch { } + + if (!retried && !isExplicit && TryAutoRebuild(ex)) + { + retried = true; + continue; + } - throw; + try { _state.Handle(ex); } catch { } + + throw; + } } } @@ -112,11 +122,11 @@ private void CommitAndReleaseTransaction(TransactionService transaction) _monitor.ReleaseTransaction(transaction); // try checkpoint when finish transaction and log file are bigger than checkpoint pragma value (in pages) - if (_header.Pragmas.Checkpoint > 0 && + if (_header.Pragmas.Checkpoint > 0 && _disk.GetFileLength(FileOrigin.Log) > (_header.Pragmas.Checkpoint * PAGE_SIZE)) { _walIndex.TryCheckpoint(); } } } -} \ No newline at end of file +} diff --git a/LiteDB/Engine/LiteEngine.cs b/LiteDB/Engine/LiteEngine.cs index 59bb84b4c..600dadced 100644 --- a/LiteDB/Engine/LiteEngine.cs +++ b/LiteDB/Engine/LiteEngine.cs @@ -7,6 +7,7 @@ using System.IO; using System.Linq; using System.Threading; + using static LiteDB.Constants; namespace LiteDB.Engine @@ -19,6 +20,7 @@ namespace LiteDB.Engine public partial class LiteEngine : ILiteEngine { #region Services instances + private volatile bool _autoRebuildInProgress; private LockService _locker; @@ -46,6 +48,8 @@ public partial class LiteEngine : ILiteEngine /// Sequence cache for collections last ID (for int/long numbers only) /// private ConcurrentDictionary _sequences; + private readonly object _reinitSync = new object(); + private readonly object _exclusiveRebuildGate = new object(); #endregion @@ -63,7 +67,7 @@ public LiteEngine() /// Initialize LiteEngine using connection string using key=value; parser /// public LiteEngine(string filename) - : this (new EngineSettings { Filename = filename }) + : this(new EngineSettings { Filename = filename }) { } @@ -98,7 +102,7 @@ internal bool Open() // initialize disk service (will create database if needed) _disk = new DiskService(_settings, _state, MEMORY_SEGMENT_SIZES); - + CleanupOrphanTempFiles(_settings.Filename); // read page with no cache ref (has a own PageBuffer) - do not Release() support var buffer = _disk.ReadFull(FileOrigin.Data).First(); @@ -161,10 +165,16 @@ internal bool Open() catch (Exception ex) { LOG(ex.Message, "ERROR"); - this.Close(ex); + + if (TryAutoRebuild(ex, viaOpen: true)) + { + return true; + } + throw; } + } /// @@ -255,6 +265,127 @@ internal List Close(Exception ex) /// public int Checkpoint() => _walIndex.Checkpoint(); + private static bool IsStructuralCorruption(Exception ex) + { + string all = ex.ToString().ToLowerInvariant(); + + if (all.Contains("empty page must be defined as empty type")) return true; + if (all.Contains("invalid page type")) return true; + if (all.Contains("page header is corrupted")) return true; + if (all.Contains("detected loop in findall")) return true; + + for (var e = ex; e != null; e = e.InnerException) + { + var msg = (e.Message ?? "").ToLowerInvariant(); + if (msg.Contains("empty page must be defined as empty type")) return true; + if (msg.Contains("invalid page type")) return true; + if (msg.Contains("page header is corrupted")) return true; + if (msg.Contains("detected loop in findall")) return true; + } + + return false; + } + private void AutoRebuildAndReopenViaOpenPath() + { + try { _disk?.MarkAsInvalidState(); } catch { } + + try + { + this.Close(); + this.Open(); + _state.Disposed = false; + } + finally + { + try { CleanupOrphanTempFiles(_settings.Filename); } catch { } + } + } + + private void AutoRebuildAndReopen() + { + lock (_reinitSync) + { + try { this.Close(); } catch { /* best effort */ } + + try + { + var file = _settings?.Filename; + PruneOldBackups(_settings.Filename); + if (!string.IsNullOrEmpty(file) && File.Exists(file)) + { + var backup = file + "-backup"; + if (!File.Exists(backup)) + File.Copy(file, backup, overwrite: true); + } + } + catch { /* best effort – backup */ } + + var collation = _header?.Pragmas?.Collation ?? Collation.Default; + + try + { + this.Recovery(collation); + this.Open(); + _state.Disposed = false; + } + finally + { + try { CleanupOrphanTempFiles(_settings.Filename); } catch { /* best effort */ } + } + } + } + + private static void PruneOldBackups(string dataFile, int keep = 1) + { + try + { + var dir = Path.GetDirectoryName(dataFile); + var baseName = Path.GetFileNameWithoutExtension(dataFile); + var pattern = $"{baseName}-backup-*.db"; + + var files = Directory.EnumerateFiles(dir, pattern) + .Select(f => new FileInfo(f)) + .OrderByDescending(fi => fi.LastWriteTimeUtc) + .ToList(); + + foreach (var fi in files.Skip(keep)) + { + try { fi.Delete(); } catch { } + } + } + catch { } + } + + private static void CleanupOrphanTempFiles(string dataFile) + { + try + { + if (string.IsNullOrEmpty(dataFile)) return; + + var dir = Path.GetDirectoryName(dataFile); + var baseName = Path.GetFileNameWithoutExtension(dataFile); + + if (string.IsNullOrEmpty(dir) || string.IsNullOrEmpty(baseName)) return; + + var patterns = new[] + { + $"{baseName}-temp*.db", + $"{baseName}-temp-*.db", + $"{baseName}-temp.db", + $"{baseName}-temp-log.db" + }; + + foreach (var pat in patterns) + { + foreach (var f in Directory.EnumerateFiles(dir, pat)) + { + try { File.Delete(f); } catch { /* ignore */ } + } + } + } + catch { /* best effort */ } + } + public void Dispose() { this.Dispose(true); diff --git a/LiteDB/Engine/Services/IndexService.cs b/LiteDB/Engine/Services/IndexService.cs index 6b8fb230d..7d2e7965f 100644 --- a/LiteDB/Engine/Services/IndexService.cs +++ b/LiteDB/Engine/Services/IndexService.cs @@ -25,6 +25,46 @@ public IndexService(Snapshot snapshot, Collation collation, uint maxItemsCount) public Collation Collation => _collation; + // GetNodeList(PageAddress) + public IEnumerable GetNodeList(PageAddress nodeAddress) + { + var node = this.GetNode(nodeAddress); + var counter = 0u; + var seen = new HashSet(); + + while (node != null) + { + ENSURE(counter++ < _maxItemsCount, "Detected loop in GetNodeList({0})", nodeAddress); + ENSURE(seen.Add(node.Position), "Detected loop in GetNodeList({0})", nodeAddress); + + yield return node; + + node = this.GetNode(node.NextNode); + } + } + + // FindAll(CollectionIndex index, int order) + public IEnumerable FindAll(CollectionIndex index, int order) + { + var cur = order == Query.Ascending ? this.GetNode(index.Head) : this.GetNode(index.Tail); + var counter = 0u; + var seen = new HashSet(); + + while (!cur.GetNextPrev(0, order).IsEmpty) + { + cur = this.GetNode(cur.GetNextPrev(0, order)); + + ENSURE(counter++ < _maxItemsCount, "Detected loop in FindAll({0})", index.Name); + ENSURE(seen.Add(cur.Position), "Detected loop in FindAll({0})", index.Name); + + // stop if node is head/tail + if (cur.Key.IsMinValue || cur.Key.IsMaxValue) yield break; + + yield return cur; + } + } + + /// /// Create a new index and returns head page address (skip list) /// @@ -196,24 +236,6 @@ public IndexNode GetNode(PageAddress address) return indexPage.GetIndexNode(address.Index); } - /// - /// Gets all node list from passed nodeAddress (forward only) - /// - public IEnumerable GetNodeList(PageAddress nodeAddress) - { - var node = this.GetNode(nodeAddress); - var counter = 0u; - - while (node != null) - { - ENSURE(counter++ < _maxItemsCount, "Detected loop in GetNodeList({0})", nodeAddress); - - yield return node; - - node = this.GetNode(node.NextNode); - } - } - /// /// Deletes all indexes nodes from pkNode /// @@ -333,28 +355,6 @@ public void DropIndex(CollectionIndex index) } #region Find - - /// - /// Return all index nodes from an index - /// - public IEnumerable FindAll(CollectionIndex index, int order) - { - var cur = order == Query.Ascending ? this.GetNode(index.Head) : this.GetNode(index.Tail); - var counter = 0u; - - while (!cur.GetNextPrev(0, order).IsEmpty) - { - ENSURE(counter++ < _maxItemsCount, "Detected loop in FindAll({0})", index.Name); - - cur = this.GetNode(cur.GetNextPrev(0, order)); - - // stop if node is head/tail - if (cur.Key.IsMinValue || cur.Key.IsMaxValue) yield break; - - yield return cur; - } - } - /// /// Find first node that index match with value . /// If index are unique, return unique value - if index are not unique, return first found (can start, middle or end) diff --git a/LiteDB/Engine/Services/RebuildService.cs b/LiteDB/Engine/Services/RebuildService.cs index b4eaa155e..ccf36fa63 100644 --- a/LiteDB/Engine/Services/RebuildService.cs +++ b/LiteDB/Engine/Services/RebuildService.cs @@ -6,6 +6,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; + using static LiteDB.Constants; namespace LiteDB.Engine @@ -37,6 +38,16 @@ public RebuildService(EngineSettings settings) _fileVersion = FileReaderV8.IsVersion(buffer) ? 8 : throw LiteException.InvalidDatabase(); } + private static uint GetSourceMaxItemsCount(EngineSettings settings) + { + long dataBytes = new FileInfo(settings.Filename).Length; + + var logFile = FileHelper.GetLogFile(settings.Filename); + long logBytes = File.Exists(logFile) ? new FileInfo(logFile).Length : 0; + // ((pages in data+log) + 10) * 255 + return (uint)((((dataBytes + logBytes) / PAGE_SIZE) + 10) * byte.MaxValue); + } + public long Rebuild(RebuildOptions options) { var backupFilename = FileHelper.GetSuffixFile(_settings.Filename, "-backup", true); @@ -62,8 +73,11 @@ public long Rebuild(RebuildOptions options) // copy all database to new Log file with NO checkpoint during all rebuild engine.Pragma(Pragmas.CHECKPOINT, 0); + // compute the correct MAX_ITEMS_COUNT from the *source* file + uint maxItemsCount = GetSourceMaxItemsCount(_settings); + // rebuild all content from reader into new engine - engine.RebuildContent(reader); + engine.RebuildContent(reader, maxItemsCount); // insert error report if (options.IncludeErrorReport && options.Errors.Count > 0) @@ -106,7 +120,7 @@ public long Rebuild(RebuildOptions options) // get difference size - return + return new FileInfo(backupFilename).Length - new FileInfo(_settings.Filename).Length; } diff --git a/LiteDB/Engine/Services/SnapShot.cs b/LiteDB/Engine/Services/SnapShot.cs index f34840e2f..09611158c 100644 --- a/LiteDB/Engine/Services/SnapShot.cs +++ b/LiteDB/Engine/Services/SnapShot.cs @@ -86,6 +86,36 @@ public Snapshot( } } + /// + /// Zwróć wszystkie strony danych kolekcji iterując przez 5 slotów listy wolnych stron + /// (0..4). Obejmuje także "pełne" strony (slot 4). + /// + public IEnumerable EnumerateDataPages() + { + ENSURE(!_disposed, "the snapshot is disposed"); + + if (_collectionPage == null) yield break; + + var counter = 0u; + + for (int slot = 0; slot < PAGE_FREE_LIST_SLOTS; slot++) + { + var next = _collectionPage.FreeDataPageList[slot]; + + while (next != uint.MaxValue) + { + var page = this.GetPage(next); + + yield return page; + + next = page.NextPageID; + + ENSURE(counter++ < _disk.MAX_ITEMS_COUNT, "Detected loop in EnumerateDataPages({0})", _collectionName); + } + } + } + + /// /// Get all snapshot pages (can or not include collectionPage) - If included, will be last page /// diff --git a/LiteDB/Engine/Services/TransactionService.cs b/LiteDB/Engine/Services/TransactionService.cs index 7373251da..94c5ce704 100644 --- a/LiteDB/Engine/Services/TransactionService.cs +++ b/LiteDB/Engine/Services/TransactionService.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Runtime.InteropServices; using System.Threading; + using static LiteDB.Constants; namespace LiteDB.Engine @@ -224,7 +225,8 @@ IEnumerable source() // persist header in log file yield return clone; } - }; + } + ; // write all dirty pages, in sequence on log-file and store references into log pages on transPages // (works only for Write snapshots) @@ -359,7 +361,8 @@ IEnumerable source() Buffer.BlockCopy(buf.Array, buf.Offset, clone.Array, clone.Offset, clone.Count); yield return clone; - }; + } + ; // create a header save point before any change var safepoint = _header.Savepoint(); @@ -398,32 +401,39 @@ protected virtual void Dispose(bool dispose) return; } - ENSURE(_state != TransactionState.Disposed, "transaction must be active before call Done"); - - // clean snapshots if there is no commit/rollback - if (_state == TransactionState.Active && _snapshots.Count > 0) + try { + ENSURE(_state != TransactionState.Disposed, "transaction must be active before call Done"); // release writable snapshots - foreach (var snapshot in _snapshots.Values.Where(x => x.Mode == LockMode.Write)) + // clean snapshots if there is no commit/rollback + if (_state == TransactionState.Active && _snapshots.Count > 0) { - // discard all dirty pages - _disk.DiscardDirtyPages(snapshot.GetWritablePages(true, true).Select(x => x.Buffer)); + // release writable snapshots + foreach (var snapshot in _snapshots.Values.Where(x => x.Mode == LockMode.Write)) + { + // discard all dirty pages + _disk.DiscardDirtyPages(snapshot.GetWritablePages(true, true).Select(x => x.Buffer)); - // discard all clean pages - _disk.DiscardCleanPages(snapshot.GetWritablePages(false, true).Select(x => x.Buffer)); - } + // discard all clean pages + _disk.DiscardCleanPages(snapshot.GetWritablePages(false, true).Select(x => x.Buffer)); + } - // release buffers in read-only snaphosts - foreach (var snapshot in _snapshots.Values.Where(x => x.Mode == LockMode.Read)) - { - foreach (var page in snapshot.LocalPages) + // release buffers in read-only snaphosts + foreach (var snapshot in _snapshots.Values.Where(x => x.Mode == LockMode.Read)) { - page.Buffer.Release(); - } + foreach (var page in snapshot.LocalPages) + { + page.Buffer.Release(); + } - snapshot.CollectionPage?.Buffer.Release(); + snapshot.CollectionPage?.Buffer.Release(); + } } } + catch (Exception ex) + { + LOG($"Error while disposing TransactionService: {ex.Message}", "ERROR"); + } _reader.Dispose(); diff --git a/LiteDB/LiteDB.csproj b/LiteDB/LiteDB.csproj index 67eeb45f1..678d8b393 100644 --- a/LiteDB/LiteDB.csproj +++ b/LiteDB/LiteDB.csproj @@ -2,17 +2,17 @@ net4.5;netstandard1.3;netstandard2.0 - 5.0.21 - 5.0.21 - 5.0.21 - 5.0.21 - Maurício David - LiteDB + 5.0.26 + 5.0.26 + 5.0.26 + 5.0.26 + Maurício David, Patryk Golus + LiteDB.fixed LiteDB - A lightweight embedded .NET NoSQL document store in a single datafile MIT en-US - LiteDB - LiteDB + LiteDB.fixed + LiteDB.fixed database nosql embedded icon_64x64.png MIT