huge refactoring

Author: formeo

.gitignore

@@ -64,3 +64,5 @@ __recovery/
 
 # Castalia statistics file (since XE7 Castalia is distributed with Delphi)
 *.stat
+* .idea
+* 

FirebirdRepairBase.dpr

@@ -15,7 +15,8 @@ uses
   uPageAnalyzer in 'uPageAnalyzer.pas',
   uBtreePage in 'uBtreePage.pas',
   uDatabaseStats in 'uDatabaseStats.pas',
-  uFlagManager in 'uFlagManager.pas';
+  uFlagManager in 'uFlagManager.pas',
+  uRecordParser in 'uRecordParser.pas';
 
 {$R *.res}
 

FirebirdRepairBase.dproj

@@ -122,6 +122,7 @@
         <DCCReference Include="uBtreePage.pas"/>
         <DCCReference Include="uDatabaseStats.pas"/>
         <DCCReference Include="uFlagManager.pas"/>
+        <DCCReference Include="uRecordParser.pas"/>
         <BuildConfiguration Include="Base">
             <Key>Base</Key>
         </BuildConfiguration>

README.md

@@ -1,32 +1,84 @@
 # FirebirdRepairBase
-Program for analize and repare broken firebird database, it is working on the low level
-This is dirty version of program, now program is in develop process.
-
-Now it features: 
-* check database pages by type
-* replace some header page params
-* set READ ONLY and FORCE WRITE flags
-* replace check sum on pages
-* generate new page
-
-# In developing
-* Analisys pages, pump data.
-* Analisys TIP sequence, generate lost TIP pages
-
-# Firebird Database Pages
-* Database Header Page
-* Page Inventory Page (PIP)
-* Transaction Inventory Page (TIP)
-* Pointer Page
-* Data Page
-* Index Root Page
-* Index B-Tree Page
-* Blob Data Page
-* Generator Page
-* Write Ahead Log Page
+
+A low-level tool for analyzing and potentially recovering data from **broken** Firebird database files (`.fdb`, `.gdb`). It bypasses the standard Firebird engine to read raw page structures and data.
+
+**Warning:** This is a **development version**. It directly manipulates database files. Always create a **backup** of your database before using this tool. Use at your own risk.
+
+## Features
+
+*   **Database Structure Analysis:**
+    *   Opens and reads Firebird database files.
+    *   Identifies and counts pages by type (Header, PIP, TIP, Pointer, Data, Index Root/BTree, Blob, Generator, WAL).
+    *   Displays basic page information (number, type, checksum, relation ID).
+*   **Header Manipulation:**
+    *   Reads and interprets database header flags.
+    *   Allows setting/clearing specific flags (e.g., Read Only, Force Write).
+*   **Page Content Inspection:**
+    *   Reads specific pages by number.
+    *   Extracts and displays raw record fragments from Data Pages.
+    *   Allows changing page type and checksum (Dangerous!).
+*   **Transaction Analysis:**
+    *   Reads Transaction Inventory Pages (TIP).
+    *   Counts transactions by state (Active, Limbo, Dead, Committed).
+*   **Basic Data Recovery:**
+    *   Extracts raw data fragments from Data Pages.
+    *   *(WIP)* Basic MVCC (Multi-Version Concurrency Control) awareness for data visibility (see TODO).
+    *   *(New)* Parses basic record headers to identify deleted records and data length.
+*   **Record Fragment Parsing:**
+    *   Introduces `uRecordParser` module for analyzing individual record fragments.
+    *   Extracts key information like transaction ID, back pointer, flags, and data length from the record header.
+
+## In Development (TODO)
+
+*   **Advanced Data Recovery:**
+    *   Implement full MVCC logic to reconstruct the correct, visible version of records based on transaction states and TIP page data.
+    *   Export recovered data to a new, valid Firebird database or other formats (e.g., CSV).
+    *   Analyze and potentially repair B-Tree index structures.
+    *   Analyze and potentially repair Blob page chains.
+*   **Deeper Integrity Checks:**
+    *   Verify logical links between pages (e.g., Pointer Pages -> Data Pages, Record Back Pointers).
+    *   Identify orphaned pages.
+*   **Page Generation/Repair:**
+    *   Implement logic to generate missing TIP or PIP pages if necessary.
+    *   Attempt to rebuild damaged page structures.
+*   **User Interface Improvements:**
+    *   Add search functionality within page data.
+    *   Provide more detailed views for different page types (B-Tree keys, Blob data).
+    *   Add a 'Restore from Backup' feature.
+    *   Improve error handling and user feedback.
+
+## Supported Firebird Pages
+
+*   Database Header Page
+*   Page Inventory Page (PIP)
+*   Transaction Inventory Page (TIP)
+*   Pointer Page
+*   Data Page
+*   Index Root Page
+*   Index B-Tree Page
+*   Blob Data Page
+*   Generator Page
+*   Write Ahead Log Page (WAL) (Note: WAL functionality might be limited)
+
+## Technical Details
+
+*   **Language:** Pascal (Delphi).
+*   **Target:** Windows.
+*   **Approach:** Direct file I/O, parsing raw page structures according to Firebird Internal Format documentation (`fbint-page-1.html`) and Firebird source code conventions.
+*   **Modules:**
+    *   `uStructs`: Core data structures (TPag, THdr, etc.).
+    *   `uPageAnalyzer`: General page reading and type identification.
+    *   `uDataPage`: Data Page header parsing and record fragment extraction.
+    *   `uTipPage`: TIP page header parsing and transaction state analysis.
+    *   `uBtreePage`: B-Tree page header parsing and key extraction.
+    *   `uRecordParser`: *(New)* Parses headers of individual record fragments.
+    *   `uDatabaseStats`: Aggregates statistics using `uPageAnalyzer`.
+    *   `uFlagManager`: Manages database header flags.
 
 ## License
-MIT:
+
+MIT
 
 ## Author
+
 Gordienko Roman

main.pas

@@ -5,9 +5,9 @@ interface
 uses
   Windows, Messages, SysUtils, Variants, Classes, Graphics, Controls, Forms,
   Dialogs, StdCtrls, FileCtrl, ComCtrls, uStructs, DB, ExtCtrls,
-  Menus, uCommon, uDatabase, XPMan, uDataPage, Grids, DBGrids, DBClient,
+  Menus, uCommon,  XPMan, uDataPage, Grids, DBGrids, DBClient,
   DBCtrls, Spin, Vcl.Buttons,
-  uPageAnalyzer, uDatabaseStats, uFlagManager;
+  uPageAnalyzer, uDatabaseStats, uFlagManager, StrUtils;
 
 type
   TfrmMain = class(TForm)
@@ -200,6 +200,7 @@ procedure TfrmMain.btnReWriteClick(Sender: TObject);
   PageInfo: TPageInfo;
   NewPageBuffer: TBytes;
   FileStream: TFileStream;
+  ModifiedHeader: TPag; // --- ÄÎÁÀÂËÅÍÎ ---
 begin
   // ReWriting page - ÂÍÈÌÀÍÈÅ: Ýòî ÎÏÀÑÍÀß îïåðàöèÿ!
   if Application.MessageBox('Do You Want to Rewrite Database Page? THIS IS DANGEROUS!',
@@ -214,10 +215,18 @@ procedure TfrmMain.btnReWriteClick(Sender: TObject);
       PageInfo := FPageAnalyzer.GetPageInfo(PageNum);
       NewPageBuffer := Copy(PageInfo.Buffer); // Êîïèðóåì òåêóùèé áóôåð
 
-      // Ìîäèôèöèðóåì çàãîëîâîê â áóôåðå
-      TPag(NewPageBuffer[0]).pag_checksum := UShort(NewChecksum);
+      // --- ÈÑÏÐÀÂËÅÍÈÅ: Èñïîëüçóåì Move äëÿ áåçîïàñíîãî êîïèðîâàíèÿ è èçìåíåíèÿ çàãîëîâêà ---
+      // Ñêîïèðóåì ñòàíäàðòíûé çàãîëîâîê èç áóôåðà â ëîêàëüíóþ ïåðåìåííóþ
+      Move(NewPageBuffer[0], ModifiedHeader, SizeOf(TPag));
+
+      // Èçìåíèì ïîëÿ â ëîêàëüíîé ïåðåìåííîé
+      ModifiedHeader.pag_checksum := UShort(NewChecksum);
       if (NewType >= 0) and (NewType <= $FF) then // Ïðîâåðÿåì äèàïàçîí
-        TPag(NewPageBuffer[0]).pag_type := SChar(NewType);
+        ModifiedHeader.pag_type := SChar(NewType);
+
+      // Ñêîïèðóåì èçìåí¸ííóþ ëîêàëüíóþ ïåðåìåííóþ îáðàòíî â áóôåð
+      Move(ModifiedHeader, NewPageBuffer[0], SizeOf(TPag));
+      // --- ÊÎÍÅÖ ÈÑÏÐÀÂËÅÍÈß ---
 
       // Çàïèñûâàåì èçìåí¸ííûé áóôåð îáðàòíî â ôàéë
       FileStream := TFileStream.Create(FPageAnalyzer.FileName, fmOpenReadWrite or fmShareExclusive);
@@ -290,20 +299,19 @@ procedure TfrmMain.btnGetHeaderFlagsClick(Sender: TObject);
   try
     Flags := FFlagManager.GetFlags;
 
-    // Ôîðìèðóåì ñòðîêó äëÿ îòîáðàæåíèÿ (íàïðèìåð, áèíàðíîå ïðåäñòàâëåíèå èëè ïðîñòî ñïèñîê)
-    // Äëÿ ïðîñòîòû, âûâåäåì ñïèñîê àêòèâíûõ ôëàãîâ
+
     edtFlags.text := 'Active flags: ';
     if Flags.ActiveShadow then edtFlags.text := edtFlags.text + 'ActiveShadow, ';
     if Flags.ForceWrite then edtFlags.text := edtFlags.text + 'ForceWrite, ';
     if Flags.NoChecksums then edtFlags.text := edtFlags.text + 'NoChecksums, ';
     if Flags.NoReserve then edtFlags.text := edtFlags.text + 'NoReserve, ';
     if Flags.SqlDialect3 then edtFlags.text := edtFlags.text + 'SqlDialect3, ';
     if Flags.ReadOnly then edtFlags.text := edtFlags.text + 'ReadOnly, ';
-    // Óáèðàåì ïîñëåäíþþ çàïÿòóþ
-    if edtFlags.text.EndsWith(', ') then
-      edtFlags.text := edtFlags.text.Remove(edtFlags.text.Length - 2);
 
-    // Èëè ìîæíî âûâåñòè âñå ôëàãè ñ óêàçàíèåì ñòàòóñà
+    if EndsStr(', ', edtFlags.text) then // Èñïîëüçóåì EndsStr èç StrUtils
+      edtFlags.text := Copy(edtFlags.text, 1, Length(edtFlags.text) - 2); // Óäàëÿåì ïîñëåäíèå 2 ñèìâîëà
+
+
     lstDBFlags.Items.Add(Format('Active Shadow: %s', [BoolToStr(Flags.ActiveShadow, True)]));
     lstDBFlags.Items.Add(Format('Force Write: %s', [BoolToStr(Flags.ForceWrite, True)]));
     lstDBFlags.Items.Add(Format('No Checksums: %s', [BoolToStr(Flags.NoChecksums, True)]));
@@ -348,28 +356,28 @@ procedure TfrmMain.checkDB;
     lstLog.Items.Add('Database page size: ' + IntToStr(FPageAnalyzer.PageSize));
     lstLog.Items.Add('Scanning pages...');
 
-    // Âû÷èñëÿåì ñòàòèñòèêó ÷åðåç FDatabaseStats
+
     FDatabaseStats.CalculateStats;
 
-    // Ïîëó÷àåì ðåçóëüòàòû
+
     PageStats := FDatabaseStats.PageStats;
     TxStats := FDatabaseStats.TransactionStats;
     RelStats := FDatabaseStats.RelationStats;
 
-    // Îáíîâëÿåì ïðîãðåññ áàð (äëÿ íàãëÿäíîñòè, õîòÿ òåïåðü ýòî áûñòðî)
+
     pbDetectedPage.Min := 0;
-    pbDetectedPage.Max := 100; // Óñëîâíî
+    pbDetectedPage.Max := 100;
     pbDetectedPage.Position := 100;
 
-    // Âûâîäèì ðåçóëüòàòû ïîäñ÷åòà
+
     lstLog.Items.Add('--- Page Type Summary ---');
     lstLog.Items.Add('Count Header Pages: ' + IntToStr(PageStats.HeaderPages));
     lstLog.Items.Add('Count Page Inventory Pages (PIP): ' + IntToStr(PageStats.PipPages));
     lstLog.Items.Add('Count Transaction Inventory Pages (TIP): ' + IntToStr(PageStats.TipPages));
     lstLog.Items.Add('Count Pointer Pages: ' + IntToStr(PageStats.PointerPages));
     lstLog.Items.Add('Count Data Pages: ' + IntToStr(PageStats.DataPages));
     lstLog.Items.Add('Count Index Root Pages: ' + IntToStr(PageStats.IndexRootPages));
-    lstLog.Items.Add('Count Index B-Tree Pages: ' + IntToStr(PageStats.IndexBtreePages)); // Èñïðàâëåíî
+    lstLog.Items.Add('Count Index B-Tree Pages: ' + IntToStr(PageStats.IndexBtreePages));
     lstLog.Items.Add('Count Blob Pages: ' + IntToStr(PageStats.BlobPages));
     lstLog.Items.Add('Count Generator Pages: ' + IntToStr(PageStats.GeneratorPages));
     lstLog.Items.Add('Count Write Ahead Log Pages: ' + IntToStr(PageStats.WalPages));
@@ -415,12 +423,10 @@ procedure TfrmMain.btnWriteFlagsClick(Sender: TObject);
   try
     NewFlags := FFlagManager.GetFlags;
 
-    // Îáíîâëÿåì ôëàãè â ñîîòâåòñòâèè ñ ÷åêáîêñàìè
+
     NewFlags.ForceWrite := chkSetFW.Checked;
     NewFlags.ReadOnly := chkReadOnly.Checked;
-    // Äîáàâüòå äðóãèå ôëàãè ïî íåîáõîäèìîñòè
 
-    // Óñòàíàâëèâàåì ôëàãè ÷åðåç FFlagManager
     FFlagManager.SetFlags(NewFlags);
 
     Application.MessageBox('Header flags updated!', 'Information', MB_OK + MB_ICONINFORMATION);
@@ -452,14 +458,14 @@ procedure TfrmMain.UpdateFlagCheckboxesFromManager;
   end;
 end;
 
-// tsFlagsShow - òåïåðü èñïîëüçóåò UpdateFlagCheckboxesFromManager
+
 procedure TfrmMain.tsFlagsShow(Sender: TObject);
 begin
-  // Îáíîâëÿåì ÷åêáîêñû ïðè ïîêàçå âêëàäêè
+
   UpdateFlagCheckboxesFromManager;
 end;
 
-// pgcServicesChange - îáíîâëÿåò MaxValue äëÿ sePosition
+
 procedure TfrmMain.pgcServicesChange(Sender: TObject);
 begin
   if tsGenerateNewPage.Showing then
@@ -513,15 +519,13 @@ procedure TfrmMain.btnGotoPageClick(Sender: TObject);
     end;
 
     btnGotoPage.Enabled := False;
-    mmoData.Clear; // Î÷èùàåì Memo ïåðåä âûâîäîì
-
-    // Èñïîëüçóåì ôóíêöèþ èç uDataPage äëÿ èçâëå÷åíèÿ ôðàãìåíòîâ
+    mmoData.Clear;
     PageFragments := ExtractDataFragments(PageInfo.Buffer, PageInfo.Size);
 
     pbDataProgress.Min := 0;
-    pbDataProgress.Max := Length(PageFragments) - 1; // Êîëè÷åñòâî ôðàãìåíòîâ
+    pbDataProgress.Max := Length(PageFragments) - 1;
 
-    for i := 0 to High(PageFragments) do // Èñïîëüçóåì High äëÿ ìàññèâà
+    for i := 0 to High(PageFragments) do
     begin
       pbDataProgress.Position := i;
       mmoData.Lines.Add('Fragment ' + IntToStr(i) + ', Offset: ' + IntToStr(PageFragments[i].Offset));