[extension/storage/filestorage] Fix recreate from panic (#41802)

<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description
* This enables recovery from a panic when the bbolt db is corrupted and
renames the file when a panic occurs.
* This changes the `recreate` behavior to not rename the file upon every
start of the collector.

<!-- Issue number (e.g. #1234) or full URL to issue, if applicable. -->
#### Link to tracking issue
Resolves
#36840
Resolves
#35899

<!--Describe what testing was performed and which tests were added.-->
#### Testing
1. Start collector with file extension configured. This should create
the file with some content within it.
2. Stop the collector.
3. Manually edit or "break" the bbolt db file by adding characters in
random places, forcefully causing a panic
4. Start the collector
5. See logs:
```json
{"level":"info","ts":"2025-07-31T23:26:34.218-0400","msg":"Setting up own telemetry...","resource":{"service.instance.id":"d830aeff-7993-49f7-9817-a0c96af3498d","service.name":"otelcontribcol","service.version":"0.131.0-dev"}}
{"level":"info","ts":"2025-07-31T23:26:34.242-0400","msg":"Starting otelcontribcol...","resource":{"service.instance.id":"d830aeff-7993-49f7-9817-a0c96af3498d","service.name":"otelcontribcol","service.version":"0.131.0-dev"},"Version":"0.131.0-dev","NumCPU":6}
{"level":"info","ts":"2025-07-31T23:26:34.242-0400","msg":"Starting extensions...","resource":{"service.instance.id":"d830aeff-7993-49f7-9817-a0c96af3498d","service.name":"otelcontribcol","service.version":"0.131.0-dev"}}
{"level":"info","ts":"2025-07-31T23:26:34.242-0400","msg":"Extension is starting...","resource":{"service.instance.id":"d830aeff-7993-49f7-9817-a0c96af3498d","service.name":"otelcontribcol","service.version":"0.131.0-dev"},"otelcol.component.id":"file_storage/persistent_queue_storage","otelcol.component.kind":"extension"}
{"level":"info","ts":"2025-07-31T23:26:34.242-0400","msg":"Extension started.","resource":{"service.instance.id":"d830aeff-7993-49f7-9817-a0c96af3498d","service.name":"otelcontribcol","service.version":"0.131.0-dev"},"otelcol.component.id":"file_storage/persistent_queue_storage","otelcol.component.kind":"extension"}
#### This line ####
{"level":"warn","ts":"2025-07-31T23:26:34.242-0400","msg":"Database corruption detected, recreating database file","resource":{"service.instance.id":"d830aeff-7993-49f7-9817-a0c96af3498d","service.name":"otelcontribcol","service.version":"0.131.0-dev"},"otelcol.component.id":"file_storage/persistent_queue_storage","otelcol.component.kind":"extension","file":"/data/otelcol/persistent_queue_storage/exporter_otlphttp_general_logs","panic":"assertion failed: Page expected to be: 96, but self identifies as 4909050520039286100"}
{"level":"info","ts":"2025-07-31T23:26:34.243-0400","msg":"Corrupted database file renamed","resource":{"service.instance.id":"d830aeff-7993-49f7-9817-a0c96af3498d","service.name":"otelcontribcol","service.version":"0.131.0-dev"},"otelcol.component.id":"file_storage/persistent_queue_storage","otelcol.component.kind":"extension","original":"/data/otelcol/persistent_queue_storage/exporter_otlphttp_general_logs","backup":"/data/otelcol/persistent_queue_storage/exporter_otlphttp_general_logs.2025-08-27T13:26:07.479.backup"}
#### through this line ####
{"level":"info","ts":"2025-07-31T23:26:34.244-0400","msg":"New queue metadata key not found, attempting to load legacy format.","resource":{"service.instance.id":"d830aeff-7993-49f7-9817-a0c96af3498d","service.name":"otelcontribcol","service.version":"0.131.0-dev"},"otelcol.component.id":"otlphttp/general","otelcol.component.kind":"exporter","otelcol.signal":"logs"}
{"level":"info","ts":"2025-07-31T23:26:34.244-0400","msg":"Initializing new persistent queue","resource":{"service.instance.id":"d830aeff-7993-49f7-9817-a0c96af3498d","service.name":"otelcontribcol","service.version":"0.131.0-dev"},"otelcol.component.id":"otlphttp/general","otelcol.component.kind":"exporter","otelcol.signal":"logs"}
{"level":"info","ts":"2025-07-31T23:26:34.244-0400","msg":"Successfully migrated to consolidated metadata format","resource":{"service.instance.id":"d830aeff-7993-49f7-9817-a0c96af3498d","service.name":"otelcontribcol","service.version":"0.131.0-dev"},"otelcol.component.id":"otlphttp/general","otelcol.component.kind":"exporter","otelcol.signal":"logs"}
{"level":"info","ts":"2025-07-31T23:26:34.255-0400","msg":"Everything is ready. Begin running and processing data.","resource":{"service.instance.id":"d830aeff-7993-49f7-9817-a0c96af3498d","service.name":"otelcontribcol","service.version":"0.131.0-dev"}}
```

<!--Describe the documentation added.-->
#### Documentation
Updated the existing documentation with behavioral changes for
`recreate` option.

<!--Please delete paragraphs that you did not use before submitting.-->

---------

Co-authored-by: Antoine Toulme <[email protected]>

Author: briandavis-viz

.chloggen/file-extension-panic-recovery-fix.yaml

@@ -0,0 +1,29 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: "bug_fix"
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: "extension/storage"
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Fix 'recreate' rename file only on panic"
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [41802]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  * This recovers from a panic when the bbolt db is corrupted and renames the file when a panic occurs.
+  * This changes the `recreate` behavior to not rename the file upon every start of the collector.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

extension/storage/filestorage/README.md

@@ -31,10 +31,14 @@ The default timeout is `1s`.
 By default, the directories will be created with `0750 (rwxr-x---)` permissions, minus the process umask.
 Use `directory_permissions` to customize directory creation permissions, minus the process umask.
 
-`recreate` when set, will rename the existing data storage to `{filename}.backup` and a new data file will be created from scratch. This option is useful if underlying database is corrupted and as a result, it can halt the entire collector process due to a panic. See (#36840)[https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/36840] for more details.
+`recreate` when set, the filestorage extension will automatically rename the corrupted bbolt database and create a new one when certain bbolt panics occur. 
+See (#35899)[https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/35899] for more details.
+
+If the database fails to open due to corruption (resulting in a panic), the corrupted file will be automatically renamed to `{filename}.{ISO 8601 timestamp}.backup` and a new data file will be created from scratch. This allows the collector to continue operating even when encountering certain bbolt panics. If no corruption is detected, the existing database continues to be used normally.
+There may still be scenarios where manually removing or renaming the file may be required, and this feature flag is not a panacea for all bbolt panics you can encounter.
 
 > [!Note]
-> Enabling `recreate` will regenerate the database files, which may lead to data duplication or data loss. 
+> When database corruption is detected and automatic recovery is triggered, the corrupted data will be moved to a `.backup` file. While this prevents complete data loss, the collector will start with a fresh database, which may lead to data duplication or loss of component state.
 
 ## Compaction
 `compaction` defines how and when files should be compacted. There are two modes of compaction available (both of which can be set concurrently):
@@ -88,6 +92,7 @@ extensions:
   file_storage/all_settings:
     directory: /var/lib/otelcol/mydir
     timeout: 1s
+    recreate: true
     compaction:
       on_start: true
       directory: /tmp/

extension/storage/filestorage/extension.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/extension"
@@ -71,12 +72,9 @@ func (lfs *localFileStorage) GetClient(_ context.Context, kind component.Kind, e
 
 	rawName = sanitize(rawName)
 	absoluteName := filepath.Join(lfs.cfg.Directory, rawName)
-	if lfs.cfg.Recreate {
-		if err := os.Rename(absoluteName, absoluteName+".backup"); err != nil {
-			return nil, fmt.Errorf("error renaming the database. Please remove %s manually: %w", absoluteName, err)
-		}
-	}
-	client, err := newClient(lfs.logger, absoluteName, lfs.cfg.Timeout, lfs.cfg.Compaction, !lfs.cfg.FSync)
+
+	// Try to create client, handling panics if recreate is enabled
+	client, err := lfs.createClientWithPanicRecovery(absoluteName)
 	if err != nil {
 		return nil, err
 	}
@@ -92,6 +90,45 @@ func (lfs *localFileStorage) GetClient(_ context.Context, kind component.Kind, e
 	return client, nil
 }
 
+// createClientWithPanicRecovery attempts to create a client, and if recreate is enabled
+// and a panic occurs (typically due to database corruption), it will rename the file
+// and try again with a fresh database
+func (lfs *localFileStorage) createClientWithPanicRecovery(absoluteName string) (client *fileStorageClient, err error) {
+	// First attempt: try to create client normally
+	if !lfs.cfg.Recreate {
+		// If recreate is disabled, just try once
+		return newClient(lfs.logger, absoluteName, lfs.cfg.Timeout, lfs.cfg.Compaction, !lfs.cfg.FSync)
+	}
+
+	// If recreate is enabled, handle potential panics during database opening
+	defer func() {
+		if r := recover(); r != nil {
+			lfs.logger.Warn("Database corruption detected, recreating database file",
+				zap.String("file", absoluteName),
+				zap.Any("panic", r))
+
+			// Rename the corrupted file with ISO 8601 timestamp
+			timestamp := time.Now().Format("2006-01-02T15:04:05.000")
+			backupName := absoluteName + "." + timestamp + ".backup"
+			if renameErr := os.Rename(absoluteName, backupName); renameErr != nil {
+				err = fmt.Errorf("error renaming corrupted database. Please remove %s manually: %w", absoluteName, renameErr)
+				return
+			}
+
+			lfs.logger.Info("Corrupted database file renamed",
+				zap.String("original", absoluteName),
+				zap.String("backup", backupName))
+
+			// Try to create client again with fresh database
+			client, err = newClient(lfs.logger, absoluteName, lfs.cfg.Timeout, lfs.cfg.Compaction, !lfs.cfg.FSync)
+		}
+	}()
+
+	// Try to create the client normally first
+	client, err = newClient(lfs.logger, absoluteName, lfs.cfg.Timeout, lfs.cfg.Compaction, !lfs.cfg.FSync)
+	return client, err
+}
+
 func kindString(k component.Kind) string {
 	switch k {
 	case component.KindReceiver:

extension/storage/filestorage/extension_test.go

@@ -666,7 +666,8 @@ func TestRecreate(t *testing.T) {
 		require.NoError(t, ext.Shutdown(ctx))
 	}
 
-	// step 3: re-create the extension, but with Recreate=true and make sure that the data is not preset
+	// step 3: re-create the extension, but with Recreate=true and make sure that the data still exists
+	// (since recreate now only happens on panic, not always when recreate=true)
 	{
 		config.Recreate = true
 		ext, err := f.Create(ctx, extensiontest.NewNopSettings(f.Type()), config)
@@ -679,9 +680,9 @@ func TestRecreate(t *testing.T) {
 		require.NoError(t, err)
 		require.NotNil(t, client)
 
-		// The data shouldn't exist.
+		// The data should still exist since no panic occurred
 		val, err := client.Get(ctx, "key")
-		require.Nil(t, val)
+		require.Equal(t, val, []byte("val"))
 		require.NoError(t, err)
 
 		// close the extension