glog: generate a Fatalf-like error message when writing to logsinks fails (#76)

Writing to logsinks can fail (for example due to "no space left on device" or I/O errors). When that happens glog has no reasonable way to continue and causes the program to exit with exit status 2.

Previously glog reused the metadata of the current call to print an error message, but that was problematic. Depending on the current call's log severity it's possible that the program just exited without printing anything. That's confusing and hard to debug.

To fix that, glog creates now a new FATAL-level metadata object and prints a clearer error message (with stacks). In most situations this will at least be logged to stderr.

Thanks @atetubou for the initial fix!

cl/750790337 (google-internal)
cl/752634801 (google-internal)

Author: chressie

glog.go

@@ -238,6 +238,8 @@ func ctxlogf(ctx context.Context, depth int, severity logsink.Severity, verbose
 	metaPool.Put(metai)
 }
 
+var sinkErrOnce sync.Once
+
 func sinkf(meta *logsink.Meta, format string, args ...any) {
 	meta.Depth++
 	n, err := logsink.Printf(meta, format, args...)
@@ -247,9 +249,20 @@ func sinkf(meta *logsink.Meta, format string, args ...any) {
 	}
 
 	if err != nil {
-		logsink.Printf(meta, "glog: exiting because of error: %s", err)
-		sinks.file.Flush()
-		os.Exit(2)
+		// Best-effort to generate a reasonable Fatalf-like
+		// error message in all sinks that are still here for
+		// the first goroutine that comes here and terminate
+		// the process.
+		sinkErrOnce.Do(func() {
+			m := &logsink.Meta{}
+			m.Time = timeNow()
+			m.Severity = logsink.Fatal
+			m.Thread = int64(pid)
+			_, m.File, m.Line, _ = runtime.Caller(0)
+			format, args := appendBacktrace(1, "log: exiting because of error writing previous log to sinks: %v", []any{err})
+			logsink.Printf(m, format, args...)
+			flushAndAbort()
+		})
 	}
 }
 
@@ -642,6 +655,10 @@ func ErrorContextDepthf(ctx context.Context, depth int, format string, args ...a
 
 func ctxfatalf(ctx context.Context, depth int, format string, args ...any) {
 	ctxlogf(ctx, depth+1, logsink.Fatal, false, withStack, format, args...)
+	flushAndAbort()
+}
+
+func flushAndAbort() {
 	sinks.file.Flush()
 
 	err := abortProcess() // Should not return.