fix: prevent leaking connection when snapshot backup fail (cloudnative-pg#6879)

The instance manager allocates a PostgreSQL connection every time a
volume snapshot backup need to be taken and then runs
`pg_backup_start` on it.  That connection will stay up until the backup
is done.

If the snapshotting process fails, the backup will be marked as failed
but the connection won't be closed.

This patch ensures that a successive backup is able to proceed correctly
by closing the previous connection.

A successive commit will allow the operator to close that stale commit
even when there's no successive backup.

Partially closes cloudnative-pg#6761 

Signed-off-by: Leonardo Cecchi <leonardo.cecchi@enterprisedb.com>
Signed-off-by: Armando Ruocco <armando.ruocco@enterprisedb.com>
Co-authored-by: Armando Ruocco <armando.ruocco@enterprisedb.com>

Author: leonardoce

pkg/management/postgres/webserver/backup_connection.go

@@ -85,6 +85,13 @@ func (bc *backupConnection) closeConnection(backupName string) error {
 	return bc.conn.Close()
 }
 
+func (bc *backupConnection) forceCloseConnection() error {
+	bc.sync.Lock()
+	defer bc.sync.Unlock()
+
+	return bc.conn.Close()
+}
+
 func (bc *backupConnection) executeWithLock(backupName string, cb func() error) {
 	bc.sync.Lock()
 	defer bc.sync.Unlock()

pkg/management/postgres/webserver/remote.go

@@ -277,7 +277,10 @@ func (ws *remoteWebserverEndpoints) backup(w http.ResponseWriter, req *http.Requ
 				sendUnprocessableEntityJSONResponse(w, "PROCESS_ALREADY_RUNNING", "")
 				return
 			}
-			if err := ws.currentBackup.closeConnection(p.BackupName); err != nil {
+			log.Info("trying to close the current backup connection",
+				"backupName", ws.currentBackup.data.BackupName,
+			)
+			if err := ws.currentBackup.forceCloseConnection(); err != nil {
 				if !errors.Is(err, sql.ErrConnDone) {
 					log.Error(err, "Error while closing backup connection (start)")
 				}