.env file support with retry mechanism

docs/include/clone.rst

@@ -43,4 +43,5 @@
      --origin                      Use this Postgres replication origin node name
      --endpos                      Stop replaying changes when reaching this LSN
      --use-copy-binary             Use the COPY BINARY format for COPY operations
+     --connection-retry-timeout    Number of seconds to retry before connection times out
 

docs/include/copy-db.rst

@@ -3,24 +3,25 @@
    pgcopydb copy db: Copy an entire database from source to target
    usage: pgcopydb copy db  --source ... --target ... [ --table-jobs ... --index-jobs ... ] 
 
-     --source              Postgres URI to the source database
-     --target              Postgres URI to the target database
-     --dir                 Work directory to use
-     --table-jobs          Number of concurrent COPY jobs to run
-     --index-jobs          Number of concurrent CREATE INDEX jobs to run
-     --restore-jobs        Number of concurrent jobs for pg_restore
-     --drop-if-exists      On the target database, clean-up from a previous run first
-     --roles               Also copy roles found on source to target
-     --no-owner            Do not set ownership of objects to match the original database
-     --no-acl              Prevent restoration of access privileges (grant/revoke commands).
-     --no-comments         Do not output commands to restore comments
-     --no-tablespaces      Do not output commands to select tablespaces
-     --skip-large-objects  Skip copying large objects (blobs)
-     --filters <filename>  Use the filters defined in <filename>
-     --fail-fast           Abort early in case of error
-     --restart             Allow restarting when temp files exist already
-     --resume              Allow resuming operations after a failure
-     --not-consistent      Allow taking a new snapshot on the source database
-     --snapshot            Use snapshot obtained with pg_export_snapshot
-     --use-copy-binary     Use the COPY BINARY format for COPY operations
+     --source                      Postgres URI to the source database
+     --target                      Postgres URI to the target database
+     --dir                         Work directory to use
+     --table-jobs                  Number of concurrent COPY jobs to run
+     --index-jobs                  Number of concurrent CREATE INDEX jobs to run
+     --restore-jobs                Number of concurrent jobs for pg_restore
+     --drop-if-exists              On the target database, clean-up from a previous run first
+     --roles                       Also copy roles found on source to target
+     --no-owner                    Do not set ownership of objects to match the original database
+     --no-acl                      Prevent restoration of access privileges (grant/revoke commands).
+     --no-comments                 Do not output commands to restore comments
+     --no-tablespaces              Do not output commands to select tablespaces
+     --skip-large-objects          Skip copying large objects (blobs)
+     --filters <filename>          Use the filters defined in <filename>
+     --fail-fast                   Abort early in case of error
+     --restart                     Allow restarting when temp files exist already
+     --resume                      Allow resuming operations after a failure
+     --not-consistent              Allow taking a new snapshot on the source database
+     --snapshot                    Use snapshot obtained with pg_export_snapshot
+     --use-copy-binary             Use the COPY BINARY format for COPY operations
+     --connection-retry-timeout    Number of seconds to retry before connection times out
 

docs/include/dump-roles.rst

@@ -3,8 +3,9 @@
    pgcopydb dump roles: Dump source database roles as custome file in work directory
    usage: pgcopydb dump roles  --source <URI>
 
-     --source            Postgres URI to the source database
-     --target            Directory where to save the dump files
-     --dir               Work directory to use
-     --no-role-passwords Do not dump passwords for roles
+     --source                      Postgres URI to the source database
+     --target                      Directory where to save the dump files
+     --dir                         Work directory to use
+     --no-role-passwords           Do not dump passwords for roles
+     --connection-retry-timeout    Number of seconds to retry before connection times out
 

docs/include/dump-schema.rst

@@ -3,10 +3,11 @@
    pgcopydb dump schema: Dump source database schema as custom files in work directory
    usage: pgcopydb dump schema  --source <URI> 
 
-     --source             Postgres URI to the source database
-     --target             Directory where to save the dump files
-     --dir                Work directory to use
-     --skip-extensions    Skip restoring extensions
-     --filters <filename> Use the filters defined in <filename>
-     --snapshot           Use snapshot obtained with pg_export_snapshot
+     --source                      Postgres URI to the source database
+     --target                      Directory where to save the dump files
+     --dir                         Work directory to use
+     --skip-extensions             Skip restoring extensions
+     --filters <filename>          Use the filters defined in <filename>
+     --snapshot                    Use snapshot obtained with pg_export_snapshot
+     --connection-retry-timeout    Number of seconds to retry before connection times out
 

docs/include/list-table-parts.rst

@@ -3,12 +3,13 @@
    pgcopydb list table-parts: List a source table copy partitions
    usage: pgcopydb list table-parts  --source ... 
 
-     --source                    Postgres URI to the source database
-     --force                     Force fetching catalogs again
-     --schema-name               Name of the schema where to find the table
-     --table-name                Name of the target table
-     --split-tables-larger-than  Size threshold to consider partitioning
-     --split-max-parts           Maximum number of jobs for Same-table concurrency 
-     --skip-split-by-ctid        Skip the ctid split
-     --estimate-table-sizes      Allow using estimates for relation sizes
+     --source                      Postgres URI to the source database
+     --force                       Force fetching catalogs again
+     --schema-name                 Name of the schema where to find the table
+     --table-name                  Name of the target table
+     --split-tables-larger-than    Size threshold to consider partitioning
+     --split-max-parts             Maximum number of jobs for Same-table concurrency 
+     --skip-split-by-ctid          Skip the ctid split
+     --estimate-table-sizes        Allow using estimates for relation sizes
+     --connection-retry-timeout    Number of seconds to retry before connection times out
 

docs/include/ping.rst

@@ -3,6 +3,7 @@
    pgcopydb ping: Attempt to connect to the source and target instances
    usage: pgcopydb ping  --source ... --target ... 
 
-     --source              Postgres URI to the source database
-     --target              Postgres URI to the target database
+     --source                      Postgres URI to the source database
+     --target                      Postgres URI to the target database
+     --connection-retry-timeout    Number of seconds to retry before connection times out
 

docs/include/restore-schema.rst

@@ -3,17 +3,18 @@
    pgcopydb restore schema: Restore a database schema from custom files to target database
    usage: pgcopydb restore schema  --dir <dir> [ --source <URI> ] --target <URI> 
 
-     --source             Postgres URI to the source database
-     --target             Postgres URI to the target database
-     --dir                Work directory to use
-     --restore-jobs       Number of concurrent jobs for pg_restore
-     --drop-if-exists     On the target database, clean-up from a previous run first
-     --no-owner           Do not set ownership of objects to match the original database
-     --no-acl             Prevent restoration of access privileges (grant/revoke commands).
-     --no-comments        Do not output commands to restore comments
-     --no-tablespaces     Do not output commands to select tablespaces
-     --filters <filename> Use the filters defined in <filename>
-     --restart            Allow restarting when temp files exist already
-     --resume             Allow resuming operations after a failure
-     --not-consistent     Allow taking a new snapshot on the source database
+     --source                      Postgres URI to the source database
+     --target                      Postgres URI to the target database
+     --dir                         Work directory to use
+     --restore-jobs                Number of concurrent jobs for pg_restore
+     --drop-if-exists              On the target database, clean-up from a previous run first
+     --no-owner                    Do not set ownership of objects to match the original database
+     --no-acl                      Prevent restoration of access privileges (grant/revoke commands).
+     --no-comments                 Do not output commands to restore comments
+     --no-tablespaces              Do not output commands to select tablespaces
+     --filters <filename>          Use the filters defined in <filename>
+     --restart                     Allow restarting when temp files exist already
+     --resume                      Allow resuming operations after a failure
+     --not-consistent              Allow taking a new snapshot on the source database
+     --connection-retry-timeout    Number of seconds to retry before connection times out
 

docs/include/snapshot.rst

@@ -9,4 +9,5 @@
      --plugin                      Output plugin to use (test_decoding, wal2json)
      --wal2json-numeric-as-string  Print numeric data type as string when using wal2json output plugin
      --slot-name                   Use this Postgres replication slot name
+     --connection-retry-timeout    Number of seconds to retry before connection times out
 

docs/include/stream-setup.rst

@@ -14,4 +14,5 @@
      --wal2json-numeric-as-string  Print numeric data type as string when using wal2json output plugin
      --slot-name                   Stream changes recorded by this slot
      --origin                      Name of the Postgres replication origin
+     --connection-retry-timeout    Number of seconds to retry before connection times out
 

docs/ref/pgcopydb_clone.rst

@@ -333,6 +333,18 @@ Here is an example implement the previous steps:
    $ kill %1
 
 
+Retry mechanism
+^^^^^^^^^^^^^^^
+
+When a connection to either the source or target database fails, the command will
+attempt to reconnect several times before giving up. The maximum duration for these
+retries is determined by the ``--connection-retry-timeout`` parameter, which
+defaults to 60 seconds.
+
+Before each retry, the command will attempt to fetch updated connection strings from
+the ``.env`` file located in either ``${XDG_CONFIG_HOME}/pgcopydb/.env`` or
+``${HOME}/.config/pgcopydb/.env``.
+
 Options
 -------
 
@@ -714,6 +726,10 @@ The following options are available to ``pgcopydb clone``:
 Environment
 -----------
 
+Environment variables that begin with ``PGCOPYDB_`` can be set directly in the
+environment or read from the ``.env`` file located in either 
+``${XDG_CONFIG_HOME}/pgcopydb/.env`` or ``${HOME}/.config/pgcopydb/.env``.
+
 PGCOPYDB_SOURCE_PGURI
 
   Connection string to the source Postgres instance. When ``--source`` is

docs/ref/pgcopydb_compare.rst

@@ -123,6 +123,10 @@ compare data`` subcommands:
 Environment
 -----------
 
+Environment variables that begin with ``PGCOPYDB_`` can be set directly in the
+environment or read from the ``.env`` file located in either 
+``${XDG_CONFIG_HOME}/pgcopydb/.env`` or ``${HOME}/.config/pgcopydb/.env``.
+
 PGCOPYDB_SOURCE_PGURI
 
   Connection string to the source Postgres instance. When ``--source`` is

docs/ref/pgcopydb_copy.rst

@@ -386,6 +386,10 @@ The following options are available to ``pgcopydb copy`` sub-commands:
 Environment
 -----------
 
+Environment variables that begin with ``PGCOPYDB_`` can be set directly in the
+environment or read from the ``.env`` file located in either 
+``${XDG_CONFIG_HOME}/pgcopydb/.env`` or ``${HOME}/.config/pgcopydb/.env``.
+
 PGCOPYDB_SOURCE_PGURI
 
   Connection string to the source Postgres instance. When ``--source`` is

docs/ref/pgcopydb_dump.rst

@@ -118,6 +118,10 @@ The following options are available to ``pgcopydb dump schema`` subcommand:
 Environment
 -----------
 
+Environment variables that begin with ``PGCOPYDB_`` can be set directly in the
+environment or read from the ``.env`` file located in either 
+``${XDG_CONFIG_HOME}/pgcopydb/.env`` or ``${HOME}/.config/pgcopydb/.env``.
+
 PGCOPYDB_SOURCE_PGURI
 
   Connection string to the source Postgres instance. When ``--source`` is

docs/ref/pgcopydb_follow.rst

@@ -504,6 +504,10 @@ The following options are available to ``pgcopydb follow``:
 Environment
 -----------
 
+Environment variables that begin with ``PGCOPYDB_`` can be set directly in the
+environment or read from the ``.env`` file located in either 
+``${XDG_CONFIG_HOME}/pgcopydb/.env`` or ``${HOME}/.config/pgcopydb/.env``.
+
 PGCOPYDB_SOURCE_PGURI
 
   Connection string to the source Postgres instance. When ``--source`` is

docs/ref/pgcopydb_list.rst

@@ -233,6 +233,10 @@ The following options are available to ``pgcopydb dump schema``:
 Environment
 -----------
 
+Environment variables that begin with ``PGCOPYDB_`` can be set directly in the
+environment or read from the ``.env`` file located in either 
+``${XDG_CONFIG_HOME}/pgcopydb/.env`` or ``${HOME}/.config/pgcopydb/.env``.
+
 PGCOPYDB_SOURCE_PGURI
 
   Connection string to the source Postgres instance. When ``--source`` is

docs/ref/pgcopydb_restore.rst

@@ -266,6 +266,10 @@ The following options are available to ``pgcopydb restore schema``:
 Environment
 -----------
 
+Environment variables that begin with ``PGCOPYDB_`` can be set directly in the
+environment or read from the ``.env`` file located in either 
+``${XDG_CONFIG_HOME}/pgcopydb/.env`` or ``${HOME}/.config/pgcopydb/.env``.
+
 PGCOPYDB_TARGET_PGURI
 
   Connection string to the target Postgres instance. When ``--target`` is

docs/ref/pgcopydb_snapshot.rst

@@ -95,6 +95,10 @@ The following options are available to ``pgcopydb snapshot``:
 Environment
 -----------
 
+Environment variables that begin with ``PGCOPYDB_`` can be set directly in the
+environment or read from the ``.env`` file located in either 
+``${XDG_CONFIG_HOME}/pgcopydb/.env`` or ``${HOME}/.config/pgcopydb/.env``.
+
 PGCOPYDB_SOURCE_PGURI
 
   Connection string to the source Postgres instance. When ``--source`` is

docs/ref/pgcopydb_stream.rst

@@ -366,6 +366,10 @@ The following options are available to ``pgcopydb stream`` sub-commands:
 Environment
 -----------
 
+Environment variables that begin with ``PGCOPYDB_`` can be set directly in the
+environment or read from the ``.env`` file located in either 
+``${XDG_CONFIG_HOME}/pgcopydb/.env`` or ``${HOME}/.config/pgcopydb/.env``.
+
 PGCOPYDB_SOURCE_PGURI
 
   Connection string to the source Postgres instance. When ``--source`` is

src/bin/pgcopydb/blobs.c

@@ -367,7 +367,8 @@ copydb_blob_worker(CopyDataSpec *specs)
 	bool dropIfExists = specs->restoreOptions.dropIfExists;
 
 	/* initialize our connection to the target database */
-	if (!pgsql_init(&dst, specs->connStrings.target_pguri, PGSQL_CONN_TARGET))
+	if (!pgsql_init(&dst, specs->connStrings.target_pguri, PGSQL_CONN_TARGET,
+					specs->connectionRetryTimeout))
 	{
 		/* errors have already been logged */
 		return false;

src/bin/pgcopydb/cli_clone_follow.c

@@ -66,6 +66,7 @@
 	"  --origin                      Use this Postgres replication origin node name\n" \
 	"  --endpos                      Stop replaying changes when reaching this LSN\n" \
 	"  --use-copy-binary             Use the COPY BINARY format for COPY operations\n" \
+	"  --connection-retry-timeout    Number of seconds to retry before connection times out\n" \
 
 CommandLine clone_command =
 	make_command(
@@ -212,7 +213,8 @@ clone_and_follow(CopyDataSpec *copySpecs)
 						   &(copySpecs->catalogs.source),
 						   copyDBoptions.stdIn,
 						   copyDBoptions.stdOut,
-						   logSQL))
+						   logSQL,
+						   copyDBoptions.connectionRetryTimeout))
 	{
 		/* errors have already been logged */
 		exit(EXIT_CODE_INTERNAL_ERROR);
@@ -390,7 +392,8 @@ cli_follow(int argc, char **argv)
 						   &(copySpecs.catalogs.source),
 						   copyDBoptions.stdIn,
 						   copyDBoptions.stdOut,
-						   logSQL))
+						   logSQL,
+						   copyDBoptions.connectionRetryTimeout))
 	{
 		/* errors have already been logged */
 		exit(EXIT_CODE_INTERNAL_ERROR);
@@ -562,7 +565,8 @@ cloneDB(CopyDataSpec *copySpecs)
 		if (!pg_copy_roles(&(copySpecs->pgPaths),
 						   &(copySpecs->connStrings),
 						   copySpecs->dumpPaths.rolesFilename,
-						   copySpecs->noRolesPasswords))
+						   copySpecs->noRolesPasswords,
+						   copySpecs->connectionRetryTimeout))
 		{
 			/* errors have already been logged */
 			return false;