bug: runner pool external dependencies inclusion fix (#5199)

* bug: runner pool external depednencies inclusion fix

* chore: discovery simplification

* discovery simplification

* chore: simplify reporting

* chore: external dependencies update

* chore: PR cleanup

* chore: strict controls update

* chore: failing tests fixes

* chore: failing tests fixes

* chore: docs simplification

Author: denis256

cli/flags/shared/shared.go

@@ -7,6 +7,7 @@ package shared
 import (
 	"github.com/gruntwork-io/terragrunt/cli/flags"
 	"github.com/gruntwork-io/terragrunt/internal/cli"
+	"github.com/gruntwork-io/terragrunt/internal/strict/controls"
 	"github.com/gruntwork-io/terragrunt/options"
 )
 
@@ -112,6 +113,13 @@ func NewQueueFlags(opts *options.TerragruntOptions, prefix flags.Prefix) cli.Fla
 				EnvVars:     tgPrefix.EnvVars(QueueExcludeExternalFlagName),
 				Destination: &opts.IgnoreExternalDependencies,
 				Usage:       "Ignore external dependencies for --all commands.",
+				Hidden:      true,
+				Action: func(ctx *cli.Context, value bool) error {
+					if value {
+						return opts.StrictControls.FilterByNames(controls.QueueExcludeExternal).Evaluate(ctx.Context)
+					}
+					return nil
+				},
 			},
 			flags.WithDeprecatedEnvVars(terragruntPrefix.EnvVars("ignore-external-dependencies"), terragruntPrefixControl),
 		),

docs-starlight/src/content/docs/04-reference/03-strict-controls.mdx

@@ -193,6 +193,19 @@ and unit-b.
 The `**` double wildcard indicates any character including `/`, this allows
 matching multiple directory depths e.g. `units/**` matches unit-a, unit-b, and unit-c.
 
+### queue-exclude-external
+
+Throw an error when using the deprecated `--queue-exclude-external` flag.
+
+**Reason**: External dependencies are now excluded by default. The `--queue-exclude-external` flag is no longer needed and has been deprecated. Use `--queue-include-external` if you need to include external dependencies.
+
+**Example**:
+```bash
+# This will show a warning (or error with strict control enabled)
+$ terragrunt run --all plan --queue-exclude-external
+WARN  The `--queue-exclude-external` flag is deprecated and will be removed in a future version of Terragrunt. External dependencies are now excluded by default.
+```
+
 ## Control Categories
 
 Certain strict controls are grouped into categories to make it easier to enable multiple strict controls at once.
@@ -217,6 +230,7 @@ Throw an error when using the deprecated flags.
 **Controls**:
 
 - [terragrunt-prefix-flags](#terragrunt-prefix-flags)
+- [queue-exclude-external](#queue-exclude-external)
 
 ### deprecated-env-vars
 

docs-starlight/src/data/flags/queue-exclude-external.mdx

@@ -6,8 +6,14 @@ env:
   - TG_QUEUE_EXCLUDE_EXTERNAL
 ---
 
-When enabled, Terragrunt will exclude external dependencies (dependencies outside the current working directory) from the queue when running commands with [`all`](/docs/reference/cli/commands/run#all).
+import { Aside } from '@astrojs/starlight/components';
 
-Note that an external dependency is a dependency that is outside the current Terragrunt working directory and is not within any directories specified by [`queue-include-dir`](/docs/reference/cli/commands/run#queue-include-dir).
+<Aside type="caution" title="Deprecated">
+This flag is deprecated and will be removed in a future version of Terragrunt. External dependencies are now excluded by default, making this flag unnecessary.
+
+Use [`--queue-include-external`](/docs/reference/cli/commands/run#queue-include-external) if you need to include external dependencies.
+</Aside>
 
-This flag is useful when you want to limit the scope of execution to only units within your current working directory structure.
+When enabled, Terragrunt will exclude external dependencies (dependencies outside the current working directory) from the queue when running commands with [`--all`](/docs/reference/cli/commands/run#all).
+
+Note that an external dependency is a dependency that is outside the current Terragrunt working directory and is not within any directories specified by [`queue-include-dir`](/docs/reference/cli/commands/run#queue-include-dir).

docs-starlight/src/data/flags/queue-include-external.mdx

@@ -6,4 +6,8 @@ env:
   - TG_QUEUE_INCLUDE_EXTERNAL
 ---
 
-When enabled, Terragrunt will include external dependencies (dependencies discovered outside the current working directory) in the queue when running commands with [`--all`](/docs/reference/cli/commands/run#all) or [`--graph`](/docs/reference/cli/commands/run#graph).
+By default, Terragrunt excludes external dependencies (dependencies discovered outside the current working directory) from the queue when running commands with [`--all`](/docs/reference/cli/commands/run#all) or [`--graph`](/docs/reference/cli/commands/run#graph).
+
+When this flag is enabled, Terragrunt will include those external dependencies in the queue.
+
+This flag is useful when you have units that depend on infrastructure outside your current working directory and need those dependencies to be processed as part of the run.

internal/discovery/dependencydiscovery.go

@@ -8,8 +8,10 @@ import (
 	"github.com/gruntwork-io/terragrunt/config/hclparse"
 	"github.com/gruntwork-io/terragrunt/internal/component"
 	"github.com/gruntwork-io/terragrunt/internal/errors"
+	"github.com/gruntwork-io/terragrunt/internal/report"
 	"github.com/gruntwork-io/terragrunt/options"
 	"github.com/gruntwork-io/terragrunt/pkg/log"
+	"github.com/gruntwork-io/terragrunt/util"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -18,6 +20,7 @@ type DependencyDiscovery struct {
 	discoveryContext     *component.DiscoveryContext
 	components           *component.ThreadSafeComponents
 	externalDependencies *component.ThreadSafeComponents
+	report               *report.Report
 	mu                   *sync.RWMutex
 	seenComponents       map[string]struct{}
 	parserOptions        []hclparse.Option
@@ -76,6 +79,13 @@ func (dd *DependencyDiscovery) WithDiscoveryContext(discoveryContext *component.
 	return dd
 }
 
+// WithReport sets the report for recording excluded external dependencies.
+func (dd *DependencyDiscovery) WithReport(r *report.Report) *DependencyDiscovery {
+	dd.report = r
+
+	return dd
+}
+
 func (dd *DependencyDiscovery) Discover(
 	ctx context.Context,
 	l log.Logger,
@@ -196,7 +206,7 @@ func (dd *DependencyDiscovery) discoverDependencies(
 
 	for _, depPath := range depPaths {
 		g.Go(func() error {
-			depComponent := dd.dependencyToDiscover(dComponent, depPath)
+			depComponent := dd.dependencyToDiscover(l, dComponent, depPath)
 			if depComponent == nil {
 				return nil
 			}
@@ -231,6 +241,7 @@ func (dd *DependencyDiscovery) discoverDependencies(
 // marking as external if it's outside the working directory of discovery, and linking dependencies.
 // Returns nil if the dependency shouldn't be involved in discovery any further (e.g., already processed or ignored).
 func (dd *DependencyDiscovery) dependencyToDiscover(
+	l log.Logger,
 	dComponent component.Component,
 	depPath string,
 ) component.Component {
@@ -274,6 +285,15 @@ func (dd *DependencyDiscovery) dependencyToDiscover(
 		existingDep, _ = dd.externalDependencies.EnsureComponent(depComponent)
 		dComponent.AddDependency(existingDep)
 
+		l.Debugf("Excluded external dependency: %s", depComponent.DisplayPath())
+
+		// Record in report as excluded external dependency
+		if dd.report != nil {
+			absPath := util.CleanPath(depPath)
+			run, _ := dd.report.EnsureRun(absPath)
+			_ = dd.report.EndRun(run.Path, report.WithResult(report.ResultExcluded), report.WithReason(report.ReasonExcludeExternal))
+		}
+
 		dd.markSeen(depPath)
 
 		return nil

internal/discovery/discovery.go

@@ -17,6 +17,7 @@ import (
 	"github.com/gruntwork-io/terragrunt/internal/component"
 	"github.com/gruntwork-io/terragrunt/internal/experiment"
 	"github.com/gruntwork-io/terragrunt/internal/filter"
+	"github.com/gruntwork-io/terragrunt/internal/report"
 	"github.com/gruntwork-io/terragrunt/shell"
 	"github.com/gruntwork-io/terragrunt/util"
 
@@ -110,6 +111,9 @@ type Discovery struct {
 	// gitExpressions contains Git filter expressions that require worktree discovery
 	gitExpressions filter.GitExpressions
 
+	// report is used for recording excluded external dependencies during discovery.
+	report *report.Report
+
 	// compiledIncludePatterns are precompiled glob patterns for includeDirs.
 	compiledIncludePatterns []CompiledPattern
 
@@ -253,6 +257,13 @@ func (d *Discovery) WithDiscoverExternalDependencies() *Discovery {
 	return d
 }
 
+// WithReport sets the report for recording excluded external dependencies.
+func (d *Discovery) WithReport(r *report.Report) *Discovery {
+	d.report = r
+
+	return d
+}
+
 // WithSuppressParseErrors sets the SuppressParseErrors flag to true.
 func (d *Discovery) WithSuppressParseErrors() *Discovery {
 	d.suppressParseErrors = true
@@ -1158,6 +1169,11 @@ func (d *Discovery) Discover(
 						dependencyDiscovery = dependencyDiscovery.WithParserOptions(d.parserOptions)
 					}
 
+					// pass report for recording excluded external dependencies
+					if d.report != nil {
+						dependencyDiscovery = dependencyDiscovery.WithReport(d.report)
+					}
+
 					discoveryErr := dependencyDiscovery.Discover(discoveryCtx, l, opts, dependencyStartingComponents)
 					if discoveryErr != nil {
 						if !d.suppressParseErrors {

internal/runner/common/options.go

@@ -47,13 +47,27 @@ func WithParseOptions(parserOptions []hclparse.Option) Option {
 	}
 }
 
+// ReportProvider exposes the report attached to an Option.
+type ReportProvider interface {
+	GetReport() *report.Report
+}
+
+// reportOption wraps a report and implements both Option and ReportProvider.
+type reportOption struct {
+	report *report.Report
+}
+
+func (o reportOption) Apply(stack StackRunner) {
+	stack.SetReport(o.report)
+}
+
+func (o reportOption) GetReport() *report.Report {
+	return o.report
+}
+
 // WithReport attaches a report collector to the stack, enabling run summaries and metrics.
 func WithReport(r *report.Report) Option {
-	return optionImpl{
-		apply: func(stack StackRunner) {
-			stack.SetReport(r)
-		},
-	}
+	return reportOption{report: r}
 }
 
 // WorktreeOption carries worktrees through the runner pipeline for git filter expressions.

internal/runner/runnerpool/builder.go

@@ -21,6 +21,5 @@ func Build(
 		return nil, err
 	}
 
-	// Create the runner
 	return createRunner(ctx, l, terragruntOptions, discovered, opts...)
 }

internal/runner/runnerpool/builder_helpers.go

@@ -8,6 +8,7 @@ import (
 	"github.com/gruntwork-io/terragrunt/internal/discovery"
 	"github.com/gruntwork-io/terragrunt/internal/errors"
 	"github.com/gruntwork-io/terragrunt/internal/filter"
+	"github.com/gruntwork-io/terragrunt/internal/report"
 	"github.com/gruntwork-io/terragrunt/internal/runner/common"
 	"github.com/gruntwork-io/terragrunt/internal/worktrees"
 	"github.com/gruntwork-io/terragrunt/options"
@@ -76,6 +77,17 @@ func extractWorktrees(opts []common.Option) *worktrees.Worktrees {
 	return nil
 }
 
+// extractReport finds ReportProvider in options and returns the report.
+func extractReport(opts []common.Option) *report.Report {
+	for _, opt := range opts {
+		if rp, ok := opt.(common.ReportProvider); ok {
+			return rp.GetReport()
+		}
+	}
+
+	return nil
+}
+
 // newBaseDiscovery constructs the base discovery with common immutable options.
 func newBaseDiscovery(
 	tgOpts *options.TerragruntOptions,
@@ -88,10 +100,9 @@ func newBaseDiscovery(
 		anyOpts[i] = v
 	}
 
-	return discovery.
+	d := discovery.
 		NewDiscovery(workingDir).
 		WithOptions(anyOpts...).
-		WithDiscoverExternalDependencies().
 		WithParseInclude().
 		WithParseExclude().
 		WithDiscoverDependencies().
@@ -102,6 +113,17 @@ func newBaseDiscovery(
 			Cmd:        tgOpts.TerraformCliArgs.First(),
 			Args:       tgOpts.TerraformCliArgs.Tail(),
 		})
+
+	// Only include external dependencies in the run queue if explicitly requested via --queue-include-external.
+	// This restores the pre-v0.94.0 behavior where external dependencies were excluded by default.
+	// External dependencies are still detected and tracked for reporting purposes, but not fully discovered
+	// unless this flag is set.
+	// See: https://github.com/gruntwork-io/terragrunt/issues/5195
+	if tgOpts.IncludeExternalDependencies {
+		d = d.WithDiscoverExternalDependencies()
+	}
+
+	return d
 }
 
 // prepareDiscovery constructs a configured discovery instance based on Terragrunt options and flags.
@@ -153,6 +175,11 @@ func prepareDiscovery(
 		d = d.WithWorktrees(w)
 	}
 
+	// Apply report for recording excluded external dependencies
+	if r := extractReport(opts); r != nil {
+		d = d.WithReport(r)
+	}
+
 	return d, nil
 }
 

internal/runner/runnerpool/runner.go

@@ -189,16 +189,6 @@ func resolveUnitsFromDiscovery(
 			AssumeAlreadyApplied: false,
 		}
 
-		// Handle external units - check if they should be excluded
-		if unit.External() {
-			if stack.Execution != nil && stack.Execution.TerragruntOptions != nil &&
-				stack.Execution.TerragruntOptions.IgnoreExternalDependencies {
-				unit.Execution.AssumeAlreadyApplied = true
-				unit.Execution.FlagExcluded = true
-				l.Infof("Excluded external dependency: %s", unit.DisplayPath())
-			}
-		}
-
 		// Store config from discovery context if available
 		if unit.DiscoveryContext() != nil && unit.Config() == nil {
 			// Config should already be set during discovery
@@ -909,6 +899,10 @@ func FilterDiscoveredUnits(discovered component.Components, units []*component.U
 // WithOptions updates the stack with the provided options.
 func (r *Runner) WithOptions(opts ...common.Option) *Runner {
 	for _, opt := range opts {
+		if opt == nil {
+			continue
+		}
+
 		opt.Apply(r)
 	}