diff --git a/CHANGELOG.md b/CHANGELOG.md
index e7f9fe30e2..299a43e1ff 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -76,6 +76,9 @@ Unreleased changes template.
 * (pypi) The PyPI extension will no longer write the lock file entries as the
   extension has been marked reproducible.
   Fixes [#2434](https://github.com/bazel-contrib/rules_python/issues/2434).
+* (gazelle) Lazily load and parse manifest files when running Gazelle. This ensures no
+  manifest files are loaded when Gazelle is run over a set of non-python directories
+  [PR #2746](https://github.com/bazel-contrib/rules_python/pull/2746).
 * (rules) {attr}`py_binary.srcs` and {attr}`py_test.srcs` is no longer mandatory when
   `main_module` is specified (for `--bootstrap_impl=script`)
 
diff --git a/gazelle/python/configure.go b/gazelle/python/configure.go
index 7b1f091b34..a00b0ba0ba 100644
--- a/gazelle/python/configure.go
+++ b/gazelle/python/configure.go
@@ -18,7 +18,6 @@ import (
 	"flag"
 	"fmt"
 	"log"
-	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -27,7 +26,6 @@ import (
 	"github.com/bazelbuild/bazel-gazelle/rule"
 	"github.com/bmatcuk/doublestar/v4"
 
-	"github.com/bazel-contrib/rules_python/gazelle/manifest"
 	"github.com/bazel-contrib/rules_python/gazelle/pythonconfig"
 )
 
@@ -228,25 +226,5 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) {
 	}
 
 	gazelleManifestPath := filepath.Join(c.RepoRoot, rel, gazelleManifestFilename)
-	gazelleManifest, err := py.loadGazelleManifest(gazelleManifestPath)
-	if err != nil {
-		log.Fatal(err)
-	}
-	if gazelleManifest != nil {
-		config.SetGazelleManifest(gazelleManifest)
-	}
-}
-
-func (py *Configurer) loadGazelleManifest(gazelleManifestPath string) (*manifest.Manifest, error) {
-	if _, err := os.Stat(gazelleManifestPath); err != nil {
-		if os.IsNotExist(err) {
-			return nil, nil
-		}
-		return nil, fmt.Errorf("failed to load Gazelle manifest at %q: %w", gazelleManifestPath, err)
-	}
-	manifestFile := new(manifest.File)
-	if err := manifestFile.Decode(gazelleManifestPath); err != nil {
-		return nil, fmt.Errorf("failed to load Gazelle manifest at %q: %w", gazelleManifestPath, err)
-	}
-	return manifestFile.Manifest, nil
+	config.SetGazelleManifestPath(gazelleManifestPath)
 }
diff --git a/gazelle/pythonconfig/pythonconfig.go b/gazelle/pythonconfig/pythonconfig.go
index 23c0cfd572..866339d449 100644
--- a/gazelle/pythonconfig/pythonconfig.go
+++ b/gazelle/pythonconfig/pythonconfig.go
@@ -16,6 +16,8 @@ package pythonconfig
 
 import (
 	"fmt"
+	"log"
+	"os"
 	"path"
 	"regexp"
 	"strings"
@@ -153,10 +155,11 @@ func (c Configs) ParentForPackage(pkg string) *Config {
 type Config struct {
 	parent *Config
 
-	extensionEnabled  bool
-	repoRoot          string
-	pythonProjectRoot string
-	gazelleManifest   *manifest.Manifest
+	extensionEnabled    bool
+	repoRoot            string
+	pythonProjectRoot   string
+	gazelleManifestPath string
+	gazelleManifest     *manifest.Manifest
 
 	excludedPatterns                          *singlylinkedlist.List
 	ignoreFiles                               map[string]struct{}
@@ -281,11 +284,26 @@ func (c *Config) SetGazelleManifest(gazelleManifest *manifest.Manifest) {
 	c.gazelleManifest = gazelleManifest
 }
 
+// SetGazelleManifestPath sets the path to the gazelle_python.yaml file
+// for the current configuration.
+func (c *Config) SetGazelleManifestPath(gazelleManifestPath string) {
+	c.gazelleManifestPath = gazelleManifestPath
+}
+
 // FindThirdPartyDependency scans the gazelle manifests for the current config
 // and the parent configs up to the root finding if it can resolve the module
 // name.
 func (c *Config) FindThirdPartyDependency(modName string) (string, string, bool) {
 	for currentCfg := c; currentCfg != nil; currentCfg = currentCfg.parent {
+		// Attempt to load the manifest if needed.
+		if currentCfg.gazelleManifestPath != "" && currentCfg.gazelleManifest == nil {
+			currentCfgManifest, err := loadGazelleManifest(currentCfg.gazelleManifestPath)
+			if err != nil {
+				log.Fatal(err)
+			}
+			currentCfg.SetGazelleManifest(currentCfgManifest)
+		}
+
 		if currentCfg.gazelleManifest != nil {
 			gazelleManifest := currentCfg.gazelleManifest
 			if distributionName, ok := gazelleManifest.ModulesMapping[modName]; ok {
@@ -526,3 +544,17 @@ func (c *Config) FormatThirdPartyDependency(repositoryName string, distributionN
 
 	return label.New(repositoryName, normConventionalDistributionName, normConventionalDistributionName)
 }
+
+func loadGazelleManifest(gazelleManifestPath string) (*manifest.Manifest, error) {
+	if _, err := os.Stat(gazelleManifestPath); err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("failed to load Gazelle manifest at %q: %w", gazelleManifestPath, err)
+	}
+	manifestFile := new(manifest.File)
+	if err := manifestFile.Decode(gazelleManifestPath); err != nil {
+		return nil, fmt.Errorf("failed to load Gazelle manifest at %q: %w", gazelleManifestPath, err)
+	}
+	return manifestFile.Manifest, nil
+}