feat: include/exclude by detected MIME type

Intended for cases like scripts, which may lack extensions indicating
their file/language type.

Author: tomeon

cmd/init/init.toml

@@ -61,6 +61,10 @@ options = []
 includes = [ "*.<language-extension>" ]
 # Glob patterns of files to exclude
 excludes = []
+# MIME types of files to include
+allowed-mimetypes = [ "text/x-mylanguage" ]
+# MIME types of files to exclude
+disallowed-mimetypes = [ "application/json" ]
 # Controls the order of application when multiple formatters match the same file
 # Lower the number, the higher the precedence
 # Default is 0

config/config.go

@@ -29,6 +29,7 @@ type Config struct {
 	CI                    bool     `mapstructure:"ci"                      toml:"-"` // not allowed in config
 	ClearCache            bool     `mapstructure:"clear-cache"             toml:"-"` // not allowed in config
 	CPUProfile            string   `mapstructure:"cpu-profile"             toml:"cpu-profile,omitempty"`
+	DisallowedMimetypes   []string `mapstructure:"disallowed-mimetypes"    toml:"excludes,omitempty"`
 	Excludes              []string `mapstructure:"excludes"                toml:"excludes,omitempty"`
 	FailOnChange          bool     `mapstructure:"fail-on-change"          toml:"fail-on-change,omitempty"`
 	Formatters            []string `mapstructure:"formatters"              toml:"formatters,omitempty"`
@@ -56,6 +57,11 @@ type Formatter struct {
 	Command string `mapstructure:"command" toml:"command"`
 	// Options are an optional list of args to be passed to Command.
 	Options []string `mapstructure:"options,omitempty" toml:"options,omitempty"`
+	// AllowedMimetypes is an optional list of MIME types used to determine whether this Formatter
+	// should be applied against a path.
+	AllowedMimetypes []string `mapstructure:"allowed-mimetypes,omitempty" toml:"includes,omitempty"`
+	// DisallowedMimetypes is an optional list of MIME types used to exclude certain files from this Formatter.
+	DisallowedMimetypes []string `mapstructure:"disallowed-mimetypes,omitempty" toml:"includes,omitempty"`
 	// Includes is a list of glob patterns used to determine whether this Formatter should be applied against a path.
 	Includes []string `mapstructure:"includes,omitempty" toml:"includes,omitempty"`
 	// Excludes is an optional list of glob patterns used to exclude certain files from this Formatter.
@@ -86,6 +92,10 @@ func SetFlags(fs *pflag.FlagSet) {
 		"cpu-profile", "",
 		"The file into which a cpu profile will be written. (env $TREEFMT_CPU_PROFILE)",
 	)
+	fs.StringSlice(
+		"disallowed-mimetypes", nil,
+		"Exclude files matching the specified MIME types. (env $TREEFMT_DISALLOWED_MIMETYPES)",
+	)
 	fs.StringSlice(
 		"excludes", nil,
 		"Exclude files or directories matching the specified globs. (env $TREEFMT_EXCLUDES)",

docs/site/getting-started/configure.md

@@ -1,3 +1,5 @@
+[MIME types]: https://github.com/gabriel-vasile/mimetype/blob/17b303270b920bc619feadef12cad28d70fcb6e0/supported_mimes.md
+
 # Configure
 
 `treefmt`'s behaviour can be influenced in one of three ways:
@@ -127,6 +129,28 @@ The file into which a [pprof](https://github.com/google/pprof) cpu profile will
     cpu-profile = "./cpu.pprof"
     ```
 
+### `disallowed-mimetypes`
+
+An optional list of [MIME types][] used to exclude files from all formatters.
+
+=== "Flag"
+
+    ```console
+    treefmt --disallowed-mimetypes application/json,image/png
+    ```
+
+=== "Env"
+
+    ```console
+    TREEFMT_DISALLOWED_MIMETYPES="application/json,image/png" treefmt
+    ```
+
+=== "Config"
+
+    ```toml
+    disallowed-mimetypes = ["application/json", "image/png"]
+    ```
+
 ### `excludes`
 
 An optional list of [glob patterns](#glob-patterns-format) used to exclude files from all formatters.
@@ -450,6 +474,11 @@ command = "deadnix"
 options = ["-e"]
 includes = ["*.nix"]
 priority = 2
+
+[formatter.shfmt]
+command = "shfmt"
+options = ["-i", "2", "-w"]
+allowed-mimetypes = ["text/x-shellscript"]
 ```
 
 ### `command`
@@ -462,19 +491,27 @@ An optional list of args to be passed to `command`.
 
 ### `includes`
 
-A list of [glob patterns](#glob-patterns-format) used to determine whether the formatter should be applied against a given path.
+An optional list of [glob patterns](#glob-patterns-format) used to determine whether the formatter should be applied against a given path.
 
 ### `excludes`
 
 An optional list of [glob patterns](#glob-patterns-format) used to exclude certain files from this formatter.
 
+### `allowed-mimetypes`
+
+An optional list of [MIME types][] used to determine whether the formatter should be applied against a given path.
+
+### `disallowed-mimetypes`
+
+An optional list of [MIME types][] used to exclude certain files from this formatter.
+
 ### `priority`
 
 Influences the order of execution. Greater precedence is given to lower numbers, with the default being `0`.
 
 ## Same file, multiple formatters?
 
-For each file, `treefmt` determines a list of formatters based on the configured `includes` / `excludes` rules. This list is
+For each file, `treefmt` determines a list of formatters based on the configured `includes` / `excludes` and `allowed-mimetypes` / `disallowed-mimetypes` rules. This list is
 then sorted, first by priority (lower the value, higher the precedence) and secondly by formatter name (lexicographically).
 
 The resultant sequence of formatters is used to create a batch key, and similarly matched files get added to that batch
@@ -486,6 +523,10 @@ Another consequence is that formatting is deterministic for a given file and a g
 By setting the priority fields appropriately, you can control the order in which those formatters are applied for any
 files they _both happen to match on_.
 
+## Required settings
+
+Each formatter must have at least one entry in the `includes` list **or** the `allowed-mimetypes` list. This is the minimal requirement for `treefmt` to identify which files are subject to a given formatter.
+
 ## Glob patterns format
 
 This is a variant of the Unix glob pattern. It supports all the usual

docs/site/guides/unmatched-formatters.md

@@ -36,7 +36,7 @@ $ treefmt --on-unmatched warn
 ### Enforcing Strict Matching
 
 Another stricter policy approach is to fail the run if any unmatched files are found.
-This can be paired with an `excludes` list to ignore specific files:
+This can be paired with `excludes` and/or `disallowed-mimetypes` lists to ignore specific files:
 
 `treefmt.toml`:
 
@@ -49,4 +49,10 @@ excludes = [
   "LICENCE",
   "go.sum",
 ]
+
+# List MIME types to explicity ignore
+disallowed-mimetypes = [
+  "image/x-portable-pixmap",
+  "text/rtf",
+]
 ```

format/composite.go

@@ -9,7 +9,6 @@ import (
 	"slices"
 
 	"github.com/charmbracelet/log"
-	"github.com/gobwas/glob"
 	"github.com/numtide/treefmt/v2/config"
 	"github.com/numtide/treefmt/v2/stats"
 	"github.com/numtide/treefmt/v2/walk"
@@ -25,9 +24,9 @@ var ErrFormattingFailures = errors.New("formatting failures detected")
 // CompositeFormatter handles the application of multiple Formatter instances based on global excludes and individual
 // formatter configuration.
 type CompositeFormatter struct {
-	cfg            *config.Config
-	stats          *stats.Stats
-	globalExcludes []glob.Glob
+	cfg           *config.Config
+	stats         *stats.Stats
+	globalMatcher *CompositeMatcher
 
 	unmatchedLevel log.Level
 
@@ -36,9 +35,9 @@ type CompositeFormatter struct {
 }
 
 // match filters the file against global excludes and returns a list of formatters that want to process the file.
-func (c *CompositeFormatter) match(file *walk.File) (bool, []*Formatter) {
+func (c *CompositeFormatter) match(file *walk.File, cache MatcherCache) (bool, []*Formatter) {
 	// first check if this file has been globally excluded
-	if pathMatches(file.RelPath, c.globalExcludes) {
+	if !c.globalMatcher.Wants(file.RelPath, cache) {
 		log.Debugf("path matched global excludes: %s", file.RelPath)
 
 		return true, nil
@@ -49,7 +48,7 @@ func (c *CompositeFormatter) match(file *walk.File) (bool, []*Formatter) {
 
 	// iterate the formatters, recording which are interested in this file
 	for _, formatter := range c.formatters {
-		if formatter.Wants(file) {
+		if formatter.Wants(file, cache) {
 			matches = append(matches, formatter)
 		}
 	}
@@ -61,9 +60,11 @@ func (c *CompositeFormatter) match(file *walk.File) (bool, []*Formatter) {
 func (c *CompositeFormatter) Apply(ctx context.Context, files []*walk.File) error {
 	var toRelease []*walk.File
 
+	cache := NewMatcherCache()
+
 	for _, file := range files {
 		// match the file against the formatters
-		globalExclude, matches := c.match(file)
+		globalExclude, matches := c.match(file, cache)
 
 		// if the file is globally excluded, we do not emit a warning
 		if globalExclude {
@@ -147,12 +148,23 @@ func NewCompositeFormatter(
 	statz *stats.Stats,
 	batchSize int,
 ) (*CompositeFormatter, error) {
+	matchers := make([]Matcher, 2)
+
+	var err error
+
 	// compile global exclude globs
-	globalExcludes, err := compileGlobs(cfg.Excludes)
+	matchers[0], err = NewGlobExclusionMatcher(cfg.Excludes)
 	if err != nil {
 		return nil, fmt.Errorf("failed to compile global excludes: %w", err)
 	}
 
+	matchers[1], err = NewMimetypeExclusionMatcher(cfg.DisallowedMimetypes)
+	if err != nil {
+		return nil, fmt.Errorf("failed to assemble global disallowed MIME types: %w", err)
+	}
+
+	globalMatcher := NewCompositeMatcher(matchers)
+
 	// parse unmatched log level
 	unmatchedLevel, err := log.ParseLevel(cfg.OnUnmatched)
 	if err != nil {
@@ -191,7 +203,7 @@ func NewCompositeFormatter(
 	return &CompositeFormatter{
 		cfg:            cfg,
 		stats:          statz,
-		globalExcludes: globalExcludes,
+		globalMatcher:  globalMatcher,
 		unmatchedLevel: unmatchedLevel,
 
 		scheduler:  scheduler,

format/formatter.go

@@ -13,7 +13,6 @@ import (
 	"time"
 
 	"github.com/charmbracelet/log"
-	"github.com/gobwas/glob"
 	"github.com/numtide/treefmt/v2/config"
 	"github.com/numtide/treefmt/v2/walk"
 	"mvdan.cc/sh/v3/expand"
@@ -37,9 +36,8 @@ type Formatter struct {
 	executable string // path to the executable described by Command
 	workingDir string
 
-	// internal, compiled versions of Includes and Excludes.
-	includes []glob.Glob
-	excludes []glob.Glob
+	// internal, compiled versions of inclusion and exclusion matchers.
+	matcher *CompositeMatcher
 }
 
 func (f *Formatter) Name() string {
@@ -123,11 +121,12 @@ func (f *Formatter) Apply(ctx context.Context, files []*walk.File) error {
 	return nil
 }
 
-// Wants is used to determine if a Formatter wants to process a path based on it's configured Includes and Excludes
-// patterns.
+// Wants is used to determine if a Formatter wants to process a path based on
+// its configured Includes and Excludes patterns, plus its AllowedMimetypes and
+// DisallowedMimetypes MIME types.
 // Returns true if the Formatter should be applied to file, false otherwise.
-func (f *Formatter) Wants(file *walk.File) bool {
-	match := !pathMatches(file.RelPath, f.excludes) && pathMatches(file.RelPath, f.includes)
+func (f *Formatter) Wants(file *walk.File, cache MatcherCache) bool {
+	match := f.matcher.Wants(file.RelPath, cache)
 	if match {
 		f.log.Debugf("match: %v", file)
 	}
@@ -171,20 +170,34 @@ func newFormatter(
 		f.log = log.WithPrefix("formatter | " + name)
 	}
 
-	// check there is at least one include
-	if len(cfg.Includes) == 0 {
-		return nil, fmt.Errorf("formatter '%v' has no includes", f.name)
+	// check there is at least one include or allowed MIME type
+	if len(cfg.Includes) == 0 && len(cfg.AllowedMimetypes) == 0 {
+		return nil, fmt.Errorf("formatter '%v' has no includes or allowed MIME types", f.name)
 	}
 
-	f.includes, err = compileGlobs(cfg.Includes)
+	matchers := make([]Matcher, 4)
+
+	matchers[0], err = NewGlobInclusionMatcher(cfg.Includes)
 	if err != nil {
 		return nil, fmt.Errorf("failed to compile formatter '%v' includes: %w", f.name, err)
 	}
 
-	f.excludes, err = compileGlobs(cfg.Excludes)
+	matchers[1], err = NewGlobExclusionMatcher(cfg.Excludes)
 	if err != nil {
 		return nil, fmt.Errorf("failed to compile formatter '%v' excludes: %w", f.name, err)
 	}
 
+	matchers[2], err = NewMimetypeInclusionMatcher(cfg.AllowedMimetypes)
+	if err != nil {
+		return nil, fmt.Errorf("failed to assemble formatter '%v' allowed MIME types: %w", f.name, err)
+	}
+
+	matchers[3], err = NewMimetypeExclusionMatcher(cfg.DisallowedMimetypes)
+	if err != nil {
+		return nil, fmt.Errorf("failed to assemble formatter '%v' disallowed MIME types: %w", f.name, err)
+	}
+
+	f.matcher = NewCompositeMatcher(matchers)
+
 	return &f, nil
 }