v1.3.0

.github/workflows/build-test.yml

@@ -14,7 +14,7 @@ jobs:
     if: "${{ !endsWith(github.actor, '[bot]') }}"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - uses: projectdiscovery/actions/setup/go@v1
       - uses: projectdiscovery/actions/golangci-lint@v1
 
@@ -28,12 +28,12 @@ jobs:
         go-version: [1.24.x]
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v6
         with:
           go-version: ${{ matrix.go-version }}
 
       - name: Check out code
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Test
         run: go test ./...

.github/workflows/codeql-analysis.yml

@@ -26,16 +26,16 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v5
+      uses: actions/checkout@v6
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
 
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v4
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v4

.github/workflows/compat-checks.yaml

@@ -13,5 +13,5 @@ jobs:
     permissions:
       contents: write
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - uses: projectdiscovery/actions/setup/go/compat-checks@v1

.github/workflows/dep-auto-merge.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.actor == 'dependabot[bot]'
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           token: ${{ secrets.DEPENDABOT_PAT }}
 

.github/workflows/discussion-triage.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Add initial response to discussions
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         with:
           script: |
             const { discussion } = context.payload;
@@ -64,7 +64,7 @@ jobs:
             }
 
       - name: Auto-label potential bug reports
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         with:
           script: |
             const { discussion } = context.payload;

.github/workflows/dockerhub-push.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest-16-cores
     steps:
       - name: Checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Get Github tag
         id: meta

.github/workflows/functional-test.yml

@@ -16,12 +16,12 @@ jobs:
         os: [ubuntu-latest, windows-latest, macOS-latest]
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v6
         with:
           go-version: 1.24.x
 
       - name: Check out code
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Functional Tests
         run: |

.github/workflows/release-binary.yml

@@ -11,12 +11,12 @@ jobs:
     runs-on: macos-latest
     steps:
       - name: "Check out code"
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with: 
           fetch-depth: 0
 
       - name: "Set up Go"
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v6
         with: 
           go-version: 1.24.x
           cache: true
@@ -34,12 +34,12 @@ jobs:
     runs-on: windows-latest-8-cores
     steps:
       - name: "Check out code"
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with: 
           fetch-depth: 0
 
       - name: "Set up Go"
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v6
         with: 
           go-version: 1.24.x
           cache: true
@@ -57,12 +57,12 @@ jobs:
     runs-on: ubuntu-latest-16-cores
     steps:
       - name: "Check out code"
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with: 
           fetch-depth: 0
 
       - name: "Set up Go"
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v6
         with: 
           go-version: 1.24.x
           cache: true

.github/workflows/release-test.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: macos-latest
     steps:
       - name: "Check out code"
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
@@ -32,12 +32,12 @@ jobs:
     runs-on: ubuntu-latest-16-cores
     steps:
       - name: "Check out code"
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v6
         with:
           go-version: 1.24.x
 
@@ -57,12 +57,12 @@ jobs:
     runs-on: windows-latest-8-cores
     steps:
       - name: "Check out code"
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v6
         with:
           go-version: 1.24.x
 
@@ -72,4 +72,4 @@ jobs:
           args: --config=.goreleaser/windows.yml
           workdir: .
         env:
-          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/security-crawl-maze-score.yaml

@@ -9,12 +9,12 @@ jobs:
     runs-on: ubuntu-latest-16-cores
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v6
         with:
           go-version: 1.24.x
 
       - name: Check out code
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Build
         run: go build .

.github/workflows/stale.yml

@@ -13,7 +13,7 @@ jobs:
       issues: write
       pull-requests: write
     steps:
-      - uses: actions/stale@v9
+      - uses: actions/stale@v10
         with:
           days-before-stale: 90
           days-before-close: 7

.github/workflows/workflow-monitor.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check Discussion vs Issue Activity
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         with:
           script: |
             const oneWeekAgo = new Date();

.gitignore

@@ -7,4 +7,6 @@ katana_*/
 dist/
 
 .vscode
-.devcontainer
+.devcontainer
+.DS_Store
+.idea

Dockerfile

@@ -1,11 +1,11 @@
-FROM golang:1.25.0-alpine AS build-env
+FROM golang:1.25.4-alpine AS build-env
 RUN apk add --no-cache git gcc musl-dev
 WORKDIR /app
 COPY . /app
 RUN go mod download
 RUN go build ./cmd/katana
 
-FROM alpine:3.22.1
+FROM alpine:3.22.2
 RUN apk add --no-cache bind-tools ca-certificates chromium
 COPY --from=build-env /app/katana /usr/local/bin/
 

README.md

@@ -166,15 +166,16 @@ SCOPE:
    -do, -display-out-scope          display external endpoint from scoped crawling
 
 FILTER:
-   -mr, -match-regex string[]       regex or list of regex to match on output url (cli, file)
-   -fr, -filter-regex string[]      regex or list of regex to filter on output url (cli, file)
-   -f, -field string                field to display in output (url,path,fqdn,rdn,rurl,qurl,qpath,file,ufile,key,value,kv,dir,udir) (Deprecated: use -output-template instead)
-   -sf, -store-field string         field to store in per-host output (url,path,fqdn,rdn,rurl,qurl,qpath,file,ufile,key,value,kv,dir,udir)
-   -em, -extension-match string[]   match output for given extension (eg, -em php,html,js)
-   -ef, -extension-filter string[]  filter output for given extension (eg, -ef png,css)
-   -mdc, -match-condition string    match response with dsl based condition
-   -fdc, -filter-condition string   filter response with dsl based condition
-   -duf, -disable-unique-filter     disable duplicate content filtering
+   -mr, -match-regex string[]             regex or list of regex to match on output url (cli, file)
+   -fr, -filter-regex string[]            regex or list of regex to filter on output url (cli, file)
+   -f, -field string                      field to display in output (url,path,fqdn,rdn,rurl,qurl,qpath,file,ufile,key,value,kv,dir,udir) (Deprecated: use -output-template instead)
+   -sf, -store-field string               field to store in per-host output (url,path,fqdn,rdn,rurl,qurl,qpath,file,ufile,key,value,kv,dir,udir)
+   -em, -extension-match string[]         match output for given extension (eg, -em php,html,js)
+   -ef, -extension-filter string[]        filter output for given extension (eg, -ef png,css)
+   -ndef, -no-default-ext-filter bool     remove default extensions from the filter list
+   -mdc, -match-condition string          match response with dsl based condition
+   -fdc, -filter-condition string         filter response with dsl based condition
+   -duf, -disable-unique-filter           disable duplicate content filtering
 
 RATE-LIMIT:
    -c, -concurrency int          number of concurrent fetchers to use (default 10)
@@ -196,6 +197,8 @@ OUTPUT:
    -sfd, -store-field-dir string     store per-host field to custom directory
    -or, -omit-raw                    omit raw requests/responses from jsonl output
    -ob, -omit-body                   omit response body from jsonl output
+   -lof, -list-output-fields         list available fields for jsonl output format
+   -eof, -exclude-output-fields      exclude fields from jsonl output
    -j, -jsonl                        write output in jsonl format
    -nc, -no-color                    disable output content coloring (ANSI escape codes)
    -silent                           display output only
@@ -731,6 +734,15 @@ Crawl output can be easily filtered for specific extension using `-ef` option wh
 
 ```
 katana -u https://tesla.com -silent -ef css,txt,md
+
+```
+*`-no-default-ext-filter`*
+---
+
+Katana filters several extensions by default. This can be disabled with the `-ndef` option.
+
+```
+katana -u https://tesla.com -silent -ndef
 ```
 
 *`-match-regex`*
@@ -777,15 +789,16 @@ katana -h filter
 
 Flags:
 FILTER:
-   -mr, -match-regex string[]       regex or list of regex to match on output url (cli, file)
-   -fr, -filter-regex string[]      regex or list of regex to filter on output url (cli, file)
-   -f, -field string                field to display in output (url,path,fqdn,rdn,rurl,qurl,qpath,file,ufile,key,value,kv,dir,udir)
-   -sf, -store-field string         field to store in per-host output (url,path,fqdn,rdn,rurl,qurl,qpath,file,ufile,key,value,kv,dir,udir)
-   -em, -extension-match string[]   match output for given extension (eg, -em php,html,js)
-   -ef, -extension-filter string[]  filter output for given extension (eg, -ef png,css)
-   -mdc, -match-condition string    match response with dsl based condition
-   -fdc, -filter-condition string   filter response with dsl based condition
-   -duf, -disable-unique-filter     disable duplicate content filtering
+   -mr, -match-regex string[]             regex or list of regex to match on output url (cli, file)
+   -fr, -filter-regex string[]            regex or list of regex to filter on output url (cli, file)
+   -f, -field string                      field to display in output (url,path,fqdn,rdn,rurl,qurl,qpath,file,ufile,key,value,kv,dir,udir)
+   -sf, -store-field string               field to store in per-host output (url,path,fqdn,rdn,rurl,qurl,qpath,file,ufile,key,value,kv,dir,udir)
+   -em, -extension-match string[]         match output for given extension (eg, -em php,html,js)
+   -ef, -extension-filter string[]        filter output for given extension (eg, -ef png,css)
+   -ndef, -no-default-ext-filter bool     remove default extensions from the filter list
+   -mdc, -match-condition string          match response with dsl based condition
+   -fdc, -filter-condition string         filter response with dsl based condition
+   -duf, -disable-unique-filter           disable duplicate content filtering
 ```
 
 
@@ -944,6 +957,24 @@ katana_response/www.iana.org/bfc096e6dd93b993ca8918bf4c08fdc707a70723.txt http:/
 
 *`-store-response` option is not supported in `-headless` mode.*
 
+*`-list-output-fields`*
+----
+
+The `-list-output-fields` or `-lof` flag displays all available fields that can be used in JSONL output format. This is useful for understanding what data is available when using custom output templates or when excluding specific fields.
+
+```console
+katana -lof
+```
+
+*`-exclude-output-fields`*
+----
+
+The `-exclude-output-fields` or `-eof` flag allows you to exclude specific fields from the JSONL output. This is useful for reducing output size or focusing on specific data by removing unwanted fields.
+
+```console
+katana -u https://example.com -jsonl -eof raw,body
+```
+
 Here are additional CLI options related to output -
 
 ```console
@@ -953,6 +984,8 @@ OUTPUT:
    -o, -output string                file to write output to
    -sr, -store-response              store http requests/responses
    -srd, -store-response-dir string  store http requests/responses to custom directory
+   -lof, -list-output-fields         list available fields for jsonl output format
+   -eof, -exclude-output-fields      exclude fields from jsonl output
    -j, -json                         write output in JSONL(ines) format
    -nc, -no-color                    disable output content coloring (ANSI escape codes)
    -silent                           display output only

cmd/integration-test/filters.go

@@ -58,7 +58,7 @@ func (h *uniqueFilterIntegrationTest) Execute() error {
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/" {
 			w.WriteHeader(http.StatusOK)
-			fmt.Fprint(w, `<html><body>
+			_, _ = fmt.Fprint(w, `<html><body>
 				<a href="/page1">Page 1</a>
 				<a href="/page2">Page 2</a>
 				<a href="/page3">Page 3</a>
@@ -67,7 +67,7 @@ func (h *uniqueFilterIntegrationTest) Execute() error {
 		} else {
 			w.WriteHeader(http.StatusNotFound)
 			// Return identical 404 content for all missing pages
-			fmt.Fprint(w, `<html><body><h1>404 - Page Not Found</h1></body></html>`)
+			_, _ = fmt.Fprint(w, `<html><body><h1>404 - Page Not Found</h1></body></html>`)
 		}
 	}))
 	defer server.Close()
@@ -89,7 +89,9 @@ func (h *uniqueFilterIntegrationTest) Execute() error {
 	if err != nil {
 		return fmt.Errorf("could not create runner: %v", err)
 	}
-	defer katanaRunner.Close()
+	defer func() {
+		_ = katanaRunner.Close()
+	}()
 
 	if err := katanaRunner.ExecuteCrawling(); err != nil {
 		return fmt.Errorf("could not execute crawling: %v", err)