feat: begin migration from python to go

Author: RoseSecurity

.gitignore

@@ -66,9 +66,6 @@ db.sqlite3-journal
 instance/
 .webassets-cache
 
-# Scrapy stuff:
-.scrapy
-
 # Sphinx documentation
 docs/_build/
 

cmd/root.go

@@ -0,0 +1,88 @@
+package cmd
+
+import (
+	"fmt"
+	"log"
+
+	tui "github.com/RoseSecurity/ScrapPY/internal/tui/utils"
+	"github.com/RoseSecurity/ScrapPY/pkg/utils"
+	"github.com/spf13/cobra"
+)
+
+var (
+	blue  = "\033[34m"
+	red   = "\033[91m"
+	green = "\033[92m"
+	norm  = "\x1b[0m"
+	tag   = "@RoseSecurity"
+)
+
+var rootCmd = &cobra.Command{
+	Use:   "scrapPY",
+	Short: "ScrapPY enumerates documents, manuals, and sensitive PDFs for key phrases and words that can be utilized in dictionary and brute force attacks.",
+	Long: `ScrapPY enumerates documents, manuals, and sensitive PDFs for key phrases and words 
+that can be utilized in dictionary and brute force attacks. These keywords are outputted 
+to a text file (ScrapPY.txt in the directory which the tool was run from) that can be read 
+by tools such as Hydra, Dirb, and other offensive security tools for initial access and 
+lateral movement.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		file, _ := cmd.Flags().GetString("file")
+		mode, _ := cmd.Flags().GetString("mode")
+		outputFile, _ := cmd.Flags().GetString("output-file")
+
+		// Print help if no file is provided
+		if file == "" {
+			tui.PrintStyledText("SCRAPPY")
+			fmt.Println(red + "Enter PDF file to scrape or use -h for help menu\n" + norm)
+			return
+		}
+
+		// Extract text from PDF
+		fileContent, err := utils.ExtractTextFromPDF(file)
+		if err != nil {
+			utils.LogErrorAndExit(err)
+		}
+
+		wordList := utils.RemoveCommonWords(fileContent)
+
+		if len(fileContent) == 0 {
+			log.Fatalf("No content found in the PDF file: %s", file)
+		}
+
+		// Process PDF content based on mode
+		var keywords []string
+		switch mode {
+		case "word-frequency":
+			keywords = utils.WordFrequency(wordList, 100)
+		case "metadata":
+			utils.PrintMetadata(file)
+			if err != nil {
+				utils.LogErrorAndExit(err)
+			}
+			return
+		case "entropy":
+			// keywords = utils.CalculateEntropy(wordList, 100)
+			// default:
+			// 	keywords = utils.ExtractKeywords(fileContent)
+		}
+
+		// Write output to file
+		if err := utils.WriteToFile(outputFile, keywords); err != nil {
+			utils.LogErrorAndExit(err)
+		}
+
+		fmt.Printf("%s has been created!\n", outputFile)
+	},
+}
+
+func init() {
+	rootCmd.Flags().StringP("file", "f", "", "PDF input file")
+	rootCmd.Flags().StringP("mode", "m", "full", "Modes of operation: full, word-frequency, metadata, entropy")
+	rootCmd.Flags().StringP("output-file", "o", "ScrapPY.txt", "Output file name")
+}
+
+func Execute() {
+	if err := rootCmd.Execute(); err != nil {
+		utils.LogErrorAndExit(err)
+	}
+}

cmd/version.go

@@ -0,0 +1,66 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/spf13/cobra"
+	"golang.org/x/mod/semver"
+)
+
+// Placeholder for builds
+var Version = "1.0.0"
+
+type Release struct {
+	TagName string `json:"tag_name"`
+}
+
+var versionCmd = &cobra.Command{
+	Use:     "version",
+	Short:   "Print the CLI version",
+	Long:    `This command prints the CLI version`,
+	Example: "scrapPY version",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("scrapPY: " + Version)
+		latestReleaseTag, err := latestRelease()
+		if err == nil && latestReleaseTag != "" {
+			latestRelease := strings.TrimPrefix(latestReleaseTag, "v")
+			currentRelease := strings.TrimPrefix(Version, "v")
+			if semver.Compare(latestRelease, currentRelease) > 0 {
+				updateScrapPY(latestRelease)
+			}
+		}
+	},
+}
+
+// Fetch latest release for comparison to current version
+func latestRelease() (string, error) {
+	resp, err := http.Get("https://api.github.com/repos/RoseSecurity/ScrapPY/releases/latest")
+	if err != nil {
+		return "", fmt.Errorf("failed to fetch version: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	var release Release
+	if err := json.Unmarshal(body, &release); err != nil {
+		return "", fmt.Errorf("failed to parse version: %w", err)
+	}
+
+	return release.TagName, nil
+}
+
+// Display out of date warning
+func updateScrapPY(latestVersion string) {
+	c1 := color.New(color.FgCyan)
+
+	c1.Println(fmt.Sprintf("\nYour version of ScrapPY is out of date. The latest version is %s\n\n", latestVersion))
+}

go.mod

@@ -0,0 +1,25 @@
+module github.com/RoseSecurity/ScrapPY
+
+go 1.23.4
+
+require (
+	github.com/arsham/figurine v1.3.0
+	github.com/barasher/go-exiftool v1.10.0
+	github.com/fatih/color v1.18.0
+	github.com/jwalton/go-supportscolor v1.2.0
+	github.com/ledongthuc/pdf v0.0.0-20240201131950-da5b75280b06
+	github.com/spf13/cobra v1.6.1
+	golang.org/x/mod v0.22.0
+)
+
+require (
+	github.com/arsham/rainbow v1.2.1 // indirect
+	github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
+)

go.sum

@@ -0,0 +1,52 @@
+github.com/arsham/figurine v1.3.0 h1:vpGbzp460B1gkdFt9jrl95v4wDE2vP3BDcg0AKWJ7J0=
+github.com/arsham/figurine v1.3.0/go.mod h1:cnw6B/y/XzRObDhQoqNJnpAGuSSrkjCcqZCcMJ1ag/I=
+github.com/arsham/rainbow v1.2.1 h1:iS8o/1WAPVFvhtMZgdiy7zM8mD+XIWZfwzGXD6manKI=
+github.com/arsham/rainbow v1.2.1/go.mod h1:vERoG76FE/wN9rGJRv9H/tTfH873AX6wfQdJqNRy6fA=
+github.com/barasher/go-exiftool v1.10.0 h1:f5JY5jc42M7tzR6tbL9508S2IXdIcG9QyieEXNMpIhs=
+github.com/barasher/go-exiftool v1.10.0/go.mod h1:F9s/a3uHSM8YniVfwF+sbQUtP8Gmh9nyzigNF+8vsWo=
+github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ=
+github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
+github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
+github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jwalton/go-supportscolor v1.2.0 h1:g6Ha4u7Vm3LIsQ5wmeBpS4gazu0UP1DRDE8y6bre4H8=
+github.com/jwalton/go-supportscolor v1.2.0/go.mod h1:hFVUAZV2cWg+WFFC4v8pT2X/S2qUUBYMioBD9AINXGs=
+github.com/ledongthuc/pdf v0.0.0-20240201131950-da5b75280b06 h1:kacRlPN7EN++tVpGUorNGPn/4DnB7/DfTY82AOn6ccU=
+github.com/ledongthuc/pdf v0.0.0-20240201131950-da5b75280b06/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
+github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
+golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
+golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/tui/utils/utils.go

@@ -0,0 +1,17 @@
+package utils
+
+import (
+	"os"
+
+	"github.com/arsham/figurine/figurine"
+	"github.com/jwalton/go-supportscolor"
+)
+
+// PrintStyledText prints a styled text to the terminal
+func PrintStyledText(text string) error {
+	// Check if the terminal supports colors
+	if supportscolor.Stdout().SupportsColor {
+		return figurine.Write(os.Stdout, text, "ANSI Regular.flf")
+	}
+	return nil
+}

main.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/RoseSecurity/ScrapPY/cmd"
+
+func main() {
+	cmd.Execute()
+}

pkg/utils/file_utils.go

@@ -0,0 +1,24 @@
+package utils
+
+import (
+	"fmt"
+	"os"
+)
+
+// WriteToFile writes the keywords to a specified output file
+func WriteToFile(filename string, keywords []string) error {
+	file, err := os.Create(filename)
+	if err != nil {
+		return fmt.Errorf("error creating output file: %v", err)
+	}
+	defer file.Close()
+
+	// Write keywords to file
+	for _, keyword := range keywords {
+		if _, err := file.WriteString(keyword + "\n"); err != nil {
+			return fmt.Errorf("error writing to file: %v", err)
+		}
+	}
+
+	return nil
+}

pkg/utils/log_utils.go

@@ -0,0 +1,44 @@
+package utils
+
+import (
+	"errors"
+	"os"
+	"os/exec"
+
+	"github.com/fatih/color"
+)
+
+const (
+	LogLevelTrace   = "Trace"
+	LogLevelDebug   = "Debug"
+	LogLevelInfo    = "Info"
+	LogLevelWarning = "Warning"
+)
+
+// LogErrorAndExit logs errors to std.Error and exits with an error code
+func LogErrorAndExit(err error) {
+	if err != nil {
+		LogError(err)
+
+		// Find the executed command's exit code from the error
+		var exitError *exec.ExitError
+		if errors.As(err, &exitError) {
+			exitCode := exitError.ExitCode()
+			os.Exit(exitCode)
+		}
+	}
+}
+
+// LogError logs errors to std.Error
+func LogError(err error) {
+	if err != nil {
+		c := color.New(color.FgRed)
+		_, err2 := c.Fprintln(color.Error, err.Error()+"\n")
+		if err2 != nil {
+			color.Red("Error logging the error:")
+			color.Red("%s\n", err2)
+			color.Red("Original error:")
+			color.Red("%s\n", err)
+		}
+	}
+}