Add: -j / --jobs flag - Allow parallelization, default is 1 parallel job.

Author: CorentinB

README.md

@@ -45,7 +45,7 @@ For more informations, refeer to the helper:
 
 [-u|--url] is required
 usage: deepsort [-h|--help] -u|--url "<value>" -i|--input "<value>"
-                [-d|--dry-run] [-R|--recursive]
+                [-R|--recursive] [-j|--jobs <integer>] [-d|--dry-run]
 
                 AI powered image tagger backed by DeepDetect
 
@@ -54,8 +54,9 @@ Arguments:
   -h  --help       Print help information
   -u  --url        URL of your DeepDetect instance (i.e: http://localhost:8080)
   -i  --input      Your input folder.
-  -d  --dry-run    Just classify images and return results, do not apply.
   -R  --recursive  Process files recursively.
+  -j  --jobs       Number of parallel jobs. Default: 1
+  -d  --dry-run    Just classify images and return results, do not apply.
 ```
 
 # (Really really quick) Benchmark

cmd/deepsort/arguments.go

@@ -14,8 +14,9 @@ func argumentParsing(args []string, argument *Arguments) {
 	// Create flags
 	URL := parser.String("u", "url", &argparse.Options{Required: true, Help: "URL of your DeepDetect instance (i.e: http://localhost:8080)"})
 	input := parser.String("i", "input", &argparse.Options{Required: true, Help: "Your input folder."})
-	dryRun := parser.Flag("d", "dry-run", &argparse.Options{Required: false, Help: "Just classify images and return results, do not apply."})
 	recursive := parser.Flag("R", "recursive", &argparse.Options{Required: false, Help: "Process files recursively."})
+	jobs := parser.Int("j", "jobs", &argparse.Options{Required: false, Help: "Number of parallel jobs", Default: 1})
+	dryRun := parser.Flag("d", "dry-run", &argparse.Options{Required: false, Help: "Just classify images and return results, do not apply."})
 	// Parse input
 	err := parser.Parse(os.Args)
 	if err != nil {
@@ -27,6 +28,7 @@ func argumentParsing(args []string, argument *Arguments) {
 	// Handle the input flag
 	inputFolder, _ := filepath.Abs(*input)
 	// Finally save the collected flags
+	argument.Jobs = *jobs
 	argument.DryRun = *dryRun
 	argument.Recursive = *recursive
 	argument.Input = inputFolder

cmd/deepsort/classify.go

@@ -6,12 +6,14 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"sync"
 
 	"github.com/CorentinB/DeepSort/pkg/logging"
 	"github.com/labstack/gommon/color"
 )
 
-func googleNetClassification(path string, arguments *Arguments) {
+func googleNetClassification(path string, arguments *Arguments, wg *sync.WaitGroup) {
+	defer wg.Done()
 	url := arguments.URL + "/predict"
 	path, _ = filepath.Abs(path)
 	var jsonStr = []byte(`{"service":"deepsort-googlenet","parameters":{"input":{"width":224,"height":224},"output":{"best":1},"mllib":{"gpu":false}},"data":["` + path + `"]}`)

cmd/deepsort/main.go

@@ -13,18 +13,28 @@ type Arguments struct {
 	URL       string
 	DryRun    bool
 	Recursive bool
+	Jobs      int
 }
 
 func main() {
 	start := time.Now()
 	arguments := new(Arguments)
+	arguments.Jobs = 1
 	argumentParsing(os.Args, arguments)
 	startGoogleNet(arguments)
 	if arguments.Recursive == true {
-		logging.Success("Starting image classification recursively..", "[GoogleNet]")
+		if arguments.Jobs == 1 {
+			logging.Success("Starting image classification recursively..", "[GoogleNet]")
+		} else {
+			logging.Success("Starting image classification recursively with "+string(arguments.Jobs)+" parallel jobs..", "[GoogleNet]")
+		}
 		runRecursively(arguments)
 	} else {
-		logging.Success("Starting image classification..", "[GoogleNet]")
+		if arguments.Jobs == 1 {
+			logging.Success("Starting image classification..", "[GoogleNet]")
+		} else {
+			logging.Success("Starting image classification with "+string(arguments.Jobs)+" parallel jobs..", "[GoogleNet]")
+		}
 		run(arguments)
 	}
 	color.Println(color.Cyan("Done in ") + color.Yellow(time.Since(start)) + color.Cyan("!"))

cmd/deepsort/process.go

@@ -5,12 +5,17 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"sync"
 
 	"github.com/CorentinB/DeepSort/pkg/logging"
 	filetype "gopkg.in/h2non/filetype.v1"
 )
 
 func run(arguments *Arguments) {
+	// Handle parallelization
+	count := 0
+	var wg sync.WaitGroup
+	// Open input folder
 	f, err := os.Open(arguments.Input)
 	if err != nil {
 		log.Fatal(err)
@@ -22,16 +27,27 @@ func run(arguments *Arguments) {
 		os.Exit(1)
 	}
 
+	// Process files in the input folder
 	for _, file := range files {
 		path := arguments.Input + "/" + file.Name()
 		buf, _ := ioutil.ReadFile(path)
 		if filetype.IsImage(buf) {
-			googleNetClassification(path, arguments)
+			count++
+			wg.Add(1)
+			go googleNetClassification(path, arguments, &wg)
+			if count == arguments.Jobs {
+				wg.Wait()
+				count = 0
+			}
 		}
 	}
 }
 
 func runRecursively(arguments *Arguments) ([]string, error) {
+	// Handle parallelization
+	count := 0
+	var wg sync.WaitGroup
+	// Open input folder
 	fileList := make([]string, 0)
 	e := filepath.Walk(arguments.Input, func(path string, f os.FileInfo, err error) error {
 		fileList = append(fileList, path)
@@ -46,7 +62,13 @@ func runRecursively(arguments *Arguments) ([]string, error) {
 	for _, file := range fileList {
 		buf, _ := ioutil.ReadFile(file)
 		if filetype.IsImage(buf) {
-			googleNetClassification(file, arguments)
+			count++
+			wg.Add(1)
+			go googleNetClassification(file, arguments, &wg)
+		}
+		if count == arguments.Jobs {
+			wg.Wait()
+			count = 0
 		}
 	}