Add records of invalid files and duplicates based on hash search

Author: vinser

cmd/flibgolite/main.go

@@ -143,48 +143,41 @@ func reindexStock() {
 	genresTree := genres.NewGenresTree(cfg.Genres.TREE_FILE)
 	hashes := hash.InitHashes(db.DB)
 
-	databaseQueue := make(chan model.Book, cfg.Database.BOOK_QUEUE_SIZE)
-	defer close(databaseQueue)
-	databaseHandler := &database.Handler{
-		CFG:    cfg,
-		DB:     db,
-		LOG:    stockLog,
-		Queue:  databaseQueue,
-		Hashes: hashes,
-	}
-	databaseHandler.StopDB = make(chan struct{})
-	defer close(databaseHandler.StopDB)
-
-	go databaseHandler.AddBooksToIndex()
-
+	bookQueue := make(chan model.Book, cfg.Database.BOOK_QUEUE_SIZE)
+	defer close(bookQueue)
 	fileQueue := make(chan stock.File, cfg.Database.FILE_QUEUE_SIZE)
 	defer close(fileQueue)
 	stockHandler := &stock.Handler{
-		CFG:    cfg,
-		LOG:    stockLog,
-		DB:     db,
-		GT:     genresTree,
-		Queue:  fileQueue,
-		Hashes: hashes,
+		CFG:       cfg,
+		LOG:       stockLog,
+		DB:        db,
+		GT:        genresTree,
+		BookQueue: bookQueue,
+		FileQueue: fileQueue,
+		Hashes:    hashes,
 	}
-	stockHandler.InitStockFolders()
+	stockHandler.StopDB = make(chan struct{})
+	defer close(stockHandler.StopDB)
 	stockHandler.StopScan = make(chan struct{})
 	defer close(stockHandler.StopScan)
+
+	stockHandler.InitStockFolders()
+	go stockHandler.AddBooksToIndex()
 	for i := 0; i < cfg.Database.MAX_SCAN_THREADS; i++ {
-		go stockHandler.ParseFB2Queue(databaseQueue)
+		go stockHandler.ParseFB2Queue()
 	}
 
 	defer func() { stockHandler.StopScan <- struct{}{} }()
 	dir := cfg.Library.STOCK_DIR
 	if len(cfg.Library.NEW_DIR) > 0 {
 		dir = cfg.Library.NEW_DIR
 	}
-	stockHandler.ScanDir(dir, databaseQueue)
+	stockHandler.ScanDir(dir)
 
 	stockHandler.StopScan <- struct{}{}
 
-	databaseHandler.StopDB <- struct{}{}
-	<-databaseHandler.StopDB
+	stockHandler.StopDB <- struct{}{}
+	<-stockHandler.StopDB
 
 	stockLog.S.Println("<<< Book stock reindex finished <<<<<<<<<<<<<<<<<<<<<<<<<<<")
 	stockLog.S.Println("Time elapsed: ", time.Since(start))
@@ -211,35 +204,28 @@ func run() {
 	genresTree := genres.NewGenresTree(cfg.Genres.TREE_FILE)
 	hashes := hash.InitHashes(db.DB)
 
-	databaseQueue := make(chan model.Book, cfg.Database.BOOK_QUEUE_SIZE)
-	defer close(databaseQueue)
-	databaseHandler := &database.Handler{
-		CFG:    cfg,
-		DB:     db,
-		LOG:    stockLog,
-		Queue:  databaseQueue,
-		Hashes: hashes,
-	}
-	databaseHandler.StopDB = make(chan struct{})
-	defer close(databaseHandler.StopDB)
-
-	go databaseHandler.AddBooksToIndex()
-
+	bookQueue := make(chan model.Book, cfg.Database.BOOK_QUEUE_SIZE)
+	defer close(bookQueue)
 	fileQueue := make(chan stock.File, cfg.Database.FILE_QUEUE_SIZE)
 	defer close(fileQueue)
 	stockHandler := &stock.Handler{
-		CFG:    cfg,
-		LOG:    stockLog,
-		DB:     db,
-		GT:     genresTree,
-		Queue:  fileQueue,
-		Hashes: hashes,
+		CFG:       cfg,
+		LOG:       stockLog,
+		DB:        db,
+		GT:        genresTree,
+		BookQueue: bookQueue,
+		FileQueue: fileQueue,
+		Hashes:    hashes,
 	}
+	stockHandler.StopDB = make(chan struct{})
+	defer close(stockHandler.StopDB)
 	stockHandler.InitStockFolders()
 	stockHandler.StopScan = make(chan struct{})
 	defer close(stockHandler.StopScan)
+
+	go stockHandler.AddBooksToIndex()
 	for i := 0; i < cfg.Database.MAX_SCAN_THREADS; i++ {
-		go stockHandler.ParseFB2Queue(databaseQueue)
+		go stockHandler.ParseFB2Queue()
 	}
 	go func() {
 		defer func() { stockHandler.StopScan <- struct{}{} }()
@@ -248,7 +234,7 @@ func run() {
 			dir = cfg.Library.NEW_DIR
 		}
 		for {
-			stockHandler.ScanDir(dir, databaseQueue)
+			stockHandler.ScanDir(dir)
 			time.Sleep(time.Duration(cfg.Database.POLL_DELAY) * time.Second)
 			select {
 			case <-stockHandler.StopScan:
@@ -293,9 +279,9 @@ func run() {
 	stockHandler.LOG.S.Printf("New acquisitions scanning was stoped correctly\n")
 
 	// Stop addind new acquisitions to index and wait for completion
-	databaseHandler.StopDB <- struct{}{}
-	<-databaseHandler.StopDB
-	databaseHandler.LOG.S.Printf("New acquisitions adding was stoped correctly\n")
+	stockHandler.StopDB <- struct{}{}
+	<-stockHandler.StopDB
+	stockHandler.LOG.S.Printf("New acquisitions adding was stoped correctly\n")
 
 	// Shutdown OPDS server
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)

pkg/config/config.go

@@ -96,12 +96,12 @@ func LoadConfig(rootDir string) *Config {
 	c := &Config{
 		Library: Library{
 			STOCK_DIR: "books/stock",
-			TRASH_DIR: "books/trash",
+			TRASH_DIR: "",
 			NEW_DIR:   "",
 		},
 		Database: Database{
 			DSN:               "dbdata/books.db",
-			POLL_DELAY:        30,
+			POLL_DELAY:        300,
 			MAX_SCAN_THREADS:  10,
 			BOOK_QUEUE_SIZE:   20000,
 			FILE_QUEUE_SIZE:   20000,
@@ -119,7 +119,7 @@ func LoadConfig(rootDir string) *Config {
 		OPDS: OPDS{
 			PORT:          8085,
 			TITLE:         "FLib Go Go Go!!!",
-			PAGE_SIZE:     30,
+			PAGE_SIZE:     20,
 			LATEST_DAYS:   14,
 			NO_CONVERSION: false,
 		},

pkg/config/config.yml

@@ -1,7 +1,7 @@
 library:
   # Book folders
   STOCK: "books/stock" # Book stock
-  TRASH: "books/trash" # Error and duplicate files and archives will be moved to this folder 
+  #TRASH: "books/trash" # Error and duplicate files and archives will be moved to this folder 
   #NEW: "books/new" # Uncomment the line to have separate folder for new acquired books
 
 genres:
@@ -10,7 +10,7 @@ genres:
 database:
   DSN: "dbdata/books.db"
   # Delay before start each new acquisitions folder processing
-  POLL_DELAY: 30 
+  POLL_DELAY: 300 
   # Maximum parallel new acquisitions processing routines
   MAX_SCAN_THREADS: 10
   # Book queue size
@@ -20,7 +20,7 @@ database:
   # Maximum number of books in one transaction
   MAX_BOOKS_IN_TX: 20000
   # Level of checking new books for duplicates: N - no check, F - fast check (default) by CRC32, S - slow check by CRC32 or title and plot comparison
-  #DEDUPLICATE_LEVEL: "N"
+  DEDUPLICATE_LEVEL: "F"
 
 logs:
   # Logs are here
@@ -36,11 +36,11 @@ opds:
   # OPDS-server title that is displayed in a book reader
   TITLE: "FLib Go Go Go!!!"
   # OPDS feeds entries page size
-  PAGE_SIZE: 30
+  PAGE_SIZE: 20
   # Latest books period in days
   LATEST_DAYS: 14
   # Do not convert FB2 to EPUB format if set to true, default: false
-  #NO_CONVERSION: true
+  NO_CONVERSION: false
 
 locales:
   # Locales folder. You can add your own locale file there like en.yml, ru.yml, uk.yml

pkg/database/db.go

@@ -7,13 +7,10 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"sync"
+
+	// "sync"
 
 	"github.com/jmoiron/sqlx"
-	"github.com/vinser/flibgolite/pkg/config"
-	"github.com/vinser/flibgolite/pkg/hash"
-	"github.com/vinser/flibgolite/pkg/model"
-	"github.com/vinser/flibgolite/pkg/rlog"
 
 	_ "embed"
 
@@ -28,18 +25,6 @@ var SQLITE_DB_INIT string
 //go:embed sqlite_db_drop.sql
 var SQLITE_DB_DROP string
 
-// ==================================
-type Handler struct {
-	CFG    *config.Config
-	Hashes *hash.BookHashes
-	DB     *DB
-	TX     *TX
-	LOG    *rlog.Log
-	WG     *sync.WaitGroup
-	Queue  <-chan model.Book
-	StopDB chan struct{}
-}
-
 type DB struct {
 	*sqlx.DB
 }
@@ -113,7 +98,7 @@ type TX struct {
 	Stmt map[string]*sqlx.Stmt
 }
 
-func (db *DB) txBegin() *TX {
+func (db *DB) TxBegin() *TX {
 	TX := &TX{
 		Tx:   db.DB.MustBegin(),
 		Stmt: map[string]*sqlx.Stmt{},
@@ -122,7 +107,7 @@ func (db *DB) txBegin() *TX {
 	return TX
 }
 
-func (tx *TX) txEnd() {
+func (tx *TX) TxEnd() {
 	defer func() {
 		for _, stmt := range tx.Stmt {
 			stmt.Close()

pkg/database/dbstock.go

@@ -2,10 +2,7 @@ package database
 
 import (
 	"database/sql"
-	"fmt"
 	"log"
-	"path/filepath"
-	"time"
 
 	"github.com/vinser/flibgolite/pkg/hash"
 	"github.com/vinser/flibgolite/pkg/model"
@@ -23,70 +20,8 @@ func (tx *TX) PrepareStatements() {
 	tx.Stmt["insertIntoSeries"] = tx.mustPrepare(`INSERT INTO series (name) VALUES (?)`)
 }
 
-func (h *Handler) AddBooksToIndex() {
-	defer func() {
-		h.TX.txEnd()
-		h.StopDB <- struct{}{}
-	}()
-	bookInTX := 0
-	for {
-		select {
-		case book := <-h.Queue:
-			if bookInTX == 0 {
-				h.TX = h.DB.txBegin()
-			}
-			h.NewBook(&book)
-			bookInTX++
-			h.LOG.I.Printf("file %s from %s has been added\n", book.File, book.Archive)
-			if bookInTX >= h.CFG.Database.MAX_BOOKS_IN_TX {
-				h.TX.txEnd()
-				bookInTX = 0
-			}
-		case <-time.After(time.Second):
-			h.LOG.D.Printf("Book queue timeout")
-			if h.TX != nil {
-				h.TX.txEnd()
-			}
-			bookInTX = 0
-		case <-h.StopDB:
-			return
-		}
-	}
-}
-
-// Files and Archives
-func (db *DB) NotInStock(name string) error {
-	var id int64
-	switch filepath.Ext(name) {
-	case ".zip":
-		q := "SELECT id FROM books WHERE archive=?"
-		err := db.QueryRow(q, name).Scan(&id)
-		if err != sql.ErrNoRows {
-			return fmt.Errorf("archive %s is in stock already and has been skipped", name)
-		}
-	case ".epub", ".fb2":
-		q := "SELECT id FROM books WHERE file=? AND archive=''"
-		err := db.QueryRow(q, name).Scan(&id)
-		if err != sql.ErrNoRows {
-			return fmt.Errorf("file %s is in stock already and has been skipped", name)
-		}
-	default:
-		return fmt.Errorf("file %s has unsupported format and has been skipped", name)
-	}
-	return nil
-}
-
 // Books
-func (h *Handler) NewBook(b *model.Book) {
-	tx := h.TX
-	switch h.Hashes.IsUnique(b) {
-	case hash.DuplicateCRC32:
-		tx.RecordBookState(b, hash.DuplicateCRC32)
-		return
-	case hash.DuplicateTitlePlot:
-		tx.RecordBookState(b, hash.DuplicateTitlePlot)
-		return
-	}
+func (tx *TX) NewBook(b *model.Book) {
 
 	languageId := tx.NewLanguage(b.Language)
 	serieId := tx.NewSerie(b.Serie)