Skip to content

Commit 450ea1a

Browse files
vinservinser
committed
Add hash
1 parent 365fe47 commit 450ea1a

File tree

6 files changed

+207
-85
lines changed

6 files changed

+207
-85
lines changed

‎cmd/flibgolite/main.go‎

Lines changed: 47 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/vinser/flibgolite/pkg/config"
1616
"github.com/vinser/flibgolite/pkg/database"
1717
"github.com/vinser/flibgolite/pkg/genres"
18+
"github.com/vinser/flibgolite/pkg/hash"
1819
"github.com/vinser/flibgolite/pkg/model"
1920
"github.com/vinser/flibgolite/pkg/opds"
2021
"github.com/vinser/flibgolite/pkg/stock"
@@ -134,51 +135,56 @@ func reindexStock() {
134135

135136
db := database.NewDB(cfg.Database.DSN)
136137
defer db.Close()
137-
db.InitDB()
138-
stockLog.S.Println("Book stock was inited. Tables were created in empty database")
138+
if !db.IsReady() {
139+
db.InitDB()
140+
stockLog.S.Println("Book stock was inited. Tables were created in empty database")
141+
}
139142

140143
genresTree := genres.NewGenresTree(cfg.Genres.TREE_FILE)
144+
hashes := hash.InitHashes(db.DB)
141145

142146
databaseQueue := make(chan model.Book, cfg.Database.BOOK_QUEUE_SIZE)
143147
defer close(databaseQueue)
144148
databaseHandler := &database.Handler{
145-
CFG: cfg,
146-
DB: db,
147-
LOG: stockLog,
148-
Queue: databaseQueue,
149+
CFG: cfg,
150+
DB: db,
151+
LOG: stockLog,
152+
Queue: databaseQueue,
153+
Hashes: hashes,
149154
}
150-
databaseHandler.Stop = make(chan struct{})
151-
defer close(databaseHandler.Stop)
155+
databaseHandler.StopDB = make(chan struct{})
156+
defer close(databaseHandler.StopDB)
152157

153158
go databaseHandler.AddBooksToIndex()
154159

155160
fileQueue := make(chan stock.File, cfg.Database.FILE_QUEUE_SIZE)
156161
defer close(fileQueue)
157162
stockHandler := &stock.Handler{
158-
CFG: cfg,
159-
LOG: stockLog,
160-
DB: db,
161-
GT: genresTree,
162-
Queue: fileQueue,
163+
CFG: cfg,
164+
LOG: stockLog,
165+
DB: db,
166+
GT: genresTree,
167+
Queue: fileQueue,
168+
Hashes: hashes,
163169
}
164170
stockHandler.InitStockFolders()
165-
stockHandler.Stop = make(chan struct{})
166-
defer close(stockHandler.Stop)
171+
stockHandler.StopScan = make(chan struct{})
172+
defer close(stockHandler.StopScan)
167173
for i := 0; i < cfg.Database.MAX_SCAN_THREADS; i++ {
168174
go stockHandler.ParseFB2Queue(databaseQueue)
169175
}
170176

171-
defer func() { stockHandler.Stop <- struct{}{} }()
177+
defer func() { stockHandler.StopScan <- struct{}{} }()
172178
dir := cfg.Library.STOCK_DIR
173179
if len(cfg.Library.NEW_DIR) > 0 {
174180
dir = cfg.Library.NEW_DIR
175181
}
176182
stockHandler.ScanDir(dir, databaseQueue)
177183

178-
stockHandler.Stop <- struct{}{}
184+
stockHandler.StopScan <- struct{}{}
179185

180-
databaseHandler.Stop <- struct{}{}
181-
<-databaseHandler.Stop
186+
databaseHandler.StopDB <- struct{}{}
187+
<-databaseHandler.StopDB
182188

183189
stockLog.S.Println("<<< Book stock reindex finished <<<<<<<<<<<<<<<<<<<<<<<<<<<")
184190
stockLog.S.Println("Time elapsed: ", time.Since(start))
@@ -203,37 +209,40 @@ func run() {
203209
}
204210

205211
genresTree := genres.NewGenresTree(cfg.Genres.TREE_FILE)
212+
hashes := hash.InitHashes(db.DB)
206213

207214
databaseQueue := make(chan model.Book, cfg.Database.BOOK_QUEUE_SIZE)
208215
defer close(databaseQueue)
209216
databaseHandler := &database.Handler{
210-
CFG: cfg,
211-
DB: db,
212-
LOG: stockLog,
213-
Queue: databaseQueue,
217+
CFG: cfg,
218+
DB: db,
219+
LOG: stockLog,
220+
Queue: databaseQueue,
221+
Hashes: hashes,
214222
}
215-
databaseHandler.Stop = make(chan struct{})
216-
defer close(databaseHandler.Stop)
223+
databaseHandler.StopDB = make(chan struct{})
224+
defer close(databaseHandler.StopDB)
217225

218226
go databaseHandler.AddBooksToIndex()
219227

220228
fileQueue := make(chan stock.File, cfg.Database.FILE_QUEUE_SIZE)
221229
defer close(fileQueue)
222230
stockHandler := &stock.Handler{
223-
CFG: cfg,
224-
LOG: stockLog,
225-
DB: db,
226-
GT: genresTree,
227-
Queue: fileQueue,
231+
CFG: cfg,
232+
LOG: stockLog,
233+
DB: db,
234+
GT: genresTree,
235+
Queue: fileQueue,
236+
Hashes: hashes,
228237
}
229238
stockHandler.InitStockFolders()
230-
stockHandler.Stop = make(chan struct{})
231-
defer close(stockHandler.Stop)
239+
stockHandler.StopScan = make(chan struct{})
240+
defer close(stockHandler.StopScan)
232241
for i := 0; i < cfg.Database.MAX_SCAN_THREADS; i++ {
233242
go stockHandler.ParseFB2Queue(databaseQueue)
234243
}
235244
go func() {
236-
defer func() { stockHandler.Stop <- struct{}{} }()
245+
defer func() { stockHandler.StopScan <- struct{}{} }()
237246
dir := cfg.Library.STOCK_DIR
238247
if len(cfg.Library.NEW_DIR) > 0 {
239248
dir = cfg.Library.NEW_DIR
@@ -242,7 +251,7 @@ func run() {
242251
stockHandler.ScanDir(dir, databaseQueue)
243252
time.Sleep(time.Duration(cfg.Database.POLL_DELAY) * time.Second)
244253
select {
245-
case <-stockHandler.Stop:
254+
case <-stockHandler.StopScan:
246255
return
247256
default:
248257
continue
@@ -279,13 +288,13 @@ func run() {
279288
opdsHandler.LOG.S.Printf("Shutdown started...\n")
280289

281290
// Stop scanning for new acquisitions and wait for completion
282-
stockHandler.Stop <- struct{}{}
283-
<-stockHandler.Stop
291+
stockHandler.StopScan <- struct{}{}
292+
<-stockHandler.StopScan
284293
stockHandler.LOG.S.Printf("New acquisitions scanning was stoped correctly\n")
285294

286295
// Stop addind new acquisitions to index and wait for completion
287-
databaseHandler.Stop <- struct{}{}
288-
<-databaseHandler.Stop
296+
databaseHandler.StopDB <- struct{}{}
297+
<-databaseHandler.StopDB
289298
databaseHandler.LOG.S.Printf("New acquisitions adding was stoped correctly\n")
290299

291300
// Shutdown OPDS server

‎pkg/database/db.go‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111

1212
"github.com/jmoiron/sqlx"
1313
"github.com/vinser/flibgolite/pkg/config"
14+
"github.com/vinser/flibgolite/pkg/hash"
1415
"github.com/vinser/flibgolite/pkg/model"
1516
"github.com/vinser/flibgolite/pkg/rlog"
1617

@@ -29,13 +30,14 @@ var SQLITE_DB_DROP string
2930

3031
// ==================================
3132
type Handler struct {
32-
CFG *config.Config
33-
DB *DB
34-
TX *TX
35-
LOG *rlog.Log
36-
WG *sync.WaitGroup
37-
Queue <-chan model.Book
38-
Stop chan struct{}
33+
CFG *config.Config
34+
Hashes *hash.BookHashes
35+
DB *DB
36+
TX *TX
37+
LOG *rlog.Log
38+
WG *sync.WaitGroup
39+
Queue <-chan model.Book
40+
StopDB chan struct{}
3941
}
4042

4143
type DB struct {
@@ -69,9 +71,7 @@ func (db *DB) Close() {
6971
}
7072

7173
func (db *DB) InitDB() {
72-
if !db.IsReady() {
73-
db.execFile(SQLITE_DB_INIT)
74-
}
74+
db.execFile(SQLITE_DB_INIT)
7575
}
7676

7777
func (db *DB) DropDB() {

‎pkg/database/dbopds.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@ func (db *DB) ListSeries(prefix, lang, abc string) []*model.Serie {
278278
SELECT sr.id, substr(sr.name,1,1) as s, count(*) as c
279279
FROM
280280
series as sr,
281-
(SELECT serie_id, language_id, count(*) as c FROM books GROUP BY serie_id HAVING c>2) as b,
281+
(SELECT serie_id, language_id, count(*) as c FROM books WHERE updated > 0 GROUP BY serie_id HAVING c>2) as b,
282282
languages as l
283283
WHERE
284284
sr.id=b.serie_id AND
@@ -293,7 +293,7 @@ func (db *DB) ListSeries(prefix, lang, abc string) []*model.Serie {
293293
SELECT sr.id, substr(sr.name,1,`, fmt.Sprint(l), `) as sn, count(*) as c
294294
FROM
295295
series as sr,
296-
(SELECT serie_id, language_id, count(*) as c FROM books GROUP BY serie_id HAVING c>2) as b,
296+
(SELECT serie_id, language_id, count(*) as c FROM books WHERE updated > 0 GROUP BY serie_id HAVING c>2) as b,
297297
languages as l
298298
WHERE
299299
sr.id=b.serie_id AND
@@ -325,7 +325,8 @@ func (db *DB) ListSeriesWithTotals(prefix, lang string) []*model.Serie {
325325
q := `
326326
SELECT s.id, s.name, count(*) as c
327327
FROM series as s, books as b, languages as l
328-
WHERE
328+
WHERE
329+
b.updated > 0 AND
329330
s.name LIKE ? AND
330331
s.id=b.serie_id AND
331332
l.id=b.language_id AND

‎pkg/database/dbstock.go‎

Lines changed: 15 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,11 @@ import (
77
"path/filepath"
88
"time"
99

10+
"github.com/vinser/flibgolite/pkg/hash"
1011
"github.com/vinser/flibgolite/pkg/model"
1112
)
1213

1314
func (tx *TX) PrepareStatements() {
14-
tx.Stmt["selectIdFromBooksS"] = tx.mustPrepare(`SELECT id FROM books WHERE crc32=? OR (title=? AND plot=?)`)
15-
tx.Stmt["selectIdFromBooksF"] = tx.mustPrepare(`SELECT id FROM books WHERE crc32=?`)
16-
tx.Stmt["insertIntoArchives"] = tx.mustPrepare(`INSERT INTO archives (name, commited) VALUES (?,?)`)
1715
tx.Stmt["selectIdFromLanguages"] = tx.mustPrepare(`SELECT id FROM languages WHERE code=?`)
1816
tx.Stmt["insertIntoLanguages"] = tx.mustPrepare(`INSERT INTO languages (code, name) VALUES (?, ?)`)
1917
tx.Stmt["insertIntoBooks"] = tx.mustPrepare(`INSERT INTO books (file, crc32, archive, size, format, title, sort, year, language_id, plot, cover, keywords, serie_id, serie_num, updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
@@ -28,7 +26,7 @@ func (tx *TX) PrepareStatements() {
2826
func (h *Handler) AddBooksToIndex() {
2927
defer func() {
3028
h.TX.txEnd()
31-
h.Stop <- struct{}{}
29+
h.StopDB <- struct{}{}
3230
}()
3331
bookInTX := 0
3432
for {
@@ -50,7 +48,7 @@ func (h *Handler) AddBooksToIndex() {
5048
h.TX.txEnd()
5149
}
5250
bookInTX = 0
53-
case <-h.Stop:
51+
case <-h.StopDB:
5452
return
5553
}
5654
}
@@ -80,11 +78,16 @@ func (db *DB) NotInStock(name string) error {
8078

8179
// Books
8280
func (h *Handler) NewBook(b *model.Book) {
83-
if h.BookDuplicateFound(b) {
81+
tx := h.TX
82+
switch h.Hashes.IsUnique(b) {
83+
case hash.DuplicateCRC32:
84+
tx.RecordBookState(b, hash.DuplicateCRC32)
85+
return
86+
case hash.DuplicateTitlePlot:
87+
tx.RecordBookState(b, hash.DuplicateTitlePlot)
8488
return
8589
}
8690

87-
tx := h.TX
8891
languageId := tx.NewLanguage(b.Language)
8992
serieId := tx.NewSerie(b.Serie)
9093
res, err := tx.Stmt["insertIntoBooks"].Exec(b.File, b.CRC32, b.Archive, b.Size, b.Format, b.Title, b.Sort, b.Year, languageId, b.Plot, b.Cover, b.Keywords, serieId, b.SerieNum, b.Updated)
@@ -119,18 +122,11 @@ func (h *Handler) NewBook(b *model.Book) {
119122
}
120123
}
121124

122-
func (h *Handler) BookDuplicateFound(b *model.Book) bool {
123-
var id int64 = 0
124-
var err error
125-
switch h.CFG.Database.DEDUPLICATE_LEVEL {
126-
case "S":
127-
err = h.TX.Stmt["selectIdFromBooksS"].QueryRow(b.CRC32, b.Title, b.Plot).Scan(&id)
128-
return err != sql.ErrNoRows
129-
case "F":
130-
err = h.TX.Stmt["selectIdFromBooksF"].QueryRow(b.CRC32).Scan(&id)
131-
return err != sql.ErrNoRows
132-
}
133-
return false
125+
func (tx *TX) RecordBookState(b *model.Book, s hash.BookState) {
126+
_, err := tx.Stmt["insertIntoBooks"].Exec(b.File, b.CRC32, b.Archive, b.Size, b.Format, b.Title, b.Sort, b.Year, 0, b.Plot, b.Cover, b.Keywords, 0, b.SerieNum, int64(s))
127+
if err != nil {
128+
log.Panicln(err)
129+
}
134130
}
135131

136132
// Languages

0 commit comments

Comments
 (0)