Skip to content

Commit fea9c0e

Browse files
vinservinser
authored
Merge pull request #50 from vinser/48-endless-processing-of-a-book-placed-in-a-zip
48 endless processing of a book placed in a zip +49 continue to process archive
2 parents 365fe47 + 0189488 commit fea9c0e

File tree

8 files changed

+412
-311
lines changed

8 files changed

+412
-311
lines changed

‎cmd/flibgolite/main.go‎

Lines changed: 52 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/vinser/flibgolite/pkg/config"
1616
"github.com/vinser/flibgolite/pkg/database"
1717
"github.com/vinser/flibgolite/pkg/genres"
18+
"github.com/vinser/flibgolite/pkg/hash"
1819
"github.com/vinser/flibgolite/pkg/model"
1920
"github.com/vinser/flibgolite/pkg/opds"
2021
"github.com/vinser/flibgolite/pkg/stock"
@@ -134,51 +135,49 @@ func reindexStock() {
134135

135136
db := database.NewDB(cfg.Database.DSN)
136137
defer db.Close()
137-
db.InitDB()
138-
stockLog.S.Println("Book stock was inited. Tables were created in empty database")
139-
140-
genresTree := genres.NewGenresTree(cfg.Genres.TREE_FILE)
141-
142-
databaseQueue := make(chan model.Book, cfg.Database.BOOK_QUEUE_SIZE)
143-
defer close(databaseQueue)
144-
databaseHandler := &database.Handler{
145-
CFG: cfg,
146-
DB: db,
147-
LOG: stockLog,
148-
Queue: databaseQueue,
138+
if !db.IsReady() {
139+
db.InitDB()
140+
stockLog.S.Println("Book stock was inited. Tables were created in empty database")
149141
}
150-
databaseHandler.Stop = make(chan struct{})
151-
defer close(databaseHandler.Stop)
152142

153-
go databaseHandler.AddBooksToIndex()
143+
genresTree := genres.NewGenresTree(cfg.Genres.TREE_FILE)
144+
hashes := hash.InitHashes(db.DB)
154145

146+
bookQueue := make(chan model.Book, cfg.Database.BOOK_QUEUE_SIZE)
147+
defer close(bookQueue)
155148
fileQueue := make(chan stock.File, cfg.Database.FILE_QUEUE_SIZE)
156149
defer close(fileQueue)
157150
stockHandler := &stock.Handler{
158-
CFG: cfg,
159-
LOG: stockLog,
160-
DB: db,
161-
GT: genresTree,
162-
Queue: fileQueue,
151+
CFG: cfg,
152+
LOG: stockLog,
153+
DB: db,
154+
GT: genresTree,
155+
BookQueue: bookQueue,
156+
FileQueue: fileQueue,
157+
Hashes: hashes,
163158
}
159+
stockHandler.StopDB = make(chan struct{})
160+
defer close(stockHandler.StopDB)
161+
stockHandler.StopScan = make(chan struct{})
162+
defer close(stockHandler.StopScan)
163+
164164
stockHandler.InitStockFolders()
165-
stockHandler.Stop = make(chan struct{})
166-
defer close(stockHandler.Stop)
165+
go stockHandler.AddBooksToIndex()
167166
for i := 0; i < cfg.Database.MAX_SCAN_THREADS; i++ {
168-
go stockHandler.ParseFB2Queue(databaseQueue)
167+
go stockHandler.ParseFB2Queue()
169168
}
170169

171-
defer func() { stockHandler.Stop <- struct{}{} }()
170+
defer func() { stockHandler.StopScan <- struct{}{} }()
172171
dir := cfg.Library.STOCK_DIR
173172
if len(cfg.Library.NEW_DIR) > 0 {
174173
dir = cfg.Library.NEW_DIR
175174
}
176-
stockHandler.ScanDir(dir, databaseQueue)
175+
stockHandler.ScanDir(dir)
177176

178-
stockHandler.Stop <- struct{}{}
177+
stockHandler.StopScan <- struct{}{}
179178

180-
databaseHandler.Stop <- struct{}{}
181-
<-databaseHandler.Stop
179+
stockHandler.StopDB <- struct{}{}
180+
<-stockHandler.StopDB
182181

183182
stockLog.S.Println("<<< Book stock reindex finished <<<<<<<<<<<<<<<<<<<<<<<<<<<")
184183
stockLog.S.Println("Time elapsed: ", time.Since(start))
@@ -203,46 +202,42 @@ func run() {
203202
}
204203

205204
genresTree := genres.NewGenresTree(cfg.Genres.TREE_FILE)
205+
hashes := hash.InitHashes(db.DB)
206206

207-
databaseQueue := make(chan model.Book, cfg.Database.BOOK_QUEUE_SIZE)
208-
defer close(databaseQueue)
209-
databaseHandler := &database.Handler{
210-
CFG: cfg,
211-
DB: db,
212-
LOG: stockLog,
213-
Queue: databaseQueue,
214-
}
215-
databaseHandler.Stop = make(chan struct{})
216-
defer close(databaseHandler.Stop)
217-
218-
go databaseHandler.AddBooksToIndex()
219-
207+
bookQueue := make(chan model.Book, cfg.Database.BOOK_QUEUE_SIZE)
208+
defer close(bookQueue)
220209
fileQueue := make(chan stock.File, cfg.Database.FILE_QUEUE_SIZE)
221210
defer close(fileQueue)
222211
stockHandler := &stock.Handler{
223-
CFG: cfg,
224-
LOG: stockLog,
225-
DB: db,
226-
GT: genresTree,
227-
Queue: fileQueue,
212+
CFG: cfg,
213+
LOG: stockLog,
214+
DB: db,
215+
GT: genresTree,
216+
BookQueue: bookQueue,
217+
FileQueue: fileQueue,
218+
Hashes: hashes,
228219
}
220+
stockHandler.StopDB = make(chan struct{})
221+
defer close(stockHandler.StopDB)
229222
stockHandler.InitStockFolders()
230-
stockHandler.Stop = make(chan struct{})
231-
defer close(stockHandler.Stop)
223+
stockHandler.StopScan = make(chan struct{})
224+
defer close(stockHandler.StopScan)
225+
226+
go stockHandler.AddBooksToIndex()
232227
for i := 0; i < cfg.Database.MAX_SCAN_THREADS; i++ {
233-
go stockHandler.ParseFB2Queue(databaseQueue)
228+
go stockHandler.ParseFB2Queue()
234229
}
235230
go func() {
236-
defer func() { stockHandler.Stop <- struct{}{} }()
231+
defer func() { stockHandler.StopScan <- struct{}{} }()
237232
dir := cfg.Library.STOCK_DIR
238233
if len(cfg.Library.NEW_DIR) > 0 {
239234
dir = cfg.Library.NEW_DIR
240235
}
241236
for {
242-
stockHandler.ScanDir(dir, databaseQueue)
237+
stockHandler.ScanDir(dir)
243238
time.Sleep(time.Duration(cfg.Database.POLL_DELAY) * time.Second)
244239
select {
245-
case <-stockHandler.Stop:
240+
case <-stockHandler.StopScan:
246241
return
247242
default:
248243
continue
@@ -279,14 +274,14 @@ func run() {
279274
opdsHandler.LOG.S.Printf("Shutdown started...\n")
280275

281276
// Stop scanning for new acquisitions and wait for completion
282-
stockHandler.Stop <- struct{}{}
283-
<-stockHandler.Stop
277+
stockHandler.StopScan <- struct{}{}
278+
<-stockHandler.StopScan
284279
stockHandler.LOG.S.Printf("New acquisitions scanning was stoped correctly\n")
285280

286281
// Stop addind new acquisitions to index and wait for completion
287-
databaseHandler.Stop <- struct{}{}
288-
<-databaseHandler.Stop
289-
databaseHandler.LOG.S.Printf("New acquisitions adding was stoped correctly\n")
282+
stockHandler.StopDB <- struct{}{}
283+
<-stockHandler.StopDB
284+
stockHandler.LOG.S.Printf("New acquisitions adding was stoped correctly\n")
290285

291286
// Shutdown OPDS server
292287
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)

‎pkg/config/config.go‎

Lines changed: 74 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -15,36 +15,42 @@ import (
1515
)
1616

1717
// See config.yml for comments about this struct
18+
19+
type Library struct {
20+
STOCK_DIR string `yaml:"STOCK"`
21+
TRASH_DIR string `yaml:"TRASH"`
22+
NEW_DIR string `yaml:"NEW"`
23+
}
24+
type Database struct {
25+
DSN string `yaml:"DSN"`
26+
POLL_DELAY int `yaml:"POLL_DELAY"`
27+
MAX_SCAN_THREADS int `yaml:"MAX_SCAN_THREADS"`
28+
BOOK_QUEUE_SIZE int `yaml:"BOOK_QUEUE_SIZE"`
29+
FILE_QUEUE_SIZE int `yaml:"FILE_QUEUE_SIZE"`
30+
MAX_BOOKS_IN_TX int `yaml:"MAX_BOOKS_IN_TX"`
31+
DEDUPLICATE_LEVEL string `yaml:"DEDUPLICATE_LEVEL"`
32+
}
33+
type Genres struct {
34+
TREE_FILE string `yaml:"TREE_FILE"`
35+
}
36+
type Logs struct {
37+
OPDS string `yaml:"OPDS"`
38+
SCAN string `yaml:"SCAN"`
39+
LEVEL string `yaml:"LEVEL"`
40+
}
41+
type OPDS struct {
42+
PORT int `yaml:"PORT"`
43+
TITLE string `yaml:"TITLE"`
44+
PAGE_SIZE int `yaml:"PAGE_SIZE"`
45+
LATEST_DAYS int `yaml:"LATEST_DAYS"`
46+
NO_CONVERSION bool `yaml:"NO_CONVERSION"`
47+
}
1848
type Config struct {
19-
Library struct {
20-
STOCK_DIR string `yaml:"STOCK"`
21-
TRASH_DIR string `yaml:"TRASH"`
22-
NEW_DIR string `yaml:"NEW"`
23-
}
24-
Database struct {
25-
DSN string `yaml:"DSN"`
26-
POLL_DELAY int `yaml:"POLL_DELAY"`
27-
MAX_SCAN_THREADS int `yaml:"MAX_SCAN_THREADS"`
28-
BOOK_QUEUE_SIZE int `yaml:"BOOK_QUEUE_SIZE"`
29-
FILE_QUEUE_SIZE int `yaml:"FILE_QUEUE_SIZE"`
30-
MAX_BOOKS_IN_TX int `yaml:"MAX_BOOKS_IN_TX"`
31-
DEDUPLICATE_LEVEL string `yaml:"DEDUPLICATE_LEVEL"`
32-
}
33-
Genres struct {
34-
TREE_FILE string `yaml:"TREE_FILE"`
35-
}
36-
Logs struct {
37-
OPDS string `yaml:"OPDS"`
38-
SCAN string `yaml:"SCAN"`
39-
LEVEL string `yaml:"LEVEL"`
40-
}
41-
OPDS struct {
42-
PORT int `yaml:"PORT"`
43-
TITLE string `yaml:"TITLE"`
44-
PAGE_SIZE int `yaml:"PAGE_SIZE"`
45-
LATEST_DAYS int `yaml:"LATEST_DAYS"`
46-
NO_CONVERSION bool `yaml:"NO_CONVERSION"`
47-
}
49+
Library Library `yaml:"library"`
50+
Database Database `yaml:"database"`
51+
Genres Genres `yaml:"genres"`
52+
Logs Logs `yaml:"logs"`
53+
OPDS OPDS `yaml:"opds"`
4854
locales.Locales
4955
}
5056

@@ -87,79 +93,50 @@ func LoadConfig(rootDir string) *Config {
8793
log.Fatal(err)
8894
}
8995
}
90-
c := &Config{}
96+
c := &Config{
97+
Library: Library{
98+
STOCK_DIR: "books/stock",
99+
TRASH_DIR: "",
100+
NEW_DIR: "",
101+
},
102+
Database: Database{
103+
DSN: "dbdata/books.db",
104+
POLL_DELAY: 300,
105+
MAX_SCAN_THREADS: 10,
106+
BOOK_QUEUE_SIZE: 20000,
107+
FILE_QUEUE_SIZE: 20000,
108+
MAX_BOOKS_IN_TX: 20000,
109+
DEDUPLICATE_LEVEL: "F",
110+
},
111+
Genres: Genres{
112+
TREE_FILE: "config/genres.xml",
113+
},
114+
Logs: Logs{
115+
OPDS: "logs/opds.log",
116+
SCAN: "logs/scan.log",
117+
LEVEL: "W",
118+
},
119+
OPDS: OPDS{
120+
PORT: 8085,
121+
TITLE: "FLib Go Go Go!!!",
122+
PAGE_SIZE: 20,
123+
LATEST_DAYS: 14,
124+
NO_CONVERSION: false,
125+
},
126+
Locales: locales.Locales{
127+
DIR: "config/locales",
128+
DEFAULT: "en",
129+
ACCEPTED: "en, ru, uk",
130+
},
131+
}
91132
if err := yaml.Unmarshal([]byte(b), c); err != nil {
92133
log.Fatal(err)
93134
}
94135

95-
if c.Library.STOCK_DIR == "" {
96-
c.Library.STOCK_DIR = "books/stock"
97-
}
98-
if c.Library.TRASH_DIR == "" {
99-
c.Library.TRASH_DIR = "books/trash"
100-
}
101-
if c.Database.DSN == "" {
102-
c.Database.DSN = "dbdata/books.db"
103-
}
104-
105-
if c.Genres.TREE_FILE == "" {
106-
c.Genres.TREE_FILE = "config/genres.xml"
107-
}
108-
109-
if c.Database.POLL_DELAY == 0 {
110-
c.Database.POLL_DELAY = 30
111-
}
112-
if c.Database.MAX_SCAN_THREADS == 0 {
113-
c.Database.MAX_SCAN_THREADS = 10
114-
}
115-
if c.Database.BOOK_QUEUE_SIZE == 0 {
116-
c.Database.BOOK_QUEUE_SIZE = 20000
117-
}
118-
if c.Database.FILE_QUEUE_SIZE == 0 {
119-
c.Database.FILE_QUEUE_SIZE = 20000
120-
}
121-
if c.Database.MAX_BOOKS_IN_TX == 0 {
122-
c.Database.MAX_BOOKS_IN_TX = 20000
123-
}
124-
if c.Database.DEDUPLICATE_LEVEL == "" {
125-
c.Database.DEDUPLICATE_LEVEL = "F"
126-
}
127-
128-
if c.Logs.OPDS == "" {
129-
c.Logs.OPDS = "logs/opds.log"
130-
}
131-
if c.Logs.SCAN == "" {
132-
c.Logs.SCAN = "logs/scan.log"
133-
}
134-
if c.Logs.LEVEL == "" {
135-
c.Logs.LEVEL = "W"
136-
}
137-
138-
if c.OPDS.PORT == 0 {
139-
c.OPDS.PORT = 8085
140-
}
141-
if c.OPDS.TITLE == "" {
142-
c.OPDS.TITLE = "FLib Go Go Go!!!"
143-
}
144-
if c.OPDS.PAGE_SIZE == 0 {
145-
c.OPDS.PAGE_SIZE = 30
146-
}
147-
if c.OPDS.LATEST_DAYS == 0 {
148-
c.OPDS.LATEST_DAYS = 14
149-
}
150-
151-
if c.Locales.DIR == "" {
152-
c.Locales.DIR = "config/locales"
153-
}
154-
if c.Locales.DEFAULT == "" {
155-
c.Locales.DEFAULT = "en"
156-
}
157-
if c.Locales.ACCEPTED == "" {
158-
c.Locales.ACCEPTED = "en, ru, uk"
159-
}
160-
161136
c.Library.STOCK_DIR = makeAbs(rootDir, c.Library.STOCK_DIR)
162-
c.Library.TRASH_DIR = makeAbs(rootDir, c.Library.TRASH_DIR)
137+
if len(c.Library.TRASH_DIR) > 0 {
138+
c.Library.TRASH_DIR = makeAbs(rootDir, c.Library.TRASH_DIR)
139+
}
163140
if len(c.Library.NEW_DIR) > 0 {
164141
c.Library.NEW_DIR = makeAbs(rootDir, c.Library.NEW_DIR)
165142
}

0 commit comments

Comments
 (0)