Skip to content

Commit 1d92da5

Browse files
shermpshermp
committed
Merge branch 'updated-series'
2 parents 24e1066 + f6f361f commit 1d92da5

File tree

4 files changed

+142
-56
lines changed

4 files changed

+142
-56
lines changed

‎go.mod‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ require (
44
github.com/BurntSushi/toml v0.3.1
55
github.com/bamiaux/rez v0.0.0-20170731184118-29f4463c688b
66
github.com/geek1011/koboutils/v2 v2.0.1-0.20200209164819-ab31aac12ee0
7-
github.com/gofrs/uuid v3.2.0+incompatible
7+
github.com/gofrs/uuid v3.2.0+incompatible // indirect
8+
github.com/google/uuid v1.1.1
89
github.com/kapmahc/epub v0.1.1
910
github.com/mattn/go-sqlite3 v2.0.3+incompatible
1011
github.com/shermp/UNCaGED v0.5.0

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ github.com/geek1011/koboutils/v2 v2.0.1-0.20200209164819-ab31aac12ee0 h1:S11UsOD
66
github.com/geek1011/koboutils/v2 v2.0.1-0.20200209164819-ab31aac12ee0/go.mod h1:c/EwVM90qr2bHUrNBVi4jBUWdz+lH1ns2xlfQoiv1vk=
77
github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
88
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
9+
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
10+
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
911
github.com/kapmahc/epub v0.1.1 h1:a4fgmhh/q2vyzFR2QXOVohR2zAuQvbacCjMZ1LGr0lw=
1012
github.com/kapmahc/epub v0.1.1/go.mod h1:UpnUbQO78vpmp6TC4emDTAIG6XVcdnZTnaTx06qbtYM=
1113
github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=

‎kobo-uncaged/device/device.go‎

Lines changed: 137 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import (
1717

1818
"github.com/bamiaux/rez"
1919
"github.com/geek1011/koboutils/v2/kobo"
20-
"github.com/gofrs/uuid"
20+
"github.com/google/uuid"
2121
"github.com/kapmahc/epub"
2222
"github.com/shermp/Kobo-UNCaGED/kobo-uncaged/kuprint"
2323
"github.com/shermp/Kobo-UNCaGED/kobo-uncaged/util"
@@ -63,6 +63,7 @@ func New(dbRootDir, sdRootDir string, updatingMD bool, opts *KuOptions, vers str
6363
}
6464
k.Passwords = newUncagedPassword(k.KuConfig.PasswordList)
6565
k.UpdatedMetadata = make(map[string]struct{}, 0)
66+
k.SeriesIDMap = make(map[string]string, 0)
6667
headerStr := "Kobo-UNCaGED " + vers
6768
if k.useSDCard {
6869
headerStr += "\nUsing SD Card"
@@ -113,7 +114,7 @@ func New(dbRootDir, sdRootDir string, updatingMD bool, opts *KuOptions, vers str
113114

114115
func (k *Kobo) openNickelDB() error {
115116
var err error
116-
dsn := "file:" + filepath.Join(k.DBRootDir, koboDBpath) + "?cache=shared&mode=rw"
117+
dsn := "file:" + filepath.Join(k.DBRootDir, koboDBpath) + "?_timeout=2000&_journal=WAL&mode=rw&_mutex=full&_sync=NORMAL"
117118
if k.nickelDB, err = sql.Open("sqlite3", dsn); err != nil {
118119
err = fmt.Errorf("openNickelDB: sql open failed: %w", err)
119120
}
@@ -128,41 +129,47 @@ func (k *Kobo) setupMetaTrigger() error {
128129
}
129130
// Table to hold temporary metadata for the trigger to use
130131
metaTableQuery := `
131-
CREATE TABLE IF NOT EXISTS _ku_meta (
132-
ContentID TEXT NOT NULL UNIQUE,
133-
Description TEXT,
134-
Series TEXT,
135-
SeriesNumber TEXT,
132+
DROP TABLE IF EXISTS _ku_meta;
133+
CREATE TABLE IF NOT EXISTS _ku_meta_new (
134+
_schema_vers INTEGER NOT NULL DEFAULT 0,
135+
ContentID TEXT NOT NULL UNIQUE,
136+
Description TEXT,
137+
Series TEXT,
138+
SeriesNumber TEXT,
139+
SeriesNumberFloat REAL,
140+
SeriesID TEXT,
136141
PRIMARY KEY(ContentID)
137142
);`
138143
if _, err = tx.Exec(metaTableQuery); err != nil {
139144
tx.Rollback()
140-
return fmt.Errorf("setupMetaTrigger: Create _ku_meta table error: %w", err)
145+
return fmt.Errorf("setupMetaTrigger: Create _ku_meta_new table error: %w", err)
141146
}
142147
// Trigger fired when Nickel inserts a book into the content table
143148
// It replaces and/or adds metadata after the record has been inserted
144149
triggerQuery := `
145-
DROP TRIGGER IF EXISTS _ku_meta_content_insert;
146-
CREATE TRIGGER _ku_meta_content_insert
150+
DROP TRIGGER IF EXISTS _ku_meta_new_content_insert;
151+
CREATE TRIGGER _ku_meta_new_content_insert
147152
AFTER INSERT ON content WHEN
148153
(new.ImageId LIKE "file____mnt_%") AND
149-
(SELECT count() FROM _ku_meta WHERE ContentID = new.ContentID)
154+
(SELECT count() FROM _ku_meta_new WHERE ContentID = new.ContentID)
150155
BEGIN
151156
UPDATE content
152157
SET
153-
Description = (SELECT Description FROM _ku_meta WHERE ContentID = new.ContentID),
154-
Series = (SELECT Series FROM _ku_meta WHERE ContentID = new.ContentID),
155-
SeriesNumber = (SELECT SeriesNumber FROM _ku_meta WHERE ContentID = new.ContentID)
158+
Description = (SELECT Description FROM _ku_meta_new WHERE ContentID = new.ContentID),
159+
Series = (SELECT Series FROM _ku_meta_new WHERE ContentID = new.ContentID),
160+
SeriesNumber = (SELECT SeriesNumber FROM _ku_meta_new WHERE ContentID = new.ContentID),
161+
SeriesNumberFloat = (SELECT SeriesNumberFloat FROM _ku_meta_new WHERE ContentID = new.ContentID),
162+
SeriesID = (SELECT SeriesID FROM _ku_meta_new WHERE ContentID = new.ContentID)
156163
WHERE ContentID = new.ContentID;
157-
DELETE FROM _ku_meta WHERE ContentID = new.ContentID;
164+
DELETE FROM _ku_meta_new WHERE ContentID = new.ContentID;
158165
END;`
159166
if _, err = tx.Exec(triggerQuery); err != nil {
160167
tx.Rollback()
161168
return fmt.Errorf("setupMetaTrigger: Create trigger error: %w", err)
162169
}
163-
// Make sure the _ku_meta has no existing records before beginning. Makes sure we aren't
170+
// Make sure the _ku_meta_new has no existing records before beginning. Makes sure we aren't
164171
// adding a duplicate row
165-
if _, err = tx.Exec(`DELETE FROM _ku_meta;`); err != nil {
172+
if _, err = tx.Exec(`DELETE FROM _ku_meta_new;`); err != nil {
166173
tx.Rollback()
167174
return fmt.Errorf("setupMetaTrigger: Delete rows error: %w", err)
168175
}
@@ -178,13 +185,13 @@ func (k *Kobo) removeMetaTrigger() error {
178185
if err != nil {
179186
return fmt.Errorf("removeMetaTrigger: Error beginning transaction: %w", err)
180187
}
181-
if _, err = tx.Exec(`DROP TABLE IF EXISTS _ku_meta;`); err != nil {
188+
if _, err = tx.Exec(`DROP TRIGGER IF EXISTS _ku_meta_new_content_insert;`); err != nil {
182189
tx.Rollback()
183-
return fmt.Errorf("removeMetaTrigger: drop _ku_meta error: %w", err)
190+
return fmt.Errorf("removeMetaTrigger: drop _ku_meta_new_content_insert error: %w", err)
184191
}
185-
if _, err = tx.Exec(`DROP TRIGGER IF EXISTS _ku_meta_content_insert;`); err != nil {
192+
if _, err = tx.Exec(`DROP TABLE IF EXISTS _ku_meta_new;`); err != nil {
186193
tx.Rollback()
187-
return fmt.Errorf("removeMetaTrigger: drop _ku_meta_content_insert error: %w", err)
194+
return fmt.Errorf("removeMetaTrigger: drop _ku_meta_new error: %w", err)
188195
}
189196
if err = tx.Commit(); err != nil {
190197
return fmt.Errorf("removeMetaTrigger: Error committing transaction: %w", err)
@@ -195,28 +202,38 @@ func (k *Kobo) removeMetaTrigger() error {
195202
// UpdateIfExists updates onboard metadata if it exists in the Nickel database
196203
func (k *Kobo) UpdateIfExists(cID string, len int) error {
197204
if _, exists := k.MetadataMap[cID]; exists {
205+
var err error
206+
tx, err := k.nickelDB.Begin()
207+
if err != nil {
208+
return fmt.Errorf("removeMetaTrigger: Error beginning transaction: %w", err)
209+
}
198210
var currSize int
199211
// Make really sure this is in the Nickel DB
200212
// The query helpfully comes from Calibre
201213
testQuery := `
202214
SELECT ___FileSize
203215
FROM content
204216
WHERE ContentID = ?
205-
AND ContentType = 6`
206-
if err := k.nickelDB.QueryRow(testQuery, cID).Scan(&currSize); err != nil {
217+
AND ContentType = 6;`
218+
if err := tx.QueryRow(testQuery, cID).Scan(&currSize); err != nil {
219+
tx.Rollback()
207220
return fmt.Errorf("UpdateIfExists: error querying row: %w", err)
208221
}
209222
if currSize != len {
210223
updateQuery := `
211224
UPDATE content
212225
SET ___FileSize = ?
213226
WHERE ContentId = ?
214-
AND ContentType = 6`
215-
if _, err := k.nickelDB.Exec(updateQuery, len, cID); err != nil {
227+
AND ContentType = 6;`
228+
if _, err := tx.Exec(updateQuery, len, cID); err != nil {
229+
tx.Rollback()
216230
return fmt.Errorf("UpdateIfExists: error updating filesize field: %w", err)
217231
}
218232
log.Println("Updated existing book file length")
219233
}
234+
if err = tx.Commit(); err != nil {
235+
return fmt.Errorf("UpdateIfExists: Error committing transaction: %w", err)
236+
}
220237
}
221238
return nil
222239
}
@@ -355,41 +372,40 @@ func (k *Kobo) readMDfile() error {
355372
// Now that we have our map, we need to check for any books in the DB not in our
356373
// metadata cache, or books that are in our cache but not in the DB
357374
var (
358-
dbCID string
359-
dbTitle *string
360-
dbAttr *string
361-
dbDesc *string
362-
dbPublisher *string
363-
dbSeries *string
364-
dbbSeriesNum *string
365-
dbContentType int
366-
dbMimeType string
375+
dbCID string
376+
dbTitle *string
377+
dbAttr *string
378+
dbDesc *string
379+
dbPublisher *string
380+
dbSeries *string
381+
dbbSeriesNum *string
382+
dbMimeType string
367383
)
368-
query := fmt.Sprintf(`
369-
SELECT ContentID, Title, Attribution, Description, Publisher, Series, SeriesNumber, ContentType, MimeType
384+
query := `
385+
SELECT ContentID, Title, Attribution, Description, Publisher, Series, SeriesNumber, MimeType
370386
FROM content
371387
WHERE ContentType=6
372388
AND MimeType NOT LIKE 'image%%'
373389
AND (IsDownloaded='true' OR IsDownloaded=1)
374390
AND ___FileSize>0
375391
AND Accessibility=-1
376-
AND ContentID LIKE '%s%%';`, k.ContentIDprefix)
392+
AND ContentID LIKE ?;`
377393

378-
bkRows, err := k.nickelDB.Query(query)
394+
bkRows, err := k.nickelDB.Query(query, fmt.Sprintf("%s%%", k.ContentIDprefix))
379395
if err != nil {
380396
return fmt.Errorf("readMDfile: error getting book rows: %w", err)
381397
}
382398
defer bkRows.Close()
383399
for bkRows.Next() {
384-
err = bkRows.Scan(&dbCID, &dbTitle, &dbAttr, &dbDesc, &dbPublisher, &dbSeries, &dbbSeriesNum, &dbContentType, &dbMimeType)
400+
err = bkRows.Scan(&dbCID, &dbTitle, &dbAttr, &dbDesc, &dbPublisher, &dbSeries, &dbbSeriesNum, &dbMimeType)
385401
if err != nil {
386402
return fmt.Errorf("readMDfile: row decoding error: %w", err)
387403
}
388404
if _, exists := tmpMap[dbCID]; !exists {
389405
log.Printf("Book not in cache: %s\n", dbCID)
390406
bkMD := uc.CalibreBookMeta{}
391407
bkMD.Lpath = util.ContentIDtoLpath(dbCID, string(onboardPrefix))
392-
uuidV4, _ := uuid.NewV4()
408+
uuidV4, _ := uuid.NewRandom()
393409
bkMD.UUID = uuidV4.String()
394410
bkMD.Comments, bkMD.Publisher, bkMD.Series = dbDesc, dbPublisher, dbSeries
395411
if dbTitle != nil {
@@ -486,7 +502,7 @@ func (k *Kobo) WriteUpdateMDfile() error {
486502
func (k *Kobo) loadDeviceInfo() error {
487503
emptyOrNotExist, err := util.ReadJSON(filepath.Join(k.BKRootDir, calibreDIfile), &k.DriveInfo.DevInfo)
488504
if emptyOrNotExist {
489-
uuid4, _ := uuid.NewV4()
505+
uuid4, _ := uuid.NewRandom()
490506
k.DriveInfo.DevInfo.LocationCode = "main"
491507
k.DriveInfo.DevInfo.DeviceName = k.Device.Family()
492508
k.DriveInfo.DevInfo.DeviceStoreUUID = uuid4.String()
@@ -573,16 +589,78 @@ func (k *Kobo) SaveCoverImage(contentID string, size image.Point, imgB64 string)
573589
func (k *Kobo) UpdateNickelDB() (bool, error) {
574590
rerun := false
575591
var err error
592+
var updateErr error
593+
var desc, series, seriesID, seriesNum *string
594+
var seriesNumFloat *float64
576595
tx, err := k.nickelDB.Begin()
577596
if err != nil {
578-
return rerun, fmt.Errorf("UpdateNickelDB: Error beginning transaction: %w", err)
597+
return rerun, fmt.Errorf("UpdateNickelDB: Error beginning SeriesID transaction: %w", err)
598+
}
599+
// Get Series and SeriesID from the DB for non-sideloaded books
600+
getSeriesQ := `
601+
SELECT DISTINCT Series, SeriesID FROM content
602+
WHERE ContentType == 6 AND ContentID NOT LIKE 'file://%' AND (Series IS NOT NULL AND Series != '') AND (SeriesID IS NOT NULL AND SeriesID != '');`
603+
seriesRows, err := tx.Query(getSeriesQ)
604+
if err != nil {
605+
tx.Rollback()
606+
return rerun, fmt.Errorf("UpdateNickelDB: error getting series rows: %w", err)
607+
}
608+
defer seriesRows.Close()
609+
for seriesRows.Next() {
610+
if err = seriesRows.Scan(&series, &seriesID); err != nil {
611+
tx.Rollback()
612+
return rerun, fmt.Errorf("UpdateNickelDB: error decoding row: %w", err)
613+
}
614+
// The WHERE clause in the SQL query should ensure we never get NULL values
615+
k.SeriesIDMap[*series] = *seriesID
616+
}
617+
if err = seriesRows.Err(); err != nil {
618+
tx.Rollback()
619+
return rerun, fmt.Errorf("UpdateNickelDB: seriesRows error: %w", err)
620+
}
621+
622+
// Update SeriesID column for all series that have Kobo derived SeriesID values
623+
// We do this because a user could download a book from Kobo which is in a series that
624+
// the user already has other (sideloaded) books on device
625+
seriesIDQuery := `UPDATE content SET SeriesID=? WHERE Series=?;`
626+
seriesIDstmt, err := tx.Prepare(seriesIDQuery)
627+
if err != nil {
628+
tx.Rollback()
629+
return rerun, fmt.Errorf("UpdateNickelDB: SeriesID prepared statement failed: %w", err)
630+
}
631+
for s, sID := range k.SeriesIDMap {
632+
if _, err = seriesIDstmt.Exec(sID, s); err != nil {
633+
tx.Rollback()
634+
return rerun, fmt.Errorf("UpdateNickelDB: %w", err)
635+
}
636+
}
637+
if err = tx.Commit(); err != nil {
638+
return rerun, fmt.Errorf("UpdateNickelDB: Error committing SeriesID transaction: %w", err)
639+
}
640+
641+
// Once we've done that, also check if there are still empty SeriesID columns that have Series set,
642+
// and update if required. This shouldn't have much, if any, effect if KU has been run before, or
643+
// the device has been connected to calibre
644+
seriesIDQuery = `
645+
UPDATE content SET SeriesID=Series
646+
WHERE ContentType == 6 AND (Series IS NOT NULL OR Series != '') AND (SeriesID IS NULL OR SeriesID == '');`
647+
if _, err = k.nickelDB.Exec(seriesIDQuery); err != nil {
648+
return rerun, fmt.Errorf("UpdateNickelDB: %w", err)
649+
}
650+
651+
// Begin a new transaction for updating metadata
652+
tx, err = k.nickelDB.Begin()
653+
if err != nil {
654+
return rerun, fmt.Errorf("UpdateNickelDB: Error beginning update transaction: %w", err)
579655
}
656+
657+
// Prepare database with some statements
580658
// Insert prepared statement if using triggers
581659
var insertStmt *sql.Stmt
582660
if k.KuConfig.AddMetadataByTrigger {
583661
insertQuery := `
584-
INSERT INTO _ku_meta (ContentID, Description, Series, SeriesNumber)
585-
VALUES (?, ?, ?, ?);`
662+
INSERT INTO _ku_meta_new (ContentID, Description, Series, SeriesNumber, SeriesNumberFloat, SeriesID)
663+
VALUES (?, ?, ?, ?, ?, ?);`
586664
insertStmt, err = tx.Prepare(insertQuery)
587665
if err != nil {
588666
tx.Rollback()
@@ -595,43 +673,47 @@ func (k *Kobo) UpdateNickelDB() (bool, error) {
595673
Description=?,
596674
Series=?,
597675
SeriesNumber=?,
598-
SeriesNumberFloat=?
676+
SeriesNumberFloat=?,
677+
SeriesID=?
599678
WHERE ContentID=?;`
600679
updateStmt, err := tx.Prepare(updateQuery)
601680
if err != nil {
602681
tx.Rollback()
603682
return rerun, fmt.Errorf("UpdateNickelDB: prepared statement failed: %w", err)
604683
}
605-
var updateErr error
606-
var desc, series, seriesNum *string
607-
var seriesNumFloat *float64
608684
for cid := range k.UpdatedMetadata {
609-
desc, series, seriesNum, seriesNumFloat = nil, nil, nil, nil
685+
desc, series, seriesID, seriesNum, seriesNumFloat = nil, nil, nil, nil, nil
610686
if k.MetadataMap[cid].Comments != nil && *k.MetadataMap[cid].Comments != "" {
611687
desc = k.MetadataMap[cid].Comments
612688
}
613689
if k.MetadataMap[cid].Series != nil && *k.MetadataMap[cid].Series != "" {
690+
// TODO: Fuzzy series matching to deal with 'The' prefixes and 'Series' postfixes?
614691
series = k.MetadataMap[cid].Series
692+
seriesID = series
693+
if sID, ok := k.SeriesIDMap[*series]; ok {
694+
seriesID = &sID
695+
}
615696
}
616697
if k.MetadataMap[cid].SeriesIndex != nil && *k.MetadataMap[cid].SeriesIndex != 0.0 {
617698
sn := strconv.FormatFloat(*k.MetadataMap[cid].SeriesIndex, 'f', -1, 64)
618699
seriesNum = &sn
619700
seriesNumFloat = k.MetadataMap[cid].SeriesIndex
620701
}
621-
// Note, not rolling back transaction on error. Is this allowed?
622-
// Don't want one bad update to derail the whole thing, hence avoiding rollback
702+
// We rollback on any sort of error, to lessen any chance of database corruption
623703
if _, ok := k.BooksInDB[cid]; ok {
624-
_, err = updateStmt.Exec(desc, series, seriesNum, seriesNumFloat, cid)
704+
_, err = updateStmt.Exec(desc, series, seriesNum, seriesNumFloat, seriesID, cid)
625705
if err != nil {
626-
updateErr = fmt.Errorf("UpdateNickelDB: %w", err)
706+
tx.Rollback()
707+
return rerun, fmt.Errorf("UpdateNickelDB: %w", err)
627708
}
628709
delete(k.UpdatedMetadata, cid)
629710
} else {
630711
rerun = true
631712
if k.KuConfig.AddMetadataByTrigger {
632-
_, err = insertStmt.Exec(cid, desc, series, seriesNum)
713+
_, err = insertStmt.Exec(cid, desc, series, seriesNum, seriesNumFloat, seriesID)
633714
if err != nil {
634-
updateErr = fmt.Errorf("UpdateNickelDB: %w", err)
715+
tx.Rollback()
716+
return rerun, fmt.Errorf("UpdateNickelDB: %w", err)
635717
}
636718
delete(k.UpdatedMetadata, cid)
637719
}

‎kobo-uncaged/device/types.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ type Kobo struct {
5959
MetadataMap map[string]uc.CalibreBookMeta
6060
UpdatedMetadata map[string]struct{}
6161
BooksInDB map[string]struct{}
62+
SeriesIDMap map[string]string
6263
Passwords *uncagedPassword
6364
DriveInfo uc.DeviceInfo
6465
nickelDB *sql.DB

0 commit comments

Comments
 (0)