fix updating

the test database is now empty but I'll be indexing them all soon

Author: noisecode3

database/README.md

@@ -1,48 +1,41 @@
 # This is a test database
 
-I will list the games in the database here by id number.
+I list some levels I have tested.
 
 You can access the game files here ~/.local/share/TombRaiderLinuxLauncher/ID.TRLE
-where ID is the number listed here.
-If you use you own database you can update it
-with updateDB_1.0.0.py then updateDB_1.1.0.py then to updateDB_1.1.1.py.
-But make backup before. I can implement in app json scraping from trcustoms.org
-to get new levels. It has a nice json api with minimal compact data for info and nice
-picture urls with md5sums. There can be and index database that
-get new level id numbers from trcustoms.org or trle.net with very minimal traffic.
+where ID is the number listed when runngin tombll_manage_data.py -l
 
-1. The Infada Cult
+3573. The Infada Cult
    * <https://www.trle.net/sc/levelfeatures.php?lid=3573>
    * <https://trcustoms.org/levels/3574>
    * have played through
    * no problems
 
-2. Templars Secret
+3082. Templars Secret
    * <https://www.trle.net/sc/levelfeatures.php?lid=3082>
    * <https://trcustoms.org/levels/3025>
    * have played through
    * no problems
 
-3. Secret of the Skull Temple
+3377. Secret of the Skull Temple
    * <https://www.trle.net/sc/levelfeatures.php?lid=3377>
    * <https://trcustoms.org/levels/3310>
    * have played through
-   * lags sometimes
-   * plying with lutris and dgvoodoo, can help
+   * lags sometimes, plying with dgvoodoo can help
 
-4. The Rescue 1-4
+318. The Rescue 1-4
    * <https://www.trle.net/sc/levelfeatures.php?lid=318>
    * <https://trcustoms.org/levels/317>
    * have not played through all of the game
    * no problems
 
-5. War of the Worlds
+3430. War of the Worlds
    * <https://www.trle.net/sc/levelfeatures.php?lid=3430>
    * <https://trcustoms.org/levels/3363>
    * have not played through all of the game
    * plying with lutris and dgvoodoo, fix flashlight bug
 
-6. The Hidden Dagger II - Dragon's Legacy
+3621. The Hidden Dagger II - Dragon's Legacy
    * <https://www.trle.net/sc/levelfeatures.php?lid=3621>
    * <https://trcustoms.org/levels/3642>
    * Starts the game and first map, backgrund audio is missing
@@ -53,13 +46,13 @@ get new level id numbers from trcustoms.org or trle.net with very minimal traffi
    * and run it from there right now
    * use dll ovride winmm=n,b
 
-7. Create a Classic 2021 - The City of Petra
+3417. Create a Classic 2021 - The City of Petra
    * <https://www.trle.net/sc/levelfeatures.php?lid=3417>
    * <https://trcustoms.org/levels/3350>
    * have not played through all of the game
    * no problems
 
-8. Into the Realm of Eternal Darkness
+3557. Into the Realm of Eternal Darkness
    * <https://www.trle.net/sc/levelfeatures.php?lid=3557>
    * <https://trcustoms.org/levels/3541>
    * have not played through all of the game
@@ -80,19 +73,19 @@ get new level id numbers from trcustoms.org or trle.net with very minimal traffi
    just package-linux
    ```
 
-9. The Experiment 3/1 - Under the Moonlight
+1388. The Experiment 3/1 - Under the Moonlight
    * <https://www.trle.net/sc/levelfeatures.php?lid=1388>
    * <https://trcustoms.org/levels/1360>
    * have not played through all of the game
    * no problems
 
-10. The Ancient Knowledge
+1964. The Ancient Knowledge
     * <https://www.trle.net/sc/levelfeatures.php?lid=1964>
     * <https://trcustoms.org/levels/1947>
     * have not played through all of the game
     * no problems
 
-11. Tomb Raider - Biohazard
+3372. Tomb Raider - Biohazard
     * <https://www.trle.net/sc/levelfeatures.php?lid=3372>
     * <https://trcustoms.org/levels/3305>
     * have not played through all of the game

database/data_factory.py

@@ -83,3 +83,12 @@ def make_zip_file():
         "release": "",
         "version": ""
     }
+
+
+def make_picture():
+    """trcustoms.org or TRLE picture data."""
+    return {
+        "position": 0,
+        "md5sum": "",
+        "md5": b''
+    }

database/index.db

@@ -1,4 +1,4 @@
-"""Main loop for CLI menu like interfaces"""
+"""Main loop for CLI menu like interfaces."""
 import sys
 import os
 import time
@@ -53,20 +53,15 @@ def test_trle_pic_local():
 
 
 def test_insert_trle_book():
-    """This must not bother the server but at the same time its not
-       meant for normal users but the database admin to run this,
-       it could take 30-45 min. there is a problem about how TRLE sort result
-       so this method might not work, cus it will miss results, there is another
-       method that is much slower but will work and accurate"""
-
+    """Make a book."""
     # Get the first page to determine the total number of records
     page = scrape_trle.get_trle_page(0)
     total_records = page['records_total']
 
     # Insert the first page of data
     index_query.insert_trle_page(page)
     print(f"Records number:20 of {total_records}")
-    time.sleep(15)
+    time.sleep(3)
 
     # Start offset at 20 and loop through all records in steps of 20
     offset = 20
@@ -82,12 +77,12 @@ def test_insert_trle_book():
             offset += 20
         print(f"Records number:{offset} of {total_records}")
 
-        # Introduce a random delay of 15 seconds
-        time.sleep(15)
+        # Introduce a random delay of 3 seconds
+        time.sleep(3)
 
 
 def show_menu():
-    """Main menu"""
+    """Print main menu."""
     print("\nPlease select a command:")
     print("1. new - Create a new empty index database and add static data")
     print("2. new - Create a new full index database, without info and walkthrough data")

database/main.py

@@ -108,6 +108,7 @@ class INT,
 C.execute('''
 CREATE TABLE Picture (
     PictureID INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    md5sum TEXT NOT NULL UNIQUE,
     data BLOB NOT NULL
 )''')
 
@@ -130,7 +131,9 @@ class INT,
 CREATE TABLE Screens (
     pictureID INTEGER NOT NULL,
     levelID INTEGER NOT NULL,
+    position INTEGER NOT NULL,
     PRIMARY KEY (pictureID, levelID),
+    UNIQUE (levelID, position),
     FOREIGN KEY (pictureID) REFERENCES Picture(PictureID),
     FOREIGN KEY (levelID) REFERENCES Level(LevelID)
 )''')

database/make_tombll_database.py

@@ -334,7 +334,7 @@ def trle_search_parser(url):
               .replace("'", r"%5C%27")
 
 
-def url_postfix(url):
+def url_basename_postfix(url):
     """
     Extract the file extension from a URL without the leading dot.
 
@@ -348,6 +348,20 @@ def url_postfix(url):
     return os.path.splitext(os.path.basename(urlparse(url).path))[1][1:]
 
 
+def url_basename_prefix(url):
+    """
+    Extract the file naem from a URL without the extension.
+
+    Args:
+        url (str): The URL to extract the file extension from.
+
+    Returns:
+        str: The file name without the extension,
+        or an empty string if no extension is present.
+    """
+    return os.path.splitext(os.path.basename(urlparse(url).path))[0]
+
+
 ###############################################################################
 # getter's for https
 ###############################################################################
@@ -391,7 +405,7 @@ def get_image(url):
     if validate_url(url) is None:
         print(f"{url} had wrong domain")
         sys.exit(1)
-    ext = url_postfix(url).lower()
+    ext = url_basename_postfix(url).lower()
     if ext in ('jpg', 'jpeg'):
         return get_jpg(url)
     if ext == 'png':

database/scrape_common.py

@@ -283,7 +283,7 @@ def get_trle_walkthrough(level_soup):
     url = "https://www.trle.net" + iframe_src
 
     # Check the type of "document"
-    typ = scrape_common.url_postfix(url)
+    typ = scrape_common.url_basename_postfix(url)
     if typ == 'jpg':
         # we should handle all images here but right now
         # we return "" this is a bit more complex