Merge pull request #10 from zSeriesGuy/beta

zSeriesGuy · web-flow · commit 233fae3f730b · 2019-10-19T18:21:41.000-04:00
v1.2.2
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Changelog
 
+## v1.2.2 (2019-10-19)
+* Add settings and process for deleting logs, transcripts, and recordings after a specified number of days.
+
 ## v1.2.1 (2019-10-19)
 * Microsoft Azure Speech-to-Text Cloud service is not available on some platforms. Added support for detecting failure to load service.
 
diff --git a/README.md b/README.md
@@ -25,7 +25,7 @@ SpeakReader requires Python 3.6 or higher.
 
 
 ## Installation
-Installation instructions can be found in the wiki.
+[Installation instructions](https://github.com/zSeriesGuy/SpeakReader/wiki) can be found in the wiki.
  
 
 ## Usage
diff --git a/html/manage.html b/html/manage.html
@@ -331,12 +331,28 @@ <h4>General</h4>
                                                 <label for="log_dir" class="font-weight-bold">Logs Folder</label>
                                                 <input id="log_dir" type="text" class="form-control" name="log_dir" value="${config['log_dir']}" size="30">
                                                 <small class="form-text">Optional: Change the folder where you would like SpeakReader to store the SpeakReader console logs.<br><strong>Restart Required.</strong> A change here will become active with the next restart.</small>
+
+                                                <div class="row mt-3">
+                                                    <label for="log_retention_days" class="font-weight-bold col-md-2 col-form-label">Retention Days</label>
+                                                    <div class="col-md-2">
+                                                        <input id="log_retention_days" type="number" class="form-control" name="log_retention_days" value="${config['log_retention_days']}" min="1">
+                                                    </div>
+                                                </div>
+                                                <small class="form-text">Number of days to keep log files before automatically deleting them. Set to blank to not automatically delete them.</small>
                                             </div>
 
                                             <div class="form-group">
                                                 <label for="transcripts_folder" class="font-weight-bold">Transcripts Folder</label>
                                                 <input id="transcripts_folder" type="text" class="form-control" name="transcripts_folder" value="${config['transcripts_folder']}" size="30">
                                                 <small class="form-text">Optional: Change the folder where you would like SpeakReader to store the transcripts.<br><strong>Restart Required.</strong> A change here will become active with the next restart.</small>
+
+                                                <div class="row mt-3">
+                                                    <label for="transcript_retention_days" class="font-weight-bold col-md-2 col-form-label">Retention Days</label>
+                                                    <div class="col-md-2">
+                                                        <input id="transcript_retention_days" type="number" class="form-control" name="transcript_retention_days" value="${config['transcript_retention_days']}" min="1">
+                                                    </div>
+                                                </div>
+                                                <small class="form-text">Number of days to keep transcript files before automatically deleting them. Set to blank to not automatically delete them.</small>
                                             </div>
 
 
@@ -351,6 +367,14 @@ <h4>General</h4>
                                                     <label for="recordings_folder" class="font-weight-bold">Recordings Folder</label>
                                                     <input id="recordings_folder" type="text" class="form-control" name="recordings_folder" value="${config['recordings_folder']}" size="30">
                                                     <small class="form-text">Optional: Change the folder where you would like SpeakReader to store the audio recordings.<br><strong>Restart Required.</strong> A change here will become active with the next restart.</small>
+
+                                                    <div class="row mt-3">
+                                                        <label for="recording_retention_days" class="font-weight-bold col-md-2 col-form-label">Retention Days</label>
+                                                        <div class="col-md-2">
+                                                            <input id="recording_retention_days" type="number" class="form-control" name="recording_retention_days" value="${config['recording_retention_days']}" min="1">
+                                                        </div>
+                                                    </div>
+                                                    <small class="form-text">Number of days to keep recording files before automatically deleting them. Set to blank to not automatically delete them.</small>
                                                 </div>
                                             </div>
                                         </div>
@@ -808,6 +832,9 @@ <h4 class="modal-title">Confirm</h4>
         $('#git_token').val(config.git_token);
         $('#git_remote').val(config.git_remote);
         $('#git_path').val(config.git_path);
+        $('#log_retention_days').val(config.log_retention_days);
+        $('#transcript_retention_days').val(config.transcript_retention_days);
+        $('#recording_retention_days').val(config.recording_retention_days);
 
         $('#git_branch').val(config.git_branch);
 
diff --git a/requirements.in b/requirements.in
@@ -19,3 +19,4 @@ tzlocal
 passlib
 wheel
 samplerate
+apscheduler
diff --git a/requirements.txt b/requirements.txt
@@ -4,6 +4,7 @@
 #
 #    pip-compile
 #
+apscheduler==3.6.1
 azure-cognitiveservices-speech==1.7.0 ; sys_platform in "win32 macos" or (sys_platform == "linux" and platform_machine in "AMD64")
 cachetools==3.1.1         # via google-auth
 certifi==2019.9.11        # via requests
@@ -39,12 +40,12 @@ pyaudio==0.2.11
 pycparser==2.19           # via cffi
 pyjwt==1.7.1
 python-dateutil==2.8.0
-pytz==2019.3              # via google-api-core, tempora, tzlocal
+pytz==2019.3              # via apscheduler, google-api-core, tempora, tzlocal
 pywin32==225 ; sys_platform == "win32"
 requests==2.22.0          # via google-api-core, ibm-cloud-sdk-core, ibm-watson
 rsa==4.0                  # via google-auth
 samplerate==0.1.0
-six==1.12.0               # via cheroot, configobj, google-api-core, google-api-python-client, google-auth, grpcio, pip-tools, protobuf, python-dateutil, tempora, websocket-client
+six==1.12.0               # via apscheduler, cheroot, configobj, google-api-core, google-api-python-client, google-auth, grpcio, pip-tools, protobuf, python-dateutil, tempora, websocket-client
 tempora==1.14.1           # via portend
 tzlocal==2.0.0
 uritemplate==3.0.0        # via google-api-python-client
@@ -54,4 +55,4 @@ wheel==0.33.6
 zc.lockfile==2.0          # via cherrypy
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools==41.4.0        # via google-api-core, protobuf, zc.lockfile
+# setuptools==41.4.0        # via apscheduler, google-api-core, protobuf, zc.lockfile
diff --git a/speakreader/__init__.py b/speakreader/__init__.py
@@ -22,6 +22,10 @@
 import pyaudio
 import cherrypy
 import json
+import datetime
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.triggers.interval import IntervalTrigger
+
 
 try:
     import webbrowser
@@ -140,6 +144,14 @@ def __init__(self, initOptions):
             if CONFIG.LAUNCH_BROWSER and not initOptions['nolaunch']:
                 launch_browser(CONFIG.HTTP_HOST, self.HTTP_PORT, CONFIG.HTTP_ROOT + 'manage')
 
+            ###################################################################################################
+            #  Run cleanup of old logs, transcripts, and recordings and start a scheduler to run every 24 hours
+            ###################################################################################################
+            self.cleanup_files()
+            self.scheduler = BackgroundScheduler()
+            self.scheduler.add_job(self.cleanup_files, 'interval', hours=24)
+            self.scheduler.start()
+
             SpeakReader._INITIALIZED = True
 
     @property
@@ -213,6 +225,7 @@ def stopTranscribeEngine(self):
     def shutdown(self, restart=False, update=False, checkout=False):
         SpeakReader._INITIALIZED = False
         self.transcribeEngine.shutdown()
+        self.scheduler.shutdown()
         CONFIG.write()
 
         if not restart and not update and not checkout:
@@ -279,6 +292,35 @@ def get_input_device_list(self):
         return deviceList
 
 
+    ###################################################################################################
+    #  Delete any files over the retention days
+    ###################################################################################################
+    def cleanup_files(self):
+        logger.info("Running File Cleanup")
+        def delete(path, days):
+            try:
+                days = int(days)
+            except ValueError:
+                return
+            delete_date = datetime.datetime.now() - datetime.timedelta(days=days)
+            with os.scandir(path=path) as files:
+                for file in files:
+                    file_info = file.stat()
+                    if datetime.datetime.fromtimestamp(file_info.st_ctime) < delete_date:
+                        filename = os.path.join(path, file.name)
+                        logger.debug("Deleting: %s" % filename)
+                        os.remove(filename)
+
+        if CONFIG.LOG_RETENTION_DAYS != "":
+            delete(CONFIG.LOG_DIR, CONFIG.LOG_RETENTION_DAYS)
+
+        if CONFIG.TRANSCRIPT_RETENTION_DAYS != "":
+            delete(CONFIG.TRANSCRIPTS_FOLDER, CONFIG.TRANSCRIPT_RETENTION_DAYS)
+
+        if CONFIG.RECORDING_RETENTION_DAYS != "":
+            delete(CONFIG.RECORDINGS_FOLDER, CONFIG.RECORDING_RETENTION_DAYS)
+
+
 def generate_uuid():
     return uuid.uuid4().hex
 
diff --git a/speakreader/config.py b/speakreader/config.py
@@ -39,13 +39,16 @@ def bool_int(value):
     'START_TRANSCRIBE_ON_STARTUP': (int, 'General', 1),
     'SHOW_INTERIM_RESULTS': (int, 'General', 1),
     'TRANSCRIPTS_FOLDER': (str, 'General', ''),
+    'TRANSCRIPT_RETENTION_DAYS': (str, 'General', '30'),
     'ENABLE_CENSORSHIP': (int, 'General', 1),
     'CENSORED_WORDS': (list, 'General', ''),
     'LOG_DIR': (str, 'General', ''),
+    'LOG_RETENTION_DAYS': (str, 'General', '30'),
     'ANON_REDIRECT': (str, 'General', 'http://www.nullrefer.com/?'),
     'SERVER_ENVIRONMENT': (str, 'Advanced', 'production'),
     'SAVE_RECORDINGS': (int, 'General', 1),
     'RECORDINGS_FOLDER': (str, 'General', ''),
+    'RECORDING_RETENTION_DAYS': (str, 'General', '30'),
     'SPEECH_TO_TEXT_SERVICE': (str, 'General', 'google'),
     'GOOGLE_CREDENTIALS_FILE': (str, 'General', ''),
     'IBM_CREDENTIALS_FILE': (str, 'General', ''),
diff --git a/speakreader/version.py b/speakreader/version.py
@@ -1,3 +1,3 @@
 PRODUCT = 'SpeakReader'
-VERSION_RELEASE = 'v1.2.1'
+VERSION_RELEASE = 'v1.2.2'
 GITHUB_BRANCH = 'master'
diff --git a/speakreader/webserve.py b/speakreader/webserve.py
@@ -107,10 +107,13 @@ def getSettings(self, **kwargs):
         config = {
             "start_transcribe_on_startup": speakreader.CONFIG.START_TRANSCRIBE_ON_STARTUP,
             "launch_browser": speakreader.CONFIG.LAUNCH_BROWSER,
-            "transcripts_folder": speakreader.CONFIG.TRANSCRIPTS_FOLDER,
             "log_dir": speakreader.CONFIG.LOG_DIR,
-            "save_recordings": speakreader.CONFIG.SAVE_RECORDINGS,
+            "transcripts_folder": speakreader.CONFIG.TRANSCRIPTS_FOLDER,
             "recordings_folder": speakreader.CONFIG.RECORDINGS_FOLDER,
+            "log_retention_days": speakreader.CONFIG.LOG_RETENTION_DAYS,
+            "transcript_retention_days": speakreader.CONFIG.TRANSCRIPT_RETENTION_DAYS,
+            "recording_retention_days": speakreader.CONFIG.TRANSCRIPT_RETENTION_DAYS,
+            "save_recordings": speakreader.CONFIG.SAVE_RECORDINGS,
             "http_port": speakreader.CONFIG.HTTP_PORT,
             "enable_https": speakreader.CONFIG.ENABLE_HTTPS,
             "https_cert": speakreader.CONFIG.HTTPS_CERT,
