Merge branch 'dev'

Author: MaxPensel

src/core/QtExtensions.py

@@ -283,7 +283,11 @@ def save_content(self):
 
             with open(self.toml_file, "w") as tf:
                 tf.write(content)
-            SimpleMessageBox("File Saved", "Successfully saved configuration.").exec()
+            SimpleMessageBox("File Saved",
+                             "Successfully saved configuration. "
+                             "Some changes only apply after restarting OpenWebScraper.").exec()
+
+            self.close()
         except toml.TomlDecodeError as exc:
             core.MASTER_LOGGER.exception(f"{type(exc).__name__}: {exc}")
             SimpleErrorInfo("Error", "Content not in correct toml formal.", details=str(exc)).exec()

src/modules/crawler/__init__.py

@@ -31,7 +31,12 @@
 
 MOD_PATH = os.path.join(APP_SETTINGS["modloader"]["mod_dir"], "crawler")
 
-SETTINGS = toml.load(os.path.join(MOD_PATH, "settings.toml"))
+
+def load_settings():
+    return toml.load(os.path.join(MOD_PATH, "settings.toml"))
+
+
+SETTINGS = load_settings()
 
 
 def init(main_window):

src/modules/crawler/controller.py

@@ -161,6 +161,9 @@ def setup_behaviour(self):
                                          self._view.parser_select)
         )
 
+        self._view.initializer_select.currentIndexChanged.connect(
+            lambda: self.set_initializer_info("")
+        )
         self._view.initializer_select.currentIndexChanged.connect(
             lambda: self.switch_sub_view(SETTINGS["ui"]["initializer"]["widgets"],
                                          self._view.initializer_container.layout(),
@@ -228,6 +231,10 @@ def register_parser_view(self, parser_view_path):
 
             self.update_view()
 
+    def set_initializer_info(self, text, color="black"):
+        self._view.initializer_info.setText(text)
+        self._view.initializer_info.setStyleSheet('QLabel { color: %s }' % color)
+
     def switch_initializer_view(self):
         key = self._view.initializer_select.currentText()
         self.register_initializer_view(SETTINGS["ui"]["initializer"]["widgets"][key])

src/modules/crawler/settings.toml

@@ -1,9 +1,9 @@
 [general]
 title       = "Crawler"
 main_widget = "modules.crawler.view.CrawlerWidget"
-version     = "0.6.0 <beta>"
+version     = "0.6.1 <beta>"
 copyright   = "2019 Maximilian Pensel"
-scrapy_wrapper_exec = '/mnt/Data/Projects/Crawler/OWS-scrapy-wrapper/venv-ubuntu/bin/python /mnt/Data/Projects/Crawler/OWS-scrapy-wrapper/src/scrapy_wrapper.py'
+scrapy_wrapper_exec = 'SPECIFY EXECUTION COMMAND FOR OWS-SCRAPY-WRAPPER'
 
 [ui]
 

src/modules/crawler/ui/initializers/local.py

@@ -20,9 +20,11 @@
 You should have received a copy of the GNU General Public License
 along with OpenWebScraper.  If not, see <https://www.gnu.org/licenses/>.
 """
+import json
 import os
 import subprocess
 import sys
+import threading
 
 from PyQt5.QtWidgets import QComboBox, QPushButton, QVBoxLayout, QGroupBox, QLineEdit, QHBoxLayout, QMessageBox
 
@@ -114,6 +116,14 @@ def setup_behaviour(self):
         else:
             self._view.crawl_name_input.textChanged.connect(self.update_model)
 
+        # test connection to scrapy wrapper and produce feedback
+        def set_info(info):
+            if "version" in info:
+                self.master_cnt.set_initializer_info(f"Connected to OWS-scrapy-wrapper version {info['version']}")
+            else:
+                self.master_cnt.set_initializer_info(info['message'], color="red")
+        thread = threading.Thread(target=check_scrapy_connection, args=[set_info])
+        thread.start()
 
     def update_model(self):
         if self.master_cnt:
@@ -269,3 +279,21 @@ def start_scrapy(settings_path):
                              close_fds=True)
     except Exception as exc:
         LOG.exception("{0}: {1}".format(type(exc).__name__, exc))
+
+
+def check_scrapy_connection(callback):
+    scrapy_script = SETTINGS["general"]["scrapy_wrapper_exec"]
+    command = scrapy_script + " INFO"
+    LOG.info("Running {0}".format(command))
+    try:
+        proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=None, shell=False)
+
+        info_json, _ = proc.communicate()
+
+        info = json.loads(info_json.splitlines()[-1])
+        LOG.info("Connected to OWS-scrapy-wrapper version {0}".format(info["version"]))
+    except Exception as exc:
+        LOG.exception("{0}: {1}".format(type(exc).__name__, exc))
+        info = dict(message="Could not connect to OWS-scrapy-wrapper. Check your config.")
+
+    callback(info)

src/modules/crawler/view.py

@@ -67,10 +67,14 @@ def __init__(self):
         self.initializer_select = QComboBox()
         self.initializer_select.setSizeAdjustPolicy(QComboBox.AdjustToContents)
 
+        self.initializer_info = QLabel()
+
         initializer_select_layout = QHBoxLayout()
         initializer_select_layout.addWidget(initializer_label)
         initializer_select_layout.addWidget(self.initializer_select)
         initializer_select_layout.addSpacerItem(QSpacerItem(100, 1, QSizePolicy.Expanding, QSizePolicy.Fixed))
+        initializer_select_layout.addWidget(self.initializer_info)
+        initializer_select_layout.addSpacerItem(QSpacerItem(100, 1, QSizePolicy.Expanding, QSizePolicy.Fixed))
 
         self.initializer_container = VerticalContainer()
         self.initializer_container.addLayout(initializer_select_layout)

src/settings.toml

@@ -1,6 +1,6 @@
 [general]
 modules    = ["crawler", "template"]
-version    = "0.6.0 <beta>"
+version    = "0.6.1 <beta>"
 copyright  = "2019 Maximilian Pensel"
 master_log = "master.log"
 

test.json

@@ -0,0 +1,15 @@
+{
+    "blacklist": [],
+    "finalizers": {},
+    "logs": "/mnt/Data/Projects/Crawler/SpiderGUI/src/default_workspace/logs/",
+    "name": "test",
+    "output": "/mnt/Data/Projects/Crawler/SpiderGUI/src/default_workspace/data/raw",
+    "parser": "parsers.RawParser",
+    "parser_data": {
+        "allowed_content_type": [
+            "text/html"
+        ]
+    },
+    "pipelines": {},
+    "urls": []
+}