feat: material changes to metadata presentation

Author: simonsdave

CHANGELOG.md

@@ -24,6 +24,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 * docker based development environment now parses repo's setup.py for pre-reqs
   that need to be install when the development docker image is built - this
   change enabled the removal of requirements.txt from the repo's root directory
+* change format of metadata returned by ```spiders.py``` and ```cloudfeaster.spider.Spider```
 
 ### Removed
 

bin/README.md

@@ -114,11 +114,121 @@ popd > /dev/null
 
 ### [run-all-spiders.sh](run-all-spiders.sh)
 
+* for use by a spider author during spider development
 * calls ```run-spider.sh``` for all spiders in a repo
+* example of command line usage
+
+```bash
+(env) dave@Daves-New-New-Mac-Mini gaming-spiders % run-all-spiders.sh
+/Users/dave/gaming-spiders/gaming_spiders/gamehouseonlinegames.py
+/Users/dave/gaming-spiders/gaming_spiders/gamesonly.py
+/Users/dave/gaming-spiders/gaming_spiders/hiddenobjectgames.py
+/Users/dave/gaming-spiders/gaming_spiders/mahjonggames.py
+/Users/dave/gaming-spiders/gaming_spiders/match3games.py
+/Users/dave/gaming-spiders/gaming_spiders/mindgames.py
+/Users/dave/gaming-spiders/gaming_spiders/miniclip.py
+/Users/dave/gaming-spiders/gaming_spiders/msnonlinegames.py
+/Users/dave/gaming-spiders/gaming_spiders/solitaireonline.py
+/Users/dave/gaming-spiders/gaming_spiders/zygomatic.py
+(env) dave@Daves-New-New-Mac-Mini gaming-spiders %
+```
 
 ### [run-spider.sh](run-spider.sh)
 
-* runs a spider in a spider repo using ```run_spider.sh miniclip```
+* for use by a spider author during spider development
+* runs a spider in a development docker container created
+  from a docker image named by the DEV_ENV_DOCKER_IMAGE
+  environment variable
+* example of command line usage
+
+```bash
+(env) dave@Daves-New-New-Mac-Mini gaming-spiders % pwd
+/Users/dave/gaming-spiders
+(env) dave@Daves-New-New-Mac-Mini gaming-spiders % ls -la gaming_spiders/*.py
+-rw-r--r--  1 dave  staff    22 26 Aug  2020 gaming_spiders/__init__.py
+-rwxr-xr-x  1 dave  staff  1780 26 Jan 20:00 gaming_spiders/gamehouseonlinegames.py
+-rwxr-xr-x  1 dave  staff  1256 26 Jan 19:58 gaming_spiders/gamesonly.py
+-rwxr-xr-x  1 dave  staff   638 26 Aug  2020 gaming_spiders/hiddenobjectgames.py
+-rwxr-xr-x  1 dave  staff   618 26 Aug  2020 gaming_spiders/mahjonggames.py
+-rwxr-xr-x  1 dave  staff   614 26 Aug  2020 gaming_spiders/match3games.py
+-rwxr-xr-x  1 dave  staff   606 26 Aug  2020 gaming_spiders/mindgames.py
+-rwxr-xr-x  1 dave  staff  1363 26 Jan 22:04 gaming_spiders/miniclip.py
+-rwxr-xr-x  1 dave  staff  1210 26 Jan 22:18 gaming_spiders/msnonlinegames.py
+-rwxr-xr-x  1 dave  staff   630 26 Aug  2020 gaming_spiders/solitaireonline.py
+-rw-r--r--  1 dave  staff  1397 26 Jan 20:11 gaming_spiders/zygomatic.py
+(env) dave@Daves-New-New-Mac-Mini gaming-spiders % run-spider.sh miniclip.py | jq .
+{
+  "1": {
+    "title": "1 8 Ball Pool",
+    "link": "https://www.miniclip.com/games/8-ball-pool-multiplayer/en/#t-w-t-H"
+  },
+  "2": {
+    "title": "2 Agar.io",
+    "link": "https://www.miniclip.com/games/agar-io/en/#t-w-t-H"
+  },
+  "3": {
+    "title": "3 Flip Master",
+    "link": "https://www.miniclip.com/games/flip-master/en/#t-w-t-H"
+  },
+  "4": {
+    "title": "4 Krunker.io",
+    "link": "https://www.miniclip.com/games/krunkerio/en/#t-w-t-H"
+  },
+  "5": {
+    "title": "5 Soccer Stars Mobile",
+    "link": "https://www.miniclip.com/games/soccer-stars-mobile/en/#t-w-t-H"
+  },
+  "6": {
+    "title": "6 Short Ride",
+    "link": "https://www.miniclip.com/games/short-ride/en/#t-w-t-H"
+  },
+  "7": {
+    "title": "7 Quick Fire Pool Instant",
+    "link": "https://www.miniclip.com/games/quickfire-pool-instant/en/#t-w-t-H"
+  },
+  "8": {
+    "title": "8 Bubble Trouble",
+    "link": "https://www.miniclip.com/games/bubble-trouble/en/#t-w-t-H"
+  },
+  "9": {
+    "title": "9 Tanki Online",
+    "link": "https://www.miniclip.com/games/tanki-online/en/#t-w-t-H"
+  },
+  "10": {
+    "title": "10 Head Ball 2",
+    "link": "https://www.miniclip.com/games/head-ball-2/en/#t-w-t-H"
+  },
+  "_metadata": {
+    "status": {
+      "code": 0,
+      "message": "Ok"
+    },
+    "spider": {
+      "name": "miniclip.py",
+      "version": "sha256:48c81ae8b86cbf71035905ebc0c4ead321b823fafe093dbf5624912808f4b954"
+    },
+    "crawlArgs": [],
+    "crawlTime": {
+      "started": "2021-03-02T16:32:56.198453+00:00",
+      "durationInMs": 4157
+    }
+  },
+  "_debug": {
+    "screenshot": "/var/folders/zc/51nmqy_93559vqw_1y526y240000gn/T/tmp.0llGnGgS/screenshot.png",
+    "crawlLog": "/var/folders/zc/51nmqy_93559vqw_1y526y240000gn/T/tmp.0llGnGgS/crawl-log.txt",
+    "chromeDriverLog": "/var/folders/zc/51nmqy_93559vqw_1y526y240000gn/T/tmp.0llGnGgS/chromedriver-log.txt"
+  }
+}
+(env) dave@Daves-New-New-Mac-Mini gaming-spiders % ls -la /var/folders/zc/51nmqy_93559vqw_1y526y240000gn/T/tmp.0llGnGgS
+total 3112
+drwx------    6 dave  staff      192  2 Mar 11:33 .
+drwx------@ 104 dave  staff     3328  2 Mar 11:32 ..
+-rw-------    1 dave  staff   170642  2 Mar 11:33 chromedriver-log.txt
+-rw-------    1 dave  staff        0  2 Mar 11:32 crawl-log.txt
+-rw-r--r--    1 dave  staff     1618  2 Mar 11:33 crawl-output.json
+-rw-------    1 dave  staff  1414454  2 Mar 11:33 screenshot.png
+(env) dave@Daves-New-New-Mac-Mini gaming-spiders %
+```
 
 ## Utilities
 

bin/int-test-run-all-spiders-in-ci-pipeline.py

@@ -61,13 +61,15 @@ def spiders(self):
             'spiders.py',
         ]
 
-        output = json.loads(subprocess.check_output(args).decode('UTF-8').strip())
+        spider_metadata_by_spider_by_category = json.loads(subprocess.check_output(args).decode('UTF-8').strip())
 
-        rv = list(output.keys())
-        rv.remove('_metadata')
-        rv.sort()
+        filenames = set()
 
-        return rv
+        for spider_metadata_by_spider in spider_metadata_by_spider_by_category.values():
+            for spider_metadata in spider_metadata_by_spider.values():
+                filenames.add(spider_metadata['absoluteFilename'])
+
+        return list(filenames)
 
 
 class CrawlContainer(object):

bin/spiders.py

@@ -2,7 +2,6 @@
 # -*- coding: utf-8 -*-
 
 import hashlib
-import importlib
 import inspect
 import json
 import logging
@@ -107,28 +106,23 @@ def parse_args(self, *args, **kwargs):
     # now discover some spiders:-)
     #
     sd = SpiderDiscovery(clo.samples)
-    metadata = sd.discover()
-    updated_metadata = {}
-    for spider_name in metadata.keys():
-        spider_module_name = '.'.join(spider_name.split('.')[:-1])
-        spider_module = importlib.import_module(spider_module_name)
-        updated_metadata[os.path.basename(spider_module.__file__)] = metadata[spider_name]
+    output = sd.discover()
 
     #
     # add _metadata
     #
     module = sys.modules['__main__']
     source = inspect.getsource(module)
-    hash = hashlib.sha256(source.encode('UTF-8'))
+    version_hash = hashlib.sha256(source.encode('UTF-8'))
 
-    updated_metadata['_metadata'] = {
+    output['_metadata'] = {
         'name': os.path.basename(__file__),
-        'version': '%s:%s' % (hash.name, hash.hexdigest()),
+        'version': '{name}:{hash_digest}'.format(name=version_hash.name, hash_digest=version_hash.hexdigest()),
     }
 
     #
     # finally generate some stdout
     #
-    print(json.dumps(updated_metadata))
+    print(json.dumps(output))
 
     sys.exit(0)

cloudfeaster/samples/xe_exchange_rates.py

@@ -16,6 +16,10 @@ class XEExchangeRatesSpider(spider.Spider):
     def get_metadata(cls):
         return {
             'url': 'https://www.xe.com/?cn=cad',
+            'categories': [
+                cls.get_default_category(),
+                'fx_rates',
+            ],
         }
 
     def crawl(self, browser):

cloudfeaster/spider.py

@@ -129,13 +129,9 @@ def get_validated_metadata(cls):
             # A spider can appear in more than one category.
             # A spider's categories are declared as part of the spider's metadata.
             # If no categories are declared in a spider's metadata then the
-            # spider's package name (per the above) is used as the spider's
-            # category. One caveat, by convention, package names often end
-            # with _spiders. When a category name is generated from a spider's
-            # package name, the trailing _spiders is removed from the package name.
+            # spider's default category is used.
             #
-            category = cls._replace_spiders_postfix_reg_ex.sub('', cls.__module__.split('.')[0])
-            metadata['categories'] = [category]
+            metadata['categories'] = [cls.get_default_category()]
 
         metadata['absoluteFilename'] = sys.modules[cls.__module__].__file__
         metadata['fullyQualifiedClassName'] = '{module}.{cls}'.format(
@@ -219,6 +215,26 @@ def get_validated_metadata(cls):
 
         return metadata
 
+    @classmethod
+    def get_default_category(cls):
+        """A spider's fully qualified name will be something like gaming_spiders.miniclip.Spider
+        A spider's default category is everything up to but not including the first period a
+        spider's fully qualified name.
+
+        get_default_category() simplifies specificiation of metadata when spiders are in
+        mulitple categories. See sample code below.
+
+            def get_metadata(cls):
+               return {
+                   'url': 'https://www.xe.com/?cn=cad',
+                   'categories': [
+                      cls.get_default_category(),
+                      'fx_rates',
+                   ],
+               }
+        """
+        return cls._replace_spiders_postfix_reg_ex.sub('', cls.__module__.split('.')[0])
+
     @classmethod
     def get_metadata(cls):
         """Spider classes should override this method to return
@@ -1020,7 +1036,26 @@ def filter(filename): return(filename.endswith('.py') and not filename.startswit
         # the concrete subclasses of ```cloudfeaster.spider.Spider```
         # which will be the spiders we're interested in
         #
-        return self._find_concrete_spider_classes(Spider)
+        concrete_spider_classes = self._find_concrete_spider_classes(Spider)
+
+        #
+        # now for some fancy formatting of the results
+        #
+        rv = {}
+        for concrete_spider_class in concrete_spider_classes:
+            metadata = concrete_spider_class.get_validated_metadata()
+            fully_qualified_class = metadata['fullyQualifiedClassName']
+            spider = fully_qualified_class.split('.')[-2]
+
+            for category in metadata['categories']:
+                if category not in rv:
+                    rv[category] = {}
+                rv[category][spider] = metadata
+
+        #
+        # all done!
+        #
+        return rv
 
     def _find_concrete_spider_classes(self, base_class):
         base_msg = "looking for concrete spider classes of base class '%s.%s'" % (
@@ -1029,22 +1064,24 @@ def _find_concrete_spider_classes(self, base_class):
         )
         _logger.info(base_msg)
 
-        rv = {}
+        concrete_spider_classes = []
         for sub_class in base_class.__subclasses__():
-            full_sub_class_name = '%s.%s' % (sub_class.__module__, sub_class.__name__)
+            fully_qualified_class_name = '%s.%s' % (sub_class.__module__, sub_class.__name__)
 
-            _logger.info("%s - assessing '%s'", base_msg, full_sub_class_name)
+            _logger.info("%s - assessing '%s'", base_msg, fully_qualified_class_name)
 
             if not sub_class.__subclasses__():
-                _logger.info("%s - identified concrete class '%s'", base_msg, full_sub_class_name)
+                msg_fmt = "{base_msg} - identified concrete class '{class_name}'"
+                msg = msg_fmt.format(base_msg=base_msg, class_name=fully_qualified_class_name)
+                _logger.info(msg)
 
-                rv[full_sub_class_name] = sub_class.get_validated_metadata()
+                concrete_spider_classes.append(sub_class)
             else:
-                _logger.info("%s - identified abstract class '%s'", base_msg, full_sub_class_name)
+                _logger.info("%s - identified abstract class '%s'", base_msg, fully_qualified_class_name)
 
-                rv.update(self._find_concrete_spider_classes(sub_class))
+                concrete_spider_classes.extend(self._find_concrete_spider_classes(sub_class))
 
-        return rv
+        return concrete_spider_classes
 
     @classmethod
     def load_and_discover_all_spiders_in_package(cls, spider_package_name):

cloudfeaster/tests/spider_tests.py

@@ -2,6 +2,7 @@
 
 import hashlib
 import http.server
+import importlib
 import inspect
 import re
 import sys
@@ -1536,27 +1537,54 @@ def test_get_selected_and_select_by_visible_text_on_non_select_element(self):
 
 
 class TestSpiderDiscovery(unittest.TestCase):
-    """A series of unit tests that validate ```spider.SpiderDiscovery```."""
+    """A series of unit tests that validate ```spider.SpiderDiscovery```.
+
+    :ADDRESS: not really super happy with this test. Feels like a lot of hard coding
+    and assumptions. Maybe not but just feels like that.
+    """
 
     def test_spiders_no_samples(self):
-        spider.SpiderDiscovery.load_and_discover_all_spiders_in_package('cloudfeaster.samples')
-        spider.SpiderDiscovery.load_and_discover_all_spiders_in_package('cloudfeaster.tests.some_test_spiders')
+        importlib.import_module('cloudfeaster.samples')
+        importlib.import_module('cloudfeaster.samples.pypi')
+        importlib.import_module('cloudfeaster.samples.pythonwheels')
+        importlib.import_module('cloudfeaster.samples.xe_exchange_rates')
+
+        importlib.import_module('cloudfeaster.tests.some_test_spiders')
+        importlib.import_module('cloudfeaster.tests.some_test_spiders.abstract')
+        importlib.import_module('cloudfeaster.tests.some_test_spiders.supersimpleconcrete')
 
         sd = spider.SpiderDiscovery()
-        spiders_by_spider_name = sd.discover()
+        spiders_by_category = sd.discover()
+
+        #
+        # only expecting a certain set of categories
+        #
+        categories = list(spiders_by_category.keys())
+        categories.sort()
+        expected_categories = [
+            'cloudfeaster',
+            'fx_rates',
+        ]
+        expected_categories.sort()
+        self.assertEqual(categories, expected_categories)
+
+        spiders_by_spider_name = spiders_by_category['cloudfeaster']
 
         #
-        # confirm available sample spiders
+        # only expecting a certain set of spiders
         #
-        def is_sample_spider_name(k):
-            return k.startswith('cloudfeaster.samples')
-        spider_names = [k for k in spiders_by_spider_name.keys() if is_sample_spider_name(k)]
+        fqcn_key = 'fullyQualifiedClassName'
+
+        def is_sample(metadata):
+            return metadata[fqcn_key].startswith('cloudfeaster.samples')
+
+        spider_names = [metadata[fqcn_key] for metadata in spiders_by_spider_name.values() if is_sample(metadata)]
         spider_names.sort()
 
         expected_spider_names = [
-            'cloudfeaster.samples.xe_exchange_rates.XEExchangeRatesSpider',
             'cloudfeaster.samples.pypi.PyPISpider',
             'cloudfeaster.samples.pythonwheels.PythonWheelsSpider',
+            'cloudfeaster.samples.xe_exchange_rates.XEExchangeRatesSpider',
         ]
         expected_spider_names.sort()
 
@@ -1565,9 +1593,12 @@ def is_sample_spider_name(k):
         #
         # confirm available test spiders which explore concrete and abstract spider classes
         #
-        def is_test_spider_name(k):
-            return k.startswith('cloudfeaster.tests.some_test_spiders')
-        spider_names = [k for k in spiders_by_spider_name.keys() if is_test_spider_name(k)]
+        fqcn_key = 'fullyQualifiedClassName'
+
+        def is_test(metadata):
+            return metadata[fqcn_key].startswith('cloudfeaster.tests.some_test_spiders')
+
+        spider_names = [metadata[fqcn_key] for metadata in spiders_by_spider_name.values() if is_test(metadata)]
         spider_names.sort()
 
         expected_spider_names = [