(WIP) Implements session_oriented.session

* Adds 'list_of' meta-datasource
* Adds documentation

Author: halfak

doc/revscoring.datasources.session_oriented.rst

@@ -0,0 +1,4 @@
+revscoring.datasources.session_oriented
+=======================================
+
+.. automodule:: revscoring.datasources.session_oriented

revscoring/datasources/__init__.py

@@ -8,7 +8,8 @@
 :class:`~revscoring.Datasource` processors are tended to
 be :func:`~revscoring.dependencies.solve`'d as dependencies. The
 provided datasources are split conceptually into a set of modules.  Currently,
-there is one module: :mod:`~revscoring.datasources.revision_oriented`.
+there are two modules: :mod:`~revscoring.datasources.revision_oriented` and
+:mod:`~revscoring.datasources.session_oriented`.
 
 Meta-datasources
 ++++++++++++++++

revscoring/datasources/meta/expanders.py

@@ -0,0 +1,13 @@
+from ..datasource import Datasource
+
+
+class list_of(Datasource):
+
+    def __init__(self, dependent, depends_on=None, name=None):
+        name = self._format_name(name, [dependent])
+        super().__init__(
+            name, self.process, depends_on=depends_on)
+        self.dependency = dependent
+
+    def process(self, *lists_of_values):
+        return [self.dependency(*values) for values in zip(*lists_of_values)]

revscoring/datasources/meta/selectors.py

@@ -116,8 +116,7 @@ class filter_keys(Datasource):
 
     :Parameters:
         table_datasource : :class:`revscoring.Datasource`
-            A datasource that generates a table including only the specified
-            keys
+            A datasource that generates a table with keys and values
         keys : `iterable` ( `hashable` )
             The keys to select from the table
         name : `str`
@@ -138,3 +137,43 @@ def process(self, table):
                 new_table[key] = table[key]
 
         return new_table
+
+
+class first(Datasource):
+    """
+    Returns the first item in an indexable collection (e.g., a list)
+
+    :Parameters:
+        items_datasource : :class:`revscoring.Datasource`
+            A datasource that generates an indexable sequence
+        name : `str`
+            A name for the datasource
+    """
+
+    def __init__(self, items_datasource, name=None):
+        name = self._format_name(
+            name, [items_datasource])
+        super().__init__(name, self.process, depend_on=[items_datasource])
+
+    def process(self, items):
+        return items[0]
+
+
+class last(Datasource):
+    """
+    Returns the last item in an indexable collection (e.g., a list)
+
+    :Parameters:
+        items_datasource : :class:`revscoring.Datasource`
+            A datasource that generates an indexable sequence
+        name : `str`
+            A name for the datasource
+    """
+
+    def __init__(self, items_datasource, name=None):
+        name = self._format_name(
+            name, [items_datasource])
+        super().__init__(name, self.process, depend_on=[items_datasource])
+
+    def process(self, items):
+        return items[-1]

revscoring/datasources/session_oriented.py

@@ -0,0 +1,60 @@
+"""
+Implements a set of datasources oriented off of a single revision.  This is
+useful for extracting features of edit and article quality.
+
+.. autodata:: revscoring.datasources.session_oriented.session
+
+"""
+
+from ..dependencies import DependentSet
+from .meta import expanders
+from .revision_oriented import Revision, User
+
+
+def list_of_tree(dependent_set, rewrite_name=None, cache=None):
+    cache = cache if cache is not None else {}
+    rewrite_name = rewrite_name if rewrite_name is not None else \
+        lambda name: name
+
+    # Rewrites all dependents.
+    for attr, dependent in dependent_set.dependents.items():
+        new_dependent = list_of_ify(dependent, rewrite_name, cache)
+        setattr(dependent_set, attr, new_dependent)
+
+    # Iterate into all sub-DependentSets
+    for attr, sub_dependent_set in dependent_set.dependent_sets.items():
+        new_dependent_set = list_of_tree(
+            sub_dependent_set, rewrite_name, cache)
+        setattr(dependent_set, attr, new_dependent_set)
+
+    return dependent_set
+
+
+def list_of_ify(dependent, rewrite_name, cache):
+    new_name = rewrite_name(dependent.name)
+    if new_name in cache:
+        return cache[new_name]
+    else:
+        new_dependencies = [list_of_ify(dependency, rewrite_name, cache)
+                            for dependency in dependent.dependencies]
+        return expanders.list_of(
+            dependent, depends_on=new_dependencies, name=new_name)
+
+
+class Session(DependentSet):
+    def __init__(self, name):
+        super().__init__(name)
+        self.revisions = list_of_tree(Revision(
+            name + ".revisions",
+            include_page_creation=True,
+            include_content=True,
+            include_user=False,
+            include_page_suggested=True))
+        self.user = list_of_tree(User(
+            name + ".user",
+            include_info=True,
+            include_last_revision=False
+        ))
+
+
+session = Session("session")

revscoring/dependencies/dependent.py

@@ -99,55 +99,60 @@ class DependentSet:
             A base name for the items in the set
     """
 
-    def __init__(self, name, _dependents=None, _dependent_sets=None):
-        self._dependents = _dependents or set()
-        self._dependent_sets = _dependent_sets or set()
-        self._name = name
+    def __init__(self, name, dependents=None, dependent_sets=None):
+        self.dependents = dependents or {}
+        self.dependent_sets = dependent_sets or {}
+        self.name = name
 
     def __setattr__(self, attr, value):
         super().__setattr__(attr, value)
 
         if isinstance(value, Dependent):
             logger.log(logging.NOTSET,
-                       "Registering {0} to {1}".format(value, self._name))
-            if value in self._dependents:
+                       "Registering {0} to {1}".format(value, self.name))
+            if value in self.dependents:
                 logger.warn("{0} has already been added to {1}.  Could be "
                             .format(value, self) + "overwritten?")
-            self._dependents.add(value)
+            self.dependents[attr] = value
         elif isinstance(value, DependentSet):
-            self._dependent_sets.add(value)
+            self.dependent_sets[attr] = value
+        else:
+            pass  # Just set it like a regular attribute
 
     # String methods
     def __str__(self):
         return self.__repr__()
 
     def __repr__(self):
-        return "{" + self._name + "}"
+        return "{" + self.name + "}"
 
     def __hash__(self):
-        return hash('dependent_set.' + self._name)
+        return hash('dependent_set.' + self.name)
 
     def __eq__(self, other):
         return hash(self) == hash(other)
 
     def __ne__(self, other):
         return not self == other
 
+    def all_dependencies(self):
+        return set(self.dependents.values()).union(*self.dependent_sets.values())
+
     # Set methods
     def __len__(self):
-        return len(self._dependents.union(*self._dependent_sets))
+        return len(self.all_dependencies())
 
     def __contains__(self, item):
-        return item in self._dependents.union(*self._dependent_sets)
+        return item in self.all_dependencies()
 
     def __iter__(self):
-        return iter(self._dependents.union(*self._dependent_sets))
+        return iter(self.all_dependencies())
 
     def __sub__(self, other):
-        return self._dependents.union(*self._dependent_sets) - other
+        return self.all_dependencies() - other
 
     def __and__(self, other):
-        return self._dependents.union(*self._dependent_sets) & other
+        return self.all_dependencies() & other
 
     def __or__(self, other):
-        return self._dependents.union(*self._dependent_sets) | other
+        return self.all_dependencies() | other

revscoring/extractors/api/extractor.py

@@ -3,11 +3,11 @@
 
 import mwapi
 
+from . import datasources
+from .. import Extractor as BaseExtractor
 from ...datasources import Datasource, revision_oriented
 from ...dependencies import expand
 from ...errors import QueryNotSupported, RevisionNotFound, UserNotFound
-from .. import Extractor as BaseExtractor
-from . import datasources
 from .revision_oriented import Revision
 from .util import REV_PROPS, USER_PROPS
 
@@ -45,7 +45,7 @@ def get_property_suggestion_search_doc(self, page):
         return datasources.PropertySuggestionDoc(page, self)
 
     def extract(self, rev_ids, dependents, context=None, caches=None,
-                cache=None, profile=None):
+                cache=None, profile=None, orientation="revision"):
         """
         Extracts a values for a set of
         :class:`~revscoring.dependents.dependent.Dependent` (e.g.

revscoring/features/meta/aggregators.py

@@ -17,6 +17,8 @@
 from ..feature import Feature
 from ..feature_vector import FeatureVector
 
+any_builtin = any
+all_builtin = all
 len_builtin = len
 sum_builtin = sum
 max_builtin = max
@@ -70,6 +72,16 @@ def wrapper(items_datasource, name=None, returns=float, vector=False):
     return wrapper
 
 
+@aggregators_factory
+def all(items_datasource, name=None, returns=bool, vector=False):
+    return all_builtin
+
+
+@aggregators_factory
+def any(items_datasource, name=None, returns=bool, vector=False):
+    return any_builtin
+
+
 @aggregators_factory
 def sum(items_datasource, name=None, returns=float, vector=False):
     return sum_builtin

revscoring/features/meta/selectors.py

@@ -0,0 +1,19 @@
+from ..aggregators import aggregators_factory
+
+
+def _first(items):
+    return items[0]
+
+
+def _last(items):
+    return items[-1]
+
+
+@aggregators_factory
+def first(items_datasource, name=None, returns=float, vector=False):
+    return _first
+
+
+@aggregators_factory
+def last(items_datasource, name=None, returns=float, vector=False):
+    return _last

tests/datasources/meta/tests/test_expanders.py

@@ -0,0 +1,21 @@
+import pickle
+
+from revscoring import Datasource, Feature
+from revscoring.datasources.meta import expanders
+from revscoring.dependencies import solve
+
+
+def process_chars(text):
+    return len(text)
+
+text = Datasource("text")
+chars = Feature("chars", process_chars, returns=int, depends_on=[text])
+many_texts = expanders.list_of(text)
+many_chars = expanders.list_of(chars, depends_on=[many_texts])
+
+
+def test_list_of():
+    assert solve(many_chars, cache={many_texts: ["foo", "barbaz"]}) == \
+           [3, 6]
+
+    assert pickle.loads(pickle.dumps(many_chars)) == many_chars