Merge branch 'main' of https://github.com/microprediction/precise into main

Author: Peter Cotton

COMMENTARY.md

@@ -0,0 +1,16 @@
+
+By topic
+
+On rebalancing frequency:
+
+ - [What does rebalancing really achieve?](https://www.semanticscholar.org/paper/What-Does-Rebalancing-Really-Achieve-Cuthbertson-Hayley/ccceb3d3dd5ba2c88394b72559f5485998c7e0cf) <-- Good lit survey
+ - [Measuring volatility pumping benefits in equity markets](https://risk.edhec.edu/sites/risk/files/measuring-volatility-pumping-benefits-in-equity-markets.pdf)
+ - [Chasing down the rebalancing premium](https://www.soa.org/globalassets/assets/library/newsletters/risks-and-rewards/2020/september/rr-2020-09-glacy.pdf)
+ - [Is Portfolio rebalancing good for investors?](https://core.ac.uk/download/pdf/84873313.pdf)
+ - [Will my risk parity portfolio outperform](https://papers.ssrn.com/sol3/papers.cfm?abstract_id=2101898)
+ - [Why regular rebalancing is key to maximizing factor premiums](https://advisors.vanguard.com/insights/article/whyregularrebalancingiskeytomaximizingfactorpremiums#:~:text=By%20breaking%20the%20returns%20down,monthly%20or%20bi%2Dannual%20rebalancing.)
+ - [The rebalancing premium](http://www.efficientfrontier.com/ef/996/rebal.htm)
+ - [Daily versus monthly...](https://www.bogleheads.org/forum/viewtopic.php?t=63189)
+ - [Estimating rebalancing premium in cryptocurrencies](https://quantpedia.com/estimating-rebalancing-premium-in-cryptocurrencies/)
+ - [An in-depth look at portfolio rebalancing strategies](https://cpb-us-w2.wpmucdn.com/sites.udel.edu/dist/a/855/files/2020/08/Rebalancing-Strategies.pdf)
+ - [Portfolio rebalancing: Hype or Hope?](https://journals.uvu.edu/index.php/jbi/article/download/90/70/)

LISTING_OF_MANAGERS.md

@@ -61,6 +61,9 @@
 | schur_vol_vol_ewa_r050_n25_s5_g100_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/schurmanagers.py |
 | schur_diag_diag_ewa_r050_n25_s5_g100_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/schurmanagers.py |
 | schur_diag_weak_pm_t0_r050_n25_s5_g100_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/schurmanagers.py |
+| schur_vol_vol_pm_t0_d0_r025_n50_s5_g100_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/schurmanagers.py |
+| schur_vol_vol_pm_t0_d0_r025_n50_s25_g100_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/schurmanagers.py |
+| schur_vol_vol_pm_t0_d0_r025_n50_s50_g100_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/schurmanagers.py |
 | schur_weak_weak_pm_t0_r025_n50_s5_g050_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/schurmanagers.py |
 | schur_weak_weak_pm_t0_r050_n25_s5_g050_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/schurmanagers.py |
 | schur_weak_weak_ewa_r025_n50_s5_g050_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/schurmanagers.py |
@@ -85,9 +88,8 @@
 | schur_vol_vol_ewa_r050_n25_s5_g000_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/schurmanagers.py |
 | schur_diag_diag_ewa_r050_n25_s5_g000_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/schurmanagers.py |
 | schur_diag_weak_pm_t0_r050_n25_s5_g000_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/schurmanagers.py |
-| rfl_hrp_var_long_manager_n200 | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/rflmanagers.py |
-| rfl_hrp_vol_long_manager_n200 | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/rflmanagers.py |
 | equal_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/equalmanagers.py |
+| equal_check_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/equalmanagers.py |
 | ldp_s5_n50_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/ldpmanagers.py |
 | ldp_s25_n50_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/ldpmanagers.py |
 | ldp_s5_n100_long_manager | https://github.com/microprediction/precise/blob/main/precise/skaters/managers/ldpmanagers.py |

README.md

@@ -1,6 +1,6 @@
 # precise ![tests](https://github.com/microprediction/precise/workflows/tests/badge.svg) ![tests-scipy-173](https://github.com/microprediction/precise/workflows/tests-scipy-173/badge.svg)![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)
 
-Online covariance and precision forecasting, portfolios, and model ensembles with ongoing benchmarking. 
+Online covariance and precision forecasting (also portfolios, and model ensembles) with ongoing benchmarking. 
 
 ## Covariance TLDR: "Functions that forecast covariance in online fashion"
 Here y is a vector:

examples_exploratory_weak/weak_entropish.py

@@ -8,8 +8,8 @@
 from precise.skaters.locationutil.vectorfunctions import scatter
 
 if __name__=='__main__':
-    k = 1  # Business days
-    n_dim = 100
+    k = 1  #
+    n_dim = 50
     n_burn = 100
     rele = list()
     s_parity = {}

examples_of_comparing/manager_speed.py

@@ -0,0 +1,7 @@
+from pprint import pprint
+from precise.skaters.managers.schurmanagers import schur_weak_weak_pm_t0_r025_n50_s25_g100_h125_long_manager as mgr
+from precise.skaters.managerutil.managertesting import manager_profile
+
+if __name__=='__main__':
+    cpu = manager_profile(mgr=mgr)
+    pprint(cpu)

precise/skaters/covariance/identity.py

@@ -0,0 +1,9 @@
+
+import numpy as np
+
+
+def identity_scov(s, y, k=1, e=1):
+    """ Useful when you don't want to waste compute """
+    # See equal_managers for usage example
+    n_dim = len(y)
+    return np.zeros(n_dim), np.eye(n_dim), s

precise/skaters/covarianceutil/covfunctions.py

@@ -2,18 +2,34 @@
 import math
 import pandas as pd
 from precise.skaters.covarianceutil.pdutil import square_to_square_dataframe, square_to_column_series, square_to_index_series
+from scipy.cluster.hierarchy import linkage, leaves_list
+from scipy.spatial.distance import squareform
 
 # Functions acting on cov, corrcoef matrices and other square matrices
 # If square pd.DataFrame are supplied instead, index and columns are preserved
 
 
+def seriation(cov, d=None):
+    if d is None:
+        d = cov_distance(cov=cov)
+    d_square = squareform(d)
+    clusters = linkage(d_square, method='single',optimal_ordering=True)
+    return leaves_list(clusters)
+
+
 def cov_to_corrcoef(a):
     if isinstance(a, pd.DataFrame):
         return square_to_square_dataframe(a, cov_to_corrcoef)
     else:
         variances = np.diagonal(a)
         denominator = np.sqrt(variances[np.newaxis, :] * variances[:, np.newaxis])
-        return a / denominator
+        with np.errstate(divide='raise'):
+            try:
+                corr = a / denominator
+                return corr
+            except (FloatingPointError, ZeroDivisionError):
+                sub_cov = np.diag(variances) + 1e-6
+                return cov_to_corrcoef(sub_cov)
 
 
 def normalize(x):
@@ -220,7 +236,20 @@ def corr_distance(corr, expon=0.5):
     if isinstance(corr, pd.DataFrame):
         return square_to_square_dataframe(corr, corr_distance, expon=expon)
     else:
-        return ((1 - np.array(corr)) / 2.) ** expon
+        if np.shape(corr)[0]==1:
+            return np.zeros_like(corr)
+        else:
+            with np.errstate(divide='raise'):
+                try:
+                    return ((1 - np.array(corr)) / 2.) ** expon
+                except (RuntimeWarning, RuntimeError):
+                    try:
+                        return ((1 - 0.5*np.array(corr)) / 2.) ** expon
+                    except (RuntimeWarning, RuntimeError):
+                        print('Failed to compute corr distance')
+                        donut = np.ones_like(corr)-np.eye(np.shape(corr)[0])
+                        return donut
+
 
 
 def cov_distance(cov, expon=0.5):
@@ -234,6 +263,7 @@ def cov_distance(cov, expon=0.5):
         return corr_distance(corr=corr, expon=expon)
 
 
+
 def try_invert(a, **affine_inversion_kwargs):
     """
        Attempt to invert a matrix by whatever means, falling back to ridge + shrinkage as required
@@ -355,4 +385,3 @@ def parity(cov,w):
     np.dot(cov,w)
 
 
-

precise/skaters/managers/allmanagers.py

@@ -2,7 +2,7 @@
 from precise.skaters.managers.ppomanagers import PPO_LONG_MANGERS
 from precise.skaters.managers.hrpmanagers import HRP_LONG_MANAGERS
 from precise.skaters.managers.schurmanagers import SCHUR_LONG_MANAGERS
-from precise.skaters.managers.rflmanagers import RFL_HRP_LONG_MANAGERS
+#from precise.skaters.managers.rflmanagers import RFL_HRP_LONG_MANAGERS
 from precise.skaters.managers.equalmanagers import EQUAL_LONG_MANAGERS
 from precise.skaters.managers.ldpmanagers import LDP_LONG_MANAGERS
 from precise.skaters.managers.molybogamanagers import MOLYBOGA_LONG_MANAGERS
@@ -12,9 +12,9 @@
 
 # d0 managers unless otherwise stated
 
-LONG_MANAGERS = WEAK_LONG_MANAGERS + PPO_LONG_MANGERS + HRP_LONG_MANAGERS + SCHUR_LONG_MANAGERS + RFL_HRP_LONG_MANAGERS
+LONG_MANAGERS = WEAK_LONG_MANAGERS + PPO_LONG_MANGERS + HRP_LONG_MANAGERS + SCHUR_LONG_MANAGERS
 LONG_MANAGERS = WEAK_LONG_MANAGERS + PPO_LONG_MANGERS[:100] + HRP_LONG_MANAGERS +\
-SCHUR_LONG_MANAGERS + RFL_HRP_LONG_MANAGERS[:2] + EQUAL_LONG_MANAGERS + LDP_LONG_MANAGERS + MOLYBOGA_LONG_MANAGERS + RP_LONG_MANAGERS
+SCHUR_LONG_MANAGERS + EQUAL_LONG_MANAGERS + LDP_LONG_MANAGERS + MOLYBOGA_LONG_MANAGERS + RP_LONG_MANAGERS
 
 RELIABLE_LONG_MANAGERS = [ m for m in LONG_MANAGERS if not 'ppo' in m.__name__]
 

precise/skaters/managers/buyandholdfactory.py

@@ -0,0 +1,66 @@
+
+import math
+from precise.skaters.locationutil.vectorfunctions import normalize
+from functools import partial, update_wrapper
+
+
+def buy_and_hold(mgr, j, q=1.0):
+    """ Make a manager with frequency j
+    :param mgr:
+    :param j:
+    :return:
+    """
+    mgr_j_q = partial(mgr,j=j, q=q)
+    mgr_j_q = update_wrapper(mgr_j_q, mgr)
+    mgr_j_q.__name__ = mgr.__name__+'_j'+str(j)+'_q'+str(int(100*q)).zfill(3)
+    return mgr_j_q
+
+
+
+def buy_and_hold_manager_factory(mgr, j:int, y, s:dict, e=1000, q=1.0):
+    """ Ignores manager preference except every j data points
+
+         For this to make any sense, 'y' must be changes in log prices.
+         For this to be efficient, the manager must respect the "e" convention. That is, the
+                                    manager must do little work when e<0
+
+    :param mgr:
+    :param j:
+    :param y:
+    :param s:              State
+    :param q:              New portfolio is  q*w + (1-q)*w_prev
+    :param mgr_kwargs:
+    :return:  w   Portfolio weights
+    """
+    if s.get('w') is None:
+        # Initialization
+        s['count'] = 0
+        s_mgr = {}
+        w, s_mgr = mgr(y=y, s=s_mgr, e=1000)
+        s['s_mgr'] = s_mgr
+        s['w'] = w
+        return w, s
+    else:
+        s['count'] = s['count']+1
+        if s['count'] % j == 0:
+            # Sporadically use the manager
+            s_mgr = s['s_mgr']
+            w_mgr, s_mgr = mgr(y=y, s=s_mgr, e=1000)
+            s['s_mgr'] = s_mgr
+            w_prev = s['w']
+            w_roll = normalize([wi * math.exp(yi) for wi, yi in zip(w_prev, y)])
+            w = [ wi*q + (1-q)*wpi for wi, wpi in zip(w_mgr, w_roll) ]
+            s['w'] = [wi for wi in w]
+            return w, s
+        else:
+            # Tell the manager not to worry too much about this data point, as the weights won't be used ...
+            s_mgr = s['s_mgr']
+            _ignore_w, s_mgr = mgr(y=y, s=s_mgr, e=-1)
+            s['s_mgr'] = s_mgr
+            # ... instead we let it ride
+            w_prev = s['w']
+            w = normalize( [ wi*math.exp(yi) for wi,yi in zip(w_prev,y)] )
+            s['w'] = [wi for wi in w]
+            return w, s
+
+

precise/skaters/managers/covmanagerfactory.py

@@ -1,10 +1,51 @@
 from precise.skaters.portfoliostatic.equalport import equal_long_port
 from precise.skaters.covarianceutil.covfunctions import cov_to_corrcoef
 import numpy as np
+import math
 from precise.skaters.locationutil.vectorfunctions import normalize
 
 
-def static_cov_manager_factory_d0(y, s, f, port, e=1, f_kwargs:dict=None, port_kwargs:dict=None, n_cold=5, zeta=0.0):
+
+def _buy_and_hold_port(port, j:int, q:float, y, s:dict, cov, port_kwargs):
+    """
+
+         Sporadically calls port() but uses buy and hold in between.
+         For this to work, 'y' must be changes in log prices.
+
+    :param port:
+    :param j:
+    :param y:
+    :param s:
+    :param cov:
+    :param port_kwargs:
+    :return:  w   Portfolio weights
+    """
+    n_dim = len(y)
+    if s.get('w') is None:
+        s['multiplier'] = [1 for _ in range(n_dim)]
+        s['count'] = 0
+        w = port(cov=cov, **port_kwargs)
+        s['w'] = [ wi for wi in w ]
+        return w, s
+    else:
+        s['count'] = s['count']+1
+        if s['count'] % j == 0:
+            # Create a compromise between the roll forward portfolio and what the optimizer wants
+            s['multiplier'] = [mi * math.exp(yi) for mi, yi in zip(s['multiplier'], y)]
+            w_roll = normalize([wi * mi for wi, mi in zip(s['w'], s['multiplier'])])
+            w_port = port(cov=cov, **port_kwargs)
+            w = [q * wi + (1 - q) * wpi for wi, wpi in zip(w_port, w_roll)]
+            s['w'] = [wi for wi in w]
+            s['multiplier'] = [1 for _ in range(n_dim)]
+            return w, s
+        else:
+            # Let it ride
+            s['multiplier'] = [ mi*math.exp(yi) for mi,yi in zip(s['multiplier'],y)]
+            w = normalize( [ wi*mi for wi,mi in zip(s['w'],s['multiplier']) ] )
+            return w, s
+
+
+def static_cov_manager_factory_d0(y, s, f, port, e=1, f_kwargs:dict=None, port_kwargs:dict=None, n_cold=5, zeta=0.0, j=1,q=1.0):
     """
        Basic manager pattern ignoring mean.
        Expects to receive changes in log(price).
@@ -14,6 +55,8 @@ def static_cov_manager_factory_d0(y, s, f, port, e=1, f_kwargs:dict=None, port_k
           :param f     cov skater ("d0")
           :param port  portfolio constructor
           :param zeta  Compromise between cov and corr portfolio (zeta=0, use cov only)
+          :param j     How often to run the static portfolio construction.
+                       (Warning: if j>1 this will be nonsense unless 'y' represents changes in log prices)
 
        :returns w, s
     """
@@ -25,13 +68,19 @@ def static_cov_manager_factory_d0(y, s, f, port, e=1, f_kwargs:dict=None, port_k
     if not s:
         s = {'f_state':{},
              'port_state':{},
-             'count':0}
+             'count':0,
+             'account_state':{}}
 
+    
     x_mean, x_cov, s['f_state'] = f(y=y,s=s['f_state'], k=1, e=e, **f_kwargs)
     s['count']+=1
     if s['count']>=n_cold and (e>0):
-        w = port(cov=x_cov, **port_kwargs)
-        if zeta>0:
+        s_account = s['account_state']
+        w, s_account = _buy_and_hold_port(port=port, y=y, j=j, q=q, s=s_account, cov=x_cov, port_kwargs=port_kwargs)
+        s['account_state'] = s_account
+        if zeta is not None and (zeta>0):
+            # Drags towards the portfolio determined by cov
+            # FIXME: This should be moved inside _buy_and_hold_port
             x_diag = np.diag(x_cov)
             x_corr = cov_to_corrcoef(x_cov)
             w_corr_raw = port(cov=x_corr, **port_kwargs)