lecture-19.tex

% !TEX spellcheck = en_US
% !TEX spellcheck = LaTeX
\documentclass[a4paper,10pt,english]{article}
\input{header}
\title{Lecture 19: Reversed processes}
\author{}
\begin{document}
\maketitle

%\noindent {\bf Recall:} A stochastic process $\{X(t) \in I: t \in T\}$ (where $T = \R$ or $T = \Z$) is \textbf{reversible} if $(X(t_i): i \in [n])$ has the same distribution as $(X(\tau-t_i): i \in [n])$ for all $t_i, \tau \in T, i \in [n]$. In general, we can ask -- what does the time-reversed version of a stochastic process look like?

\section{Reversed Processes}
%\begin{defn} 
Let $\{X(t): t \in T\}$ be a stochastic process with index set $T$ being $\R$ or $\Z$. 
Then, $\{X(\tau-t): t \in T\}$ is the \textbf{reversed process}, for some $\tau \in T$. 
%\end{defn}
\begin{lem} 
If $\{X(t)\}_{t \in \Z}$ is a Markov chain, then the reversed process $\{X(\tau -t)\}_{t \in \Z}$ is also a Markov chain.
\end{lem}
\begin{proof} For each $m \in \Z$, let $\F_m = \sigma \left(\cup_{k \geq m} X_k\right)$ represent the $\sigma$-algebra generated by the random variables $\{X_k: k \geq m\}$. Then, we can write
\begin{align*}
\Pr\{X_{m-1}=i|X_m=j,\F_{m+1}\} %&= \frac{\Pr\{X_{m-1}=i,X_m=j, \F_{m+1} \}}{\Pr\{X_m=j, \F_{m+1}\}}\\
&=\frac{\Pr\{X_{m-1}=i|X_m=j\}\Pr\{\F_{m+1} |X_{m-1}=i,X_m=j \}}{\Pr\{\F_{m+1} |X_m=j\}}.
\end{align*}
Using the Markov property of $X(t)$, i.e.,
\begin{align*}
\Pr\{\F_{m+1} |X_m= j, X_{m-1} = i \} &=\Pr\{\F_{m+1}|X_m=j\},
\end{align*}
thus implies $\Pr\{X_{m-1}=i|X_m=j,\F_{m+1}\} = \Pr\{X_{m-1}=i|X_m=j\}$, showing that the reversed process is indeed a Markov chain. 
\end{proof}

\begin{rem}
Even if the forward process $X$ is time-homogeneous, the reversed process need not be time-homogeneous. 
\end{rem}


\begin{lem}  
\label{lem:reversedchain}
If $\{X(t): t \in \Z\}$ is a stationary, time homogeneous Markov chain with transition matrix $P$ and equilibrium distribution $\alpha$, 
then the reversed chain $\{X(\tau -t): t \in \Z\}$ is a stationary, time homogeneous Markov chain with the same equilibrium distribution $\alpha$, and transition matrix $P^{\ast}$ given by
\begin{align*}
P^{\ast}_{ij} = \frac{\alpha_j}{\alpha_i}P_{ji}.
\end{align*}
\end{lem}
\begin{proof} 
It is clear that the reversed chain must have the same stationary distribution $\alpha$. For computing its transition matrix, use Bayes' formula to get
\begin{align*}
\Pr\{X_{m-1}=i|X_m=j\} &= \frac{\Pr\{X_{m-1}=i\} \, P_{ij}  }{ \Pr\{X_{m}=j\} } =\frac{\alpha_i \, P_{ij}  }{ \alpha_j },
\end{align*}
yielding the desired result. 
\end{proof}

\begin{lem} 
If $X(t)$ is a stationary Markov process with generator matrix $Q$ and equilibrium distribution $\pi$, then the reversed process $X(\tau -t)$ is a stationary Markov process with same equilibrium distribution $\pi$ and generator matrix $Q^{\ast}$ such that
\begin{align*}
Q^{\ast}_{ij} = \frac{\pi_j}{\pi_i}Q_{ji}.
\end{align*}
\end{lem}
\begin{proof} 
Follows from calculations analogous to those in the previous result. 
\end{proof}

In fact, the following powerful result allows us to `guess' both the transition behaviour of the reversed chain and the stationary distribution together!

\begin{thm}
\label{thm:guessReversal}
Consider an irreducible, discrete time Markov chain with transition matrix $P$. If one can find non-negative vectors $\pi$ and a transition matrix $P^\ast$ such that $\sum_j \pi_j =1$ and $\pi_iP_{ij}=\pi_jP^*_{ji}$, then $\pi$ is the stationary probability vector of the Markov chain, and $P^*$ is the transition matrix for the reversed chain.
\end{thm}
\begin{proof}
Summing $\pi_iP_{ij}=\pi_jP_{ji}^*$ over $i$ gives, $\sum_{i}\pi_iP_{ij}=\pi_j$. Hence $\pi_i$s are the stationary probabilities of the forward and reverse process. Since $P_{ji}^*=\frac{\pi_iP_{ij}}{\pi_j}$ (Lemma \ref{lem:reversedchain}), $P_{ij}^*$ are the transition probabilities of the reverse chain.
\end{proof} 

\begin{shaded*}
\begin{exmp}[Age and excess time of a renewal process]
Consider a discrete-time renewal process with the associated age process $\{A_n\}_{n \in \Z}$, i.e., $A_n$ represents the time elapsed since the last renewal instant (assume $A_n = 0$ if there is a renewal at time instant $n$). Let the interarrival times be distributed according to the probability mass function $p_i$,  $i = 0, 1, 2, \ldots$
\end{exmp}
\end{shaded*}
\begin{lem}
$\{A_n\}$ is a DTMC, with transition probabilities $P_{i,0} = p_i/\sum_{j \geq i} p_j = 1 - P_{i,i+1}$ $\forall i $, and $P_{i,j} = 0$ otherwise.  
\end{lem}
\noindent (Work out the proof if needed.) \\

\noindent {\bf Question.} What would the time-reversed version of this DTMC look like? 

Qualitatively, in the time reversed process, the state would decrease down to $0$ by one unit at each time, and then jump to an arbitrary state decided by the interarrival time distribution. This appears suspiciously similar to the description of the excess time process, i.e., the process that tracks the time remaining until the next renewal instant!

Let us guess that the reversed process is the excess time process $\{B_n\}$. Analogous to the previous lemma, $\{B_n\}$ is a DTMC, and its transition probabilities must be $P^\ast_{i,i-1} = 1$, $i \geq 1$, $P^\ast_{0,i} = p_i$, $i \geq 0$, and $P^\ast_{i,j} = 0$ otherwise. By Theorem \ref{thm:guessReversal}, if we can show for some positive, probability vector $\pi$ that $\pi_i P_{i,j} = \pi_j P^\ast_{j,i}$ $\forall i, j \geq 0$, then we would have shown that the excess time process is indeed the time-reversed version of the age process, and in the process also have computed the stationary probabilities of both processes. 

Writing down $\pi_i P_{i,0} = \pi_0 P^\ast_{0,i}$ gives $\pi_i = \pi_0 \Pr\{X \geq i\}$, where $X$ represents an interarrival time. Since $\sum \pi_i = 1$, this implies that $\pi_i = \Pr\{X \geq i\}/\E[X]$. The only remaining task is to check that $\pi_i P_{i,i+1} = \pi_{i+1} P^\ast_{i+1,i}$ for any $i \geq 0$, which is equivalent to $ \Pr\{X \geq i\} \left( 1 - p_i/ \Pr\{X \geq i\}\right) =  \Pr\{X \geq i+1\}$, a true statement.  

\end{document}