\section{Generative and control readouts}\label{sec:readouts}

\subsection{Exact reference-measure factorization}

Define the trajectory base measure and reward-tilted density
\begin{align}
 m_{\refp}(\xi\mid x)&=\prod_{t=1}^{K}\pi_{\refp}(a_t\mid s_t)\mu(e_t\mid s'_t),\label{eq:base}\\
 p^\tau_\gen(\xi\mid x)&=\frac{m_{\refp}(\xi\mid x)\exp\{R(\xi)/\tau\}}{Z_\tau(x)}.\label{eq:ebm}
\end{align}

\begin{proposition}[Generative factorization]\label{prop:factor}
With terminal value zero, define
\begin{align}
W^\tau_\gen(s')&=\tau\log\Exp_{e\sim\mu(\cdot\mid s')}\exp\left\{\frac{r(s',e)+V^\tau_\gen(s'\oplus e)}{\tau}\right\},\label{eq:wgen}\\
Q^\tau_\gen(s,a)&=r(s,a)+W^\tau_\gen(s\oplus a),\\
V^\tau_\gen(s)&=\tau\log\Exp_{a\sim\pi_{\refp}(\cdot\mid s)}\exp\{Q^\tau_\gen(s,a)/\tau\}.\label{eq:vgen}
\end{align}
Then \eqref{eq:ebm} has exact conditionals
\begin{align}
p^\tau_\gen(a\mid s)&=\pi_{\refp}(a\mid s)\exp\{(Q^\tau_\gen(s,a)-V^\tau_\gen(s))/\tau\},\label{eq:genaction}\\
p^\tau_\gen(e\mid s')&=\mu(e\mid s')\exp\{(r(s',e)+V^\tau_\gen(s'\oplus e)-W^\tau_\gen(s'))/\tau\}.\label{eq:genevent}
\end{align}
Their chain product equals $p^\tau_\gen(\xi\mid x)$.
\end{proposition}

The proof is a telescoping cancellation of adjacent value terms (Appendix~\ref{app:proofs}). The proposition extends reference-measure ARM--EBM factorization to typed action--event sequences. It is a \emph{generative} recursion: both decisions and events are exponentially tilted toward high reward.

\subsection{Control preserves exogenous chance}

The optimum of \eqref{eq:objective} instead satisfies
\begin{align}
W_\ctrl(s')&=\Exp_{e\sim\mu(\cdot\mid s')}[r(s',e)+V_\ctrl(s'\oplus e)],\label{eq:wctrl}\\
Q_\ctrl(s,a)&=r(s,a)+W_\ctrl(s\oplus a),\\
V_\ctrl(s)&=\tau\log\Exp_{a\sim\pi_{\refp}(\cdot\mid s)}\exp\{Q_\ctrl(s,a)/\tau\},\label{eq:vctrl}\\
\pi^*_\ctrl(a\mid s)&=\pi_{\refp}(a\mid s)\exp\{(Q_\ctrl(s,a)-V_\ctrl(s))/\tau\}.\label{eq:ctrlpolicy}
\end{align}
The action operator is the same as in generation; the chance operator is not. Generation uses an exponential log-moment in \eqref{eq:wgen}, whereas control uses the fixed-kernel expectation in \eqref{eq:wctrl}.

For a continuation function $U$, define the local chance-readout gap
\begin{equation}\label{eq:localgap}
g_\tau(s';U)=\tau\log\Exp_\mu\exp\{[r(s',E)+U(s'\oplus E)]/\tau\}
-\Exp_\mu[r(s',E)+U(s'\oplus E)].
\end{equation}

\begin{proposition}[Local-to-global gap recursion]\label{prop:gap}
Let $\Delta V=V^\tau_\gen-V_\ctrl$ and $\Delta W=W^\tau_\gen-W_\ctrl$. Then
\begin{align}
\Delta W(s')&=g_\tau(s';V^\tau_\gen)+\Exp_{e\sim\mu(\cdot\mid s')}\Delta V(s'\oplus e),\label{eq:gapw}\\
\Delta V(s)&=\tau\log\Exp_{a\sim\pi^*_\ctrl(\cdot\mid s)}\exp\{\Delta W(s\oplus a)/\tau\}.\label{eq:gapv}
\end{align}
Consequently $\Delta V,\Delta W\ge0$. If $\bar g_j$ is the largest local gap at chance layer $j$, then $\Delta V(s_t)\le\sum_{j=t}^{K}\bar g_j$. The two readouts agree exactly when every reachable chance return $r(s',E)+V^\tau_\gen(s'\oplus E)$ is $\mu$-almost surely constant.
\end{proposition}

Equation~\eqref{eq:gapw} separates optimism newly created at the current event from gap inherited from future events. By Jensen's inequality $g_\tau\ge0$. For centered $Z=r(s',E)+U(s'\oplus E)$ with small fluctuation,
\begin{equation}\label{eq:cumulant}
    g_\tau(s';U)=\frac{\operatorname{Var}_\mu(Z)}{2\tau}
      +O\!\left(\frac{\Exp_\mu|Z-\Exp Z|^3}{\tau^2}\right).
\end{equation}
Thus event entropy is not the causal variable. A high-entropy event law gives zero gap if all events have the same continuation value; a low-entropy law can give a large gap when a rare event has an extreme return.

\begin{remark}[Known optimism, new localization]
Naive inference-based control under stochastic dynamics can act as though favorable outcomes were controllable \citep{levine2018}. Proposition~\ref{prop:gap} does not rename that phenomenon. It localizes the mismatch in the ARM--EBM readout and gives its exact multi-round propagation.
\end{remark}
