\section{Proofs}\label{app:proofs}

\subsection{Proof of Proposition~\ref{prop:factor}}

At a decision state, \eqref{eq:genaction} is normalized because its denominator is $\exp\{V^\tau_\gen(s)/\tau\}$; at a chance state, \eqref{eq:genevent} is normalized because its denominator is $\exp\{W^\tau_\gen(s')/\tau\}$. Multiplying the two conditionals for round $t$ gives
\[
\pi_{\refp}(a_t\mid s_t)\mu(e_t\mid s'_t)
\exp\left\{\frac{r(s_t,a_t)+r(s'_t,e_t)+V^\tau_\gen(s_{t+1})-V^\tau_\gen(s_t)}{\tau}\right\}.
\]
Over $K$ rounds the value terms telescope. With terminal value zero, the chain product is
\[
m_{\refp}(\xi\mid x)\exp\{[R(\xi)-V^\tau_\gen(x)]/\tau\}.
\]
Summing over $\xi$ shows that $Z_\tau(x)=\exp\{V^\tau_\gen(x)/\tau\}$, which proves the factorization.

For the teacher-forcing statement, expand expected NLL by position and condition on the reached prefix. Every term is a nonnegative occupancy weight times a conditional cross-entropy. On a positive-occupancy prefix, the cross-entropy is minimized exactly by the data conditional. Applying this separately to the tagged action and event positions of \eqref{eq:observational} yields $\pi_\beh$ and $\mu$ on reachable prefixes. \qed

\subsection{Proof of Proposition~\ref{prop:gap}}

At a chance state set $Z_\gen(e)=r(s',e)+V^\tau_\gen(s'\oplus e)$. Since $V_\ctrl=V^\tau_\gen-\Delta V$,
\[
W_\ctrl(s')=\Exp_\mu[Z_\gen(E)-\Delta V(s'\oplus E)].
\]
Subtracting this expression from \eqref{eq:wgen} gives \eqref{eq:gapw}. At a decision state,
$Q^\tau_\gen(s,a)=Q_\ctrl(s,a)+\Delta W(s\oplus a)$. Substitute \eqref{eq:ctrlpolicy} into the ratio of the two action partitions to obtain \eqref{eq:gapv}.

Jensen's inequality gives $g_\tau\ge0$. Starting from zero terminal gap, \eqref{eq:gapw} and \eqref{eq:gapv} imply nonnegativity by backward induction. Also,
\[
\Exp X\le\max X,\qquad \tau\log\Exp\exp(X/\tau)\le\max X,
\]
so the same induction gives the sum of layerwise maxima. Strict convexity of the exponential gives $g_\tau=0$ exactly when its argument is constant $\mu$-almost surely. Finally, \eqref{eq:cumulant} is the cumulant expansion of $\tau\log\Exp\exp\{(Z-\Exp Z)/\tau\}$. \qed

\subsection{Proof of Proposition~\ref{prop:audit}}

Rearranging \eqref{eq:localscore},
\[
q_b(s,z)=\tau\log\frac{\hat p(z\mid s)}{b(z\mid s)}+V_{q_b}(s).
\]
Therefore
\[
r_b(s,z)=\tau\log\frac{\hat p(z\mid s)}{b(z\mid s)}+V_{q_b}(s)-V_{q_b}(s\oplus z).
\]
Summation along a complete trajectory telescopes the values and multiplies the conditional ratios, giving \eqref{eq:audit}. Setting $\hat\rho=\rho_\beh$ and $b=\pi_{\refp}$ at actions and $b=\mu$ at events cancels every event ratio and yields \eqref{eq:eventcancel}. Under a uniform counting base, neither factor cancels. \qed

\subsection{Proof of Theorem~\ref{thm:pathwise}}

At each round, \eqref{eq:advantage} and \eqref{eq:eventimpact} give
\begin{align*}
A_\ctrl(s_t,a_t)+I_{\mathrm{event}}(s'_t,e_t)
&=Q_\ctrl(s_t,a_t)-V_\ctrl(s_t)\\
&\quad+r(s'_t,e_t)+V_\ctrl(s_{t+1})-W_\ctrl(s'_t)\\
&=r(s_t,a_t)+r(s'_t,e_t)+V_\ctrl(s_{t+1})-V_\ctrl(s_t).
\end{align*}
Summing over $t$ telescopes the value terms and proves \eqref{eq:pathwise}. Centering in \eqref{eq:eventimpact} follows directly from \eqref{eq:wctrl}. \qed

\subsection{Proof of the KL-regret identity \eqref{eq:klregret}}

The Gibbs variational identity applied at any decision state gives, for every distribution $\pi_s$,
\begin{align*}
V_\ctrl(s)&-\left\{\Exp_{a\sim\pi_s}Q_\ctrl(s,a)-\tau\KL(\pi_s\|\pi_{\refp,s})\right\}\\
&=\tau\KL(\pi_s\|\pi^*_{\ctrl,s}).
\end{align*}
Take expectation under the state occupancy generated by $\pi$ and $\mu$, sum over time, and expand $Q_\ctrl=r+\Exp_\mu[r+V_\ctrl]$. Adjacent expected values telescope, leaving the difference between $V_\ctrl(x)=J_M^\tau(\pi^*_\ctrl;x)$ and $J_M^\tau(\pi;x)$. \qed

\subsection{Proof of Theorem~\ref{thm:error}}

Fix a policy $\pi$ and let
$\delta_h=\sup_s|V_{M,h}^{\pi}(s)-V_{\hat M,h}^{\pi}(s)|$.
The statewise KL penalty is identical in the two models and cancels. At a chance node, add and subtract the $\mu$-expectation of
$f(e)=\hat r(s',e)+V_{\hat M,h-1}^{\pi}(s'\oplus e)$. Using
\[
|\Exp_Pf-\Exp_Qf|\le\TV(P,Q)\operatorname{span}(f)
\]
and one action-position plus one event-position reward error gives
\begin{equation}\label{eq:delta-recursion}
\delta_h\le\delta_{h-1}+2\epsilon_r
+\epsilon_\mu\left[2R_{\max}+\operatorname{span}(V_{\hat M,h-1}^{\pi})\right].
\end{equation}
For any policy, a one-round expected reward plus KL penalty lies in
$[-2R_{\max}-\tau B_{\refp},,2R_{\max}]$. Expectations do not expand span, hence
\[
\operatorname{span}(V_{\hat M,h}^{\pi})\le h(4R_{\max}+\tau B_{\refp}).
\]
Substitution in \eqref{eq:delta-recursion} and summation from $\delta_0=0$ give \eqref{eq:ch}.

The bound is uniform in $\pi$, so $|J_M^\tau(\pi)-J_{\hat M}^\tau(\pi)|\le C_K$. With $\pi_M^*$ optimal in $M$ and $\hat\pi$ $\eta$-optimal in $\hat M$,
\begin{align*}
J_M^\tau(\pi_M^*)-J_M^\tau(\hat\pi)
&\le[J_M^\tau(\pi_M^*)-J_{\hat M}^\tau(\pi_M^*)]+\eta\\
&\quad+[J_{\hat M}^\tau(\hat\pi)-J_M^\tau(\hat\pi)]\\
&\le2C_K+\eta.
\end{align*}
This proves the theorem. \qed

\subsection{Proof of Corollary~\ref{cor:gate}}

Apply Theorem~\ref{thm:error} after restricting the policy class to $\Pi_G$, where its assumptions hold. Add and subtract $J_M^\tau(\pi_G^*)$:
\begin{align*}
J_M^\tau(\pi_M^*)-J_M^\tau(\hat\pi_G)
&=[J_M^\tau(\pi_M^*)-J_M^\tau(\pi_G^*)]\\
&\quad+[J_M^\tau(\pi_G^*)-J_M^\tau(\hat\pi_G)].
\end{align*}
The first bracket is the coverage-restriction cost; the second is at most $2C_K(G)+\eta$. \qed
