SoftMax Algorithm

\begin{algorithm}
\caption{Geometric Algorithm MDP-IC (MDP Inspired Feedback Control).}\label{alg:mdp-ic-geo}
\begin{algorithmic}[1]
\Require{$\vec{\mathrm{Q}} = \mathrm{vec}(Q), \vec{\mathrm{W}} = \mathrm{vec}(\mathrm{err}), \vec{\theta}{s_0}=\mathrm{vec}((Q, \pi^{\vec{\theta}{s_0}})), q = \langle q_i \rangle, w = \langle w_i \rangle, \theta_{s_0} = \langle Q^{{(\sigma_i)}}, \pi^{\theta_i}{s_0}\rangle$} \Ensure{$\vec{\theta}{s_0}$ or a sample $\pi^{\theta_{s_0}}={\mathrm{vec}}^{-1}(\vec{\theta}{s_0})$ if coefficient is $\alpha$.} %\For{$i \leq n$} \Comment{Train Iterations} \State{Train $\vec{X}{\vec{\theta}_{s_0}}$ from $q, w, s$} \Comment{Fixed Inner Loop Update}
%\EndFor

   % \State{$\mathrm{err}_{\vec{\theta}_{s_0}}\left(\vec{\mathrm{Q}},{\mathrm{vec^{-1}(\vec{\mathrm{Q}})}},q,w,s\right)\;$: Initialize function approximator}

    \For{$t \leq t_{max}$} \Comment{Main Loop}
        \State{$s \sim p_{0}$}\Comment{Generate $s$ from $p_{0}$}
        \State{$a \sim \pi^{\theta_{s_0}}(a|s)$} \Comment{Execute action $a$}
        %\State{$\tilde{s} \sim p(\tilde{s}|s,a)$} \Comment{Observe $s' \leftarrow \tilde{s}$}
        \State{$s' \sim p^{\pi_{s_0}}(\cdot|s,a)$} \Comment{Generate $s'$ from $\pi^{\theta_{s_0}}$}

        \State{$Q(s,a)$}\Comment{Create $\vec{\mathrm{Q}}_t$ from $(s,a,\tilde{s},r)$} 
        \State{$\mathrm{err}_{\vec{\theta}_{s_0}}\left(\vec{\mathrm{Q}_t},\vec{w}, q_{\tilde{s}, s'}, s\right)$}\Comment{Create $\vec{w}_t$ from $(q,w)$}
        %\If{$\epsilon_\mathrm{pred}\left(\vec{\mathrm{Q}}, {\vec{\theta}_{s_0}},\vec{w},q,s\right)$ $\leq$ $\epsilon_\mathrm{goal}$} 
        \State{$x = H(\epsilon_\mathrm{pred}\left(\vec{\mathrm{Q}}, q'\right))$} \Comment{Decay over time}
        \State{$y = L(\epsilon_\mathrm{pred}\left(\vec{\mathrm{Q}}, q'\right))$} \Comment{Learned trajectory}
        \State{Construct $\mathrm{err}_{\vec{\theta}_{s_0}}\left(\vec{\mathrm{Q}}_t,\vec{w}_t,q_{\tilde{s}, s'}, s\right)$}\Comment{Construct error term $\times x,y,\vec{\mathrm{Q}}_t$}
            \State{$\theta_{\tilde{s}} = {\mathrm{vec}}^{-1}\left({\vec{\theta}_{s_0}}\right)$}
        \If{Norm of $\nabla_{\theta} \mathrm{err}_{\vec{\theta}_{s_0}}\left(\vec{\mathrm{Q}}_t,\vec{w}_t,q_{\tilde{s}, s'}, s\right) \leq \epsilon_\mathrm{goal}$} \Comment{Norm of Gradient}
            \State{\textbf{break}}
        \EndIf
    \EndFor

    \State{Compute $d_s = ||f_{\delta}(\mathcal{S}^\prime) - f_{\delta}(s)||_2$}
    \State{Adjust the parameters of $\pi_{s_0}$ using $d_s$ and $\alpha$}
    \State{\Return $\pi_{s_0}(a|s)$ and $\theta_{s_0}$}
\end{algorithmic}

\end{algorithm}