Benchmark Weight-Update Rules
Table of Contents
Benchmark 1
\[
{0.9995}^{- t - 0.9995} \Delta w^{\text{A}\hspace{-.018cm}\text{dadelta}}_{j, m, t=\tau} + 0.9 \Delta w^{\text{A}\hspace{-.018cm}\text{dadelta}}_{j, m, t=\tau}
\]
MSE: 2.4550
Benchmark 2
\[
\sin{\left(\frac{\mu_{j,m,t=\tau}}{- 2.0 t + \nu_{j,m,t=\tau}} \right)} + 1.0 \cdot 10^{-8}
\]
MSE: 2.5329
Benchmark 3
\[
- \frac{0.02 y_{j}}{d_{j} - \frac{\widehat{\mu}_{j,m,t=\tau}}{d_{j}}} + 1.0 \cdot 10^{-8}
\]
MSE: 2.5362
Benchmark 4
\[
g_{j, m, t=\tau} + e^{\frac{\Delta w^{\text{A}\hspace{-.018cm}\text{dadelta}}_{j, m, t=\tau} \nu_{j,m,t=\tau}}{\nu_{j,m,t=\tau} \frac{1}{v_{j, m, t=\tau}}}}
\]
MSE: 7.7460
Benchmark 5
\[
\frac{E\left[\Delta w_{j, m}^2\right]_{t=\tau}}{\left(\sigma_{\hspace{-.05cm}g^{2}_{j,m}}\right)^{\widehat{\mu}_{j,m,t=\tau} - v_{j, m, t=\tau}} + \tanh{\left(E\left[\Delta w_{j, m}^2\right]_{t=\tau} \right)} + 0.9001}
\]
MSE: 9.3206
Benchmark 6
\[
\frac{0.01 \left(d_{j}^{\mathrm{Nesterov}} + \cos{\left(0.999 t \right)} - 0.01\right)}{E\left[\Delta w_{j, m}^2\right]_{t=\tau}}
\]
MSE: 7.9954
Benchmark 7
\[
\frac{\Delta w_{j, m, t=\tau} \sigma_{\hspace{-.05cm}g^{2}_{j,m}}}{E\left[\Delta w_{j, m}^2\right]_{t=\tau} \left(- E\left[g_{j, m}^2\right]_{t=\tau} + \frac{\mu_{j,m,t=\tau}}{\Delta w_{j, m, t=\tau}}\right)}
\]
MSE: 237418000.0000
Benchmark 8
\[
\frac{E\left[g_{j, m}^2\right]_{t=\tau}}{\Delta w^{\text{A}\hspace{-.018cm}\text{dadelta}}_{j, m, t=\tau} \sigma_{\hspace{-.05cm}g^{2}_{j,m}} - \widehat{\nu}_{j,m,t=\tau} - y_{j} + v_{j, m, t=\tau} + 1.0 \cdot 10^{-8}}
\]
MSE: 2876810.0000
Benchmark 9
\[
\frac{1.001 g_{j, m, t=\tau} + \nu_{j,m,t=\tau} + y_{j}}{- E\left[g_{j, m}^2\right]_{t=\tau} - g_{j, m, t=\tau} - t + \nu_{j,m,t=\tau} + y_{j}}
\]
MSE: 245055000.0000
Benchmark 10
\[
\frac{{0.999}^{y_{j}^{0.25}}}{- d_{j} - E\left[g_{j, m}^2\right]_{t=\tau} + \mu_{j,m,t=\tau} - w_{j,m,t=\tau} - 0.01}
\]
MSE: 2.7822
Benchmark 11
\[
0.01 \left(- \frac{0.9 - g_{j, m, t=\tau}}{E\left[g_{j, m}^2\right]_{t=\tau} + \mu_{j,m,t=\tau}} + 0.01\right) \cos{\left(\Delta w^{\text{A}\hspace{-.018cm}\text{dadelta}}_{j, m, t=\tau} \right)}
\]
MSE: 3.6288
Benchmark 12
\[
\frac{y_{j}}{- 0.001 d_{j} - \Delta w_{j, m, t=\tau} + E\left[g_{j, m}^2\right]_{t=\tau} - 0.5236}
\]
MSE: 3.2924
Benchmark 13
\[
\frac{y_{j}}{0.01 E\left[\Delta w_{j, m}^2\right]_{t=\tau} + v_{j, m, t=\tau} + 1.4}
\]
MSE: 13.6686
Benchmark 14
\[
\frac{d_{j}^{\mathrm{Nesterov}} t}{d_{j} + d_{j}^{\mathrm{Nesterov}} + \nu_{j,m,t=\tau} - 0.7833}
\]
MSE: 15.4005
Benchmark 15
\[
0.01 \Delta w^{\text{A}\hspace{-.018cm}\text{dadelta}}_{j, m, t=\tau} - \frac{0.01 E\left[\Delta w_{j, m}^2\right]_{t=\tau}}{- 1.0 E\left[g_{j, m}^2\right]_{t=\tau} + 1.0 v_{j, m, t=\tau}}
\]
MSE: 14.1013
Benchmark 16
\[
\frac{\Delta w_{j, m, t=\tau} + t}{- v_{j, m, t=\tau} - \left(- 1.111 \Delta w_{j, m, t=\tau} + 1.111 w_{j,m,t=\tau}\right) \tanh{\left(\sigma_{\hspace{-.05cm}g^{2}_{j,m}} \right)} + 1.0 \cdot 10^{-8}}
\]
MSE: 0.2727
Benchmark 17
\[
\frac{E\left[g_{j, m}^2\right]_{t=\tau}}{\left(- \mu_{j,m,t=\tau} + \tanh{\left(d_{j} \right)}\right) \left(- \widehat{\nu}_{j,m,t=\tau} - \frac{\sin{\left(w_{j,m,t=\tau} \right)}}{\sigma_{\hspace{-.05cm}g^{2}_{j,m}} \widehat{\nu}_{j,m,t=\tau}}\right)}
\]
MSE: 0.2840
Benchmark 18
\[
\frac{E\left[g_{j, m}^2\right]_{t=\tau}}{\left(0.01 - \mu_{j,m,t=\tau}\right) \left(- \widehat{\nu}_{j,m,t=\tau}^{2.0 \cdot 10^{-8}} - \widehat{\nu}_{j,m,t=\tau}\right)}
\]
MSE: 0.2688
Benchmark 19
\[
\frac{\Delta w_{j, m, t=\tau} - \mu_{j,m,t=\tau} - 0.9}{- \Delta w^{\text{A}\hspace{-.018cm}\text{dadelta}}_{j, m, t=\tau} + t + v_{j, m, t=\tau}}
\]
MSE: 379.5750
Benchmark 20
\[
\frac{0.999}{\frac{d_{j}}{0.01 - 0.999 g_{j, m, t=\tau}} + y_{j}}
\]
MSE: 318.1030
Benchmark 21
\[
\frac{0.999}{\mu_{j,m,t=\tau} + \nu_{j,m,t=\tau} \widehat{\nu}_{j,m,t=\tau}^{\Delta w^{\text{A}\hspace{-.018cm}\text{dadelta}}_{j, m, t=\tau} - v_{j, m, t=\tau}}}
\]
MSE: 373.3070
Benchmark 22
\[
- 2.866 {0.9}^{0.999 \Delta w^{\text{A}\hspace{-.018cm}\text{dadelta}}_{j, m, t=\tau} - 0.999 \nu_{j,m,t=\tau}}
\]
MSE: 11.2288
Benchmark 23
\[
\frac{\mu_{j,m,t=\tau} t + \mu_{j,m,t=\tau} + v_{j, m, t=\tau}}{0.01 d_{j}^{\mathrm{Nesterov}} - \left(E\left[g_{j, m}^2\right]_{t=\tau}\right)^{0.9} - \widehat{\mu}_{j,m,t=\tau} + v_{j, m, t=\tau}}
\]
MSE: 16.0370
Benchmark 24
\[
- 0.01526 \mu_{j,m,t=\tau} - 0.01526 t^{0.5} + 1.526 \cdot 10^{-10}
\]
MSE: 16.8173
Benchmark 25
\[
- \frac{1.998 t \operatorname{acos}{\left(\sin{\left(\Delta w_{j, m, t=\tau} \right)} \right)}}{\widehat{\mu}_{j,m,t=\tau} \left(\mu_{j,m,t=\tau} - y_{j} - 0.01\right)}
\]
MSE: 3405.0500
Benchmark 26
\[
\frac{0.998 t}{\left(\mu_{j,m,t=\tau} - 0.91\right) \left(d_{j}^{\mathrm{Nesterov}} E\left[\Delta w_{j, m}^2\right]_{t=\tau} + 0.9\right)}
\]
MSE: 2938.7000
Benchmark 27
\[
\frac{E\left[\Delta w_{j, m}^2\right]_{t=\tau} y_{j} \left(\tanh{\left(\widehat{\mu}_{j,m,t=\tau} \right)} + 0.9\right)}{- E\left[\Delta w_{j, m}^2\right]_{t=\tau} + \mu_{j,m,t=\tau} + \widehat{\mu}_{j,m,t=\tau} - t - y_{j}}
\]
MSE: 3506.8600
Benchmark 28
\[
\frac{1.449 t}{\Delta w_{j, m, t=\tau} g_{j, m, t=\tau} + t + \sin{\left(\sigma_{\hspace{-.05cm}g^{2}_{j,m}} \right)}}
\]
MSE: 2818.0700
Benchmark 29
\[
\frac{0.999 t \left(\cos{\left(\Delta w_{j, m, t=\tau} \right)} + 0.9\right)}{\Delta w_{j, m, t=\tau} g_{j, m, t=\tau} + y_{j} - 0.999}
\]
MSE: 3113.2000
Benchmark 30
\[
\frac{2.46 \left(0.5 t + 0.9\right)}{{0.999}^{g_{j, m, t=\tau}} - \mu_{j,m,t=\tau} + 0.9}
\]
MSE: 3236.6700