Learning Rate Schedules

1
#include <bits/stdc++.h>
2
using namespace std;
3
4
// Step Decay scheduler: eta_t = eta0 * gamma^{floor(t / step_size)}
5
struct StepDecay {
6
    double eta0;     // initial learning rate
7
    double gamma;    // decay factor (0 < gamma < 1)
8
    int step_size;   // drop every step_size steps
9
    StepDecay(double eta0_, double gamma_, int step_size_)
10
        : eta0(eta0_), gamma(gamma_), step_size(step_size_) {}
11
    double operator()(int t) const {
12
        int k = t / step_size; // number of decays so far
13
        // pow with small integer exponent is fine; could precompute if desired
14
        return eta0 * pow(gamma, k);
15
    }
16
};
17
18
// Minimize f(x) = x^2 with gradient 2x using SGD-like updates
19
int main() {
20
    // Hyperparameters
21
    const double x_init = 10.0;    // start far from optimum (0)
22
    const int T = 100;             // total steps
23
    StepDecay sched(0.2, 0.1, 30); // eta0=0.2, drop x10 every 30 steps
24
25
    double x = x_init;
26
    for (int t = 0; t < T; ++t) {
27
        double lr = sched(t);
28
        double grad = 2.0 * x; // derivative of x^2
29
        x -= lr * grad;        // SGD update: x := x - lr * grad
30
        if (t % 10 == 0) {
31
            cout << "t=" << t << ", lr=" << lr << ", x=" << x 
32
                 << ", f(x)=" << x * x << "\n";
33
        }
34
    }
35
    cout << "Final x=" << x << ", f(x)=" << x * x << "\n";
36
    return 0;
37
}
38

1
#include <bits/stdc++.h>
2
using namespace std;
3
4
struct LinearWarmup {
5
    double eta_start, eta_target; int T_warm;
6
    LinearWarmup(double s, double t, int Tw): eta_start(s), eta_target(t), T_warm(Tw) {}
7
    double operator()(int t) const {
8
        if (T_warm <= 0) return eta_target;
9
        if (t >= T_warm) return eta_target;
10
        double alpha = static_cast<double>(t) / max(1, T_warm);
11
        return eta_start + alpha * (eta_target - eta_start);
12
    }
13
};
14
15
struct CosineAnnealing {
16
    double eta_max, eta_min; int T_max;
17
    CosineAnnealing(double mx, double mn, int T): eta_max(mx), eta_min(mn), T_max(T) {}
18
    double operator()(int t) const {
19
        int tt = min(t, T_max);
20
        double cosv = cos(M_PI * (static_cast<double>(tt) / max(1, T_max)));
21
        return eta_min + 0.5 * (eta_max - eta_min) * (1.0 + cosv);
22
    }
23
};
24
25
// Compose: use warmup for first T_warm steps, then cosine (with time offset)
26
struct WarmupThenCosine {
27
    LinearWarmup warm; CosineAnnealing cos; int T_warm;
28
    WarmupThenCosine(LinearWarmup w, CosineAnnealing c)
29
        : warm(w), cos(c), T_warm(w.T_warm) {}
30
    double operator()(int t) const {
31
        if (t <= T_warm) return warm(t);
32
        return cos(t - T_warm);
33
    }
34
};
35
36
int main() {
37
    // Settings: warm up to 0.1 in 20 steps, then cosine down to 1e-4 over 200 steps
38
    WarmupThenCosine sched(
39
        LinearWarmup(0.0, 0.1, 20),
40
        CosineAnnealing(0.1, 1e-4, 200)
41
    );
42
43
    double x = 10.0; // minimize f(x)=x^2
44
    const int T_total = 250;
45
    for (int t = 0; t < T_total; ++t) {
46
        double lr = sched(t);
47
        double grad = 2.0 * x;
48
        x -= lr * grad;
49
        if (t % 25 == 0) {
50
            cout << "t=" << t << ", lr=" << lr << ", x=" << x 
51
                 << ", f(x)=" << x * x << "\n";
52
        }
53
    }
54
    cout << "Final x=" << x << ", f(x)=" << x * x << "\n";
55
    return 0;
56
}
57

1
#include <bits/stdc++.h>
2
using namespace std;
3
4
// Cosine with warm restarts: cycles of length T_i; within a cycle use cosine from eta_max to eta_min
5
struct CosineWithRestarts {
6
    double eta_max, eta_min; 
7
    vector<int> cycle_lengths; // e.g., {50, 50, 100} or geometric growth
8
    CosineWithRestarts(double mx, double mn, vector<int> L): eta_max(mx), eta_min(mn), cycle_lengths(move(L)) {}
9
    double operator()(int t) const {
10
        int acc = 0;
11
        for (size_t i = 0; i < cycle_lengths.size(); ++i) {
12
            int Ti = cycle_lengths[i];
13
            if (t < acc + Ti) {
14
                int tprime = t - acc;
15
                double cosv = cos(M_PI * (static_cast<double>(tprime) / max(1, Ti)));
16
                return eta_min + 0.5 * (eta_max - eta_min) * (1.0 + cosv);
17
            }
18
            acc += Ti;
19
        }
20
        // If beyond provided cycles, stay at eta_min (or repeat last cycle pattern)
21
        return eta_min;
22
    }
23
};
24
25
int main() {
26
    CosineWithRestarts sched(0.1, 1e-4, {50, 50, 100}); // three cycles
27
28
    // Print the LR curve to visualize cycles
29
    for (int t = 0; t < 220; ++t) {
30
        double lr = sched(t);
31
        if (t % 10 == 0) {
32
            cout << "t=" << t << ", lr=" << lr << "\n";
33
        }
34
    }
35
    return 0;
36
}
37