Cross-Entropy Loss

1
#include <bits/stdc++.h>
2
using namespace std;
3
4
// Compute softmax probabilities using the log-sum-exp trick for stability.
5
vector<double> softmax(const vector<double>& logits) {
6
    double m = *max_element(logits.begin(), logits.end());
7
    double sumExp = 0.0;
8
    vector<double> exps(logits.size());
9
    for (size_t i = 0; i < logits.size(); ++i) {
10
        exps[i] = exp(logits[i] - m);
11
        sumExp += exps[i];
12
    }
13
    vector<double> probs(logits.size());
14
    for (size_t i = 0; i < logits.size(); ++i) probs[i] = exps[i] / sumExp;
15
    return probs;
16
}
17
18
// Cross-entropy from probabilities and one-hot label.
19
double cross_entropy_from_probs(const vector<double>& probs, const vector<int>& one_hot) {
20
    const double eps = 1e-15; // avoid log(0)
21
    if (probs.size() != one_hot.size()) throw runtime_error("Size mismatch");
22
    double loss = 0.0;
23
    for (size_t i = 0; i < probs.size(); ++i) {
24
        if (one_hot[i]) {
25
            loss = -log(max(probs[i], eps));
26
            break; // one-hot has a single 1
27
        }
28
    }
29
    return loss;
30
}
31
32
// Cross-entropy directly from logits and true class index using stable log-sum-exp.
33
double cross_entropy_from_logits(const vector<double>& logits, int y) {
34
    if (y < 0 || y >= (int)logits.size()) throw runtime_error("Label out of range");
35
    double m = *max_element(logits.begin(), logits.end());
36
    double sumExp = 0.0;
37
    for (double z : logits) sumExp += exp(z - m);
38
    // L = -(z_y - m) + log(sum_j exp(z_j - m))
39
    return -(logits[y] - m) + log(sumExp);
40
}
41
42
int main() {
43
    // Example: 4-class problem, logits for one sample
44
    vector<double> logits = {2.3, -1.2, 0.7, 1.1};
45
    int true_class = 0; // class index 0..3
46
47
    vector<double> probs = softmax(logits);
48
    vector<int> one_hot = {1, 0, 0, 0};
49
50
    double ce_probs = cross_entropy_from_probs(probs, one_hot);
51
    double ce_logits = cross_entropy_from_logits(logits, true_class);
52
53
    cout << fixed << setprecision(6);
54
    cout << "Probabilities: ";
55
    for (double p : probs) cout << p << ' ';
56
    cout << "\nCE (from probs): " << ce_probs << "\nCE (from logits): " << ce_logits << "\n";
57
    return 0;
58
}
59

1
#include <bits/stdc++.h>
2
using namespace std;
3
4
// Numerically stable sigmoid
5
static inline double sigmoid(double z) {
6
    if (z >= 0) {
7
        double ez = exp(-z);
8
        return 1.0 / (1.0 + ez);
9
    } else {
10
        double ez = exp(z);
11
        return ez / (1.0 + ez);
12
    }
13
}
14
15
// Stable BCE computed directly from logit z and label y in {0,1}
16
static inline double bce_from_logit(double z, int y) {
17
    // L = max(z,0) - z*y + log(1 + exp(-|z|))
18
    double a = fabs(z);
19
    return max(z, 0.0) - z * (double)y + log1p(exp(-a));
20
}
21
22
// Gradient dL/dz = sigmoid(z) - y
23
static inline double bce_grad_wrt_logit(double z, int y) {
24
    return sigmoid(z) - (double)y;
25
}
26
27
int main() {
28
    vector<double> logits = {5.0, -5.0, 0.0, 2.0, -2.0};
29
    vector<int> labels = {1, 0, 1, 0, 1};
30
31
    cout << fixed << setprecision(6);
32
    for (size_t i = 0; i < logits.size(); ++i) {
33
        double z = logits[i];
34
        int y = labels[i];
35
        double loss = bce_from_logit(z, y);
36
        double grad = bce_grad_wrt_logit(z, y);
37
        double p = sigmoid(z);
38
        cout << "z=" << z << ", y=" << y
39
             << ", p(sigmoid)=" << p
40
             << ", loss(BCE)=" << loss
41
             << ", grad(dL/dz)=" << grad << "\n";
42
    }
43
    return 0;
44
}
45

1
#include <bits/stdc++.h>
2
using namespace std;
3
4
// Utilities for stable softmax and loss
5
vector<double> softmax(const vector<double>& z) {
6
    double m = *max_element(z.begin(), z.end());
7
    double sumExp = 0.0;
8
    vector<double> e(z.size());
9
    for (size_t i = 0; i < z.size(); ++i) { e[i] = exp(z[i] - m); sumExp += e[i]; }
10
    for (size_t i = 0; i < z.size(); ++i) e[i] /= sumExp;
11
    return e;
12
}
13
14
double nll_from_logits(const vector<double>& z, int y) {
15
    double m = *max_element(z.begin(), z.end());
16
    double sumExp = 0.0;
17
    for (double zi : z) sumExp += exp(zi - m);
18
    return -(z[y] - m) + log(sumExp);
19
}
20
21
// Matrix-vector multiply: y = W x + b, W: k x d, x: d
22
vector<double> affine_logits(const vector<vector<double>>& W, const vector<double>& b, const vector<double>& x) {
23
    size_t k = W.size(), d = x.size();
24
    vector<double> z(k);
25
    for (size_t i = 0; i < k; ++i) {
26
        double s = b[i];
27
        for (size_t j = 0; j < d; ++j) s += W[i][j] * x[j];
28
        z[i] = s;
29
    }
30
    return z;
31
}
32
33
int main() {
34
    // Create a toy 2D, 3-class dataset (linearly separable-ish)
35
    std::mt19937 rng(42);
36
    normal_distribution<double> noise(0.0, 0.2);
37
    vector<vector<double>> X; vector<int> Y;
38
    int n_per_class = 60; // total n = 180
39
    vector<vector<double>> centers = {{2,2}, {-2,0}, {0,-2}};
40
    for (int c = 0; c < 3; ++c) {
41
        for (int i = 0; i < n_per_class; ++i) {
42
            vector<double> x = {centers[c][0] + noise(rng), centers[c][1] + noise(rng)};
43
            X.push_back(x); Y.push_back(c);
44
        }
45
    }
46
47
    // Model: W (k x d), b (k)
48
    size_t k = 3, d = 2;
49
    vector<vector<double>> W(k, vector<double>(d));
50
    vector<double> b(k, 0.0);
51
    // Initialize small random weights
52
    uniform_real_distribution<double> uni(-0.01, 0.01);
53
    for (size_t i = 0; i < k; ++i) for (size_t j = 0; j < d; ++j) W[i][j] = uni(rng);
54
55
    double lr = 0.1; int epochs = 50; int n = (int)X.size();
56
57
    for (int ep = 1; ep <= epochs; ++ep) {
58
        // Shuffle data each epoch
59
        vector<int> idx(n); iota(idx.begin(), idx.end(), 0);
60
        shuffle(idx.begin(), idx.end(), rng);
61
62
        double epoch_loss = 0.0;
63
        // Simple SGD with batch size 1 (for brevity)
64
        for (int t = 0; t < n; ++t) {
65
            int i = idx[t];
66
            const vector<double>& x = X[i];
67
            int y = Y[i];
68
            // Forward
69
            vector<double> z = affine_logits(W, b, x);
70
            vector<double> p = softmax(z);
71
            double loss = -log(max(1e-15, p[y]));
72
            epoch_loss += loss;
73
            // Gradients: dL/dz = p - y_onehot
74
            vector<double> dz = p; dz[y] -= 1.0; // size k
75
            // Update W and b: dL/dW = dz * x^T, dL/db = dz
76
            for (size_t r = 0; r < k; ++r) {
77
                for (size_t j = 0; j < d; ++j) W[r][j] -= lr * dz[r] * x[j];
78
                b[r] -= lr * dz[r];
79
            }
80
        }
81
        cout << "Epoch " << ep << ": avg loss = " << (epoch_loss / n) << "\n";
82
    }
83
84
    // Evaluate accuracy
85
    int correct = 0;
86
    for (int i = 0; i < n; ++i) {
87
        vector<double> z = affine_logits(W, b, X[i]);
88
        vector<double> p = softmax(z);
89
        int pred = (int)(max_element(p.begin(), p.end()) - p.begin());
90
        if (pred == Y[i]) ++correct;
91
    }
92
    cout << "Training accuracy: " << (100.0 * correct / n) << "%\n";
93
    return 0;
94
}
95