Skip to content

Commit 7f27857

Browse files
committed
Fixing Adaboost algorithm
1 parent e318b88 commit 7f27857

File tree

1 file changed

+73
-30
lines changed

1 file changed

+73
-30
lines changed

machine_learning/adaboost.py

Lines changed: 73 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -18,73 +18,103 @@
1818

1919
class AdaBoost:
2020
def __init__(self, n_estimators: int = 50) -> None:
21-
"""Initialize AdaBoost classifier.
21+
"""
22+
Initialize AdaBoost classifier.
23+
2224
Args:
23-
n_estimators: Number of boosting rounds.
25+
n_estimators: Number of boosting rounds (weak learners).
2426
"""
2527
self.n_estimators: int = n_estimators
26-
self.alphas: list[float] = [] # Weights for each weak learner
27-
self.models: list[dict[str, Any]] = [] # List of weak learners (stumps)
28+
self.alphas: list[float] = [] # Weights assigned to each weak learner
29+
self.models: list[dict[str, Any]] = [] # Stores each decision stump
2830

2931
def fit(self, feature_matrix: np.ndarray, target: np.ndarray) -> None:
30-
"""Fit AdaBoost model.
32+
"""
33+
Train AdaBoost model using decision stumps.
34+
3135
Args:
32-
feature_matrix: (n_samples, n_features) feature matrix
33-
target: (n_samples,) labels (0 or 1)
36+
feature_matrix: 2D array of shape (n_samples, n_features)
37+
target: 1D array of binary labels (0 or 1)
3438
"""
35-
n_samples, _n_features = feature_matrix.shape
36-
sample_weights = np.ones(n_samples) / n_samples # Initialize sample weights
39+
n_samples, _ = feature_matrix.shape
40+
41+
# Initialize uniform sample weights
42+
sample_weights = np.ones(n_samples) / n_samples
43+
44+
# Reset model state
3745
self.models = []
3846
self.alphas = []
39-
y_signed = np.where(target == 0, -1, 1) # Convert labels to -1, 1
47+
48+
# Convert labels to {-1, 1} for boosting
49+
y_signed = np.where(target == 0, -1, 1)
50+
4051
for _ in range(self.n_estimators):
41-
# Train a decision stump with weighted samples
52+
# Train a weighted decision stump
4253
stump = self._build_stump(feature_matrix, y_signed, sample_weights)
4354
pred = stump["pred"]
4455
err = stump["error"]
45-
# Compute alpha (learner weight)
56+
57+
# Compute alpha (learner weight) with numerical stability
4658
alpha = 0.5 * np.log((1 - err) / (err + 1e-10))
47-
# Update sample weights
59+
60+
# Update sample weights to focus on misclassified points
4861
sample_weights *= np.exp(-alpha * y_signed * pred)
4962
sample_weights /= np.sum(sample_weights)
63+
64+
# Store the stump and its weight
5065
self.models.append(stump)
5166
self.alphas.append(alpha)
5267

5368
def predict(self, feature_matrix: np.ndarray) -> np.ndarray:
54-
"""Predict class labels for samples in feature_matrix.
69+
"""
70+
Predict binary class labels for input samples.
71+
5572
Args:
56-
feature_matrix: (n_samples, n_features) feature matrix
73+
feature_matrix: 2D array of shape (n_samples, n_features)
74+
5775
Returns:
58-
(n_samples,) predicted labels (0 or 1)
59-
>>> import numpy as np
60-
>>> features = np.array([[0, 0], [1, 1], [1, 0], [0, 1]])
61-
>>> labels = np.array([0, 1, 1, 0])
62-
>>> clf = AdaBoost(n_estimators=5)
63-
>>> clf.fit(features, labels)
64-
>>> clf.predict(np.array([[0, 0], [1, 1]]))
65-
array([0, 1])
76+
1D array of predicted labels (0 or 1)
6677
"""
6778
clf_preds = np.zeros(feature_matrix.shape[0])
79+
80+
# Aggregate predictions from all stumps
6881
for alpha, stump in zip(self.alphas, self.models):
6982
pred = self._stump_predict(
70-
feature_matrix, stump["feature"], stump["threshold"], stump["polarity"]
83+
feature_matrix,
84+
stump["feature"],
85+
stump["threshold"],
86+
stump["polarity"],
7187
)
7288
clf_preds += alpha * pred
89+
90+
# Final prediction: sign of weighted sum
7391
return np.where(clf_preds >= 0, 1, 0)
7492

7593
def _build_stump(
76-
self, feature_matrix: np.ndarray, target_signed: np.ndarray, sample_weights: np.ndarray
94+
self,
95+
feature_matrix: np.ndarray,
96+
target_signed: np.ndarray,
97+
sample_weights: np.ndarray,
7798
) -> dict[str, Any]:
78-
"""Find the best decision stump for current weights."""
79-
_n_samples, n_features = feature_matrix.shape
99+
"""
100+
Build the best decision stump for current sample weights.
101+
102+
Returns:
103+
Dictionary containing stump parameters and predictions.
104+
"""
105+
_, n_features = feature_matrix.shape
80106
min_error = float("inf")
81107
best_stump: dict[str, Any] = {}
108+
109+
# Iterate over all features and thresholds
82110
for feature in range(n_features):
83111
thresholds = np.unique(feature_matrix[:, feature])
84112
for threshold in thresholds:
85113
for polarity in [1, -1]:
86114
pred = self._stump_predict(feature_matrix, feature, threshold, polarity)
87115
error = np.sum(sample_weights * (pred != target_signed))
116+
117+
# Keep stump with lowest weighted error
88118
if error < min_error:
89119
min_error = error
90120
best_stump = {
@@ -94,15 +124,28 @@ def _build_stump(
94124
"error": error,
95125
"pred": pred.copy(),
96126
}
127+
97128
return best_stump
98129

99130
def _stump_predict(
100-
self, feature_matrix: np.ndarray, feature: int, threshold: float, polarity: int
131+
self,
132+
feature_matrix: np.ndarray,
133+
feature: int,
134+
threshold: float,
135+
polarity: int,
101136
) -> np.ndarray:
102-
"""Predict using a single decision stump."""
137+
"""
138+
Predict using a single decision stump.
139+
140+
Returns:
141+
1D array of predictions in {-1, 1}
142+
"""
103143
pred = np.ones(feature_matrix.shape[0])
144+
145+
# Apply polarity to threshold comparison
104146
if polarity == 1:
105147
pred[feature_matrix[:, feature] < threshold] = -1
106148
else:
107149
pred[feature_matrix[:, feature] > threshold] = -1
108-
return pred
150+
151+
return pred

0 commit comments

Comments
 (0)