Skip to content

Commit c1ff9b4

Browse files
bit-web24bit-web24
committed
chore(load-test): stabilize auth load testing and K8s local setup
This commit introduces several adjustments to enable local load testing without saturating local resources or hitting false negatives. ### Auth Service - **Async BCrypt:** Wrapped all `bcrypt` hashing operations in `tokio::task::spawn_blocking` to prevent CPU-bound tasks from blocking the tokio runtime under high concurrency. - **Dynamic Cost:** Lowered the BCrypt cost to `4` when `MODE=development` to prevent CPU starvation on local machines during heavy load tests, while defaulting to `10` in production. - **OTP Debug Store:** Exposed `/api/auth/debug/otp-store` strictly in development mode to allow `k6` scripts to fetch OTPs in memory and provision users dynamically. ### URL Service - **Cache Fix:** Migrated from Moka's `get_with` to `try_get_with` in `get_original_url` to ensure transient database errors aren't incorrectly cached as 404s. ### K8s & Infrastructure - **Ingress:** Temporarily dropped the `limit-rps` annotation to prevent rate limits from interfering with raw load test measurements, and improved path routing. - **ConfigMap:** Flipped the default MODE to `development`. - **k8s.sh Deploy Script:** Added image caching commands to preload third-party dependencies (Postgres, Redis, RabbitMQ, Jaeger) into `kind`, dramatically reducing local cluster boot times. Changed wait checks to wait on deployments instead of individual pods. ### Load Tests (k6) - **auth.js:** Added a new load test explicitly for stressing the login and token generation endpoints, ensuring SLI adherence. - **redirects.js & shorten.js:** Adjusted test setups to autonomously provision VUs utilizing the new development OTP endpoint, generating valid JWTs for the tests.
1 parent 9bbc990 commit c1ff9b4

9 files changed

Lines changed: 691 additions & 93 deletions

File tree

‎k8s/base/configmap.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ metadata:
55
namespace: bittuly
66
data:
77
# Shared
8-
MODE: "production"
8+
MODE: "development"
99
RUST_LOG: "auth_service=info,url_service=info,consumer_service=info,shared=info"
1010

1111
# Auth service

‎k8s/base/ingress.yaml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@ metadata:
55
namespace: bittuly
66
annotations:
77
nginx.ingress.kubernetes.io/use-regex: "true"
8-
# Rate limiting (increased for load testing, was 20)
9-
nginx.ingress.kubernetes.io/limit-rps: "2000"
108
spec:
119
ingressClassName: nginx
1210
rules:
@@ -26,6 +24,13 @@ spec:
2624
name: url-service
2725
port:
2826
number: 3002
27+
- path: /(?!(api|login|signup|verify-otp|dashboard|insights|profile|settings|health|unavailable|assets))([a-zA-Z0-9_-]{3,8})
28+
pathType: ImplementationSpecific
29+
backend:
30+
service:
31+
name: url-service
32+
port:
33+
number: 3002
2934
- path: /
3035
pathType: Prefix
3136
backend:

‎scripts/k8s.sh‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,7 @@ cmd_up() {
4646
log "Installing NGINX Ingress Controller..."
4747
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
4848
kubectl wait --namespace ingress-nginx \
49-
--for=condition=ready pod \
50-
--selector=app.kubernetes.io/component=controller \
49+
--for=condition=available deployment/ingress-nginx-controller \
5150
--timeout=120s
5251

5352
log "Building and deploying all services..."
@@ -62,12 +61,18 @@ cmd_deploy() {
6261
docker build -f services/consumer-service/Dockerfile -t bittuly/consumer-service:local .
6362
docker build -f web/Dockerfile -t bittuly/frontend-service:local web/
6463

65-
log "Loading images into kind..."
64+
log "Loading custom application images into kind..."
6665
kind load docker-image bittuly/auth-service:local --name "${CLUSTER_NAME}"
6766
kind load docker-image bittuly/url-service:local --name "${CLUSTER_NAME}"
6867
kind load docker-image bittuly/consumer-service:local --name "${CLUSTER_NAME}"
6968
kind load docker-image bittuly/frontend-service:local --name "${CLUSTER_NAME}"
7069

70+
log "Pre-loading third-party dependencies from local Docker cache to speed up boot..."
71+
for img in postgres:17-alpine redis:7-alpine rabbitmq:3-management jaegertracing/all-in-one:latest; do
72+
docker pull "$img" >/dev/null 2>&1 || true
73+
kind load docker-image "$img" --name "${CLUSTER_NAME}" 2>/dev/null || true
74+
done
75+
7176
log "Applying Kubernetes manifests..."
7277
kubectl apply -f k8s/base/namespace.yaml
7378
kubectl apply -f k8s/base/configmap.yaml
@@ -81,8 +86,8 @@ cmd_deploy() {
8186
kubectl apply -f k8s/base/jaeger/jaeger.yaml
8287

8388
log "Waiting for databases to be ready..."
84-
kubectl wait -n "${NAMESPACE}" --for=condition=ready pod --selector=app=postgres-auth --timeout=120s
85-
kubectl wait -n "${NAMESPACE}" --for=condition=ready pod --selector=app=postgres-urls --timeout=120s
89+
kubectl wait -n "${NAMESPACE}" --for=condition=available deployment/postgres-auth --timeout=300s
90+
kubectl wait -n "${NAMESPACE}" --for=condition=available deployment/postgres-urls --timeout=300s
8691

8792
kubectl apply -f k8s/base/auth-service/auth-service.yaml
8893
kubectl apply -f k8s/base/url-service/url-service.yaml

‎services/auth-service/src/routes.rs‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,25 @@ pub fn auth_routes() -> Router<AuthStateRef> {
1818
.route("/logout", post(logout))
1919
.layer(middleware::from_fn(jwt_auth));
2020

21-
Router::new()
21+
let mut router = Router::new()
2222
.route("/signup", post(request_signup_handler)) // Step 1: send OTP
2323
.route("/verify-otp", post(verify_otp_handler)) // Step 2: verify OTP → create user + JWT
2424
.route("/login", post(login))
2525
.route("/health", get(health))
2626
.route("/metrics", get(metrics))
2727
.merge(protected)
2828
.fallback(|| async { axum::http::StatusCode::NOT_FOUND })
29-
.layer(middleware::from_fn(shared::metrics::track_metrics))
29+
.layer(middleware::from_fn(shared::metrics::track_metrics));
30+
31+
// Only expose the OTP debug endpoint in development mode.
32+
// This lets load tests retrieve OTPs programmatically without real emails.
33+
// NEVER present in production builds.
34+
if std::env::var("MODE").unwrap_or_default() == "development" {
35+
router = router.route(
36+
"/debug/otp-store",
37+
get(crate::debug_handler::debug_otp_store_handler),
38+
);
39+
}
40+
41+
router
3042
}

‎services/auth-service/src/services.rs‎

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,21 @@ use uuid::Uuid;
99

1010
type ServiceResult<T> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
1111

12+
#[inline]
13+
fn get_bcrypt_cost() -> u32 {
14+
if std::env::var("MODE").unwrap_or_default() == "development" {
15+
4 // Minimum cost for fast local load testing without saturating CPU
16+
} else {
17+
10 // Production cost (12 is default, but 10 meets our 400ms SLO under moderate load better)
18+
}
19+
}
20+
1221
#[allow(dead_code)]
1322
pub async fn create_user(
1423
repo: &dyn UserRepo,
1524
mut payload: CreateUserPayload,
1625
) -> ServiceResult<AuthUserResponse> {
17-
payload.password = bcrypt::hash(&payload.password, bcrypt::DEFAULT_COST)?;
26+
payload.password = bcrypt::hash(&payload.password, get_bcrypt_cost())?;
1827
let user = repo.create_user(payload).await?;
1928
let token = create_access_token(user.id)?;
2029
let refresh_token = create_refresh_token(user.id)?;
@@ -44,13 +53,20 @@ pub async fn request_signup(
4453
}
4554

4655
// Hash the password before it goes anywhere near a JWT.
47-
payload.password = bcrypt::hash(&payload.password, bcrypt::DEFAULT_COST)?;
56+
let plain_password = payload.password.clone();
57+
payload.password =
58+
tokio::task::spawn_blocking(move || bcrypt::hash(&plain_password, get_bcrypt_cost()))
59+
.await
60+
.map_err(|_| "task panicked")??;
4861

4962
// Generate a 6-digit OTP and bcrypt-hash it for the pending token.
5063
let otp: String = rand::rng()
5164
.random_range(100_000u32..=999_999u32)
5265
.to_string();
53-
let otp_hash = bcrypt::hash(&otp, bcrypt::DEFAULT_COST)?;
66+
let otp_clone = otp.clone();
67+
let otp_hash = tokio::task::spawn_blocking(move || bcrypt::hash(&otp_clone, get_bcrypt_cost()))
68+
.await
69+
.map_err(|_| "task panicked")??;
5470

5571
// Fire the email first — if it fails we return early without issuing a token.
5672
send_otp_email(&payload.email, &otp).await?;
@@ -79,7 +95,13 @@ pub async fn verify_otp(
7995
let claims = decode_pending_token(pending_token)?;
8096

8197
// Verify the submitted OTP against the hashed one inside the token.
82-
if !bcrypt::verify(otp, &claims.otp_hash)? {
98+
let otp_clone = otp.to_string();
99+
let hash_clone = claims.otp_hash.clone();
100+
let is_valid = tokio::task::spawn_blocking(move || bcrypt::verify(&otp_clone, &hash_clone))
101+
.await
102+
.map_err(|_| "task panicked")??;
103+
104+
if !is_valid {
83105
return Err("invalid OTP".into());
84106
}
85107

@@ -111,7 +133,14 @@ pub async fn login(
111133
.await?
112134
.ok_or("invalid credentials")?;
113135

114-
if !bcrypt::verify(password, &user.password)? {
136+
let password_clone = password.to_string();
137+
let hash_clone = user.password.clone();
138+
let is_valid =
139+
tokio::task::spawn_blocking(move || bcrypt::verify(&password_clone, &hash_clone))
140+
.await
141+
.map_err(|_| "task panicked")??;
142+
143+
if !is_valid {
115144
return Err("invalid credentials".into());
116145
}
117146

@@ -138,7 +167,12 @@ pub async fn update_user(
138167
mut payload: UpdateUserPayload,
139168
) -> ServiceResult<AuthUserResponse> {
140169
if let Some(ref plain) = payload.password {
141-
payload.password = Some(bcrypt::hash(plain, bcrypt::DEFAULT_COST)?);
170+
let plain_clone = plain.clone();
171+
payload.password = Some(
172+
tokio::task::spawn_blocking(move || bcrypt::hash(&plain_clone, get_bcrypt_cost()))
173+
.await
174+
.map_err(|_| "task panicked")??,
175+
);
142176
}
143177
let user = repo.update_user(user_id, payload).await?;
144178
let token = create_access_token(user.id)?;
@@ -178,7 +212,7 @@ mod tests {
178212
repo.create_user(CreateUserPayload {
179213
username: "testuser".into(),
180214
email: "test@example.com".into(),
181-
password: bcrypt::hash("password123", bcrypt::DEFAULT_COST).unwrap(),
215+
password: bcrypt::hash("password123", get_bcrypt_cost()).unwrap(),
182216
})
183217
.await
184218
.unwrap();
@@ -199,7 +233,7 @@ mod tests {
199233
repo.create_user(CreateUserPayload {
200234
username: "testuser".into(),
201235
email: "test@example.com".into(),
202-
password: bcrypt::hash("password123", bcrypt::DEFAULT_COST).unwrap(),
236+
password: bcrypt::hash("password123", get_bcrypt_cost()).unwrap(),
203237
})
204238
.await
205239
.unwrap();

‎services/url-service/src/handlers.rs‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ pub async fn get_original_url(
112112
// Singleflight Cache Coalescing (Thundering Herd Protection)
113113
let result = state
114114
.l1_cache
115-
.get_with(short_code.clone(), async {
115+
.try_get_with(short_code.clone(), async {
116116
let mut redis = state.redis.clone();
117117

118118
let cached: Option<String> = {
@@ -128,7 +128,7 @@ pub async fn get_original_url(
128128

129129
if let Some(original_url) = cached {
130130
metrics::counter!("cache_hits").increment(1);
131-
return Some((original_url, None));
131+
return Ok::<_, axum::http::StatusCode>(Some((original_url, None)));
132132
}
133133

134134
// 2. Try DB
@@ -154,16 +154,20 @@ pub async fn get_original_url(
154154
.unwrap_or_default();
155155
}
156156
});
157-
Some((original_url, expires_at))
157+
Ok(Some((original_url, expires_at)))
158+
}
159+
Ok(None) => Ok(None), // True 404 Not Found
160+
Err(e) => {
161+
tracing::error!("DB error during redirect: {:?}", e);
162+
Err(axum::http::StatusCode::SERVICE_UNAVAILABLE)
158163
}
159-
_ => None, // Not found or error
160164
}
161165
})
162166
.await;
163167

164168
// Process Result
165169
match result {
166-
Some((original_url, expires_at)) => {
170+
Ok(Some((original_url, expires_at))) => {
167171
// Re-verify expiration in case it expired while sitting in the 3-second L1 microcache
168172
if let Some(exp) = expires_at
169173
&& exp < chrono::Utc::now()
@@ -210,7 +214,12 @@ pub async fn get_original_url(
210214

211215
Redirect::temporary(&original_url).into_response()
212216
}
213-
None => StatusCode::NOT_FOUND.into_response(),
217+
Ok(None) => StatusCode::NOT_FOUND.into_response(),
218+
Err(e) => {
219+
// Moka wraps the error in Arc
220+
let status = *e;
221+
status.into_response()
222+
}
214223
}
215224
}
216225

0 commit comments

Comments
 (0)