Skip to content

Commit e2f534c

Browse files
cbauguscbaugus
authored
Merge pull request #112 from cbaugus/dev
fix: workers running past duration + POST /config race condition (closes #109, #110)
2 parents cd7f3f8 + b0d6e0e commit e2f534c

6 files changed

Lines changed: 253 additions & 35 deletions

File tree

‎.github/workflows/release.yaml‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
name: Release
2+
3+
on:
4+
push:
5+
tags:
6+
- "v*.*.*"
7+
8+
env:
9+
REGISTRY: docker.io
10+
IMAGE_NAME: cbaugus/rust_loadtest
11+
12+
jobs:
13+
# ── 1. Lint ────────────────────────────────────────────────────────────────
14+
lint:
15+
name: Lint (rustfmt & clippy)
16+
runs-on: ubuntu-latest
17+
timeout-minutes: 10
18+
steps:
19+
- uses: actions/checkout@v4
20+
21+
- name: Install Rust toolchain
22+
uses: dtolnay/rust-toolchain@stable
23+
with:
24+
components: rustfmt, clippy
25+
26+
- name: Cache Rust dependencies
27+
uses: Swatinem/rust-cache@v2
28+
with:
29+
shared-key: lint
30+
31+
- name: Check formatting
32+
run: cargo fmt --all --check
33+
34+
- name: Run clippy
35+
run: cargo clippy --all-targets --all-features -- -D warnings
36+
37+
# ── 2. Test ────────────────────────────────────────────────────────────────
38+
test:
39+
name: Test Suite
40+
runs-on: ubuntu-latest
41+
timeout-minutes: 15
42+
steps:
43+
- uses: actions/checkout@v4
44+
45+
- name: Install Rust toolchain
46+
uses: dtolnay/rust-toolchain@stable
47+
48+
- name: Cache Rust dependencies
49+
uses: Swatinem/rust-cache@v2
50+
with:
51+
shared-key: test
52+
53+
- name: Run unit tests
54+
run: cargo test --lib --all-features --verbose -- --test-threads=1
55+
timeout-minutes: 10
56+
57+
- name: Run integration tests
58+
run: cargo test --test '*' --all-features --verbose -- --test-threads=1
59+
timeout-minutes: 10
60+
61+
# ── 3. Build & publish Docker images ──────────────────────────────────────
62+
build-and-release:
63+
name: Build, Push & Release
64+
runs-on: ubuntu-latest
65+
needs: [lint, test]
66+
timeout-minutes: 30
67+
permissions:
68+
contents: write # needed to create GitHub releases
69+
steps:
70+
- name: Checkout
71+
uses: actions/checkout@v4
72+
73+
# Strip the leading 'v' to get a bare semver (e.g. v1.5.3 → 1.5.3)
74+
- name: Derive version strings
75+
id: version
76+
run: |
77+
TAG="${{ github.ref_name }}"
78+
SEMVER="${TAG#v}"
79+
echo "tag=${TAG}" >> $GITHUB_OUTPUT
80+
echo "semver=${SEMVER}" >> $GITHUB_OUTPUT
81+
82+
- name: Set up QEMU
83+
uses: docker/setup-qemu-action@v3
84+
85+
- name: Set up Docker Buildx
86+
uses: docker/setup-buildx-action@v3
87+
88+
- name: Log in to Docker Hub
89+
uses: docker/login-action@v3
90+
with:
91+
username: ${{ secrets.DOCKERHUB_USERNAME }}
92+
password: ${{ secrets.DOCKERHUB_TOKEN }}
93+
94+
# ── Standard image ──────────────────────────────────────────────────
95+
- name: Build standard image (load for SBOM)
96+
uses: docker/build-push-action@v5
97+
with:
98+
context: .
99+
file: ./Dockerfile
100+
platforms: linux/amd64
101+
tags: ${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}
102+
push: false
103+
load: true
104+
cache-from: type=gha
105+
cache-to: type=gha,mode=max
106+
107+
# ── Chainguard image ─────────────────────────────────────────────────
108+
- name: Build Chainguard image (load for SBOM)
109+
uses: docker/build-push-action@v5
110+
with:
111+
context: .
112+
file: ./Dockerfile.chainguard
113+
platforms: linux/amd64
114+
tags: ${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}-Chainguard
115+
push: false
116+
load: true
117+
cache-from: type=gha
118+
119+
# ── SBOMs ─────────────────────────────────────────────────────────────
120+
- name: Install Syft
121+
run: curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
122+
123+
- name: Generate SBOM — standard
124+
run: |
125+
syft "docker:${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}" \
126+
-o cyclonedx-json > sbom-standard.cyclonedx.json
127+
128+
- name: Generate SBOM — Chainguard
129+
run: |
130+
syft "docker:${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}-Chainguard" \
131+
-o cyclonedx-json > sbom-chainguard.cyclonedx.json
132+
133+
# ── Push standard: version tag + latest ───────────────────────────────
134+
- name: Push standard image
135+
uses: docker/build-push-action@v5
136+
with:
137+
context: .
138+
file: ./Dockerfile
139+
platforms: linux/amd64
140+
tags: |
141+
${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}
142+
${{ env.IMAGE_NAME }}:latest
143+
provenance: true
144+
push: true
145+
cache-from: type=gha
146+
147+
# ── Push Chainguard: version tag + latest-Chainguard ─────────────────
148+
- name: Push Chainguard image
149+
uses: docker/build-push-action@v5
150+
with:
151+
context: .
152+
file: ./Dockerfile.chainguard
153+
platforms: linux/amd64
154+
tags: |
155+
${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}-Chainguard
156+
${{ env.IMAGE_NAME }}:latest-Chainguard
157+
provenance: true
158+
push: true
159+
cache-from: type=gha
160+
161+
# ── Update Docker Hub repository description ──────────────────────────
162+
- name: Update Docker Hub description
163+
uses: peter-evans/dockerhub-description@v4
164+
with:
165+
username: ${{ secrets.DOCKERHUB_USERNAME }}
166+
password: ${{ secrets.DOCKERHUB_TOKEN }}
167+
repository: ${{ env.IMAGE_NAME }}
168+
readme-filepath: ./DOCKER_HUB_OVERVIEW.md
169+
170+
# ── GitHub Release ────────────────────────────────────────────────────
171+
- name: Create GitHub Release
172+
uses: softprops/action-gh-release@v2
173+
with:
174+
tag_name: ${{ steps.version.outputs.tag }}
175+
name: ${{ steps.version.outputs.tag }}
176+
generate_release_notes: true
177+
files: |
178+
sbom-standard.cyclonedx.json
179+
sbom-chainguard.cyclonedx.json
180+
body: |
181+
## Docker images
182+
183+
| Variant | Pull command |
184+
|---------|-------------|
185+
| Standard (Ubuntu) | `docker pull ${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}` |
186+
| Chainguard (minimal) | `docker pull ${{ env.IMAGE_NAME }}:${{ steps.version.outputs.tag }}-Chainguard` |
187+
188+
`latest` and `latest-Chainguard` are also updated to this release.
189+
190+
## SBOMs
191+
CycloneDX SBOMs for both images are attached to this release.

‎src/client.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ pub struct ClientConfig {
1515
pub client_key_path: Option<String>,
1616
pub custom_headers: Option<String>,
1717
pub pool_config: Option<PoolConfig>,
18+
/// Enable per-request cookie jar (required for scenario session isolation).
19+
pub cookie_store: bool,
1820
}
1921

2022
/// Result of building the client, includes parsed headers for logging.
@@ -60,6 +62,11 @@ pub fn build_client(
6062
pool_config.max_idle_per_host, pool_config.idle_timeout
6163
);
6264

65+
// Cookie store for session isolation (scenario workers)
66+
if config.cookie_store {
67+
client_builder = client_builder.cookie_store(true);
68+
}
69+
6370
// Build client with TLS settings
6471
let client = if config.skip_tls_verify {
6572
println!("WARNING: Skipping TLS certificate verification.");

‎src/config.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -719,6 +719,7 @@ impl Config {
719719
client_key_path: self.client_key_path.clone(),
720720
custom_headers: self.custom_headers.clone(),
721721
pool_config: Some(crate::connection_pool::PoolConfig::from_env()),
722+
cookie_store: false,
722723
}
723724
}
724725

‎src/main.rs‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -714,12 +714,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
714714
// Startup standby config: fallback when a test YAML has no `standby:` block.
715715
// Nodes auto-revert to their startup state (typically TARGET_RPS=0) after a test ends.
716716
let startup_standby: Arc<StandbyRunConfig> = Arc::new(StandbyRunConfig {
717-
workers: config.num_concurrent_tasks,
718-
rps: if let LoadModel::Rps { target_rps } = &config.load_model {
719-
*target_rps
720-
} else {
721-
0.0
722-
},
717+
workers: 2,
718+
rps: 0.0,
723719
url: config.target_url.clone(),
724720
request_type: config.request_type.clone(),
725721
send_json: config.send_json,
@@ -1120,6 +1116,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
11201116
"Config submitted via POST /config — draining worker pool"
11211117
);
11221118

1119+
// Bump generation first — invalidates any in-flight completion watcher
1120+
// so it exits at Check 2 rather than re-spawning standby workers on top
1121+
// of the workers we are about to start.
1122+
{
1123+
let mut ts = test_state_for_watcher.lock().unwrap();
1124+
ts.generation += 1;
1125+
}
11231126
// Signal graceful stop (workers exit after current request).
11241127
{
11251128
let state = pool_for_watcher.lock().await;
@@ -1182,12 +1185,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
11821185
tenant: new_tenant.clone().unwrap_or_default(),
11831186
node_id: node_id_for_watcher.clone(),
11841187
run_id: new_run_id.clone(),
1188+
skip_tls_verify: new_cfg.skip_tls_verify,
1189+
resolve_target_addr: new_cfg.resolve_target_addr.clone(),
11851190
};
1186-
tokio::spawn(run_scenario_worker(
1187-
new_client.clone(),
1188-
sc,
1189-
new_start,
1190-
))
1191+
tokio::spawn(run_scenario_worker(sc, new_start))
11911192
})
11921193
.collect()
11931194
}
@@ -1474,7 +1475,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
14741475
(None, None, None)
14751476
};
14761477
(
1477-
remaining as i64,
1478+
(remaining as i64).max(0),
14781479
ts.yaml.clone(),
14791480
ts.node_state.to_string(),
14801481
started_at,

‎src/worker.rs‎

Lines changed: 32 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ fn should_sample(rate: u8) -> bool {
2020
counter % 100 < rate as u64
2121
}
2222

23+
use crate::client::{build_client, ClientConfig};
2324
use crate::connection_pool::GLOBAL_POOL_STATS;
2425
use crate::errors::ErrorCategory;
2526
use crate::executor::{ScenarioExecutor, SessionStore};
@@ -142,6 +143,8 @@ pub async fn run_worker(client: reqwest::Client, config: WorkerConfig, start_tim
142143
// immediately next iteration (Concurrent) or we set a long pause (0 RPS).
143144
if current_target_rps == 0.0 {
144145
next_fire = now + Duration::from_secs(3600);
146+
// rps=0 means idle standby — skip request entirely and wait for the next cycle.
147+
continue;
145148
}
146149
// For Concurrent (f64::MAX), next_fire stays in the past → fires immediately.
147150
}
@@ -373,6 +376,10 @@ pub struct ScenarioWorkerConfig {
373376
pub node_id: String,
374377
/// Run identifier (Issue #106). Unique per test dispatch.
375378
pub run_id: String,
379+
/// Skip TLS certificate verification (propagated from global config).
380+
pub skip_tls_verify: bool,
381+
/// DNS override string in `hostname:ip:port` format (propagated from global config).
382+
pub resolve_target_addr: Option<String>,
376383
}
377384

378385
/// Runs a scenario-based worker task that executes multi-step scenarios according to the load model.
@@ -382,13 +389,10 @@ pub struct ScenarioWorkerConfig {
382389
///
383390
/// # Cookie and Session Management
384391
///
385-
/// For proper session isolation, each scenario execution gets its own cookie-enabled
386-
/// HTTP client. This ensures cookies from one virtual user don't leak to another.
387-
pub async fn run_scenario_worker(
388-
_client: reqwest::Client, // Ignored - we create per-execution clients
389-
config: ScenarioWorkerConfig,
390-
start_time: Instant,
391-
) {
392+
/// Each scenario execution gets its own cookie-enabled HTTP client built from the
393+
/// worker config (DNS override, TLS settings). This ensures cookies from one virtual
394+
/// user don't leak to another while preserving global client settings.
395+
pub async fn run_scenario_worker(config: ScenarioWorkerConfig, start_time: Instant) {
392396
debug!(
393397
task_id = config.task_id,
394398
scenario = %config.scenario.name,
@@ -416,6 +420,23 @@ pub async fn run_scenario_worker(
416420
// subsequent iterations skip the HTTP request until the TTL expires.
417421
let mut session = SessionStore::new();
418422

423+
// Build the HTTP client once per worker with DNS override, TLS, and cookie store enabled.
424+
// Building once avoids log flooding and expensive reconstruction on every loop iteration.
425+
let worker_client = build_client(&ClientConfig {
426+
skip_tls_verify: config.skip_tls_verify,
427+
resolve_target_addr: config.resolve_target_addr.clone(),
428+
client_cert_path: None,
429+
client_key_path: None,
430+
custom_headers: None,
431+
pool_config: None,
432+
cookie_store: true,
433+
})
434+
.map(|r| r.client)
435+
.unwrap_or_else(|e| {
436+
error!(error = %e, "Failed to build scenario worker client; falling back to default");
437+
reqwest::Client::new()
438+
});
439+
419440
loop {
420441
time::sleep_until(next_fire).await;
421442

@@ -444,20 +465,14 @@ pub async fn run_scenario_worker(
444465
next_fire += Duration::from_millis(cycle_ms);
445466
} else if current_target_sps == 0.0 {
446467
next_fire = now + Duration::from_secs(3600);
468+
// rps=0 means idle standby — skip scenario execution entirely and wait for the next cycle.
469+
continue;
447470
}
448471

449-
// Create new cookie-enabled client for this virtual user
450-
// This ensures cookie isolation between scenario executions
451-
let client = reqwest::Client::builder()
452-
.cookie_store(true) // Enable automatic cookie management
453-
.timeout(std::time::Duration::from_secs(30))
454-
.build()
455-
.unwrap_or_else(|_| reqwest::Client::new());
456-
457-
// Create executor with isolated client
472+
// Create executor with the worker's configured client
458473
let executor = ScenarioExecutor::new(
459474
config.base_url.clone(),
460-
client,
475+
worker_client.clone(),
461476
config.node_id.clone(),
462477
config.run_id.clone(),
463478
);

0 commit comments

Comments
 (0)