Skip to content

feat: dead zone logic for auto-zoom to prevent jittery segments #1664

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
namearth5005 wants to merge 7 commits into
base: main
Choose a base branch
from
Open

feat: dead zone logic for auto-zoom to prevent jittery segments #1664

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5d54edd
Add AutoZoomConfig struct to extract hardcoded zoom constants into co…
namearth5005 Mar 18, 2026
ae39052
feat(settings): add auto_zoom_config to GeneralSettingsStore with ser…
namearth5005 Mar 18, 2026
04a7fed
feat(recording): wire AutoZoomConfig into zoom segment generation, re…
namearth5005 Mar 18, 2026
b893118
feat(ipc): pass AutoZoomConfig from settings through generate_zoom_se…
namearth5005 Mar 18, 2026
c866805
feat(ui): add zoom amount, sensitivity, and smoothing sliders for aut…
namearth5005 Mar 18, 2026
680a8e5
style: apply cargo fmt and biome formatting
namearth5005 Mar 18, 2026
e308026
feat(recording): add dead zone logic to prevent jittery auto-zoom seg…
namearth5005 Mar 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions apps/desktop/src-tauri/src/general_settings.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,8 @@ pub struct GeneralSettingsStore {
#[serde(default)]
pub auto_zoom_on_clicks: bool,
#[serde(default)]
pub auto_zoom_config: cap_project::AutoZoomConfig,
#[serde(default)]
pub post_deletion_behaviour: PostDeletionBehaviour,
#[serde(default = "default_excluded_windows")]
pub excluded_windows: Vec<WindowExclusion>,
Expand Down Expand Up @@ -203,6 +205,7 @@ impl Default for GeneralSettingsStore {
recording_countdown: Some(3),
enable_native_camera_preview: default_enable_native_camera_preview(),
auto_zoom_on_clicks: false,
auto_zoom_config: cap_project::AutoZoomConfig::default(),
post_deletion_behaviour: PostDeletionBehaviour::DoNothing,
excluded_windows: default_excluded_windows(),
delete_instant_recordings_after_upload: false,
Expand Down
9 changes: 7 additions & 2 deletions apps/desktop/src-tauri/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2111,14 +2111,19 @@ async fn update_project_config_in_memory(

#[tauri::command]
#[specta::specta]
#[instrument(skip(editor_instance))]
#[instrument(skip(app, editor_instance))]
async fn generate_zoom_segments_from_clicks(
app: AppHandle,
editor_instance: WindowEditorInstance,
) -> Result<Vec<ZoomSegment>, String> {
let settings = GeneralSettingsStore::get(&app)
.unwrap_or(None)
.unwrap_or_default();
let meta = editor_instance.meta();
let recordings = &editor_instance.recordings;

let zoom_segments = recording::generate_zoom_segments_for_project(meta, recordings);
let zoom_segments =
recording::generate_zoom_segments_for_project(meta, recordings, &settings.auto_zoom_config);

Ok(zoom_segments)
}
Expand Down
170 changes: 131 additions & 39 deletions apps/desktop/src-tauri/src/recording.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1989,30 +1989,30 @@ async fn finalize_studio_recording(
Ok(())
}

/// Core logic for generating zoom segments based on mouse click events.
/// This is an experimental feature that automatically creates zoom effects
/// around user interactions to highlight important moments.
fn generate_zoom_segments_from_clicks_impl(
mut clicks: Vec<CursorClickEvent>,
mut moves: Vec<CursorMoveEvent>,
max_duration: f64,
config: &cap_project::AutoZoomConfig,
) -> Vec<ZoomSegment> {
const STOP_PADDING_SECONDS: f64 = 0.5;
const CLICK_GROUP_TIME_THRESHOLD_SECS: f64 = 2.5;
const CLICK_GROUP_SPATIAL_THRESHOLD: f64 = 0.15;
const CLICK_PRE_PADDING: f64 = 0.4;
const CLICK_POST_PADDING: f64 = 1.8;
const MOVEMENT_PRE_PADDING: f64 = 0.3;
const MOVEMENT_POST_PADDING: f64 = 1.5;
const MERGE_GAP_THRESHOLD: f64 = 0.8;
const MIN_SEGMENT_DURATION: f64 = 1.0;
const MOVEMENT_WINDOW_SECONDS: f64 = 1.5;
const MOVEMENT_EVENT_DISTANCE_THRESHOLD: f64 = 0.02;
const MOVEMENT_WINDOW_DISTANCE_THRESHOLD: f64 = 0.08;
const AUTO_ZOOM_AMOUNT: f64 = 1.5;
const SHAKE_FILTER_THRESHOLD: f64 = 0.33;
const SHAKE_FILTER_WINDOW_MS: f64 = 150.0;

let click_group_time_threshold_secs = config.click_group_time_threshold;
let click_group_spatial_threshold = config.click_group_spatial_threshold;
let click_pre_padding = config.click_pre_padding;
let click_post_padding = config.click_post_padding;
let movement_pre_padding = config.movement_pre_padding;
let movement_post_padding = config.movement_post_padding;
let merge_gap_threshold = config.merge_gap_threshold;
let min_segment_duration = config.min_segment_duration;
let movement_event_distance_threshold = config.movement_event_distance_threshold;
let movement_window_distance_threshold = config.movement_window_distance_threshold;
let auto_zoom_amount = config.zoom_amount;
let dead_zone_radius = config.dead_zone_radius;

if max_duration <= 0.0 {
return Vec::new();
}
Expand Down Expand Up @@ -2073,24 +2073,42 @@ fn generate_zoom_segments_from_clicks_impl(

let mut found_group = false;
for group in click_groups.iter_mut() {
let can_join = group.iter().any(|&group_idx| {
let time_and_spatial = group.iter().any(|&group_idx| {
let group_click = &clicks[group_idx];
let group_time = group_click.time_ms / 1000.0;
let time_close = (click_time - group_time).abs() < CLICK_GROUP_TIME_THRESHOLD_SECS;
let time_close = (click_time - group_time).abs() < click_group_time_threshold_secs;

let spatial_close = match (click_pos, click_positions.get(&group_idx)) {
(Some((x1, y1)), Some((x2, y2))) => {
let dx = x1 - x2;
let dy = y1 - y2;
(dx * dx + dy * dy).sqrt() < CLICK_GROUP_SPATIAL_THRESHOLD
(dx * dx + dy * dy).sqrt() < click_group_spatial_threshold
}
_ => true,
};

time_close && spatial_close
});

if can_join {
let in_dead_zone = dead_zone_radius > 0.0 && click_pos.is_some() && {
let (cx, cy) = click_pos.unwrap();
let group_positions: Vec<(f64, f64)> = group
.iter()
.filter_map(|&gi| click_positions.get(&gi).copied())
.collect();
if group_positions.is_empty() {
false
} else {
let count = group_positions.len() as f64;
let centroid_x = group_positions.iter().map(|(x, _)| x).sum::<f64>() / count;
let centroid_y = group_positions.iter().map(|(_, y)| y).sum::<f64>() / count;
let dx = cx - centroid_x;
let dy = cy - centroid_y;
(dx * dx + dy * dy).sqrt() < dead_zone_radius
}
Comment on lines +2101 to +2108
Copy link
Contributor

@greptile-apps greptile-apps bot Mar 18, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Centroid drift: repeated near-boundary clicks can shift the dead zone across the screen

The centroid used for the dead zone check is recomputed as the average of all positions already in the group, including those added via previous dead zone merges. Each time a click near the boundary of the dead zone is accepted, the centroid shifts slightly in that direction, which may then accept the next click that is slightly further away, and so on.

Concretely with dead_zone_radius = 0.1:

  • Click at (0.50, 0.50) → centroid (0.50, 0.50)
  • Click at (0.56, 0.56) (dist ≈ 0.085 < 0.1) → merges, centroid shifts to (0.53, 0.53)
  • Click at (0.62, 0.62) (dist from new centroid ≈ 0.127 > 0.1) — just misses in this case, but with slightly smaller steps the group can creep across the screen.

For the typical use case (same button clicked repeatedly) this is harmless, but a user doing a slow drag-with-pauses could inadvertently merge distant clicks into one group. One mitigation is to compute the centroid only from the first click in the group (i.e. the "anchor"), rather than from the running average of all members.

Prompt To Fix With AI 
This is a comment left during a code review.
Path: apps/desktop/src-tauri/src/recording.rs
Line: 2101-2108

Comment:
**Centroid drift: repeated near-boundary clicks can shift the dead zone across the screen**

The centroid used for the dead zone check is recomputed as the average of **all** positions already in the group, including those added via previous dead zone merges. Each time a click near the boundary of the dead zone is accepted, the centroid shifts slightly in that direction, which may then accept the next click that is slightly further away, and so on.

Concretely with `dead_zone_radius = 0.1`:
- Click at `(0.50, 0.50)` → centroid `(0.50, 0.50)`
- Click at `(0.56, 0.56)` (dist ≈ 0.085 < 0.1) → merges, centroid shifts to `(0.53, 0.53)`
- Click at `(0.62, 0.62)` (dist from new centroid ≈ 0.127 > 0.1) — just misses in this case, but with slightly smaller steps the group can creep across the screen.

For the typical use case (same button clicked repeatedly) this is harmless, but a user doing a slow drag-with-pauses could inadvertently merge distant clicks into one group. One mitigation is to compute the centroid only from the **first** click in the group (i.e. the "anchor"), rather than from the running average of all members.

How can I resolve this? If you propose a fix, please make it concise.

};

if time_and_spatial || in_dead_zone {
group.push(*idx);
found_group = true;
break;
Expand All @@ -2116,8 +2134,8 @@ fn generate_zoom_segments_from_clicks_impl(
let group_start = times.iter().cloned().fold(f64::INFINITY, f64::min);
let group_end = times.iter().cloned().fold(f64::NEG_INFINITY, f64::max);

let start = (group_start - CLICK_PRE_PADDING).max(0.0);
let end = (group_end + CLICK_POST_PADDING).min(activity_end_limit);
let start = (group_start - click_pre_padding).max(0.0);
let end = (group_end + click_post_padding).min(activity_end_limit);

if end > start {
intervals.push((start, end));
Expand Down Expand Up @@ -2199,15 +2217,15 @@ fn generate_zoom_segments_from_clicks_impl(
window_distance = 0.0;
}

let significant_movement = distance >= MOVEMENT_EVENT_DISTANCE_THRESHOLD
|| window_distance >= MOVEMENT_WINDOW_DISTANCE_THRESHOLD;
let significant_movement = distance >= movement_event_distance_threshold
|| window_distance >= movement_window_distance_threshold;

if !significant_movement {
continue;
}

let start = (time - MOVEMENT_PRE_PADDING).max(0.0);
let end = (time + MOVEMENT_POST_PADDING).min(activity_end_limit);
let start = (time - movement_pre_padding).max(0.0);
let end = (time + movement_post_padding).min(activity_end_limit);

if end > start {
intervals.push((start, end));
Expand All @@ -2223,7 +2241,7 @@ fn generate_zoom_segments_from_clicks_impl(
let mut merged: Vec<(f64, f64)> = Vec::new();
for interval in intervals {
if let Some(last) = merged.last_mut()
&& interval.0 <= last.1 + MERGE_GAP_THRESHOLD
&& interval.0 <= last.1 + merge_gap_threshold
{
last.1 = last.1.max(interval.1);
continue;
Expand All @@ -2235,14 +2253,14 @@ fn generate_zoom_segments_from_clicks_impl(
.into_iter()
.filter_map(|(start, end)| {
let duration = end - start;
if duration < MIN_SEGMENT_DURATION {
if duration < min_segment_duration {
return None;
}

Some(ZoomSegment {
start,
end,
amount: AUTO_ZOOM_AMOUNT,
amount: auto_zoom_amount,
mode: ZoomMode::Auto,
glide_direction: GlideDirection::None,
glide_speed: 0.5,
Expand All @@ -2253,13 +2271,11 @@ fn generate_zoom_segments_from_clicks_impl(
.collect()
}

/// Generates zoom segments based on mouse click events during recording.
/// Used during the recording completion process.
pub fn generate_zoom_segments_from_clicks(
recording: &studio_recording::CompletedRecording,
recordings: &ProjectRecordingsMeta,
config: &cap_project::AutoZoomConfig,
) -> Vec<ZoomSegment> {
// Build a temporary RecordingMeta so we can use the common implementation
let recording_meta = RecordingMeta {
platform: None,
project_path: recording.project_path.clone(),
Expand All @@ -2269,14 +2285,13 @@ pub fn generate_zoom_segments_from_clicks(
upload: None,
};

generate_zoom_segments_for_project(&recording_meta, recordings)
generate_zoom_segments_for_project(&recording_meta, recordings, config)
}

/// Generates zoom segments from clicks for an existing project.
/// Used in the editor context where we have RecordingMeta.
pub fn generate_zoom_segments_for_project(
recording_meta: &RecordingMeta,
recordings: &ProjectRecordingsMeta,
config: &cap_project::AutoZoomConfig,
) -> Vec<ZoomSegment> {
let RecordingMetaInner::Studio(studio_meta) = &recording_meta.inner else {
return Vec::new();
Expand Down Expand Up @@ -2309,7 +2324,7 @@ pub fn generate_zoom_segments_for_project(
}
}

generate_zoom_segments_from_clicks_impl(all_clicks, all_moves, recordings.duration())
generate_zoom_segments_from_clicks_impl(all_clicks, all_moves, recordings.duration(), config)
}

fn project_config_from_recording(
Expand Down Expand Up @@ -2355,7 +2370,11 @@ fn project_config_from_recording(
.collect::<Vec<_>>();

let zoom_segments = if settings.auto_zoom_on_clicks {
generate_zoom_segments_from_clicks(completed_recording, recordings)
generate_zoom_segments_from_clicks(
completed_recording,
recordings,
&settings.auto_zoom_config,
)
} else {
Vec::new()
};
Expand Down Expand Up @@ -2429,8 +2448,12 @@ mod tests {

#[test]
fn skips_trailing_stop_click() {
let segments =
generate_zoom_segments_from_clicks_impl(vec![click_event(11_900.0)], vec![], 12.0);
let segments = generate_zoom_segments_from_clicks_impl(
vec![click_event(11_900.0)],
vec![],
12.0,
&cap_project::AutoZoomConfig::default(),
);

assert!(
segments.is_empty(),
Expand All @@ -2447,7 +2470,12 @@ mod tests {
move_event(1_940.0, 0.74, 0.78),
];

let segments = generate_zoom_segments_from_clicks_impl(clicks, moves, 20.0);
let segments = generate_zoom_segments_from_clicks_impl(
clicks,
moves,
20.0,
&cap_project::AutoZoomConfig::default(),
);

assert!(
!segments.is_empty(),
Expand All @@ -2469,11 +2497,75 @@ mod tests {
})
.collect::<Vec<_>>();

let segments = generate_zoom_segments_from_clicks_impl(Vec::new(), jitter_moves, 15.0);
let segments = generate_zoom_segments_from_clicks_impl(
Vec::new(),
jitter_moves,
15.0,
&cap_project::AutoZoomConfig::default(),
);

assert!(
segments.is_empty(),
"small jitter should not generate segments"
);
}

#[test]
fn dead_zone_merges_nearby_clicks() {
let clicks = vec![click_event(1_000.0), click_event(5_000.0)];
let moves = vec![move_event(999.0, 0.5, 0.5), move_event(4_999.0, 0.55, 0.55)];

let config = cap_project::AutoZoomConfig {
dead_zone_radius: 0.1,
..Default::default()
};

let segments = generate_zoom_segments_from_clicks_impl(clicks, moves, 20.0, &config);

assert!(
segments.len() <= 1,
"nearby clicks within dead zone should merge into at most 1 segment, got {}",
segments.len()
);
}

#[test]
fn dead_zone_allows_distant_clicks() {
let clicks = vec![click_event(1_000.0), click_event(5_000.0)];
let moves = vec![move_event(999.0, 0.2, 0.2), move_event(4_999.0, 0.8, 0.8)];

let config = cap_project::AutoZoomConfig {
dead_zone_radius: 0.1,
..Default::default()
};

let segments = generate_zoom_segments_from_clicks_impl(clicks, moves, 20.0, &config);

assert_eq!(
segments.len(),
2,
"distant clicks outside dead zone should produce 2 separate segments, got {}",
segments.len()
);
}

#[test]
fn dead_zone_zero_disables() {
let clicks = vec![click_event(1_000.0), click_event(5_000.0)];
let moves = vec![move_event(999.0, 0.5, 0.5), move_event(4_999.0, 0.55, 0.55)];

let config = cap_project::AutoZoomConfig {
dead_zone_radius: 0.0,
..Default::default()
};

let segments = generate_zoom_segments_from_clicks_impl(clicks, moves, 20.0, &config);

assert_eq!(
segments.len(),
2,
"with dead_zone_radius=0.0, nearby clicks should produce 2 segments, got {}",
segments.len()
);
}
}
Loading