Skip to content

Commit 6b7b44d

Browse files
authored
fix(skills): prevent low-confidence skill injection (#2512, #2513, #2501) (#2517)
Add min_injection_score config field (default 0.20) to filter out skills that score below threshold before injection. This closes the structural gap where any top-N skill was injected regardless of absolute score. Also raise disambiguation_threshold default from 0.05 to 0.20 to prevent LLM disambiguation on clearly non-matching candidates. Fix SKILL.md descriptions to eliminate specific false positives: - process-management: replace "memory" with "RAM" to avoid matching user queries containing the word "memory" (#2513) - os-automation: replace catch-all phrase with explicit negative scope sentence to prevent triggering on generic shell commands (#2501) - rust-agent-handoff: remove noisy Keywords line, tighten description to exclude generic update/memory prompts (#2512) Fix pre-existing clippy::float_cmp violations in admission.rs tests.
1 parent e6306f8 commit 6b7b44d

File tree

11 files changed

+40
-15
lines changed

11 files changed

+40
-15
lines changed

.zeph/skills/os-automation/SKILL.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ description: >
88
create scheduled tasks and cron jobs, and manage display brightness.
99
Use when the user asks to send a notification, copy to clipboard, take a
1010
screenshot, open a file or URL, launch an app, adjust volume, check WiFi,
11-
schedule a task, or automate any OS-level action on macOS, Windows, or Linux.
11+
schedule a task, or control OS UI and system settings on macOS, Windows, or Linux.
12+
Do not use for running arbitrary shell commands or scripts — use the shell tool instead.
1213
license: MIT
1314
compatibility: Uses built-in OS utilities; some features require specific tools (see per-platform references)
1415
metadata:

.zeph/skills/process-management/SKILL.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ description: >
55
Monitor, manage, and control system processes. Use when the user asks to list
66
running processes, find a process by name or PID, kill or terminate processes,
77
manage background jobs, check resource usage, set resource limits, manage
8-
system services with systemctl or launchctl, or troubleshoot CPU and memory
9-
consumption with ps, top, htop, kill, pgrep, and related tools.
8+
system services with systemctl or launchctl, or troubleshoot CPU and RAM usage
9+
with ps, top, htop, kill, pgrep, and related tools.
1010
license: MIT
1111
metadata:
1212
author: zeph

.zeph/skills/rust-agent-handoff/SKILL.md

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@
22
name: rust-agent-handoff
33
category: dev
44
description: >
5-
Handoff protocol for Rust multi-agent development system with structured YAML communication.
6-
Use when working as rust-architect, rust-developer, rust-testing-engineer,
7-
rust-performance-engineer, rust-security-maintenance, rust-code-reviewer,
8-
rust-cicd-devops, rust-debugger, or rust-critic. ALWAYS read on agent startup.
9-
Keywords: handoff, agent, subagent, workflow, orchestration, yaml, protocol, team.
5+
Handoff protocol for the Rust multi-agent development team (rust-architect, rust-developer,
6+
rust-testing-engineer, rust-performance-engineer, rust-security-maintenance,
7+
rust-code-reviewer, rust-cicd-devops, rust-debugger, rust-critic). Use only when
8+
orchestrating subagents via structured YAML files in .local/handoff/. Not for general
9+
agent interactions or memory updates.
1010
metadata:
1111
author: zeph
1212
version: "1.0"

CHANGELOG.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
66

77
## [Unreleased]
88

9+
### Fixed
10+
11+
- skills: raise `disambiguation_threshold` default from 0.05 to 0.20 to prevent low-confidence skill injection (#2512)
12+
- skills: add `min_injection_score` config field (default 0.20) — skills scoring below threshold are no longer injected (#2512)
13+
- skills: fix `process-management` SKILL.md false positive on user queries containing "memory" by replacing with "RAM" (#2513)
14+
- skills: fix `os-automation` SKILL.md over-triggering on generic shell commands — add explicit negative scope sentence (#2501)
15+
- skills: fix `rust-agent-handoff` SKILL.md — remove noisy Keywords line, tighten description to exclude generic update/memory prompts (#2512)
16+
917
## [0.18.1] - 2026-03-31
1018

1119
### Fixed

crates/zeph-config/src/features.rs

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,11 @@ use crate::learning::LearningConfig;
88
use crate::security::TrustConfig;
99

1010
fn default_disambiguation_threshold() -> f32 {
11-
0.05
11+
0.20
12+
}
13+
14+
fn default_min_injection_score() -> f32 {
15+
0.20
1216
}
1317

1418
fn default_cosine_weight() -> f32 {
@@ -117,6 +121,8 @@ pub struct SkillsConfig {
117121
pub max_active_skills: usize,
118122
#[serde(default = "default_disambiguation_threshold")]
119123
pub disambiguation_threshold: f32,
124+
#[serde(default = "default_min_injection_score")]
125+
pub min_injection_score: f32,
120126
#[serde(default = "default_cosine_weight")]
121127
pub cosine_weight: f32,
122128
#[serde(default = "default_hybrid_search")]

crates/zeph-config/src/root.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,8 @@ impl Default for Config {
156156
skills: SkillsConfig {
157157
paths: default_skill_paths(),
158158
max_active_skills: 5,
159-
disambiguation_threshold: 0.05,
159+
disambiguation_threshold: 0.20,
160+
min_injection_score: 0.20,
160161
cosine_weight: 0.7,
161162
hybrid_search: true,
162163
learning: LearningConfig::default(),

crates/zeph-core/config/default.toml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,9 @@ max_active_skills = 5
167167
# Prompt mode: "full" (inject full SKILL.md), "compact" (name+description only), "auto" (compact if budget < 8192)
168168
# prompt_mode = "auto"
169169
# Minimum score delta for skill disambiguation (0.0-1.0)
170-
# disambiguation_threshold = 0.05
170+
# disambiguation_threshold = 0.20
171+
# Minimum absolute score a skill must reach to be injected into context (0.0-1.0)
172+
# min_injection_score = 0.20
171173

172174
[skills.learning]
173175
# Enable self-learning skill improvement (feature enabled by default, runtime toggle)

crates/zeph-core/src/agent/context/assembly.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1327,6 +1327,10 @@ impl<C: Channel> Agent<C> {
13271327
tracing::warn!("skill matcher returned no results, falling back to all skills");
13281328
(0..all_meta.len()).collect()
13291329
} else {
1330+
// Drop skills whose score falls below the minimum injection floor.
1331+
let min_score = self.skill_state.min_injection_score;
1332+
scored.retain(|s| s.score >= min_score);
1333+
13301334
// Capture the names of skills that had real embedding scores for
13311335
// usage stats — before disambiguation may reorder indices.
13321336
skills_to_record = scored

crates/zeph-core/src/agent/mod.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,8 @@ impl<C: Channel> Agent<C> {
334334
trust_config: crate::config::TrustConfig::default(),
335335
matcher,
336336
max_active_skills,
337-
disambiguation_threshold: 0.05,
337+
disambiguation_threshold: 0.20,
338+
min_injection_score: 0.20,
338339
embedding_model: String::new(),
339340
skill_reload_rx: None,
340341
active_skill_names: Vec::new(),
@@ -4807,6 +4808,7 @@ impl<C: Channel> Agent<C> {
48074808
self.memory_state.summarization_threshold = config.memory.summarization_threshold;
48084809
self.skill_state.max_active_skills = config.skills.max_active_skills;
48094810
self.skill_state.disambiguation_threshold = config.skills.disambiguation_threshold;
4811+
self.skill_state.min_injection_score = config.skills.min_injection_score;
48104812
self.skill_state.cosine_weight = config.skills.cosine_weight.clamp(0.0, 1.0);
48114813
self.skill_state.hybrid_search = config.skills.hybrid_search;
48124814
self.skill_state.two_stage_matching = config.skills.two_stage_matching;

crates/zeph-core/src/agent/state/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ pub(crate) struct SkillState {
8080
pub(crate) matcher: Option<SkillMatcherBackend>,
8181
pub(crate) max_active_skills: usize,
8282
pub(crate) disambiguation_threshold: f32,
83+
pub(crate) min_injection_score: f32,
8384
pub(crate) embedding_model: String,
8485
pub(crate) skill_reload_rx: Option<mpsc::Receiver<SkillEvent>>,
8586
pub(crate) active_skill_names: Vec<String>,

0 commit comments

Comments
 (0)