Add hierarchical agent prompt (openai#8996)

Author: pakrym-oai

codex-rs/core/hierarchical_agents_message.md

@@ -0,0 +1,7 @@
+Files called AGENTS.md commonly appear in many places inside a container - at "/", in "~", deep within git repositories, or in any other directory; their location is not limited to version-controlled folders.
+
+Their purpose is to pass along human guidance to you, the agent. Such guidance can include coding standards, explanations of the project layout, steps for building or testing, and even wording that must accompany a GitHub pull-request description produced by the agent; all of it is to be followed.
+
+Each AGENTS.md governs the entire directory that contains it and every child directory beneath that point. Whenever you change a file, you have to comply with every AGENTS.md whose scope covers that file. Naming conventions, stylistic rules and similar directives are restricted to the code that falls inside that scope unless the document explicitly states otherwise.
+
+When two AGENTS.md files disagree, the one located deeper in the directory structure overrides the higher-level file, while instructions given directly in the prompt by the system, developer, or user outrank any AGENTS.md content.

codex-rs/core/src/features.rs

@@ -86,6 +86,8 @@ pub enum Feature {
     RemoteModels,
     /// Experimental shell snapshotting.
     ShellSnapshot,
+    /// Append additional AGENTS.md guidance to user instructions.
+    HierarchicalAgents,
     /// Experimental TUI v2 (viewport) implementation.
     Tui2,
     /// Enforce UTF8 output in Powershell.
@@ -352,6 +354,12 @@ pub const FEATURES: &[FeatureSpec] = &[
         },
         default_enabled: false,
     },
+    FeatureSpec {
+        id: Feature::HierarchicalAgents,
+        key: "hierarchical_agents",
+        stage: Stage::Experimental,
+        default_enabled: false,
+    },
     FeatureSpec {
         id: Feature::ApplyPatchFreeform,
         key: "apply_patch_freeform",

codex-rs/core/src/project_doc.rs

@@ -14,13 +14,17 @@
 //! 3.  We do **not** walk past the Git root.
 
 use crate::config::Config;
+use crate::features::Feature;
 use crate::skills::SkillMetadata;
 use crate::skills::render_skills_section;
 use dunce::canonicalize as normalize_path;
 use std::path::PathBuf;
 use tokio::io::AsyncReadExt;
 use tracing::error;
 
+pub(crate) const HIERARCHICAL_AGENTS_MESSAGE: &str =
+    include_str!("../hierarchical_agents_message.md");
+
 /// Default filename scanned for project-level docs.
 pub const DEFAULT_PROJECT_DOC_FILENAME: &str = "AGENTS.md";
 /// Preferred local override for project-level docs.
@@ -36,35 +40,46 @@ pub(crate) async fn get_user_instructions(
     config: &Config,
     skills: Option<&[SkillMetadata]>,
 ) -> Option<String> {
-    let skills_section = skills.and_then(render_skills_section);
+    let project_docs = read_project_docs(config).await;
+
+    let mut output = String::new();
+
+    if let Some(instructions) = config.user_instructions.clone() {
+        output.push_str(&instructions);
+    }
 
-    let project_docs = match read_project_docs(config).await {
-        Ok(docs) => docs,
+    match project_docs {
+        Ok(Some(docs)) => {
+            if !output.is_empty() {
+                output.push_str(PROJECT_DOC_SEPARATOR);
+            }
+            output.push_str(&docs);
+        }
+        Ok(None) => {}
         Err(e) => {
             error!("error trying to find project doc: {e:#}");
-            return config.user_instructions.clone();
         }
     };
 
-    let combined_project_docs = merge_project_docs_with_skills(project_docs, skills_section);
-
-    let mut parts: Vec<String> = Vec::new();
-
-    if let Some(instructions) = config.user_instructions.clone() {
-        parts.push(instructions);
+    let skills_section = skills.and_then(render_skills_section);
+    if let Some(skills_section) = skills_section {
+        if !output.is_empty() {
+            output.push_str("\n\n");
+        }
+        output.push_str(&skills_section);
     }
 
-    if let Some(project_doc) = combined_project_docs {
-        if !parts.is_empty() {
-            parts.push(PROJECT_DOC_SEPARATOR.to_string());
+    if config.features.enabled(Feature::HierarchicalAgents) {
+        if !output.is_empty() {
+            output.push_str("\n\n");
         }
-        parts.push(project_doc);
+        output.push_str(HIERARCHICAL_AGENTS_MESSAGE);
     }
 
-    if parts.is_empty() {
-        None
+    if !output.is_empty() {
+        Some(output)
     } else {
-        Some(parts.concat())
+        None
     }
 }
 
@@ -217,18 +232,6 @@ fn candidate_filenames<'a>(config: &'a Config) -> Vec<&'a str> {
     names
 }
 
-fn merge_project_docs_with_skills(
-    project_doc: Option<String>,
-    skills_section: Option<String>,
-) -> Option<String> {
-    match (project_doc, skills_section) {
-        (Some(doc), Some(skills)) => Some(format!("{doc}\n\n{skills}")),
-        (Some(doc), None) => Some(doc),
-        (None, Some(skills)) => Some(skills),
-        (None, None) => None,
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

codex-rs/core/tests/suite/hierarchical_agents.rs

@@ -0,0 +1,71 @@
+use codex_core::features::Feature;
+use core_test_support::load_sse_fixture_with_id;
+use core_test_support::responses::mount_sse_once;
+use core_test_support::responses::start_mock_server;
+use core_test_support::test_codex::test_codex;
+
+const HIERARCHICAL_AGENTS_SNIPPET: &str =
+    "Files called AGENTS.md commonly appear in many places inside a container";
+
+fn sse_completed(id: &str) -> String {
+    load_sse_fixture_with_id("../fixtures/completed_template.json", id)
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn hierarchical_agents_appends_to_project_doc_in_user_instructions() {
+    let server = start_mock_server().await;
+    let resp_mock = mount_sse_once(&server, sse_completed("resp1")).await;
+
+    let mut builder = test_codex().with_config(|config| {
+        config.features.enable(Feature::HierarchicalAgents);
+        std::fs::write(config.cwd.join("AGENTS.md"), "be nice").expect("write AGENTS.md");
+    });
+    let test = builder.build(&server).await.expect("build test codex");
+
+    test.submit_turn("hello").await.expect("submit turn");
+
+    let request = resp_mock.single_request();
+    let user_messages = request.message_input_texts("user");
+    let instructions = user_messages
+        .iter()
+        .find(|text| text.starts_with("# AGENTS.md instructions for "))
+        .expect("instructions message");
+    assert!(
+        instructions.contains("be nice"),
+        "expected AGENTS.md text included: {instructions}"
+    );
+    let snippet_pos = instructions
+        .find(HIERARCHICAL_AGENTS_SNIPPET)
+        .expect("expected hierarchical agents snippet");
+    let base_pos = instructions
+        .find("be nice")
+        .expect("expected AGENTS.md text");
+    assert!(
+        snippet_pos > base_pos,
+        "expected hierarchical agents message appended after base instructions: {instructions}"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn hierarchical_agents_emits_when_no_project_doc() {
+    let server = start_mock_server().await;
+    let resp_mock = mount_sse_once(&server, sse_completed("resp1")).await;
+
+    let mut builder = test_codex().with_config(|config| {
+        config.features.enable(Feature::HierarchicalAgents);
+    });
+    let test = builder.build(&server).await.expect("build test codex");
+
+    test.submit_turn("hello").await.expect("submit turn");
+
+    let request = resp_mock.single_request();
+    let user_messages = request.message_input_texts("user");
+    let instructions = user_messages
+        .iter()
+        .find(|text| text.starts_with("# AGENTS.md instructions for "))
+        .expect("instructions message");
+    assert!(
+        instructions.contains(HIERARCHICAL_AGENTS_SNIPPET),
+        "expected hierarchical agents message appended: {instructions}"
+    );
+}

codex-rs/core/tests/suite/mod.rs

@@ -30,6 +30,7 @@ mod exec;
 mod exec_policy;
 mod fork_thread;
 mod grep_files;
+mod hierarchical_agents;
 mod items;
 mod json_result;
 mod list_dir;

docs/agents_md.md

@@ -1,3 +1,7 @@
 # AGENTS.md
 
 For information about AGENTS.md, see [this documentation](https://developers.openai.com/codex/guides/agents-md).
+
+## Hierarchical agents message
+
+When the `hierarchical_agents` feature flag is enabled (via `[features]` in `config.toml`), Codex appends additional guidance about AGENTS.md scope and precedence to the user instructions message and emits that message even when no AGENTS.md is present.