Add check_path_exists tool and fix uv environment setup

New features:
- Add check_path_exists tool to verify bucket/path accessibility
- If path doesn't exist, automatically notifies support channel
- Detects common errors: NoSuchBucket, AccessDenied, NoSuchKey

Bug fixes:
- Add uv sync to prepare script to ensure environment is up to date
- Use 'uv run xfer' instead of bare 'xfer' commands
- Add XFER_INSTALL_DIR config option for xfer repo location

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: fluidnumerics-joe

src/xfer/slackbot/claude_agent.py

@@ -14,6 +14,7 @@
 from .config import BotConfig
 from .slurm_tools import (
     cancel_job,
+    check_path_exists,
     get_allowed_backends,
     get_job_status,
     get_jobs_by_thread,
@@ -179,6 +180,29 @@
             "required": ["source"],
         },
     },
+    {
+        "name": "check_path_exists",
+        "description": """Check if a bucket or path exists at a remote endpoint.
+
+Use this to verify that a source or destination path is accessible before starting a transfer.
+If the path doesn't exist, this will notify the support team so they can help resolve the issue.
+
+Common reasons for paths not existing:
+- Bucket name is misspelled
+- Bucket hasn't been created yet
+- Credentials don't have access to the bucket
+- Wrong endpoint/region configured""",
+        "input_schema": {
+            "type": "object",
+            "properties": {
+                "path": {
+                    "type": "string",
+                    "description": "Path to check in rclone format (remote:bucket/path)",
+                },
+            },
+            "required": ["path"],
+        },
+    },
 ]
 
 SYSTEM_PROMPT = """You are a helpful data transfer assistant for an HPC cluster. You help researchers submit and monitor data transfer jobs via Slurm.
@@ -190,6 +214,7 @@
 - Cancel jobs if requested
 - Request access to new backends on behalf of users
 - Scan source paths to get file statistics and transfer estimates
+- Verify that buckets/paths exist before starting transfers
 
 Guidelines:
 1. Always validate that backends are allowed before submitting transfers
@@ -404,6 +429,79 @@ def execute_tool(
                 result["error"] = stats.error
             return json.dumps(result)
 
+        elif tool_name == "check_path_exists":
+            path = tool_input["path"]
+            check_result = check_path_exists(path, self.config)
+
+            # If path doesn't exist, notify support channel
+            support_notified = False
+            if not check_result.exists and self.slack_client and self.config.support_channel:
+                try:
+                    self.slack_client.chat_postMessage(
+                        channel=self.config.support_channel,
+                        text=f"Path access issue: {path}",
+                        blocks=[
+                            {
+                                "type": "header",
+                                "text": {
+                                    "type": "plain_text",
+                                    "text": "Path Access Issue",
+                                },
+                            },
+                            {
+                                "type": "section",
+                                "fields": [
+                                    {
+                                        "type": "mrkdwn",
+                                        "text": f"*Path:*\n`{path}`",
+                                    },
+                                    {
+                                        "type": "mrkdwn",
+                                        "text": f"*Error:*\n{check_result.error or 'Unknown'}",
+                                    },
+                                ],
+                            },
+                            {
+                                "type": "section",
+                                "text": {
+                                    "type": "mrkdwn",
+                                    "text": f"*Details:*\n```{check_result.details[:500] if check_result.details else 'No details'}```",
+                                },
+                            },
+                            {
+                                "type": "context",
+                                "elements": [
+                                    {
+                                        "type": "mrkdwn",
+                                        "text": f"<https://slack.com/archives/{channel_id}/p{thread_ts.replace('.', '')}|View original thread>",
+                                    }
+                                ],
+                            },
+                        ],
+                    )
+                    support_notified = True
+                except Exception as e:
+                    import logging
+
+                    logging.getLogger(__name__).error(
+                        f"Failed to post to support channel: {e}"
+                    )
+
+            return json.dumps(
+                {
+                    "path": check_result.path,
+                    "exists": check_result.exists,
+                    "error": check_result.error,
+                    "support_notified": support_notified,
+                    "message": (
+                        f"Path '{path}' is accessible."
+                        if check_result.exists
+                        else f"Path '{path}' is not accessible: {check_result.error}. "
+                        + ("Support team has been notified." if support_notified else "")
+                    ),
+                }
+            )
+
         else:
             return json.dumps({"error": f"Unknown tool: {tool_name}"})
 

src/xfer/slackbot/config.py

@@ -66,6 +66,9 @@ class BotConfig:
     allowed_backends_file: Optional[Path] = (
         None  # Path to YAML/JSON listing allowed backends
     )
+    xfer_install_dir: Optional[Path] = field(
+        default_factory=lambda: Path(__file__).parent.parent.parent.parent.resolve()
+    )  # Path to xfer git repo for uv sync
 
     # Defaults
     slurm: SlurmDefaults = field(default_factory=SlurmDefaults)
@@ -100,6 +103,9 @@ def from_env(cls) -> "BotConfig":
         if rclone_config := os.environ.get("XFER_RCLONE_CONFIG"):
             config.rclone.config_path = Path(rclone_config)
 
+        if xfer_install_dir := os.environ.get("XFER_INSTALL_DIR"):
+            config.xfer_install_dir = Path(xfer_install_dir)
+
         return config
 
 

src/xfer/slackbot/slurm_tools.py

@@ -195,6 +195,9 @@ def _write_prepare_script(
     # User flags to append after analysis-suggested flags
     user_rclone_flags = rclone_flags or ""
 
+    # Get xfer install directory for uv
+    xfer_dir = config.xfer_install_dir or Path(__file__).parent.parent.parent.parent.resolve()
+
     script_content = f"""#!/usr/bin/env bash
 #SBATCH --job-name={job_name}-prepare
 #SBATCH --output={run_dir}/prepare-%j.out
@@ -209,13 +212,21 @@ def _write_prepare_script(
 
 set -euo pipefail
 
+# Setup uv environment
+XFER_DIR="{xfer_dir}"
+echo "=== Setting up uv environment at $(date -Is) ==="
+echo "XFER_DIR: $XFER_DIR"
+cd "$XFER_DIR"
+uv sync
+echo "=== uv sync complete ==="
+
 echo "=== Starting manifest build at $(date -Is) ==="
 echo "Source: {source}"
 echo "Dest: {dest}"
 echo "Run dir: {run_dir}"
 
 # Phase 1: Build manifest
-xfer manifest build \\
+uv run xfer manifest build \\
     --source {shlex.quote(source)} \\
     --dest {shlex.quote(dest)} \\
     --out {run_dir}/manifest.jsonl \\
@@ -226,7 +237,7 @@ def _write_prepare_script(
 
 # Phase 2: Analyze manifest to determine optimal rclone flags
 echo "=== Analyzing file size distribution ==="
-xfer manifest analyze \\
+uv run xfer manifest analyze \\
     --in {run_dir}/manifest.jsonl \\
     --out {run_dir}/analysis.json
 
@@ -247,15 +258,15 @@ def _write_prepare_script(
 echo "=== Analysis complete at $(date -Is) ==="
 
 # Phase 3: Shard manifest
-xfer manifest shard \\
+uv run xfer manifest shard \\
     --in {run_dir}/manifest.jsonl \\
     --outdir {run_dir}/shards \\
     --num-shards {num_shards}
 
 echo "=== Sharding complete at $(date -Is) ==="
 
 # Phase 4: Render Slurm scripts
-xfer slurm render \\
+uv run xfer slurm render \\
     --run-dir {run_dir} \\
     --num-shards {num_shards} \\
     --array-concurrency {array_concurrency} \\
@@ -273,7 +284,7 @@ def _write_prepare_script(
 echo "=== Render complete at $(date -Is) ==="
 
 # Phase 5: Submit transfer array job
-xfer slurm submit --run-dir {run_dir}
+uv run xfer slurm submit --run-dir {run_dir}
 
 echo "=== Transfer job submitted at $(date -Is) ==="
 """
@@ -779,3 +790,99 @@ def get_source_stats(source: str, config: BotConfig) -> SourceStats:
         histogram_text=histogram_text,
         error=None,
     )
+
+
+@dataclass
+class PathCheckResult:
+    """Result of checking if a path exists."""
+
+    path: str
+    exists: bool
+    error: Optional[str] = None
+    details: Optional[str] = None
+
+
+def check_path_exists(path: str, config: BotConfig) -> PathCheckResult:
+    """
+    Check if a bucket/path exists at a remote endpoint.
+
+    Uses rclone lsf with --max-depth 0 to check if the path is accessible.
+    """
+    # Validate backend first
+    valid, msg = validate_backend(path, config)
+    if not valid:
+        return PathCheckResult(
+            path=path,
+            exists=False,
+            error=msg,
+            details="Backend not in allowed list",
+        )
+
+    # Build rclone lsf command to check if path exists
+    # Using lsf with --max-depth 0 and --dirs-only is a quick way to check
+    rclone_cmd = [
+        "rclone",
+        "lsf",
+        path,
+        "--max-depth",
+        "0",
+        "--config",
+        config.rclone.container_conf_path,
+    ]
+
+    # Build srun command with container
+    mounts = f"{config.rclone.config_path}:{config.rclone.container_conf_path}:ro"
+
+    srun_cmd = [
+        "srun",
+        "-n",
+        "1",
+        "-c",
+        "2",
+        "--container-image",
+        config.rclone.image,
+        "--container-mounts",
+        mounts,
+        "--no-container-remap-root",
+    ] + rclone_cmd
+
+    try:
+        result = run_cmd(srun_cmd, capture=True, check=True)
+        # If command succeeded, path exists
+        return PathCheckResult(
+            path=path,
+            exists=True,
+            details="Path is accessible",
+        )
+    except subprocess.CalledProcessError as e:
+        error_output = e.stderr or str(e)
+
+        # Check for common error patterns
+        if "NoSuchBucket" in error_output or "bucket does not exist" in error_output.lower():
+            return PathCheckResult(
+                path=path,
+                exists=False,
+                error="Bucket does not exist",
+                details=error_output,
+            )
+        elif "AccessDenied" in error_output or "access denied" in error_output.lower():
+            return PathCheckResult(
+                path=path,
+                exists=False,
+                error="Access denied - credentials may not have permission",
+                details=error_output,
+            )
+        elif "NoSuchKey" in error_output or "not found" in error_output.lower():
+            return PathCheckResult(
+                path=path,
+                exists=False,
+                error="Path does not exist within the bucket",
+                details=error_output,
+            )
+        else:
+            return PathCheckResult(
+                path=path,
+                exists=False,
+                error=f"Failed to access path: {error_output[:200]}",
+                details=error_output,
+            )