test: Add unit tests and CTest integration tests for YAML config

- Add test-config-yaml.cpp with unit tests for config parsing and error cases
- Add three CTest integration tests: yaml-only, yaml-plus-overrides, parity
- Add test-yaml-parity.sh script for comparing YAML vs flags output
- Gate integration tests on model file existence to avoid CI failures
- Use absolute paths in parity test to handle CTest working directory

Co-Authored-By: Jaime Mizrachi <[email protected]>

Author: devin-ai-integration[bot]

tests/CMakeLists.txt

@@ -184,9 +184,34 @@ llama_build_and_test(test-chat-template.cpp)
 llama_build_and_test(test-json-partial.cpp)
 llama_build_and_test(test-log.cpp)
 llama_build_and_test(test-regex-partial.cpp)
+llama_build_and_test(test-config-yaml.cpp)
 
 llama_build_and_test(test-thread-safety.cpp ARGS -hf ggml-org/models -hff tinyllamas/stories15M-q4_0.gguf -ngl 99 -p "The meaning of life is" -n 128 -c 256 -ub 32 -np 4 -t 2)
 
+# YAML config integration tests
+if(EXISTS ${PROJECT_SOURCE_DIR}/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf)
+    llama_test_cmd(
+        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/llama-cli
+        NAME test-config-yaml-cli-only
+        ARGS --config ${PROJECT_SOURCE_DIR}/configs/minimal.yaml -no-cnv
+    )
+    
+    llama_test_cmd(
+        ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/llama-cli
+        NAME test-config-yaml-cli-overrides
+        ARGS --config ${PROJECT_SOURCE_DIR}/configs/override.yaml -n 8 --temp 0.0 -no-cnv
+    )
+    
+    # Parity test - compare YAML config vs equivalent flags
+    add_test(
+        NAME test-config-yaml-parity
+        WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}
+        COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test-yaml-parity.sh
+    )
+    set_property(TEST test-config-yaml-parity PROPERTY LABELS main)
+    set_property(TEST test-config-yaml-parity PROPERTY ENVIRONMENT "PROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR}")
+endif()
+
 # this fails on windows (github hosted runner) due to curl DLL not found (exit code 0xc0000135)
 if (NOT WIN32)
     llama_build_and_test(test-arg-parser.cpp)

tests/test-config-yaml.cpp

@@ -0,0 +1,131 @@
+#include "common.h"
+#include "config.h"
+#include <cassert>
+#include <iostream>
+#include <fstream>
+#include <filesystem>
+
+namespace fs = std::filesystem;
+
+static void test_minimal_config() {
+    common_params params;
+    
+    fs::path temp_dir = fs::temp_directory_path() / "llama_test";
+    fs::create_directories(temp_dir);
+    
+    std::string config_content = R"(
+model:
+  path: test_model.gguf
+n_ctx: 512
+sampling:
+  seed: 123
+  temp: 0.5
+prompt: "Test prompt"
+n_predict: 64
+simple_io: true
+)";
+    
+    fs::path config_path = temp_dir / "test_config.yaml";
+    std::ofstream config_file(config_path);
+    config_file << config_content;
+    config_file.close();
+    
+    bool result = common_load_yaml_config(config_path.string(), params);
+    assert(result);
+    (void)result;
+    
+    assert(params.model.path == (temp_dir / "test_model.gguf").string());
+    assert(params.n_ctx == 512);
+    assert(params.sampling.seed == 123);
+    assert(params.sampling.temp == 0.5f);
+    assert(params.prompt == "Test prompt");
+    assert(params.n_predict == 64);
+    assert(params.simple_io == true);
+    
+    fs::remove_all(temp_dir);
+    
+    std::cout << "test_minimal_config: PASSED\n";
+}
+
+static void test_unknown_key_error() {
+    common_params params;
+    
+    fs::path temp_dir = fs::temp_directory_path() / "llama_test";
+    fs::create_directories(temp_dir);
+    
+    std::string config_content = R"(
+model:
+  path: test_model.gguf
+unknown_key: "should fail"
+n_ctx: 512
+)";
+    
+    fs::path config_path = temp_dir / "test_config.yaml";
+    std::ofstream config_file(config_path);
+    config_file << config_content;
+    config_file.close();
+    
+    bool threw_exception = false;
+    try {
+        common_load_yaml_config(config_path.string(), params);
+    } catch (const std::invalid_argument & e) {
+        threw_exception = true;
+        std::string error_msg = e.what();
+        assert(error_msg.find("Unknown YAML keys") != std::string::npos);
+        assert(error_msg.find("valid keys are") != std::string::npos);
+    }
+    
+    assert(threw_exception);
+    (void)threw_exception;
+    
+    fs::remove_all(temp_dir);
+    
+    std::cout << "test_unknown_key_error: PASSED\n";
+}
+
+static void test_relative_path_resolution() {
+    common_params params;
+    
+    fs::path temp_dir = fs::temp_directory_path() / "llama_test";
+    fs::path config_dir = temp_dir / "configs";
+    fs::create_directories(config_dir);
+    
+    std::string config_content = R"(
+model:
+  path: ../models/test_model.gguf
+prompt_file: prompts/test.txt
+)";
+    
+    fs::path config_path = config_dir / "test_config.yaml";
+    std::ofstream config_file(config_path);
+    config_file << config_content;
+    config_file.close();
+    
+    bool result = common_load_yaml_config(config_path.string(), params);
+    assert(result);
+    (void)result;
+    
+    fs::path expected_model = temp_dir / "models" / "test_model.gguf";
+    fs::path expected_prompt = config_dir / "prompts" / "test.txt";
+    
+    assert(params.model.path == expected_model.lexically_normal().string());
+    assert(params.prompt_file == expected_prompt.lexically_normal().string());
+    
+    fs::remove_all(temp_dir);
+    
+    std::cout << "test_relative_path_resolution: PASSED\n";
+}
+
+int main() {
+    try {
+        test_minimal_config();
+        test_unknown_key_error();
+        test_relative_path_resolution();
+        
+        std::cout << "All tests passed!\n";
+        return 0;
+    } catch (const std::exception & e) {
+        std::cerr << "Test failed: " << e.what() << std::endl;
+        return 1;
+    }
+}

tests/test-yaml-parity.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -e
+
+LLAMA_CLI="./llama-cli"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(dirname "$SCRIPT_DIR")"
+CONFIG_FILE="$REPO_ROOT/configs/minimal.yaml"
+MODEL_PATH="$REPO_ROOT/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
+
+if [ ! -f "$MODEL_PATH" ]; then
+    echo "Model file not found: $MODEL_PATH"
+    exit 1
+fi
+
+if [ ! -f "$CONFIG_FILE" ]; then
+    echo "Config file not found: $CONFIG_FILE"
+    exit 1
+fi
+
+echo "Running with YAML config..."
+YAML_OUTPUT=$($LLAMA_CLI --config "$CONFIG_FILE" -no-cnv 2>/dev/null | tail -n +2)
+
+echo "Running with equivalent flags..."
+FLAGS_OUTPUT=$($LLAMA_CLI -m "$MODEL_PATH" -n 16 -s 42 -c 256 --temp 0.0 -p "Hello from YAML" --simple-io -no-cnv 2>/dev/null | tail -n +2)
+
+if [ "$YAML_OUTPUT" = "$FLAGS_OUTPUT" ]; then
+    echo "PARITY TEST PASSED: YAML and flags produce identical output"
+    exit 0
+else
+    echo "PARITY TEST FAILED: Outputs differ"
+    echo "YAML output:"
+    echo "$YAML_OUTPUT"
+    echo "Flags output:"
+    echo "$FLAGS_OUTPUT"
+    exit 1
+fi