Skip to content

Commit 26c175b

Browse files
author
ochafik
committed
json: build_grammar helper
1 parent eaca756 commit 26c175b

File tree

2 files changed

+71
-45
lines changed

2 files changed

+71
-45
lines changed

‎common/json-schema-to-grammar.cpp‎

Lines changed: 59 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,6 @@
1111

1212
using json = nlohmann::ordered_json;
1313

14-
template <typename Iterator>
15-
static std::string join(Iterator begin, Iterator end, const std::string & separator);
16-
1714
static std::string repeat(const std::string & str, size_t n);
1815

1916
static std::string build_repetition(const std::string & item_rule, int min_items, int max_items, const std::string & separator_rule = "") {
@@ -397,22 +394,6 @@ class SchemaConverter {
397394
std::vector<std::string> _errors;
398395
std::vector<std::string> _warnings;
399396

400-
std::string _add_rule(const std::string & name, const std::string & rule) {
401-
std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-");
402-
if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) {
403-
_rules[esc_name] = rule;
404-
return esc_name;
405-
} else {
406-
int i = 0;
407-
while (_rules.find(esc_name + std::to_string(i)) != _rules.end() && _rules[esc_name + std::to_string(i)] != rule) {
408-
i++;
409-
}
410-
std::string key = esc_name + std::to_string(i);
411-
_rules[key] = rule;
412-
return key;
413-
}
414-
}
415-
416397
std::string _generate_union_rule(const std::string & name, const std::vector<json> & alt_schemas) {
417398
std::vector<std::string> rules;
418399
for (size_t i = 0; i < alt_schemas.size(); i++) {
@@ -449,7 +430,7 @@ class SchemaConverter {
449430
} else {
450431
rule = "[^\\x0A\\x0D]";
451432
}
452-
return _add_rule("dot", rule);
433+
return add_rule("dot", rule);
453434
};
454435

455436
// Joins the sequence, merging consecutive literals together.
@@ -566,7 +547,7 @@ class SchemaConverter {
566547
if (!sub_is_literal) {
567548
std::string & sub_id = sub_rule_ids[sub];
568549
if (sub_id.empty()) {
569-
sub_id = _add_rule(name + "-" + std::to_string(sub_rule_ids.size()), sub);
550+
sub_id = add_rule(name + "-" + std::to_string(sub_rule_ids.size()), sub);
570551
}
571552
sub = sub_id;
572553
}
@@ -611,7 +592,7 @@ class SchemaConverter {
611592
}
612593
return join_seq();
613594
};
614-
return _add_rule(name, "\"\\\"\" " + to_rule(transform()) + " \"\\\"\" space");
595+
return add_rule(name, "\"\\\"\" " + to_rule(transform()) + " \"\\\"\" space");
615596
}
616597

617598
/*
@@ -709,7 +690,7 @@ class SchemaConverter {
709690
const auto &prop_schema = kv.second;
710691

711692
std::string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name);
712-
prop_kv_rule_names[prop_name] = _add_rule(
693+
prop_kv_rule_names[prop_name] = add_rule(
713694
name + (name.empty() ? "" : "-") + prop_name + "-kv",
714695
format_literal(json(prop_name).dump()) + " space \":\" space " + prop_rule_name
715696
);
@@ -728,8 +709,8 @@ class SchemaConverter {
728709

729710
auto key_rule =
730711
prop_names.empty() ? _add_primitive("string", PRIMITIVE_RULES.at("string"))
731-
: _add_rule(sub_name + "-k", _not_strings(prop_names));
732-
std::string kv_rule = _add_rule(sub_name + "-kv", key_rule + " \":\" space " + value_rule);
712+
: add_rule(sub_name + "-k", _not_strings(prop_names));
713+
std::string kv_rule = add_rule(sub_name + "-kv", key_rule + " \":\" space " + value_rule);
733714
prop_kv_rule_names["*"] = kv_rule;
734715
optional_props.push_back("*");
735716
}
@@ -762,7 +743,7 @@ class SchemaConverter {
762743
res = kv_rule_name + (k == "*" ? " " + comma_ref + "*" : "");
763744
}
764745
if (ks.size() > 1) {
765-
res += " " + _add_rule(
746+
res += " " + add_rule(
766747
name + (name.empty() ? "" : "-") + k + "-rest",
767748
get_recursive_refs(std::vector<std::string>(ks.begin() + 1, ks.end()), true)
768749
);
@@ -788,7 +769,7 @@ class SchemaConverter {
788769
}
789770

790771
std::string _add_primitive(const std::string & name, const BuiltinRule & rule) {
791-
auto n = _add_rule(name, rule.content);
772+
auto n = add_rule(name, rule.content);
792773
for (const auto & dep : rule.deps) {
793774
BuiltinRule dep_rule;
794775
auto it = PRIMITIVE_RULES.find(dep);
@@ -815,6 +796,22 @@ class SchemaConverter {
815796
_rules["space"] = SPACE_RULE;
816797
}
817798

799+
std::string add_rule(const std::string & name, const std::string & rule) {
800+
std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-");
801+
if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) {
802+
_rules[esc_name] = rule;
803+
return esc_name;
804+
} else {
805+
int i = 0;
806+
while (_rules.find(esc_name + std::to_string(i)) != _rules.end() && _rules[esc_name + std::to_string(i)] != rule) {
807+
i++;
808+
}
809+
std::string key = esc_name + std::to_string(i);
810+
_rules[key] = rule;
811+
return key;
812+
}
813+
}
814+
818815
void resolve_refs(json & schema, const std::string & url) {
819816
/*
820817
* Resolves all $ref fields in the given schema, fetching any remote schemas,
@@ -886,26 +883,26 @@ class SchemaConverter {
886883
std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name;
887884

888885
if (schema.contains("$ref")) {
889-
return _add_rule(rule_name, _resolve_ref(schema["$ref"]));
886+
return add_rule(rule_name, _resolve_ref(schema["$ref"]));
890887
} else if (schema.contains("oneOf") || schema.contains("anyOf")) {
891888
std::vector<json> alt_schemas = schema.contains("oneOf") ? schema["oneOf"].get<std::vector<json>>() : schema["anyOf"].get<std::vector<json>>();
892-
return _add_rule(rule_name, _generate_union_rule(name, alt_schemas));
889+
return add_rule(rule_name, _generate_union_rule(name, alt_schemas));
893890
} else if (schema_type.is_array()) {
894891
std::vector<json> schema_types;
895892
for (const auto & t : schema_type) {
896893
json schema_copy(schema);
897894
schema_copy["type"] = t;
898895
schema_types.push_back(schema_copy);
899896
}
900-
return _add_rule(rule_name, _generate_union_rule(name, schema_types));
897+
return add_rule(rule_name, _generate_union_rule(name, schema_types));
901898
} else if (schema.contains("const")) {
902-
return _add_rule(rule_name, _generate_constant_rule(schema["const"]) + " space");
899+
return add_rule(rule_name, _generate_constant_rule(schema["const"]) + " space");
903900
} else if (schema.contains("enum")) {
904901
std::vector<std::string> enum_values;
905902
for (const auto & v : schema["enum"]) {
906903
enum_values.push_back(_generate_constant_rule(v));
907904
}
908-
return _add_rule(rule_name, "(" + join(enum_values.begin(), enum_values.end(), " | ") + ") space");
905+
return add_rule(rule_name, "(" + join(enum_values.begin(), enum_values.end(), " | ") + ") space");
909906
} else if ((schema_type.is_null() || schema_type == "object")
910907
&& (schema.contains("properties") ||
911908
(schema.contains("additionalProperties") && schema["additionalProperties"] != true))) {
@@ -923,7 +920,7 @@ class SchemaConverter {
923920
properties.emplace_back(prop.key(), prop.value());
924921
}
925922
}
926-
return _add_rule(rule_name,
923+
return add_rule(rule_name,
927924
_build_object_rule(
928925
properties, required, name,
929926
schema.contains("additionalProperties") ? schema["additionalProperties"] : json()));
@@ -954,7 +951,7 @@ class SchemaConverter {
954951
add_component(t, true);
955952
}
956953
}
957-
return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json()));
954+
return add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json()));
958955
} else if ((schema_type.is_null() || schema_type == "array") && (schema.contains("items") || schema.contains("prefixItems"))) {
959956
json items = schema.contains("items") ? schema["items"] : schema["prefixItems"];
960957
if (items.is_array()) {
@@ -966,27 +963,27 @@ class SchemaConverter {
966963
rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i));
967964
}
968965
rule += " \"]\" space";
969-
return _add_rule(rule_name, rule);
966+
return add_rule(rule_name, rule);
970967
} else {
971968
std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item");
972969
int min_items = schema.contains("minItems") ? schema["minItems"].get<int>() : 0;
973970
json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json();
974971
int max_items = max_items_json.is_number_integer() ? max_items_json.get<int>() : std::numeric_limits<int>::max();
975972

976-
return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " \"]\" space");
973+
return add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " \"]\" space");
977974
}
978975
} else if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) {
979976
return _visit_pattern(schema["pattern"], rule_name);
980977
} else if ((schema_type.is_null() || schema_type == "string") && std::regex_match(schema_format, std::regex("^uuid[1-5]?$"))) {
981978
return _add_primitive(rule_name == "root" ? "root" : schema_format, PRIMITIVE_RULES.at("uuid"));
982979
} else if ((schema_type.is_null() || schema_type == "string") && STRING_FORMAT_RULES.find(schema_format + "-string") != STRING_FORMAT_RULES.end()) {
983980
auto prim_name = schema_format + "-string";
984-
return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name)));
981+
return add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name)));
985982
} else if (schema_type == "string" && (schema.contains("minLength") || schema.contains("maxLength"))) {
986983
std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
987984
int min_len = schema.contains("minLength") ? schema["minLength"].get<int>() : 0;
988985
int max_len = schema.contains("maxLength") ? schema["maxLength"].get<int>() : std::numeric_limits<int>::max();
989-
return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\" space");
986+
return add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\" space");
990987
} else if (schema_type == "integer" && (schema.contains("minimum") || schema.contains("exclusiveMinimum") || schema.contains("maximum") || schema.contains("exclusiveMaximum"))) {
991988
int min_value = std::numeric_limits<int>::min();
992989
int max_value = std::numeric_limits<int>::max();
@@ -1004,9 +1001,9 @@ class SchemaConverter {
10041001
out << "(";
10051002
_build_min_max_int(min_value, max_value, out);
10061003
out << ") space";
1007-
return _add_rule(rule_name, out.str());
1004+
return add_rule(rule_name, out.str());
10081005
} else if (schema.empty() || schema_type == "object") {
1009-
return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object")));
1006+
return add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object")));
10101007
} else {
10111008
if (!schema_type.is_string() || PRIMITIVE_RULES.find(schema_type.get<std::string>()) == PRIMITIVE_RULES.end()) {
10121009
_errors.push_back("Unrecognized schema: " + schema.dump());
@@ -1036,10 +1033,28 @@ class SchemaConverter {
10361033
};
10371034

10381035
std::string json_schema_to_grammar(const json & schema) {
1039-
SchemaConverter converter([](const std::string &) { return json::object(); }, /* dotall= */ false);
1040-
auto copy = schema;
1041-
converter.resolve_refs(copy, "input");
1042-
converter.visit(copy, "");
1036+
return build_grammar([&](const llama_grammar_builder & callbacks) {
1037+
auto copy = schema;
1038+
callbacks.resolve_refs(copy);
1039+
callbacks.add_schema("root", copy);
1040+
});
1041+
}
1042+
1043+
std::string build_grammar(const std::function<void(const llama_grammar_builder &)> & cb) {
1044+
SchemaConverter converter([&](const std::string & name) { return json(); }, /* dotall= */ false);
1045+
llama_grammar_builder builder {
1046+
.add_rule = [&](const std::string & name, const std::string & rule) {
1047+
return converter.add_rule(name, rule);
1048+
},
1049+
.add_schema = [&](const std::string & name, const nlohmann::ordered_json & schema) {
1050+
return converter.visit(schema, name);
1051+
},
1052+
.resolve_refs = [&](nlohmann::ordered_json & schema) {
1053+
converter.resolve_refs(schema, "");
1054+
}
1055+
};
1056+
cb(builder);
10431057
converter.check_errors();
10441058
return converter.format_grammar();
10451059
}
1060+

‎common/json-schema-to-grammar.h‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,15 @@
55
#define JSON_ASSERT GGML_ASSERT
66
#include "json.hpp"
77

8-
std::string json_schema_to_grammar(const nlohmann::ordered_json& schema);
8+
template <typename Iterator>
9+
std::string join(Iterator begin, Iterator end, const std::string & separator);
10+
11+
std::string json_schema_to_grammar(const nlohmann::ordered_json & schema);
12+
13+
struct llama_grammar_builder {
14+
std::function<std::string(const std::string &, const std::string &)> add_rule;
15+
std::function<std::string(const std::string &, const nlohmann::ordered_json &)> add_schema;
16+
std::function<void(nlohmann::ordered_json &)> resolve_refs;
17+
};
18+
19+
std::string build_grammar(const std::function<void(const llama_grammar_builder &)> & cb);

0 commit comments

Comments
 (0)