Skip to content

Commit d77fecc

Browse files
author
Olivier Chafik
committed
shrink diff in json conversion code
1 parent 3972945 commit d77fecc

File tree

1 file changed

+39
-38
lines changed

1 file changed

+39
-38
lines changed

‎common/json-schema-to-grammar.cpp‎

Lines changed: 39 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,7 @@ static std::string format_literal(const std::string & literal) {
386386

387387
class SchemaConverter {
388388
private:
389+
friend std::string build_grammar(const std::function<void(const llama_grammar_builder &)> & cb);
389390
std::function<json(const std::string &)> _fetch_json;
390391
bool _dotall;
391392
std::map<std::string, std::string> _rules;
@@ -394,6 +395,22 @@ class SchemaConverter {
394395
std::vector<std::string> _errors;
395396
std::vector<std::string> _warnings;
396397

398+
std::string _add_rule(const std::string & name, const std::string & rule) {
399+
std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-");
400+
if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) {
401+
_rules[esc_name] = rule;
402+
return esc_name;
403+
} else {
404+
int i = 0;
405+
while (_rules.find(esc_name + std::to_string(i)) != _rules.end() && _rules[esc_name + std::to_string(i)] != rule) {
406+
i++;
407+
}
408+
std::string key = esc_name + std::to_string(i);
409+
_rules[key] = rule;
410+
return key;
411+
}
412+
}
413+
397414
std::string _generate_union_rule(const std::string & name, const std::vector<json> & alt_schemas) {
398415
std::vector<std::string> rules;
399416
for (size_t i = 0; i < alt_schemas.size(); i++) {
@@ -430,7 +447,7 @@ class SchemaConverter {
430447
} else {
431448
rule = "[^\\x0A\\x0D]";
432449
}
433-
return add_rule("dot", rule);
450+
return _add_rule("dot", rule);
434451
};
435452

436453
// Joins the sequence, merging consecutive literals together.
@@ -547,7 +564,7 @@ class SchemaConverter {
547564
if (!sub_is_literal) {
548565
std::string & sub_id = sub_rule_ids[sub];
549566
if (sub_id.empty()) {
550-
sub_id = add_rule(name + "-" + std::to_string(sub_rule_ids.size()), sub);
567+
sub_id = _add_rule(name + "-" + std::to_string(sub_rule_ids.size()), sub);
551568
}
552569
sub = sub_id;
553570
}
@@ -592,7 +609,7 @@ class SchemaConverter {
592609
}
593610
return join_seq();
594611
};
595-
return add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\" space");
612+
return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\" space");
596613
}
597614

598615
/*
@@ -690,7 +707,7 @@ class SchemaConverter {
690707
const auto &prop_schema = kv.second;
691708

692709
std::string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name);
693-
prop_kv_rule_names[prop_name] = add_rule(
710+
prop_kv_rule_names[prop_name] = _add_rule(
694711
name + (name.empty() ? "" : "-") + prop_name + "-kv",
695712
format_literal(json(prop_name).dump()) + " space \":\" space " + prop_rule_name
696713
);
@@ -709,8 +726,8 @@ class SchemaConverter {
709726

710727
auto key_rule =
711728
prop_names.empty() ? _add_primitive("string", PRIMITIVE_RULES.at("string"))
712-
: add_rule(sub_name + "-k", _not_strings(prop_names));
713-
std::string kv_rule = add_rule(sub_name + "-kv", key_rule + " \":\" space " + value_rule);
729+
: _add_rule(sub_name + "-k", _not_strings(prop_names));
730+
std::string kv_rule = _add_rule(sub_name + "-kv", key_rule + " \":\" space " + value_rule);
714731
prop_kv_rule_names["*"] = kv_rule;
715732
optional_props.push_back("*");
716733
}
@@ -743,7 +760,7 @@ class SchemaConverter {
743760
res = kv_rule_name + (k == "*" ? " " + comma_ref + "*" : "");
744761
}
745762
if (ks.size() > 1) {
746-
res += " " + add_rule(
763+
res += " " + _add_rule(
747764
name + (name.empty() ? "" : "-") + k + "-rest",
748765
get_recursive_refs(std::vector<std::string>(ks.begin() + 1, ks.end()), true)
749766
);
@@ -769,7 +786,7 @@ class SchemaConverter {
769786
}
770787

771788
std::string _add_primitive(const std::string & name, const BuiltinRule & rule) {
772-
auto n = add_rule(name, rule.content);
789+
auto n = _add_rule(name, rule.content);
773790
for (const auto & dep : rule.deps) {
774791
BuiltinRule dep_rule;
775792
auto it = PRIMITIVE_RULES.find(dep);
@@ -796,22 +813,6 @@ class SchemaConverter {
796813
_rules["space"] = SPACE_RULE;
797814
}
798815

799-
std::string add_rule(const std::string & name, const std::string & rule) {
800-
std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-");
801-
if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) {
802-
_rules[esc_name] = rule;
803-
return esc_name;
804-
} else {
805-
int i = 0;
806-
while (_rules.find(esc_name + std::to_string(i)) != _rules.end() && _rules[esc_name + std::to_string(i)] != rule) {
807-
i++;
808-
}
809-
std::string key = esc_name + std::to_string(i);
810-
_rules[key] = rule;
811-
return key;
812-
}
813-
}
814-
815816
void resolve_refs(json & schema, const std::string & url) {
816817
/*
817818
* Resolves all $ref fields in the given schema, fetching any remote schemas,
@@ -883,26 +884,26 @@ class SchemaConverter {
883884
std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name;
884885

885886
if (schema.contains("$ref")) {
886-
return add_rule(rule_name, _resolve_ref(schema["$ref"]));
887+
return _add_rule(rule_name, _resolve_ref(schema["$ref"]));
887888
} else if (schema.contains("oneOf") || schema.contains("anyOf")) {
888889
std::vector<json> alt_schemas = schema.contains("oneOf") ? schema["oneOf"].get<std::vector<json>>() : schema["anyOf"].get<std::vector<json>>();
889-
return add_rule(rule_name, _generate_union_rule(name, alt_schemas));
890+
return _add_rule(rule_name, _generate_union_rule(name, alt_schemas));
890891
} else if (schema_type.is_array()) {
891892
std::vector<json> schema_types;
892893
for (const auto & t : schema_type) {
893894
json schema_copy(schema);
894895
schema_copy["type"] = t;
895896
schema_types.push_back(schema_copy);
896897
}
897-
return add_rule(rule_name, _generate_union_rule(name, schema_types));
898+
return _add_rule(rule_name, _generate_union_rule(name, schema_types));
898899
} else if (schema.contains("const")) {
899-
return add_rule(rule_name, _generate_constant_rule(schema["const"]) + " space");
900+
return _add_rule(rule_name, _generate_constant_rule(schema["const"]) + " space");
900901
} else if (schema.contains("enum")) {
901902
std::vector<std::string> enum_values;
902903
for (const auto & v : schema["enum"]) {
903904
enum_values.push_back(_generate_constant_rule(v));
904905
}
905-
return add_rule(rule_name, "(" + join(enum_values.begin(), enum_values.end(), " | ") + ") space");
906+
return _add_rule(rule_name, "(" + join(enum_values.begin(), enum_values.end(), " | ") + ") space");
906907
} else if ((schema_type.is_null() || schema_type == "object")
907908
&& (schema.contains("properties") ||
908909
(schema.contains("additionalProperties") && schema["additionalProperties"] != true))) {
@@ -920,7 +921,7 @@ class SchemaConverter {
920921
properties.emplace_back(prop.key(), prop.value());
921922
}
922923
}
923-
return add_rule(rule_name,
924+
return _add_rule(rule_name,
924925
_build_object_rule(
925926
properties, required, name,
926927
schema.contains("additionalProperties") ? schema["additionalProperties"] : json()));
@@ -951,7 +952,7 @@ class SchemaConverter {
951952
add_component(t, true);
952953
}
953954
}
954-
return add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json()));
955+
return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json()));
955956
} else if ((schema_type.is_null() || schema_type == "array") && (schema.contains("items") || schema.contains("prefixItems"))) {
956957
json items = schema.contains("items") ? schema["items"] : schema["prefixItems"];
957958
if (items.is_array()) {
@@ -963,27 +964,27 @@ class SchemaConverter {
963964
rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i));
964965
}
965966
rule += " \"]\" space";
966-
return add_rule(rule_name, rule);
967+
return _add_rule(rule_name, rule);
967968
} else {
968969
std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item");
969970
int min_items = schema.contains("minItems") ? schema["minItems"].get<int>() : 0;
970971
json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json();
971972
int max_items = max_items_json.is_number_integer() ? max_items_json.get<int>() : std::numeric_limits<int>::max();
972973

973-
return add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " \"]\" space");
974+
return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " \"]\" space");
974975
}
975976
} else if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) {
976977
return _visit_pattern(schema["pattern"], rule_name);
977978
} else if ((schema_type.is_null() || schema_type == "string") && std::regex_match(schema_format, std::regex("^uuid[1-5]?$"))) {
978979
return _add_primitive(rule_name == "root" ? "root" : schema_format, PRIMITIVE_RULES.at("uuid"));
979980
} else if ((schema_type.is_null() || schema_type == "string") && STRING_FORMAT_RULES.find(schema_format + "-string") != STRING_FORMAT_RULES.end()) {
980981
auto prim_name = schema_format + "-string";
981-
return add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name)));
982+
return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name)));
982983
} else if (schema_type == "string" && (schema.contains("minLength") || schema.contains("maxLength"))) {
983984
std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
984985
int min_len = schema.contains("minLength") ? schema["minLength"].get<int>() : 0;
985986
int max_len = schema.contains("maxLength") ? schema["maxLength"].get<int>() : std::numeric_limits<int>::max();
986-
return add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\" space");
987+
return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\" space");
987988
} else if (schema_type == "integer" && (schema.contains("minimum") || schema.contains("exclusiveMinimum") || schema.contains("maximum") || schema.contains("exclusiveMaximum"))) {
988989
int min_value = std::numeric_limits<int>::min();
989990
int max_value = std::numeric_limits<int>::max();
@@ -1001,9 +1002,9 @@ class SchemaConverter {
10011002
out << "(";
10021003
_build_min_max_int(min_value, max_value, out);
10031004
out << ") space";
1004-
return add_rule(rule_name, out.str());
1005+
return _add_rule(rule_name, out.str());
10051006
} else if (schema.empty() || schema_type == "object") {
1006-
return add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object")));
1007+
return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object")));
10071008
} else {
10081009
if (!schema_type.is_string() || PRIMITIVE_RULES.find(schema_type.get<std::string>()) == PRIMITIVE_RULES.end()) {
10091010
_errors.push_back("Unrecognized schema: " + schema.dump());
@@ -1044,7 +1045,7 @@ std::string build_grammar(const std::function<void(const llama_grammar_builder &
10441045
SchemaConverter converter([&](const std::string &) { return json(); }, /* dotall= */ false);
10451046
llama_grammar_builder builder {
10461047
/* .add_rule = */ [&](const std::string & name, const std::string & rule) {
1047-
return converter.add_rule(name, rule);
1048+
return converter._add_rule(name, rule);
10481049
},
10491050
/* .add_schema = */ [&](const std::string & name, const nlohmann::ordered_json & schema) {
10501051
return converter.visit(schema, name == "root" ? "" : name);

0 commit comments

Comments
 (0)