common_peg_parse_result operator()(const common_peg_gbnf_parser & p) {
return arena.parse(p.child, ctx, start_pos);
}
+
+ common_peg_parse_result operator()(const common_peg_ac_parser & p) {
+ return arena.parse(p.child, ctx, start_pos);
+ }
};
common_peg_parse_result common_peg_arena::parse(common_peg_parse_context & ctx, size_t start) const {
std::is_same_v<T, common_peg_not_parser> ||
std::is_same_v<T, common_peg_tag_parser> ||
std::is_same_v<T, common_peg_atomic_parser> ||
- std::is_same_v<T, common_peg_gbnf_parser>) {
+ std::is_same_v<T, common_peg_gbnf_parser> ||
+ std::is_same_v<T, common_peg_ac_parser>) {
p.child = resolve_ref(p.child);
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
p.child = resolve_ref(p.child);
return "Atomic(" + dump_impl(p.child, visited) + ")";
} else if constexpr (std::is_same_v<T, common_peg_gbnf_parser>) {
return "Gbnf(" + p.grammar + ", " + dump_impl(p.child, visited) + ")";
+ } else if constexpr (std::is_same_v<T, common_peg_ac_parser>) {
+ return "Ac(" + string_join(p.delimiters, " | ") + ", " + dump_impl(p.child, visited) + ")";
} else if constexpr (std::is_same_v<T, common_peg_any_parser>) {
return "Any";
} else if constexpr (std::is_same_v<T, common_peg_space_parser>) {
});
}
+common_peg_parser common_peg_parser_builder::ac(const common_peg_parser & p, const std::vector<std::string> & delimiters) {
+ if (delimiters.empty()) {
+ throw std::runtime_error("ac parser requires at least one delimiter");
+ }
+ return add(common_peg_ac_parser{p, delimiters});
+}
+
static std::string gbnf_escape_char_class(uint32_t c) {
if (c == '-' || c == ']' || c == '[' || c == '\\') {
return "\\" + std::string(1, (char) c);
return std::string(buf);
}
-// GBNF grammar matching strings that contain no string in `strings` as a
-// substring. Emits the complement of an Aho-Corasick automaton DFA and returns
-// the start state rule name.
-//
-// ref: https://github.com/ggml-org/llama.cpp/pull/24839
-static std::string gbnf_excluding_grammar(const common_grammar_builder & builder,
- const std::string & prefix,
- const std::vector<std::string> & strings) {
+static std::string gbnf_char_class(const std::vector<uint32_t> & chars, bool negate) {
+ std::string s = negate ? "[^" : "[";
+ for (uint32_t ch : chars) {
+ s += gbnf_escape_char_class(ch);
+ }
+ return s + "]";
+}
+
+static std::string gbnf_ac_grammar(
+ const common_grammar_builder & builder,
+ const std::string & prefix,
+ const std::vector<std::string> & strings,
+ const std::function<std::string(const std::vector<uint32_t> &,
+ const std::map<size_t, std::vector<uint32_t>> &,
+ const std::vector<uint32_t> &,
+ const std::function<std::string(size_t)> &)> & build_rule) {
aho_corasick ac(strings);
auto state_name = [&](size_t s) -> std::string {
return prefix + "-" + num;
};
- auto char_class = [](const std::vector<uint32_t> & chars, bool negate) {
- std::string s = negate ? "[^" : "[";
- for (uint32_t ch : chars) {
- s += gbnf_escape_char_class(ch);
- }
- return s + "]";
- };
-
for (size_t q = 0; q < ac.num_states(); q++) {
if (ac.is_terminal(q)) {
- continue; // match states are dropped
+ continue; // match states
}
std::map<size_t, std::vector<uint32_t>> buckets;
- std::vector<uint32_t> excluded;
+ std::vector<uint32_t> completing; // chars that complete a delimiter
+ std::vector<uint32_t> specific; // chars with an explicit transition
for (uint32_t c : ac.alphabet) {
size_t d = ac.next(q, c);
if (ac.is_terminal(d)) {
- excluded.push_back(c); // completes a forbidden string -> omit
+ completing.push_back(c);
+ specific.push_back(c);
} else if (d != 0) {
buckets[d].push_back(c); // specific non-root destination
- excluded.push_back(c);
+ specific.push_back(c);
}
}
- std::string rhs = "|"; // every state is accepting
- for (const auto & [d, chars] : buckets) {
- rhs += " " + char_class(chars, false) + " " + state_name(d) + " |";
- }
- rhs += " " + char_class(excluded, true) + " " + state_name(0);
-
- builder.add_rule(state_name(q), rhs);
+ builder.add_rule(state_name(q), build_rule(completing, buckets, specific, state_name));
}
// An empty delimiter makes the start state terminal. Emit an entry rule
- // that matches nothing so the returned reference stays valid.
+ // that matches the empty string so the returned reference stays valid.
if (ac.is_terminal(0)) {
builder.add_rule(prefix, "|");
}
return state_name(0);
}
+// GBNF grammar matching strings that contain no string in `strings` as a
+// substring. Emits the complement of an Aho-Corasick automaton DFA and returns
+// the start state rule name.
+//
+// ref: https://github.com/ggml-org/llama.cpp/pull/24839
+static std::string gbnf_excluding_grammar(const common_grammar_builder & builder,
+ const std::string & prefix,
+ const std::vector<std::string> & strings) {
+ return gbnf_ac_grammar(builder, prefix, strings,
+ [](const std::vector<uint32_t> & /*completing*/,
+ const std::map<size_t, std::vector<uint32_t>> & buckets,
+ const std::vector<uint32_t> & specific,
+ const std::function<std::string(size_t)> & state_name) {
+ // every state is accepting and completing chars get no
+ // alternative, so a forbidden string can never be matched
+ std::string rhs = "|";
+ for (const auto & [d, chars] : buckets) {
+ rhs += " " + gbnf_char_class(chars, false) + " " + state_name(d) + " |";
+ }
+ rhs += " " + gbnf_char_class(specific, true) + " " + state_name(0);
+ return rhs;
+ });
+}
+
+// GBNF grammar matching everything up to and including the first occurrence of
+// any string in `strings`. Emits the Aho-Corasick automaton DFA and returns
+// the start state rule name.
+static std::string gbnf_including_grammar(const common_grammar_builder & builder,
+ const std::string & prefix,
+ const std::vector<std::string> & strings) {
+ return gbnf_ac_grammar(builder, prefix, strings,
+ [](const std::vector<uint32_t> & completing,
+ const std::map<size_t, std::vector<uint32_t>> & buckets,
+ const std::vector<uint32_t> & specific,
+ const std::function<std::string(size_t)> & state_name) {
+ std::vector<std::string> alts;
+ if (!completing.empty()) {
+ alts.push_back(gbnf_char_class(completing, false)); // terminate on match
+ }
+ for (const auto & [d, chars] : buckets) {
+ alts.push_back(gbnf_char_class(chars, false) + " " + state_name(d));
+ }
+ // every other character keeps scanning from the start state
+ alts.push_back(gbnf_char_class(specific, true) + " " + state_name(0));
+ return string_join(alts, " | ");
+ });
+}
+
static std::set<std::string> collect_reachable_rules(
const common_peg_arena & arena,
const common_peg_parser_id & rule
std::is_same_v<T, common_peg_tag_parser> ||
std::is_same_v<T, common_peg_atomic_parser> ||
std::is_same_v<T, common_peg_gbnf_parser> ||
+ std::is_same_v<T, common_peg_ac_parser> ||
std::is_same_v<T, common_peg_schema_parser>) {
visit(p.child);
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
return to_gbnf(p.child);
} else if constexpr (std::is_same_v<T, common_peg_gbnf_parser>) {
return p.grammar;
+ } else if constexpr (std::is_same_v<T, common_peg_ac_parser>) {
+ return gbnf_including_grammar(builder, "ac-" + std::to_string(id), p.delimiters);
} else {
static_assert(is_always_false_v<T>);
}
};
} else if constexpr (std::is_same_v<T, common_peg_gbnf_parser>) {
return json{{"type", "gbnf"}, {"child", p.child}, {"grammar", p.grammar}};
+ } else if constexpr (std::is_same_v<T, common_peg_ac_parser>) {
+ return json{{"type", "ac"}, {"child", p.child}, {"delimiters", p.delimiters}};
}
}, variant);
}
};
}
+ if (type == "ac") {
+ if (!j.contains("child") || !j.contains("delimiters") || !j["delimiters"].is_array() || j["delimiters"].empty()) {
+ throw std::runtime_error("ac parser requires 'child' and a non-empty 'delimiters' array");
+ }
+ return common_peg_ac_parser{
+ j["child"].get<common_peg_parser_id>(),
+ j["delimiters"].get<std::vector<std::string>>(),
+ };
+ }
+
throw std::runtime_error("Unknown parser type: " + type);
}
)""", gbnf);
});
+ t.test("ac grammar", [](testing &t) {
+ auto parser = build_peg_parser([](common_peg_parser_builder & p) {
+ return p.ac(p.until("</tag>") + p.literal("</tag>"), "</tag>");
+ });
+
+ auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
+ parser.build_grammar(builder);
+ });
+
+ assert_gbnf_equal(t, R"""(
+ ac-3 ::= [<] ac-3-01 | [^<] ac-3
+ ac-3-01 ::= [<] ac-3-01 | [/] ac-3-02 | [^/<] ac-3
+ ac-3-02 ::= [<] ac-3-01 | [t] ac-3-03 | [^<t] ac-3
+ ac-3-03 ::= [<] ac-3-01 | [a] ac-3-04 | [^<a] ac-3
+ ac-3-04 ::= [<] ac-3-01 | [g] ac-3-05 | [^<g] ac-3
+ ac-3-05 ::= [>] | [<] ac-3-01 | [^<>] ac-3
+ root ::= ac-3
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
+ )""", gbnf);
+ });
+
+ t.test("ac grammar terminates at first delimiter", [](testing &t) {
+ auto parser = build_peg_parser([](common_peg_parser_builder & p) {
+ return p.ac(p.until("\n</parameter>\n") + p.literal("\n</parameter>\n"), "\n</parameter>\n");
+ });
+
+ auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
+ parser.build_grammar(builder);
+ });
+
+ assert_gbnf_equal(t, R"""(
+ ac-3 ::= [\n] ac-3-01 | [^\n] ac-3
+ ac-3-01 ::= [\n] ac-3-01 | [<] ac-3-02 | [^\n<] ac-3
+ ac-3-02 ::= [\n] ac-3-01 | [/] ac-3-03 | [^\n/] ac-3
+ ac-3-03 ::= [\n] ac-3-01 | [p] ac-3-04 | [^\np] ac-3
+ ac-3-04 ::= [\n] ac-3-01 | [a] ac-3-05 | [^\na] ac-3
+ ac-3-05 ::= [\n] ac-3-01 | [r] ac-3-06 | [^\nr] ac-3
+ ac-3-06 ::= [\n] ac-3-01 | [a] ac-3-07 | [^\na] ac-3
+ ac-3-07 ::= [\n] ac-3-01 | [m] ac-3-08 | [^\nm] ac-3
+ ac-3-08 ::= [\n] ac-3-01 | [e] ac-3-09 | [^\ne] ac-3
+ ac-3-09 ::= [\n] ac-3-01 | [t] ac-3-10 | [^\nt] ac-3
+ ac-3-10 ::= [\n] ac-3-01 | [e] ac-3-11 | [^\ne] ac-3
+ ac-3-11 ::= [\n] ac-3-01 | [r] ac-3-12 | [^\nr] ac-3
+ ac-3-12 ::= [\n] ac-3-01 | [>] ac-3-13 | [^\n>] ac-3
+ ac-3-13 ::= [\n] | [^\n] ac-3
+ root ::= ac-3
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
+ )""", gbnf);
+ });
+
+ t.test("ac grammar multiple delimiters", [](testing &t) {
+ auto parser = build_peg_parser([](common_peg_parser_builder & p) {
+ return p.ac(p.eps(), std::vector<std::string>{"ab", "cd", "ef"});
+ });
+
+ auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
+ parser.build_grammar(builder);
+ });
+
+ assert_gbnf_equal(t, R"""(
+ ac-1 ::= [a] ac-1-01 | [c] ac-1-03 | [e] ac-1-05 | [^ace] ac-1
+ ac-1-01 ::= [b] | [a] ac-1-01 | [c] ac-1-03 | [e] ac-1-05 | [^abce] ac-1
+ ac-1-03 ::= [d] | [a] ac-1-01 | [c] ac-1-03 | [e] ac-1-05 | [^acde] ac-1
+ ac-1-05 ::= [f] | [a] ac-1-01 | [c] ac-1-03 | [e] ac-1-05 | [^acef] ac-1
+ root ::= ac-1
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
+ )""", gbnf);
+ });
+
t.test("complex expressions with parentheses", [](testing &t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
return p.one_or_more(p.literal("a") | p.literal("b"));
overview / pkg / ggml / sources / llama.cpp / commitdiff