return tokens[current + offset];
}
+ const token & next() {
+ if (current >= tokens.size()) {
+ throw parser_exception("Parser Error: Unexpected EOF", source, tokens.empty() ? 0 : tokens.back().pos);
+ }
+ return tokens[current++];
+ }
+
token expect(token::type type, const std::string& error) {
const auto & t = peek();
if (t.t != type) {
size_t start_pos = current;
switch (peek().t) {
case token::comment:
- return mk_stmt<comment_statement>(start_pos, tokens[current++].value);
+ return mk_stmt<comment_statement>(start_pos, next().value);
case token::text:
- return mk_stmt<string_literal>(start_pos, tokens[current++].value);
+ return mk_stmt<string_literal>(start_pos, next().value);
case token::open_statement:
return parse_jinja_statement();
case token::open_expression:
}
size_t start_pos = current;
- std::string name = peek().value;
- current++; // consume identifier
+ std::string name = next().value;
statement_ptr result;
if (name == "set") {
// Ignore generation blocks (transformers-specific)
// See https://github.com/huggingface/transformers/pull/30650 for more information.
result = mk_stmt<noop_statement>(start_pos);
- current++;
+ ++current;
} else {
throw std::runtime_error("Unknown statement: " + name);
statements body;
if (is(token::equals)) {
- current++;
+ ++current;
value = parse_expression_sequence();
} else {
// parsing multiline set here
exprs.push_back(primary ? parse_primary_expression() : parse_expression());
bool is_tuple = is(token::comma);
while (is(token::comma)) {
- current++; // consume comma
+ ++current; // consume comma
exprs.push_back(primary ? parse_primary_expression() : parse_expression());
}
return is_tuple ? mk_stmt<tuple_literal>(start_pos, std::move(exprs)) : std::move(exprs[0]);
// e.g., `message` in `for message in messages`
auto loop_var = parse_expression_sequence(true); // should be an identifier/tuple
if (!is_identifier("in")) throw std::runtime_error("Expected 'in'");
- current++;
+ ++current; // consume 'in'
// `messages` in `for message in messages`
auto iterable = parse_expression();
}
if (is_statement({"else"})) {
- current += 2;
+ ++current; // consume {%
+ ++current; // consume 'else'
expect(token::close_statement, "Expected %}");
while (!is_statement({"endfor"})) {
alternate.push_back(parse_any());
auto left = parse_logical_and_expression();
while (is_identifier("or")) {
size_t start_pos = current;
- token op = tokens[current++];
+ token op = next();
left = mk_stmt<binary_expression>(start_pos, op, std::move(left), parse_logical_and_expression());
}
return left;
auto left = parse_logical_negation_expression();
while (is_identifier("and")) {
size_t start_pos = current;
- auto op = tokens[current++];
+ auto op = next();
left = mk_stmt<binary_expression>(start_pos, op, std::move(left), parse_logical_negation_expression());
}
return left;
// Try parse unary operators
if (is_identifier("not")) {
size_t start_pos = current;
- auto op = tokens[current++];
+ auto op = next();
return mk_stmt<unary_expression>(start_pos, op, parse_logical_negation_expression());
}
return parse_comparison_expression();
size_t start_pos = current;
if (is_identifier("not") && peek(1).t == token::identifier && peek(1).value == "in") {
op = {token::identifier, "not in", tokens[current].pos};
- current += 2;
+ ++current; // consume 'not'
+ ++current; // consume 'in'
} else if (is_identifier("in")) {
- op = tokens[current++];
+ op = next();
} else if (is(token::comparison_binary_operator)) {
- op = tokens[current++];
+ op = next();
} else break;
left = mk_stmt<binary_expression>(start_pos, op, std::move(left), parse_additive_expression());
}
auto left = parse_multiplicative_expression();
while (is(token::additive_binary_operator)) {
size_t start_pos = current;
- auto op = tokens[current++];
+ auto op = next();
left = mk_stmt<binary_expression>(start_pos, op, std::move(left), parse_multiplicative_expression());
}
return left;
auto left = parse_test_expression();
while (is(token::multiplicative_binary_operator)) {
size_t start_pos = current;
- auto op = tokens[current++];
+ auto op = next();
left = mk_stmt<binary_expression>(start_pos, op, std::move(left), parse_test_expression());
}
return left;
auto operand = parse_filter_expression();
while (is_identifier("is")) {
size_t start_pos = current;
- current++;
+ ++current; // consume 'is'
bool negate = false;
- if (is_identifier("not")) { current++; negate = true; }
+ if (is_identifier("not")) { ++current; negate = true; }
auto test_id = parse_primary_expression();
// FIXME: tests can also be expressed like this: if x is eq 3
if (is(token::open_paren)) test_id = parse_call_expression(std::move(test_id));
auto operand = parse_call_member_expression();
while (is(token::pipe)) {
size_t start_pos = current;
- current++;
+ ++current; // consume pipe
auto filter = parse_primary_expression();
if (is(token::open_paren)) filter = parse_call_expression(std::move(filter));
operand = mk_stmt<filter_expression>(start_pos, std::move(operand), std::move(filter));
statement_ptr parse_member_expression(statement_ptr object) {
size_t start_pos = current;
while (is(token::dot) || is(token::open_square_bracket)) {
- auto op = tokens[current++];
+ auto op = next();
bool computed = op.t == token::open_square_bracket;
statement_ptr prop;
if (computed) {
statement_ptr parse_primary_expression() {
size_t start_pos = current;
- auto t = tokens[current++];
+ auto t = next();
switch (t.t) {
case token::numeric_literal:
if (t.value.find('.') != std::string::npos) {
case token::string_literal: {
std::string val = t.value;
while (is(token::string_literal)) {
- val += tokens[current++].value;
+ val += next().value;
}
return mk_stmt<string_literal>(start_pos, val);
}
statements vals;
while (!is(token::close_square_bracket)) {
vals.push_back(parse_expression());
- if (is(token::comma)) current++;
+ if (is(token::comma)) ++current;
}
- current++;
+ ++current;
return mk_stmt<array_literal>(start_pos, std::move(vals));
}
case token::open_curly_bracket: {
auto key = parse_expression();
expect(token::colon, "Expected :");
pairs.push_back({std::move(key), parse_expression()});
- if (is(token::comma)) current++;
+ if (is(token::comma)) ++current;
}
- current++;
+ ++current;
return mk_stmt<object_literal>(start_pos, std::move(pairs));
}
default:
overview / pkg / ggml / sources / llama.cpp / commitdiff