•

Implements SPPF, for efficient parsing and storing of ambiguous grammars.

•

Implements a parse-aware lexer that provides a better power of expression than traditional LALR implementations (such as ply).

rule_name : list of rules and TERMINALS to match


          | another possible list of items


          | etc.
TERMINAL: "some text to match"

value: dict


     | list


     | STRING


     | NUMBER


     | "true" | "false" | "null"
list : "[" [value ("," value)*] "]"
dict : "{" [pair ("," pair)*] "}"
pair : STRING ":" value

%import common.ESCAPED_STRING   -> STRING
%import common.SIGNED_NUMBER    -> NUMBER

%import common.WS
%ignore WS

NUMBER : /-?\d+(\.\d+)?([eE][+-]?\d+)?/
STRING : /".*?(?<!\\)"/
%ignore /[ \t\n\f\r]+/

from lark import Lark
json_parser = Lark(r"""


    value: dict


         | list


         | ESCAPED_STRING


         | SIGNED_NUMBER


         | "true" | "false" | "null"


    list : "[" [value ("," value)*] "]"


    dict : "{" [pair ("," pair)*] "}"


    pair : ESCAPED_STRING ":" value


    %import common.ESCAPED_STRING


    %import common.SIGNED_NUMBER


    %import common.WS


    %ignore WS


    """, start='value')

>>> text = '{"key": ["item0", "item1", 3.14]}'
>>> json_parser.parse(text)
Tree(value, [Tree(dict, [Tree(pair, [Token(STRING, "key"), Tree(value, [Tree(list, [Tree(value, [Token(STRING, "item0")]), Tree(value, [Token(STRING, "item1")]), Tree(value, [Token(NUMBER, 3.14)])])])])])])
>>> print( _.pretty() )
value


  dict


    pair


      "key"


      value


        list


          value	"item0"


          value	"item1"


          value	3.14

?value: dict


      | list


      | string


      | SIGNED_NUMBER      -> number


      | "true"             -> true


      | "false"            -> false


      | "null"             -> null
...
string : ESCAPED_STRING

from lark import Lark
json_parser = Lark(r"""


    ?value: dict


          | list


          | string


          | SIGNED_NUMBER      -> number


          | "true"             -> true


          | "false"            -> false


          | "null"             -> null


    list : "[" [value ("," value)*] "]"


    dict : "{" [pair ("," pair)*] "}"


    pair : string ":" value


    string : ESCAPED_STRING


    %import common.ESCAPED_STRING


    %import common.SIGNED_NUMBER


    %import common.WS


    %ignore WS


    """, start='value')

>>> text = '{"key": ["item0", "item1", 3.14, true]}'
>>> print( json_parser.parse(text).pretty() )
dict


  pair


    string	"key"


    list


      string	"item0"


      string	"item1"


      number	3.14


      true

from lark import Transformer
class MyTransformer(Transformer):


    def list(self, items):


        return list(items)


    def pair(self, key_value):


        k, v = key_value


        return k, v


    def dict(self, items):


        return dict(items)

>>> tree = json_parser.parse(text)
>>> MyTransformer().transform(tree)
{Tree(string, [Token(ANONRE_1, "key")]): [Tree(string, [Token(ANONRE_1, "item0")]), Tree(string, [Token(ANONRE_1, "item1")]), Tree(number, [Token(ANONRE_0, 3.14)]), Tree(true, [])]}

from lark import Transformer
class TreeToJson(Transformer):


    def string(self, s):


        (s,) = s


        return s[1:-1]


    def number(self, n):


        (n,) = n


        return float(n)


    list = list


    pair = tuple


    dict = dict


    null = lambda self, _: None


    true = lambda self, _: True


    false = lambda self, _: False

>>> tree = json_parser.parse(text)
>>> TreeToJson().transform(tree)
{u'key': [u'item0', u'item1', 3.14, True]}

import sys
from lark import Lark, Transformer
json_grammar = r"""


    ?value: dict


          | list


          | string


          | SIGNED_NUMBER      -> number


          | "true"             -> true


          | "false"            -> false


          | "null"             -> null


    list : "[" [value ("," value)*] "]"


    dict : "{" [pair ("," pair)*] "}"


    pair : string ":" value


    string : ESCAPED_STRING


    %import common.ESCAPED_STRING


    %import common.SIGNED_NUMBER


    %import common.WS


    %ignore WS


    """
class TreeToJson(Transformer):


    def string(self, s):


        (s,) = s


        return s[1:-1]


    def number(self, n):


        (n,) = n


        return float(n)


    list = list


    pair = tuple


    dict = dict


    null = lambda self, _: None


    true = lambda self, _: True


    false = lambda self, _: False
json_parser = Lark(json_grammar, start='value', lexer='standard')
if __name__ == '__main__':


    with open(sys.argv[1]) as f:


        tree = json_parser.parse(f.read())


        print(TreeToJson().transform(tree))

$ time python tutorial_json.py json_data > /dev/null
real	0m36.257s
user	0m34.735s
sys         0m1.361s

json_parser = Lark(json_grammar, start='value', parser='lalr')

$ time python tutorial_json.py json_data > /dev/null
real        0m7.554s
user        0m7.352s
sys         0m0.148s

json_parser = Lark(json_grammar, start='value', parser='lalr', transformer=TreeToJson())
if __name__ == '__main__':


    with open(sys.argv[1]) as f:


        print( json_parser.parse(f.read()) )

real	0m4.866s
user	0m4.722s
sys 	0m0.121s

$ time pypy tutorial_json.py json_data > /dev/null
real	0m1.397s
user	0m1.296s
sys 	0m0.083s

import logging
from lark import Lark, logger
logger.setLevel(logging.DEBUG)
collision_grammar = '''
start: as as
as: a*
a: "a"
'''
p = Lark(collision_grammar, parser='lalr', debug=True)

python -m tests

pypy -m tests

##   test_package test_class_name.test_function_name
python -m tests TestLalrStandard.test_lexer_error_recovering

pytest tests

python setup.py test 

from lark import Lark, Transformer
class T(Transformer):


    def INT(self, tok):


        "Convert the value of `tok` from string to int, while maintaining line number & column."


        return tok.update(value=int(tok))
parser = Lark("""
start: INT*
%import common.INT
%ignore " "
""", parser="lalr", transformer=T())
print(parser.parse('3 14 159'))

Tree(start, [Token(INT, 3), Token(INT, 14), Token(INT, 159)])

{TOKEN_TYPE: callback}

from lark import Lark
comments = []
parser = Lark("""


    start: INT*


    COMMENT: /#.*/


    %import common (INT, WS)


    %ignore COMMENT


    %ignore WS
""", parser="lalr", lexer_callbacks={'COMMENT': comments.append})
parser.parse("""
1 2 3  # hello
# world
4 5 6
""")
print(comments)

[Token(COMMENT, '# hello'), Token(COMMENT, '# world')]

from lark import Lark, Tree, Transformer
from lark.visitors import CollapseAmbiguities
grammar = """


    !start: x y


    !x: "a" "b"


      | "ab"


      | "abc"


    !y: "c" "d"


      | "cd"


      | "d"
"""
parser = Lark(grammar, ambiguity='explicit')
t = parser.parse('abcd')
for x in CollapseAmbiguities().transform(t):


    print(x.pretty())

start
x


    a


    b
y


    c


    d
start
x     ab
y     cd
start
x     abc
y     d

class Parent(Visitor):


    def __default__(self, tree):


        for subtree in tree.children:


            if isinstance(subtree, Tree):


                assert not hasattr(subtree, 'parent')


                subtree.parent = tree

from lark import Lark
from lark.indenter import Indenter
tree_grammar = r"""


    ?start: _NL* tree


    tree: NAME _NL [_INDENT tree+ _DEDENT]


    %import common.CNAME -> NAME


    %import common.WS_INLINE


    %declare _INDENT _DEDENT


    %ignore WS_INLINE


    _NL: /(\r?\n[\t ]*)+/
"""
class TreeIndenter(Indenter):


    NL_type = '_NL'


    OPEN_PAREN_types = []


    CLOSE_PAREN_types = []


    INDENT_type = '_INDENT'


    DEDENT_type = '_DEDENT'


    tab_len = 8
parser = Lark(tree_grammar, parser='lalr', postlex=TreeIndenter())
test_tree = """
a


    b


    c


        d


        e


    f


        g
"""
def test():


    print(parser.parse(test_tree).pretty())
if __name__ == '__main__':


    test()

import lark
from pathlib import Path
parser = lark.Lark.open('lark.lark', rel_to=__file__, parser="lalr")
examples_path = Path(__file__).parent
lark_path = Path(lark.__file__).parent
grammar_files = [


    examples_path / 'lark.lark',


    examples_path / 'advanced/python2.lark',


    examples_path / 'advanced/python3.lark',


    examples_path / 'relative-imports/multiples.lark',


    examples_path / 'relative-imports/multiple2.lark',


    examples_path / 'relative-imports/multiple3.lark',


    examples_path / 'tests/no_newline_at_end.lark',


    examples_path / 'tests/negative_priority.lark',


    lark_path / 'grammars/common.lark',
]
def test():


    for grammar_file in grammar_files:


        tree = parser.parse(open(grammar_file).read())


    print("All grammars parsed successfully")
if __name__ == '__main__':


    test()

import sys
from lark import Lark, tree
grammar = """


    sentence: noun verb noun        -> simple


            | noun verb "like" noun -> comparative


    noun: adj? NOUN


    verb: VERB


    adj: ADJ


    NOUN: "flies" | "bananas" | "fruit"


    VERB: "like" | "flies"


    ADJ: "fruit"


    %import common.WS


    %ignore WS
"""
parser = Lark(grammar, start='sentence', ambiguity='explicit')
sentence = 'fruit flies like bananas'
def make_png(filename):


    tree.pydot__tree_to_png( parser.parse(sentence), filename)
def make_dot(filename):


    tree.pydot__tree_to_dot( parser.parse(sentence), filename)
if __name__ == '__main__':


    print(parser.parse(sentence).pretty())


    # make_png(sys.argv[1])


    # make_dot(sys.argv[1])
# Output:
#
# _ambig
#   comparative
#     noun  fruit
#     verb  flies
#     noun  bananas
#   simple
#     noun
#       fruit
#       flies
#     verb  like
#     noun  bananas
#
# (or view a nicer version at "./fruitflies.png")

from lark import Lark, Transformer, v_args
try:


    input = raw_input   # For Python2 compatibility
except NameError:


    pass
calc_grammar = """


    ?start: sum


          | NAME "=" sum    -> assign_var


    ?sum: product


        | sum "+" product   -> add


        | sum "-" product   -> sub


    ?product: atom


        | product "*" atom  -> mul


        | product "/" atom  -> div


    ?atom: NUMBER           -> number


         | "-" atom         -> neg


         | NAME             -> var


         | "(" sum ")"


    %import common.CNAME -> NAME


    %import common.NUMBER


    %import common.WS_INLINE


    %ignore WS_INLINE
"""
@v_args(inline=True)    # Affects the signatures of the methods
class CalculateTree(Transformer):


    from operator import add, sub, mul, truediv as div, neg


    number = float


    def __init__(self):


        self.vars = {}


    def assign_var(self, name, value):


        self.vars[name] = value


        return value


    def var(self, name):


        try:


            return self.vars[name]


        except KeyError:


            raise Exception("Variable not found: %s" % name)
calc_parser = Lark(calc_grammar, parser='lalr', transformer=CalculateTree())
calc = calc_parser.parse
def main():


    while True:


        try:


            s = input('> ')


        except EOFError:


            break


        print(calc(s))
def test():


    print(calc("a = 1+2"))


    print(calc("1+a*-3"))
if __name__ == '__main__':


    # test()


    main()

try:


    input = raw_input   # For Python2 compatibility
except NameError:


    pass
import turtle
from lark import Lark
turtle_grammar = """


    start: instruction+


    instruction: MOVEMENT NUMBER            -> movement


               | "c" COLOR [COLOR]          -> change_color


               | "fill" code_block          -> fill


               | "repeat" NUMBER code_block -> repeat


    code_block: "{" instruction+ "}"


    MOVEMENT: "f"|"b"|"l"|"r"


    COLOR: LETTER+


    %import common.LETTER


    %import common.INT -> NUMBER


    %import common.WS


    %ignore WS
"""
parser = Lark(turtle_grammar)
def run_instruction(t):


    if t.data == 'change_color':


        turtle.color(*t.children)   # We just pass the color names as-is


    elif t.data == 'movement':


        name, number = t.children


        { 'f': turtle.fd,


          'b': turtle.bk,


          'l': turtle.lt,


          'r': turtle.rt, }[name](int(number))


    elif t.data == 'repeat':


        count, block = t.children


        for i in range(int(count)):


            run_instruction(block)


    elif t.data == 'fill':


        turtle.begin_fill()


        run_instruction(t.children[0])


        turtle.end_fill()


    elif t.data == 'code_block':


        for cmd in t.children:


            run_instruction(cmd)


    else:


        raise SyntaxError('Unknown instruction: %s' % t.data)
def run_turtle(program):


    parse_tree = parser.parse(program)


    for inst in parse_tree.children:


        run_instruction(inst)
def main():


    while True:


        code = input('> ')


        try:


            run_turtle(code)


        except Exception as e:


            print(e)
def test():


    text = """


        c red yellow


        fill { repeat 36 {


            f200 l170


        }}


    """


    run_turtle(text)
if __name__ == '__main__':


    # test()


    main()

import sys
from lark import Lark, Transformer, v_args
json_grammar = r"""


    ?start: value


    ?value: object


          | array


          | string


          | SIGNED_NUMBER      -> number


          | "true"             -> true


          | "false"            -> false


          | "null"             -> null


    array  : "[" [value ("," value)*] "]"


    object : "{" [pair ("," pair)*] "}"


    pair   : string ":" value


    string : ESCAPED_STRING


    %import common.ESCAPED_STRING


    %import common.SIGNED_NUMBER


    %import common.WS


    %ignore WS
"""
class TreeToJson(Transformer):


    @v_args(inline=True)


    def string(self, s):


        return s[1:-1].replace('\\"', '"')


    array = list


    pair = tuple


    object = dict


    number = v_args(inline=True)(float)


    null = lambda self, _: None


    true = lambda self, _: True


    false = lambda self, _: False
### Create the JSON parser with Lark, using the Earley algorithm
# json_parser = Lark(json_grammar, parser='earley', lexer='standard')
# def parse(x):
#     return TreeToJson().transform(json_parser.parse(x))
### Create the JSON parser with Lark, using the LALR algorithm
json_parser = Lark(json_grammar, parser='lalr',


                   # Using the standard lexer isn't required, and isn't usually recommended.


                   # But, it's good enough for JSON, and it's slightly faster.


                   lexer='standard',


                   # Disabling propagate_positions and placeholders slightly improves speed


                   propagate_positions=False,


                   maybe_placeholders=False,


                   # Using an internal transformer is faster and more memory efficient


                   transformer=TreeToJson())
parse = json_parser.parse
def test():


    test_json = '''


        {


            "empty_object" : {},


            "empty_array"  : [],


            "booleans"     : { "YES" : true, "NO" : false },


            "numbers"      : [ 0, 1, -2, 3.3, 4.4e5, 6.6e-7 ],


            "strings"      : [ "This", [ "And" , "That", "And a \\"b" ] ],


            "nothing"      : null


        }


    '''


    j = parse(test_json)


    print(j)


    import json


    assert j == json.loads(test_json)
if __name__ == '__main__':


    # test()


    with open(sys.argv[1]) as f:


        print(parse(f.read()))

from lark import Lark
parser = Lark(r"""


        start: _NL? section+


        section: "[" NAME "]" _NL item+


        item: NAME "=" VALUE? _NL


        VALUE: /./+


        %import common.CNAME -> NAME


        %import common.NEWLINE -> _NL


        %import common.WS_INLINE


        %ignore WS_INLINE


    """, parser="lalr")
sample_conf = """
[bla]
a=Hello
this="that",4
empty=
"""
print(parser.parse(sample_conf).pretty())

from lark import Lark
grammar = r"""
start: list | dict
list: "[" _seperated{atom, ","} "]"
dict: "{" _seperated{key_value, ","} "}"
key_value: atom ":" atom
_seperated{x, sep}: x (sep x)*  // Define a sequence of 'x sep x sep x ...'
atom: NUMBER | ESCAPED_STRING
%import common (NUMBER, ESCAPED_STRING, WS)
%ignore WS
"""
parser = Lark(grammar)
print(parser.parse('[1, "a", 2]'))
print(parser.parse('{"a": 2, "b": 6}'))

from lark import Lark
parser = Lark(r"""


        start: _NL? section+


        section: "[" NAME "]" _NL item+


        item: NAME "=" VALUE? _NL


        VALUE: /./+


        %import common.CNAME -> NAME


        %import common.NEWLINE -> _NL


        %import common.WS_INLINE


        %ignore WS_INLINE


    """, parser="earley")
def test():


    sample_conf = """
[bla]
a=Hello
this="that",4
empty=
"""


    r = parser.parse(sample_conf)


    print (r.pretty())
if __name__ == '__main__':


    test()

from lark import Token
from _json_parser import json_parser
def ignore_errors(e):


    if e.token.type == 'COMMA':


        # Skip comma


        return True


    elif e.token.type == 'SIGNED_NUMBER':


        # Try to feed a comma and retry the number


        e.puppet.feed_token(Token('COMMA', ','))


        e.puppet.feed_token(e.token)


        return True


    # Unhandled error. Will stop parse and raise exception


    return False
def main():


    s = "[0 1, 2,, 3,,, 4, 5 6 ]"


    res = json_parser.parse(s, on_error=ignore_errors)


    print(res)      # prints [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0]
main()

import json
from lark import Lark
from lark.reconstruct import Reconstructor
from _json_parser import json_grammar
test_json = '''


    {


        "empty_object" : {},


        "empty_array"  : [],


        "booleans"     : { "YES" : true, "NO" : false },


        "numbers"      : [ 0, 1, -2, 3.3, 4.4e5, 6.6e-7 ],


        "strings"      : [ "This", [ "And" , "That", "And a \\"b" ] ],


        "nothing"      : null


    }
'''
def test_earley():


    json_parser = Lark(json_grammar, maybe_placeholders=False)


    tree = json_parser.parse(test_json)


    new_json = Reconstructor(json_parser).reconstruct(tree)


    print (new_json)


    print (json.loads(new_json) == json.loads(test_json))
def test_lalr():


    json_parser = Lark(json_grammar, parser='lalr', maybe_placeholders=False)


    tree = json_parser.parse(test_json)


    new_json = Reconstructor(json_parser).reconstruct(tree)


    print (new_json)


    print (json.loads(new_json) == json.loads(test_json))
test_earley()
test_lalr()

from lark import Lark, Transformer, v_args
from lark.lexer import Lexer, Token
class TypeLexer(Lexer):


    def __init__(self, lexer_conf):


        pass


    def lex(self, data):


        for obj in data:


            if isinstance(obj, int):


                yield Token('INT', obj)


            elif isinstance(obj, (type(''), type(u''))):


                yield Token('STR', obj)


            else:


                raise TypeError(obj)
parser = Lark("""


        start: data_item+


        data_item: STR INT*


        %declare STR INT


        """, parser='lalr', lexer=TypeLexer)
class ParseToDict(Transformer):


    @v_args(inline=True)


    def data_item(self, name, *numbers):


        return name.value, [n.value for n in numbers]


    start = dict
def test():


    data = ['alice', 1, 27, 3, 'bob', 4, 'carrie', 'dan', 8, 6]


    print(data)


    tree = parser.parse(data)


    res = ParseToDict().transform(tree)


    print('-->')


    print(res) # prints {'alice': [1, 27, 3], 'bob': [4], 'carrie': [], 'dan': [8, 6]}
if __name__ == '__main__':


    test()

from lark import Lark
from lark.parsers.earley_forest import TreeForestTransformer, handles_ambiguity, Discard
class CustomTransformer(TreeForestTransformer):


    @handles_ambiguity


    def sentence(self, trees):


        return next(tree for tree in trees if tree.data == 'simple')


    def simple(self, children):


        children.append('.')


        return self.tree_class('simple', children)


    def adj(self, children):


        raise Discard()


    def __default_token__(self, token):


        return token.capitalize()
grammar = """


    sentence: noun verb noun        -> simple


            | noun verb "like" noun -> comparative


    noun: adj? NOUN


    verb: VERB


    adj: ADJ


    NOUN: "flies" | "bananas" | "fruit"


    VERB: "like" | "flies"


    ADJ: "fruit"


    %import common.WS


    %ignore WS
"""
parser = Lark(grammar, start='sentence', ambiguity='forest')
sentence = 'fruit flies like bananas'
forest = parser.parse(sentence)
tree = CustomTransformer(resolve_ambiguity=False).transform(forest)
print(tree.pretty())
# Output:
#
# simple
#   noun  Flies
#   verb  Like
#   noun  Bananas
#   .
#

import sys
from lark import Lark, Transformer, v_args
json_grammar = r"""


    ?start: value


    ?value: object


          | array


          | string


          | SIGNED_NUMBER      -> number


          | "true"             -> true


          | "false"            -> false


          | "null"             -> null


    array  : "[" [value ("," value)*] "]"


    object : "{" [pair ("," pair)*] "}"


    pair   : string ":" value


    string : ESCAPED_STRING


    %import common.ESCAPED_STRING


    %import common.SIGNED_NUMBER


    %import common.WS


    %ignore WS
"""
class TreeToJson(Transformer):


    @v_args(inline=True)


    def string(self, s):


        return s[1:-1].replace('\\"', '"')


    array = list


    pair = tuple


    object = dict


    number = v_args(inline=True)(float)


    null = lambda self, _: None


    true = lambda self, _: True


    false = lambda self, _: False
### Create the JSON parser with Lark, using the LALR algorithm
json_parser = Lark(json_grammar, parser='lalr',


                   # Using the standard lexer isn't required, and isn't usually recommended.


                   # But, it's good enough for JSON, and it's slightly faster.


                   lexer='standard',


                   # Disabling propagate_positions and placeholders slightly improves speed


                   propagate_positions=False,


                   maybe_placeholders=False,


                   # Using an internal transformer is faster and more memory efficient


                   transformer=TreeToJson())

from lark import Lark
from lark.parsers.earley_forest import ForestVisitor, TreeForestTransformer
class TokenPrioritizer(ForestVisitor):


    def visit_symbol_node_in(self, node):


        # visit the entire forest by returning node.children


        return node.children


    def visit_packed_node_in(self, node):


        return node.children


    def visit_symbol_node_out(self, node):


        priority = 0


        for child in node.children:


            # Tokens do not have a priority attribute


            # count them as -1


            priority += getattr(child, 'priority', -1)


        node.priority = priority


    def visit_packed_node_out(self, node):


        priority = 0


        for child in node.children:


            priority += getattr(child, 'priority', -1)


        node.priority = priority


    def on_cycle(self, node, path):


        raise Exception("Oops, we encountered a cycle.")
grammar = """
start: hello " " world | hello_world
hello: "Hello"
world: "World"
hello_world: "Hello World"
"""
parser = Lark(grammar, parser='earley', ambiguity='forest')
forest = parser.parse("Hello World")
print("Default prioritizer:")
tree = TreeForestTransformer(resolve_ambiguity=True).transform(forest)
print(tree.pretty())
forest = parser.parse("Hello World")
print("Custom prioritizer:")
tree = TreeForestTransformer(resolve_ambiguity=True, prioritizer=TokenPrioritizer()).transform(forest)
print(tree.pretty())
# Output:
#
# Default prioritizer:
# start
#   hello Hello
#
#   world World
#
# Custom prioritizer:
# start
#   hello_world   Hello World

$ pip install bytecode

from lark import Lark, Transformer, v_args
from lark.indenter import Indenter
from bytecode import Instr, Bytecode
class PythonIndenter(Indenter):


    NL_type = '_NEWLINE'


    OPEN_PAREN_types = ['LPAR', 'LSQB', 'LBRACE']


    CLOSE_PAREN_types = ['RPAR', 'RSQB', 'RBRACE']


    INDENT_type = '_INDENT'


    DEDENT_type = '_DEDENT'


    tab_len = 8
@v_args(inline=True)
class Compile(Transformer):


    def number(self, n):


        return [Instr('LOAD_CONST', int(n))]


    def string(self, s):


        return [Instr('LOAD_CONST', s[1:-1])]


    def var(self, n):


        return [Instr('LOAD_NAME', n)]


    def arith_expr(self, a, op, b):


        # TODO support chain arithmetic


        assert op == '+'


        return a + b + [Instr('BINARY_ADD')]


    def arguments(self, args):


        return args


    def funccall(self, name, args):


        return name + args + [Instr('CALL_FUNCTION', 1)]


    @v_args(inline=False)


    def file_input(self, stmts):


        return sum(stmts, []) + [Instr("RETURN_VALUE")]


    def expr_stmt(self, lval, rval):


        # TODO more complicated than that


        name ,= lval


        assert name.name == 'LOAD_NAME' # XXX avoid with another layer of abstraction


        return rval + [Instr("STORE_NAME", name.arg)]


    def __default__(self, *args):


        assert False, args
python_parser3 = Lark.open('python3.lark', rel_to=__file__, start='file_input',


                           parser='lalr', postlex=PythonIndenter(),


                           transformer=Compile(), propagate_positions=False)
def compile_python(s):


    insts = python_parser3.parse(s+"\n")


    return Bytecode(insts).to_code()
code = compile_python("""
a = 3
b = 5
print("Hello World!")
print(a+(b+2))
print((a+b)+2)
""")
exec(code)
# -- Output --
# Hello World!
# 10
# 10

import sys
import os, os.path
from io import open
import glob, time
from lark import Lark
from lark.indenter import Indenter
# __path__ = os.path.dirname(__file__)
class PythonIndenter(Indenter):


    NL_type = '_NEWLINE'


    OPEN_PAREN_types = ['LPAR', 'LSQB', 'LBRACE']


    CLOSE_PAREN_types = ['RPAR', 'RSQB', 'RBRACE']


    INDENT_type = '_INDENT'


    DEDENT_type = '_DEDENT'


    tab_len = 8
kwargs = dict(rel_to=__file__, postlex=PythonIndenter(), start='file_input')
python_parser2 = Lark.open('python2.lark', parser='lalr', **kwargs)
python_parser3 = Lark.open('python3.lark',parser='lalr', **kwargs)
python_parser2_earley = Lark.open('python2.lark', parser='earley', lexer='standard', **kwargs)
try:


    xrange
except NameError:


    chosen_parser = python_parser3
else:


    chosen_parser = python_parser2
def _read(fn, *args):


    kwargs = {'encoding': 'iso-8859-1'}


    with open(fn, *args, **kwargs) as f:


        return f.read()
def _get_lib_path():


    if os.name == 'nt':


        if 'PyPy' in sys.version:


            return os.path.join(sys.prefix, 'lib-python', sys.winver)


        else:


            return os.path.join(sys.prefix, 'Lib')


    else:


        return [x for x in sys.path if x.endswith('%s.%s' % sys.version_info[:2])][0]
def test_python_lib():


    path = _get_lib_path()


    start = time.time()


    files = glob.glob(path+'/*.py')


    for f in files:


        print( f )


        chosen_parser.parse(_read(os.path.join(path, f)) + '\n')


    end = time.time()


    print( "test_python_lib (%d files), time: %s secs"%(len(files), end-start) )
def test_earley_equals_lalr():


    path = _get_lib_path()


    files = glob.glob(path+'/*.py')


    for f in files:


        print( f )


        tree1 = python_parser2.parse(_read(os.path.join(path, f)) + '\n')


        tree2 = python_parser2_earley.parse(_read(os.path.join(path, f)) + '\n')


        assert tree1 == tree2
if __name__ == '__main__':


    test_python_lib()


    # test_earley_equals_lalr()


    # python_parser3.parse(_read(sys.argv[1]) + '\n')

from lark import Lark, UnexpectedInput
from _json_parser import json_grammar   # Using the grammar from the json_parser example
json_parser = Lark(json_grammar, parser='lalr')
class JsonSyntaxError(SyntaxError):


    def __str__(self):


        context, line, column = self.args


        return '%s at line %s, column %s.\n\n%s' % (self.label, line, column, context)
class JsonMissingValue(JsonSyntaxError):


    label = 'Missing Value'
class JsonMissingOpening(JsonSyntaxError):


    label = 'Missing Opening'
class JsonMissingClosing(JsonSyntaxError):


    label = 'Missing Closing'
class JsonMissingComma(JsonSyntaxError):


    label = 'Missing Comma'
class JsonTrailingComma(JsonSyntaxError):


    label = 'Trailing Comma'
def parse(json_text):


    try:


        j = json_parser.parse(json_text)


    except UnexpectedInput as u:


        exc_class = u.match_examples(json_parser.parse, {


            JsonMissingOpening: ['{"foo": ]}',


                                 '{"foor": }}',


                                 '{"foo": }'],


            JsonMissingClosing: ['{"foo": [}',


                                 '{',


                                 '{"a": 1',


                                 '[1'],


            JsonMissingComma: ['[1 2]',


                               '[false 1]',


                               '["b" 1]',


                               '{"a":true 1:4}',


                               '{"a":1 1:4}',


                               '{"a":"b" 1:4}'],


            JsonTrailingComma: ['[,]',


                                '[1,]',


                                '[1,2,]',


                                '{"foo":1,}',


                                '{"foo":false,"bar":true,}']


        }, use_accepts=True)


        if not exc_class:


            raise


        raise exc_class(u.get_context(json_text), u.line, u.column)
def test():


    try:


        parse('{"example1": "value"')


    except JsonMissingClosing as e:


        print(e)


    try:


        parse('{"example2": ] ')


    except JsonMissingOpening as e:


        print(e)
if __name__ == '__main__':


    test()

import sys
import textwrap
from PyQt5.Qt import *  # noqa
from PyQt5.Qsci import QsciScintilla
from PyQt5.Qsci import QsciLexerCustom
from lark import Lark
class LexerJson(QsciLexerCustom):


    def __init__(self, parent=None):


        super().__init__(parent)


        self.create_parser()


        self.create_styles()


    def create_styles(self):


        deeppink = QColor(249, 38, 114)


        khaki = QColor(230, 219, 116)


        mediumpurple = QColor(174, 129, 255)


        mediumturquoise = QColor(81, 217, 205)


        yellowgreen = QColor(166, 226, 46)


        lightcyan = QColor(213, 248, 232)


        darkslategrey = QColor(39, 40, 34)


        styles = {


            0: mediumturquoise,


            1: mediumpurple,


            2: yellowgreen,


            3: deeppink,


            4: khaki,


            5: lightcyan


        }


        for style, color in styles.items():


            self.setColor(color, style)


            self.setPaper(darkslategrey, style)


            self.setFont(self.parent().font(), style)


        self.token_styles = {


            "COLON": 5,


            "COMMA": 5,


            "LBRACE": 5,


            "LSQB": 5,


            "RBRACE": 5,


            "RSQB": 5,


            "FALSE": 0,


            "NULL": 0,


            "TRUE": 0,


            "STRING": 4,


            "NUMBER": 1,


        }


    def create_parser(self):


        grammar = '''


            anons: ":" "{" "}" "," "[" "]"


            TRUE: "true"


            FALSE: "false"


            NULL: "NULL"


            %import common.ESCAPED_STRING -> STRING


            %import common.SIGNED_NUMBER  -> NUMBER


            %import common.WS


            %ignore WS


        '''


        self.lark = Lark(grammar, parser=None, lexer='standard')


        # All tokens: print([t.name for t in self.lark.parser.lexer.tokens])


    def defaultPaper(self, style):


        return QColor(39, 40, 34)


    def language(self):


        return "Json"


    def description(self, style):


        return {v: k for k, v in self.token_styles.items()}.get(style, "")


    def styleText(self, start, end):


        self.startStyling(start)


        text = self.parent().text()[start:end]


        last_pos = 0


        try:


            for token in self.lark.lex(text):


                ws_len = token.pos_in_stream - last_pos


                if ws_len:


                    self.setStyling(ws_len, 0)    # whitespace


                token_len = len(bytearray(token, "utf-8"))


                self.setStyling(


                    token_len, self.token_styles.get(token.type, 0))


                last_pos = token.pos_in_stream + token_len


        except Exception as e:


            print(e)
class EditorAll(QsciScintilla):


    def __init__(self, parent=None):


        super().__init__(parent)


        # Set font defaults


        font = QFont()


        font.setFamily('Consolas')


        font.setFixedPitch(True)


        font.setPointSize(8)


        font.setBold(True)


        self.setFont(font)


        # Set margin defaults


        fontmetrics = QFontMetrics(font)


        self.setMarginsFont(font)


        self.setMarginWidth(0, fontmetrics.width("000") + 6)


        self.setMarginLineNumbers(0, True)


        self.setMarginsForegroundColor(QColor(128, 128, 128))


        self.setMarginsBackgroundColor(QColor(39, 40, 34))


        self.setMarginType(1, self.SymbolMargin)


        self.setMarginWidth(1, 12)


        # Set indentation defaults


        self.setIndentationsUseTabs(False)


        self.setIndentationWidth(4)


        self.setBackspaceUnindents(True)


        self.setIndentationGuides(True)


        # self.setFolding(QsciScintilla.CircledFoldStyle)


        # Set caret defaults


        self.setCaretForegroundColor(QColor(247, 247, 241))


        self.setCaretWidth(2)


        # Set selection color defaults


        self.setSelectionBackgroundColor(QColor(61, 61, 52))


        self.resetSelectionForegroundColor()


        # Set multiselection defaults


        self.SendScintilla(QsciScintilla.SCI_SETMULTIPLESELECTION, True)


        self.SendScintilla(QsciScintilla.SCI_SETMULTIPASTE, 1)


        self.SendScintilla(


            QsciScintilla.SCI_SETADDITIONALSELECTIONTYPING, True)


        lexer = LexerJson(self)


        self.setLexer(lexer)
EXAMPLE_TEXT = textwrap.dedent("""\


        {


            "_id": "5b05ffcbcf8e597939b3f5ca",


            "about": "Excepteur consequat commodo esse voluptate aute aliquip ad sint deserunt commodo eiusmod irure. Sint aliquip sit magna duis eu est culpa aliqua excepteur ut tempor nulla. Aliqua ex pariatur id labore sit. Quis sit ex aliqua veniam exercitation laboris anim adipisicing. Lorem nisi reprehenderit ullamco labore qui sit ut aliqua tempor consequat pariatur proident.",


            "address": "665 Malbone Street, Thornport, Louisiana, 243",


            "age": 23,


            "balance": "$3,216.91",


            "company": "BULLJUICE",


            "email": "elisekelley@bulljuice.com",


            "eyeColor": "brown",


            "gender": "female",


            "guid": "d3a6d865-0f64-4042-8a78-4f53de9b0707",


            "index": 0,


            "isActive": false,


            "isActive2": true,


            "latitude": -18.660714,


            "longitude": -85.378048,


            "name": "Elise Kelley",


            "phone": "+1 (808) 543-3966",


            "picture": "http://placehold.it/32x32",


            "registered": "2017-09-30T03:47:40 -02:00",


            "tags": [


                "et",


                "nostrud",


                "in",


                "fugiat",


                "incididunt",


                "labore",


                "nostrud"


            ]


        }\


    """)
def main():


    app = QApplication(sys.argv)


    ex = EditorAll()


    ex.setWindowTitle(__file__)


    ex.setText(EXAMPLE_TEXT)


    ex.resize(800, 600)


    ex.show()


    sys.exit(app.exec_())
if __name__ == "__main__":


    main()

  a b? c    ->    (a c | a b c)

  a: b*    ->    a: _b_tag


                 _b_tag: (_b_tag b)?

  rule: <EBNF EXPRESSION>


      | etc.


  TERM: <EBNF EXPRESSION>   // Rules aren't allowed

<NAME> [. <priority>] : <literals-and-or-terminals>

  my_template{param1, param2, ...}: <EBNF EXPRESSION>

some_rule: my_template{arg1, arg2, ...}

_separated{x, sep}: x (sep x)*  // Define a sequence of 'x sep x sep x ...'
num_list: "[" _separated{NUMBER, ","} "]"   // Will match "[1, 2, 3]" etc.

SELECT: "select"i     //# Will ignore case, and match SELECT or Select, etc.
MULTILINE_TEXT: /.+/s

IF: "if"
INTEGER : /[0-9]+/
INTEGER2 : ("0".."9")+          //# Same as INTEGER
DECIMAL.2: INTEGER? "." INTEGER  //# Will be matched before INTEGER
WHITESPACE: (" " | /\t/ )+
SQL_SELECT: "select"i

A1: "a" | "b"
A2: /a|b/

start           : (A | B)+
A               : "a" | "ab"
B               : "b"

>>> p.parse("ab")
Tree(start, [Token(A, 'a'), Token(B, 'b')])

>>> p = Lark("""start: (a | b)+
...             !a: "a" | "ab"
...             !b: "b"
...             """, ambiguity="explicit")
>>> print(p.parse("ab").pretty())
_ambig


  start


    a   ab


  start


    a   a


    b   b

<name> : <items-to-match>  [-> <alias> ]


       | ...

hello_world: "hello" "world"
mul: (mul "*")? number     //# Left-recursion is allowed and encouraged!
expr: expr operator expr


    | value               //# Multi-line, belongs to expr
four_words: word ~ 4

%ignore <TERMINAL>

%ignore " "
COMMENT: "#" /[^\n]/*
%ignore COMMENT

%import <module>.<TERMINAL>
%import <module>.<rule>
%import <module>.<TERMINAL> -> <NEWTERMINAL>
%import <module>.<rule> -> <newrule>
%import <module> (<TERM1>, <TERM2>, <rule1>, <rule2>)

%import common.NUMBER
%import .terminals_file (A, B, C)
%import .rules_file.rulea -> ruleb

• Unnamed literals (like "keyword" or "+")
• Terminals whose name starts with an underscore (like _DIGIT)

• Unnamed regular expressions (like /[0-9]/)
• Named terminals whose name starts with a letter (like DIGIT)

start:  PNAME pname
PNAME:  "(" NAME ")"
pname:  "(" NAME ")"
NAME:   /\w+/
%ignore /\s+/

start


    (Hello)


    pname World

    expr: "(" expr ")"


        | NAME+


    NAME: /\w+/


    %ignore " "

expr


    expr


        expr


            "hello"


            "world"

    start: "(" _greet ")"


    _greet: /\w+/ /\w+/

start


    "hello"


    "world"

    start: greet greet


    ?greet: "(" /\w+/ ")"


          | /\w+/ /\w+/

start


    greet


        "hello"


        "world"


    "planet"

    !expr: "(" expr ")"


         | NAME+


    NAME: /\w+/


    %ignore " "

expr


  (


  expr


    (


    expr


      hello


      world


    )


  )

    start: greet greet


    greet: "hello"


         | "world" -> planet

start


    greet


    planet

Parameters

start

The start symbol. Either a string, or a list of strings for multiple possible starts (Default: "start")

debug

Display debug information, such as warnings (default: False)

transformer

Applies the transformer to every parse tree (equivlent to applying it after the parse, but faster)

propagate_positions

Propagates (line, column, end_line, end_column) attributes into all tree branches.

maybe_placeholders

When True, the [] operator returns None when not matched.

When False, [] behaves like the ? operator, and returns no value at all. (default= False. Recommended to set to True)

cache

Cache the results of the Lark grammar analysis, for x2 to x3 faster loading. LALR only for now.

regex

When True, uses the regex module instead of the stdlib re.

g_regex_flags

Flags that are applied to all terminals (both regex and strings)

keep_all_tokens

Prevent the tree builder from automagically removing "punctuation" tokens (default: False)

tree_class

Lark will produce trees comprised of instances of this class instead of the default lark.Tree.

parser

Decides which parser engine to use. Accepts "earley" or "lalr". (Default: "earley"). (there is also a "cyk" option for legacy)

lexer

Decides whether or not to use a lexer stage

ambiguity

Decides how to handle ambiguity in the parse. Only relevant if parser="earley"

postlex

Lexer post-processing (Default: None) Only works with the standard and contextual lexers.

priority

How priorities should be evaluated - auto, none, normal, invert (Default: auto)

lexer_callbacks

Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution.

use_bytes

Accept an input of type bytes instead of str (Python 3 only).

edit_terminals

A callback for editing the terminals before parse.

save(f)

Saves the instance into the given file object

Useful for caching and multiprocessing.

classmethod load(f)

Loads an instance from the given file object

Useful for caching and multiprocessing.

classmethod open(grammar_filename, rel_to=None, **options)

Create an instance of Lark with the grammar given by its filename

If rel_to is provided, the function will find the grammar filename in relation to it.

Example

>>> Lark.open("grammar_file.lark", rel_to=__file__, parser="lalr")
Lark(...)
    

parse(text, start=None, on_error=None)

Parse the given text, according to the options provided.

from lark import Lark
>>> g = Lark(r"""


                    ?start: NAME


                    NAME: ID_START ID_CONTINUE*


                    ID_START: /[\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Lo}\p{Nl}_]+/


                    ID_CONTINUE: ID_START | /[\p{Mn}\p{Mc}\p{Nd}\p{Pc}·]+/


                """, regex=True)
>>> g.parse('வணக்கம்')
'வணக்கம்'

Parameters

pretty(indent_str=' ')

Returns an indented string representation of the tree.

Great for debugging.

iter_subtrees()

Depth-first iteration.

Iterates over all the subtrees, never returning to the same node twice (Lark's parse-tree is actually a DAG).

find_pred(pred)

Returns all nodes of the tree that evaluate pred(node) as true.

find_data(data)

Returns all nodes of the tree whose data equals the given data.

iter_subtrees_topdown()

Breadth-first iteration.

Iterates over all the subtrees, return nodes in order like pretty() does.

type

Name of the token (as specified in grammar)

value

Value of the token (redundant, as token.value == token will always be true)

pos_in_stream

The index of the token in the text

line

The line of the token in the text (starting with 1)

column

The column of the token in the text (starting with 1)

end_line

The line where the token ends

end_column

The next column after the end of the token. For example, if the token is a single character with a column value of 4, end_column will be 5.

end_pos

the index where the token ends (basically pos_in_stream + len(token))

• UnexpectedToken: The parser received an unexpected token
• UnexpectedCharacters: The lexer encountered an unexpected string

get_context(text, span=40)

Returns a pretty string pinpointing the error in the text, with span amount of context characters around it.

NOTE:

match_examples(parse_fn, examples, token_type_match_fallback=False, use_accepts=False)

Allows you to detect what's wrong in the input text by matching against example errors.

Given a parser instance and a dictionary mapping some label with some malformed syntax examples, it'll return the label for the example that bests matches the current error. The function will iterate the dictionary until it finds a matching error, and return the corresponding value.

For an example usage, see examples/error_reporting_lalr.py

feed_token(token)

Feed the parser with a token, and advance it to the next state, as if it received it from the lexer.

Note that token has to be an instance of Token.

copy()

Create a new puppet with a separate state.

Calls to feed_token() won't affect the old puppet, and vice-versa.

pretty()

Print the output of choices() in a way that's easier to read.

choices()

Returns a dictionary of token types, matched to their action in the parser.

Only returns token types that are accepted by the current state.

Updated by feed_token().

resume_parse()

Resume parsing from the current puppet state.

• Transformer - Recursively transforms the tree. This is the one you probably want.
• Transformer_InPlace - Non-recursive. Changes the tree in-place instead of returning new instances
• Transformer_InPlaceRecursive - Recursive. Changes the tree in-place instead of returning new instances

Parameters

visit_tokens -- By default, transformers only visit rules. visit_tokens=True will tell Transformer to visit tokens as well. This is a slightly slower alternative to lexer_callbacks but it's easier to maintain and works for all algorithms (even when there isn't a lexer).

__default__(data, children, meta)

Default operation on tree (for override)

Function that is called on if a function with a corresponding name has not been found. Defaults to reconstruct the Tree.

__default_token__(token)

Default operation on token (for override)

Function that is called on if a function with a corresponding name has not been found. Defaults to just return the argument.

Parameters

Variables

property is_ambiguous

Returns True if this node is ambiguous.

property children

Returns a list of this node's children sorted from greatest to least priority.

Variables

property children

Returns a list of this node's children.

visit_token_node(node)

Called when a Token is visited. Token nodes are always leaves.

visit_symbol_node_in(node)

Called when a symbol node is visited. Nodes that are returned will be scheduled to be visited. If visit_intermediate_node_in is not implemented, this function will be called for intermediate nodes as well.

visit_symbol_node_out(node)

Called after all nodes returned from a corresponding visit_symbol_node_in call have been visited. If visit_intermediate_node_out is not implemented, this function will be called for intermediate nodes as well.

visit_packed_node_in(node)

Called when a packed node is visited. Nodes that are returned will be scheduled to be visited.

visit_packed_node_out(node)

Called after all nodes returned from a corresponding visit_packed_node_in call have been visited.

on_cycle(node, path)

Called when a cycle is encountered.

get_cycle_in_path(node, path)

A utility function for use in on_cycle to obtain a slice of path that only contains the nodes that make up the cycle.

transform(root)

Perform a transformation on an SPPF.

transform_symbol_node(node, data)

Transform a symbol node.

transform_intermediate_node(node, data)

Transform an intermediate node.

transform_packed_node(node, data)

Transform a packed node.

transform_token_node(node)

Transform a Token.

Parameters

__default__(name, data)

Default operation on tree (for override).

Returns a tree with name with data as children.

__default_ambig__(name, data)

Default operation on ambiguous rule (for override).

Wraps data in an '_ambig_' node if it contains more than one element.

__default_token__(node)

Default operation on Token (for override).

Returns node.

pip install lark-parser[nearley]

git clone https://github.com/Hardmath123/nearley

git clone https://github.com/Hardmath123/nearley
python -m lark.tools.nearley nearley/examples/calculator/arithmetic.ne main nearley > ncalc.py

>>> import ncalc
>>> ncalc.parse('sin(pi/4) ^ e')
0.38981434460254655

git clone https://github.com/Hardmath123/nearley
python -m lark.tools.nearley nearley/examples/calculator/arithmetic.ne main nearley --es6 > ncalc.py