diff options
103 files changed, 11483 insertions, 6260 deletions
@@ -159,5 +159,3 @@ venv.bak/ # For developers on OSX .DS_Store - - diff --git a/.travis.yml b/.travis.yml index 1762181..eae572c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,18 +2,25 @@ language: python matrix: include: + - python: 3.8 + env: TOXENV=black - python: 3.5 + env: TOXENV=py35 - python: 3.6 + env: TOXENV=py36 - python: 3.7 + env: TOXENV=py37 - python: 3.8 + env: TOXENV=py38 - python: pypy3 + env: TOXENV=pypy3 fast_finish: true install: - - pip install codecov + - pip install tox codecov script: - - python -m unittest + - tox after_success: - - codecov run -m unittest tests.test_unit + - codecov diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0c405a0..6dcb0f3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,22 +39,22 @@ If you have a question on using pyparsing, there are a number of resources avail If you are considering proposing updates to pyparsing, please bear in mind the following guidelines. -Please review [_The Zen of Pyparsing_ and _The Zen of Pyparsing -Development_](https://github.com/pyparsing/pyparsing/wiki/Zen) -article on the pyparsing wiki, to get a general feel for the historical and future approaches to pyparsing's +Please review [_The Zen of Pyparsing_ and _The Zen of Pyparsing +Development_](https://github.com/pyparsing/pyparsing/wiki/Zen) +article on the pyparsing wiki, to get a general feel for the historical and future approaches to pyparsing's design, and intended developer experience as an embedded DSL. ## Some design points -- Minimize additions to the module namespace. Over time, pyparsing's namespace has acquired a *lot* of names. - New features have been encapsulated into namespace classes to try to hold back the name flooding when importing +- Minimize additions to the module namespace. Over time, pyparsing's namespace has acquired a *lot* of names. + New features have been encapsulated into namespace classes to try to hold back the name flooding when importing pyparsing. - New operator overloads will need to show broad applicability. - Performance tuning should focus on parse time performance. Optimizing parser definition performance is secondary. -- New external dependencies will require substantial justification, and if included, will need to be guarded for +- New external dependencies will require substantial justification, and if included, will need to be guarded for `ImportError`s raised if the external module is not installed. ## Some coding points @@ -66,7 +66,7 @@ These coding styles are encouraged whether submitting code for core pyparsing or future trend in coding styles. There are plans to convert these names to PEP8-conformant snake case, but this will be done over several releases to provide a migration path for current pyparsing-dependent applications. See more information at the [PEP8 wiki page](https://github.com/pyparsing/pyparsing/wiki/PEP-8-planning). - + If you wish to submit a new example, please follow PEP8 name and coding guidelines. Example code must be available for distribution with the rest of pyparsing under the MIT open source license. @@ -84,7 +84,7 @@ These coding styles are encouraged whether submitting code for core pyparsing or - str.format() statements should use named format arguments (unless this proves to be a slowdown at parse time). -- List, tuple, and dict literals should include a trailing comma after the last element, which reduces changeset +- List, tuple, and dict literals should include a trailing comma after the last element, which reduces changeset clutter when another element gets added to the end. - Examples should import pyparsing and the common namespace classes as: @@ -99,19 +99,19 @@ These coding styles are encouraged whether submitting code for core pyparsing or - Where possible use operators to create composite parse expressions: expr = expr_a + expr_b | expr_c - + instead of: - + expr = pp.MatchFirst([pp.And([expr_a, expr_b]), expr_c]) Exception: if using a generator to create an expression: - + import keyword python_keywords = keyword.kwlist - any_keyword = pp.MatchFirst(pp.Keyword(kw) + any_keyword = pp.MatchFirst(pp.Keyword(kw) for kw in python_keywords)) -- Learn [The Classic Blunders](https://github.com/pyparsing/pyparsing/wiki/The-Classic-Blunders) and +- Learn [The Classic Blunders](https://github.com/pyparsing/pyparsing/wiki/The-Classic-Blunders) and how to avoid them when developing new examples. - New features should be accompanied with updates to unitTests.py and a bullet in the CHANGES file. diff --git a/docs/conf.py b/docs/conf.py index c4a7698..a26ed9b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,15 +13,16 @@ # import os import sys -sys.path.insert(0, os.path.abspath('..')) + +sys.path.insert(0, os.path.abspath("..")) from pyparsing import __version__ as pyparsing_version # -- Project information ----------------------------------------------------- -project = 'PyParsing' -copyright = '2018, Paul T. McGuire' -author = 'Paul T. McGuire' +project = "PyParsing" +copyright = "2018, Paul T. McGuire" +author = "Paul T. McGuire" # The short X.Y version version = pyparsing_version @@ -39,20 +40,20 @@ release = pyparsing_version # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', + "sphinx.ext.autodoc", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -67,7 +68,7 @@ language = None exclude_patterns = [] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # -- Options for HTML output ------------------------------------------------- @@ -75,7 +76,7 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -86,7 +87,7 @@ html_theme = 'alabaster' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -102,7 +103,7 @@ html_static_path = ['_static'] # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'PyParsingdoc' +htmlhelp_basename = "PyParsingdoc" # -- Options for LaTeX output ------------------------------------------------ @@ -111,15 +112,12 @@ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -129,8 +127,13 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'PyParsing.tex', 'PyParsing Documentation', - 'Paul T. McGuire', 'manual'), + ( + master_doc, + "PyParsing.tex", + "PyParsing Documentation", + "Paul T. McGuire", + "manual", + ), ] @@ -138,10 +141,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'pyparsing', 'PyParsing Documentation', - [author], 1) -] +man_pages = [(master_doc, "pyparsing", "PyParsing Documentation", [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -150,9 +150,15 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'PyParsing', 'PyParsing Documentation', - author, 'PyParsing', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "PyParsing", + "PyParsing Documentation", + author, + "PyParsing", + "One line description of project.", + "Miscellaneous", + ), ] @@ -174,7 +180,7 @@ epub_copyright = copyright # epub_uid = '' # A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] +epub_exclude_files = ["search.html"] # -- Extension configuration ------------------------------------------------- diff --git a/examples/LAparser.py b/examples/LAparser.py index 330b8f5..b72166f 100644 --- a/examples/LAparser.py +++ b/examples/LAparser.py @@ -57,41 +57,54 @@ Usage: To process LA equations embedded in source files, import this module and """ -import re,sys -from pyparsing import Word, alphas, ParseException, Literal, CaselessLiteral \ -, Combine, Optional, nums, Forward, ZeroOrMore, \ - StringEnd, alphanums +import re, sys +from pyparsing import ( + Word, + alphas, + ParseException, + Literal, + CaselessLiteral, + Combine, + Optional, + nums, + Forward, + ZeroOrMore, + StringEnd, + alphanums, +) # Debugging flag can be set to either "debug_flag=True" or "debug_flag=False" -debug_flag=False +debug_flag = False -#---------------------------------------------------------------------------- +# ---------------------------------------------------------------------------- # Variables that hold intermediate parsing results and a couple of # helper functions. -exprStack = [] # Holds operators and operands parsed from input. -targetvar = None # Holds variable name to left of '=' sign in LA equation. +exprStack = [] # Holds operators and operands parsed from input. +targetvar = None # Holds variable name to left of '=' sign in LA equation. -def _pushFirst( str, loc, toks ): - if debug_flag: print("pushing ", toks[0], "str is ", str) - exprStack.append( toks[0] ) +def _pushFirst(str, loc, toks): + if debug_flag: + print("pushing ", toks[0], "str is ", str) + exprStack.append(toks[0]) -def _assignVar( str, loc, toks ): + +def _assignVar(str, loc, toks): global targetvar - targetvar = toks[0] + targetvar = toks[0] + -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # The following statements define the grammar for the parser. -point = Literal('.') -e = CaselessLiteral('E') -plusorminus = Literal('+') | Literal('-') +point = Literal(".") +e = CaselessLiteral("E") +plusorminus = Literal("+") | Literal("-") number = Word(nums) -integer = Combine( Optional(plusorminus) + number ) -floatnumber = Combine( integer + - Optional( point + Optional(number) ) + - Optional( e + integer ) - ) +integer = Combine(Optional(plusorminus) + number) +floatnumber = Combine( + integer + Optional(point + Optional(number)) + Optional(e + integer) +) lbracket = Literal("[") rbracket = Literal("]") @@ -100,57 +113,66 @@ ident = Forward() ## can include references to array elements, rows and columns, e.g., a = b[i] + 5. ## Expressions within []'s are not presently supported, so a = b[i+1] will raise ## a ParseException. -ident = Combine(Word(alphas + '-',alphanums + '_') + \ - ZeroOrMore(lbracket + (Word(alphas + '-',alphanums + '_')|integer) + rbracket) \ - ) - -plus = Literal( "+" ) -minus = Literal( "-" ) -mult = Literal( "*" ) -div = Literal( "/" ) -outer = Literal( "@" ) -lpar = Literal( "(" ).suppress() -rpar = Literal( ")" ).suppress() -addop = plus | minus +ident = Combine( + Word(alphas + "-", alphanums + "_") + + ZeroOrMore(lbracket + (Word(alphas + "-", alphanums + "_") | integer) + rbracket) +) + +plus = Literal("+") +minus = Literal("-") +mult = Literal("*") +div = Literal("/") +outer = Literal("@") +lpar = Literal("(").suppress() +rpar = Literal(")").suppress() +addop = plus | minus multop = mult | div | outer -expop = Literal( "^" ) -assignop = Literal( "=" ) +expop = Literal("^") +assignop = Literal("=") expr = Forward() -atom = ( ( e | floatnumber | integer | ident ).setParseAction(_pushFirst) | - ( lpar + expr.suppress() + rpar ) - ) +atom = (e | floatnumber | integer | ident).setParseAction(_pushFirst) | ( + lpar + expr.suppress() + rpar +) factor = Forward() -factor << atom + ZeroOrMore( ( expop + factor ).setParseAction( _pushFirst ) ) +factor << atom + ZeroOrMore((expop + factor).setParseAction(_pushFirst)) -term = factor + ZeroOrMore( ( multop + factor ).setParseAction( _pushFirst ) ) -expr << term + ZeroOrMore( ( addop + term ).setParseAction( _pushFirst ) ) +term = factor + ZeroOrMore((multop + factor).setParseAction(_pushFirst)) +expr << term + ZeroOrMore((addop + term).setParseAction(_pushFirst)) equation = (ident + assignop).setParseAction(_assignVar) + expr + StringEnd() # End of grammar definition -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- ## The following are helper variables and functions used by the Binary Infix Operator ## Functions described below. -vprefix = 'V3_' +vprefix = "V3_" vplen = len(vprefix) -mprefix = 'M3_' +mprefix = "M3_" mplen = len(mprefix) ## We don't support unary negation for vectors and matrices -class UnaryUnsupportedError(Exception): pass +class UnaryUnsupportedError(Exception): + pass + def _isvec(ident): - if ident[0] == '-' and ident[1:vplen+1] == vprefix: - raise UnaryUnsupportedError - else: return ident[0:vplen] == vprefix + if ident[0] == "-" and ident[1 : vplen + 1] == vprefix: + raise UnaryUnsupportedError + else: + return ident[0:vplen] == vprefix + def _ismat(ident): - if ident[0] == '-' and ident[1:mplen+1] == mprefix: - raise UnaryUnsupportedError - else: return ident[0:mplen] == mprefix + if ident[0] == "-" and ident[1 : mplen + 1] == mprefix: + raise UnaryUnsupportedError + else: + return ident[0:mplen] == mprefix + + +def _isscalar(ident): + return not (_isvec(ident) or _ismat(ident)) -def _isscalar(ident): return not (_isvec(ident) or _ismat(ident)) ## Binary infix operator (BIO) functions. These are called when the stack evaluator ## pops a binary operator like '+' or '*". The stack evaluator pops the two operand, a and b, @@ -164,80 +186,121 @@ def _isscalar(ident): return not (_isvec(ident) or _ismat(ident)) ## the appropriate prefix is placed on the outer function for removal later as the stack evaluation ## recurses toward the final assignment statement. -def _addfunc(a,b): - if _isscalar(a) and _isscalar(b): return "(%s+%s)"%(a,b) - if _isvec(a) and _isvec(b): return "%svAdd(%s,%s)"%(vprefix,a[vplen:],b[vplen:]) - if _ismat(a) and _ismat(b): return "%smAdd(%s,%s)"%(mprefix,a[mplen:],b[mplen:]) - else: raise TypeError - -def _subfunc(a,b): - if _isscalar(a) and _isscalar(b): return "(%s-%s)"%(a,b) - if _isvec(a) and _isvec(b): return "%svSubtract(%s,%s)"%(vprefix,a[vplen:],b[vplen:]) - if _ismat(a) and _ismat(b): return "%smSubtract(%s,%s)"%(mprefix,a[mplen:],b[mplen:]) - else: raise TypeError - -def _mulfunc(a,b): - if _isscalar(a) and _isscalar(b): return "%s*%s"%(a,b) - if _isvec(a) and _isvec(b): return "vDot(%s,%s)"%(a[vplen:],b[vplen:]) - if _ismat(a) and _ismat(b): return "%smMultiply(%s,%s)"%(mprefix,a[mplen:],b[mplen:]) - if _ismat(a) and _isvec(b): return "%smvMultiply(%s,%s)"%(vprefix,a[mplen:],b[vplen:]) - if _ismat(a) and _isscalar(b): return "%smScale(%s,%s)"%(mprefix,a[mplen:],b) - if _isvec(a) and _isscalar(b): return "%svScale(%s,%s)"%(vprefix,a[mplen:],b) - else: raise TypeError - -def _outermulfunc(a,b): - ## The '@' operator is used for the vector outer product. - if _isvec(a) and _isvec(b): - return "%svOuterProduct(%s,%s)"%(mprefix,a[vplen:],b[vplen:]) - else: raise TypeError - -def _divfunc(a,b): - ## The '/' operator is used only for scalar division - if _isscalar(a) and _isscalar(b): return "%s/%s"%(a,b) - else: raise TypeError - -def _expfunc(a,b): - ## The '^' operator is used for exponentiation on scalars and - ## as a marker for unary operations on vectors and matrices. - if _isscalar(a) and _isscalar(b): return "pow(%s,%s)"%(str(a),str(b)) - if _ismat(a) and b=='-1': return "%smInverse(%s)"%(mprefix,a[mplen:]) - if _ismat(a) and b=='T': return "%smTranspose(%s)"%(mprefix,a[mplen:]) - if _ismat(a) and b=='Det': return "mDeterminant(%s)"%(a[mplen:]) - if _isvec(a) and b=='Mag': return "sqrt(vMagnitude2(%s))"%(a[vplen:]) - if _isvec(a) and b=='Mag2': return "vMagnitude2(%s)"%(a[vplen:]) - else: raise TypeError - -def _assignfunc(a,b): - ## The '=' operator is used for assignment - if _isscalar(a) and _isscalar(b): return "%s=%s"%(a,b) - if _isvec(a) and _isvec(b): return "vCopy(%s,%s)"%(a[vplen:],b[vplen:]) - if _ismat(a) and _ismat(b): return "mCopy(%s,%s)"%(a[mplen:],b[mplen:]) - else: raise TypeError + +def _addfunc(a, b): + if _isscalar(a) and _isscalar(b): + return "(%s+%s)" % (a, b) + if _isvec(a) and _isvec(b): + return "%svAdd(%s,%s)" % (vprefix, a[vplen:], b[vplen:]) + if _ismat(a) and _ismat(b): + return "%smAdd(%s,%s)" % (mprefix, a[mplen:], b[mplen:]) + else: + raise TypeError + + +def _subfunc(a, b): + if _isscalar(a) and _isscalar(b): + return "(%s-%s)" % (a, b) + if _isvec(a) and _isvec(b): + return "%svSubtract(%s,%s)" % (vprefix, a[vplen:], b[vplen:]) + if _ismat(a) and _ismat(b): + return "%smSubtract(%s,%s)" % (mprefix, a[mplen:], b[mplen:]) + else: + raise TypeError + + +def _mulfunc(a, b): + if _isscalar(a) and _isscalar(b): + return "%s*%s" % (a, b) + if _isvec(a) and _isvec(b): + return "vDot(%s,%s)" % (a[vplen:], b[vplen:]) + if _ismat(a) and _ismat(b): + return "%smMultiply(%s,%s)" % (mprefix, a[mplen:], b[mplen:]) + if _ismat(a) and _isvec(b): + return "%smvMultiply(%s,%s)" % (vprefix, a[mplen:], b[vplen:]) + if _ismat(a) and _isscalar(b): + return "%smScale(%s,%s)" % (mprefix, a[mplen:], b) + if _isvec(a) and _isscalar(b): + return "%svScale(%s,%s)" % (vprefix, a[mplen:], b) + else: + raise TypeError + + +def _outermulfunc(a, b): + ## The '@' operator is used for the vector outer product. + if _isvec(a) and _isvec(b): + return "%svOuterProduct(%s,%s)" % (mprefix, a[vplen:], b[vplen:]) + else: + raise TypeError + + +def _divfunc(a, b): + ## The '/' operator is used only for scalar division + if _isscalar(a) and _isscalar(b): + return "%s/%s" % (a, b) + else: + raise TypeError + + +def _expfunc(a, b): + ## The '^' operator is used for exponentiation on scalars and + ## as a marker for unary operations on vectors and matrices. + if _isscalar(a) and _isscalar(b): + return "pow(%s,%s)" % (str(a), str(b)) + if _ismat(a) and b == "-1": + return "%smInverse(%s)" % (mprefix, a[mplen:]) + if _ismat(a) and b == "T": + return "%smTranspose(%s)" % (mprefix, a[mplen:]) + if _ismat(a) and b == "Det": + return "mDeterminant(%s)" % (a[mplen:]) + if _isvec(a) and b == "Mag": + return "sqrt(vMagnitude2(%s))" % (a[vplen:]) + if _isvec(a) and b == "Mag2": + return "vMagnitude2(%s)" % (a[vplen:]) + else: + raise TypeError + + +def _assignfunc(a, b): + ## The '=' operator is used for assignment + if _isscalar(a) and _isscalar(b): + return "%s=%s" % (a, b) + if _isvec(a) and _isvec(b): + return "vCopy(%s,%s)" % (a[vplen:], b[vplen:]) + if _ismat(a) and _ismat(b): + return "mCopy(%s,%s)" % (a[mplen:], b[mplen:]) + else: + raise TypeError + ## End of BIO func definitions ##---------------------------------------------------------------------------- # Map operator symbols to corresponding BIO funcs -opn = { "+" : ( _addfunc ), - "-" : ( _subfunc ), - "*" : ( _mulfunc ), - "@" : ( _outermulfunc ), - "/" : ( _divfunc), - "^" : ( _expfunc ), } +opn = { + "+": (_addfunc), + "-": (_subfunc), + "*": (_mulfunc), + "@": (_outermulfunc), + "/": (_divfunc), + "^": (_expfunc), +} ##---------------------------------------------------------------------------- # Recursive function that evaluates the expression stack -def _evaluateStack( s ): - op = s.pop() - if op in "+-*/@^": - op2 = _evaluateStack( s ) - op1 = _evaluateStack( s ) - result = opn[op]( op1, op2 ) - if debug_flag: print(result) - return result - else: - return op +def _evaluateStack(s): + op = s.pop() + if op in "+-*/@^": + op2 = _evaluateStack(s) + op1 = _evaluateStack(s) + result = opn[op](op1, op2) + if debug_flag: + print(result) + return result + else: + return op + ##---------------------------------------------------------------------------- # The parse function that invokes all of the above. @@ -248,59 +311,76 @@ def parse(input_string): calls that implement the expression. """ - global exprStack + global exprStack global targetvar # Start with a blank exprStack and a blank targetvar exprStack = [] - targetvar=None - - if input_string != '': - # try parsing the input string - try: - L=equation.parseString( input_string ) - except ParseException as err: - print('Parse Failure', file=sys.stderr) - print(err.line, file=sys.stderr) - print(" "*(err.column-1) + "^", file=sys.stderr) - print(err, file=sys.stderr) - raise - - # show result of parsing the input string - if debug_flag: - print(input_string, "->", L) - print("exprStack=", exprStack) - - # Evaluate the stack of parsed operands, emitting C code. - try: - result=_evaluateStack(exprStack) - except TypeError: - print("Unsupported operation on right side of '%s'.\nCheck for missing or incorrect tags on non-scalar operands."%input_string, file=sys.stderr) - raise - except UnaryUnsupportedError: - print("Unary negation is not supported for vectors and matrices: '%s'"%input_string, file=sys.stderr) - raise - - # Create final assignment and print it. - if debug_flag: print("var=",targetvar) - if targetvar != None: - try: - result = _assignfunc(targetvar,result) - except TypeError: - print("Left side tag does not match right side of '%s'"%input_string, file=sys.stderr) + targetvar = None + + if input_string != "": + # try parsing the input string + try: + L = equation.parseString(input_string) + except ParseException as err: + print("Parse Failure", file=sys.stderr) + print(err.line, file=sys.stderr) + print(" " * (err.column - 1) + "^", file=sys.stderr) + print(err, file=sys.stderr) raise - except UnaryUnsupportedError: - print("Unary negation is not supported for vectors and matrices: '%s'"%input_string, file=sys.stderr) + + # show result of parsing the input string + if debug_flag: + print(input_string, "->", L) + print("exprStack=", exprStack) + + # Evaluate the stack of parsed operands, emitting C code. + try: + result = _evaluateStack(exprStack) + except TypeError: + print( + "Unsupported operation on right side of '%s'.\nCheck for missing or incorrect tags on non-scalar operands." + % input_string, + file=sys.stderr, + ) + raise + except UnaryUnsupportedError: + print( + "Unary negation is not supported for vectors and matrices: '%s'" + % input_string, + file=sys.stderr, + ) raise - return result - else: - print("Empty left side in '%s'"%input_string, file=sys.stderr) - raise TypeError + # Create final assignment and print it. + if debug_flag: + print("var=", targetvar) + if targetvar != None: + try: + result = _assignfunc(targetvar, result) + except TypeError: + print( + "Left side tag does not match right side of '%s'" % input_string, + file=sys.stderr, + ) + raise + except UnaryUnsupportedError: + print( + "Unary negation is not supported for vectors and matrices: '%s'" + % input_string, + file=sys.stderr, + ) + raise + + return result + else: + print("Empty left side in '%s'" % input_string, file=sys.stderr) + raise TypeError + ##----------------------------------------------------------------------------------- -def fprocess(infilep,outfilep): - """ +def fprocess(infilep, outfilep): + """ Scans an input file for LA equations between double square brackets, e.g. [[ M3_mymatrix = M3_anothermatrix^-1 ]], and replaces the expression with a comment containing the equation followed by nested function calls @@ -313,106 +393,139 @@ def fprocess(infilep,outfilep): The arguments are file objects (NOT file names) opened for reading and writing, respectively. """ - pattern = r'\[\[\s*(.*?)\s*\]\]' - eqn = re.compile(pattern,re.DOTALL) - s = infilep.read() - def parser(mo): - ccode = parse(mo.group(1)) - return "/* %s */\n%s;\nLAParserBufferReset();\n"%(mo.group(1),ccode) + pattern = r"\[\[\s*(.*?)\s*\]\]" + eqn = re.compile(pattern, re.DOTALL) + s = infilep.read() + + def parser(mo): + ccode = parse(mo.group(1)) + return "/* %s */\n%s;\nLAParserBufferReset();\n" % (mo.group(1), ccode) + + content = eqn.sub(parser, s) + outfilep.write(content) - content = eqn.sub(parser,s) - outfilep.write(content) ##----------------------------------------------------------------------------------- def test(): - """ + """ Tests the parsing of various supported expressions. Raises an AssertError if the output is not what is expected. Prints the input, expected output, and actual output for all tests. """ - print("Testing LAParser") - testcases = [ - ("Scalar addition","a = b+c","a=(b+c)"), - ("Vector addition","V3_a = V3_b + V3_c","vCopy(a,vAdd(b,c))"), - ("Vector addition","V3_a=V3_b+V3_c","vCopy(a,vAdd(b,c))"), - ("Matrix addition","M3_a = M3_b + M3_c","mCopy(a,mAdd(b,c))"), - ("Matrix addition","M3_a=M3_b+M3_c","mCopy(a,mAdd(b,c))"), - ("Scalar subtraction","a = b-c","a=(b-c)"), - ("Vector subtraction","V3_a = V3_b - V3_c","vCopy(a,vSubtract(b,c))"), - ("Matrix subtraction","M3_a = M3_b - M3_c","mCopy(a,mSubtract(b,c))"), - ("Scalar multiplication","a = b*c","a=b*c"), - ("Scalar division","a = b/c","a=b/c"), - ("Vector multiplication (dot product)","a = V3_b * V3_c","a=vDot(b,c)"), - ("Vector multiplication (outer product)","M3_a = V3_b @ V3_c","mCopy(a,vOuterProduct(b,c))"), - ("Matrix multiplication","M3_a = M3_b * M3_c","mCopy(a,mMultiply(b,c))"), - ("Vector scaling","V3_a = V3_b * c","vCopy(a,vScale(b,c))"), - ("Matrix scaling","M3_a = M3_b * c","mCopy(a,mScale(b,c))"), - ("Matrix by vector multiplication","V3_a = M3_b * V3_c","vCopy(a,mvMultiply(b,c))"), - ("Scalar exponentiation","a = b^c","a=pow(b,c)"), - ("Matrix inversion","M3_a = M3_b^-1","mCopy(a,mInverse(b))"), - ("Matrix transpose","M3_a = M3_b^T","mCopy(a,mTranspose(b))"), - ("Matrix determinant","a = M3_b^Det","a=mDeterminant(b)"), - ("Vector magnitude squared","a = V3_b^Mag2","a=vMagnitude2(b)"), - ("Vector magnitude","a = V3_b^Mag","a=sqrt(vMagnitude2(b))"), - ("Complicated expression", "myscalar = (M3_amatrix * V3_bvector)^Mag + 5*(-xyz[i] + 2.03^2)","myscalar=(sqrt(vMagnitude2(mvMultiply(amatrix,bvector)))+5*(-xyz[i]+pow(2.03,2)))"), - ("Complicated Multiline", "myscalar = \n(M3_amatrix * V3_bvector)^Mag +\n 5*(xyz + 2.03^2)","myscalar=(sqrt(vMagnitude2(mvMultiply(amatrix,bvector)))+5*(xyz+pow(2.03,2)))") - - ] - - - all_passed = [True] - - def post_test(test, parsed): - - # copy exprStack to evaluate and clear before running next test - parsed_stack = exprStack[:] - exprStack.clear() - - name, testcase, expected = next(tc for tc in testcases if tc[1] == test) - - this_test_passed = False - try: - try: - result=_evaluateStack(parsed_stack) - except TypeError: - print("Unsupported operation on right side of '%s'.\nCheck for missing or incorrect tags on non-scalar operands."%input_string, file=sys.stderr) - raise - except UnaryUnsupportedError: - print("Unary negation is not supported for vectors and matrices: '%s'"%input_string, file=sys.stderr) - raise - - # Create final assignment and print it. - if debug_flag: print("var=",targetvar) - if targetvar != None: - try: - result = _assignfunc(targetvar,result) - except TypeError: - print("Left side tag does not match right side of '%s'"%input_string, file=sys.stderr) + print("Testing LAParser") + testcases = [ + ("Scalar addition", "a = b+c", "a=(b+c)"), + ("Vector addition", "V3_a = V3_b + V3_c", "vCopy(a,vAdd(b,c))"), + ("Vector addition", "V3_a=V3_b+V3_c", "vCopy(a,vAdd(b,c))"), + ("Matrix addition", "M3_a = M3_b + M3_c", "mCopy(a,mAdd(b,c))"), + ("Matrix addition", "M3_a=M3_b+M3_c", "mCopy(a,mAdd(b,c))"), + ("Scalar subtraction", "a = b-c", "a=(b-c)"), + ("Vector subtraction", "V3_a = V3_b - V3_c", "vCopy(a,vSubtract(b,c))"), + ("Matrix subtraction", "M3_a = M3_b - M3_c", "mCopy(a,mSubtract(b,c))"), + ("Scalar multiplication", "a = b*c", "a=b*c"), + ("Scalar division", "a = b/c", "a=b/c"), + ("Vector multiplication (dot product)", "a = V3_b * V3_c", "a=vDot(b,c)"), + ( + "Vector multiplication (outer product)", + "M3_a = V3_b @ V3_c", + "mCopy(a,vOuterProduct(b,c))", + ), + ("Matrix multiplication", "M3_a = M3_b * M3_c", "mCopy(a,mMultiply(b,c))"), + ("Vector scaling", "V3_a = V3_b * c", "vCopy(a,vScale(b,c))"), + ("Matrix scaling", "M3_a = M3_b * c", "mCopy(a,mScale(b,c))"), + ( + "Matrix by vector multiplication", + "V3_a = M3_b * V3_c", + "vCopy(a,mvMultiply(b,c))", + ), + ("Scalar exponentiation", "a = b^c", "a=pow(b,c)"), + ("Matrix inversion", "M3_a = M3_b^-1", "mCopy(a,mInverse(b))"), + ("Matrix transpose", "M3_a = M3_b^T", "mCopy(a,mTranspose(b))"), + ("Matrix determinant", "a = M3_b^Det", "a=mDeterminant(b)"), + ("Vector magnitude squared", "a = V3_b^Mag2", "a=vMagnitude2(b)"), + ("Vector magnitude", "a = V3_b^Mag", "a=sqrt(vMagnitude2(b))"), + ( + "Complicated expression", + "myscalar = (M3_amatrix * V3_bvector)^Mag + 5*(-xyz[i] + 2.03^2)", + "myscalar=(sqrt(vMagnitude2(mvMultiply(amatrix,bvector)))+5*(-xyz[i]+pow(2.03,2)))", + ), + ( + "Complicated Multiline", + "myscalar = \n(M3_amatrix * V3_bvector)^Mag +\n 5*(xyz + 2.03^2)", + "myscalar=(sqrt(vMagnitude2(mvMultiply(amatrix,bvector)))+5*(xyz+pow(2.03,2)))", + ), + ] + + all_passed = [True] + + def post_test(test, parsed): + + # copy exprStack to evaluate and clear before running next test + parsed_stack = exprStack[:] + exprStack.clear() + + name, testcase, expected = next(tc for tc in testcases if tc[1] == test) + + this_test_passed = False + try: + try: + result = _evaluateStack(parsed_stack) + except TypeError: + print( + "Unsupported operation on right side of '%s'.\nCheck for missing or incorrect tags on non-scalar operands." + % input_string, + file=sys.stderr, + ) raise - except UnaryUnsupportedError: - print("Unary negation is not supported for vectors and matrices: '%s'"%input_string, file=sys.stderr) + except UnaryUnsupportedError: + print( + "Unary negation is not supported for vectors and matrices: '%s'" + % input_string, + file=sys.stderr, + ) raise - else: - print("Empty left side in '%s'"%input_string, file=sys.stderr) - raise TypeError - - parsed['result'] = result - parsed['passed'] = this_test_passed = result == expected + # Create final assignment and print it. + if debug_flag: + print("var=", targetvar) + if targetvar != None: + try: + result = _assignfunc(targetvar, result) + except TypeError: + print( + "Left side tag does not match right side of '%s'" + % input_string, + file=sys.stderr, + ) + raise + except UnaryUnsupportedError: + print( + "Unary negation is not supported for vectors and matrices: '%s'" + % input_string, + file=sys.stderr, + ) + raise + + else: + print("Empty left side in '%s'" % input_string, file=sys.stderr) + raise TypeError + + parsed["result"] = result + parsed["passed"] = this_test_passed = result == expected + + finally: + all_passed[0] = all_passed[0] and this_test_passed + print("\n" + name) + + equation.runTests((t[1] for t in testcases), postParse=post_test) + + ##TODO: Write testcases with invalid expressions and test that the expected + ## exceptions are raised. + + print("Tests completed!") + print("PASSED" if all_passed[0] else "FAILED") + assert all_passed[0] - finally: - all_passed[0] = all_passed[0] and this_test_passed - print('\n' + name) - - equation.runTests((t[1] for t in testcases), postParse=post_test) - - - ##TODO: Write testcases with invalid expressions and test that the expected - ## exceptions are raised. - - print("Tests completed!") - print("PASSED" if all_passed[0] else "FAILED") - assert all_passed[0] ##---------------------------------------------------------------------------- ## The following is executed only when this module is executed as @@ -420,43 +533,44 @@ def test(): ## and then enters an interactive loop where you ## can enter expressions and see the resulting C code as output. -if __name__ == '__main__': +if __name__ == "__main__": - import sys - if not sys.flags.interactive: - # run testcases - test() - sys.exit(0) + import sys - # input_string - input_string='' + if not sys.flags.interactive: + # run testcases + test() + sys.exit(0) - # Display instructions on how to use the program interactively - interactiveusage = """ + # input_string + input_string = "" + + # Display instructions on how to use the program interactively + interactiveusage = """ Entering interactive mode: Type in an equation to be parsed or 'quit' to exit the program. Type 'debug on' to print parsing details as each string is processed. Type 'debug off' to stop printing parsing details """ - print(interactiveusage) - input_string = input("> ") - - while input_string != 'quit': - if input_string == "debug on": - debug_flag = True - elif input_string == "debug off": - debug_flag = False - else: - try: - print(parse(input_string)) - except Exception: - pass - - # obtain new input string + print(interactiveusage) input_string = input("> ") - # if user types 'quit' then say goodbye - print("Good bye!") - import os - os._exit(0) - + while input_string != "quit": + if input_string == "debug on": + debug_flag = True + elif input_string == "debug off": + debug_flag = False + else: + try: + print(parse(input_string)) + except Exception: + pass + + # obtain new input string + input_string = input("> ") + + # if user types 'quit' then say goodbye + print("Good bye!") + import os + + os._exit(0) diff --git a/examples/SimpleCalc.py b/examples/SimpleCalc.py index 5e1d835..7ace9ae 100644 --- a/examples/SimpleCalc.py +++ b/examples/SimpleCalc.py @@ -23,13 +23,12 @@ # - # Uncomment the line below for readline support on interactive terminal # import readline from pyparsing import ParseException, Word, alphas, alphanums # Debugging flag can be set to either "debug_flag=True" or "debug_flag=False" -debug_flag=False +debug_flag = False variables = {} @@ -59,64 +58,67 @@ from fourFn import BNF, exprStack, evaluate_stack arithExpr = BNF() ident = Word(alphas, alphanums).setName("identifier") -assignment = ident("varname") + '=' + arithExpr +assignment = ident("varname") + "=" + arithExpr pattern = assignment | arithExpr -if __name__ == '__main__': - # input_string - input_string='' +if __name__ == "__main__": + # input_string + input_string = "" - # Display instructions on how to quit the program - print("Type in the string to be parsed or 'quit' to exit the program") - input_string = input("> ") + # Display instructions on how to quit the program + print("Type in the string to be parsed or 'quit' to exit the program") + input_string = input("> ") - while input_string.strip().lower() != 'quit': - if input_string.strip().lower() == 'debug': - debug_flag=True + while input_string.strip().lower() != "quit": + if input_string.strip().lower() == "debug": + debug_flag = True + input_string = input("> ") + continue + + # Reset to an empty exprStack + del exprStack[:] + + if input_string != "": + # try parsing the input string + try: + L = pattern.parseString(input_string, parseAll=True) + except ParseException as err: + L = ["Parse Failure", input_string, (str(err), err.line, err.column)] + + # show result of parsing the input string + if debug_flag: + print(input_string, "->", L) + if len(L) == 0 or L[0] != "Parse Failure": + if debug_flag: + print("exprStack=", exprStack) + + for i, ob in enumerate(exprStack): + if isinstance(ob, str) and ob in variables: + exprStack[i] = str(variables[ob]) + + # calculate result , store a copy in ans , display the result to user + try: + result = evaluate_stack(exprStack) + except Exception as e: + print(str(e)) + else: + variables["ans"] = result + print(result) + + # Assign result to a variable if required + if L.varname: + variables[L.varname] = result + if debug_flag: + print("variables=", variables) + else: + print("Parse Failure") + err_str, err_line, err_col = L[-1] + print(err_line) + print(" " * (err_col - 1) + "^") + print(err_str) + + # obtain new input string input_string = input("> ") - continue - - # Reset to an empty exprStack - del exprStack[:] - - if input_string != '': - # try parsing the input string - try: - L = pattern.parseString(input_string, parseAll=True) - except ParseException as err: - L = ['Parse Failure', input_string, (str(err), err.line, err.column)] - - # show result of parsing the input string - if debug_flag: print(input_string, "->", L) - if len(L)==0 or L[0] != 'Parse Failure': - if debug_flag: print("exprStack=", exprStack) - - for i, ob in enumerate(exprStack): - if isinstance(ob, str) and ob in variables: - exprStack[i] = str(variables[ob]) - - # calculate result , store a copy in ans , display the result to user - try: - result=evaluate_stack(exprStack) - except Exception as e: - print(str(e)) - else: - variables['ans']=result - print(result) - - # Assign result to a variable if required - if L.varname: - variables[L.varname] = result - if debug_flag: print("variables=", variables) - else: - print('Parse Failure') - err_str, err_line, err_col = L[-1] - print(err_line) - print(" "*(err_col-1) + "^") - print(err_str) - - # obtain new input string - input_string = input("> ") - # if user type 'quit' then say goodbye - print("Good bye!") + # if user type 'quit' then say goodbye + print("Good bye!") diff --git a/examples/TAP.py b/examples/TAP.py index cb3afff..8676e7e 100644 --- a/examples/TAP.py +++ b/examples/TAP.py @@ -22,11 +22,24 @@ # Copyright 2008, by Paul McGuire
#
-from pyparsing import ParserElement,LineEnd,Optional,Word,nums,Regex,\
- Literal,CaselessLiteral,Group,OneOrMore,Suppress,restOfLine,\
- FollowedBy,empty
-
-__all__ = ['tapOutputParser', 'TAPTest', 'TAPSummary']
+from pyparsing import (
+ ParserElement,
+ LineEnd,
+ Optional,
+ Word,
+ nums,
+ Regex,
+ Literal,
+ CaselessLiteral,
+ Group,
+ OneOrMore,
+ Suppress,
+ restOfLine,
+ FollowedBy,
+ empty,
+)
+
+__all__ = ["tapOutputParser", "TAPTest", "TAPSummary"]
# newlines are significant whitespace, so set default skippable
# whitespace to just spaces and tabs
@@ -34,51 +47,58 @@ ParserElement.setDefaultWhitespaceChars(" \t") NL = LineEnd().suppress()
integer = Word(nums)
-plan = '1..' + integer("ubound")
+plan = "1.." + integer("ubound")
-OK,NOT_OK = map(Literal,['ok','not ok'])
-testStatus = (OK | NOT_OK)
+OK, NOT_OK = map(Literal, ["ok", "not ok"])
+testStatus = OK | NOT_OK
description = Regex("[^#\n]+")
-description.setParseAction(lambda t:t[0].lstrip('- '))
-
-TODO,SKIP = map(CaselessLiteral,'TODO SKIP'.split())
-directive = Group(Suppress('#') + (TODO + restOfLine |
- FollowedBy(SKIP) +
- restOfLine.copy().setParseAction(lambda t:['SKIP',t[0]]) ))
+description.setParseAction(lambda t: t[0].lstrip("- "))
+
+TODO, SKIP = map(CaselessLiteral, "TODO SKIP".split())
+directive = Group(
+ Suppress("#")
+ + (
+ TODO + restOfLine
+ | FollowedBy(SKIP) + restOfLine.copy().setParseAction(lambda t: ["SKIP", t[0]])
+ )
+)
commentLine = Suppress("#") + empty + restOfLine
testLine = Group(
- Optional(OneOrMore(commentLine + NL))("comments") +
- testStatus("passed") +
- Optional(integer)("testNumber") +
- Optional(description)("description") +
- Optional(directive)("directive")
- )
-bailLine = Group(Literal("Bail out!")("BAIL") +
- empty + Optional(restOfLine)("reason"))
+ Optional(OneOrMore(commentLine + NL))("comments")
+ + testStatus("passed")
+ + Optional(integer)("testNumber")
+ + Optional(description)("description")
+ + Optional(directive)("directive")
+)
+bailLine = Group(Literal("Bail out!")("BAIL") + empty + Optional(restOfLine)("reason"))
+
+tapOutputParser = Optional(Group(plan)("plan") + NL) & Group(
+ OneOrMore((testLine | bailLine) + NL)
+)("tests")
-tapOutputParser = Optional(Group(plan)("plan") + NL) & \
- Group(OneOrMore((testLine|bailLine) + NL))("tests")
class TAPTest:
- def __init__(self,results):
+ def __init__(self, results):
self.num = results.testNumber
- self.passed = (results.passed=="ok")
+ self.passed = results.passed == "ok"
self.skipped = self.todo = False
if results.directive:
- self.skipped = (results.directive[0][0]=='SKIP')
- self.todo = (results.directive[0][0]=='TODO')
+ self.skipped = results.directive[0][0] == "SKIP"
+ self.todo = results.directive[0][0] == "TODO"
+
@classmethod
- def bailedTest(cls,num):
+ def bailedTest(cls, num):
ret = TAPTest(empty.parseString(""))
ret.num = num
ret.skipped = True
return ret
+
class TAPSummary:
- def __init__(self,results):
+ def __init__(self, results):
self.passedTests = []
self.failedTests = []
self.skippedTests = []
@@ -86,22 +106,22 @@ class TAPSummary: self.bonusTests = []
self.bail = False
if results.plan:
- expected = list(range(1, int(results.plan.ubound)+1))
+ expected = list(range(1, int(results.plan.ubound) + 1))
else:
- expected = list(range(1,len(results.tests)+1))
+ expected = list(range(1, len(results.tests) + 1))
- for i,res in enumerate(results.tests):
+ for i, res in enumerate(results.tests):
# test for bail out
if res.BAIL:
- #~ print "Test suite aborted: " + res.reason
- #~ self.failedTests += expected[i:]
+ # ~ print "Test suite aborted: " + res.reason
+ # ~ self.failedTests += expected[i:]
self.bail = True
- self.skippedTests += [ TAPTest.bailedTest(ii) for ii in expected[i:] ]
+ self.skippedTests += [TAPTest.bailedTest(ii) for ii in expected[i:]]
self.bailReason = res.reason
break
- #~ print res.dump()
- testnum = i+1
+ # ~ print res.dump()
+ testnum = i + 1
if res.testNumber != "":
if testnum != int(res.testNumber):
print("ERROR! test %(testNumber)s out of sequence" % res)
@@ -113,31 +133,37 @@ class TAPSummary: self.passedTests.append(test)
else:
self.failedTests.append(test)
- if test.skipped: self.skippedTests.append(test)
- if test.todo: self.todoTests.append(test)
- if test.todo and test.passed: self.bonusTests.append(test)
+ if test.skipped:
+ self.skippedTests.append(test)
+ if test.todo:
+ self.todoTests.append(test)
+ if test.todo and test.passed:
+ self.bonusTests.append(test)
- self.passedSuite = not self.bail and (set(self.failedTests)-set(self.todoTests) == set())
+ self.passedSuite = not self.bail and (
+ set(self.failedTests) - set(self.todoTests) == set()
+ )
def summary(self, showPassed=False, showAll=False):
- testListStr = lambda tl : "[" + ",".join(str(t.num) for t in tl) + "]"
+ testListStr = lambda tl: "[" + ",".join(str(t.num) for t in tl) + "]"
summaryText = []
if showPassed or showAll:
- summaryText.append( "PASSED: %s" % testListStr(self.passedTests) )
+ summaryText.append("PASSED: %s" % testListStr(self.passedTests))
if self.failedTests or showAll:
- summaryText.append( "FAILED: %s" % testListStr(self.failedTests) )
+ summaryText.append("FAILED: %s" % testListStr(self.failedTests))
if self.skippedTests or showAll:
- summaryText.append( "SKIPPED: %s" % testListStr(self.skippedTests) )
+ summaryText.append("SKIPPED: %s" % testListStr(self.skippedTests))
if self.todoTests or showAll:
- summaryText.append( "TODO: %s" % testListStr(self.todoTests) )
+ summaryText.append("TODO: %s" % testListStr(self.todoTests))
if self.bonusTests or showAll:
- summaryText.append( "BONUS: %s" % testListStr(self.bonusTests) )
+ summaryText.append("BONUS: %s" % testListStr(self.bonusTests))
if self.passedSuite:
- summaryText.append( "PASSED" )
+ summaryText.append("PASSED")
else:
- summaryText.append( "FAILED" )
+ summaryText.append("FAILED")
return "\n".join(summaryText)
+
# create TAPSummary objects from tapOutput parsed results, by setting
# class as parse action
tapOutputParser.setParseAction(TAPSummary)
@@ -210,7 +236,7 @@ if __name__ == "__main__": 1..7
"""
- for test in (test1,test2,test3,test4,test5,test6):
+ for test in (test1, test2, test3, test4, test5, test6):
print(test)
tapResult = tapOutputParser.parseString(test)[0]
print(tapResult.summary(showAll=True))
diff --git a/examples/adventureEngine.py b/examples/adventureEngine.py index f4ef392..a6a44ad 100644 --- a/examples/adventureEngine.py +++ b/examples/adventureEngine.py @@ -8,59 +8,64 @@ from pyparsing import * import random
import string
-def aOrAn( item ):
+
+def aOrAn(item):
if item.desc[0] in "aeiou":
return "an " + item.desc
else:
return "a " + item.desc
+
def enumerateItems(l):
- if len(l) == 0: return "nothing"
+ if len(l) == 0:
+ return "nothing"
out = []
if len(l) > 1:
- out.append(', '.join(aOrAn(item) for item in l[:-1]))
- out.append('and')
+ out.append(", ".join(aOrAn(item) for item in l[:-1]))
+ out.append("and")
out.append(aOrAn(l[-1]))
return " ".join(out)
+
def enumerateDoors(l):
- if len(l) == 0: return ""
+ if len(l) == 0:
+ return ""
out = []
if len(l) > 1:
- out.append(', '.join(l[:-1]))
+ out.append(", ".join(l[:-1]))
out.append("and")
out.append(l[-1])
return " ".join(out)
+
class Room:
def __init__(self, desc):
self.desc = desc
self.inv = []
self.gameOver = False
- self.doors = [None,None,None,None]
-
- def __getattr__(self,attr):
- return \
- {
- "n":self.doors[0],
- "s":self.doors[1],
- "e":self.doors[2],
- "w":self.doors[3],
- }[attr]
-
- def enter(self,player):
+ self.doors = [None, None, None, None]
+
+ def __getattr__(self, attr):
+ return {
+ "n": self.doors[0],
+ "s": self.doors[1],
+ "e": self.doors[2],
+ "w": self.doors[3],
+ }[attr]
+
+ def enter(self, player):
if self.gameOver:
player.gameOver = True
def addItem(self, it):
self.inv.append(it)
- def removeItem(self,it):
+ def removeItem(self, it):
self.inv.remove(it)
def describe(self):
print(self.desc)
- visibleItems = [ it for it in self.inv if it.isVisible ]
+ visibleItems = [it for it in self.inv if it.isVisible]
if random.random() > 0.5:
if len(visibleItems) > 1:
is_form = "are"
@@ -75,13 +80,13 @@ class Exit(Room): def __init__(self):
super().__init__("")
- def enter(self,player):
+ def enter(self, player):
player.gameOver = True
-
class Item:
items = {}
+
def __init__(self, desc):
self.desc = desc
self.isDeadly = False
@@ -106,7 +111,7 @@ class Item: def isUsable(self, player, target):
if self.usableConditionTest:
- return self.usableConditionTest( player, target )
+ return self.usableConditionTest(player, target)
else:
return False
@@ -114,6 +119,7 @@ class Item: if self.useAction:
self.useAction(player, self, target)
+
class OpenableItem(Item):
def __init__(self, desc, contents=None):
super().__init__(desc)
@@ -121,7 +127,9 @@ class OpenableItem(Item): self.isOpened = False
if contents is not None:
if isinstance(contents, Item):
- self.contents = [contents,]
+ self.contents = [
+ contents,
+ ]
else:
self.contents = contents
else:
@@ -132,7 +140,7 @@ class OpenableItem(Item): self.isOpened = not self.isOpened
if self.contents is not None:
for item in self.contents:
- player.room.addItem( item )
+ player.room.addItem(item)
self.contents = []
self.desc = "open " + self.desc
@@ -145,6 +153,7 @@ class OpenableItem(Item): class Command:
"Base class for commands"
+
def __init__(self, verb, verbProg):
self.verb = verb
self.verbProg = verbProg
@@ -156,8 +165,8 @@ class Command: def _doCommand(self, player):
pass
- def __call__(self, player ):
- print(self.verbProg.capitalize()+"...")
+ def __call__(self, player):
+ print(self.verbProg.capitalize() + "...")
self._doCommand(player)
@@ -173,16 +182,9 @@ class MoveCommand(Command): def _doCommand(self, player):
rm = player.room
- nextRoom = rm.doors[
- {
- "N":0,
- "S":1,
- "E":2,
- "W":3,
- }[self.direction]
- ]
+ nextRoom = rm.doors[{"N": 0, "S": 1, "E": 2, "W": 3,}[self.direction]]
if nextRoom:
- player.moveTo( nextRoom )
+ player.moveTo(nextRoom)
else:
print("Can't go that way.")
@@ -227,6 +229,7 @@ class DropCommand(Command): else:
print("You don't have %s." % (aOrAn(subj)))
+
class InventoryCommand(Command):
def __init__(self, quals):
super().__init__("INV", "taking inventory")
@@ -236,7 +239,8 @@ class InventoryCommand(Command): return "INVENTORY or INV or I - lists what items you have"
def _doCommand(self, player):
- print("You have %s." % enumerateItems( player.inv ))
+ print("You have %s." % enumerateItems(player.inv))
+
class LookCommand(Command):
def __init__(self, quals):
@@ -249,6 +253,7 @@ class LookCommand(Command): def _doCommand(self, player):
player.room.describe()
+
class DoorsCommand(Command):
def __init__(self, quals):
super().__init__("DOORS", "looking for doors")
@@ -267,13 +272,17 @@ class DoorsCommand(Command): reply = "There is a door to the "
else:
reply = "There are doors to the "
- doorNames = [ {0:"north", 1:"south", 2:"east", 3:"west"}[i]
- for i,d in enumerate(rm.doors) if d is not None ]
- #~ print doorNames
- reply += enumerateDoors( doorNames )
+ doorNames = [
+ {0: "north", 1: "south", 2: "east", 3: "west"}[i]
+ for i, d in enumerate(rm.doors)
+ if d is not None
+ ]
+ # ~ print doorNames
+ reply += enumerateDoors(doorNames)
reply += "."
print(reply)
+
class UseCommand(Command):
def __init__(self, quals):
super().__init__("USE", "using")
@@ -291,13 +300,14 @@ class UseCommand(Command): rm = player.room
availItems = rm.inv + player.inv
if self.subject in availItems:
- if self.subject.isUsable( player, self.target ):
- self.subject.useItem( player, self.target )
+ if self.subject.isUsable(player, self.target):
+ self.subject.useItem(player, self.target)
else:
print("You can't use that here.")
else:
print("There is no %s here to use." % self.subject)
+
class OpenCommand(Command):
def __init__(self, quals):
super().__init__("OPEN", "opening")
@@ -309,11 +319,11 @@ class OpenCommand(Command): def _doCommand(self, player):
rm = player.room
- availItems = rm.inv+player.inv
+ availItems = rm.inv + player.inv
if self.subject in availItems:
if self.subject.isOpenable:
if not self.subject.isOpened:
- self.subject.openItem( player )
+ self.subject.openItem(player)
else:
print("It's already open.")
else:
@@ -321,6 +331,7 @@ class OpenCommand(Command): else:
print("There is no %s here to open." % self.subject)
+
class CloseCommand(Command):
def __init__(self, quals):
super().__init__("CLOSE", "closing")
@@ -332,11 +343,11 @@ class CloseCommand(Command): def _doCommand(self, player):
rm = player.room
- availItems = rm.inv+player.inv
+ availItems = rm.inv + player.inv
if self.subject in availItems:
if self.subject.isOpenable:
if self.subject.isOpened:
- self.subject.closeItem( player )
+ self.subject.closeItem(player)
else:
print("You can't close that, it's not open.")
else:
@@ -344,6 +355,7 @@ class CloseCommand(Command): else:
print("There is no %s here to close." % self.subject)
+
class QuitCommand(Command):
def __init__(self, quals):
super().__init__("QUIT", "quitting")
@@ -356,6 +368,7 @@ class QuitCommand(Command): print("Ok....")
player.gameOver = True
+
class HelpCommand(Command):
def __init__(self, quals):
super().__init__("HELP", "helping")
@@ -378,13 +391,15 @@ class HelpCommand(Command): DoorsCommand,
QuitCommand,
HelpCommand,
- ]:
+ ]:
print(" - %s" % cmd.helpDescription())
print()
+
class AppParseException(ParseException):
pass
+
class Parser:
def __init__(self):
self.bnf = self.makeBNF()
@@ -392,8 +407,9 @@ class Parser: def makeBNF(self):
invVerb = oneOf("INV INVENTORY I", caseless=True)
dropVerb = oneOf("DROP LEAVE", caseless=True)
- takeVerb = oneOf("TAKE PICKUP", caseless=True) | \
- (CaselessLiteral("PICK") + CaselessLiteral("UP") )
+ takeVerb = oneOf("TAKE PICKUP", caseless=True) | (
+ CaselessLiteral("PICK") + CaselessLiteral("UP")
+ )
moveVerb = oneOf("MOVE GO", caseless=True) | empty
useVerb = oneOf("USE U", caseless=True)
openVerb = oneOf("OPEN O", caseless=True)
@@ -401,21 +417,24 @@ class Parser: quitVerb = oneOf("QUIT Q", caseless=True)
lookVerb = oneOf("LOOK L", caseless=True)
doorsVerb = CaselessLiteral("DOORS")
- helpVerb = oneOf("H HELP ?",caseless=True)
+ helpVerb = oneOf("H HELP ?", caseless=True)
- itemRef = OneOrMore(Word(alphas)).setParseAction( self.validateItemName )
- nDir = oneOf("N NORTH",caseless=True).setParseAction(replaceWith("N"))
- sDir = oneOf("S SOUTH",caseless=True).setParseAction(replaceWith("S"))
- eDir = oneOf("E EAST",caseless=True).setParseAction(replaceWith("E"))
- wDir = oneOf("W WEST",caseless=True).setParseAction(replaceWith("W"))
+ itemRef = OneOrMore(Word(alphas)).setParseAction(self.validateItemName)
+ nDir = oneOf("N NORTH", caseless=True).setParseAction(replaceWith("N"))
+ sDir = oneOf("S SOUTH", caseless=True).setParseAction(replaceWith("S"))
+ eDir = oneOf("E EAST", caseless=True).setParseAction(replaceWith("E"))
+ wDir = oneOf("W WEST", caseless=True).setParseAction(replaceWith("W"))
moveDirection = nDir | sDir | eDir | wDir
invCommand = invVerb
dropCommand = dropVerb + itemRef("item")
takeCommand = takeVerb + itemRef("item")
- useCommand = useVerb + itemRef("usedObj") + \
- Optional(oneOf("IN ON",caseless=True)) + \
- Optional(itemRef,default=None)("targetObj")
+ useCommand = (
+ useVerb
+ + itemRef("usedObj")
+ + Optional(oneOf("IN ON", caseless=True))
+ + Optional(itemRef, default=None)("targetObj")
+ )
openCommand = openVerb + itemRef("item")
closeCommand = closeVerb + itemRef("item")
moveCommand = moveVerb + moveDirection("direction")
@@ -438,22 +457,24 @@ class Parser: helpCommand.setParseAction(HelpCommand)
# define parser using all command expressions
- return ( invCommand |
- useCommand |
- openCommand |
- closeCommand |
- dropCommand |
- takeCommand |
- moveCommand |
- lookCommand |
- doorsCommand |
- helpCommand |
- quitCommand )("command") + LineEnd()
-
- def validateItemName(self,s,l,t):
+ return (
+ invCommand
+ | useCommand
+ | openCommand
+ | closeCommand
+ | dropCommand
+ | takeCommand
+ | moveCommand
+ | lookCommand
+ | doorsCommand
+ | helpCommand
+ | quitCommand
+ )("command") + LineEnd()
+
+ def validateItemName(self, s, l, t):
iname = " ".join(t)
if iname not in Item.items:
- raise AppParseException(s,l,"No such item '%s'." % iname)
+ raise AppParseException(s, l, "No such item '%s'." % iname)
return iname
def parseCmd(self, cmdstr):
@@ -463,11 +484,18 @@ class Parser: except AppParseException as pe:
print(pe.msg)
except ParseException as pe:
- print(random.choice([ "Sorry, I don't understand that.",
- "Huh?",
- "Excuse me?",
- "???",
- "What?" ] ))
+ print(
+ random.choice(
+ [
+ "Sorry, I don't understand that.",
+ "Huh?",
+ "Excuse me?",
+ "???",
+ "What?",
+ ]
+ )
+ )
+
class Player:
def __init__(self, name):
@@ -485,20 +513,20 @@ class Player: else:
rm.describe()
- def take(self,it):
+ def take(self, it):
if it.isDeadly:
print("Aaaagh!...., the %s killed me!" % it)
self.gameOver = True
else:
self.inv.append(it)
- def drop(self,it):
+ def drop(self, it):
self.inv.remove(it)
if it.isFragile:
it.breakItem()
-def createRooms( rm ):
+def createRooms(rm):
"""
create rooms, using multiline string showing map layout
string contains symbols for the following:
@@ -521,8 +549,8 @@ def createRooms( rm ): # scan through input string looking for connections between rooms
rows = rm.split("\n")
- for row,line in enumerate(rows):
- for col,c in enumerate(line):
+ for row, line in enumerate(rows):
+ for col, c in enumerate(line):
if c in string.ascii_letters:
room = ret[c]
n = None
@@ -533,46 +561,52 @@ def createRooms( rm ): # look in neighboring cells for connection symbols (must take
# care to guard that neighboring cells exist before testing
# contents)
- if col > 0 and line[col-1] in "<-":
- other = line[col-2]
+ if col > 0 and line[col - 1] in "<-":
+ other = line[col - 2]
w = ret[other]
- if col < len(line)-1 and line[col+1] in "->":
- other = line[col+2]
+ if col < len(line) - 1 and line[col + 1] in "->":
+ other = line[col + 2]
e = ret[other]
- if row > 1 and col < len(rows[row-1]) and rows[row-1][col] in '|^':
- other = rows[row-2][col]
+ if row > 1 and col < len(rows[row - 1]) and rows[row - 1][col] in "|^":
+ other = rows[row - 2][col]
n = ret[other]
- if row < len(rows)-1 and col < len(rows[row+1]) and rows[row+1][col] in '|.':
- other = rows[row+2][col]
+ if (
+ row < len(rows) - 1
+ and col < len(rows[row + 1])
+ and rows[row + 1][col] in "|."
+ ):
+ other = rows[row + 2][col]
s = ret[other]
# set connections to neighboring rooms
- room.doors=[n,s,e,w]
+ room.doors = [n, s, e, w]
return ret
+
# put items in rooms
-def putItemInRoom(i,r):
- if isinstance(r,str):
+def putItemInRoom(i, r):
+ if isinstance(r, str):
r = rooms[r]
- r.addItem( Item.items[i] )
+ r.addItem(Item.items[i])
-def playGame(p,startRoom):
+
+def playGame(p, startRoom):
# create parser
parser = Parser()
- p.moveTo( startRoom )
+ p.moveTo(startRoom)
while not p.gameOver:
cmdstr = input(">> ")
cmd = parser.parseCmd(cmdstr)
if cmd is not None:
- cmd.command( p )
+ cmd.command(p)
print()
print("You ended the game with:")
for i in p.inv:
print(" -", aOrAn(i))
-#====================
+# ====================
# start game definition
roomMap = """
d-Z
@@ -583,7 +617,7 @@ roomMap = """ |
A
"""
-rooms = createRooms( roomMap )
+rooms = createRooms(roomMap)
rooms["A"].desc = "You are standing on the front porch of a wooden shack."
rooms["b"].desc = "You are in a garden."
rooms["c"].desc = "You are in a kitchen."
@@ -595,29 +629,39 @@ rooms["q"].gameOver = True # define global variables for referencing rooms
frontPorch = rooms["A"]
-garden = rooms["b"]
-kitchen = rooms["c"]
-backPorch = rooms["d"]
-library = rooms["e"]
-patio = rooms["f"]
+garden = rooms["b"]
+kitchen = rooms["c"]
+backPorch = rooms["d"]
+library = rooms["e"]
+patio = rooms["f"]
# create items
-itemNames = """sword.diamond.apple.flower.coin.shovel.book.mirror.telescope.gold bar""".split(".")
+itemNames = """sword.diamond.apple.flower.coin.shovel.book.mirror.telescope.gold bar""".split(
+ "."
+)
for itemName in itemNames:
- Item( itemName )
+ Item(itemName)
Item.items["apple"].isDeadly = True
Item.items["mirror"].isFragile = True
Item.items["coin"].isVisible = False
-Item.items["shovel"].usableConditionTest = ( lambda p,t: p.room is garden )
-def useShovel(p,subj,target):
+Item.items["shovel"].usableConditionTest = lambda p, t: p.room is garden
+
+
+def useShovel(p, subj, target):
coin = Item.items["coin"]
if not coin.isVisible and coin in p.room.inv:
coin.isVisible = True
+
+
Item.items["shovel"].useAction = useShovel
Item.items["telescope"].isTakeable = False
-def useTelescope(p,subj,target):
+
+
+def useTelescope(p, subj, target):
print("You don't see anything.")
+
+
Item.items["telescope"].useAction = useTelescope
OpenableItem("treasure chest", Item.items["gold bar"])
@@ -642,7 +686,7 @@ putItemInRoom("treasure chest", patio) # create player
plyr = Player("Bob")
-plyr.take( Item.items["sword"] )
+plyr.take(Item.items["sword"])
# start game
-playGame( plyr, frontPorch )
+playGame(plyr, frontPorch)
diff --git a/examples/antlr_grammar.py b/examples/antlr_grammar.py index c131cfb..49151ee 100644 --- a/examples/antlr_grammar.py +++ b/examples/antlr_grammar.py @@ -1,4 +1,4 @@ -''' +""" antlr_grammar.py Created on 4 sept. 2010 @@ -8,196 +8,299 @@ Created on 4 sept. 2010 Submitted by Luca DallOlio, September, 2010 (Minor updates by Paul McGuire, June, 2012) (Code idiom updates by Paul McGuire, April, 2019) -''' -from pyparsing import (Word, ZeroOrMore, printables, Suppress, OneOrMore, Group, - LineEnd, Optional, White, originalTextFor, hexnums, nums, Combine, Literal, Keyword, - cStyleComment, Regex, Forward, MatchFirst, And, oneOf, alphas, alphanums, - delimitedList, Char) +""" +from pyparsing import ( + Word, + ZeroOrMore, + printables, + Suppress, + OneOrMore, + Group, + LineEnd, + Optional, + White, + originalTextFor, + hexnums, + nums, + Combine, + Literal, + Keyword, + cStyleComment, + Regex, + Forward, + MatchFirst, + And, + oneOf, + alphas, + alphanums, + delimitedList, + Char, +) # http://www.antlr.org/grammar/ANTLR/ANTLRv3.g -QUOTE,APOS,EQ,LBRACK,RBRACK,LBRACE,RBRACE,LPAR,RPAR,ROOT,BANG,AT,TIL,SEMI,COLON,VERT = map(Suppress, - '"\'=[]{}()^!@~;:|') -BSLASH = Literal('\\') -keywords = (SRC_, SCOPE_, OPTIONS_, TOKENS_, FRAGMENT, ID, LEXER, PARSER, GRAMMAR, TREE, CATCH, FINALLY, - THROWS, PROTECTED, PUBLIC, PRIVATE, ) = map(Keyword, - """src scope options tokens fragment id lexer parser grammar tree catch finally throws protected - public private """.split()) +( + QUOTE, + APOS, + EQ, + LBRACK, + RBRACK, + LBRACE, + RBRACE, + LPAR, + RPAR, + ROOT, + BANG, + AT, + TIL, + SEMI, + COLON, + VERT, +) = map(Suppress, "\"'=[]{}()^!@~;:|") +BSLASH = Literal("\\") +keywords = ( + SRC_, + SCOPE_, + OPTIONS_, + TOKENS_, + FRAGMENT, + ID, + LEXER, + PARSER, + GRAMMAR, + TREE, + CATCH, + FINALLY, + THROWS, + PROTECTED, + PUBLIC, + PRIVATE, +) = map( + Keyword, + """src scope options tokens fragment id lexer parser grammar tree catch finally throws protected + public private """.split(), +) KEYWORD = MatchFirst(keywords) # Tokens -EOL = Suppress(LineEnd()) # $ +EOL = Suppress(LineEnd()) # $ SGL_PRINTABLE = Char(printables) -singleTextString = originalTextFor(ZeroOrMore(~EOL + (White(" \t") | Word(printables)))).leaveWhitespace() +singleTextString = originalTextFor( + ZeroOrMore(~EOL + (White(" \t") | Word(printables))) +).leaveWhitespace() XDIGIT = hexnums INT = Word(nums) -ESC = BSLASH + (oneOf(list(r'nrtbf\">'+"'")) | ('u' + Word(hexnums, exact=4)) | SGL_PRINTABLE) +ESC = BSLASH + ( + oneOf(list(r"nrtbf\">" + "'")) | ("u" + Word(hexnums, exact=4)) | SGL_PRINTABLE +) LITERAL_CHAR = ESC | ~(APOS | BSLASH) + SGL_PRINTABLE CHAR_LITERAL = APOS + LITERAL_CHAR + APOS STRING_LITERAL = APOS + Combine(OneOrMore(LITERAL_CHAR)) + APOS DOUBLE_QUOTE_STRING_LITERAL = '"' + ZeroOrMore(LITERAL_CHAR) + '"' -DOUBLE_ANGLE_STRING_LITERAL = '<<' + ZeroOrMore(SGL_PRINTABLE) + '>>' -TOKEN_REF = Word(alphas.upper(), alphanums+'_') -RULE_REF = Word(alphas.lower(), alphanums+'_') -ACTION_ESC = (BSLASH.suppress() + APOS - | BSLASH.suppress() - | BSLASH.suppress() + (~(APOS | QUOTE) + SGL_PRINTABLE) - ) -ACTION_CHAR_LITERAL = (APOS + (ACTION_ESC | ~(BSLASH | APOS) + SGL_PRINTABLE) + APOS) -ACTION_STRING_LITERAL = (QUOTE + ZeroOrMore(ACTION_ESC | ~(BSLASH | QUOTE) + SGL_PRINTABLE) + QUOTE) +DOUBLE_ANGLE_STRING_LITERAL = "<<" + ZeroOrMore(SGL_PRINTABLE) + ">>" +TOKEN_REF = Word(alphas.upper(), alphanums + "_") +RULE_REF = Word(alphas.lower(), alphanums + "_") +ACTION_ESC = ( + BSLASH.suppress() + APOS + | BSLASH.suppress() + | BSLASH.suppress() + (~(APOS | QUOTE) + SGL_PRINTABLE) +) +ACTION_CHAR_LITERAL = APOS + (ACTION_ESC | ~(BSLASH | APOS) + SGL_PRINTABLE) + APOS +ACTION_STRING_LITERAL = ( + QUOTE + ZeroOrMore(ACTION_ESC | ~(BSLASH | QUOTE) + SGL_PRINTABLE) + QUOTE +) SRC = SRC_.suppress() + ACTION_STRING_LITERAL("file") + INT("line") id = TOKEN_REF | RULE_REF -SL_COMMENT = Suppress('//') + Suppress('$ANTLR') + SRC | ZeroOrMore(~EOL + Word(printables)) + EOL +SL_COMMENT = ( + Suppress("//") + Suppress("$ANTLR") + SRC + | ZeroOrMore(~EOL + Word(printables)) + EOL +) ML_COMMENT = cStyleComment -WS = OneOrMore(Suppress(' ') | Suppress('\t') | (Optional(Suppress('\r')) + Literal('\n'))) +WS = OneOrMore( + Suppress(" ") | Suppress("\t") | (Optional(Suppress("\r")) + Literal("\n")) +) WS_LOOP = ZeroOrMore(SL_COMMENT | ML_COMMENT) NESTED_ARG_ACTION = Forward() -NESTED_ARG_ACTION << (LBRACK - + ZeroOrMore(NESTED_ARG_ACTION - | ACTION_STRING_LITERAL - | ACTION_CHAR_LITERAL) - + RBRACK) +NESTED_ARG_ACTION << ( + LBRACK + + ZeroOrMore(NESTED_ARG_ACTION | ACTION_STRING_LITERAL | ACTION_CHAR_LITERAL) + + RBRACK +) ARG_ACTION = NESTED_ARG_ACTION NESTED_ACTION = Forward() -NESTED_ACTION << (LBRACE - + ZeroOrMore(NESTED_ACTION - | SL_COMMENT - | ML_COMMENT - | ACTION_STRING_LITERAL - | ACTION_CHAR_LITERAL) - + RBRACE) -ACTION = NESTED_ACTION + Optional('?') +NESTED_ACTION << ( + LBRACE + + ZeroOrMore( + NESTED_ACTION + | SL_COMMENT + | ML_COMMENT + | ACTION_STRING_LITERAL + | ACTION_CHAR_LITERAL + ) + + RBRACE +) +ACTION = NESTED_ACTION + Optional("?") SCOPE = SCOPE_.suppress() -OPTIONS = OPTIONS_.suppress() + LBRACE # + WS_LOOP + Suppress('{') -TOKENS = TOKENS_.suppress() + LBRACE # + WS_LOOP + Suppress('{') +OPTIONS = OPTIONS_.suppress() + LBRACE # + WS_LOOP + Suppress('{') +TOKENS = TOKENS_.suppress() + LBRACE # + WS_LOOP + Suppress('{') TREE_BEGIN = ROOT + LPAR -RANGE = Suppress('..') -REWRITE = Suppress('->') +RANGE = Suppress("..") +REWRITE = Suppress("->") # General Parser Definitions # Grammar heading -optionValue = id | STRING_LITERAL | CHAR_LITERAL | INT | Literal('*').setName("s") +optionValue = id | STRING_LITERAL | CHAR_LITERAL | INT | Literal("*").setName("s") option = Group(id("id") + EQ + optionValue("value"))("option") optionsSpec = OPTIONS + Group(OneOrMore(option + SEMI))("options") + RBRACE -tokenSpec = Group(TOKEN_REF("token_ref") - + (EQ + (STRING_LITERAL | CHAR_LITERAL)("lit")))("token") + SEMI +tokenSpec = ( + Group(TOKEN_REF("token_ref") + (EQ + (STRING_LITERAL | CHAR_LITERAL)("lit")))( + "token" + ) + + SEMI +) tokensSpec = TOKENS + Group(OneOrMore(tokenSpec))("tokens") + RBRACE attrScope = SCOPE_.suppress() + id + ACTION grammarType = LEXER + PARSER + TREE actionScopeName = id | LEXER("l") | PARSER("p") -action = AT + Optional(actionScopeName + Suppress('::')) + id + ACTION - -grammarHeading = (Optional(ML_COMMENT("ML_COMMENT")) - + Optional(grammarType) - + GRAMMAR - + id("grammarName") + SEMI - + Optional(optionsSpec) - + Optional(tokensSpec) - + ZeroOrMore(attrScope) - + ZeroOrMore(action)) +action = AT + Optional(actionScopeName + Suppress("::")) + id + ACTION + +grammarHeading = ( + Optional(ML_COMMENT("ML_COMMENT")) + + Optional(grammarType) + + GRAMMAR + + id("grammarName") + + SEMI + + Optional(optionsSpec) + + Optional(tokensSpec) + + ZeroOrMore(attrScope) + + ZeroOrMore(action) +) modifier = PROTECTED | PUBLIC | PRIVATE | FRAGMENT ruleAction = AT + id + ACTION throwsSpec = THROWS.suppress() + delimitedList(id) -ruleScopeSpec = ((SCOPE_.suppress() + ACTION) - | (SCOPE_.suppress() + delimitedList(id) + SEMI) - | (SCOPE_.suppress() + ACTION + SCOPE_.suppress() + delimitedList(id) + SEMI)) +ruleScopeSpec = ( + (SCOPE_.suppress() + ACTION) + | (SCOPE_.suppress() + delimitedList(id) + SEMI) + | (SCOPE_.suppress() + ACTION + SCOPE_.suppress() + delimitedList(id) + SEMI) +) unary_op = oneOf("^ !") notTerminal = CHAR_LITERAL | TOKEN_REF | STRING_LITERAL -terminal = (CHAR_LITERAL | TOKEN_REF + Optional(ARG_ACTION) | STRING_LITERAL | '.') + Optional(unary_op) +terminal = ( + CHAR_LITERAL | TOKEN_REF + Optional(ARG_ACTION) | STRING_LITERAL | "." +) + Optional(unary_op) block = Forward() notSet = TIL + (notTerminal | block) rangeNotPython = CHAR_LITERAL("c1") + RANGE + CHAR_LITERAL("c2") -atom = Group((rangeNotPython + Optional(unary_op)("op")) - | terminal - | (notSet + Optional(unary_op)("op")) - | (RULE_REF + Optional(ARG_ACTION("arg")) + Optional(unary_op)("op")) - ) +atom = Group( + (rangeNotPython + Optional(unary_op)("op")) + | terminal + | (notSet + Optional(unary_op)("op")) + | (RULE_REF + Optional(ARG_ACTION("arg")) + Optional(unary_op)("op")) +) element = Forward() -treeSpec = ROOT + LPAR + element*(2,) + RPAR +treeSpec = ROOT + LPAR + element * (2,) + RPAR ebnfSuffix = oneOf("? * +") -ebnf = block + Optional(ebnfSuffix("op") | '=>') -elementNoOptionSpec = ((id("result_name") + oneOf('= +=')("labelOp") + atom("atom") + Optional(ebnfSuffix)) - | (id("result_name") + oneOf('= +=')("labelOp") + block + Optional(ebnfSuffix)) - | atom("atom") + Optional(ebnfSuffix) - | ebnf - | ACTION - | (treeSpec + Optional(ebnfSuffix)) - ) # | SEMPRED ( '=>' -> GATED_SEMPRED | -> SEMPRED ) +ebnf = block + Optional(ebnfSuffix("op") | "=>") +elementNoOptionSpec = ( + (id("result_name") + oneOf("= +=")("labelOp") + atom("atom") + Optional(ebnfSuffix)) + | (id("result_name") + oneOf("= +=")("labelOp") + block + Optional(ebnfSuffix)) + | atom("atom") + Optional(ebnfSuffix) + | ebnf + | ACTION + | (treeSpec + Optional(ebnfSuffix)) +) # | SEMPRED ( '=>' -> GATED_SEMPRED | -> SEMPRED ) element <<= Group(elementNoOptionSpec)("element") # Do not ask me why group is needed twice... seems like the xml that you see is not always the real structure? alternative = Group(Group(OneOrMore(element))("elements")) -rewrite = Optional(Literal('TODO REWRITE RULES TODO')) -block <<= (LPAR - + Optional(Optional(optionsSpec("opts")) + COLON) - + Group(alternative('a1') - + rewrite - + Group(ZeroOrMore(VERT - + alternative('a2') - + rewrite))("alternatives"))("block") - + RPAR) -altList = alternative('a1') + rewrite + Group(ZeroOrMore(VERT + alternative('a2') + rewrite))("alternatives") +rewrite = Optional(Literal("TODO REWRITE RULES TODO")) +block <<= ( + LPAR + + Optional(Optional(optionsSpec("opts")) + COLON) + + Group( + alternative("a1") + + rewrite + + Group(ZeroOrMore(VERT + alternative("a2") + rewrite))("alternatives") + )("block") + + RPAR +) +altList = ( + alternative("a1") + + rewrite + + Group(ZeroOrMore(VERT + alternative("a2") + rewrite))("alternatives") +) exceptionHandler = CATCH.suppress() + ARG_ACTION + ACTION finallyClause = FINALLY.suppress() + ACTION exceptionGroup = (OneOrMore(exceptionHandler) + Optional(finallyClause)) | finallyClause -ruleHeading = (Optional(ML_COMMENT)("ruleComment") - + Optional(modifier)("modifier") - + id("ruleName") - + Optional("!") - + Optional(ARG_ACTION("arg")) - + Optional(Suppress('returns') + ARG_ACTION("rt")) - + Optional(throwsSpec) - + Optional(optionsSpec) - + Optional(ruleScopeSpec) - + ZeroOrMore(ruleAction)) +ruleHeading = ( + Optional(ML_COMMENT)("ruleComment") + + Optional(modifier)("modifier") + + id("ruleName") + + Optional("!") + + Optional(ARG_ACTION("arg")) + + Optional(Suppress("returns") + ARG_ACTION("rt")) + + Optional(throwsSpec) + + Optional(optionsSpec) + + Optional(ruleScopeSpec) + + ZeroOrMore(ruleAction) +) rule = Group(ruleHeading + COLON + altList + SEMI + Optional(exceptionGroup))("rule") grammarDef = grammarHeading + Group(OneOrMore(rule))("rules") + def grammar(): return grammarDef + def __antlrAlternativesConverter(pyparsingRules, antlrBlock): rule = None - if hasattr(antlrBlock, 'alternatives') and antlrBlock.alternatives != '' and len(antlrBlock.alternatives) > 0: + if ( + hasattr(antlrBlock, "alternatives") + and antlrBlock.alternatives != "" + and len(antlrBlock.alternatives) > 0 + ): alternatives = [] alternatives.append(__antlrAlternativeConverter(pyparsingRules, antlrBlock.a1)) for alternative in antlrBlock.alternatives: - alternatives.append(__antlrAlternativeConverter(pyparsingRules, alternative)) + alternatives.append( + __antlrAlternativeConverter(pyparsingRules, alternative) + ) rule = MatchFirst(alternatives)("anonymous_or") - elif hasattr(antlrBlock, 'a1') and antlrBlock.a1 != '': + elif hasattr(antlrBlock, "a1") and antlrBlock.a1 != "": rule = __antlrAlternativeConverter(pyparsingRules, antlrBlock.a1) else: - raise Exception('Not yet implemented') + raise Exception("Not yet implemented") assert rule != None return rule + def __antlrAlternativeConverter(pyparsingRules, antlrAlternative): elementList = [] for element in antlrAlternative.elements: rule = None - if hasattr(element.atom, 'c1') and element.atom.c1 != '': - regex = r'['+str(element.atom.c1[0])+'-'+str(element.atom.c2[0]+']') + if hasattr(element.atom, "c1") and element.atom.c1 != "": + regex = r"[" + str(element.atom.c1[0]) + "-" + str(element.atom.c2[0] + "]") rule = Regex(regex)("anonymous_regex") - elif hasattr(element, 'block') and element.block != '': + elif hasattr(element, "block") and element.block != "": rule = __antlrAlternativesConverter(pyparsingRules, element.block) else: ruleRef = element.atom[0] assert ruleRef in pyparsingRules rule = pyparsingRules[ruleRef](ruleRef) - if hasattr(element, 'op') and element.op != '': - if element.op == '+': + if hasattr(element, "op") and element.op != "": + if element.op == "+": rule = Group(OneOrMore(rule))("anonymous_one_or_more") - elif element.op == '*': + elif element.op == "*": rule = Group(ZeroOrMore(rule))("anonymous_zero_or_more") - elif element.op == '?': + elif element.op == "?": rule = Optional(rule) else: - raise Exception('rule operator not yet implemented : ' + element.op) + raise Exception("rule operator not yet implemented : " + element.op) rule = rule elementList.append(rule) if len(elementList) > 1: @@ -207,6 +310,7 @@ def __antlrAlternativeConverter(pyparsingRules, antlrAlternative): assert rule is not None return rule + def __antlrRuleConverter(pyparsingRules, antlrRule): rule = None rule = __antlrAlternativesConverter(pyparsingRules, antlrRule) @@ -214,6 +318,7 @@ def __antlrRuleConverter(pyparsingRules, antlrRule): rule(antlrRule.ruleName) return rule + def antlrConverter(antlrGrammarTree): pyparsingRules = {} @@ -226,7 +331,7 @@ def antlrConverter(antlrGrammarTree): antlrRules = {} for antlrRule in antlrGrammarTree.rules: antlrRules[antlrRule.ruleName] = antlrRule - pyparsingRules[antlrRule.ruleName] = Forward() # antlr is a top down grammar + pyparsingRules[antlrRule.ruleName] = Forward() # antlr is a top down grammar for antlrRuleName, antlrRule in list(antlrRules.items()): pyparsingRule = __antlrRuleConverter(pyparsingRules, antlrRule) assert pyparsingRule != None @@ -234,6 +339,7 @@ def antlrConverter(antlrGrammarTree): return pyparsingRules + if __name__ == "__main__": text = """\ diff --git a/examples/antlr_grammar_tests.py b/examples/antlr_grammar_tests.py index 57d6cb6..17d8fa0 100644 --- a/examples/antlr_grammar_tests.py +++ b/examples/antlr_grammar_tests.py @@ -1,21 +1,20 @@ -''' +""" Created on 4 sept. 2010 @author: luca Submitted by Luca DallOlio, September, 2010 -''' +""" import unittest from . import antlr_grammar -class Test(unittest.TestCase): - +class Test(unittest.TestCase): def testOptionsSpec(self): text = """options { language = Python; }""" - antlr_grammar.optionsSpec.parseString(text) #@UndefinedVariable + antlr_grammar.optionsSpec.parseString(text) # @UndefinedVariable def testTokensSpec(self): text = """tokens { @@ -24,23 +23,23 @@ class Test(unittest.TestCase): MULT = '*' ; DIV = '/' ; }""" - antlr_grammar.tokensSpec.parseString(text) #@UndefinedVariable + antlr_grammar.tokensSpec.parseString(text) # @UndefinedVariable def testBlock(self): text = """( PLUS | MINUS )""" - antlr_grammar.block.parseString(text) #@UndefinedVariable + antlr_grammar.block.parseString(text) # @UndefinedVariable def testRule(self): text = """expr : term ( ( PLUS | MINUS ) term )* ;""" - antlr_grammar.rule.parseString(text) #@UndefinedVariable + antlr_grammar.rule.parseString(text) # @UndefinedVariable def testLexerRule(self): text = """fragment DIGIT : '0'..'9' ;""" - antlr_grammar.rule.parseString(text) #@UndefinedVariable + antlr_grammar.rule.parseString(text) # @UndefinedVariable def testLexerRule2(self): text = """WHITESPACE : ( '\t' | ' ' | '\r' | '\n'| '\u000C' )+ { $channel = HIDDEN; } ;""" - #antlr_grammar.rule.parseString(text) #@UndefinedVariable + # antlr_grammar.rule.parseString(text) #@UndefinedVariable def testGrammar(self): text = """grammar SimpleCalc; @@ -76,16 +75,28 @@ NUMBER : (DIGIT)+ ; /* WHITESPACE : ( '\t' | ' ' | '\r' | '\n'| '\u000C' )+ { $channel = HIDDEN; } ; */ fragment DIGIT : '0'..'9' ;""" - antlrGrammarTree = antlr_grammar.grammarDef.parseString(text) #@UndefinedVariable + antlrGrammarTree = antlr_grammar.grammarDef.parseString( + text + ) # @UndefinedVariable pyparsingRules = antlr_grammar.antlrConverter(antlrGrammarTree) pyparsingRule = pyparsingRules["expr"] pyparsingTree = pyparsingRule.parseString("2 - 5 * 42 + 7 / 25") pyparsingTreeList = pyparsingTree.asList() print(pyparsingTreeList) - self.assertEqual(pyparsingTreeList, - [[[['2'], []], [['-', [['5'], [['*', ['4', '2']]]]], ['+', [['7'], [['/', ['2', '5']]]]]]]] - ) + self.assertEqual( + pyparsingTreeList, + [ + [ + [["2"], []], + [ + ["-", [["5"], [["*", ["4", "2"]]]]], + ["+", [["7"], [["/", ["2", "5"]]]]], + ], + ] + ], + ) + if __name__ == "__main__": - #import sys;sys.argv = ['', 'Test.testOptionsSpec'] + # import sys;sys.argv = ['', 'Test.testOptionsSpec'] unittest.main() diff --git a/examples/apicheck.py b/examples/apicheck.py index cd35a9a..1905d4a 100644 --- a/examples/apicheck.py +++ b/examples/apicheck.py @@ -9,25 +9,24 @@ from pyparsing import *
# define punctuation and simple tokens for locating API calls
-LBRACK,RBRACK,LBRACE,RBRACE = map(Suppress,"[]{}")
-ident = Word(alphas,alphanums+"_") | QuotedString("{",endQuoteChar="}")
+LBRACK, RBRACK, LBRACE, RBRACE = map(Suppress, "[]{}")
+ident = Word(alphas, alphanums + "_") | QuotedString("{", endQuoteChar="}")
arg = "$" + ident
# define an API call with a specific number of arguments - using '-'
# will ensure that after matching procname, an incorrect number of args will
# raise a ParseSyntaxException, which will interrupt the scanString
def apiProc(name, numargs):
- return LBRACK + Keyword(name)("procname") - arg*numargs + RBRACK
+ return LBRACK + Keyword(name)("procname") - arg * numargs + RBRACK
+
# create an apiReference, listing all API functions to be scanned for, and
# their respective number of arguments. Beginning the overall expression
# with FollowedBy allows us to quickly rule out non-api calls while scanning,
# since all of the api calls begin with a "["
-apiRef = FollowedBy("[") + MatchFirst([
- apiProc("procname1", 2),
- apiProc("procname2", 1),
- apiProc("procname3", 2),
- ])
+apiRef = FollowedBy("[") + MatchFirst(
+ [apiProc("procname1", 2), apiProc("procname2", 1), apiProc("procname3", 2),]
+)
test = """[ procname1 $par1 $par2 ]
other code here
@@ -45,13 +44,13 @@ test = """[ procname1 $par1 $par2 ] api_scanner = apiRef.scanString(test)
while 1:
try:
- t,s,e = next(api_scanner)
- print("found %s on line %d" % (t.procname, lineno(s,test)))
+ t, s, e = next(api_scanner)
+ print("found %s on line %d" % (t.procname, lineno(s, test)))
except ParseSyntaxException as pe:
print("invalid arg count on line", pe.lineno)
- print(pe.lineno,':',pe.line)
+ print(pe.lineno, ":", pe.line)
# reset api scanner to start after this exception location
- test = "\n"*(pe.lineno-1)+test[pe.loc+1:]
+ test = "\n" * (pe.lineno - 1) + test[pe.loc + 1 :]
api_scanner = apiRef.scanString(test)
except StopIteration:
break
diff --git a/examples/bigquery_view_parser.py b/examples/bigquery_view_parser.py index 0277a17..695ca30 100644 --- a/examples/bigquery_view_parser.py +++ b/examples/bigquery_view_parser.py @@ -16,6 +16,7 @@ from pyparsing import infixNotation, opAssoc, OneOrMore, Regex, nums class BigQueryViewParser: """Parser to extract table info from BigQuery view definitions""" + _parser = None _table_identifiers = set() _with_aliases = set() @@ -27,7 +28,8 @@ class BigQueryViewParser: # relevant- aliases are not case sensitive lower_aliases = BigQueryViewParser.lowercase_set_of_tuples(with_aliases) tables = { - x for x in table_identifiers + x + for x in table_identifiers if not BigQueryViewParser.lowercase_of_tuple(x) in lower_aliases } @@ -62,6 +64,147 @@ class BigQueryViewParser: # keywords ( + UNION, + ALL, + AND, + INTERSECT, + EXCEPT, + COLLATE, + ASC, + DESC, + ON, + USING, + NATURAL, + INNER, + CROSS, + LEFT, + RIGHT, + OUTER, + FULL, + JOIN, + AS, + INDEXED, + NOT, + SELECT, + DISTINCT, + FROM, + WHERE, + GROUP, + BY, + HAVING, + ORDER, + BY, + LIMIT, + OFFSET, + OR, + CAST, + ISNULL, + NOTNULL, + NULL, + IS, + BETWEEN, + ELSE, + END, + CASE, + WHEN, + THEN, + EXISTS, + COLLATE, + IN, + LIKE, + GLOB, + REGEXP, + MATCH, + ESCAPE, + CURRENT_TIME, + CURRENT_DATE, + CURRENT_TIMESTAMP, + WITH, + EXTRACT, + PARTITION, + ROWS, + RANGE, + UNBOUNDED, + PRECEDING, + CURRENT, + ROW, + FOLLOWING, + OVER, + INTERVAL, + DATE_ADD, + DATE_SUB, + ADDDATE, + SUBDATE, + REGEXP_EXTRACT, + SPLIT, + ORDINAL, + FIRST_VALUE, + LAST_VALUE, + NTH_VALUE, + LEAD, + LAG, + PERCENTILE_CONT, + PRECENTILE_DISC, + RANK, + DENSE_RANK, + PERCENT_RANK, + CUME_DIST, + NTILE, + ROW_NUMBER, + DATE, + TIME, + DATETIME, + TIMESTAMP, + UNNEST, + INT64, + NUMERIC, + FLOAT64, + BOOL, + BYTES, + GEOGRAPHY, + ARRAY, + STRUCT, + SAFE_CAST, + ANY_VALUE, + ARRAY_AGG, + ARRAY_CONCAT_AGG, + AVG, + BIT_AND, + BIT_OR, + BIT_XOR, + COUNT, + COUNTIF, + LOGICAL_AND, + LOGICAL_OR, + MAX, + MIN, + STRING_AGG, + SUM, + CORR, + COVAR_POP, + COVAR_SAMP, + STDDEV_POP, + STDDEV_SAMP, + STDDEV, + VAR_POP, + VAR_SAMP, + VARIANCE, + TIMESTAMP_ADD, + TIMESTAMP_SUB, + GENERATE_ARRAY, + GENERATE_DATE_ARRAY, + GENERATE_TIMESTAMP_ARRAY, + FOR, + SYSTEMTIME, + AS, + OF, + WINDOW, + RESPECT, + IGNORE, + NULLS, + ) = map( + CaselessKeyword, + """ UNION, ALL, AND, INTERSECT, EXCEPT, COLLATE, ASC, DESC, ON, USING, NATURAL, INNER, CROSS, LEFT, RIGHT, OUTER, FULL, JOIN, AS, INDEXED, NOT, SELECT, DISTINCT, FROM, WHERE, GROUP, BY, HAVING, ORDER, BY, @@ -81,50 +224,100 @@ class BigQueryViewParser: VAR_SAMP, VARIANCE, TIMESTAMP_ADD, TIMESTAMP_SUB, GENERATE_ARRAY, GENERATE_DATE_ARRAY, GENERATE_TIMESTAMP_ARRAY, FOR, SYSTEMTIME, AS, OF, WINDOW, RESPECT, IGNORE, NULLS - ) = map(CaselessKeyword, - """ - UNION, ALL, AND, INTERSECT, EXCEPT, COLLATE, ASC, DESC, ON, USING, - NATURAL, INNER, CROSS, LEFT, RIGHT, OUTER, FULL, JOIN, AS, INDEXED, - NOT, SELECT, DISTINCT, FROM, WHERE, GROUP, BY, HAVING, ORDER, BY, - LIMIT, OFFSET, OR, CAST, ISNULL, NOTNULL, NULL, IS, BETWEEN, ELSE, - END, CASE, WHEN, THEN, EXISTS, COLLATE, IN, LIKE, GLOB, REGEXP, - MATCH, ESCAPE, CURRENT_TIME, CURRENT_DATE, CURRENT_TIMESTAMP, WITH, - EXTRACT, PARTITION, ROWS, RANGE, UNBOUNDED, PRECEDING, CURRENT, - ROW, FOLLOWING, OVER, INTERVAL, DATE_ADD, DATE_SUB, ADDDATE, - SUBDATE, REGEXP_EXTRACT, SPLIT, ORDINAL, FIRST_VALUE, LAST_VALUE, - NTH_VALUE, LEAD, LAG, PERCENTILE_CONT, PRECENTILE_DISC, RANK, - DENSE_RANK, PERCENT_RANK, CUME_DIST, NTILE, ROW_NUMBER, DATE, TIME, - DATETIME, TIMESTAMP, UNNEST, INT64, NUMERIC, FLOAT64, BOOL, BYTES, - GEOGRAPHY, ARRAY, STRUCT, SAFE_CAST, ANY_VALUE, ARRAY_AGG, - ARRAY_CONCAT_AGG, AVG, BIT_AND, BIT_OR, BIT_XOR, COUNT, COUNTIF, - LOGICAL_AND, LOGICAL_OR, MAX, MIN, STRING_AGG, SUM, CORR, - COVAR_POP, COVAR_SAMP, STDDEV_POP, STDDEV_SAMP, STDDEV, VAR_POP, - VAR_SAMP, VARIANCE, TIMESTAMP_ADD, TIMESTAMP_SUB, GENERATE_ARRAY, - GENERATE_DATE_ARRAY, GENERATE_TIMESTAMP_ARRAY, FOR, SYSTEMTIME, AS, - OF, WINDOW, RESPECT, IGNORE, NULLS - """.replace(",", "").split()) + """.replace( + ",", "" + ).split(), + ) - keyword_nonfunctions = MatchFirst(( - UNION, ALL, INTERSECT, EXCEPT, COLLATE, ASC, DESC, ON, USING, - NATURAL, INNER, CROSS, LEFT, RIGHT, OUTER, FULL, JOIN, AS, INDEXED, - NOT, SELECT, DISTINCT, FROM, WHERE, GROUP, BY, HAVING, ORDER, BY, - LIMIT, OFFSET, CAST, ISNULL, NOTNULL, NULL, IS, BETWEEN, ELSE, END, - CASE, WHEN, THEN, EXISTS, COLLATE, IN, LIKE, GLOB, REGEXP, MATCH, - STRUCT, WINDOW)) + keyword_nonfunctions = MatchFirst( + ( + UNION, + ALL, + INTERSECT, + EXCEPT, + COLLATE, + ASC, + DESC, + ON, + USING, + NATURAL, + INNER, + CROSS, + LEFT, + RIGHT, + OUTER, + FULL, + JOIN, + AS, + INDEXED, + NOT, + SELECT, + DISTINCT, + FROM, + WHERE, + GROUP, + BY, + HAVING, + ORDER, + BY, + LIMIT, + OFFSET, + CAST, + ISNULL, + NOTNULL, + NULL, + IS, + BETWEEN, + ELSE, + END, + CASE, + WHEN, + THEN, + EXISTS, + COLLATE, + IN, + LIKE, + GLOB, + REGEXP, + MATCH, + STRUCT, + WINDOW, + ) + ) - keyword = keyword_nonfunctions | MatchFirst(( - ESCAPE, CURRENT_TIME, CURRENT_DATE, CURRENT_TIMESTAMP, DATE_ADD, - DATE_SUB, ADDDATE, SUBDATE, INTERVAL, STRING_AGG, REGEXP_EXTRACT, - SPLIT, ORDINAL, UNNEST, SAFE_CAST, PARTITION, TIMESTAMP_ADD, - TIMESTAMP_SUB, ARRAY, GENERATE_ARRAY, GENERATE_DATE_ARRAY, - GENERATE_TIMESTAMP_ARRAY)) + keyword = keyword_nonfunctions | MatchFirst( + ( + ESCAPE, + CURRENT_TIME, + CURRENT_DATE, + CURRENT_TIMESTAMP, + DATE_ADD, + DATE_SUB, + ADDDATE, + SUBDATE, + INTERVAL, + STRING_AGG, + REGEXP_EXTRACT, + SPLIT, + ORDINAL, + UNNEST, + SAFE_CAST, + PARTITION, + TIMESTAMP_ADD, + TIMESTAMP_SUB, + ARRAY, + GENERATE_ARRAY, + GENERATE_DATE_ARRAY, + GENERATE_TIMESTAMP_ARRAY, + ) + ) - identifier_word = Word(alphas + '_@#', alphanums + '@$#_') + identifier_word = Word(alphas + "_@#", alphanums + "@$#_") identifier = ~keyword + identifier_word.copy() collation_name = identifier.copy() # NOTE: Column names can be keywords. Doc says they cannot, but in practice it seems to work. column_name = identifier_word.copy() - qualified_column_name = Combine(column_name + ('.' + column_name) * (0, 6)) + qualified_column_name = Combine(column_name + ("." + column_name) * (0, 6)) # NOTE: As with column names, column aliases can be keywords, e.g. functions like `current_time`. Other # keywords, e.g. `from` make parsing pretty difficult (e.g. "SELECT a from from b" is confusing.) column_alias = ~keyword_nonfunctions + column_name.copy() @@ -135,15 +328,14 @@ class BigQueryViewParser: parameter_name = identifier.copy() # NOTE: The expression in a CASE statement can be an integer. E.g. this is valid SQL: # select CASE 1 WHEN 1 THEN -1 ELSE -2 END from test_table - unquoted_case_identifier = ~keyword + Word(alphanums + '$_') + unquoted_case_identifier = ~keyword + Word(alphanums + "$_") quoted_case_identifier = ~keyword + ( - QuotedString('"') ^ Suppress('`') - + CharsNotIn('`') + Suppress('`') + QuotedString('"') ^ Suppress("`") + CharsNotIn("`") + Suppress("`") ) - case_identifier = (quoted_case_identifier | unquoted_case_identifier) + case_identifier = quoted_case_identifier | unquoted_case_identifier case_expr = ( - Optional(case_identifier + Suppress('.')) - + Optional(case_identifier + Suppress('.')) + Optional(case_identifier + Suppress(".")) + + Optional(case_identifier + Suppress(".")) + case_identifier ) @@ -152,82 +344,109 @@ class BigQueryViewParser: integer = Regex(r"[+-]?\d+") numeric_literal = Regex(r"[+-]?\d*\.?\d+([eE][+-]?\d+)?") - string_literal = ( - QuotedString("'") - | QuotedString('"') - | QuotedString('`') - ) + string_literal = QuotedString("'") | QuotedString('"') | QuotedString("`") regex_literal = "r" + string_literal blob_literal = Regex(r"[xX]'[0-9A-Fa-f]+'") - date_or_time_literal = ( - (DATE | TIME | DATETIME | TIMESTAMP) - + string_literal - ) + date_or_time_literal = (DATE | TIME | DATETIME | TIMESTAMP) + string_literal literal_value = ( - numeric_literal | string_literal | regex_literal - | blob_literal | date_or_time_literal | NULL + numeric_literal + | string_literal + | regex_literal + | blob_literal + | date_or_time_literal + | NULL | CURRENT_TIME + Optional(LPAR + Optional(string_literal) + RPAR) | CURRENT_DATE + Optional(LPAR + Optional(string_literal) + RPAR) - | CURRENT_TIMESTAMP - + Optional(LPAR + Optional(string_literal) + RPAR) - ) - bind_parameter = ( - Word("?", nums) - | Combine(oneOf(": @ $") + parameter_name) + | CURRENT_TIMESTAMP + Optional(LPAR + Optional(string_literal) + RPAR) ) - type_name = oneOf("""TEXT REAL INTEGER BLOB NULL TIMESTAMP STRING DATE + bind_parameter = Word("?", nums) | Combine(oneOf(": @ $") + parameter_name) + type_name = oneOf( + """TEXT REAL INTEGER BLOB NULL TIMESTAMP STRING DATE INT64 NUMERIC FLOAT64 BOOL BYTES DATETIME GEOGRAPHY TIME ARRAY - STRUCT""", caseless=True) - date_part = oneOf("""DAY DAY_HOUR DAY_MICROSECOND DAY_MINUTE DAY_SECOND + STRUCT""", + caseless=True, + ) + date_part = oneOf( + """DAY DAY_HOUR DAY_MICROSECOND DAY_MINUTE DAY_SECOND HOUR HOUR_MICROSECOND HOUR_MINUTE HOUR_SECOND MICROSECOND MINUTE MINUTE_MICROSECOND MINUTE_SECOND MONTH QUARTER SECOND - SECOND_MICROSECOND WEEK YEAR YEAR_MONTH""", caseless=True) + SECOND_MICROSECOND WEEK YEAR YEAR_MONTH""", + caseless=True, + ) datetime_operators = ( - DATE_ADD | DATE_SUB | ADDDATE | SUBDATE | TIMESTAMP_ADD - | TIMESTAMP_SUB + DATE_ADD | DATE_SUB | ADDDATE | SUBDATE | TIMESTAMP_ADD | TIMESTAMP_SUB ) grouping_term = expr.copy() ordering_term = Group( - expr('order_key') - + Optional(COLLATE + collation_name('collate')) - + Optional(ASC | DESC)('direction') + expr("order_key") + + Optional(COLLATE + collation_name("collate")) + + Optional(ASC | DESC)("direction") )("ordering_term") function_arg = expr.copy()("function_arg") function_args = Optional( "*" - | Optional(DISTINCT) + delimitedList(function_arg) + Optional((RESPECT | IGNORE) + NULLS) + | Optional(DISTINCT) + + delimitedList(function_arg) + + Optional((RESPECT | IGNORE) + NULLS) )("function_args") function_call = ( (function_name | keyword)("function_name") - + LPAR + Group(function_args)("function_args_group") + RPAR + + LPAR + + Group(function_args)("function_args_group") + + RPAR ) navigation_function_name = ( - FIRST_VALUE | LAST_VALUE | NTH_VALUE | LEAD | LAG - | PERCENTILE_CONT | PRECENTILE_DISC + FIRST_VALUE + | LAST_VALUE + | NTH_VALUE + | LEAD + | LAG + | PERCENTILE_CONT + | PRECENTILE_DISC ) aggregate_function_name = ( - ANY_VALUE | ARRAY_AGG | ARRAY_CONCAT_AGG | AVG | BIT_AND | BIT_OR - | BIT_XOR | COUNT | COUNTIF | LOGICAL_AND | LOGICAL_OR | MAX | MIN - | STRING_AGG | SUM + ANY_VALUE + | ARRAY_AGG + | ARRAY_CONCAT_AGG + | AVG + | BIT_AND + | BIT_OR + | BIT_XOR + | COUNT + | COUNTIF + | LOGICAL_AND + | LOGICAL_OR + | MAX + | MIN + | STRING_AGG + | SUM ) statistical_aggregate_function_name = ( - CORR | COVAR_POP | COVAR_SAMP | STDDEV_POP | STDDEV_SAMP | STDDEV - | VAR_POP | VAR_SAMP | VARIANCE + CORR + | COVAR_POP + | COVAR_SAMP + | STDDEV_POP + | STDDEV_SAMP + | STDDEV + | VAR_POP + | VAR_SAMP + | VARIANCE ) numbering_function_name = ( - RANK | DENSE_RANK | PERCENT_RANK | CUME_DIST | NTILE | ROW_NUMBER) + RANK | DENSE_RANK | PERCENT_RANK | CUME_DIST | NTILE | ROW_NUMBER + ) analytic_function_name = ( navigation_function_name | aggregate_function_name | statistical_aggregate_function_name | numbering_function_name )("analytic_function_name") - partition_expression_list = delimitedList( - grouping_term - )("partition_expression_list") + partition_expression_list = delimitedList(grouping_term)( + "partition_expression_list" + ) window_frame_boundary_start = ( UNBOUNDED + PRECEDING | numeric_literal + (PRECEDING | FOLLOWING) @@ -239,15 +458,9 @@ class BigQueryViewParser: | CURRENT + ROW ) window_frame_clause = (ROWS | RANGE) + ( - ( - (UNBOUNDED + PRECEDING) - | (numeric_literal + PRECEDING) - | (CURRENT + ROW) - ) | - ( - BETWEEN + window_frame_boundary_start - + AND + window_frame_boundary_end - )) + ((UNBOUNDED + PRECEDING) | (numeric_literal + PRECEDING) | (CURRENT + ROW)) + | (BETWEEN + window_frame_boundary_start + AND + window_frame_boundary_end) + ) window_name = identifier.copy()("window_name") window_specification = ( Optional(window_name) @@ -257,8 +470,11 @@ class BigQueryViewParser: ) analytic_function = ( analytic_function_name - + LPAR + function_args + RPAR - + OVER + (window_name | LPAR + Optional(window_specification) + RPAR) + + LPAR + + function_args + + RPAR + + OVER + + (window_name | LPAR + Optional(window_specification) + RPAR) )("analytic_function") string_agg_term = ( @@ -268,9 +484,7 @@ class BigQueryViewParser: + expr + Optional(COMMA + string_literal) + Optional( - ORDER + BY + expr - + Optional(ASC | DESC) - + Optional(LIMIT + integer) + ORDER + BY + expr + Optional(ASC | DESC) + Optional(LIMIT + integer) ) + RPAR )("string_agg") @@ -317,25 +531,22 @@ class BigQueryViewParser: CASE + Optional(case_expr.copy()) + case_clauses("case_clauses") - + Optional(case_else) + END + + Optional(case_else) + + END )("case") expr_term = ( (analytic_function)("analytic_function") | (CAST + LPAR + expr + AS + type_name + RPAR)("cast") | (SAFE_CAST + LPAR + expr + AS + type_name + RPAR)("safe_cast") - | ( - Optional(EXISTS) - + LPAR + ungrouped_select_stmt + RPAR - )("subselect") + | (Optional(EXISTS) + LPAR + ungrouped_select_stmt + RPAR)("subselect") | (literal_value)("literal") | (bind_parameter)("bind_parameter") | (EXTRACT + LPAR + expr + FROM + expr + RPAR)("extract") | case_stmt - | ( - datetime_operators - + LPAR + expr + COMMA + interval + RPAR - )("date_operation") + | (datetime_operators + LPAR + expr + COMMA + interval + RPAR)( + "date_operation" + ) | string_agg_term("string_agg_term") | array_literal("array_literal") | array_generator("array_generator") @@ -343,49 +554,52 @@ class BigQueryViewParser: | explicit_struct("explicit_struct") | function_call("function_call") | qualified_column_name("column") - ) + Optional( - LBRACKET - + (OFFSET | ORDINAL) - + LPAR + expr + RPAR - + RBRACKET - )("offset_ordinal") + ) + Optional(LBRACKET + (OFFSET | ORDINAL) + LPAR + expr + RPAR + RBRACKET)( + "offset_ordinal" + ) - struct_term = (LPAR + delimitedList(expr_term) + RPAR) + struct_term = LPAR + delimitedList(expr_term) + RPAR UNARY, BINARY, TERNARY = 1, 2, 3 - expr << infixNotation((expr_term | struct_term), [ - (oneOf('- + ~') | NOT, UNARY, opAssoc.RIGHT), - (ISNULL | NOTNULL | NOT + NULL, UNARY, opAssoc.LEFT), - ('||', BINARY, opAssoc.LEFT), - (oneOf('* / %'), BINARY, opAssoc.LEFT), - (oneOf('+ -'), BINARY, opAssoc.LEFT), - (oneOf('<< >> & |'), BINARY, opAssoc.LEFT), - (oneOf("= > < >= <= <> != !< !>"), BINARY, opAssoc.LEFT), - ( - IS + Optional(NOT) - | Optional(NOT) + IN - | Optional(NOT) + LIKE - | GLOB - | MATCH - | REGEXP, BINARY, opAssoc.LEFT - ), - ((BETWEEN, AND), TERNARY, opAssoc.LEFT), - ( - Optional(NOT) + IN - + LPAR - + Group(ungrouped_select_stmt | delimitedList(expr)) - + RPAR, - UNARY, - opAssoc.LEFT - ), - (AND, BINARY, opAssoc.LEFT), - (OR, BINARY, opAssoc.LEFT), - ]) + expr << infixNotation( + (expr_term | struct_term), + [ + (oneOf("- + ~") | NOT, UNARY, opAssoc.RIGHT), + (ISNULL | NOTNULL | NOT + NULL, UNARY, opAssoc.LEFT), + ("||", BINARY, opAssoc.LEFT), + (oneOf("* / %"), BINARY, opAssoc.LEFT), + (oneOf("+ -"), BINARY, opAssoc.LEFT), + (oneOf("<< >> & |"), BINARY, opAssoc.LEFT), + (oneOf("= > < >= <= <> != !< !>"), BINARY, opAssoc.LEFT), + ( + IS + Optional(NOT) + | Optional(NOT) + IN + | Optional(NOT) + LIKE + | GLOB + | MATCH + | REGEXP, + BINARY, + opAssoc.LEFT, + ), + ((BETWEEN, AND), TERNARY, opAssoc.LEFT), + ( + Optional(NOT) + + IN + + LPAR + + Group(ungrouped_select_stmt | delimitedList(expr)) + + RPAR, + UNARY, + opAssoc.LEFT, + ), + (AND, BINARY, opAssoc.LEFT), + (OR, BINARY, opAssoc.LEFT), + ], + ) quoted_expr = ( expr ^ Suppress('"') + expr + Suppress('"') ^ Suppress("'") + expr + Suppress("'") - ^ Suppress('`') + expr + Suppress('`') + ^ Suppress("`") + expr + Suppress("`") )("quoted_expr") compound_operator = ( @@ -399,10 +613,9 @@ class BigQueryViewParser: join_constraint = Group( Optional( ON + expr - | USING + LPAR - + Group(delimitedList(qualified_column_name)) - + RPAR - ))("join_constraint") + | USING + LPAR + Group(delimitedList(qualified_column_name)) + RPAR + ) + )("join_constraint") join_op = ( COMMA @@ -418,7 +631,8 @@ class BigQueryViewParser: | FULL + OUTER | OUTER | FULL - ) + JOIN + ) + + JOIN ) )("join_op") @@ -449,8 +663,8 @@ class BigQueryViewParser: standard_table_part = ~keyword + Word(alphanums + "_") standard_table_identifier = ( - Optional(standard_table_part("project") + Suppress('.')) - + Optional(standard_table_part("dataset") + Suppress('.')) + Optional(standard_table_part("project") + Suppress(".")) + + Optional(standard_table_part("dataset") + Suppress(".")) + standard_table_part("table") ).setParseAction(lambda t: record_table_identifier(t)) @@ -465,13 +679,13 @@ class BigQueryViewParser: | Suppress("`") + CharsNotIn("`.") + Suppress("`") ) quoted_table_parts_identifier = ( - Optional(quoted_project_part("project") + Suppress('.')) - + Optional(quoted_table_part("dataset") + Suppress('.')) + Optional(quoted_project_part("project") + Suppress(".")) + + Optional(quoted_table_part("dataset") + Suppress(".")) + quoted_table_part("table") ).setParseAction(lambda t: record_table_identifier(t)) def record_quoted_table_identifier(t): - identifier_list = t.asList()[0].split('.') + identifier_list = t.asList()[0].split(".") first = ".".join(identifier_list[0:-2]) or None second = identifier_list[-2] third = identifier_list[-1] @@ -486,19 +700,16 @@ class BigQueryViewParser: ).setParseAction(lambda t: record_quoted_table_identifier(t)) table_identifier = ( - standard_table_identifier | - quoted_table_parts_identifier | - quotable_table_parts_identifier + standard_table_identifier + | quoted_table_parts_identifier + | quotable_table_parts_identifier ) single_source = ( table_identifier + Optional(Optional(AS) + table_alias("table_alias*")) + Optional(FOR + SYSTEMTIME + AS + OF + string_literal) - + Optional( - INDEXED + BY + index_name("name") - | NOT + INDEXED - )("index") + + Optional(INDEXED + BY + index_name("name") | NOT + INDEXED)("index") | ( LPAR + ungrouped_select_stmt @@ -506,23 +717,18 @@ class BigQueryViewParser: + Optional(Optional(AS) + table_alias) ) | (LPAR + join_source + RPAR) - | (UNNEST + LPAR + expr + RPAR) - + Optional(Optional(AS) + column_alias) + | (UNNEST + LPAR + expr + RPAR) + Optional(Optional(AS) + column_alias) ) join_source << ( - Group( - single_source - + OneOrMore(join_op + single_source + join_constraint) - ) + Group(single_source + OneOrMore(join_op + single_source + join_constraint)) | single_source ) - over_partition = ( - PARTITION + BY - + delimitedList(partition_expression_list) - )("over_partition") - over_order = (ORDER + BY + delimitedList(ordering_term)) + over_partition = (PARTITION + BY + delimitedList(partition_expression_list))( + "over_partition" + ) + over_order = ORDER + BY + delimitedList(ordering_term) over_unsigned_value_specification = expr over_window_frame_preceding = ( UNBOUNDED + PRECEDING @@ -535,17 +741,15 @@ class BigQueryViewParser: | CURRENT + ROW ) over_window_frame_bound = ( - over_window_frame_preceding - | over_window_frame_following + over_window_frame_preceding | over_window_frame_following ) over_window_frame_between = ( BETWEEN + over_window_frame_bound + AND + over_window_frame_bound ) over_window_frame_extent = ( - over_window_frame_preceding - | over_window_frame_between + over_window_frame_preceding | over_window_frame_between ) - over_row_or_range = ((ROWS | RANGE) + over_window_frame_extent) + over_row_or_range = (ROWS | RANGE) + over_window_frame_extent over = ( OVER + LPAR @@ -555,18 +759,13 @@ class BigQueryViewParser: + RPAR )("over") - result_column = ( - Optional(table_name + ".") - + "*" - + Optional(EXCEPT + LPAR + delimitedList(column_name) + RPAR) - | Group( - quoted_expr - + Optional(over) - + Optional(Optional(AS) + column_alias) - ) - ) + result_column = Optional(table_name + ".") + "*" + Optional( + EXCEPT + LPAR + delimitedList(column_name) + RPAR + ) | Group(quoted_expr + Optional(over) + Optional(Optional(AS) + column_alias)) - window_select_clause = WINDOW + identifier + AS + LPAR + window_specification + RPAR + window_select_clause = ( + WINDOW + identifier + AS + LPAR + window_specification + RPAR + ) select_core = ( SELECT @@ -575,17 +774,13 @@ class BigQueryViewParser: + Optional(FROM + join_source("from*")) + Optional(WHERE + expr) + Optional( - GROUP + BY - + Group(delimitedList(grouping_term))("group_by_terms") + GROUP + BY + Group(delimitedList(grouping_term))("group_by_terms") ) + Optional(HAVING + expr("having_expr")) + Optional( - ORDER + BY - + Group(delimitedList(ordering_term))("order_by_terms") - ) - + Optional( - delimitedList(window_select_clause) + ORDER + BY + Group(delimitedList(ordering_term))("order_by_terms") ) + + Optional(delimitedList(window_select_clause)) ) grouped_select_core = select_core | (LPAR + select_core + RPAR) @@ -594,17 +789,15 @@ class BigQueryViewParser: + ZeroOrMore(compound_operator + grouped_select_core) + Optional( LIMIT - + ( - Group(expr + OFFSET + expr) - | Group(expr + COMMA + expr) - | expr - )("limit") + + (Group(expr + OFFSET + expr) | Group(expr + COMMA + expr) | expr)( + "limit" + ) ) )("select") select_stmt = ungrouped_select_stmt | (LPAR + ungrouped_select_stmt + RPAR) # define comment format, and ignore them - sql_comment = (oneOf("-- #") + restOfLine | cStyleComment) + sql_comment = oneOf("-- #") + restOfLine | cStyleComment select_stmt.ignore(sql_comment) def record_with_alias(t): @@ -616,16 +809,18 @@ class BigQueryViewParser: with_clause = Group( identifier.setParseAction(lambda t: record_with_alias(t)) + AS - + LPAR + (select_stmt | with_stmt) + RPAR + + LPAR + + (select_stmt | with_stmt) + + RPAR ) - with_core = (WITH + delimitedList(with_clause)) + with_core = WITH + delimitedList(with_clause) with_stmt << (with_core + ungrouped_select_stmt) with_stmt.ignore(sql_comment) - select_or_with = (select_stmt | with_stmt) + select_or_with = select_stmt | with_stmt select_or_with_parens = LPAR + select_or_with + RPAR - cls._parser = (select_or_with | select_or_with_parens) + cls._parser = select_or_with | select_or_with_parens return cls._parser TEST_CASES = [ @@ -633,86 +828,61 @@ class BigQueryViewParser: """ SELECT x FROM y.a, b """, - [ - (None, "y", "a"), - (None, None, "b",), - ] + [(None, "y", "a"), (None, None, "b",),], ], [ """ SELECT x FROM y.a JOIN b """, - [ - (None, "y", "a"), - (None, None, "b"), - ] + [(None, "y", "a"), (None, None, "b"),], ], [ """ select * from xyzzy where z > 100 """, - [ - (None, None, "xyzzy"), - ] + [(None, None, "xyzzy"),], ], [ """ select * from xyzzy where z > 100 order by zz """, - [ - (None, None, "xyzzy"), - ] + [(None, None, "xyzzy"),], ], [ """ select * from xyzzy """, - [ - (None, None, "xyzzy",), - ] + [(None, None, "xyzzy",),], ], [ """ select z.* from xyzzy """, - [ - (None, None, "xyzzy",), - ] + [(None, None, "xyzzy",),], ], [ """ select a, b from test_table where 1=1 and b='yes' """, - [ - (None, None, "test_table"), - ] + [(None, None, "test_table"),], ], [ """ select a, b from test_table where 1=1 and b in (select bb from foo) """, - [ - (None, None, "test_table"), - (None, None, "foo"), - ] + [(None, None, "test_table"), (None, None, "foo"),], ], [ """ select z.a, b from test_table where 1=1 and b in (select bb from foo) """, - [ - (None, None, "test_table"), - (None, None, "foo"), - ] + [(None, None, "test_table"), (None, None, "foo"),], ], [ """ select z.a, b from test_table where 1=1 and b in (select bb from foo) order by b,c desc,d """, - [ - (None, None, "test_table"), - (None, None, "foo"), - ] + [(None, None, "test_table"), (None, None, "foo"),], ], [ """ @@ -722,41 +892,31 @@ class BigQueryViewParser: (None, None, "test_table"), (None, None, "test2_table"), (None, None, "foo"), - ] + ], ], [ """ select a, db.table.b as BBB from db.table where 1=1 and BBB='yes' """, - [ - (None, "db", "table"), - ] + [(None, "db", "table"),], ], [ """ select a, db.table.b as BBB from test_table,db.table where 1=1 and BBB='yes' """, - [ - (None, None, "test_table"), - (None, "db", "table"), - ] + [(None, None, "test_table"), (None, "db", "table"),], ], [ """ select a, db.table.b as BBB from test_table,db.table where 1=1 and BBB='yes' limit 50 """, - [ - (None, None, "test_table"), - (None, "db", "table"), - ] + [(None, None, "test_table"), (None, "db", "table"),], ], [ """ select a, b from test_table where (1=1 or 2=3) and b='yes' group by zx having b=2 order by 1 """, - [ - (None, None, "test_table"), - ] + [(None, None, "test_table"),], ], [ """ @@ -771,44 +931,31 @@ class BigQueryViewParser: #yup, a comment group by zx having b=2 order by 1 """, - [ - (None, None, "test_table"), - ] + [(None, None, "test_table"),], ], [ """ SELECT COUNT(DISTINCT foo) FROM bar JOIN baz ON bar.baz_id = baz.id """, - [ - (None, None, "bar"), - (None, None, "baz"), - ] + [(None, None, "bar"), (None, None, "baz"),], ], [ """ SELECT COUNT(DISTINCT foo) FROM bar, baz WHERE bar.baz_id = baz.id """, - [ - (None, None, "bar"), - (None, None, "baz"), - ] + [(None, None, "bar"), (None, None, "baz"),], ], [ """ WITH one AS (SELECT id FROM foo) SELECT one.id """, - [ - (None, None, "foo"), - ] + [(None, None, "foo"),], ], [ """ WITH one AS (SELECT id FROM foo), two AS (select id FROM bar) SELECT one.id, two.id """, - [ - (None, None, "foo"), - (None, None, "bar"), - ] + [(None, None, "foo"), (None, None, "bar"),], ], [ """ @@ -818,9 +965,7 @@ class BigQueryViewParser: ROW_NUMBER() OVER (PARTITION BY x ORDER BY y) AS row_num FROM a """, - [ - (None, None, "a",), - ] + [(None, None, "a",),], ], [ """ @@ -828,9 +973,7 @@ class BigQueryViewParser: RANGE BETWEEN 2 PRECEDING AND 2 FOLLOWING ) AS count_x FROM T """, - [ - (None, None, "T",), - ] + [(None, None, "T",),], ], [ """ @@ -838,11 +981,8 @@ class BigQueryViewParser: RANK() OVER ( PARTITION BY department ORDER BY startdate ) AS rank FROM Employees """, - [ - (None, None, "Employees"), - ] + [(None, None, "Employees"),], ], - # A fragment from https://cloud.google.com/bigquery/docs/reference/standard-sql/navigation_functions [ """ @@ -859,10 +999,8 @@ class BigQueryViewParser: UNION ALL SELECT 'Carly Forte', TIMESTAMP '2016-10-18 3:08:58', 'F25-29' UNION ALL SELECT 'Lauren Reasoner', TIMESTAMP '2016-10-18 3:10:14', 'F30-34' """, - [ - ] + [], ], - # From https://cloud.google.com/bigquery/docs/reference/standard-sql/navigation_functions [ """ @@ -893,10 +1031,8 @@ class BigQueryViewParser: ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS fastest_time FROM finishers) """, - [ - ] + [], ], - # From https://cloud.google.com/bigquery/docs/reference/standard-sql/navigation_functions [ """ @@ -927,10 +1063,8 @@ class BigQueryViewParser: ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS slowest_time FROM finishers) """, - [ - ] + [], ], - # From https://cloud.google.com/bigquery/docs/reference/standard-sql/navigation_functions [ """ @@ -965,10 +1099,8 @@ class BigQueryViewParser: PARTITION BY division ORDER BY finish_time ASC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) """, - [ - ] + [], ], - # From https://cloud.google.com/bigquery/docs/reference/standard-sql/navigation_functions [ """ @@ -992,10 +1124,8 @@ class BigQueryViewParser: OVER (PARTITION BY division ORDER BY finish_time ASC) AS followed_by FROM finishers """, - [ - ] + [], ], - # From https://cloud.google.com/bigquery/docs/reference/standard-sql/navigation_functions [ """ @@ -1019,10 +1149,8 @@ class BigQueryViewParser: OVER (PARTITION BY division ORDER BY finish_time ASC) AS two_runners_back FROM finishers """, - [ - ] + [], ], - # From https://cloud.google.com/bigquery/docs/reference/standard-sql/navigation_functions [ """ @@ -1046,10 +1174,8 @@ class BigQueryViewParser: OVER (PARTITION BY division ORDER BY finish_time ASC) AS preceding_runner FROM finishers """, - [ - ] + [], ], - # From https://cloud.google.com/bigquery/docs/reference/standard-sql/navigation_functions [ """ @@ -1061,10 +1187,8 @@ class BigQueryViewParser: PERCENTILE_CONT(x, 1) OVER() AS max FROM UNNEST([0, 3, NULL, 1, 2]) AS x LIMIT 1 """, - [ - ] + [], ], - # From https://cloud.google.com/bigquery/docs/reference/standard-sql/navigation_functions [ """ @@ -1075,10 +1199,8 @@ class BigQueryViewParser: PERCENTILE_DISC(x, 1) OVER() AS max FROM UNNEST(['c', NULL, 'b', 'a']) AS x """, - [ - ] + [], ], - # From https://cloud.google.com/bigquery/docs/reference/standard-sql/timestamp_functions [ """ @@ -1086,10 +1208,8 @@ class BigQueryViewParser: TIMESTAMP "2008-12-25 15:30:00 UTC" as original, TIMESTAMP_ADD(TIMESTAMP "2008-12-25 15:30:00 UTC", INTERVAL 10 MINUTE) AS later """, - [ - ] + [], ], - # Previously hosted on https://cloud.google.com/bigquery/docs/reference/standard-sql/timestamp_functions, but # appears to no longer be there [ @@ -1178,7 +1298,7 @@ class BigQueryViewParser: [ (None, "date_hour_slots", "full_timestamps"), (None, "full_timestamps", "dt_range"), - ] + ], ], [ """ @@ -1198,17 +1318,14 @@ class BigQueryViewParser: FROM bar """, - [ - (None, None, "bar"), - ] + [(None, None, "bar"),], ], [ """ SELECT GENERATE_ARRAY(start, 5) AS example_array FROM UNNEST([3, 4, 5]) AS start """, - [ - ] + [], ], [ """ @@ -1221,8 +1338,7 @@ class BigQueryViewParser: SELECT GENERATE_DATE_ARRAY(date_start, date_end, INTERVAL 1 WEEK) AS date_range FROM StartsAndEnds """, - [ - ] + [], ], [ """ @@ -1241,15 +1357,13 @@ class BigQueryViewParser: TIMESTAMP '2016-10-05 23:59:00' AS start_timestamp, TIMESTAMP '2016-10-06 01:59:00' AS end_timestamp) """, - [ - ] + [], ], [ """ SELECT DATE_SUB(current_date("-08:00")), INTERVAL 2 DAY) """, - [ - ] + [], ], [ """ @@ -1257,9 +1371,7 @@ class BigQueryViewParser: case when (a) then b else c end FROM d """, - [ - (None, None, "d",), - ] + [(None, None, "d",),], ], [ """ @@ -1268,9 +1380,7 @@ class BigQueryViewParser: case when (f) then g else h end FROM i """, - [ - (None, None, "i",), - ] + [(None, None, "i",),], ], [ """ @@ -1278,9 +1388,7 @@ class BigQueryViewParser: case when j then k else l end FROM m """, - [ - (None, None, "m",), - ] + [(None, None, "m",),], ], [ """ @@ -1289,9 +1397,7 @@ class BigQueryViewParser: case when o then p else q end FROM r """, - [ - (None, None, "r",), - ] + [(None, None, "r",),], ], [ """ @@ -1299,9 +1405,7 @@ class BigQueryViewParser: case s when (t) then u else v end FROM w """, - [ - (None, None, "w",), - ] + [(None, None, "w",),], ], [ """ @@ -1310,18 +1414,15 @@ class BigQueryViewParser: case y when (z) then aa else ab end FROM ac """, - [ - (None, None, "ac",), - ] + [(None, None, "ac",),], ], [ """ SELECT case ad when ae then af else ag end FROM ah - """, [ - (None, None, "ah",), - ] + """, + [(None, None, "ah",),], ], [ """ @@ -1330,9 +1431,7 @@ class BigQueryViewParser: case aj when ak then al else am end FROM an """, - [ - (None, None, "an",), - ] + [(None, None, "an",),], ], [ """ @@ -1341,10 +1440,7 @@ class BigQueryViewParser: TWO AS (select a FROM b) SELECT y FROM onE JOIN TWo """, - [ - (None, None, "y",), - (None, None, "b",), - ] + [(None, None, "y",), (None, None, "b",),], ], [ """ @@ -1353,89 +1449,67 @@ class BigQueryViewParser: (SELECT b FROM oNE) FROM OnE """, - [ - (None, None, "oNE",), - (None, None, "OnE",), - ] + [(None, None, "oNE",), (None, None, "OnE",),], ], [ """ SELECT * FROM `a.b.c` """, - [ - ("a", "b", "c"), - ] + [("a", "b", "c"),], ], [ """ SELECT * FROM `b.c` """, - [ - (None, "b", "c"), - ] + [(None, "b", "c"),], ], [ """ SELECT * FROM `c` """, - [ - (None, None, "c"), - ] - ], [ + [(None, None, "c"),], + ], + [ """ SELECT * FROM a.b.c """, - [ - ("a", "b", "c"), - ] + [("a", "b", "c"),], ], [ """ SELECT * FROM "a"."b"."c" """, - [ - ("a", "b", "c"), - ] + [("a", "b", "c"),], ], [ """ SELECT * FROM 'a'.'b'.'c' """, - [ - ("a", "b", "c"), - ] + [("a", "b", "c"),], ], [ """ SELECT * FROM `a`.`b`.`c` """, - [ - ("a", "b", "c"), - ] + [("a", "b", "c"),], ], [ """ SELECT * FROM "a.b.c" """, - [ - ("a", "b", "c"), - ] + [("a", "b", "c"),], ], [ """ SELECT * FROM 'a.b.c' """, - [ - ("a", "b", "c"), - ] + [("a", "b", "c"),], ], [ """ SELECT * FROM `a.b.c` """, - [ - ("a", "b", "c"), - ] + [("a", "b", "c"),], ], [ """ @@ -1444,21 +1518,14 @@ class BigQueryViewParser: WHERE t1.a IN (SELECT t2.a FROM t2 ) FOR SYSTEM_TIME AS OF t1.timestamp_column) """, - [ - (None, None, "t1"), - (None, None, "t2"), - ] + [(None, None, "t1"), (None, None, "t2"),], ], [ """ WITH a AS (SELECT b FROM c) SELECT d FROM A JOIN e ON f = g JOIN E ON h = i """, - [ - (None, None, "c"), - (None, None, "e"), - (None, None, "E"), - ] + [(None, None, "c"), (None, None, "e"), (None, None, "E"),], ], [ """ @@ -1478,11 +1545,7 @@ class BigQueryViewParser: select g from h """, - [ - (None, None, 'd'), - (None, None, 'f'), - (None, None, 'h'), - ] + [(None, None, "d"), (None, None, "f"), (None, None, "h"),], ], [ """ @@ -1494,9 +1557,7 @@ class BigQueryViewParser: e AS DATE_ADD FROM x """, - [ - (None, None, 'x'), - ] + [(None, None, "x"),], ], [ """ @@ -1507,22 +1568,16 @@ class BigQueryViewParser: ) SELECT y FROM z """, - [ - (None, None, 'b'), - (None, None, 'z') - ] + [(None, None, "b"), (None, None, "z")], ], - [ """ SELECT DISTINCT FIRST_VALUE(x IGNORE NULLS) OVER (PARTITION BY y) FROM z """, - [ - (None, None, 'z') - ] - ] + [(None, None, "z")], + ], ] def test(self): @@ -1533,8 +1588,10 @@ class BigQueryViewParser: expected_tables_set = set(expected_tables) if expected_tables_set != found_tables: - raise Exception(f"Test {test_index} failed- expected {expected_tables_set} but got {found_tables}") + raise Exception( + f"Test {test_index} failed- expected {expected_tables_set} but got {found_tables}" + ) -if __name__ == '__main__': +if __name__ == "__main__": BigQueryViewParser().test() diff --git a/examples/booleansearchparser.py b/examples/booleansearchparser.py index d970e98..7ac502c 100644 --- a/examples/booleansearchparser.py +++ b/examples/booleansearchparser.py @@ -64,8 +64,8 @@ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. CONTRIBUTORS: @@ -81,46 +81,56 @@ TODO: - add more kinds of wildcards ('*' at the beginning and '*' inside a word)? """ -from pyparsing import Word, alphanums, Keyword, Group, Forward, Suppress, OneOrMore, oneOf +from pyparsing import ( + Word, + alphanums, + Keyword, + Group, + Forward, + Suppress, + OneOrMore, + oneOf, +) import re alphabet_ranges = [ ##CYRILIC: https://en.wikipedia.org/wiki/Cyrillic_(Unicode_block) - [int("0400",16), int("04FF",16)], + [int("0400", 16), int("04FF", 16)], ##THAI: https://en.wikipedia.org/wiki/Thai_(Unicode_block) - [int("0E00",16), int("0E7F",16)], + [int("0E00", 16), int("0E7F", 16)], ##ARABIC: https://en.wikipedia.org/wiki/Arabic_(Unicode_block) (Arabic (0600–06FF)+ Syriac (0700–074F)+ Arabic Supplement (0750–077F) ) - [int("0600",16), int("07FF",16)], + [int("0600", 16), int("07FF", 16)], ##CHINESE: https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) - [int("0400",16), int("09FF",16)], - #JAPANESE : https://en.wikipedia.org/wiki/Japanese_writing_system - [int("3040",16), int("30FF",16)], - #KOREAN : https://en.wikipedia.org/wiki/Hangul - [int("AC00",16), int("D7AF",16)], - [int("1100",16), int("11FF",16)], - [int("3130",16), int("318F",16)], - [int("3200",16), int("32FF",16)], - [int("A960",16), int("A97F",16)], - [int("D7B0",16), int("D7FF",16)], - [int("FF00",16), int("FFEF",16)], + [int("0400", 16), int("09FF", 16)], + # JAPANESE : https://en.wikipedia.org/wiki/Japanese_writing_system + [int("3040", 16), int("30FF", 16)], + # KOREAN : https://en.wikipedia.org/wiki/Hangul + [int("AC00", 16), int("D7AF", 16)], + [int("1100", 16), int("11FF", 16)], + [int("3130", 16), int("318F", 16)], + [int("3200", 16), int("32FF", 16)], + [int("A960", 16), int("A97F", 16)], + [int("D7B0", 16), int("D7FF", 16)], + [int("FF00", 16), int("FFEF", 16)], ] -class BooleanSearchParser: - def __init__(self,only_parse=False): + +class BooleanSearchParser: + def __init__(self, only_parse=False): self._methods = { - 'and': self.evaluateAnd, - 'or': self.evaluateOr, - 'not': self.evaluateNot, - 'parenthesis': self.evaluateParenthesis, - 'quotes': self.evaluateQuotes, - 'word': self.evaluateWord, - 'wordwildcardprefix': self.evaluateWordWildcardPrefix, - 'wordwildcardsufix': self.evaluateWordWildcardSufix, + "and": self.evaluateAnd, + "or": self.evaluateOr, + "not": self.evaluateNot, + "parenthesis": self.evaluateParenthesis, + "quotes": self.evaluateQuotes, + "word": self.evaluateWord, + "wordwildcardprefix": self.evaluateWordWildcardPrefix, + "wordwildcardsufix": self.evaluateWordWildcardSufix, } - self._parser = self.parser() - self.text = '' - self.words = [] + self._parser = self.parser() + self.text = "" + self.words = [] def parser(self): """ @@ -141,47 +151,54 @@ class BooleanSearchParser: alphabet = alphanums - #suport for non-western alphabets + # suport for non-western alphabets for r in alphabet_ranges: - alphabet += ''.join(chr(c) for c in range(*r) if not chr(c).isspace()) - - operatorWord = Group( - Word(alphabet + '*') - ).setResultsName('word*') + alphabet += "".join(chr(c) for c in range(*r) if not chr(c).isspace()) + operatorWord = Group(Word(alphabet + "*")).setResultsName("word*") operatorQuotesContent = Forward() - operatorQuotesContent << ( - (operatorWord + operatorQuotesContent) | operatorWord - ) + operatorQuotesContent << ((operatorWord + operatorQuotesContent) | operatorWord) - operatorQuotes = Group( - Suppress('"') + operatorQuotesContent + Suppress('"') - ).setResultsName("quotes") | operatorWord + operatorQuotes = ( + Group(Suppress('"') + operatorQuotesContent + Suppress('"')).setResultsName( + "quotes" + ) + | operatorWord + ) - operatorParenthesis = Group( - Suppress("(") + operatorOr + Suppress(")") - ).setResultsName("parenthesis") | operatorQuotes + operatorParenthesis = ( + Group(Suppress("(") + operatorOr + Suppress(")")).setResultsName( + "parenthesis" + ) + | operatorQuotes + ) operatorNot = Forward() - operatorNot << (Group( - Suppress(Keyword("not", caseless=True)) + operatorNot - ).setResultsName("not") | operatorParenthesis) + operatorNot << ( + Group(Suppress(Keyword("not", caseless=True)) + operatorNot).setResultsName( + "not" + ) + | operatorParenthesis + ) operatorAnd = Forward() operatorAnd << ( Group( operatorNot + Suppress(Keyword("and", caseless=True)) + operatorAnd - ).setResultsName("and")| - Group( + ).setResultsName("and") + | Group( operatorNot + OneOrMore(~oneOf("and or") + operatorAnd) - ).setResultsName("and") | - operatorNot + ).setResultsName("and") + | operatorNot ) - operatorOr << (Group( - operatorAnd + Suppress(Keyword("or", caseless=True)) + operatorOr - ).setResultsName("or") | operatorAnd) + operatorOr << ( + Group( + operatorAnd + Suppress(Keyword("or", caseless=True)) + operatorOr + ).setResultsName("or") + | operatorAnd + ) return operatorOr.parseString @@ -204,26 +221,26 @@ class BooleanSearchParser: function GetQuoted to only return the subset of ID's that contain the literal string. """ - #r = set() + # r = set() r = False search_terms = [] for item in argument: search_terms.append(item[0]) r = r and self.evaluate(item) - return self.GetQuotes(' '.join(search_terms), r) + return self.GetQuotes(" ".join(search_terms), r) def evaluateWord(self, argument): wildcard_count = argument[0].count("*") if wildcard_count > 0: if wildcard_count == 1 and argument[0].startswith("*"): - return self.GetWordWildcard(argument[0][1:], method = "endswith") + return self.GetWordWildcard(argument[0][1:], method="endswith") if wildcard_count == 1 and argument[0].endswith("*"): - return self.GetWordWildcard(argument[0][:-1], method = "startswith") + return self.GetWordWildcard(argument[0][:-1], method="startswith") else: - _regex = argument[0].replace("*",".+") + _regex = argument[0].replace("*", ".+") matched = False for w in self.words: - matched = bool(re.search(_regex,w)) + matched = bool(re.search(_regex, w)) if matched: break return matched @@ -231,10 +248,10 @@ class BooleanSearchParser: return self.GetWord(argument[0]) def evaluateWordWildcardPrefix(self, argument): - return self.GetWordWildcard(argument[0], method = "endswith") + return self.GetWordWildcard(argument[0], method="endswith") def evaluateWordWildcardSufix(self, argument): - return self.GetWordWildcard(argument[0], method = "startswith") + return self.GetWordWildcard(argument[0], method="startswith") def evaluate(self, argument): return self._methods[argument.getName()](argument) @@ -245,10 +262,10 @@ class BooleanSearchParser: def GetWord(self, word): return word in self.words - def GetWordWildcard(self, word, method = "startswith"): + def GetWordWildcard(self, word, method="startswith"): matched = False for w in self.words: - matched = getattr(w,method)(word) + matched = getattr(w, method)(word) if matched: break return matched @@ -265,24 +282,22 @@ class BooleanSearchParser: def GetQuotes(self, search_string, tmp_result): return search_string in self.text - def GetNot(self, not_set): return not not_set - - def _split_words(self,text): + def _split_words(self, text): words = [] """ >>> import string >>> string.punctuation '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~' """ - #it will keep @, # and - #usernames and hashtags can contain dots, so a double check is done - r = re.compile(r'[\s{}]+'.format(re.escape('!"$%&\'()*+,-/:;<=>?[\\]^`{|}~'))) - _words = r.split(text) + # it will keep @, # and + # usernames and hashtags can contain dots, so a double check is done + r = re.compile(r"[\s{}]+".format(re.escape("!\"$%&'()*+,-/:;<=>?[\\]^`{|}~"))) + _words = r.split(text) for _w in _words: - if '.' in _w and not _w.startswith("#") and not _w.startswith("@"): + if "." in _w and not _w.startswith("#") and not _w.startswith("@"): for __w in _w.split("."): words.append(__w) continue @@ -291,9 +306,9 @@ class BooleanSearchParser: return words - def match(self,text,expr): - self.text = text - self.words = self._split_words(text) + def match(self, text, expr): + self.text = text + self.words = self._split_words(text) return self.Parse(expr) @@ -305,72 +320,132 @@ class ParserTest(BooleanSearchParser): def Test(self): exprs = { - '0' : 'help', - '1' : 'help or hulp', - '2' : 'help and hulp', - '3' : 'help hulp', - '4' : 'help and hulp or hilp', - '5' : 'help or hulp and hilp', - '6' : 'help or hulp or hilp or halp', - '7' : '(help or hulp) and (hilp or halp)', - '8' : 'help and (hilp or halp)', - '9' : '(help and (hilp or halp)) or hulp', - '10': 'not help', - '11': 'not hulp and halp', - '12': 'not (help and halp)', - '13': '"help me please"', - '14': '"help me please" or hulp', - '15': '"help me please" or (hulp and halp)', - '16': 'help*', - '17': 'help or hulp*', - '18': 'help* and hulp', - '19': 'help and hulp* or hilp', - '20': 'help* or hulp or hilp or halp', - '21': '(help or hulp*) and (hilp* or halp)', - '22': 'help* and (hilp* or halp*)', - '23': '(help and (hilp* or halp)) or hulp*', - '24': 'not help* and halp', - '25': 'not (help* and helpe*)', - '26': '"help* me please"', - '27': '"help* me* please" or hulp*', - '28': '"help me please*" or (hulp and halp)', - '29': '"help me please" not (hulp and halp)', - '30': '"help me please" hulp', - '31': 'help and hilp and not holp', - '32': 'help hilp not holp', - '33': 'help hilp and not holp', - '34': '*lp and halp', - '35': '*신은 and 어떠세요', + "0": "help", + "1": "help or hulp", + "2": "help and hulp", + "3": "help hulp", + "4": "help and hulp or hilp", + "5": "help or hulp and hilp", + "6": "help or hulp or hilp or halp", + "7": "(help or hulp) and (hilp or halp)", + "8": "help and (hilp or halp)", + "9": "(help and (hilp or halp)) or hulp", + "10": "not help", + "11": "not hulp and halp", + "12": "not (help and halp)", + "13": '"help me please"', + "14": '"help me please" or hulp', + "15": '"help me please" or (hulp and halp)', + "16": "help*", + "17": "help or hulp*", + "18": "help* and hulp", + "19": "help and hulp* or hilp", + "20": "help* or hulp or hilp or halp", + "21": "(help or hulp*) and (hilp* or halp)", + "22": "help* and (hilp* or halp*)", + "23": "(help and (hilp* or halp)) or hulp*", + "24": "not help* and halp", + "25": "not (help* and helpe*)", + "26": '"help* me please"', + "27": '"help* me* please" or hulp*', + "28": '"help me please*" or (hulp and halp)', + "29": '"help me please" not (hulp and halp)', + "30": '"help me please" hulp', + "31": "help and hilp and not holp", + "32": "help hilp not holp", + "33": "help hilp and not holp", + "34": "*lp and halp", + "35": "*신은 and 어떠세요", } texts_matcheswith = { "halp thinks he needs help": [ - "25", "22", "20", "21", "11", "17", "16", "23", "34", "1", "0", "5", "7", "6", "9", "8" - ], - "he needs halp": [ - "24", "25", "20", "11", "10", "12", "34", "6" - ], - "help": [ - "25", "20", "12", "17", "16", "1", "0", "5", "6" + "25", + "22", + "20", + "21", + "11", + "17", + "16", + "23", + "34", + "1", + "0", + "5", + "7", + "6", + "9", + "8", ], + "he needs halp": ["24", "25", "20", "11", "10", "12", "34", "6"], + "help": ["25", "20", "12", "17", "16", "1", "0", "5", "6"], "help hilp": [ - "25", "22", "20", "32", "21", "12", "17", "16", "19", "31", "23", "1", "0", "5", "4", "7", "6", "9", "8", "33" + "25", + "22", + "20", + "32", + "21", + "12", + "17", + "16", + "19", + "31", + "23", + "1", + "0", + "5", + "4", + "7", + "6", + "9", + "8", + "33", ], "help me please hulp": [ - "30", "25", "27", "20", "13", "12", "15", "14", "17", "16", "19", "18", "23", "29", "1", "0", "3", "2", "5", "4", "6", "9" - ], - "helper": [ - "20", "10", "12", "16" + "30", + "25", + "27", + "20", + "13", + "12", + "15", + "14", + "17", + "16", + "19", + "18", + "23", + "29", + "1", + "0", + "3", + "2", + "5", + "4", + "6", + "9", ], + "helper": ["20", "10", "12", "16"], "hulp hilp": [ - "25", "27", "20", "21", "10", "12", "14", "17", "19", "23", "1", "5", "4", "7", "6", "9" + "25", + "27", + "20", + "21", + "10", + "12", + "14", + "17", + "19", + "23", + "1", + "5", + "4", + "7", + "6", + "9", ], - "nothing": [ - "25", "10", "12" - ], - "안녕하세요, 당신은 어떠세요?": [ - "10", "12", "25", "35" - ] + "nothing": ["25", "10", "12"], + "안녕하세요, 당신은 어떠세요?": ["10", "12", "25", "35"], } all_ok = True @@ -382,16 +457,17 @@ class ParserTest(BooleanSearchParser): test_passed = sorted(matches) == sorted(_matches) if not test_passed: - print('Failed', repr(text), 'expected', matches, 'matched', _matches) + print("Failed", repr(text), "expected", matches, "matched", _matches) all_ok = all_ok and test_passed return all_ok -if __name__=='__main__': + +if __name__ == "__main__": if ParserTest().Test(): - print ('All tests OK') + print("All tests OK") exit(0) else: - print ('One or more tests FAILED') + print("One or more tests FAILED") exit(1) diff --git a/examples/btpyparse.py b/examples/btpyparse.py index 39e5261..81ca4b0 100644 --- a/examples/btpyparse.py +++ b/examples/btpyparse.py @@ -10,22 +10,36 @@ Submitted by Matthew Brett, 2010 Simplified BSD license
"""
-from pyparsing import (Regex, Suppress, ZeroOrMore, Group, Optional, Forward,
- SkipTo, CaselessLiteral, Dict)
+from pyparsing import (
+ Regex,
+ Suppress,
+ ZeroOrMore,
+ Group,
+ Optional,
+ Forward,
+ SkipTo,
+ CaselessLiteral,
+ Dict,
+)
class Macro:
""" Class to encapsulate undefined macro references """
+
def __init__(self, name):
self.name = name
+
def __repr__(self):
return 'Macro("%s")' % self.name
+
def __eq__(self, other):
return self.name == other.name
# Character literals
-LCURLY,RCURLY,LPAREN,RPAREN,QUOTE,COMMA,AT,EQUALS,HASH = map(Suppress,'{}()",@=#')
+LCURLY, RCURLY, LPAREN, RPAREN, QUOTE, COMMA, AT, EQUALS, HASH = map(
+ Suppress, '{}()",@=#'
+)
def bracketed(expr):
@@ -48,31 +62,30 @@ quoted_item = Group(curly_string) | chars_no_quotecurly quoted_string = QUOTE + ZeroOrMore(quoted_item) + QUOTE
# Numbers can just be numbers. Only integers though.
-number = Regex('[0-9]+')
+number = Regex("[0-9]+")
# Basis characters (by exclusion) for variable / field names. The following
# list of characters is from the btparse documentation
-any_name = Regex('[^\\s"#%\'(),={}]+')
+any_name = Regex("[^\\s\"#%'(),={}]+")
# btparse says, and the test bibs show by experiment, that macro and field names
# cannot start with a digit. In fact entry type names cannot start with a digit
# either (see tests/bibs). Cite keys can start with a digit
-not_digname = Regex('[^\\d\\s"#%\'(),={}][^\\s"#%\'(),={}]*')
+not_digname = Regex("[^\\d\\s\"#%'(),={}][^\\s\"#%'(),={}]*")
# Comment comments out to end of line
-comment = (AT + CaselessLiteral('comment') +
- Regex(r"[\s{(].*").leaveWhitespace())
+comment = AT + CaselessLiteral("comment") + Regex(r"[\s{(].*").leaveWhitespace()
# The name types with their digiteyness
not_dig_lower = not_digname.copy().setParseAction(lambda t: t[0].lower())
macro_def = not_dig_lower.copy()
-macro_ref = not_dig_lower.copy().setParseAction(lambda t : Macro(t[0].lower()))
+macro_ref = not_dig_lower.copy().setParseAction(lambda t: Macro(t[0].lower()))
field_name = not_dig_lower.copy()
# Spaces in names mean they cannot clash with field names
-entry_type = not_dig_lower('entry_type')
-cite_key = any_name('cite_key')
+entry_type = not_dig_lower("entry_type")
+cite_key = any_name("cite_key")
# Number has to be before macro name
-string = (number | macro_ref | quoted_string | curly_string)
+string = number | macro_ref | quoted_string | curly_string
# There can be hash concatenation
field_value = string + ZeroOrMore(HASH + string)
@@ -80,25 +93,21 @@ field_def = Group(field_name + EQUALS + field_value) entry_contents = Dict(ZeroOrMore(field_def + COMMA) + Optional(field_def))
# Entry is surrounded either by parentheses or curlies
-entry = (AT + entry_type + bracketed(cite_key + COMMA + entry_contents))
+entry = AT + entry_type + bracketed(cite_key + COMMA + entry_contents)
# Preamble is a macro-like thing with no name
-preamble = AT + CaselessLiteral('preamble') + bracketed(field_value)
+preamble = AT + CaselessLiteral("preamble") + bracketed(field_value)
# Macros (aka strings)
macro_contents = macro_def + EQUALS + field_value
-macro = AT + CaselessLiteral('string') + bracketed(macro_contents)
+macro = AT + CaselessLiteral("string") + bracketed(macro_contents)
# Implicit comments
-icomment = SkipTo('@').setParseAction(lambda t : t.insert(0, 'icomment'))
+icomment = SkipTo("@").setParseAction(lambda t: t.insert(0, "icomment"))
# entries are last in the list (other than the fallback) because they have
# arbitrary start patterns that would match comments, preamble or macro
-definitions = Group(comment |
- preamble |
- macro |
- entry |
- icomment)
+definitions = Group(comment | preamble | macro | entry | icomment)
# Start symbol
bibfile = ZeroOrMore(definitions)
@@ -108,7 +117,7 @@ def parse_str(str): return bibfile.parseString(str)
-if __name__ == '__main__':
+if __name__ == "__main__":
# Run basic test
txt = """
Some introductory text
@@ -124,4 +133,4 @@ Some introductory text number = {2}
}
"""
- print('\n\n'.join(defn.dump() for defn in parse_str(txt)))
+ print("\n\n".join(defn.dump() for defn in parse_str(txt)))
diff --git a/examples/builtin_parse_action_demo.py b/examples/builtin_parse_action_demo.py index 3ec6af8..36b3a98 100644 --- a/examples/builtin_parse_action_demo.py +++ b/examples/builtin_parse_action_demo.py @@ -7,7 +7,7 @@ from pyparsing import *
-integer = Word(nums).setParseAction(lambda t : int(t[0]))
+integer = Word(nums).setParseAction(lambda t: int(t[0]))
# make an expression that will match a list of ints (which
# will be converted to actual ints by the parse action attached
@@ -23,7 +23,7 @@ for fn in (sum, max, min, len, sorted, reversed, list, tuple, set, any, all): fn_name = fn.__name__
if fn is reversed:
# reversed returns an iterator, we really want to show the list of items
- fn = lambda x : list(reversed(x))
+ fn = lambda x: list(reversed(x))
# show how each builtin works as a free-standing parse action
print(fn_name, nums.setParseAction(fn).parseString(test))
diff --git a/examples/cLibHeader.py b/examples/cLibHeader.py index 6bb1c25..10a0c77 100644 --- a/examples/cLibHeader.py +++ b/examples/cLibHeader.py @@ -6,7 +6,17 @@ # Copyright, 2012 - Paul McGuire
#
-from pyparsing import Word, alphas, alphanums, Combine, oneOf, Optional, delimitedList, Group, Keyword
+from pyparsing import (
+ Word,
+ alphas,
+ alphanums,
+ Combine,
+ oneOf,
+ Optional,
+ delimitedList,
+ Group,
+ Keyword,
+)
testdata = """
int func1(float *vec, int len, double arg1);
@@ -14,12 +24,12 @@ testdata = """ """
ident = Word(alphas, alphanums + "_")
-vartype = Combine( oneOf("float double int char") + Optional(Word("*")), adjacent = False)
+vartype = Combine(oneOf("float double int char") + Optional(Word("*")), adjacent=False)
arglist = delimitedList(Group(vartype("type") + ident("name")))
functionCall = Keyword("int") + ident("name") + "(" + arglist("args") + ")" + ";"
-for fn,s,e in functionCall.scanString(testdata):
+for fn, s, e in functionCall.scanString(testdata):
print(fn.name)
for a in fn.args:
print(" - %(name)s (%(type)s)" % a)
diff --git a/examples/chemicalFormulas.py b/examples/chemicalFormulas.py index 1b41871..f4725ed 100644 --- a/examples/chemicalFormulas.py +++ b/examples/chemicalFormulas.py @@ -7,12 +7,12 @@ import pyparsing as pp
atomicWeight = {
- "O" : 15.9994,
- "H" : 1.00794,
- "Na" : 22.9897,
- "Cl" : 35.4527,
- "C" : 12.0107,
- }
+ "O": 15.9994,
+ "H": 1.00794,
+ "Na": 22.9897,
+ "Cl": 35.4527,
+ "C": 12.0107,
+}
digits = "0123456789"
@@ -26,58 +26,78 @@ element = pp.Word(pp.alphas.upper(), pp.alphas.lower(), max=2) elementRef = pp.Group(element + pp.Optional(pp.Word(digits), default="1"))
formula = elementRef[...]
-fn = lambda elemList : sum(atomicWeight[elem]*int(qty) for elem,qty in elemList)
-formula.runTests("""\
+fn = lambda elemList: sum(atomicWeight[elem] * int(qty) for elem, qty in elemList)
+formula.runTests(
+ """\
H2O
C6H5OH
NaCl
""",
- fullDump=False, postParse=lambda _, tokens: "Molecular weight: {}".format(fn(tokens)))
+ fullDump=False,
+ postParse=lambda _, tokens: "Molecular weight: {}".format(fn(tokens)),
+)
print()
# Version 2 - access parsed items by results name
-elementRef = pp.Group(element("symbol") + pp.Optional(pp.Word(digits), default="1")("qty"))
+elementRef = pp.Group(
+ element("symbol") + pp.Optional(pp.Word(digits), default="1")("qty")
+)
formula = elementRef[...]
-fn = lambda elemList : sum(atomicWeight[elem.symbol]*int(elem.qty) for elem in elemList)
-formula.runTests("""\
+fn = lambda elemList: sum(
+ atomicWeight[elem.symbol] * int(elem.qty) for elem in elemList
+)
+formula.runTests(
+ """\
H2O
C6H5OH
NaCl
""",
- fullDump=False, postParse=lambda _, tokens: "Molecular weight: {}".format(fn(tokens)))
+ fullDump=False,
+ postParse=lambda _, tokens: "Molecular weight: {}".format(fn(tokens)),
+)
print()
# Version 3 - convert integers during parsing process
-integer = pp.Word(digits).setParseAction(lambda t:int(t[0]))
+integer = pp.Word(digits).setParseAction(lambda t: int(t[0]))
elementRef = pp.Group(element("symbol") + pp.Optional(integer, default=1)("qty"))
formula = elementRef[...]
-fn = lambda elemList : sum(atomicWeight[elem.symbol]*elem.qty for elem in elemList)
-formula.runTests("""\
+fn = lambda elemList: sum(atomicWeight[elem.symbol] * elem.qty for elem in elemList)
+formula.runTests(
+ """\
H2O
C6H5OH
NaCl
""",
- fullDump=False, postParse=lambda _, tokens: "Molecular weight: {}".format(fn(tokens)))
+ fullDump=False,
+ postParse=lambda _, tokens: "Molecular weight: {}".format(fn(tokens)),
+)
print()
# Version 4 - parse and convert integers as subscript digits
subscript_digits = "₀₁₂₃₄₅₆₇₈₉"
subscript_int_map = {e[1]: e[0] for e in enumerate(subscript_digits)}
+
+
def cvt_subscript_int(s):
ret = 0
for c in s[0]:
- ret = ret*10 + subscript_int_map[c]
+ ret = ret * 10 + subscript_int_map[c]
return ret
+
+
subscript_int = pp.Word(subscript_digits).addParseAction(cvt_subscript_int)
elementRef = pp.Group(element("symbol") + pp.Optional(subscript_int, default=1)("qty"))
formula = elementRef[...]
-formula.runTests("""\
+formula.runTests(
+ """\
H₂O
C₆H₅OH
NaCl
""",
- fullDump=False, postParse=lambda _, tokens: "Molecular weight: {}".format(fn(tokens)))
+ fullDump=False,
+ postParse=lambda _, tokens: "Molecular weight: {}".format(fn(tokens)),
+)
print()
diff --git a/examples/commasep.py b/examples/commasep.py index 067647d..c3557b6 100644 --- a/examples/commasep.py +++ b/examples/commasep.py @@ -10,6 +10,7 @@ # import pyparsing as pp + ppc = pp.pyparsing_common testData = [ @@ -19,6 +20,6 @@ testData = [ "John Doe, 123 Main St., Cleveland, Ohio", "Jane Doe, 456 St. James St., Los Angeles , California ", "", - ] +] ppc.comma_separated_list.runTests(testData) diff --git a/examples/configParse.py b/examples/configParse.py index db7b6c7..02727e9 100644 --- a/examples/configParse.py +++ b/examples/configParse.py @@ -6,13 +6,24 @@ # Copyright (c) 2003, Paul McGuire
#
-from pyparsing import \
- Literal, Word, ZeroOrMore, Group, Dict, Optional, \
- printables, ParseException, restOfLine, empty
+from pyparsing import (
+ Literal,
+ Word,
+ ZeroOrMore,
+ Group,
+ Dict,
+ Optional,
+ printables,
+ ParseException,
+ restOfLine,
+ empty,
+)
import pprint
inibnf = None
+
+
def inifile_BNF():
global inibnf
@@ -22,50 +33,53 @@ def inifile_BNF(): lbrack = Literal("[").suppress()
rbrack = Literal("]").suppress()
equals = Literal("=").suppress()
- semi = Literal(";")
+ semi = Literal(";")
- comment = semi + Optional( restOfLine )
+ comment = semi + Optional(restOfLine)
- nonrbrack = "".join( [ c for c in printables if c != "]" ] ) + " \t"
- nonequals = "".join( [ c for c in printables if c != "=" ] ) + " \t"
+ nonrbrack = "".join([c for c in printables if c != "]"]) + " \t"
+ nonequals = "".join([c for c in printables if c != "="]) + " \t"
- sectionDef = lbrack + Word( nonrbrack ) + rbrack
- keyDef = ~lbrack + Word( nonequals ) + equals + empty + restOfLine
+ sectionDef = lbrack + Word(nonrbrack) + rbrack
+ keyDef = ~lbrack + Word(nonequals) + equals + empty + restOfLine
# strip any leading or trailing blanks from key
def stripKey(tokens):
tokens[0] = tokens[0].strip()
+
keyDef.setParseAction(stripKey)
# using Dict will allow retrieval of named data fields as attributes of the parsed results
- inibnf = Dict( ZeroOrMore( Group( sectionDef + Dict( ZeroOrMore( Group( keyDef ) ) ) ) ) )
+ inibnf = Dict(ZeroOrMore(Group(sectionDef + Dict(ZeroOrMore(Group(keyDef))))))
- inibnf.ignore( comment )
+ inibnf.ignore(comment)
return inibnf
pp = pprint.PrettyPrinter(2)
-def test( strng ):
+
+def test(strng):
print(strng)
try:
iniFile = open(strng)
- iniData = "".join( iniFile.readlines() )
+ iniData = "".join(iniFile.readlines())
bnf = inifile_BNF()
- tokens = bnf.parseString( iniData )
- pp.pprint( tokens.asList() )
+ tokens = bnf.parseString(iniData)
+ pp.pprint(tokens.asList())
except ParseException as err:
print(err.line)
- print(" "*(err.column-1) + "^")
+ print(" " * (err.column - 1) + "^")
print(err)
iniFile.close()
print()
return tokens
+
if __name__ == "__main__":
- ini = test("setup.ini")
- print("ini['Startup']['modemid'] =", ini['Startup']['modemid'])
- print("ini.Startup =", ini.Startup)
- print("ini.Startup.modemid =", ini.Startup.modemid)
+ ini = test("setup.ini")
+ print("ini['Startup']['modemid'] =", ini["Startup"]["modemid"])
+ print("ini.Startup =", ini.Startup)
+ print("ini.Startup.modemid =", ini.Startup.modemid)
diff --git a/examples/cpp_enum_parser.py b/examples/cpp_enum_parser.py index ca2c04b..26dde7c 100644 --- a/examples/cpp_enum_parser.py +++ b/examples/cpp_enum_parser.py @@ -10,8 +10,9 @@ #
from pyparsing import *
+
# sample string with enums and other stuff
-sample = '''
+sample = """
stuff before
enum hello {
Zero,
@@ -31,22 +32,22 @@ sample = ''' zeta = 50
};
at the end
- '''
+ """
# syntax we don't want to see in the final parse tree
-LBRACE,RBRACE,EQ,COMMA = map(Suppress,"{}=,")
-_enum = Suppress('enum')
-identifier = Word(alphas,alphanums+'_')
+LBRACE, RBRACE, EQ, COMMA = map(Suppress, "{}=,")
+_enum = Suppress("enum")
+identifier = Word(alphas, alphanums + "_")
integer = Word(nums)
-enumValue = Group(identifier('name') + Optional(EQ + integer('value')))
+enumValue = Group(identifier("name") + Optional(EQ + integer("value")))
enumList = Group(enumValue + ZeroOrMore(COMMA + enumValue))
-enum = _enum + identifier('enum') + LBRACE + enumList('names') + RBRACE
+enum = _enum + identifier("enum") + LBRACE + enumList("names") + RBRACE
# find instances of enums ignoring other syntax
-for item,start,stop in enum.scanString(sample):
+for item, start, stop in enum.scanString(sample):
id = 0
for entry in item.names:
- if entry.value != '':
+ if entry.value != "":
id = int(entry.value)
- print('%s_%s = %d' % (item.enum.upper(),entry.name.upper(),id))
+ print("%s_%s = %d" % (item.enum.upper(), entry.name.upper(), id))
id += 1
diff --git a/examples/datetimeParseActions.py b/examples/datetimeParseActions.py index e5ae2b9..f7c4fc9 100644 --- a/examples/datetimeParseActions.py +++ b/examples/datetimeParseActions.py @@ -12,32 +12,44 @@ from pyparsing import pyparsing_common as ppc # define an integer string, and a parse action to convert it
# to an integer at parse time
integer = pp.Word(pp.nums).setName("integer")
+
+
def convertToInt(tokens):
# no need to test for validity - we can't get here
# unless tokens[0] contains all numeric digits
return int(tokens[0])
+
+
integer.setParseAction(convertToInt)
# or can be written as one line as
-#integer = Word(nums).setParseAction(lambda t: int(t[0]))
+# integer = Word(nums).setParseAction(lambda t: int(t[0]))
# define a pattern for a year/month/day date
-date_expr = integer('year') + '/' + integer('month') + '/' + integer('day')
+date_expr = integer("year") + "/" + integer("month") + "/" + integer("day")
date_expr.ignore(pp.pythonStyleComment)
-def convertToDatetime(s,loc,tokens):
+
+def convertToDatetime(s, loc, tokens):
try:
# note that the year, month, and day fields were already
# converted to ints from strings by the parse action defined
# on the integer expression above
return datetime(tokens.year, tokens.month, tokens.day).date()
except Exception as ve:
- errmsg = "'%s/%s/%s' is not a valid date, %s" % \
- (tokens.year, tokens.month, tokens.day, ve)
+ errmsg = "'%s/%s/%s' is not a valid date, %s" % (
+ tokens.year,
+ tokens.month,
+ tokens.day,
+ ve,
+ )
raise pp.ParseException(s, loc, errmsg)
+
+
date_expr.setParseAction(convertToDatetime)
-date_expr.runTests("""\
+date_expr.runTests(
+ """\
2000/1/1
# invalid month
@@ -48,14 +60,16 @@ date_expr.runTests("""\ # but 2000 was
2000/2/29
- """)
+ """
+)
# if dates conform to ISO8601, use definitions in pyparsing_common
date_expr = ppc.iso8601_date.setParseAction(ppc.convertToDate())
date_expr.ignore(pp.pythonStyleComment)
-date_expr.runTests("""\
+date_expr.runTests(
+ """\
2000-01-01
# invalid month
@@ -66,4 +80,5 @@ date_expr.runTests("""\ # but 2000 was
2000-02-29
- """)
+ """
+)
diff --git a/examples/decaf_parser.py b/examples/decaf_parser.py index e6b1abb..be3a1e9 100644 --- a/examples/decaf_parser.py +++ b/examples/decaf_parser.py @@ -12,46 +12,75 @@ """ Program ::= Decl+ Decl ::= VariableDecl | FunctionDecl | ClassDecl | InterfaceDecl - VariableDecl ::= Variable ; - Variable ::= Type ident - Type ::= int | double | bool | string | ident | Type [] - FunctionDecl ::= Type ident ( Formals ) StmtBlock | void ident ( Formals ) StmtBlock - Formals ::= Variable+, | e - ClassDecl ::= class ident <extends ident> <implements ident + ,> { Field* } - Field ::= VariableDecl | FunctionDecl - InterfaceDecl ::= interface ident { Prototype* } - Prototype ::= Type ident ( Formals ) ; | void ident ( Formals ) ; - StmtBlock ::= { VariableDecl* Stmt* } - Stmt ::= <Expr> ; | IfStmt | WhileStmt | ForStmt | BreakStmt | ReturnStmt | PrintStmt | StmtBlock - IfStmt ::= if ( Expr ) Stmt <else Stmt> - WhileStmt ::= while ( Expr ) Stmt - ForStmt ::= for ( <Expr> ; Expr ; <Expr> ) Stmt - ReturnStmt ::= return <Expr> ; - BreakStmt ::= break ; - PrintStmt ::= Print ( Expr+, ) ; + VariableDecl ::= Variable ; + Variable ::= Type ident + Type ::= int | double | bool | string | ident | Type [] + FunctionDecl ::= Type ident ( Formals ) StmtBlock | void ident ( Formals ) StmtBlock + Formals ::= Variable+, | e + ClassDecl ::= class ident <extends ident> <implements ident + ,> { Field* } + Field ::= VariableDecl | FunctionDecl + InterfaceDecl ::= interface ident { Prototype* } + Prototype ::= Type ident ( Formals ) ; | void ident ( Formals ) ; + StmtBlock ::= { VariableDecl* Stmt* } + Stmt ::= <Expr> ; | IfStmt | WhileStmt | ForStmt | BreakStmt | ReturnStmt | PrintStmt | StmtBlock + IfStmt ::= if ( Expr ) Stmt <else Stmt> + WhileStmt ::= while ( Expr ) Stmt + ForStmt ::= for ( <Expr> ; Expr ; <Expr> ) Stmt + ReturnStmt ::= return <Expr> ; + BreakStmt ::= break ; + PrintStmt ::= Print ( Expr+, ) ; Expr ::= LValue = Expr | Constant | LValue | this | Call - | ( Expr ) - | Expr + Expr | Expr - Expr | Expr * Expr | Expr / Expr | Expr % Expr | - Expr - | Expr < Expr | Expr <= Expr | Expr > Expr | Expr >= Expr | Expr == Expr | Expr != Expr - | Expr && Expr | Expr || Expr | ! Expr - | ReadInteger ( ) | ReadLine ( ) | new ident | NewArray ( Expr , Typev) - LValue ::= ident | Expr . ident | Expr [ Expr ] - Call ::= ident ( Actuals ) | Expr . ident ( Actuals ) - Actuals ::= Expr+, | e + | ( Expr ) + | Expr + Expr | Expr - Expr | Expr * Expr | Expr / Expr | Expr % Expr | - Expr + | Expr < Expr | Expr <= Expr | Expr > Expr | Expr >= Expr | Expr == Expr | Expr != Expr + | Expr && Expr | Expr || Expr | ! Expr + | ReadInteger ( ) | ReadLine ( ) | new ident | NewArray ( Expr , Typev) + LValue ::= ident | Expr . ident | Expr [ Expr ] + Call ::= ident ( Actuals ) | Expr . ident ( Actuals ) + Actuals ::= Expr+, | e Constant ::= intConstant | doubleConstant | boolConstant | stringConstant | null """ import pyparsing as pp from pyparsing import pyparsing_common as ppc + pp.ParserElement.enablePackrat() # keywords -keywords = (VOID, INT, DOUBLE, BOOL, STRING, CLASS, INTERFACE, NULL, THIS, EXTENDS, IMPLEMENTS, FOR, WHILE, - IF, ELSE, RETURN, BREAK, NEW, NEWARRAY, PRINT, READINTEGER, READLINE, TRUE, FALSE) = map(pp.Keyword, - """void int double bool string class interface null this extends implements or while - if else return break new NewArray Print ReadInteger ReadLine true false""".split()) +keywords = ( + VOID, + INT, + DOUBLE, + BOOL, + STRING, + CLASS, + INTERFACE, + NULL, + THIS, + EXTENDS, + IMPLEMENTS, + FOR, + WHILE, + IF, + ELSE, + RETURN, + BREAK, + NEW, + NEWARRAY, + PRINT, + READINTEGER, + READLINE, + TRUE, + FALSE, +) = map( + pp.Keyword, + """void int double bool string class interface null this extends implements or while + if else return break new NewArray Print ReadInteger ReadLine true false""".split(), +) keywords = pp.MatchFirst(list(keywords)) -LPAR, RPAR, LBRACE, RBRACE, LBRACK, RBRACK, DOT, EQ, COMMA, SEMI = map(pp.Suppress, "(){}[].=,;") +LPAR, RPAR, LBRACE, RBRACE, LBRACK, RBRACK, DOT, EQ, COMMA, SEMI = map( + pp.Suppress, "(){}[].=,;" +) hexConstant = pp.Regex(r"0[xX][0-9a-fA-F]+").addParseAction(lambda t: int(t[0][2:], 16)) intConstant = hexConstant | ppc.integer doubleConstant = ppc.real @@ -59,7 +88,7 @@ boolConstant = TRUE | FALSE stringConstant = pp.dblQuotedString null = NULL constant = doubleConstant | boolConstant | intConstant | stringConstant | null -ident = ~keywords + pp.Word(pp.alphas, pp.alphanums+'_') +ident = ~keywords + pp.Word(pp.alphas, pp.alphanums + "_") type_ = pp.Group((INT | DOUBLE | BOOL | STRING | ident) + pp.ZeroOrMore("[]")) variable = type_ + ident @@ -68,86 +97,142 @@ variable_decl = variable + SEMI expr = pp.Forward() expr_parens = pp.Group(LPAR + expr + RPAR) actuals = pp.Optional(pp.delimitedList(expr)) -call = pp.Group(ident("call_ident") + LPAR + actuals("call_args") + RPAR - | (expr_parens + pp.ZeroOrMore(DOT + ident))("call_ident_expr") + LPAR + actuals("call_args") + RPAR) -lvalue = ((ident | expr_parens) - + pp.ZeroOrMore(DOT + (ident | expr_parens)) - + pp.ZeroOrMore(LBRACK + expr + RBRACK)) +call = pp.Group( + ident("call_ident") + LPAR + actuals("call_args") + RPAR + | (expr_parens + pp.ZeroOrMore(DOT + ident))("call_ident_expr") + + LPAR + + actuals("call_args") + + RPAR +) +lvalue = ( + (ident | expr_parens) + + pp.ZeroOrMore(DOT + (ident | expr_parens)) + + pp.ZeroOrMore(LBRACK + expr + RBRACK) +) assignment = pp.Group(lvalue("lhs") + EQ + expr("rhs")) read_integer = pp.Group(READINTEGER + LPAR + RPAR) read_line = pp.Group(READLINE + LPAR + RPAR) new_statement = pp.Group(NEW + ident) new_array = pp.Group(NEWARRAY + LPAR + expr + COMMA + type_ + RPAR) rvalue = constant | call | read_integer | read_line | new_statement | new_array | ident -arith_expr = pp.infixNotation(rvalue, +arith_expr = pp.infixNotation( + rvalue, [ - ('-', 1, pp.opAssoc.RIGHT,), - (pp.oneOf("* / %"), 2, pp.opAssoc.LEFT,), - (pp.oneOf("+ -"), 2, pp.opAssoc.LEFT,), - ]) -comparison_expr = pp.infixNotation(arith_expr, + ("-", 1, pp.opAssoc.RIGHT,), + (pp.oneOf("* / %"), 2, pp.opAssoc.LEFT,), + (pp.oneOf("+ -"), 2, pp.opAssoc.LEFT,), + ], +) +comparison_expr = pp.infixNotation( + arith_expr, [ - ('!', 1, pp.opAssoc.RIGHT,), - (pp.oneOf("< > <= >="), 2, pp.opAssoc.LEFT,), - (pp.oneOf("== !="), 2, pp.opAssoc.LEFT,), - (pp.oneOf("&&"), 2, pp.opAssoc.LEFT,), - (pp.oneOf("||"), 2, pp.opAssoc.LEFT,), - ]) -expr <<= (assignment - | call - | THIS - | comparison_expr - | arith_expr - | lvalue - | constant - | read_integer - | read_line - | new_statement - | new_array - ) + ("!", 1, pp.opAssoc.RIGHT,), + (pp.oneOf("< > <= >="), 2, pp.opAssoc.LEFT,), + (pp.oneOf("== !="), 2, pp.opAssoc.LEFT,), + (pp.oneOf("&&"), 2, pp.opAssoc.LEFT,), + (pp.oneOf("||"), 2, pp.opAssoc.LEFT,), + ], +) +expr <<= ( + assignment + | call + | THIS + | comparison_expr + | arith_expr + | lvalue + | constant + | read_integer + | read_line + | new_statement + | new_array +) stmt = pp.Forward() -print_stmt = pp.Group(PRINT("statement") + LPAR + pp.Group(pp.Optional(pp.delimitedList(expr)))("args") + RPAR + SEMI) +print_stmt = pp.Group( + PRINT("statement") + + LPAR + + pp.Group(pp.Optional(pp.delimitedList(expr)))("args") + + RPAR + + SEMI +) break_stmt = pp.Group(BREAK("statement") + SEMI) return_stmt = pp.Group(RETURN("statement") + expr + SEMI) -for_stmt = pp.Group(FOR("statement") + LPAR + pp.Optional(expr) + SEMI + expr + SEMI + pp.Optional(expr) + RPAR + stmt) +for_stmt = pp.Group( + FOR("statement") + + LPAR + + pp.Optional(expr) + + SEMI + + expr + + SEMI + + pp.Optional(expr) + + RPAR + + stmt +) while_stmt = pp.Group(WHILE("statement") + LPAR + expr + RPAR + stmt) -if_stmt = pp.Group(IF("statement") - + LPAR + pp.Group(expr)("condition") + RPAR - + pp.Group(stmt)("then_statement") - + pp.Group(pp.Optional(ELSE + stmt))("else_statement")) -stmt_block = pp.Group(LBRACE + pp.ZeroOrMore(variable_decl) + pp.ZeroOrMore(stmt) + RBRACE) -stmt <<= (if_stmt - | while_stmt - | for_stmt - | break_stmt - | return_stmt - | print_stmt - | stmt_block - | pp.Group(expr + SEMI) - ) +if_stmt = pp.Group( + IF("statement") + + LPAR + + pp.Group(expr)("condition") + + RPAR + + pp.Group(stmt)("then_statement") + + pp.Group(pp.Optional(ELSE + stmt))("else_statement") +) +stmt_block = pp.Group( + LBRACE + pp.ZeroOrMore(variable_decl) + pp.ZeroOrMore(stmt) + RBRACE +) +stmt <<= ( + if_stmt + | while_stmt + | for_stmt + | break_stmt + | return_stmt + | print_stmt + | stmt_block + | pp.Group(expr + SEMI) +) formals = pp.Optional(pp.delimitedList(variable)) -prototype = pp.Group((type_ | VOID)("return_type") - + ident("function_name") - + LPAR + formals("args") + RPAR + SEMI)("prototype") -function_decl = pp.Group((type_ | VOID)("return_type") + ident("function_name") - + LPAR + formals("args") + RPAR - + stmt_block("body"))("function_decl") +prototype = pp.Group( + (type_ | VOID)("return_type") + + ident("function_name") + + LPAR + + formals("args") + + RPAR + + SEMI +)("prototype") +function_decl = pp.Group( + (type_ | VOID)("return_type") + + ident("function_name") + + LPAR + + formals("args") + + RPAR + + stmt_block("body") +)("function_decl") -interface_decl = pp.Group(INTERFACE + ident("interface_name") - + LBRACE + pp.ZeroOrMore(prototype)("prototypes") + RBRACE)("interface") +interface_decl = pp.Group( + INTERFACE + + ident("interface_name") + + LBRACE + + pp.ZeroOrMore(prototype)("prototypes") + + RBRACE +)("interface") field = variable_decl | function_decl -class_decl = pp.Group(CLASS + ident("class_name") - + pp.Optional(EXTENDS + ident)("extends") - + pp.Optional(IMPLEMENTS + pp.delimitedList(ident))("implements") - + LBRACE + pp.ZeroOrMore(field)("fields") + RBRACE)("class_decl") +class_decl = pp.Group( + CLASS + + ident("class_name") + + pp.Optional(EXTENDS + ident)("extends") + + pp.Optional(IMPLEMENTS + pp.delimitedList(ident))("implements") + + LBRACE + + pp.ZeroOrMore(field)("fields") + + RBRACE +)("class_decl") decl = variable_decl | function_decl | class_decl | interface_decl | prototype program = pp.OneOrMore(pp.Group(decl)) decaf_parser = program -stmt.runTests("""\ +stmt.runTests( + """\ sin(30); a = 1; b = 1 + 1; @@ -158,7 +243,8 @@ stmt.runTests("""\ a[100] = b; a[0][0] = 2; a = 0x1234; -""") +""" +) test_program = """ void getenv(string var); diff --git a/examples/delta_time.py b/examples/delta_time.py index e079094..237414f 100644 --- a/examples/delta_time.py +++ b/examples/delta_time.py @@ -39,22 +39,34 @@ __all__ = ["time_expression"] # basic grammar definitions def make_integer_word_expr(int_name, int_value): return pp.CaselessKeyword(int_name).addParseAction(pp.replaceWith(int_value)) -integer_word = pp.MatchFirst(make_integer_word_expr(int_str, int_value) - for int_value, int_str - in enumerate("one two three four five six seven eight nine ten" - " eleven twelve thirteen fourteen fifteen sixteen" - " seventeen eighteen nineteen twenty".split(), start=1)) + + +integer_word = pp.MatchFirst( + make_integer_word_expr(int_str, int_value) + for int_value, int_str in enumerate( + "one two three four five six seven eight nine ten" + " eleven twelve thirteen fourteen fifteen sixteen" + " seventeen eighteen nineteen twenty".split(), + start=1, + ) +) integer = pp.pyparsing_common.integer | integer_word CK = pp.CaselessKeyword CL = pp.CaselessLiteral -today, tomorrow, yesterday, noon, midnight, now = map(CK, "today tomorrow yesterday noon midnight now".split()) +today, tomorrow, yesterday, noon, midnight, now = map( + CK, "today tomorrow yesterday noon midnight now".split() +) + + def plural(s): - return CK(s) | CK(s + 's').addParseAction(pp.replaceWith(s)) + return CK(s) | CK(s + "s").addParseAction(pp.replaceWith(s)) + + week, day, hour, minute, second = map(plural, "week day hour minute second".split()) am = CL("am") pm = CL("pm") -COLON = pp.Suppress(':') +COLON = pp.Suppress(":") in_ = CK("in").setParseAction(pp.replaceWith(1)) from_ = CK("from").setParseAction(pp.replaceWith(1)) @@ -66,51 +78,59 @@ last_ = CK("last").setParseAction(pp.replaceWith(-1)) at_ = CK("at") on_ = CK("on") -couple = (pp.Optional(CK("a")) + CK("couple") + pp.Optional(CK("of"))).setParseAction(pp.replaceWith(2)) +couple = (pp.Optional(CK("a")) + CK("couple") + pp.Optional(CK("of"))).setParseAction( + pp.replaceWith(2) +) a_qty = (CK("a") | CK("an")).setParseAction(pp.replaceWith(1)) the_qty = CK("the").setParseAction(pp.replaceWith(1)) qty = pp.ungroup(integer | couple | a_qty | the_qty) -time_ref_present = pp.Empty().addParseAction(pp.replaceWith(True))('time_ref_present') +time_ref_present = pp.Empty().addParseAction(pp.replaceWith(True))("time_ref_present") + def fill_24hr_time_fields(t): - t['HH'] = t[0] - t['MM'] = t[1] - t['SS'] = 0 - t['ampm'] = ('am','pm')[t.HH >= 12] + t["HH"] = t[0] + t["MM"] = t[1] + t["SS"] = 0 + t["ampm"] = ("am", "pm")[t.HH >= 12] + def fill_default_time_fields(t): - for fld in 'HH MM SS'.split(): + for fld in "HH MM SS".split(): if fld not in t: t[fld] = 0 + weekday_name_list = list(calendar.day_name) weekday_name = pp.oneOf(weekday_name_list) -_24hour_time = pp.Word(pp.nums, exact=4).addParseAction(lambda t: [int(t[0][:2]),int(t[0][2:])], - fill_24hr_time_fields) +_24hour_time = pp.Word(pp.nums, exact=4).addParseAction( + lambda t: [int(t[0][:2]), int(t[0][2:])], fill_24hr_time_fields +) _24hour_time.setName("0000 time") ampm = am | pm -timespec = (integer("HH") - + pp.Optional(CK("o'clock") - | - COLON + integer("MM") - + pp.Optional(COLON + integer("SS")) - ) - + (am | pm)("ampm") - ).addParseAction(fill_default_time_fields) +timespec = ( + integer("HH") + + pp.Optional( + CK("o'clock") | COLON + integer("MM") + pp.Optional(COLON + integer("SS")) + ) + + (am | pm)("ampm") +).addParseAction(fill_default_time_fields) absolute_time = _24hour_time | timespec absolute_time_of_day = noon | midnight | now | absolute_time + def add_computed_time(t): - if t[0] in 'now noon midnight'.split(): - t['computed_time'] = {'now': datetime.now().time().replace(microsecond=0), - 'noon': time(hour=12), - 'midnight': time()}[t[0]] + if t[0] in "now noon midnight".split(): + t["computed_time"] = { + "now": datetime.now().time().replace(microsecond=0), + "noon": time(hour=12), + "midnight": time(), + }[t[0]] else: - t['HH'] = {'am': int(t['HH']) % 12, - 'pm': int(t['HH']) % 12 + 12}[t.ampm] - t['computed_time'] = time(hour=t.HH, minute=t.MM, second=t.SS) + t["HH"] = {"am": int(t["HH"]) % 12, "pm": int(t["HH"]) % 12 + 12}[t.ampm] + t["computed_time"] = time(hour=t.HH, minute=t.MM, second=t.SS) + absolute_time_of_day.addParseAction(add_computed_time) @@ -119,42 +139,49 @@ absolute_time_of_day.addParseAction(add_computed_time) # | qty time_units 'ago' # | 'in' qty time_units time_units = hour | minute | second -relative_time_reference = (qty('qty') + time_units('units') + ago('dir') - | qty('qty') + time_units('units') - + (from_ | before | after)('dir') - + pp.Group(absolute_time_of_day)('ref_time') - | in_('dir') + qty('qty') + time_units('units') - ) +relative_time_reference = ( + qty("qty") + time_units("units") + ago("dir") + | qty("qty") + + time_units("units") + + (from_ | before | after)("dir") + + pp.Group(absolute_time_of_day)("ref_time") + | in_("dir") + qty("qty") + time_units("units") +) + def compute_relative_time(t): - if 'ref_time' not in t: - t['ref_time'] = datetime.now().time().replace(microsecond=0) + if "ref_time" not in t: + t["ref_time"] = datetime.now().time().replace(microsecond=0) else: - t['ref_time'] = t.ref_time.computed_time - delta_seconds = {'hour': 3600, - 'minute': 60, - 'second': 1}[t.units] * t.qty - t['time_delta'] = timedelta(seconds=t.dir * delta_seconds) + t["ref_time"] = t.ref_time.computed_time + delta_seconds = {"hour": 3600, "minute": 60, "second": 1}[t.units] * t.qty + t["time_delta"] = timedelta(seconds=t.dir * delta_seconds) + relative_time_reference.addParseAction(compute_relative_time) time_reference = absolute_time_of_day | relative_time_reference + + def add_default_time_ref_fields(t): - if 'time_delta' not in t: - t['time_delta'] = timedelta() + if "time_delta" not in t: + t["time_delta"] = timedelta() + + time_reference.addParseAction(add_default_time_ref_fields) # absolute_day_reference ::= 'today' | 'tomorrow' | 'yesterday' | ('next' | 'last') weekday_name # day_units ::= 'days' | 'weeks' day_units = day | week -weekday_reference = pp.Optional(next_ | last_, 1)('dir') + weekday_name('day_name') +weekday_reference = pp.Optional(next_ | last_, 1)("dir") + weekday_name("day_name") + def convert_abs_day_reference_to_date(t): now = datetime.now().replace(microsecond=0) # handle day reference by weekday name - if 'day_name' in t: + if "day_name" in t: todaynum = now.weekday() daynames = [n.lower() for n in weekday_name_list] nameddaynum = daynames.index(t.day_name.lower()) @@ -168,84 +195,111 @@ def convert_abs_day_reference_to_date(t): else: name = t[0] t["abs_date"] = { - "now" : now, - "today" : datetime(now.year, now.month, now.day), - "yesterday" : datetime(now.year, now.month, now.day) + timedelta(days=-1), - "tomorrow" : datetime(now.year, now.month, now.day) + timedelta(days=+1), - }[name] + "now": now, + "today": datetime(now.year, now.month, now.day), + "yesterday": datetime(now.year, now.month, now.day) + timedelta(days=-1), + "tomorrow": datetime(now.year, now.month, now.day) + timedelta(days=+1), + }[name] -absolute_day_reference = today | tomorrow | yesterday | now + time_ref_present | weekday_reference + +absolute_day_reference = ( + today | tomorrow | yesterday | now + time_ref_present | weekday_reference +) absolute_day_reference.addParseAction(convert_abs_day_reference_to_date) # relative_day_reference ::= 'in' qty day_units # | qty day_units 'ago' # | 'qty day_units ('from' | 'before' | 'after') absolute_day_reference -relative_day_reference = (in_('dir') + qty('qty') + day_units('units') - | qty('qty') + day_units('units') + ago('dir') - | qty('qty') + day_units('units') + (from_ | before | after)('dir') - + absolute_day_reference('ref_day') - ) +relative_day_reference = ( + in_("dir") + qty("qty") + day_units("units") + | qty("qty") + day_units("units") + ago("dir") + | qty("qty") + + day_units("units") + + (from_ | before | after)("dir") + + absolute_day_reference("ref_day") +) + def compute_relative_date(t): now = datetime.now().replace(microsecond=0) - if 'ref_day' in t: - t['computed_date'] = t.ref_day + if "ref_day" in t: + t["computed_date"] = t.ref_day else: - t['computed_date'] = now.date() - day_diff = t.dir * t.qty * {'week': 7, 'day': 1}[t.units] - t['date_delta'] = timedelta(days=day_diff) + t["computed_date"] = now.date() + day_diff = t.dir * t.qty * {"week": 7, "day": 1}[t.units] + t["date_delta"] = timedelta(days=day_diff) + + relative_day_reference.addParseAction(compute_relative_date) # combine expressions for absolute and relative day references day_reference = relative_day_reference | absolute_day_reference + + def add_default_date_fields(t): - if 'date_delta' not in t: - t['date_delta'] = timedelta() + if "date_delta" not in t: + t["date_delta"] = timedelta() + + day_reference.addParseAction(add_default_date_fields) # combine date and time expressions into single overall parser -time_and_day = (time_reference + time_ref_present + pp.Optional(pp.Optional(on_) + day_reference) - | day_reference + pp.Optional(at_ + absolute_time_of_day + time_ref_present)) +time_and_day = time_reference + time_ref_present + pp.Optional( + pp.Optional(on_) + day_reference +) | day_reference + pp.Optional(at_ + absolute_time_of_day + time_ref_present) # parse actions for total time_and_day expression def save_original_string(s, l, t): # save original input string and reference time - t['original'] = ' '.join(s.strip().split()) - t['relative_to'] = datetime.now().replace(microsecond=0) + t["original"] = " ".join(s.strip().split()) + t["relative_to"] = datetime.now().replace(microsecond=0) + def compute_timestamp(t): # accumulate values from parsed time and day subexpressions - fill in defaults for omitted parts now = datetime.now().replace(microsecond=0) - if 'computed_time' not in t: - t['computed_time'] = t.ref_time or now.time() - if 'abs_date' not in t: - t['abs_date'] = now + if "computed_time" not in t: + t["computed_time"] = t.ref_time or now.time() + if "abs_date" not in t: + t["abs_date"] = now # roll up all fields and apply any time or day deltas - t['computed_dt'] = ( - t.abs_date.replace(hour=t.computed_time.hour, minute=t.computed_time.minute, second=t.computed_time.second) + t["computed_dt"] = ( + t.abs_date.replace( + hour=t.computed_time.hour, + minute=t.computed_time.minute, + second=t.computed_time.second, + ) + (t.time_delta or timedelta(0)) + (t.date_delta or timedelta(0)) ) # if time just given in terms of day expressions, zero out time fields if not t.time_ref_present: - t['computed_dt'] = t.computed_dt.replace(hour=0, minute=0, second=0) + t["computed_dt"] = t.computed_dt.replace(hour=0, minute=0, second=0) # add results name compatible with previous version - t['calculatedTime'] = t.computed_dt + t["calculatedTime"] = t.computed_dt # add time_offset fields - t['time_offset'] = t.computed_dt - t.relative_to + t["time_offset"] = t.computed_dt - t.relative_to + def remove_temp_keys(t): # strip out keys that are just used internally all_keys = list(t.keys()) for k in all_keys: - if k not in ('computed_dt', 'original', 'relative_to', 'time_offset', 'calculatedTime'): + if k not in ( + "computed_dt", + "original", + "relative_to", + "time_offset", + "calculatedTime", + ): del t[k] + time_and_day.addParseAction(save_original_string, compute_timestamp, remove_temp_keys) @@ -304,50 +358,56 @@ if __name__ == "__main__": last Sunday at 2pm """ - time_of_day = timedelta(hours=current_time.hour, - minutes=current_time.minute, - seconds=current_time.second) + time_of_day = timedelta( + hours=current_time.hour, + minutes=current_time.minute, + seconds=current_time.second, + ) expected = { - 'now' : timedelta(0), - '10 minutes ago': timedelta(minutes=-10), - '10 minutes from now': timedelta(minutes=10), - 'in 10 minutes': timedelta(minutes=10), - 'in a minute': timedelta(minutes=1), - 'in a couple of minutes': timedelta(minutes=2), - '20 seconds ago': timedelta(seconds=-20), - 'in 30 seconds': timedelta(seconds=30), - 'in an hour': timedelta(hours=1), - 'in a couple hours': timedelta(hours=2), - 'a week from now': timedelta(days=7), - '3 days from now': timedelta(days=3), - 'a couple of days from now': timedelta(days=2), - 'an hour ago': timedelta(hours=-1), - 'in a couple days': timedelta(days=2) - time_of_day, - 'a week from today': timedelta(days=7) - time_of_day, - 'three weeks ago': timedelta(days=-21) - time_of_day, - 'a day ago': timedelta(days=-1) - time_of_day, - 'in a couple of days': timedelta(days=2) - time_of_day, - 'a couple of days from today': timedelta(days=2) - time_of_day, - '2 weeks after today': timedelta(days=14) - time_of_day, - 'in 2 weeks': timedelta(days=14) - time_of_day, - 'the day after tomorrow': timedelta(days=2) - time_of_day, - 'tomorrow': timedelta(days=1) - time_of_day, - 'the day before yesterday': timedelta(days=-2) - time_of_day, - 'yesterday': timedelta(days=-1) - time_of_day, - 'today': -time_of_day, - 'midnight': -time_of_day, - 'in a day': timedelta(days=1) - time_of_day, - '3 days ago': timedelta(days=-3) - time_of_day, - 'noon tomorrow': timedelta(days=1) - time_of_day + timedelta(hours=12), - '6am tomorrow': timedelta(days=1) - time_of_day + timedelta(hours=6), - '0800 yesterday': timedelta(days=-1) - time_of_day + timedelta(hours=8), - '1700 tomorrow': timedelta(days=1) - time_of_day + timedelta(hours=17), - '12:15 AM today': -time_of_day + timedelta(minutes=15), - '3pm 2 days from today': timedelta(days=2) - time_of_day + timedelta(hours=15), - 'ten seconds before noon tomorrow': timedelta(days=1) - time_of_day - + timedelta(hours=12) + timedelta(seconds=-10), - '20 seconds before noon': -time_of_day + timedelta(hours=12) + timedelta(seconds=-20), - 'in 3 days at 5pm': timedelta(days=3) - time_of_day + timedelta(hours=17), + "now": timedelta(0), + "10 minutes ago": timedelta(minutes=-10), + "10 minutes from now": timedelta(minutes=10), + "in 10 minutes": timedelta(minutes=10), + "in a minute": timedelta(minutes=1), + "in a couple of minutes": timedelta(minutes=2), + "20 seconds ago": timedelta(seconds=-20), + "in 30 seconds": timedelta(seconds=30), + "in an hour": timedelta(hours=1), + "in a couple hours": timedelta(hours=2), + "a week from now": timedelta(days=7), + "3 days from now": timedelta(days=3), + "a couple of days from now": timedelta(days=2), + "an hour ago": timedelta(hours=-1), + "in a couple days": timedelta(days=2) - time_of_day, + "a week from today": timedelta(days=7) - time_of_day, + "three weeks ago": timedelta(days=-21) - time_of_day, + "a day ago": timedelta(days=-1) - time_of_day, + "in a couple of days": timedelta(days=2) - time_of_day, + "a couple of days from today": timedelta(days=2) - time_of_day, + "2 weeks after today": timedelta(days=14) - time_of_day, + "in 2 weeks": timedelta(days=14) - time_of_day, + "the day after tomorrow": timedelta(days=2) - time_of_day, + "tomorrow": timedelta(days=1) - time_of_day, + "the day before yesterday": timedelta(days=-2) - time_of_day, + "yesterday": timedelta(days=-1) - time_of_day, + "today": -time_of_day, + "midnight": -time_of_day, + "in a day": timedelta(days=1) - time_of_day, + "3 days ago": timedelta(days=-3) - time_of_day, + "noon tomorrow": timedelta(days=1) - time_of_day + timedelta(hours=12), + "6am tomorrow": timedelta(days=1) - time_of_day + timedelta(hours=6), + "0800 yesterday": timedelta(days=-1) - time_of_day + timedelta(hours=8), + "1700 tomorrow": timedelta(days=1) - time_of_day + timedelta(hours=17), + "12:15 AM today": -time_of_day + timedelta(minutes=15), + "3pm 2 days from today": timedelta(days=2) - time_of_day + timedelta(hours=15), + "ten seconds before noon tomorrow": timedelta(days=1) + - time_of_day + + timedelta(hours=12) + + timedelta(seconds=-10), + "20 seconds before noon": -time_of_day + + timedelta(hours=12) + + timedelta(seconds=-20), + "in 3 days at 5pm": timedelta(days=3) - time_of_day + timedelta(hours=17), } def verify_offset(instring, parsed): @@ -355,9 +415,9 @@ if __name__ == "__main__": if instring in expected: # allow up to a second time discrepancy due to test processing time if (parsed.time_offset - expected[instring]) <= time_epsilon: - parsed['verify_offset'] = 'PASS' + parsed["verify_offset"] = "PASS" else: - parsed['verify_offset'] = 'FAIL' + parsed["verify_offset"] = "FAIL" print("(relative to %s)" % datetime.now()) time_expression.runTests(tests, postParse=verify_offset) diff --git a/examples/dfmparse.py b/examples/dfmparse.py index 5a1d2a0..5d9b1b1 100644 --- a/examples/dfmparse.py +++ b/examples/dfmparse.py @@ -8,21 +8,36 @@ __version__ = "1.0" __author__ = "Daniel 'Dang' Griffith <pythondev - dang at lazytwinacres . net>"
-from pyparsing import Literal, CaselessLiteral, Word, delimitedList \
- , Optional, Combine, Group, alphas, nums, alphanums, Forward \
- , oneOf, OneOrMore, ZeroOrMore, CharsNotIn
+from pyparsing import (
+ Literal,
+ CaselessLiteral,
+ Word,
+ delimitedList,
+ Optional,
+ Combine,
+ Group,
+ alphas,
+ nums,
+ alphanums,
+ Forward,
+ oneOf,
+ OneOrMore,
+ ZeroOrMore,
+ CharsNotIn,
+)
# This converts DFM character constants into Python string (unicode) values.
def to_chr(x):
"""chr(x) if 0 < x < 128 ; unicode(x) if x > 127."""
- return 0 < x < 128 and chr(x) or eval("u'\\u%d'" % x )
+ return 0 < x < 128 and chr(x) or eval("u'\\u%d'" % x)
+
#################
# BEGIN GRAMMAR
#################
-COLON = Literal(":").suppress()
+COLON = Literal(":").suppress()
CONCAT = Literal("+").suppress()
EQUALS = Literal("=").suppress()
LANGLE = Literal("<").suppress()
@@ -33,66 +48,100 @@ RANGLE = Literal(">").suppress() RBRACE = Literal("]").suppress()
RPAREN = Literal(")").suppress()
-CATEGORIES = CaselessLiteral("categories").suppress()
-END = CaselessLiteral("end").suppress()
-FONT = CaselessLiteral("font").suppress()
-HINT = CaselessLiteral("hint").suppress()
-ITEM = CaselessLiteral("item").suppress()
-OBJECT = CaselessLiteral("object").suppress()
+CATEGORIES = CaselessLiteral("categories").suppress()
+END = CaselessLiteral("end").suppress()
+FONT = CaselessLiteral("font").suppress()
+HINT = CaselessLiteral("hint").suppress()
+ITEM = CaselessLiteral("item").suppress()
+OBJECT = CaselessLiteral("object").suppress()
-attribute_value_pair = Forward() # this is recursed in item_list_entry
+attribute_value_pair = Forward() # this is recursed in item_list_entry
simple_identifier = Word(alphas, alphanums + "_")
-identifier = Combine( simple_identifier + ZeroOrMore( Literal(".") + simple_identifier ))
+identifier = Combine(simple_identifier + ZeroOrMore(Literal(".") + simple_identifier))
object_name = identifier
object_type = identifier
# Integer and floating point values are converted to Python longs and floats, respectively.
-int_value = Combine(Optional("-") + Word(nums)).setParseAction(lambda s,l,t: [ int(t[0]) ] )
-float_value = Combine(Optional("-") + Optional(Word(nums)) + "." + Word(nums)).setParseAction(lambda s,l,t: [ float(t[0]) ] )
+int_value = Combine(Optional("-") + Word(nums)).setParseAction(
+ lambda s, l, t: [int(t[0])]
+)
+float_value = Combine(
+ Optional("-") + Optional(Word(nums)) + "." + Word(nums)
+).setParseAction(lambda s, l, t: [float(t[0])])
number_value = float_value | int_value
# Base16 constants are left in string form, including the surrounding braces.
-base16_value = Combine(Literal("{") + OneOrMore(Word("0123456789ABCDEFabcdef")) + Literal("}"), adjacent=False)
+base16_value = Combine(
+ Literal("{") + OneOrMore(Word("0123456789ABCDEFabcdef")) + Literal("}"),
+ adjacent=False,
+)
# This is the first part of a hack to convert the various delphi partial sglQuotedStrings
# into a single sglQuotedString equivalent. The gist of it is to combine
# all sglQuotedStrings (with their surrounding quotes removed (suppressed))
# with sequences of #xyz character constants, with "strings" concatenated
# with a '+' sign.
-unquoted_sglQuotedString = Combine( Literal("'").suppress() + ZeroOrMore( CharsNotIn("'\n\r") ) + Literal("'").suppress() )
+unquoted_sglQuotedString = Combine(
+ Literal("'").suppress() + ZeroOrMore(CharsNotIn("'\n\r")) + Literal("'").suppress()
+)
# The parse action on this production converts repetitions of constants into a single string.
pound_char = Combine(
- OneOrMore((Literal("#").suppress()+Word(nums)
- ).setParseAction( lambda s, l, t: to_chr(int(t[0]) ))))
+ OneOrMore(
+ (Literal("#").suppress() + Word(nums)).setParseAction(
+ lambda s, l, t: to_chr(int(t[0]))
+ )
+ )
+)
# This is the second part of the hack. It combines the various "unquoted"
# partial strings into a single one. Then, the parse action puts
# a single matched pair of quotes around it.
delphi_string = Combine(
- OneOrMore(CONCAT | pound_char | unquoted_sglQuotedString)
- , adjacent=False
- ).setParseAction(lambda s, l, t: "'%s'" % t[0])
+ OneOrMore(CONCAT | pound_char | unquoted_sglQuotedString), adjacent=False
+).setParseAction(lambda s, l, t: "'%s'" % t[0])
string_value = delphi_string | base16_value
-list_value = LBRACE + Optional(Group(delimitedList(identifier | number_value | string_value))) + RBRACE
-paren_list_value = LPAREN + ZeroOrMore(identifier | number_value | string_value) + RPAREN
+list_value = (
+ LBRACE
+ + Optional(Group(delimitedList(identifier | number_value | string_value)))
+ + RBRACE
+)
+paren_list_value = (
+ LPAREN + ZeroOrMore(identifier | number_value | string_value) + RPAREN
+)
item_list_entry = ITEM + ZeroOrMore(attribute_value_pair) + END
item_list = LANGLE + ZeroOrMore(item_list_entry) + RANGLE
generic_value = identifier
-value = item_list | number_value | string_value | list_value | paren_list_value | generic_value
+value = (
+ item_list
+ | number_value
+ | string_value
+ | list_value
+ | paren_list_value
+ | generic_value
+)
category_attribute = CATEGORIES + PERIOD + oneOf("strings itemsvisibles visibles", True)
-event_attribute = oneOf("onactivate onclosequery onclose oncreate ondeactivate onhide onshow", True)
+event_attribute = oneOf(
+ "onactivate onclosequery onclose oncreate ondeactivate onhide onshow", True
+)
font_attribute = FONT + PERIOD + oneOf("charset color height name style", True)
hint_attribute = HINT
layout_attribute = oneOf("left top width height", True)
generic_attribute = identifier
-attribute = (category_attribute | event_attribute | font_attribute | hint_attribute | layout_attribute | generic_attribute)
+attribute = (
+ category_attribute
+ | event_attribute
+ | font_attribute
+ | hint_attribute
+ | layout_attribute
+ | generic_attribute
+)
category_attribute_value_pair = category_attribute + EQUALS + paren_list_value
event_attribute_value_pair = event_attribute + EQUALS + value
@@ -101,31 +150,36 @@ hint_attribute_value_pair = hint_attribute + EQUALS + value layout_attribute_value_pair = layout_attribute + EQUALS + value
generic_attribute_value_pair = attribute + EQUALS + value
attribute_value_pair << Group(
- category_attribute_value_pair
+ category_attribute_value_pair
| event_attribute_value_pair
| font_attribute_value_pair
| hint_attribute_value_pair
| layout_attribute_value_pair
| generic_attribute_value_pair
- )
+)
object_declaration = Group(OBJECT + object_name + COLON + object_type)
object_attributes = Group(ZeroOrMore(attribute_value_pair))
nested_object = Forward()
-object_definition = object_declaration + object_attributes + ZeroOrMore(nested_object) + END
+object_definition = (
+ object_declaration + object_attributes + ZeroOrMore(nested_object) + END
+)
nested_object << Group(object_definition)
#################
# END GRAMMAR
#################
+
def printer(s, loc, tok):
- print(tok, end=' ')
+ print(tok, end=" ")
return tok
+
def get_filename_list(tf):
import sys, glob
+
if tf == None:
if len(sys.argv) > 1:
tf = sys.argv[1:]
@@ -138,6 +192,7 @@ def get_filename_list(tf): testfiles.extend(glob.glob(arg))
return testfiles
+
def main(testfiles=None, action=printer):
"""testfiles can be None, in which case the command line arguments are used as filenames.
testfiles can be a string, in which case that file is parsed.
@@ -165,8 +220,8 @@ def main(testfiles=None, action=printer): failures.append(f)
if failures:
- print('\nfailed while processing %s' % ', '.join(failures))
- print('\nsucceeded on %d of %d files' %(success, len(testfiles)))
+ print("\nfailed while processing %s" % ", ".join(failures))
+ print("\nsucceeded on %d of %d files" % (success, len(testfiles)))
if len(retval) == 1 and len(testfiles) == 1:
# if only one file is parsed, return the parseResults directly
@@ -175,5 +230,6 @@ def main(testfiles=None, action=printer): # else, return a dictionary of parseResults
return retval
+
if __name__ == "__main__":
main()
diff --git a/examples/dhcpd_leases_parser.py b/examples/dhcpd_leases_parser.py index a885051..e9f64bd 100644 --- a/examples/dhcpd_leases_parser.py +++ b/examples/dhcpd_leases_parser.py @@ -44,28 +44,32 @@ lease 192.168.0.239 { """
from pyparsing import *
-import datetime,time
+import datetime, time
-LBRACE,RBRACE,SEMI,QUOTE = map(Suppress,'{};"')
-ipAddress = Combine(Word(nums) + ('.' + Word(nums))*3)
-hexint = Word(hexnums,exact=2)
-macAddress = Combine(hexint + (':'+hexint)*5)
+LBRACE, RBRACE, SEMI, QUOTE = map(Suppress, '{};"')
+ipAddress = Combine(Word(nums) + ("." + Word(nums)) * 3)
+hexint = Word(hexnums, exact=2)
+macAddress = Combine(hexint + (":" + hexint) * 5)
hdwType = Word(alphanums)
-yyyymmdd = Combine((Word(nums,exact=4)|Word(nums,exact=2))+
- ('/'+Word(nums,exact=2))*2)
-hhmmss = Combine(Word(nums,exact=2)+(':'+Word(nums,exact=2))*2)
-dateRef = oneOf(list("0123456"))("weekday") + yyyymmdd("date") + \
- hhmmss("time")
+yyyymmdd = Combine(
+ (Word(nums, exact=4) | Word(nums, exact=2)) + ("/" + Word(nums, exact=2)) * 2
+)
+hhmmss = Combine(Word(nums, exact=2) + (":" + Word(nums, exact=2)) * 2)
+dateRef = oneOf(list("0123456"))("weekday") + yyyymmdd("date") + hhmmss("time")
+
def utcToLocalTime(tokens):
- utctime = datetime.datetime.strptime("%(date)s %(time)s" % tokens,
- "%Y/%m/%d %H:%M:%S")
- localtime = utctime-datetime.timedelta(0,time.timezone,0)
- tokens["utcdate"],tokens["utctime"] = tokens["date"],tokens["time"]
- tokens["localdate"],tokens["localtime"] = str(localtime).split()
+ utctime = datetime.datetime.strptime(
+ "%(date)s %(time)s" % tokens, "%Y/%m/%d %H:%M:%S"
+ )
+ localtime = utctime - datetime.timedelta(0, time.timezone, 0)
+ tokens["utcdate"], tokens["utctime"] = tokens["date"], tokens["time"]
+ tokens["localdate"], tokens["localtime"] = str(localtime).split()
del tokens["date"]
del tokens["time"]
+
+
dateRef.setParseAction(utcToLocalTime)
startsStmt = "starts" + dateRef + SEMI
@@ -76,12 +80,18 @@ hdwStmt = "hardware" + hdwType("type") + macAddress("mac") + SEMI uidStmt = "uid" + QuotedString('"')("uid") + SEMI
bindingStmt = "binding" + Word(alphanums) + Word(alphanums) + SEMI
-leaseStatement = startsStmt | endsStmt | tstpStmt | tsfpStmt | hdwStmt | \
- uidStmt | bindingStmt
-leaseDef = "lease" + ipAddress("ipaddress") + LBRACE + \
- Dict(ZeroOrMore(Group(leaseStatement))) + RBRACE
+leaseStatement = (
+ startsStmt | endsStmt | tstpStmt | tsfpStmt | hdwStmt | uidStmt | bindingStmt
+)
+leaseDef = (
+ "lease"
+ + ipAddress("ipaddress")
+ + LBRACE
+ + Dict(ZeroOrMore(Group(leaseStatement)))
+ + RBRACE
+)
for lease in leaseDef.searchString(sample):
print(lease.dump())
- print(lease.ipaddress,'->',lease.hardware.mac)
+ print(lease.ipaddress, "->", lease.hardware.mac)
print()
diff --git a/examples/dictExample.py b/examples/dictExample.py index 7d3d45d..ebc437f 100644 --- a/examples/dictExample.py +++ b/examples/dictExample.py @@ -19,15 +19,19 @@ testData = """ """
# define grammar for datatable
-heading = (pp.Literal(
-"+-------+------+------+------+------+------+------+------+------+") +
-"| | A1 | B1 | C1 | D1 | A2 | B2 | C2 | D2 |" +
-"+=======+======+======+======+======+======+======+======+======+").suppress()
+heading = (
+ pp.Literal("+-------+------+------+------+------+------+------+------+------+")
+ + "| | A1 | B1 | C1 | D1 | A2 | B2 | C2 | D2 |"
+ + "+=======+======+======+======+======+======+======+======+======+"
+).suppress()
vert = pp.Literal("|").suppress()
number = pp.Word(pp.nums)
-rowData = pp.Group( vert + pp.Word(pp.alphas) + vert + pp.delimitedList(number,"|") + vert )
+rowData = pp.Group(
+ vert + pp.Word(pp.alphas) + vert + pp.delimitedList(number, "|") + vert
+)
trailing = pp.Literal(
-"+-------+------+------+------+------+------+------+------+------+").suppress()
+ "+-------+------+------+------+------+------+------+------+------+"
+).suppress()
datatable = heading + pp.Dict(pp.ZeroOrMore(rowData)) + trailing
@@ -42,7 +46,7 @@ data.pprint() print("data keys=", list(data.keys()))
# use dict-style access to values
-print("data['min']=", data['min'])
+print("data['min']=", data["min"])
# use attribute-style access to values (if key is a valid Python identifier)
print("data.max", data.max)
diff --git a/examples/dictExample2.py b/examples/dictExample2.py index fa1b866..16590a3 100644 --- a/examples/dictExample2.py +++ b/examples/dictExample2.py @@ -6,7 +6,17 @@ #
# Copyright (c) 2004, Paul McGuire
#
-from pyparsing import Literal, Word, Group, Dict, ZeroOrMore, alphas, nums, delimitedList, pyparsing_common as ppc
+from pyparsing import (
+ Literal,
+ Word,
+ Group,
+ Dict,
+ ZeroOrMore,
+ alphas,
+ nums,
+ delimitedList,
+ pyparsing_common as ppc,
+)
testData = """
+-------+------+------+------+------+------+------+------+------+
@@ -25,34 +35,34 @@ number = ppc.integer vert = Literal("|").suppress()
-rowDelim = ("+" + ZeroOrMore( underline + "+" ) ).suppress()
+rowDelim = ("+" + ZeroOrMore(underline + "+")).suppress()
columnHeader = Group(vert + vert + delimitedList(Word(alphas + nums), "|") + vert)
heading = rowDelim + columnHeader("columns") + rowDelim
-rowData = Group( vert + Word(alphas) + vert + delimitedList(number,"|") + vert )
+rowData = Group(vert + Word(alphas) + vert + delimitedList(number, "|") + vert)
trailing = rowDelim
-datatable = heading + Dict( ZeroOrMore(rowData) ) + trailing
+datatable = heading + Dict(ZeroOrMore(rowData)) + trailing
# now parse data and print results
data = datatable.parseString(testData)
print(data.dump())
print("data keys=", list(data.keys()))
-print("data['min']=", data['min'])
-print("sum(data['min']) =", sum(data['min']))
+print("data['min']=", data["min"])
+print("sum(data['min']) =", sum(data["min"]))
print("data.max =", data.max)
print("sum(data.max) =", sum(data.max))
# now print transpose of data table, using column labels read from table header and
# values from data lists
print()
-print(" " * 5, end=' ')
-for i in range(1,len(data)):
- print("|%5s" % data[i][0], end=' ')
+print(" " * 5, end=" ")
+for i in range(1, len(data)):
+ print("|%5s" % data[i][0], end=" ")
print()
-print(("-" * 6) + ("+------" * (len(data)-1)))
+print(("-" * 6) + ("+------" * (len(data) - 1)))
for i in range(len(data.columns)):
- print("%5s" % data.columns[i], end=' ')
+ print("%5s" % data.columns[i], end=" ")
for j in range(len(data) - 1):
- print('|%5s' % data[j + 1][i + 1], end=' ')
+ print("|%5s" % data[j + 1][i + 1], end=" ")
print()
diff --git a/examples/ebnf.py b/examples/ebnf.py index bb19155..4843d40 100644 --- a/examples/ebnf.py +++ b/examples/ebnf.py @@ -11,7 +11,7 @@ from pyparsing import *
-all_names = '''
+all_names = """
integer
meta_identifier
terminal_string
@@ -25,29 +25,36 @@ single_definition definitions_list
syntax_rule
syntax
-'''.split()
+""".split()
integer = Word(nums)
-meta_identifier = Word(alphas, alphanums + '_')
-terminal_string = Suppress("'") + CharsNotIn("'") + Suppress("'") ^ \
- Suppress('"') + CharsNotIn('"') + Suppress('"')
+meta_identifier = Word(alphas, alphanums + "_")
+terminal_string = Suppress("'") + CharsNotIn("'") + Suppress("'") ^ Suppress(
+ '"'
+) + CharsNotIn('"') + Suppress('"')
definitions_list = Forward()
-optional_sequence = Suppress('[') + definitions_list + Suppress(']')
-repeated_sequence = Suppress('{') + definitions_list + Suppress('}')
-grouped_sequence = Suppress('(') + definitions_list + Suppress(')')
-syntactic_primary = optional_sequence ^ repeated_sequence ^ \
- grouped_sequence ^ meta_identifier ^ terminal_string
-syntactic_factor = Optional(integer + Suppress('*')) + syntactic_primary
-syntactic_term = syntactic_factor + Optional(Suppress('-') + syntactic_factor)
-single_definition = delimitedList(syntactic_term, ',')
-definitions_list << delimitedList(single_definition, '|')
-syntax_rule = meta_identifier + Suppress('=') + definitions_list + \
- Suppress(';')
-
-ebnfComment = ( "(*" +
- ZeroOrMore( CharsNotIn("*") | ( "*" + ~Literal(")") ) ) +
- "*)" ).streamline().setName("ebnfComment")
+optional_sequence = Suppress("[") + definitions_list + Suppress("]")
+repeated_sequence = Suppress("{") + definitions_list + Suppress("}")
+grouped_sequence = Suppress("(") + definitions_list + Suppress(")")
+syntactic_primary = (
+ optional_sequence
+ ^ repeated_sequence
+ ^ grouped_sequence
+ ^ meta_identifier
+ ^ terminal_string
+)
+syntactic_factor = Optional(integer + Suppress("*")) + syntactic_primary
+syntactic_term = syntactic_factor + Optional(Suppress("-") + syntactic_factor)
+single_definition = delimitedList(syntactic_term, ",")
+definitions_list << delimitedList(single_definition, "|")
+syntax_rule = meta_identifier + Suppress("=") + definitions_list + Suppress(";")
+
+ebnfComment = (
+ ("(*" + ZeroOrMore(CharsNotIn("*") | ("*" + ~Literal(")"))) + "*)")
+ .streamline()
+ .setName("ebnfComment")
+)
syntax = OneOrMore(syntax_rule)
syntax.ignore(ebnfComment)
@@ -56,6 +63,7 @@ syntax.ignore(ebnfComment) def do_integer(str, loc, toks):
return int(toks[0])
+
def do_meta_identifier(str, loc, toks):
if toks[0] in symbol_table:
return symbol_table[toks[0]]
@@ -64,28 +72,35 @@ def do_meta_identifier(str, loc, toks): symbol_table[toks[0]] = Forward()
return symbol_table[toks[0]]
+
def do_terminal_string(str, loc, toks):
return Literal(toks[0])
+
def do_optional_sequence(str, loc, toks):
return Optional(toks[0])
+
def do_repeated_sequence(str, loc, toks):
return ZeroOrMore(toks[0])
+
def do_grouped_sequence(str, loc, toks):
return Group(toks[0])
+
def do_syntactic_primary(str, loc, toks):
return toks[0]
+
def do_syntactic_factor(str, loc, toks):
if len(toks) == 2:
# integer * syntactic_primary
return And([toks[1]] * toks[0])
else:
# syntactic_primary
- return [ toks[0] ]
+ return [toks[0]]
+
def do_syntactic_term(str, loc, toks):
if len(toks) == 2:
@@ -93,7 +108,8 @@ def do_syntactic_term(str, loc, toks): return NotAny(toks[1]) + toks[0]
else:
# syntactic_factor
- return [ toks[0] ]
+ return [toks[0]]
+
def do_single_definition(str, loc, toks):
toks = toks.asList()
@@ -102,7 +118,8 @@ def do_single_definition(str, loc, toks): return And(toks)
else:
# syntactic_term
- return [ toks[0] ]
+ return [toks[0]]
+
def do_definitions_list(str, loc, toks):
toks = toks.asList()
@@ -111,31 +128,36 @@ def do_definitions_list(str, loc, toks): return Or(toks)
else:
# single_definition
- return [ toks[0] ]
+ return [toks[0]]
+
def do_syntax_rule(str, loc, toks):
# meta_identifier = definitions_list ;
assert toks[0].expr is None, "Duplicate definition"
forward_count.value -= 1
toks[0] << toks[1]
- return [ toks[0] ]
+ return [toks[0]]
+
def do_syntax(str, loc, toks):
# syntax_rule syntax_rule ...
return symbol_table
-
symbol_table = {}
+
+
class forward_count:
pass
+
+
forward_count.value = 0
for name in all_names:
expr = vars()[name]
- action = vars()['do_' + name]
+ action = vars()["do_" + name]
expr.setName(name)
expr.setParseAction(action)
- #~ expr.setDebug()
+ # ~ expr.setDebug()
def parse(ebnf, given_table={}):
@@ -147,5 +169,5 @@ def parse(ebnf, given_table={}): for name in table:
expr = table[name]
expr.setName(name)
- #~ expr.setDebug()
+ # ~ expr.setDebug()
return table
diff --git a/examples/ebnftest.py b/examples/ebnftest.py index 40772ee..7b1ff75 100644 --- a/examples/ebnftest.py +++ b/examples/ebnftest.py @@ -5,14 +5,14 @@ #
# Submitted 2004 by Seo Sanghyeon
#
-print('Importing pyparsing...')
+print("Importing pyparsing...")
from pyparsing import *
-print('Constructing EBNF parser with pyparsing...')
+print("Constructing EBNF parser with pyparsing...")
import ebnf
-grammar = '''
+grammar = """
syntax = (syntax_rule), {(syntax_rule)};
syntax_rule = meta_identifier, '=', definitions_list, ';';
definitions_list = single_definition, {'|', single_definition};
@@ -30,43 +30,46 @@ terminal_string = "'", character - "'", {character - "'"}, "'" | meta_identifier = letter, {letter | digit};
integer = digit, {digit};
*)
-'''
+"""
table = {}
-#~ table['character'] = Word(printables, exact=1)
-#~ table['letter'] = Word(alphas + '_', exact=1)
-#~ table['digit'] = Word(nums, exact=1)
-table['terminal_string'] = sglQuotedString
-table['meta_identifier'] = Word(alphas+"_", alphas+"_"+nums)
-table['integer'] = Word(nums)
+# ~ table['character'] = Word(printables, exact=1)
+# ~ table['letter'] = Word(alphas + '_', exact=1)
+# ~ table['digit'] = Word(nums, exact=1)
+table["terminal_string"] = sglQuotedString
+table["meta_identifier"] = Word(alphas + "_", alphas + "_" + nums)
+table["integer"] = Word(nums)
-print('Parsing EBNF grammar with EBNF parser...')
+print("Parsing EBNF grammar with EBNF parser...")
parsers = ebnf.parse(grammar, table)
-ebnf_parser = parsers['syntax']
+ebnf_parser = parsers["syntax"]
commentcharcount = 0
commentlocs = set()
-def tallyCommentChars(s,l,t):
- global commentcharcount,commentlocs
+
+
+def tallyCommentChars(s, l, t):
+ global commentcharcount, commentlocs
# only count this comment if we haven't seen it before
if l not in commentlocs:
- charCount = ( len(t[0]) - len(list(filter(str.isspace, t[0]))) )
+ charCount = len(t[0]) - len(list(filter(str.isspace, t[0])))
commentcharcount += charCount
commentlocs.add(l)
- return l,t
+ return l, t
+
-#ordinarily, these lines wouldn't be necessary, but we are doing extra stuff with the comment expression
-ebnf.ebnfComment.setParseAction( tallyCommentChars )
-ebnf_parser.ignore( ebnf.ebnfComment )
+# ordinarily, these lines wouldn't be necessary, but we are doing extra stuff with the comment expression
+ebnf.ebnfComment.setParseAction(tallyCommentChars)
+ebnf_parser.ignore(ebnf.ebnfComment)
-print('Parsing EBNF grammar with generated EBNF parser...\n')
+print("Parsing EBNF grammar with generated EBNF parser...\n")
parsed_chars = ebnf_parser.parseString(grammar)
parsed_char_len = len(parsed_chars)
-print("],\n".join(str( parsed_chars.asList() ).split("],")))
+print("],\n".join(str(parsed_chars.asList()).split("],")))
-#~ grammar_length = len(grammar) - len(filter(str.isspace, grammar))-commentcharcount
+# ~ grammar_length = len(grammar) - len(filter(str.isspace, grammar))-commentcharcount
-#~ assert parsed_char_len == grammar_length
+# ~ assert parsed_char_len == grammar_length
-print('Ok!')
+print("Ok!")
diff --git a/examples/eval_arith.py b/examples/eval_arith.py index 8f3e996..bfd0ce0 100644 --- a/examples/eval_arith.py +++ b/examples/eval_arith.py @@ -8,28 +8,43 @@ # Added support for exponentiation, using right-to-left evaluation of # operands # -from pyparsing import Word, nums, alphas, Combine, oneOf, \ - opAssoc, infixNotation, Literal +from pyparsing import ( + Word, + nums, + alphas, + Combine, + oneOf, + opAssoc, + infixNotation, + Literal, +) + class EvalConstant: "Class to evaluate a parsed constant or variable" vars_ = {} + def __init__(self, tokens): self.value = tokens[0] + def eval(self): if self.value in EvalConstant.vars_: return EvalConstant.vars_[self.value] else: return float(self.value) + class EvalSignOp: "Class to evaluate expressions with a leading + or - sign" + def __init__(self, tokens): self.sign, self.value = tokens[0] + def eval(self): - mult = {'+':1, '-':-1}[self.sign] + mult = {"+": 1, "-": -1}[self.sign] return mult * self.value.eval() + def operatorOperands(tokenlist): "generator to extract operators and operands in pairs" it = iter(tokenlist) @@ -39,67 +54,79 @@ def operatorOperands(tokenlist): except StopIteration: break + class EvalPowerOp: "Class to evaluate multiplication and division expressions" + def __init__(self, tokens): self.value = tokens[0] + def eval(self): res = self.value[-1].eval() for val in self.value[-3::-2]: - res = val.eval()**res + res = val.eval() ** res return res + class EvalMultOp: "Class to evaluate multiplication and division expressions" + def __init__(self, tokens): self.value = tokens[0] + def eval(self): prod = self.value[0].eval() - for op,val in operatorOperands(self.value[1:]): - if op == '*': + for op, val in operatorOperands(self.value[1:]): + if op == "*": prod *= val.eval() - if op == '/': + if op == "/": prod /= val.eval() return prod + class EvalAddOp: "Class to evaluate addition and subtraction expressions" + def __init__(self, tokens): self.value = tokens[0] + def eval(self): sum = self.value[0].eval() - for op,val in operatorOperands(self.value[1:]): - if op == '+': + for op, val in operatorOperands(self.value[1:]): + if op == "+": sum += val.eval() - if op == '-': + if op == "-": sum -= val.eval() return sum + class EvalComparisonOp: "Class to evaluate comparison expressions" opMap = { - "<" : lambda a,b : a < b, - "<=" : lambda a,b : a <= b, - ">" : lambda a,b : a > b, - ">=" : lambda a,b : a >= b, - "!=" : lambda a,b : a != b, - "=" : lambda a,b : a == b, - "LT" : lambda a,b : a < b, - "LE" : lambda a,b : a <= b, - "GT" : lambda a,b : a > b, - "GE" : lambda a,b : a >= b, - "NE" : lambda a,b : a != b, - "EQ" : lambda a,b : a == b, - "<>" : lambda a,b : a != b, - } + "<": lambda a, b: a < b, + "<=": lambda a, b: a <= b, + ">": lambda a, b: a > b, + ">=": lambda a, b: a >= b, + "!=": lambda a, b: a != b, + "=": lambda a, b: a == b, + "LT": lambda a, b: a < b, + "LE": lambda a, b: a <= b, + "GT": lambda a, b: a > b, + "GE": lambda a, b: a >= b, + "NE": lambda a, b: a != b, + "EQ": lambda a, b: a == b, + "<>": lambda a, b: a != b, + } + def __init__(self, tokens): self.value = tokens[0] + def eval(self): val1 = self.value[0].eval() - for op,val in operatorOperands(self.value[1:]): + for op, val in operatorOperands(self.value[1:]): fn = EvalComparisonOp.opMap[op] val2 = val.eval() - if not fn(val1,val2): + if not fn(val1, val2): break val1 = val2 else: @@ -110,104 +137,116 @@ class EvalComparisonOp: # define the parser integer = Word(nums) real = Combine(Word(nums) + "." + Word(nums)) -variable = Word(alphas,exact=1) +variable = Word(alphas, exact=1) operand = real | integer | variable -signop = oneOf('+ -') -multop = oneOf('* /') -plusop = oneOf('+ -') -expop = Literal('**') +signop = oneOf("+ -") +multop = oneOf("* /") +plusop = oneOf("+ -") +expop = Literal("**") # use parse actions to attach EvalXXX constructors to sub-expressions operand.setParseAction(EvalConstant) -arith_expr = infixNotation(operand, +arith_expr = infixNotation( + operand, [ - (signop, 1, opAssoc.RIGHT, EvalSignOp), - (expop, 2, opAssoc.LEFT, EvalPowerOp), - (multop, 2, opAssoc.LEFT, EvalMultOp), - (plusop, 2, opAssoc.LEFT, EvalAddOp), - ]) + (signop, 1, opAssoc.RIGHT, EvalSignOp), + (expop, 2, opAssoc.LEFT, EvalPowerOp), + (multop, 2, opAssoc.LEFT, EvalMultOp), + (plusop, 2, opAssoc.LEFT, EvalAddOp), + ], +) comparisonop = oneOf("< <= > >= != = <> LT GT LE GE EQ NE") -comp_expr = infixNotation(arith_expr, - [ - (comparisonop, 2, opAssoc.LEFT, EvalComparisonOp), - ]) +comp_expr = infixNotation( + arith_expr, [(comparisonop, 2, opAssoc.LEFT, EvalComparisonOp),] +) + def main(): # sample expressions posted on comp.lang.python, asking for advice # in safely evaluating them - rules=[ - '( A - B ) = 0', - '(A + B + C + D + E + F + G + H + I) = J', - '(A + B + C + D + E + F + G + H) = I', - '(A + B + C + D + E + F) = G', - '(A + B + C + D + E) = (F + G + H + I + J)', - '(A + B + C + D + E) = (F + G + H + I)', - '(A + B + C + D + E) = F', - '(A + B + C + D) = (E + F + G + H)', - '(A + B + C) = (D + E + F)', - '(A + B) = (C + D + E + F)', - '(A + B) = (C + D)', - '(A + B) = (C - D + E - F - G + H + I + J)', - '(A + B) = C', - '(A + B) = 0', - '(A+B+C+D+E) = (F+G+H+I+J)', - '(A+B+C+D) = (E+F+G+H)', - '(A+B+C+D)=(E+F+G+H)', - '(A+B+C)=(D+E+F)', - '(A+B)=(C+D)', - '(A+B)=C', - '(A-B)=C', - '(A/(B+C))', - '(B/(C+D))', - '(G + H) = I', - '-0.99 LE ((A+B+C)-(D+E+F+G)) LE 0.99', - '-0.99 LE (A-(B+C)) LE 0.99', - '-1000.00 LE A LE 0.00', - '-5000.00 LE A LE 0.00', - 'A < B', - 'A < 7000', - 'A = -(B)', - 'A = C', - 'A = 0', - 'A GT 0', - 'A GT 0.00', - 'A GT 7.00', - 'A LE B', - 'A LT -1000.00', - 'A LT -5000', - 'A LT 0', - 'A=(B+C+D)', - 'A=B', - 'I = (G + H)', - '0.00 LE A LE 4.00', - '4.00 LT A LE 7.00', - '0.00 LE A LE 4.00 LE E > D', - '2**2**(A+3)', - ] - vars_={'A': 0, 'B': 1.1, 'C': 2.2, 'D': 3.3, 'E': 4.4, 'F': 5.5, 'G': - 6.6, 'H':7.7, 'I':8.8, 'J':9.9} + rules = [ + "( A - B ) = 0", + "(A + B + C + D + E + F + G + H + I) = J", + "(A + B + C + D + E + F + G + H) = I", + "(A + B + C + D + E + F) = G", + "(A + B + C + D + E) = (F + G + H + I + J)", + "(A + B + C + D + E) = (F + G + H + I)", + "(A + B + C + D + E) = F", + "(A + B + C + D) = (E + F + G + H)", + "(A + B + C) = (D + E + F)", + "(A + B) = (C + D + E + F)", + "(A + B) = (C + D)", + "(A + B) = (C - D + E - F - G + H + I + J)", + "(A + B) = C", + "(A + B) = 0", + "(A+B+C+D+E) = (F+G+H+I+J)", + "(A+B+C+D) = (E+F+G+H)", + "(A+B+C+D)=(E+F+G+H)", + "(A+B+C)=(D+E+F)", + "(A+B)=(C+D)", + "(A+B)=C", + "(A-B)=C", + "(A/(B+C))", + "(B/(C+D))", + "(G + H) = I", + "-0.99 LE ((A+B+C)-(D+E+F+G)) LE 0.99", + "-0.99 LE (A-(B+C)) LE 0.99", + "-1000.00 LE A LE 0.00", + "-5000.00 LE A LE 0.00", + "A < B", + "A < 7000", + "A = -(B)", + "A = C", + "A = 0", + "A GT 0", + "A GT 0.00", + "A GT 7.00", + "A LE B", + "A LT -1000.00", + "A LT -5000", + "A LT 0", + "A=(B+C+D)", + "A=B", + "I = (G + H)", + "0.00 LE A LE 4.00", + "4.00 LT A LE 7.00", + "0.00 LE A LE 4.00 LE E > D", + "2**2**(A+3)", + ] + vars_ = { + "A": 0, + "B": 1.1, + "C": 2.2, + "D": 3.3, + "E": 4.4, + "F": 5.5, + "G": 6.6, + "H": 7.7, + "I": 8.8, + "J": 9.9, + } # define tests from given rules tests = [] for t in rules: t_orig = t - t = t.replace("=","==") - t = t.replace("EQ","==") - t = t.replace("LE","<=") - t = t.replace("GT",">") - t = t.replace("LT","<") - t = t.replace("GE",">=") - t = t.replace("LE","<=") - t = t.replace("NE","!=") - t = t.replace("<>","!=") - tests.append( (t_orig,eval(t,vars_)) ) + t = t.replace("=", "==") + t = t.replace("EQ", "==") + t = t.replace("LE", "<=") + t = t.replace("GT", ">") + t = t.replace("LT", "<") + t = t.replace("GE", ">=") + t = t.replace("LE", "<=") + t = t.replace("NE", "!=") + t = t.replace("<>", "!=") + tests.append((t_orig, eval(t, vars_))) # copy vars_ to EvalConstant lookup dict EvalConstant.vars_ = vars_ failed = 0 - for test,expected in tests: + for test, expected in tests: ret = comp_expr.parseString(test)[0] parsedvalue = ret.eval() print(test, expected, parsedvalue) @@ -215,9 +254,9 @@ def main(): print("<<< FAIL") failed += 1 else: - print('') + print("") - print('') + print("") if failed: print(failed, "tests FAILED") return 1 @@ -225,5 +264,6 @@ def main(): print("all tests PASSED") return 0 -if __name__=='__main__': + +if __name__ == "__main__": exit(main()) diff --git a/examples/excelExpr.py b/examples/excelExpr.py index 86237ef..cea2eea 100644 --- a/examples/excelExpr.py +++ b/examples/excelExpr.py @@ -4,36 +4,64 @@ #
# A partial implementation of a parser of Excel formula expressions.
#
-from pyparsing import (CaselessKeyword, Suppress, Word, alphas,
- alphanums, nums, Optional, Group, oneOf, Forward,
- infixNotation, opAssoc, dblQuotedString, delimitedList,
- Combine, Literal, QuotedString, ParserElement, pyparsing_common as ppc)
+from pyparsing import (
+ CaselessKeyword,
+ Suppress,
+ Word,
+ alphas,
+ alphanums,
+ nums,
+ Optional,
+ Group,
+ oneOf,
+ Forward,
+ infixNotation,
+ opAssoc,
+ dblQuotedString,
+ delimitedList,
+ Combine,
+ Literal,
+ QuotedString,
+ ParserElement,
+ pyparsing_common as ppc,
+)
+
ParserElement.enablePackrat()
-EQ,LPAR,RPAR,COLON,COMMA = map(Suppress, '=():,')
-EXCL, DOLLAR = map(Literal,"!$")
-sheetRef = Word(alphas, alphanums) | QuotedString("'",escQuote="''")
-colRef = Optional(DOLLAR) + Word(alphas,max=2)
+EQ, LPAR, RPAR, COLON, COMMA = map(Suppress, "=():,")
+EXCL, DOLLAR = map(Literal, "!$")
+sheetRef = Word(alphas, alphanums) | QuotedString("'", escQuote="''")
+colRef = Optional(DOLLAR) + Word(alphas, max=2)
rowRef = Optional(DOLLAR) + Word(nums)
-cellRef = Combine(Group(Optional(sheetRef + EXCL)("sheet") + colRef("col") +
- rowRef("row")))
+cellRef = Combine(
+ Group(Optional(sheetRef + EXCL)("sheet") + colRef("col") + rowRef("row"))
+)
-cellRange = (Group(cellRef("start") + COLON + cellRef("end"))("range")
- | cellRef | Word(alphas,alphanums))
+cellRange = (
+ Group(cellRef("start") + COLON + cellRef("end"))("range")
+ | cellRef
+ | Word(alphas, alphanums)
+)
expr = Forward()
COMPARISON_OP = oneOf("< = > >= <= != <>")
condExpr = expr + COMPARISON_OP + expr
-ifFunc = (CaselessKeyword("if")
- - LPAR
- + Group(condExpr)("condition")
- + COMMA + Group(expr)("if_true")
- + COMMA + Group(expr)("if_false")
- + RPAR)
+ifFunc = (
+ CaselessKeyword("if")
+ - LPAR
+ + Group(condExpr)("condition")
+ + COMMA
+ + Group(expr)("if_true")
+ + COMMA
+ + Group(expr)("if_false")
+ + RPAR
+)
-statFunc = lambda name : Group(CaselessKeyword(name) + Group(LPAR + delimitedList(expr) + RPAR))
+statFunc = lambda name: Group(
+ CaselessKeyword(name) + Group(LPAR + delimitedList(expr) + RPAR)
+)
sumFunc = statFunc("sum")
minFunc = statFunc("min")
maxFunc = statFunc("max")
@@ -44,22 +72,18 @@ multOp = oneOf("* /") addOp = oneOf("+ -")
numericLiteral = ppc.number
operand = numericLiteral | funcCall | cellRange | cellRef
-arithExpr = infixNotation(operand,
- [
- (multOp, 2, opAssoc.LEFT),
- (addOp, 2, opAssoc.LEFT),
- ])
+arithExpr = infixNotation(
+ operand, [(multOp, 2, opAssoc.LEFT), (addOp, 2, opAssoc.LEFT),]
+)
textOperand = dblQuotedString | cellRef
-textExpr = infixNotation(textOperand,
- [
- ('&', 2, opAssoc.LEFT),
- ])
+textExpr = infixNotation(textOperand, [("&", 2, opAssoc.LEFT),])
expr << (arithExpr | textExpr)
-(EQ + expr).runTests("""\
+(EQ + expr).runTests(
+ """\
=3*A7+5
=3*Sheet1!$A$7+5
=3*'Sheet 1'!$A$7+5"
@@ -67,4 +91,5 @@ expr << (arithExpr | textExpr) =if(Sum(A1:A25)>42,Min(B1:B25),if(Sum(C1:C25)>3.14, (Min(C1:C25)+3)*18,Max(B1:B25)))
=sum(a1:a25,10,min(b1,c2,d3))
=if("T"&a2="TTime", "Ready", "Not ready")
-""")
+"""
+)
diff --git a/examples/fourFn.py b/examples/fourFn.py index e5be215..68441b8 100644 --- a/examples/fourFn.py +++ b/examples/fourFn.py @@ -10,24 +10,40 @@ # # Copyright 2003-2019 by Paul McGuire # -from pyparsing import (Literal, Word, Group, Forward, alphas, alphanums, Regex, ParseException, - CaselessKeyword, Suppress, delimitedList) +from pyparsing import ( + Literal, + Word, + Group, + Forward, + alphas, + alphanums, + Regex, + ParseException, + CaselessKeyword, + Suppress, + delimitedList, +) import math import operator exprStack = [] + def push_first(toks): exprStack.append(toks[0]) + def push_unary_minus(toks): for t in toks: - if t == '-': - exprStack.append('unary -') + if t == "-": + exprStack.append("unary -") else: break + bnf = None + + def BNF(): """ expop :: '^' @@ -52,7 +68,7 @@ def BNF(): # or use provided pyparsing_common.number, but convert back to str: # fnumber = ppc.number().addParseAction(lambda t: str(t[0])) fnumber = Regex(r"[+-]?\d+(?:\.\d*)?(?:[eE][+-]?\d+)?") - ident = Word(alphas, alphanums+"_$") + ident = Word(alphas, alphanums + "_$") plus, minus, mult, div = map(Literal, "+-*/") lpar, rpar = map(Suppress, "()") @@ -63,11 +79,16 @@ def BNF(): expr = Forward() expr_list = delimitedList(Group(expr)) # add parse action that replaces the function identifier with a (name, number of args) tuple - fn_call = (ident + lpar - Group(expr_list) + rpar).setParseAction(lambda t: t.insert(0, (t.pop(0), len(t[0])))) - atom = (addop[...] + - ((fn_call | pi | e | fnumber | ident).setParseAction(push_first) - | Group(lpar + expr + rpar)) - ).setParseAction(push_unary_minus) + fn_call = (ident + lpar - Group(expr_list) + rpar).setParseAction( + lambda t: t.insert(0, (t.pop(0), len(t[0]))) + ) + atom = ( + addop[...] + + ( + (fn_call | pi | e | fnumber | ident).setParseAction(push_first) + | Group(lpar + expr + rpar) + ) + ).setParseAction(push_unary_minus) # by defining exponentiation as "atom [ ^ factor ]..." instead of "atom [ ^ atom ]...", we get right-to-left # exponents, instead of left-to-right that is, 2^3^2 = 2^(3^2), not (2^3)^2. @@ -78,6 +99,7 @@ def BNF(): bnf = expr return bnf + # map operator symbols to corresponding arithmetic operations epsilon = 1e-12 opn = { @@ -85,8 +107,8 @@ opn = { "-": operator.sub, "*": operator.mul, "/": operator.truediv, - "^": operator.pow - } + "^": operator.pow, +} fn = { "sin": math.sin, @@ -96,14 +118,15 @@ fn = { "abs": abs, "trunc": lambda a: int(a), "round": round, - "sgn": lambda a: -1 if a < -epsilon else 1 if a > epsilon else 0 - } + "sgn": lambda a: -1 if a < -epsilon else 1 if a > epsilon else 0, +} + def evaluate_stack(s): op, num_args = s.pop(), 0 if isinstance(op, tuple): op, num_args = op - if op == 'unary -': + if op == "unary -": return -evaluate_stack(s) if op in "+-*/^": # note: operands are pushed onto the stack in reverse order @@ -113,7 +136,7 @@ def evaluate_stack(s): elif op == "PI": return math.pi # 3.1415926535 elif op == "E": - return math.e # 2.718281828 + return math.e # 2.718281828 elif op in fn: # note: args are pushed onto the stack in reverse order args = reversed([evaluate_stack(s) for _ in range(num_args)]) @@ -152,37 +175,37 @@ if __name__ == "__main__": test("9 + 3 + 6", 9 + 3 + 6) test("9 + 3 / 11", 9 + 3.0 / 11) test("(9 + 3)", (9 + 3)) - test("(9+3) / 11", (9+3.0) / 11) + test("(9+3) / 11", (9 + 3.0) / 11) test("9 - 12 - 6", 9 - 12 - 6) test("9 - (12 - 6)", 9 - (12 - 6)) - test("2*3.14159", 2*3.14159) - test("3.1415926535*3.1415926535 / 10", 3.1415926535*3.1415926535 / 10) + test("2*3.14159", 2 * 3.14159) + test("3.1415926535*3.1415926535 / 10", 3.1415926535 * 3.1415926535 / 10) test("PI * PI / 10", math.pi * math.pi / 10) - test("PI*PI/10", math.pi*math.pi/10) - test("PI^2", math.pi**2) - test("round(PI^2)", round(math.pi**2)) - test("6.02E23 * 8.048", 6.02E23 * 8.048) + test("PI*PI/10", math.pi * math.pi / 10) + test("PI^2", math.pi ** 2) + test("round(PI^2)", round(math.pi ** 2)) + test("6.02E23 * 8.048", 6.02e23 * 8.048) test("e / 3", math.e / 3) - test("sin(PI/2)", math.sin(math.pi/2)) - test("10+sin(PI/4)^2", 10 + math.sin(math.pi/4)**2) + test("sin(PI/2)", math.sin(math.pi / 2)) + test("10+sin(PI/4)^2", 10 + math.sin(math.pi / 4) ** 2) test("trunc(E)", int(math.e)) test("trunc(-E)", int(-math.e)) test("round(E)", round(math.e)) test("round(-E)", round(-math.e)) - test("E^PI", math.e**math.pi) + test("E^PI", math.e ** math.pi) test("exp(0)", 1) test("exp(1)", math.e) - test("2^3^2", 2**3**2) - test("(2^3)^2", (2**3)**2) - test("2^3+2", 2**3+2) - test("2^3+5", 2**3+5) - test("2^9", 2**9) + test("2^3^2", 2 ** 3 ** 2) + test("(2^3)^2", (2 ** 3) ** 2) + test("2^3+2", 2 ** 3 + 2) + test("2^3+5", 2 ** 3 + 5) + test("2^9", 2 ** 9) test("sgn(-2)", -1) test("sgn(0)", 0) test("sgn(0.1)", 1) test("foo(0.1)", None) test("round(E, 3)", round(math.e, 3)) - test("round(PI^2, 3)", round(math.pi**2, 3)) + test("round(PI^2, 3)", round(math.pi ** 2, 3)) test("sgn(cos(PI/4))", 1) test("sgn(cos(PI/2))", 0) test("sgn(cos(PI*3/4))", -1) diff --git a/examples/gen_ctypes.py b/examples/gen_ctypes.py index 325aa28..176644f 100644 --- a/examples/gen_ctypes.py +++ b/examples/gen_ctypes.py @@ -8,43 +8,43 @@ from pyparsing import *
typemap = {
- "byte" : "c_byte",
- "char" : "c_char",
- "char *" : "c_char_p",
- "double" : "c_double",
- "float" : "c_float",
- "int" : "c_int",
- "int16" : "c_int16",
- "int32" : "c_int32",
- "int64" : "c_int64",
- "int8" : "c_int8",
- "long" : "c_long",
- "longlong" : "c_longlong",
- "short" : "c_short",
- "size_t" : "c_size_t",
- "ubyte" : "c_ubyte",
- "uchar" : "c_ubyte",
- "u_char" : "c_ubyte",
- "uint" : "c_uint",
- "u_int" : "c_uint",
- "uint16" : "c_uint16",
- "uint32" : "c_uint32",
- "uint64" : "c_uint64",
- "uint8" : "c_uint8",
- "u_long" : "c_ulong",
- "ulong" : "c_ulong",
- "ulonglong" : "c_ulonglong",
- "ushort" : "c_ushort",
- "u_short" : "c_ushort",
- "void *" : "c_void_p",
- "voidp" : "c_voidp",
- "wchar" : "c_wchar",
- "wchar *" : "c_wchar_p",
- "Bool" : "c_bool",
- "void" : "None",
- }
-
-LPAR,RPAR,LBRACE,RBRACE,COMMA,SEMI = map(Suppress,"(){},;")
+ "byte": "c_byte",
+ "char": "c_char",
+ "char *": "c_char_p",
+ "double": "c_double",
+ "float": "c_float",
+ "int": "c_int",
+ "int16": "c_int16",
+ "int32": "c_int32",
+ "int64": "c_int64",
+ "int8": "c_int8",
+ "long": "c_long",
+ "longlong": "c_longlong",
+ "short": "c_short",
+ "size_t": "c_size_t",
+ "ubyte": "c_ubyte",
+ "uchar": "c_ubyte",
+ "u_char": "c_ubyte",
+ "uint": "c_uint",
+ "u_int": "c_uint",
+ "uint16": "c_uint16",
+ "uint32": "c_uint32",
+ "uint64": "c_uint64",
+ "uint8": "c_uint8",
+ "u_long": "c_ulong",
+ "ulong": "c_ulong",
+ "ulonglong": "c_ulonglong",
+ "ushort": "c_ushort",
+ "u_short": "c_ushort",
+ "void *": "c_void_p",
+ "voidp": "c_voidp",
+ "wchar": "c_wchar",
+ "wchar *": "c_wchar_p",
+ "Bool": "c_bool",
+ "void": "None",
+}
+
+LPAR, RPAR, LBRACE, RBRACE, COMMA, SEMI = map(Suppress, "(){},;")
ident = Word(alphas, alphanums + "_")
integer = Regex(r"[+-]?\d+")
hexinteger = Regex(r"0x[0-9a-fA-F]+")
@@ -52,32 +52,50 @@ hexinteger = Regex(r"0x[0-9a-fA-F]+") const = Suppress("const")
primitiveType = oneOf(t for t in typemap if not t.endswith("*"))
structType = Suppress("struct") + ident
-vartype = (Optional(const) +
- (primitiveType | structType | ident) +
- Optional(Word("*")("ptr")))
+vartype = (
+ Optional(const) + (primitiveType | structType | ident) + Optional(Word("*")("ptr"))
+)
+
+
def normalizetype(t):
if isinstance(t, ParseResults):
- return ' '.join(t)
- #~ ret = ParseResults([' '.join(t)])
- #~ return ret
+ return " ".join(t)
+ # ~ ret = ParseResults([' '.join(t)])
+ # ~ return ret
+
vartype.setParseAction(normalizetype)
arg = Group(vartype("argtype") + Optional(ident("argname")))
-func_def = (vartype("fn_type") + ident("fn_name") +
- LPAR + Optional(delimitedList(arg|"..."))("fn_args") + RPAR + SEMI)
+func_def = (
+ vartype("fn_type")
+ + ident("fn_name")
+ + LPAR
+ + Optional(delimitedList(arg | "..."))("fn_args")
+ + RPAR
+ + SEMI
+)
+
+
def derivefields(t):
if t.fn_args and t.fn_args[-1] == "...":
- t["varargs"]=True
+ t["varargs"] = True
+
+
func_def.setParseAction(derivefields)
fn_typedef = "typedef" + func_def
var_typedef = "typedef" + primitiveType("primType") + ident("name") + SEMI
-enum_def = (Keyword("enum") + LBRACE +
- delimitedList(Group(ident("name") + '=' + (hexinteger|integer)("value")))("evalues")
- + Optional(COMMA)
- + RBRACE)
+enum_def = (
+ Keyword("enum")
+ + LBRACE
+ + delimitedList(Group(ident("name") + "=" + (hexinteger | integer)("value")))(
+ "evalues"
+ )
+ + Optional(COMMA)
+ + RBRACE
+)
c_header = open("snmp_api.h").read()
@@ -91,19 +109,23 @@ functions = [] enum_constants = []
# add structures commonly included from std lib headers
-def addStdType(t,namespace=""):
- fullname = namespace+'_'+t if namespace else t
+def addStdType(t, namespace=""):
+ fullname = namespace + "_" + t if namespace else t
typemap[t] = fullname
user_defined_types.add(t)
+
+
addStdType("fd_set", "sys_select")
addStdType("timeval", "sys_time")
+
def getUDType(typestr):
key = typestr.rstrip(" *")
if key not in typemap:
user_defined_types.add(key)
typemap[key] = "{}_{}".format(module, key)
+
def typeAsCtypes(typestr):
if typestr in typemap:
return typemap[typestr]
@@ -111,9 +133,10 @@ def typeAsCtypes(typestr): return "POINTER(%s)" % typeAsCtypes(typestr.rstrip(" *"))
return typestr
+
# scan input header text for primitive typedefs
-for td,_,_ in var_typedef.scanString(c_header):
- typedefs.append( (td.name, td.primType) )
+for td, _, _ in var_typedef.scanString(c_header):
+ typedefs.append((td.name, td.primType))
# add typedef type to typemap to map to itself
typemap[td.name] = td.name
@@ -124,8 +147,11 @@ for fntd in fn_typedefs: typemap[fntd.fn_name] = fntd.fn_name
# scan input header text, and keep running list of user-defined types
-for fn,_,_ in (cStyleComment.suppress() | fn_typedef.suppress() | func_def).scanString(c_header):
- if not fn: continue
+for fn, _, _ in (
+ cStyleComment.suppress() | fn_typedef.suppress() | func_def
+).scanString(c_header):
+ if not fn:
+ continue
getUDType(fn.fn_type)
for arg in fn.fn_args:
if arg != "...":
@@ -135,26 +161,29 @@ for fn,_,_ in (cStyleComment.suppress() | fn_typedef.suppress() | func_def).scan # scan input header text for enums
enum_def.ignore(cppStyleComment)
-for en_,_,_ in enum_def.scanString(c_header):
+for en_, _, _ in enum_def.scanString(c_header):
for ev in en_.evalues:
- enum_constants.append( (ev.name, ev.value) )
+ enum_constants.append((ev.name, ev.value))
print("from ctypes import *")
print("{} = CDLL('{}.dll')".format(module, module))
print()
print("# user defined types")
-for tdname,tdtyp in typedefs:
+for tdname, tdtyp in typedefs:
print("{} = {}".format(tdname, typemap[tdtyp]))
for fntd in fn_typedefs:
- print("{} = CFUNCTYPE({})".format(fntd.fn_name,
- ',\n '.join(typeAsCtypes(a.argtype) for a in fntd.fn_args)))
+ print(
+ "{} = CFUNCTYPE({})".format(
+ fntd.fn_name, ",\n ".join(typeAsCtypes(a.argtype) for a in fntd.fn_args)
+ )
+ )
for udtype in user_defined_types:
print("class %s(Structure): pass" % typemap[udtype])
print()
print("# constant definitions")
-for en,ev in enum_constants:
- print("{} = {}".format(en,ev))
+for en, ev in enum_constants:
+ print("{} = {}".format(en, ev))
print()
print("# functions")
@@ -166,7 +195,11 @@ for fn in functions: print("# warning - %s takes variable argument list" % prefix)
del fn.fn_args[-1]
- if fn.fn_args.asList() != [['void']]:
- print("{}.argtypes = ({},)".format(prefix, ','.join(typeAsCtypes(a.argtype) for a in fn.fn_args)))
+ if fn.fn_args.asList() != [["void"]]:
+ print(
+ "{}.argtypes = ({},)".format(
+ prefix, ",".join(typeAsCtypes(a.argtype) for a in fn.fn_args)
+ )
+ )
else:
print("%s.argtypes = ()" % (prefix))
diff --git a/examples/getNTPserversNew.py b/examples/getNTPserversNew.py index 3ee5e06..5fcd9d1 100644 --- a/examples/getNTPserversNew.py +++ b/examples/getNTPserversNew.py @@ -7,16 +7,25 @@ # September, 2010 - updated to more current use of setResultsName, new NIST URL # import pyparsing as pp + ppc = pp.pyparsing_common from urllib.request import urlopen integer = pp.Word(pp.nums) ipAddress = ppc.ipv4_address() -hostname = pp.delimitedList(pp.Word(pp.alphas, pp.alphanums+"-_"), ".", combine=True) +hostname = pp.delimitedList(pp.Word(pp.alphas, pp.alphanums + "-_"), ".", combine=True) tdStart, tdEnd = pp.makeHTMLTags("td") -timeServerPattern = (tdStart + hostname("hostname") + tdEnd - + tdStart + ipAddress("ipAddr") + tdEnd - + tdStart + tdStart.tag_body("loc") + tdEnd) +timeServerPattern = ( + tdStart + + hostname("hostname") + + tdEnd + + tdStart + + ipAddress("ipAddr") + + tdEnd + + tdStart + + tdStart.tag_body("loc") + + tdEnd +) # get list of time servers nistTimeServerURL = "https://tf.nist.gov/tf-cgi/servers.cgi#" diff --git a/examples/greeting.py b/examples/greeting.py index 6b1cfe3..28a534a 100644 --- a/examples/greeting.py +++ b/examples/greeting.py @@ -14,12 +14,14 @@ greet = pp.Word(pp.alphas) + "," + pp.Word(pp.alphas) + pp.oneOf("! ? .") hello = "Hello, World!" # parse input string -print(hello, "->", greet.parseString( hello )) +print(hello, "->", greet.parseString(hello)) # parse a bunch of input strings -greet.runTests("""\ +greet.runTests( + """\ Hello, World! Ahoy, Matey! Howdy, Pardner! Morning, Neighbor! - """)
\ No newline at end of file + """ +) diff --git a/examples/greetingInGreek.py b/examples/greetingInGreek.py index 56117d2..ed98e9a 100644 --- a/examples/greetingInGreek.py +++ b/examples/greetingInGreek.py @@ -9,7 +9,7 @@ from pyparsing import Word, pyparsing_unicode as ppu # define grammar
alphas = ppu.Greek.alphas
-greet = Word(alphas) + ',' + Word(alphas) + '!'
+greet = Word(alphas) + "," + Word(alphas) + "!"
# input string
hello = "Καλημέρα, κόσμε!"
diff --git a/examples/greetingInKorean.py b/examples/greetingInKorean.py index 9e881ac..00ea9bc 100644 --- a/examples/greetingInKorean.py +++ b/examples/greetingInKorean.py @@ -14,7 +14,7 @@ koreanWord = Word(koreanChars, min=2) greet = koreanWord + "," + koreanWord + "!"
# input string
-hello = '안녕, 여러분!' #"Hello, World!" in Korean
+hello = "안녕, 여러분!" # "Hello, World!" in Korean
# parse input string
print(greet.parseString(hello))
diff --git a/examples/holaMundo.py b/examples/holaMundo.py index 02be601..0589a0d 100644 --- a/examples/holaMundo.py +++ b/examples/holaMundo.py @@ -1,7 +1,15 @@ # escrito por Marco Alfonso, 2004 Noviembre
# importamos los símbolos requeridos desde el módulo
-from pyparsing import Word, alphas, oneOf, nums, Group, OneOrMore, pyparsing_unicode as ppu
+from pyparsing import (
+ Word,
+ alphas,
+ oneOf,
+ nums,
+ Group,
+ OneOrMore,
+ pyparsing_unicode as ppu,
+)
# usamos las letras en latin1, que incluye las como 'ñ', 'á', 'é', etc.
alphas = ppu.Latin1.alphas
@@ -10,42 +18,45 @@ alphas = ppu.Latin1.alphas # una palabra compuesta de caracteres alfanumericos
# (Word(alphas)) mas una ',' mas otra palabra alfanumerica,
# mas '!' y esos seian nuestros tokens
-saludo = Word(alphas) + ',' + Word(alphas) + oneOf('! . ?')
+saludo = Word(alphas) + "," + Word(alphas) + oneOf("! . ?")
tokens = saludo.parseString("Hola, Mundo !")
# Ahora parseamos una cadena, "Hola, Mundo!",
# el metodo parseString, nos devuelve una lista con los tokens
# encontrados, en caso de no haber errores...
for i, token in enumerate(tokens):
- print ("Token %d -> %s" % (i,token))
+ print("Token %d -> %s" % (i, token))
-#imprimimos cada uno de los tokens Y listooo!!, he aquí a salida
+# imprimimos cada uno de los tokens Y listooo!!, he aquí a salida
# Token 0 -> Hola
# Token 1 -> ,
# Token 2-> Mundo
# Token 3 -> !
# ahora cambia el parseador, aceptando saludos con mas que una sola palabra antes que ','
-saludo = Group(OneOrMore(Word(alphas))) + ',' + Word(alphas) + oneOf('! . ?')
+saludo = Group(OneOrMore(Word(alphas))) + "," + Word(alphas) + oneOf("! . ?")
tokens = saludo.parseString("Hasta mañana, Mundo !")
for i, token in enumerate(tokens):
- print ("Token %d -> %s" % (i,token))
+ print("Token %d -> %s" % (i, token))
# Ahora parseamos algunas cadenas, usando el metodo runTests
-saludo.runTests("""\
+saludo.runTests(
+ """\
Hola, Mundo!
Hasta mañana, Mundo !
-""", fullDump=False)
+""",
+ fullDump=False,
+)
# Por supuesto, se pueden "reutilizar" gramáticas, por ejemplo:
-numimag = Word(nums) + 'i'
+numimag = Word(nums) + "i"
numreal = Word(nums)
-numcomplex = numreal + '+' + numimag
-print (numcomplex.parseString("3+5i"))
+numcomplex = numreal + "+" + numimag
+print(numcomplex.parseString("3+5i"))
# Cambiar a complejo numero durante parsear:
-numcomplex.setParseAction(lambda t: complex(''.join(t).replace('i','j')))
-print (numcomplex.parseString("3+5i"))
+numcomplex.setParseAction(lambda t: complex("".join(t).replace("i", "j")))
+print(numcomplex.parseString("3+5i"))
# Excelente!!, bueno, los dejo, me voy a seguir tirando código...
diff --git a/examples/htmlStripper.py b/examples/htmlStripper.py index bd99b77..6a209fa 100644 --- a/examples/htmlStripper.py +++ b/examples/htmlStripper.py @@ -7,8 +7,16 @@ # Copyright (c) 2006, 2016, Paul McGuire # from urllib.request import urlopen -from pyparsing import (makeHTMLTags, commonHTMLEntity, replaceHTMLEntity, - htmlComment, anyOpenTag, anyCloseTag, LineEnd, replaceWith) +from pyparsing import ( + makeHTMLTags, + commonHTMLEntity, + replaceHTMLEntity, + htmlComment, + anyOpenTag, + anyCloseTag, + LineEnd, + replaceWith, +) scriptOpen, scriptClose = makeHTMLTags("script") scriptBody = scriptOpen + scriptOpen.tag_body + scriptClose @@ -16,15 +24,18 @@ commonHTMLEntity.setParseAction(replaceHTMLEntity) # get some HTML targetURL = "https://wiki.python.org/moin/PythonDecoratorLibrary" -with urlopen( targetURL ) as targetPage: +with urlopen(targetURL) as targetPage: targetHTML = targetPage.read().decode("UTF-8") # first pass, strip out tags and translate entities -firstPass = (htmlComment | scriptBody | commonHTMLEntity | - anyOpenTag | anyCloseTag ).suppress().transformString(targetHTML) +firstPass = ( + (htmlComment | scriptBody | commonHTMLEntity | anyOpenTag | anyCloseTag) + .suppress() + .transformString(targetHTML) +) # first pass leaves many blank lines, collapse these down -repeatedNewlines = LineEnd()*(2,) +repeatedNewlines = LineEnd() * (2,) repeatedNewlines.setParseAction(replaceWith("\n\n")) secondPass = repeatedNewlines.transformString(firstPass) diff --git a/examples/htmlTableParser.py b/examples/htmlTableParser.py index 35cdd03..e96a913 100644 --- a/examples/htmlTableParser.py +++ b/examples/htmlTableParser.py @@ -11,42 +11,51 @@ import urllib.request # define basic HTML tags, and compose into a Table -table, table_end = pp.makeHTMLTags('table') -thead, thead_end = pp.makeHTMLTags('thead') -tbody, tbody_end = pp.makeHTMLTags('tbody') -tr, tr_end = pp.makeHTMLTags('tr') -th, th_end = pp.makeHTMLTags('th') -td, td_end = pp.makeHTMLTags('td') -a, a_end = pp.makeHTMLTags('a') +table, table_end = pp.makeHTMLTags("table") +thead, thead_end = pp.makeHTMLTags("thead") +tbody, tbody_end = pp.makeHTMLTags("tbody") +tr, tr_end = pp.makeHTMLTags("tr") +th, th_end = pp.makeHTMLTags("th") +td, td_end = pp.makeHTMLTags("td") +a, a_end = pp.makeHTMLTags("a") # method to strip HTML tags from a string - will be used to clean up content of table cells strip_html = (pp.anyOpenTag | pp.anyCloseTag).suppress().transformString # expression for parsing <a href="url">text</a> links, returning a (text, url) tuple -link = pp.Group(a + a.tag_body('text') + a_end.suppress()) +link = pp.Group(a + a.tag_body("text") + a_end.suppress()) link.addParseAction(lambda t: (t[0].text, t[0].href)) # method to create table rows of header and data tags def table_row(start_tag, end_tag): body = start_tag.tag_body - body.addParseAction(pp.tokenMap(str.strip), - pp.tokenMap(strip_html)) - row = pp.Group(tr.suppress() - + pp.ZeroOrMore(start_tag.suppress() - + body - + end_tag.suppress()) - + tr_end.suppress()) + body.addParseAction(pp.tokenMap(str.strip), pp.tokenMap(strip_html)) + row = pp.Group( + tr.suppress() + + pp.ZeroOrMore(start_tag.suppress() + body + end_tag.suppress()) + + tr_end.suppress() + ) return row + th_row = table_row(th, th_end) td_row = table_row(td, td_end) # define expression for overall table - may vary slightly for different pages -html_table = table + tbody + pp.Optional(th_row('headers')) + pp.ZeroOrMore(td_row)('rows') + tbody_end + table_end +html_table = ( + table + + tbody + + pp.Optional(th_row("headers")) + + pp.ZeroOrMore(td_row)("rows") + + tbody_end + + table_end +) # read in a web page containing an interesting HTML table -with urllib.request.urlopen("https://en.wikipedia.org/wiki/List_of_tz_database_time_zones") as page: +with urllib.request.urlopen( + "https://en.wikipedia.org/wiki/List_of_tz_database_time_zones" +) as page: page_html = page.read().decode() tz_table = html_table.searchString(page_html)[0] @@ -55,7 +64,8 @@ tz_table = html_table.searchString(page_html)[0] rows = [dict(zip(tz_table.headers, row)) for row in tz_table.rows] # make a dict keyed by TZ database name -tz_db = {row['TZ database name']: row for row in rows} +tz_db = {row["TZ database name"]: row for row in rows} from pprint import pprint -pprint(tz_db['America/Chicago']) + +pprint(tz_db["America/Chicago"]) diff --git a/examples/httpServerLogParser.py b/examples/httpServerLogParser.py index b10678b..c84337f 100644 --- a/examples/httpServerLogParser.py +++ b/examples/httpServerLogParser.py @@ -23,39 +23,69 @@ Referer Client Software """ -from pyparsing import alphas,nums, dblQuotedString, Combine, Word, Group, delimitedList, Suppress, removeQuotes +from pyparsing import ( + alphas, + nums, + dblQuotedString, + Combine, + Word, + Group, + delimitedList, + Suppress, + removeQuotes, +) import string -def getCmdFields( s, l, t ): - t["method"],t["requestURI"],t["protocolVersion"] = t[0].strip('"').split() + +def getCmdFields(s, l, t): + t["method"], t["requestURI"], t["protocolVersion"] = t[0].strip('"').split() + logLineBNF = None + + def getLogLineBNF(): global logLineBNF if logLineBNF is None: - integer = Word( nums ) - ipAddress = delimitedList( integer, ".", combine=True ) + integer = Word(nums) + ipAddress = delimitedList(integer, ".", combine=True) - timeZoneOffset = Word("+-",nums) + timeZoneOffset = Word("+-", nums) month = Word(string.ascii_uppercase, string.ascii_lowercase, exact=3) - serverDateTime = Group( Suppress("[") + - Combine( integer + "/" + month + "/" + integer + - ":" + integer + ":" + integer + ":" + integer ) + - timeZoneOffset + - Suppress("]") ) + serverDateTime = Group( + Suppress("[") + + Combine( + integer + + "/" + + month + + "/" + + integer + + ":" + + integer + + ":" + + integer + + ":" + + integer + ) + + timeZoneOffset + + Suppress("]") + ) - logLineBNF = ( ipAddress.setResultsName("ipAddr") + - Suppress("-") + - ("-" | Word( alphas+nums+"@._" )).setResultsName("auth") + - serverDateTime.setResultsName("timestamp") + - dblQuotedString.setResultsName("cmd").setParseAction(getCmdFields) + - (integer | "-").setResultsName("statusCode") + - (integer | "-").setResultsName("numBytesSent") + - dblQuotedString.setResultsName("referrer").setParseAction(removeQuotes) + - dblQuotedString.setResultsName("clientSfw").setParseAction(removeQuotes) ) + logLineBNF = ( + ipAddress.setResultsName("ipAddr") + + Suppress("-") + + ("-" | Word(alphas + nums + "@._")).setResultsName("auth") + + serverDateTime.setResultsName("timestamp") + + dblQuotedString.setResultsName("cmd").setParseAction(getCmdFields) + + (integer | "-").setResultsName("statusCode") + + (integer | "-").setResultsName("numBytesSent") + + dblQuotedString.setResultsName("referrer").setParseAction(removeQuotes) + + dblQuotedString.setResultsName("clientSfw").setParseAction(removeQuotes) + ) return logLineBNF + testdata = """ 195.146.134.15 - - [20/Jan/2003:08:55:36 -0800] "GET /path/to/page.html HTTP/1.0" 200 4649 "http://www.somedomain.com/020602/page.html" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)" 111.111.111.11 - - [16/Feb/2004:04:09:49 -0800] "GET /ads/redirectads/336x280redirect.htm HTTP/1.1" 304 - "http://www.foobarp.org/theme_detail.php?type=vs&cat=0&mid=27512" "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)" @@ -63,10 +93,11 @@ testdata = """ 127.0.0.1 - u.surname@domain.com [12/Sep/2006:14:13:53 +0300] "GET /skins/monobook/external.png HTTP/1.0" 304 - "http://wiki.mysite.com/skins/monobook/main.css" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.6) Gecko/20060728 Firefox/1.5.0.6" """ for line in testdata.split("\n"): - if not line: continue + if not line: + continue fields = getLogLineBNF().parseString(line) print(fields.dump()) - #~ print repr(fields) - #~ for k in fields.keys(): - #~ print "fields." + k + " =", fields[k] + # ~ print repr(fields) + # ~ for k in fields.keys(): + # ~ print "fields." + k + " =", fields[k] print() diff --git a/examples/idlParse.py b/examples/idlParse.py index 1daaf24..52ed3c3 100644 --- a/examples/idlParse.py +++ b/examples/idlParse.py @@ -6,85 +6,200 @@ # Copyright (c) 2003, Paul McGuire
#
-from pyparsing import Literal, Word, OneOrMore, ZeroOrMore, \
- Forward, delimitedList, Group, Optional, alphas, restOfLine, cStyleComment, \
- alphanums, quotedString, ParseException, Keyword, Regex
+from pyparsing import (
+ Literal,
+ Word,
+ OneOrMore,
+ ZeroOrMore,
+ Forward,
+ delimitedList,
+ Group,
+ Optional,
+ alphas,
+ restOfLine,
+ cStyleComment,
+ alphanums,
+ quotedString,
+ ParseException,
+ Keyword,
+ Regex,
+)
import pprint
-#~ import tree2image
+
+# ~ import tree2image
bnf = None
+
+
def CORBA_IDL_BNF():
global bnf
if not bnf:
# punctuation
- (colon,lbrace,rbrace,lbrack,rbrack,lparen,rparen,
- equals,comma,dot,slash,bslash,star,semi,langle,rangle) = map(Literal, r":{}[]()=,./\*;<>")
+ (
+ colon,
+ lbrace,
+ rbrace,
+ lbrack,
+ rbrack,
+ lparen,
+ rparen,
+ equals,
+ comma,
+ dot,
+ slash,
+ bslash,
+ star,
+ semi,
+ langle,
+ rangle,
+ ) = map(Literal, r":{}[]()=,./\*;<>")
# keywords
- (any_, attribute_, boolean_, case_, char_, const_, context_, default_, double_, enum_, exception_,
- FALSE_, fixed_, float_, inout_, interface_, in_, long_, module_, Object_, octet_, oneway_, out_, raises_,
- readonly_, sequence_, short_, string_, struct_, switch_, TRUE_, typedef_, unsigned_, union_, void_,
- wchar_, wstring_) = map(Keyword, """any attribute boolean case char const context
+ (
+ any_,
+ attribute_,
+ boolean_,
+ case_,
+ char_,
+ const_,
+ context_,
+ default_,
+ double_,
+ enum_,
+ exception_,
+ FALSE_,
+ fixed_,
+ float_,
+ inout_,
+ interface_,
+ in_,
+ long_,
+ module_,
+ Object_,
+ octet_,
+ oneway_,
+ out_,
+ raises_,
+ readonly_,
+ sequence_,
+ short_,
+ string_,
+ struct_,
+ switch_,
+ TRUE_,
+ typedef_,
+ unsigned_,
+ union_,
+ void_,
+ wchar_,
+ wstring_,
+ ) = map(
+ Keyword,
+ """any attribute boolean case char const context
default double enum exception FALSE fixed float inout interface in long module
Object octet oneway out raises readonly sequence short string struct switch
- TRUE typedef unsigned union void wchar wstring""".split())
+ TRUE typedef unsigned union void wchar wstring""".split(),
+ )
- identifier = Word( alphas, alphanums + "_" ).setName("identifier")
+ identifier = Word(alphas, alphanums + "_").setName("identifier")
real = Regex(r"[+-]?\d+\.\d*([Ee][+-]?\d+)?").setName("real")
integer = Regex(r"0x[0-9a-fA-F]+|[+-]?\d+").setName("int")
- udTypeName = delimitedList( identifier, "::", combine=True ).setName("udType")
- typeName = ( any_ | boolean_ | char_ | double_ | fixed_ |
- float_ | long_ | octet_ | short_ | string_ |
- wchar_ | wstring_ | udTypeName ).setName("type")
+ udTypeName = delimitedList(identifier, "::", combine=True).setName("udType")
+ typeName = (
+ any_
+ | boolean_
+ | char_
+ | double_
+ | fixed_
+ | float_
+ | long_
+ | octet_
+ | short_
+ | string_
+ | wchar_
+ | wstring_
+ | udTypeName
+ ).setName("type")
sequenceDef = Forward().setName("seq")
- sequenceDef << Group( sequence_ + langle + ( sequenceDef | typeName ) + rangle )
- typeDef = sequenceDef | ( typeName + Optional( lbrack + integer + rbrack ) )
- typedefDef = Group( typedef_ + typeDef + identifier + semi ).setName("typedef")
+ sequenceDef << Group(sequence_ + langle + (sequenceDef | typeName) + rangle)
+ typeDef = sequenceDef | (typeName + Optional(lbrack + integer + rbrack))
+ typedefDef = Group(typedef_ + typeDef + identifier + semi).setName("typedef")
moduleDef = Forward()
- constDef = Group( const_ + typeDef + identifier + equals + ( real | integer | quotedString ) + semi ) #| quotedString )
- exceptionItem = Group( typeDef + identifier + semi )
- exceptionDef = ( exception_ + identifier + lbrace + ZeroOrMore( exceptionItem ) + rbrace + semi )
- attributeDef = Optional( readonly_ ) + attribute_ + typeDef + identifier + semi
- paramlist = delimitedList( Group( ( inout_ | in_ | out_ ) + typeName + identifier ) ).setName( "paramlist" )
- operationDef = ( ( void_ ^ typeDef ) + identifier + lparen + Optional( paramlist ) + rparen + \
- Optional( raises_ + lparen + Group( delimitedList( typeName ) ) + rparen ) + semi )
- interfaceItem = ( constDef | exceptionDef | attributeDef | operationDef )
- interfaceDef = Group( interface_ + identifier + Optional( colon + delimitedList( typeName ) ) + lbrace + \
- ZeroOrMore( interfaceItem ) + rbrace + semi ).setName("opnDef")
- moduleItem = ( interfaceDef | exceptionDef | constDef | typedefDef | moduleDef )
- moduleDef << module_ + identifier + lbrace + ZeroOrMore( moduleItem ) + rbrace + semi
-
- bnf = ( moduleDef | OneOrMore( moduleItem ) )
+ constDef = Group(
+ const_
+ + typeDef
+ + identifier
+ + equals
+ + (real | integer | quotedString)
+ + semi
+ ) # | quotedString )
+ exceptionItem = Group(typeDef + identifier + semi)
+ exceptionDef = (
+ exception_ + identifier + lbrace + ZeroOrMore(exceptionItem) + rbrace + semi
+ )
+ attributeDef = Optional(readonly_) + attribute_ + typeDef + identifier + semi
+ paramlist = delimitedList(
+ Group((inout_ | in_ | out_) + typeName + identifier)
+ ).setName("paramlist")
+ operationDef = (
+ (void_ ^ typeDef)
+ + identifier
+ + lparen
+ + Optional(paramlist)
+ + rparen
+ + Optional(raises_ + lparen + Group(delimitedList(typeName)) + rparen)
+ + semi
+ )
+ interfaceItem = constDef | exceptionDef | attributeDef | operationDef
+ interfaceDef = Group(
+ interface_
+ + identifier
+ + Optional(colon + delimitedList(typeName))
+ + lbrace
+ + ZeroOrMore(interfaceItem)
+ + rbrace
+ + semi
+ ).setName("opnDef")
+ moduleItem = interfaceDef | exceptionDef | constDef | typedefDef | moduleDef
+ moduleDef << module_ + identifier + lbrace + ZeroOrMore(
+ moduleItem
+ ) + rbrace + semi
+
+ bnf = moduleDef | OneOrMore(moduleItem)
singleLineComment = "//" + restOfLine
- bnf.ignore( singleLineComment )
- bnf.ignore( cStyleComment )
+ bnf.ignore(singleLineComment)
+ bnf.ignore(cStyleComment)
return bnf
+
testnum = 1
-def test( strng ):
+
+
+def test(strng):
global testnum
print(strng)
try:
bnf = CORBA_IDL_BNF()
- tokens = bnf.parseString( strng )
+ tokens = bnf.parseString(strng)
print("tokens = ")
- pprint.pprint( tokens.asList() )
+ pprint.pprint(tokens.asList())
imgname = "idlParse%02d.bmp" % testnum
testnum += 1
- #~ tree2image.str2image( str(tokens.asList()), imgname )
+ # ~ tree2image.str2image( str(tokens.asList()), imgname )
except ParseException as err:
print(err.line)
- print(" "*(err.column-1) + "^")
+ print(" " * (err.column - 1) + "^")
print(err)
print()
+
if __name__ == "__main__":
test(
"""
@@ -101,7 +216,7 @@ if __name__ == "__main__": string method3();
};
"""
- )
+ )
test(
"""
/*
@@ -122,7 +237,7 @@ if __name__ == "__main__": string method3();
};
"""
- )
+ )
test(
r"""
const string test="Test String\n";
@@ -141,7 +256,7 @@ if __name__ == "__main__": void method1( in string arg1, inout long arg2 );
};
"""
- )
+ )
test(
"""
module Test1
@@ -158,7 +273,7 @@ if __name__ == "__main__": };
};
"""
- )
+ )
test(
"""
module Test1
@@ -170,4 +285,4 @@ if __name__ == "__main__": };
"""
- )
+ )
diff --git a/examples/include_preprocessor.py b/examples/include_preprocessor.py index 0b0d742..294d658 100644 --- a/examples/include_preprocessor.py +++ b/examples/include_preprocessor.py @@ -9,19 +9,20 @@ import pyparsing as pp from pathlib import Path # parser elements to be used to assemble into #include parser -SEMI = pp.Suppress(';') +SEMI = pp.Suppress(";") INCLUDE = pp.Keyword("#include") quoted_string = pp.quotedString.addParseAction(pp.removeQuotes) -file_ref = (quoted_string - | pp.Word(pp.printables, excludeChars=';')) +file_ref = quoted_string | pp.Word(pp.printables, excludeChars=";") # parser for parsing "#include xyz.dat;" directives -include_directive = (INCLUDE + file_ref("include_file_name") + SEMI) +include_directive = INCLUDE + file_ref("include_file_name") + SEMI # add parse action that will recursively pull in included files - when # using transformString, the value returned from the parse action will replace # the text matched by the attached expression seen = set() + + def read_include_contents(s, l, t): include_file_ref = t.include_file_name include_echo = "/* {} */".format(pp.line(l, s).strip()) @@ -30,18 +31,22 @@ def read_include_contents(s, l, t): if include_file_ref not in seen: seen.add(include_file_ref) included_file_contents = Path(include_file_ref).read_text() - return (include_echo + '\n' - + include_directive.transformString(included_file_contents)) + return ( + include_echo + + "\n" + + include_directive.transformString(included_file_contents) + ) else: - lead = ' '*(pp.col(l, s) - 1) + lead = " " * (pp.col(l, s) - 1) return "/* recursive include! */\n{}{}".format(lead, include_echo) + # attach include processing method as parse action (parse-time callback) # to include_directive expression include_directive.addParseAction(read_include_contents) -if __name__ == '__main__': +if __name__ == "__main__": # demo @@ -49,35 +54,40 @@ if __name__ == '__main__': # - a.txt includes b.txt # - b.txt includes c.txt # - c.txt includes b.txt (must catch infinite recursion) - Path('a.txt').write_text("""\ + Path("a.txt").write_text( + """\ /* a.txt */ int i; - - /* sometimes included files aren't in quotes */ + + /* sometimes included files aren't in quotes */ #include b.txt; - """) + """ + ) - Path('b.txt').write_text("""\ + Path("b.txt").write_text( + """\ i = 100; #include 'c.txt'; - """) + """ + ) - Path('c.txt').write_text("""\ + Path("c.txt").write_text( + """\ i += 1; - + /* watch out! this might be recursive if this file included by b.txt */ #include b.txt; - """) - + """ + ) # use include_directive.transformString to perform includes # read contents of original file - initial_file = Path('a.txt').read_text() + initial_file = Path("a.txt").read_text() # print original file print(initial_file) - print('-----------------') + print("-----------------") # expand includes in source file (and any included files) and print the result expanded_source = include_directive.transformString(initial_file) diff --git a/examples/indentedGrammarExample.py b/examples/indentedGrammarExample.py index 0133a9e..c761393 100644 --- a/examples/indentedGrammarExample.py +++ b/examples/indentedGrammarExample.py @@ -37,14 +37,16 @@ stmt = Forward() suite = indentedBlock(stmt, indentStack)
identifier = Word(alphas, alphanums)
-funcDecl = ("def" + identifier + Group( "(" + Optional( delimitedList(identifier) ) + ")" ) + ":")
-funcDef = Group( funcDecl + suite )
+funcDecl = (
+ "def" + identifier + Group("(" + Optional(delimitedList(identifier)) + ")") + ":"
+)
+funcDef = Group(funcDecl + suite)
rvalue = Forward()
funcCall = Group(identifier + "(" + Optional(delimitedList(rvalue)) + ")")
rvalue << (funcCall | identifier | Word(nums))
assignment = Group(identifier + "=" + rvalue)
-stmt << ( funcDef | assignment | identifier )
+stmt << (funcDef | assignment | identifier)
module_body = OneOrMore(stmt)
diff --git a/examples/invRegex.py b/examples/invRegex.py index 1777635..f0bfe31 100644 --- a/examples/invRegex.py +++ b/examples/invRegex.py @@ -11,85 +11,122 @@ # - () grouping
# - | alternation
#
-__all__ = ["count","invert"]
+__all__ = ["count", "invert"]
+
+from pyparsing import (
+ Literal,
+ oneOf,
+ printables,
+ ParserElement,
+ Combine,
+ SkipTo,
+ infixNotation,
+ ParseFatalException,
+ Word,
+ nums,
+ opAssoc,
+ Suppress,
+ ParseResults,
+ srange,
+)
-from pyparsing import (Literal, oneOf, printables, ParserElement, Combine,
- SkipTo, infixNotation, ParseFatalException, Word, nums, opAssoc,
- Suppress, ParseResults, srange)
class CharacterRangeEmitter:
- def __init__(self,chars):
+ def __init__(self, chars):
# remove duplicate chars in character range, but preserve original order
seen = set()
- self.charset = "".join( seen.add(c) or c for c in chars if c not in seen )
+ self.charset = "".join(seen.add(c) or c for c in chars if c not in seen)
+
def __str__(self):
- return '['+self.charset+']'
+ return "[" + self.charset + "]"
+
def __repr__(self):
- return '['+self.charset+']'
+ return "[" + self.charset + "]"
+
def makeGenerator(self):
def genChars():
- yield from self.charset + yield from self.charset
+
return genChars
+
class OptionalEmitter:
- def __init__(self,expr):
+ def __init__(self, expr):
self.expr = expr
+
def makeGenerator(self):
def optionalGen():
yield ""
- yield from self.expr.makeGenerator()() + yield from self.expr.makeGenerator()()
+
return optionalGen
+
class DotEmitter:
def makeGenerator(self):
def dotGen():
- yield from printables + yield from printables
+
return dotGen
+
class GroupEmitter:
- def __init__(self,exprs):
+ def __init__(self, exprs):
self.exprs = ParseResults(exprs)
+
def makeGenerator(self):
def groupGen():
def recurseList(elist):
- if len(elist)==1:
- yield from elist[0].makeGenerator()() + if len(elist) == 1:
+ yield from elist[0].makeGenerator()()
else:
for s in elist[0].makeGenerator()():
for s2 in recurseList(elist[1:]):
yield s + s2
+
if self.exprs:
- yield from recurseList(self.exprs) + yield from recurseList(self.exprs)
+
return groupGen
+
class AlternativeEmitter:
- def __init__(self,exprs):
+ def __init__(self, exprs):
self.exprs = exprs
+
def makeGenerator(self):
def altGen():
for e in self.exprs:
- yield from e.makeGenerator()() + yield from e.makeGenerator()()
+
return altGen
+
class LiteralEmitter:
- def __init__(self,lit):
+ def __init__(self, lit):
self.lit = lit
+
def __str__(self):
- return "Lit:"+self.lit
+ return "Lit:" + self.lit
+
def __repr__(self):
- return "Lit:"+self.lit
+ return "Lit:" + self.lit
+
def makeGenerator(self):
def litGen():
yield self.lit
+
return litGen
+
def handleRange(toks):
return CharacterRangeEmitter(srange(toks[0]))
+
def handleRepetition(toks):
- toks=toks[0]
+ toks = toks[0]
if toks[1] in "*+":
- raise ParseFatalException("",0,"unbounded repetition operators not supported")
+ raise ParseFatalException("", 0, "unbounded repetition operators not supported")
if toks[1] == "?":
return OptionalEmitter(toks[0])
if "count" in toks:
@@ -100,24 +137,26 @@ def handleRepetition(toks): optcount = maxcount - mincount
if optcount:
opt = OptionalEmitter(toks[0])
- for i in range(1,optcount):
- opt = OptionalEmitter(GroupEmitter([toks[0],opt]))
+ for i in range(1, optcount):
+ opt = OptionalEmitter(GroupEmitter([toks[0], opt]))
return GroupEmitter([toks[0]] * mincount + [opt])
else:
return [toks[0]] * mincount
+
def handleLiteral(toks):
lit = ""
for t in toks:
if t[0] == "\\":
if t[1] == "t":
- lit += '\t'
+ lit += "\t"
else:
lit += t[1]
else:
lit += t
return LiteralEmitter(lit)
+
def handleMacro(toks):
macroChar = toks[0][1]
if macroChar == "d":
@@ -127,60 +166,74 @@ def handleMacro(toks): elif macroChar == "s":
return LiteralEmitter(" ")
else:
- raise ParseFatalException("",0,"unsupported macro character (" + macroChar + ")")
+ raise ParseFatalException(
+ "", 0, "unsupported macro character (" + macroChar + ")"
+ )
+
def handleSequence(toks):
return GroupEmitter(toks[0])
+
def handleDot():
return CharacterRangeEmitter(printables)
+
def handleAlternative(toks):
return AlternativeEmitter(toks[0])
_parser = None
+
+
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
- lbrack,rbrack,lbrace,rbrace,lparen,rparen,colon,qmark = map(Literal,"[]{}():?")
+ lbrack, rbrack, lbrace, rbrace, lparen, rparen, colon, qmark = map(
+ Literal, "[]{}():?"
+ )
reMacro = Combine("\\" + oneOf(list("dws")))
escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables)))
- reLiteralChar = "".join(c for c in printables if c not in r"\[]{}().*?+|") + " \t"
+ reLiteralChar = (
+ "".join(c for c in printables if c not in r"\[]{}().*?+|") + " \t"
+ )
- reRange = Combine(lbrack + SkipTo(rbrack,ignore=escapedChar) + rbrack)
- reLiteral = ( escapedChar | oneOf(list(reLiteralChar)) )
+ reRange = Combine(lbrack + SkipTo(rbrack, ignore=escapedChar) + rbrack)
+ reLiteral = escapedChar | oneOf(list(reLiteralChar))
reNonCaptureGroup = Suppress("?:")
reDot = Literal(".")
repetition = (
- ( lbrace + Word(nums)("count") + rbrace ) |
- ( lbrace + Word(nums)("minCount")+","+ Word(nums)("maxCount") + rbrace ) |
- oneOf(list("*+?"))
- )
+ (lbrace + Word(nums)("count") + rbrace)
+ | (lbrace + Word(nums)("minCount") + "," + Word(nums)("maxCount") + rbrace)
+ | oneOf(list("*+?"))
+ )
reRange.setParseAction(handleRange)
reLiteral.setParseAction(handleLiteral)
reMacro.setParseAction(handleMacro)
reDot.setParseAction(handleDot)
- reTerm = ( reLiteral | reRange | reMacro | reDot | reNonCaptureGroup)
- reExpr = infixNotation( reTerm,
+ reTerm = reLiteral | reRange | reMacro | reDot | reNonCaptureGroup
+ reExpr = infixNotation(
+ reTerm,
[
- (repetition, 1, opAssoc.LEFT, handleRepetition),
- (None, 2, opAssoc.LEFT, handleSequence),
- (Suppress('|'), 2, opAssoc.LEFT, handleAlternative),
- ]
- )
+ (repetition, 1, opAssoc.LEFT, handleRepetition),
+ (None, 2, opAssoc.LEFT, handleSequence),
+ (Suppress("|"), 2, opAssoc.LEFT, handleAlternative),
+ ],
+ )
_parser = reExpr
return _parser
+
def count(gen):
"""Simple function to count the number of elements returned by a generator."""
return sum(1 for _ in gen)
+
def invert(regex):
r"""Call this routine as a generator to return all the strings that
match the input regular expression.
@@ -190,6 +243,7 @@ def invert(regex): invReGenerator = GroupEmitter(parser().parseString(regex)).makeGenerator()
return invReGenerator()
+
def main():
tests = r"""
[A-EA]
@@ -225,12 +279,15 @@ def main(): [ABCDEFG](?:#|##|b|bb)?(?:maj|min|m|sus|aug|dim)?[0-9]?(?:/[ABCDEFG](?:#|##|b|bb)?)?
(Fri|Mon|S(atur|un)|T(hur|ue)s|Wednes)day
A(pril|ugust)|((Dec|Nov|Sept)em|Octo)ber|(Febr|Jan)uary|Ju(ly|ne)|Ma(rch|y)
- """.split('\n')
+ """.split(
+ "\n"
+ )
for t in tests:
t = t.strip()
- if not t: continue
- print('-'*50)
+ if not t:
+ continue
+ print("-" * 50)
print(t)
try:
num = count(invert(t))
@@ -243,9 +300,10 @@ def main(): break
except ParseFatalException as pfe:
print(pfe.msg)
- print('')
+ print("")
continue
- print('')
+ print("")
+
if __name__ == "__main__":
main()
diff --git a/examples/jsonParser.py b/examples/jsonParser.py index fbf76b4..3dd9b69 100644 --- a/examples/jsonParser.py +++ b/examples/jsonParser.py @@ -36,11 +36,14 @@ value import pyparsing as pp
from pyparsing import pyparsing_common as ppc
+
def make_keyword(kwd_str, kwd_value):
return pp.Keyword(kwd_str).setParseAction(pp.replaceWith(kwd_value))
-TRUE = make_keyword("true", True)
+
+
+TRUE = make_keyword("true", True)
FALSE = make_keyword("false", False)
-NULL = make_keyword("null", None)
+NULL = make_keyword("null", None)
LBRACK, RBRACK, LBRACE, RBRACE, COLON = map(pp.Suppress, "[]{}:")
@@ -49,9 +52,11 @@ jsonNumber = ppc.number() jsonObject = pp.Forward()
jsonValue = pp.Forward()
-jsonElements = pp.delimitedList( jsonValue )
+jsonElements = pp.delimitedList(jsonValue)
jsonArray = pp.Group(LBRACK + pp.Optional(jsonElements, []) + RBRACK)
-jsonValue << (jsonString | jsonNumber | pp.Group(jsonObject) | jsonArray | TRUE | FALSE | NULL)
+jsonValue << (
+ jsonString | jsonNumber | pp.Group(jsonObject) | jsonArray | TRUE | FALSE | NULL
+)
memberDef = pp.Group(jsonString + COLON + jsonValue)
jsonMembers = pp.delimitedList(memberDef)
jsonObject << pp.Dict(LBRACE + pp.Optional(jsonMembers) + RBRACE)
@@ -94,12 +99,14 @@ if __name__ == "__main__": results = jsonObject.parseString(testdata)
results.pprint()
print()
+
def testPrint(x):
print(type(x), repr(x))
+
print(list(results.glossary.GlossDiv.GlossList.keys()))
- testPrint( results.glossary.title )
- testPrint( results.glossary.GlossDiv.GlossList.ID )
- testPrint( results.glossary.GlossDiv.GlossList.FalseValue )
- testPrint( results.glossary.GlossDiv.GlossList.Acronym )
- testPrint( results.glossary.GlossDiv.GlossList.EvenPrimesGreaterThan2 )
- testPrint( results.glossary.GlossDiv.GlossList.PrimesLessThan10 )
+ testPrint(results.glossary.title)
+ testPrint(results.glossary.GlossDiv.GlossList.ID)
+ testPrint(results.glossary.GlossDiv.GlossList.FalseValue)
+ testPrint(results.glossary.GlossDiv.GlossList.Acronym)
+ testPrint(results.glossary.GlossDiv.GlossList.EvenPrimesGreaterThan2)
+ testPrint(results.glossary.GlossDiv.GlossList.PrimesLessThan10)
diff --git a/examples/linenoExample.py b/examples/linenoExample.py index f343869..d21e502 100644 --- a/examples/linenoExample.py +++ b/examples/linenoExample.py @@ -15,16 +15,20 @@ to come to the aid of their country."""
# demonstrate use of lineno, line, and col in a parse action
-def reportLongWords(st,locn,toks):
+def reportLongWords(st, locn, toks):
word = toks[0]
if len(word) > 3:
- print("Found '%s' on line %d at column %d" % (word, lineno(locn,st), col(locn,st)))
+ print(
+ "Found '%s' on line %d at column %d"
+ % (word, lineno(locn, st), col(locn, st))
+ )
print("The full line of text was:")
- print("'%s'" % line(locn,st))
- print((" "*col(locn,st))+" ^")
+ print("'%s'" % line(locn, st))
+ print((" " * col(locn, st)) + " ^")
print()
-wd = Word(alphas).setParseAction( reportLongWords )
+
+wd = Word(alphas).setParseAction(reportLongWords)
OneOrMore(wd).parseString(data)
@@ -34,16 +38,19 @@ class Token: def __init__(self, st, locn, tokString):
self.tokenString = tokString
self.locn = locn
- self.sourceLine = line(locn,st)
- self.lineNo = lineno(locn,st)
- self.col = col(locn,st)
+ self.sourceLine = line(locn, st)
+ self.lineNo = lineno(locn, st)
+ self.col = col(locn, st)
+
def __str__(self):
return "%(tokenString)s (line: %(lineNo)d, col: %(col)d)" % self.__dict__
-def createTokenObject(st,locn,toks):
- return Token(st,locn, toks[0])
-wd = Word(alphas).setParseAction( createTokenObject )
+def createTokenObject(st, locn, toks):
+ return Token(st, locn, toks[0])
+
+
+wd = Word(alphas).setParseAction(createTokenObject)
for tokenObj in OneOrMore(wd).parseString(data):
print(tokenObj)
diff --git a/examples/list1.py b/examples/list1.py index 49a0bff..53673db 100644 --- a/examples/list1.py +++ b/examples/list1.py @@ -11,8 +11,9 @@ from pyparsing import * lbrack = Literal("[")
rbrack = Literal("]")
integer = Word(nums).setName("integer")
-real = Combine(Optional(oneOf("+ -")) + Word(nums) + "." +
- Optional(Word(nums))).setName("real")
+real = Combine(
+ Optional(oneOf("+ -")) + Word(nums) + "." + Optional(Word(nums))
+).setName("real")
listItem = real | integer | quotedString
@@ -26,12 +27,15 @@ print(listStr.parseString(test)) # second pass, cleanup and add converters
lbrack = Literal("[").suppress()
rbrack = Literal("]").suppress()
-cvtInt = lambda s,l,toks: int(toks[0])
-cvtReal = lambda s,l,toks: float(toks[0])
-integer = Word(nums).setName("integer").setParseAction( cvtInt )
-real = Combine(Optional(oneOf("+ -")) + Word(nums) + "." +
- Optional(Word(nums))).setName("real").setParseAction( cvtReal )
-listItem = real | integer | quotedString.setParseAction( removeQuotes )
+cvtInt = lambda s, l, toks: int(toks[0])
+cvtReal = lambda s, l, toks: float(toks[0])
+integer = Word(nums).setName("integer").setParseAction(cvtInt)
+real = (
+ Combine(Optional(oneOf("+ -")) + Word(nums) + "." + Optional(Word(nums)))
+ .setName("real")
+ .setParseAction(cvtReal)
+)
+listItem = real | integer | quotedString.setParseAction(removeQuotes)
listStr = lbrack + delimitedList(listItem) + rbrack
@@ -45,12 +49,12 @@ lbrack, rbrack = map(Suppress, "[]") cvtInt = tokenMap(int)
cvtReal = tokenMap(float)
-integer = Word(nums).setName("integer").setParseAction( cvtInt )
-real = Regex(r"[+-]?\d+\.\d*").setName("real").setParseAction( cvtReal )
+integer = Word(nums).setName("integer").setParseAction(cvtInt)
+real = Regex(r"[+-]?\d+\.\d*").setName("real").setParseAction(cvtReal)
listStr = Forward()
listItem = real | integer | quotedString.setParseAction(removeQuotes) | Group(listStr)
listStr << lbrack + delimitedList(listItem) + rbrack
test = "['a', 100, 3.14, [ +2.718, 'xyzzy', -1.414] ]"
-print(listStr.parseString(test))
\ No newline at end of file +print(listStr.parseString(test))
diff --git a/examples/listAllMatches.py b/examples/listAllMatches.py index 7301c84..02ef481 100644 --- a/examples/listAllMatches.py +++ b/examples/listAllMatches.py @@ -8,11 +8,11 @@ from pyparsing import oneOf, OneOrMore, printables, StringEnd
test = "The quick brown fox named 'Aloysius' lives at 123 Main Street (and jumps over lazy dogs in his spare time)."
-nonAlphas = [ c for c in printables if not c.isalpha() ]
+nonAlphas = [c for c in printables if not c.isalpha()]
print("Extract vowels, consonants, and special characters from this test string:")
print("'" + test + "'")
-print('')
+print("")
print("Define grammar using normal results names")
print("(only last matching symbol is saved)")
@@ -26,7 +26,7 @@ print(results) print(results.vowels)
print(results.cons)
print(results.others)
-print('')
+print("")
print("Define grammar using results names, with listAllMatches=True")
@@ -40,12 +40,12 @@ letters = OneOrMore(cons | vowels | other) results = letters.parseString(test, parseAll=True)
print(results)
print(sorted(set(results)))
-print('')
+print("")
print(results.vowels)
print(sorted(set(results.vowels)))
-print('')
+print("")
print(results.cons)
print(sorted(set(results.cons)))
-print('')
+print("")
print(results.others)
print(sorted(set(results.others)))
diff --git a/examples/lucene_grammar.py b/examples/lucene_grammar.py index bf92509..ee4633c 100644 --- a/examples/lucene_grammar.py +++ b/examples/lucene_grammar.py @@ -9,19 +9,22 @@ import pyparsing as pp from pyparsing import pyparsing_common as ppc + pp.ParserElement.enablePackrat() -COLON,LBRACK,RBRACK,LBRACE,RBRACE,TILDE,CARAT = map(pp.Literal,":[]{}~^") -LPAR,RPAR = map(pp.Suppress,"()") +COLON, LBRACK, RBRACK, LBRACE, RBRACE, TILDE, CARAT = map(pp.Literal, ":[]{}~^") +LPAR, RPAR = map(pp.Suppress, "()") and_, or_, not_, to_ = map(pp.CaselessKeyword, "AND OR NOT TO".split()) keyword = and_ | or_ | not_ | to_ expression = pp.Forward() -valid_word = pp.Regex(r'([a-zA-Z0-9*_+.-]|\\\\|\\([+\-!(){}\[\]^"~*?:]|\|\||&&))+').setName("word") +valid_word = pp.Regex( + r'([a-zA-Z0-9*_+.-]|\\\\|\\([+\-!(){}\[\]^"~*?:]|\|\||&&))+' +).setName("word") valid_word.setParseAction( - lambda t : t[0].replace('\\\\',chr(127)).replace('\\','').replace(chr(127),'\\') - ) + lambda t: t[0].replace("\\\\", chr(127)).replace("\\", "").replace(chr(127), "\\") +) string = pp.QuotedString('"') @@ -37,24 +40,28 @@ field_name = valid_word().setName("fieldname") incl_range_search = pp.Group(LBRACK - term("lower") + to_ + term("upper") + RBRACK) excl_range_search = pp.Group(LBRACE - term("lower") + to_ + term("upper") + RBRACE) range_search = incl_range_search("incl_range") | excl_range_search("excl_range") -boost = (CARAT - number("boost")) +boost = CARAT - number("boost") string_expr = pp.Group(string + proximity_modifier) | string word_expr = pp.Group(valid_word + fuzzy_modifier) | valid_word -term << (pp.Optional(field_name("field") + COLON) - + (word_expr | string_expr | range_search | pp.Group(LPAR + expression + RPAR)) - + pp.Optional(boost)) -term.setParseAction(lambda t:[t] if 'field' in t or 'boost' in t else None) +term << ( + pp.Optional(field_name("field") + COLON) + + (word_expr | string_expr | range_search | pp.Group(LPAR + expression + RPAR)) + + pp.Optional(boost) +) +term.setParseAction(lambda t: [t] if "field" in t or "boost" in t else None) -expression << pp.infixNotation(term, +expression << pp.infixNotation( + term, [ - (required_modifier | prohibit_modifier, 1, pp.opAssoc.RIGHT), - ((not_ | '!').setParseAction(lambda: "NOT"), 1, pp.opAssoc.RIGHT), - ((and_ | '&&').setParseAction(lambda: "AND"), 2, pp.opAssoc.LEFT), - (pp.Optional(or_ | '||').setParseAction(lambda: "OR"), 2, pp.opAssoc.LEFT), - ]) + (required_modifier | prohibit_modifier, 1, pp.opAssoc.RIGHT), + ((not_ | "!").setParseAction(lambda: "NOT"), 1, pp.opAssoc.RIGHT), + ((and_ | "&&").setParseAction(lambda: "AND"), 2, pp.opAssoc.LEFT), + (pp.Optional(or_ | "||").setParseAction(lambda: "OR"), 2, pp.opAssoc.LEFT), + ], +) -if __name__ == '__main__': +if __name__ == "__main__": # test strings taken from grammar description doc, and TestQueryParser.java tests = r""" @@ -329,4 +336,5 @@ if __name__ == '__main__': if not (success1 and success2): import sys + sys.exit(1) diff --git a/examples/macroExpander.py b/examples/macroExpander.py index c6b7034..3b06250 100644 --- a/examples/macroExpander.py +++ b/examples/macroExpander.py @@ -16,7 +16,7 @@ from pyparsing import * # define the structure of a macro definition (the empty term is used
# to advance to the next non-whitespace character)
-identifier = Word(alphas+"_",alphanums+"_")
+identifier = Word(alphas + "_", alphanums + "_")
macroDef = "#def" + identifier("macro") + empty + restOfLine("value")
# define a placeholder for defined macros - initially nothing
@@ -27,16 +27,18 @@ macroExpr << NoMatch() macros = {}
# parse action for macro definitions
-def processMacroDefn(s,l,t):
+def processMacroDefn(s, l, t):
macroVal = macroExpander.transformString(t.value)
macros[t.macro] = macroVal
macroExpr << MatchFirst(map(Keyword, macros.keys()))
return "#def " + t.macro + " " + macroVal
+
# parse action to replace macro references with their respective definition
-def processMacroRef(s,l,t):
+def processMacroRef(s, l, t):
return macros[t[0]]
+
# attach parse actions to expressions
macroExpr.setParseAction(processMacroRef)
macroDef.setParseAction(processMacroDefn)
@@ -45,7 +47,6 @@ macroDef.setParseAction(processMacroDefn) macroExpander = macroExpr | macroDef
-
# test macro substitution using transformString
testString = """
#def A 100
diff --git a/examples/matchPreviousDemo.py b/examples/matchPreviousDemo.py index 34991f8..caa76e1 100644 --- a/examples/matchPreviousDemo.py +++ b/examples/matchPreviousDemo.py @@ -23,7 +23,7 @@ identifier = Word(alphas) classIdent = identifier("classname") # note that this also makes a copy of identifier
classHead = "class" + classIdent
classBody = "..."
-classEnd = "end" + matchPreviousLiteral(classIdent) + ';'
+classEnd = "end" + matchPreviousLiteral(classIdent) + ";"
classDefn = classHead + classBody + classEnd
# use this form to catch syntax error
diff --git a/examples/mozillaCalendarParser.py b/examples/mozillaCalendarParser.py index 2633eb8..5000cfe 100644 --- a/examples/mozillaCalendarParser.py +++ b/examples/mozillaCalendarParser.py @@ -1,4 +1,13 @@ -from pyparsing import Optional, oneOf, Literal, Word, printables, Group, OneOrMore, ZeroOrMore +from pyparsing import ( + Optional, + oneOf, + Literal, + Word, + printables, + Group, + OneOrMore, + ZeroOrMore, +) """ A simple parser for calendar (*.ics) files, @@ -14,68 +23,73 @@ License: Free for any use # TERMINALS -BEGIN = Literal("BEGIN:").suppress() -END = Literal("END:").suppress() -valstr = printables + "\xe4\xf6\xe5\xd6\xc4\xc5 " +BEGIN = Literal("BEGIN:").suppress() +END = Literal("END:").suppress() +valstr = printables + "\xe4\xf6\xe5\xd6\xc4\xc5 " -EQ = Literal("=").suppress() -SEMI = Literal(";").suppress() -COLON = Literal(":").suppress() +EQ = Literal("=").suppress() +SEMI = Literal(";").suppress() +COLON = Literal(":").suppress() -EVENT = Literal("VEVENT").suppress() -CALENDAR = Literal("VCALENDAR").suppress() -ALARM = Literal("VALARM").suppress() +EVENT = Literal("VEVENT").suppress() +CALENDAR = Literal("VCALENDAR").suppress() +ALARM = Literal("VALARM").suppress() # TOKENS -CALPROP = oneOf("VERSION PRODID METHOD") -ALMPROP = oneOf("TRIGGER") -EVTPROP = oneOf("X-MOZILLA-RECUR-DEFAULT-INTERVAL \ +CALPROP = oneOf("VERSION PRODID METHOD") +ALMPROP = oneOf("TRIGGER") +EVTPROP = oneOf( + "X-MOZILLA-RECUR-DEFAULT-INTERVAL \ X-MOZILLA-RECUR-DEFAULT-UNITS \ - UID DTSTAMP LAST-MODIFIED X RRULE EXDATE") + UID DTSTAMP LAST-MODIFIED X RRULE EXDATE" +) -propval = Word(valstr) -typeval = Word(valstr) -typename = oneOf("VALUE MEMBER FREQ UNTIL INTERVAL") +propval = Word(valstr) +typeval = Word(valstr) +typename = oneOf("VALUE MEMBER FREQ UNTIL INTERVAL") proptype = Group(SEMI + typename + EQ + typeval).suppress() calprop = Group(CALPROP + ZeroOrMore(proptype) + COLON + propval) almprop = Group(ALMPROP + ZeroOrMore(proptype) + COLON + propval) -evtprop = Group(EVTPROP + ZeroOrMore(proptype) + COLON + propval).suppress() \ - | "CATEGORIES" + COLON + propval.setResultsName("categories") \ - | "CLASS" + COLON + propval.setResultsName("class") \ - | "DESCRIPTION" + COLON + propval.setResultsName("description") \ - | "DTSTART" + proptype + COLON + propval.setResultsName("begin") \ - | "DTEND" + proptype + COLON + propval.setResultsName("end") \ - | "LOCATION" + COLON + propval.setResultsName("location") \ - | "PRIORITY" + COLON + propval.setResultsName("priority") \ - | "STATUS" + COLON + propval.setResultsName("status") \ - | "SUMMARY" + COLON + propval.setResultsName("summary") \ - | "URL" + COLON + propval.setResultsName("url") \ - +evtprop = ( + Group(EVTPROP + ZeroOrMore(proptype) + COLON + propval).suppress() + | "CATEGORIES" + COLON + propval.setResultsName("categories") + | "CLASS" + COLON + propval.setResultsName("class") + | "DESCRIPTION" + COLON + propval.setResultsName("description") + | "DTSTART" + proptype + COLON + propval.setResultsName("begin") + | "DTEND" + proptype + COLON + propval.setResultsName("end") + | "LOCATION" + COLON + propval.setResultsName("location") + | "PRIORITY" + COLON + propval.setResultsName("priority") + | "STATUS" + COLON + propval.setResultsName("status") + | "SUMMARY" + COLON + propval.setResultsName("summary") + | "URL" + COLON + propval.setResultsName("url") +) calprops = Group(OneOrMore(calprop)).suppress() evtprops = Group(OneOrMore(evtprop)) almprops = Group(OneOrMore(almprop)).suppress() -alarm = BEGIN + ALARM + almprops + END + ALARM -event = BEGIN + EVENT + evtprops + Optional(alarm) + END + EVENT -events = Group(OneOrMore(event)) -calendar = BEGIN + CALENDAR + calprops + ZeroOrMore(event) + END + CALENDAR -calendars = OneOrMore(calendar) +alarm = BEGIN + ALARM + almprops + END + ALARM +event = BEGIN + EVENT + evtprops + Optional(alarm) + END + EVENT +events = Group(OneOrMore(event)) +calendar = BEGIN + CALENDAR + calprops + ZeroOrMore(event) + END + CALENDAR +calendars = OneOrMore(calendar) # PARSE ACTIONS -def gotEvent(s,loc,toks): - for event in toks: - print(event.dump()) + +def gotEvent(s, loc, toks): + for event in toks: + print(event.dump()) + event.setParseAction(gotEvent) # MAIN PROGRAM -if __name__=="__main__": +if __name__ == "__main__": - calendars.parseFile("mozilla.ics") + calendars.parseFile("mozilla.ics") diff --git a/examples/nested.py b/examples/nested.py index 218c10b..c696b48 100644 --- a/examples/nested.py +++ b/examples/nested.py @@ -21,8 +21,8 @@ data = """ # use {}'s for nested lists
nestedItems = nestedExpr("{", "}")
-print( (nestedItems+stringEnd).parseString(data).asList() )
+print((nestedItems + stringEnd).parseString(data).asList())
# use default delimiters of ()'s
mathExpr = nestedExpr()
-print( mathExpr.parseString( "((( ax + by)*C) *(Z | (E^F) & D))") )
+print(mathExpr.parseString("((( ax + by)*C) *(Z | (E^F) & D))"))
diff --git a/examples/nested_markup.py b/examples/nested_markup.py index 6d83636..3e3aafa 100644 --- a/examples/nested_markup.py +++ b/examples/nested_markup.py @@ -12,27 +12,34 @@ wiki_markup = pp.Forward() # a method that will construct and return a parse action that will # do the proper wrapping in opening and closing HTML, and recursively call # wiki_markup.transformString on the markup body text -def convert_markup_to_html(opening,closing): +def convert_markup_to_html(opening, closing): def conversionParseAction(s, l, t): return opening + wiki_markup.transformString(t[1][1:-1]) + closing + return conversionParseAction + # use a nestedExpr with originalTextFor to parse nested braces, but return the # parsed text as a single string containing the outermost nested braces instead # of a nested list of parsed tokens -markup_body = pp.originalTextFor(pp.nestedExpr('{', '}')) -italicized = ('ital' + markup_body).setParseAction(convert_markup_to_html("<I>", "</I>")) -bolded = ('bold' + markup_body).setParseAction(convert_markup_to_html("<B>", "</B>")) +markup_body = pp.originalTextFor(pp.nestedExpr("{", "}")) +italicized = ("ital" + markup_body).setParseAction( + convert_markup_to_html("<I>", "</I>") +) +bolded = ("bold" + markup_body).setParseAction(convert_markup_to_html("<B>", "</B>")) # another markup and parse action to parse links - again using transform string # to recursively parse any markup in the link text def convert_link_to_html(s, l, t): link_text, url = t._skipped - t['link_text'] = wiki_markup.transformString(link_text) - t['url'] = url + t["link_text"] = wiki_markup.transformString(link_text) + t["url"] = url return '<A href="{url}">{link_text}</A>'.format_map(t) -urlRef = (pp.Keyword('link') + '{' + ... + '->' + ... + '}').setParseAction(convert_link_to_html) + +urlRef = (pp.Keyword("link") + "{" + ... + "->" + ... + "}").setParseAction( + convert_link_to_html +) # now inject all the markup bits as possible markup expressions wiki_markup <<= urlRef | italicized | bolded diff --git a/examples/numerics.py b/examples/numerics.py index 0af3cae..3a1f9e9 100644 --- a/examples/numerics.py +++ b/examples/numerics.py @@ -48,15 +48,19 @@ tests = """\ from pyparsing import Regex
-comma_decimal = Regex(r'\d{1,2}(([ .])\d\d\d(\2\d\d\d)*)?,\d*')
-comma_decimal.setParseAction(lambda t: float(t[0].replace(' ','').replace('.','').replace(',','.')))
+comma_decimal = Regex(r"\d{1,2}(([ .])\d\d\d(\2\d\d\d)*)?,\d*")
+comma_decimal.setParseAction(
+ lambda t: float(t[0].replace(" ", "").replace(".", "").replace(",", "."))
+)
-dot_decimal = Regex(r'\d{1,2}(([ ,])\d\d\d(\2\d\d\d)*)?\.\d*')
-dot_decimal.setParseAction(lambda t: float(t[0].replace(' ','').replace(',','')))
+dot_decimal = Regex(r"\d{1,2}(([ ,])\d\d\d(\2\d\d\d)*)?\.\d*")
+dot_decimal.setParseAction(lambda t: float(t[0].replace(" ", "").replace(",", "")))
decimal = comma_decimal ^ dot_decimal
decimal.runTests(tests, parseAll=True)
-grouped_integer = Regex(r'\d{1,2}(([ .,])\d\d\d(\2\d\d\d)*)?')
-grouped_integer.setParseAction(lambda t: int(t[0].replace(' ','').replace(',','').replace('.','')))
+grouped_integer = Regex(r"\d{1,2}(([ .,])\d\d\d(\2\d\d\d)*)?")
+grouped_integer.setParseAction(
+ lambda t: int(t[0].replace(" ", "").replace(",", "").replace(".", ""))
+)
grouped_integer.runTests(tests, parseAll=False)
diff --git a/examples/oc.py b/examples/oc.py index c8b95b1..f19a2b0 100644 --- a/examples/oc.py +++ b/examples/oc.py @@ -71,13 +71,15 @@ The following is a description of the OC grammar: """
from pyparsing import *
+
ParserElement.enablePackrat()
-LPAR,RPAR,LBRACK,RBRACK,LBRACE,RBRACE,SEMI,COMMA = map(Suppress, "()[]{};,")
-INT, CHAR, WHILE, DO, IF, ELSE, RETURN = map(Keyword,
- "int char while do if else return".split())
+LPAR, RPAR, LBRACK, RBRACK, LBRACE, RBRACE, SEMI, COMMA = map(Suppress, "()[]{};,")
+INT, CHAR, WHILE, DO, IF, ELSE, RETURN = map(
+ Keyword, "int char while do if else return".split()
+)
-NAME = Word(alphas+"_", alphanums+"_")
+NAME = Word(alphas + "_", alphanums + "_")
integer = Regex(r"[+-]?\d+")
char = Regex(r"'.'")
string_ = dblQuotedString
@@ -86,19 +88,20 @@ TYPE = Group((INT | CHAR) + ZeroOrMore("*")) expr = Forward()
func_call = Group(NAME + LPAR + Group(Optional(delimitedList(expr))) + RPAR)
operand = func_call | NAME | integer | char | string_
-expr <<= (infixNotation(operand,
+expr <<= infixNotation(
+ operand,
[
- (oneOf('! - *'), 1, opAssoc.RIGHT),
- (oneOf('++ --'), 1, opAssoc.RIGHT),
- (oneOf('++ --'), 1, opAssoc.LEFT),
- (oneOf('* / %'), 2, opAssoc.LEFT),
- (oneOf('+ -'), 2, opAssoc.LEFT),
- (oneOf('< == > <= >= !='), 2, opAssoc.LEFT),
- (Regex(r'(?<!=)=(?!=)'), 2, opAssoc.LEFT),
- ]) +
- Optional( LBRACK + expr + RBRACK |
- LPAR + Group(Optional(delimitedList(expr))) + RPAR )
- )
+ (oneOf("! - *"), 1, opAssoc.RIGHT),
+ (oneOf("++ --"), 1, opAssoc.RIGHT),
+ (oneOf("++ --"), 1, opAssoc.LEFT),
+ (oneOf("* / %"), 2, opAssoc.LEFT),
+ (oneOf("+ -"), 2, opAssoc.LEFT),
+ (oneOf("< == > <= >= !="), 2, opAssoc.LEFT),
+ (Regex(r"(?<!=)=(?!=)"), 2, opAssoc.LEFT),
+ ],
+) + Optional(
+ LBRACK + expr + RBRACK | LPAR + Group(Optional(delimitedList(expr))) + RPAR
+)
stmt = Forward()
@@ -107,34 +110,46 @@ whilestmt = WHILE - LPAR + expr + RPAR + stmt dowhilestmt = DO - stmt + WHILE + LPAR + expr + RPAR + SEMI
returnstmt = RETURN - expr + SEMI
-stmt << Group( ifstmt |
- whilestmt |
- dowhilestmt |
- returnstmt |
- expr + SEMI |
- LBRACE + ZeroOrMore(stmt) + RBRACE |
- SEMI)
+stmt << Group(
+ ifstmt
+ | whilestmt
+ | dowhilestmt
+ | returnstmt
+ | expr + SEMI
+ | LBRACE + ZeroOrMore(stmt) + RBRACE
+ | SEMI
+)
vardecl = Group(TYPE + NAME + Optional(LBRACK + integer + RBRACK)) + SEMI
arg = Group(TYPE + NAME)
body = ZeroOrMore(vardecl) + ZeroOrMore(stmt)
-fundecl = Group(TYPE + NAME + LPAR + Optional(Group(delimitedList(arg))) + RPAR +
- LBRACE + Group(body) + RBRACE)
+fundecl = Group(
+ TYPE
+ + NAME
+ + LPAR
+ + Optional(Group(delimitedList(arg)))
+ + RPAR
+ + LBRACE
+ + Group(body)
+ + RBRACE
+)
decl = fundecl | vardecl
program = ZeroOrMore(decl)
program.ignore(cStyleComment)
# set parser element names
-for vname in ("ifstmt whilestmt dowhilestmt returnstmt TYPE "
- "NAME fundecl vardecl program arg body stmt".split()):
+for vname in (
+ "ifstmt whilestmt dowhilestmt returnstmt TYPE "
+ "NAME fundecl vardecl program arg body stmt".split()
+):
v = vars()[vname]
v.setName(vname)
-#~ for vname in "fundecl stmt".split():
- #~ v = vars()[vname]
- #~ v.setDebug()
+# ~ for vname in "fundecl stmt".split():
+# ~ v = vars()[vname]
+# ~ v.setDebug()
test = r"""
/* A factorial program */
diff --git a/examples/parseListString.py b/examples/parseListString.py index d5723b0..f34f614 100644 --- a/examples/parseListString.py +++ b/examples/parseListString.py @@ -9,8 +9,9 @@ from pyparsing import * lbrack = Literal("[")
rbrack = Literal("]")
integer = Word(nums).setName("integer")
-real = Combine(Optional(oneOf("+ -")) + Word(nums) + "." +
- Optional(Word(nums))).setName("real")
+real = Combine(
+ Optional(oneOf("+ -")) + Word(nums) + "." + Optional(Word(nums))
+).setName("real")
listItem = real | integer | quotedString
@@ -24,11 +25,11 @@ print(listStr.parseString(test)) # second pass, cleanup and add converters
lbrack = Literal("[").suppress()
rbrack = Literal("]").suppress()
-cvtInt = lambda s,l,toks: int(toks[0])
-integer = Word(nums).setName("integer").setParseAction( cvtInt )
-cvtReal = lambda s,l,toks: float(toks[0])
-real = Regex(r'[+-]?\d+\.\d*').setName("floating-point number").setParseAction( cvtReal )
-listItem = real | integer | quotedString.setParseAction( removeQuotes )
+cvtInt = lambda s, l, toks: int(toks[0])
+integer = Word(nums).setName("integer").setParseAction(cvtInt)
+cvtReal = lambda s, l, toks: float(toks[0])
+real = Regex(r"[+-]?\d+\.\d*").setName("floating-point number").setParseAction(cvtReal)
+listItem = real | integer | quotedString.setParseAction(removeQuotes)
listStr = lbrack + delimitedList(listItem) + rbrack
@@ -37,46 +38,75 @@ test = "['a', 100, 3.14]" print(listStr.parseString(test))
# third pass, add nested list support, and tuples, too!
-cvtInt = lambda s,l,toks: int(toks[0])
-cvtReal = lambda s,l,toks: float(toks[0])
+cvtInt = lambda s, l, toks: int(toks[0])
+cvtReal = lambda s, l, toks: float(toks[0])
lbrack = Literal("[").suppress()
rbrack = Literal("]").suppress()
-integer = Word(nums).setName("integer").setParseAction( cvtInt )
-real = Regex(r'[+-]?\d+\.\d*').setName("floating-point number").setParseAction( cvtReal )
+integer = Word(nums).setName("integer").setParseAction(cvtInt)
+real = Regex(r"[+-]?\d+\.\d*").setName("floating-point number").setParseAction(cvtReal)
tupleStr = Forward()
listStr = Forward()
-listItem = real | integer | quotedString.setParseAction(removeQuotes) | Group(listStr) | tupleStr
-tupleStr << ( Suppress("(") + delimitedList(listItem) + Optional(Suppress(",")) + Suppress(")") )
-tupleStr.setParseAction( lambda t:tuple(t.asList()) )
+listItem = (
+ real
+ | integer
+ | quotedString.setParseAction(removeQuotes)
+ | Group(listStr)
+ | tupleStr
+)
+tupleStr << (
+ Suppress("(") + delimitedList(listItem) + Optional(Suppress(",")) + Suppress(")")
+)
+tupleStr.setParseAction(lambda t: tuple(t.asList()))
listStr << lbrack + delimitedList(listItem) + Optional(Suppress(",")) + rbrack
test = "['a', 100, ('A', [101,102]), 3.14, [ +2.718, 'xyzzy', -1.414] ]"
print(listStr.parseString(test))
# fourth pass, add parsing of dicts
-cvtInt = lambda s,l,toks: int(toks[0])
-cvtReal = lambda s,l,toks: float(toks[0])
-cvtDict = lambda s,l,toks: dict(toks[0])
+cvtInt = lambda s, l, toks: int(toks[0])
+cvtReal = lambda s, l, toks: float(toks[0])
+cvtDict = lambda s, l, toks: dict(toks[0])
lbrack = Literal("[").suppress()
rbrack = Literal("]").suppress()
lbrace = Literal("{").suppress()
rbrace = Literal("}").suppress()
colon = Literal(":").suppress()
-integer = Word(nums).setName("integer").setParseAction( cvtInt )
-real = Regex(r'[+-]?\d+\.\d*').setName("real").setParseAction( cvtReal )
+integer = Word(nums).setName("integer").setParseAction(cvtInt)
+real = Regex(r"[+-]?\d+\.\d*").setName("real").setParseAction(cvtReal)
tupleStr = Forward()
listStr = Forward()
dictStr = Forward()
-listItem = real | integer | quotedString.setParseAction(removeQuotes) | Group(listStr) | tupleStr | dictStr
-tupleStr <<= ( Suppress("(") + delimitedList(listItem) + Optional(Suppress(",")) + Suppress(")") )
-tupleStr.setParseAction( lambda t:tuple(t.asList()) )
-listStr <<= (lbrack + Optional(delimitedList(listItem)) + Optional(Suppress(",")) + rbrack)
+listItem = (
+ real
+ | integer
+ | quotedString.setParseAction(removeQuotes)
+ | Group(listStr)
+ | tupleStr
+ | dictStr
+)
+tupleStr <<= (
+ Suppress("(") + delimitedList(listItem) + Optional(Suppress(",")) + Suppress(")")
+)
+tupleStr.setParseAction(lambda t: tuple(t.asList()))
+listStr <<= (
+ lbrack + Optional(delimitedList(listItem)) + Optional(Suppress(",")) + rbrack
+)
dictKeyStr = real | integer | quotedString.setParseAction(removeQuotes)
-dictStr <<= lbrace + Optional(delimitedList( Group( dictKeyStr + colon + listItem ))) + Optional(Suppress(",")) + rbrace
-dictStr.setParseAction(lambda t: {k_v[0]:(k_v[1].asList() if isinstance(k_v[1],ParseResults) else k_v[1]) for k_v in t})
-
-test = '[{0: [2], 1: []}, {0: [], 1: [], 2: [,]}, {0: [1, 2,],}]'
+dictStr <<= (
+ lbrace
+ + Optional(delimitedList(Group(dictKeyStr + colon + listItem)))
+ + Optional(Suppress(","))
+ + rbrace
+)
+dictStr.setParseAction(
+ lambda t: {
+ k_v[0]: (k_v[1].asList() if isinstance(k_v[1], ParseResults) else k_v[1])
+ for k_v in t
+ }
+)
+
+test = "[{0: [2], 1: []}, {0: [], 1: [], 2: [,]}, {0: [1, 2,],}]"
print(listStr.parseString(test))
diff --git a/examples/parsePythonValue.py b/examples/parsePythonValue.py index 259473f..6a75d48 100644 --- a/examples/parsePythonValue.py +++ b/examples/parsePythonValue.py @@ -5,52 +5,55 @@ import pyparsing as pp -cvtBool = lambda t:t[0]=='True' +cvtBool = lambda t: t[0] == "True" cvtInt = lambda toks: int(toks[0]) cvtReal = lambda toks: float(toks[0]) -cvtTuple = lambda toks : tuple(toks.asList()) +cvtTuple = lambda toks: tuple(toks.asList()) cvtDict = lambda toks: dict(toks.asList()) cvtList = lambda toks: [toks.asList()] # define punctuation as suppressed literals -lparen, rparen, lbrack, rbrack, lbrace, rbrace, colon, comma = map(pp.Suppress,"()[]{}:,") +lparen, rparen, lbrack, rbrack, lbrace, rbrace, colon, comma = map( + pp.Suppress, "()[]{}:," +) -integer = pp.Regex(r"[+-]?\d+").setName("integer").setParseAction(cvtInt ) +integer = pp.Regex(r"[+-]?\d+").setName("integer").setParseAction(cvtInt) real = pp.Regex(r"[+-]?\d+\.\d*([Ee][+-]?\d+)?").setName("real").setParseAction(cvtReal) tupleStr = pp.Forward() listStr = pp.Forward() dictStr = pp.Forward() -pp.unicodeString.setParseAction(lambda t:t[0][2:-1]) -pp.quotedString.setParseAction(lambda t:t[0][1:-1]) +pp.unicodeString.setParseAction(lambda t: t[0][2:-1]) +pp.quotedString.setParseAction(lambda t: t[0][1:-1]) boolLiteral = pp.oneOf("True False").setParseAction(cvtBool) noneLiteral = pp.Literal("None").setParseAction(pp.replaceWith(None)) -listItem = (real - | integer - | pp.quotedString - | pp.unicodeString - | boolLiteral - | noneLiteral - | pp.Group(listStr) - | tupleStr - | dictStr) +listItem = ( + real + | integer + | pp.quotedString + | pp.unicodeString + | boolLiteral + | noneLiteral + | pp.Group(listStr) + | tupleStr + | dictStr +) -tupleStr << (lparen - + pp.Optional(pp.delimitedList(listItem)) - + pp.Optional(comma) - + rparen) +tupleStr << ( + lparen + pp.Optional(pp.delimitedList(listItem)) + pp.Optional(comma) + rparen +) tupleStr.setParseAction(cvtTuple) -listStr << (lbrack - + pp.Optional(pp.delimitedList(listItem) + pp.Optional(comma)) - + rbrack) +listStr << ( + lbrack + pp.Optional(pp.delimitedList(listItem) + pp.Optional(comma)) + rbrack +) listStr.setParseAction(cvtList, lambda t: t[0]) dictEntry = pp.Group(listItem + colon + listItem) -dictStr << (lbrace - + pp.Optional(pp.delimitedList(dictEntry) + pp.Optional(comma)) - + rbrace) +dictStr << ( + lbrace + pp.Optional(pp.delimitedList(dictEntry) + pp.Optional(comma)) + rbrace +) dictStr.setParseAction(cvtDict) tests = """['a', 100, ('A', [101,102]), 3.14, [ +2.718, 'xyzzy', -1.414] ] diff --git a/examples/parseResultsSumExample.py b/examples/parseResultsSumExample.py index 2c0f9fc..2341b7c 100644 --- a/examples/parseResultsSumExample.py +++ b/examples/parseResultsSumExample.py @@ -10,13 +10,19 @@ samplestr3 = "garbage;DOB 10-10-2010" samplestr4 = "garbage;ID PARI12345678;more garbage- I am cool"
from pyparsing import *
+
dob_ref = "DOB" + Regex(r"\d{2}-\d{2}-\d{4}")("dob")
-id_ref = "ID" + Word(alphanums,exact=12)("id")
+id_ref = "ID" + Word(alphanums, exact=12)("id")
info_ref = "-" + restOfLine("info")
person_data = dob_ref | id_ref | info_ref
-for test in (samplestr1,samplestr2,samplestr3,samplestr4,):
+for test in (
+ samplestr1,
+ samplestr2,
+ samplestr3,
+ samplestr4,
+):
person = sum(person_data.searchString(test))
print(person.id)
print(person.dump())
diff --git a/examples/parseTabularData.py b/examples/parseTabularData.py index da19033..3c898e0 100644 --- a/examples/parseTabularData.py +++ b/examples/parseTabularData.py @@ -7,7 +7,7 @@ #
# Copyright 2015, Paul McGuire
#
-from pyparsing import col,Word,Optional,alphas,nums
+from pyparsing import col, Word, Optional, alphas, nums
table = """\
1 2
@@ -19,32 +19,41 @@ GREEN 3 5 PURPLE 8"""
# function to create column-specific parse conditions
-def mustMatchCols(startloc,endloc):
- return lambda s,l,t: startloc <= col(l,s) <= endloc
+def mustMatchCols(startloc, endloc):
+ return lambda s, l, t: startloc <= col(l, s) <= endloc
+
# helper to define values in a space-delimited table
# (change empty_cell_is_zero to True if a value of 0 is desired for empty cells)
def tableValue(expr, colstart, colend):
empty_cell_is_zero = False
if empty_cell_is_zero:
- return Optional(expr.copy().addCondition(mustMatchCols(colstart,colend),
- message="text not in expected columns"),
- default=0)
+ return Optional(
+ expr.copy().addCondition(
+ mustMatchCols(colstart, colend), message="text not in expected columns"
+ ),
+ default=0,
+ )
else:
- return Optional(expr.copy().addCondition(mustMatchCols(colstart,colend),
- message="text not in expected columns"))
+ return Optional(
+ expr.copy().addCondition(
+ mustMatchCols(colstart, colend), message="text not in expected columns"
+ )
+ )
# define the grammar for this simple table
colorname = Word(alphas)
integer = Word(nums).setParseAction(lambda t: int(t[0])).setName("integer")
-row = (colorname("name") +
- tableValue(integer, 11, 12)("S") +
- tableValue(integer, 15, 16)("M") +
- tableValue(integer, 19, 20)("L"))
+row = (
+ colorname("name")
+ + tableValue(integer, 11, 12)("S")
+ + tableValue(integer, 15, 16)("M")
+ + tableValue(integer, 19, 20)("L")
+)
# parse the sample text - skip over the header and counter lines
for line in table.splitlines()[3:]:
print(line)
print(row.parseString(line).dump())
- print('')
+ print("")
diff --git a/examples/partial_gene_match.py b/examples/partial_gene_match.py index e4c59af..fe62e77 100644 --- a/examples/partial_gene_match.py +++ b/examples/partial_gene_match.py @@ -17,12 +17,19 @@ Sample header: >NC_001799-6-2978-2778 | organism=Toxoplasma_gondii_RH | location=NC_001799:2778-2978(-) | length=201 """ integer = pp.pyparsing_common.integer -genebit = pp.Group(">" + pp.Word(pp.alphanums.upper() + "-_")("gene_id") - + "|" + pp.Word(pp.printables)("organism") - + "|" + pp.Word(pp.printables)("location") - + "|" + "length=" + integer("gene_len") - + pp.LineEnd() - + pp.Word("ACGTN")[1, ...].addParseAction(''.join)("gene")) +genebit = pp.Group( + ">" + + pp.Word(pp.alphanums.upper() + "-_")("gene_id") + + "|" + + pp.Word(pp.printables)("organism") + + "|" + + pp.Word(pp.printables)("location") + + "|" + + "length=" + + integer("gene_len") + + pp.LineEnd() + + pp.Word("ACGTN")[1, ...].addParseAction("".join)("gene") +) # read gene data from .fasta file - takes just a few seconds # An important aspect of this parsing process is the reassembly of all the separate lines of the @@ -45,12 +52,17 @@ for g in genedata: show_header = False matched = t[0] - mismatches = t['mismatches'] + mismatches = t["mismatches"] print("MATCH:", searchseq.match_string) print("FOUND:", matched) if mismatches: - print(" ", ''.join('*' if i in mismatches else ' ' - for i, c in enumerate(searchseq.match_string))) + print( + " ", + "".join( + "*" if i in mismatches else " " + for i, c in enumerate(searchseq.match_string) + ), + ) else: print("<exact match>") print("at location", startLoc) diff --git a/examples/pgn.py b/examples/pgn.py index fc19c98..d9889d6 100644 --- a/examples/pgn.py +++ b/examples/pgn.py @@ -10,7 +10,18 @@ # Copyright 2004, by Alberto Santini http://www.albertosantini.it/chess/
#
from pyparsing import alphanums, nums, quotedString
-from pyparsing import Combine, Forward, Group, Literal, oneOf, OneOrMore, Optional, Suppress, ZeroOrMore, Word
+from pyparsing import (
+ Combine,
+ Forward,
+ Group,
+ Literal,
+ oneOf,
+ OneOrMore,
+ Optional,
+ Suppress,
+ ZeroOrMore,
+ Word,
+)
from pyparsing import ParseException
#
@@ -30,14 +41,14 @@ castle_queenside = oneOf("O-O-O 0-0-0 o-o-o") castle_kingside = oneOf("O-O 0-0 o-o")
move_number = Optional(comment) + Word(nums) + dot
-m1 = file_coord + rank_coord # pawn move e.g. d4
-m2 = file_coord + capture + file_coord + rank_coord # pawn capture move e.g. dxe5
-m3 = file_coord + "8" + promote + piece # pawn promotion e.g. e8=Q
-m4 = piece + file_coord + rank_coord # piece move e.g. Be6
-m5 = piece + file_coord + file_coord + rank_coord # piece move e.g. Nbd2
-m6 = piece + rank_coord + file_coord + rank_coord # piece move e.g. R4a7
-m7 = piece + capture + file_coord + rank_coord # piece capture move e.g. Bxh7
-m8 = castle_queenside | castle_kingside # castling e.g. o-o
+m1 = file_coord + rank_coord # pawn move e.g. d4
+m2 = file_coord + capture + file_coord + rank_coord # pawn capture move e.g. dxe5
+m3 = file_coord + "8" + promote + piece # pawn promotion e.g. e8=Q
+m4 = piece + file_coord + rank_coord # piece move e.g. Be6
+m5 = piece + file_coord + file_coord + rank_coord # piece move e.g. Nbd2
+m6 = piece + rank_coord + file_coord + rank_coord # piece move e.g. R4a7
+m7 = piece + capture + file_coord + rank_coord # piece capture move e.g. Bxh7
+m8 = castle_queenside | castle_kingside # castling e.g. o-o
check = oneOf("+ ++")
mate = Literal("#")
@@ -46,8 +57,11 @@ nag = " $" + Word(nums) decoration = check | mate | annotation | nag
variant = Forward()
-half_move = Combine((m3 | m1 | m2 | m4 | m5 | m6 | m7 | m8) + Optional(decoration)) \
- + Optional(comment) +Optional(variant)
+half_move = (
+ Combine((m3 | m1 | m2 | m4 | m5 | m6 | m7 | m8) + Optional(decoration))
+ + Optional(comment)
+ + Optional(variant)
+)
move = Suppress(move_number) + half_move + Optional(half_move)
variant << "(" + OneOrMore(move) + ")"
# grouping the plies (half-moves) for each move: useful to group annotations, variants...
@@ -56,19 +70,23 @@ move = Group(Suppress(move_number) + half_move + Optional(half_move)) variant << Group("(" + OneOrMore(move) + ")")
game_terminator = oneOf("1-0 0-1 1/2-1/2 *")
-pgnGrammar = Suppress(ZeroOrMore(tag)) + ZeroOrMore(move) + Optional(Suppress(game_terminator))
+pgnGrammar = (
+ Suppress(ZeroOrMore(tag)) + ZeroOrMore(move) + Optional(Suppress(game_terminator))
+)
+
+
+def parsePGN(pgn, bnf=pgnGrammar, fn=None):
+ try:
+ return bnf.parseString(pgn)
+ except ParseException as err:
+ print(err.line)
+ print(" " * (err.column - 1) + "^")
+ print(err)
-def parsePGN( pgn, bnf=pgnGrammar, fn=None ):
- try:
- return bnf.parseString( pgn )
- except ParseException as err:
- print(err.line)
- print(" "*(err.column-1) + "^")
- print(err)
if __name__ == "__main__":
- # input string
- pgn = """
+ # input string
+ pgn = """
[Event "ICC 5 0 u"]
[Site "Internet Chess Club"]
[Date "2004.01.25"]
@@ -89,6 +107,6 @@ Bg4 8. Nbd2 c5 9. h3 Be6 10. O-O-O Nc6 11. g4 Bd6 12. g5 Nd7 13. Rg1 d4 14. g6 fxg6 15. Bg5 Rf8 16. a3 Bd5 17. Re1+ Nde5 18. Nxe5 Nxe5 19. Bf4 Rf5 20.
Bxe5 Rxe5 21. Rg5 Rxe1# {Black wins} 0-1
"""
- # parse input string
- tokens = parsePGN(pgn, pgnGrammar)
- print(tokens.dump())
+ # parse input string
+ tokens = parsePGN(pgn, pgnGrammar)
+ print(tokens.dump())
diff --git a/examples/position.py b/examples/position.py index d88c14a..a6f03b9 100644 --- a/examples/position.py +++ b/examples/position.py @@ -11,7 +11,7 @@ mollit anim id est laborum""" # find all words beginning with a vowel
vowels = "aeiouAEIOU"
-initialVowelWord = Word(vowels,alphas)
+initialVowelWord = Word(vowels, alphas)
# Unfortunately, searchString will advance character by character through
# the input text, so it will detect that the initial "Lorem" is not an
@@ -20,34 +20,38 @@ initialVowelWord = Word(vowels,alphas) # consonants, but we will just throw them away when we match them. The key is
# that, in having been matched, the parser will skip over them entirely when
# looking for initialVowelWords.
-consonants = ''.join(c for c in alphas if c not in vowels)
+consonants = "".join(c for c in alphas if c not in vowels)
initialConsWord = Word(consonants, alphas).suppress()
# using scanString to locate where tokens are matched
-for t,start,end in (initialConsWord|initialVowelWord).scanString(text):
+for t, start, end in (initialConsWord | initialVowelWord).scanString(text):
if t:
- print(start,':', t[0])
+ print(start, ":", t[0])
# add parse action to annotate the parsed tokens with their location in the
# input string
-def addLocnToTokens(s,l,t):
- t['locn'] = l
- t['word'] = t[0]
+def addLocnToTokens(s, l, t):
+ t["locn"] = l
+ t["word"] = t[0]
+
+
initialVowelWord.setParseAction(addLocnToTokens)
for ivowelInfo in (initialConsWord | initialVowelWord).searchString(text):
if not ivowelInfo:
continue
- print(ivowelInfo.locn, ':', ivowelInfo.word)
+ print(ivowelInfo.locn, ":", ivowelInfo.word)
# alternative - add an Empty that will save the current location
def location(name):
- return Empty().setParseAction(lambda s,l,t: t.__setitem__(name,l))
+ return Empty().setParseAction(lambda s, l, t: t.__setitem__(name, l))
+
+
locateInitialVowels = location("locn") + initialVowelWord("word")
# search through the input text
for ivowelInfo in (initialConsWord | locateInitialVowels).searchString(text):
if not ivowelInfo:
continue
- print(ivowelInfo.locn, ':', ivowelInfo.word)
+ print(ivowelInfo.locn, ":", ivowelInfo.word)
diff --git a/examples/protobuf_parser.py b/examples/protobuf_parser.py index ae5b5d3..afc8296 100644 --- a/examples/protobuf_parser.py +++ b/examples/protobuf_parser.py @@ -5,14 +5,27 @@ # Copyright 2010, Paul McGuire # -from pyparsing import (Word, alphas, alphanums, Regex, Suppress, Forward, - Group, oneOf, ZeroOrMore, Optional, delimitedList, - restOfLine, quotedString, Dict) - -ident = Word(alphas+"_",alphanums+"_").setName("identifier") +from pyparsing import ( + Word, + alphas, + alphanums, + Regex, + Suppress, + Forward, + Group, + oneOf, + ZeroOrMore, + Optional, + delimitedList, + restOfLine, + quotedString, + Dict, +) + +ident = Word(alphas + "_", alphanums + "_").setName("identifier") integer = Regex(r"[+-]?\d+") -LBRACE,RBRACE,LBRACK,RBRACK,LPAR,RPAR,EQ,SEMI = map(Suppress,"{}[]()=;") +LBRACE, RBRACE, LBRACK, RBRACK, LPAR, RPAR, EQ, SEMI = map(Suppress, "{}[]()=;") kwds = """message required optional repeated enum extensions extends extend to package service rpc returns true false option import syntax""" @@ -23,15 +36,34 @@ messageBody = Forward() messageDefn = MESSAGE_ - ident("messageId") + LBRACE + messageBody("body") + RBRACE -typespec = oneOf("""double float int32 int64 uint32 uint64 sint32 sint64 - fixed32 fixed64 sfixed32 sfixed64 bool string bytes""") | ident +typespec = ( + oneOf( + """double float int32 int64 uint32 uint64 sint32 sint64 + fixed32 fixed64 sfixed32 sfixed64 bool string bytes""" + ) + | ident +) rvalue = integer | TRUE_ | FALSE_ | ident fieldDirective = LBRACK + Group(ident + EQ + rvalue) + RBRACK -fieldDefnPrefix = REQUIRED_ | OPTIONAL_ | REPEATED_ -fieldDefn = (Optional(fieldDefnPrefix) + typespec("typespec") + ident("ident") + EQ + integer("fieldint") + ZeroOrMore(fieldDirective) + SEMI) +fieldDefnPrefix = REQUIRED_ | OPTIONAL_ | REPEATED_ +fieldDefn = ( + Optional(fieldDefnPrefix) + + typespec("typespec") + + ident("ident") + + EQ + + integer("fieldint") + + ZeroOrMore(fieldDirective) + + SEMI +) # enumDefn ::= 'enum' ident '{' { ident '=' integer ';' }* '}' -enumDefn = ENUM_("typespec") - ident('name') + LBRACE + Dict( ZeroOrMore( Group(ident + EQ + integer + SEMI) ))('values') + RBRACE +enumDefn = ( + ENUM_("typespec") + - ident("name") + + LBRACE + + Dict(ZeroOrMore(Group(ident + EQ + integer + SEMI)))("values") + + RBRACE +) # extensionsDefn ::= 'extensions' integer 'to' integer ';' extensionsDefn = EXTENSIONS_ - integer + TO_ + integer + SEMI @@ -40,28 +72,52 @@ extensionsDefn = EXTENSIONS_ - integer + TO_ + integer + SEMI messageExtension = EXTEND_ - ident + LBRACE + messageBody + RBRACE # messageBody ::= { fieldDefn | enumDefn | messageDefn | extensionsDefn | messageExtension }* -messageBody << Group(ZeroOrMore( Group(fieldDefn | enumDefn | messageDefn | extensionsDefn | messageExtension) )) +messageBody << Group( + ZeroOrMore( + Group(fieldDefn | enumDefn | messageDefn | extensionsDefn | messageExtension) + ) +) # methodDefn ::= 'rpc' ident '(' [ ident ] ')' 'returns' '(' [ ident ] ')' ';' -methodDefn = (RPC_ - ident("methodName") + - LPAR + Optional(ident("methodParam")) + RPAR + - RETURNS_ + LPAR + Optional(ident("methodReturn")) + RPAR) +methodDefn = ( + RPC_ + - ident("methodName") + + LPAR + + Optional(ident("methodParam")) + + RPAR + + RETURNS_ + + LPAR + + Optional(ident("methodReturn")) + + RPAR +) # serviceDefn ::= 'service' ident '{' methodDefn* '}' -serviceDefn = SERVICE_ - ident("serviceName") + LBRACE + ZeroOrMore(Group(methodDefn)) + RBRACE +serviceDefn = ( + SERVICE_ - ident("serviceName") + LBRACE + ZeroOrMore(Group(methodDefn)) + RBRACE +) syntaxDefn = SYNTAX_ + EQ - quotedString("syntaxString") + SEMI # packageDirective ::= 'package' ident [ '.' ident]* ';' -packageDirective = Group(PACKAGE_ - delimitedList(ident, '.', combine=True) + SEMI) +packageDirective = Group(PACKAGE_ - delimitedList(ident, ".", combine=True) + SEMI) -comment = '//' + restOfLine +comment = "//" + restOfLine importDirective = IMPORT_ - quotedString("importFileSpec") + SEMI -optionDirective = OPTION_ - ident("optionName") + EQ + quotedString("optionValue") + SEMI - -topLevelStatement = Group(messageDefn | messageExtension | enumDefn | serviceDefn | importDirective | optionDirective | syntaxDefn) +optionDirective = ( + OPTION_ - ident("optionName") + EQ + quotedString("optionValue") + SEMI +) + +topLevelStatement = Group( + messageDefn + | messageExtension + | enumDefn + | serviceDefn + | importDirective + | optionDirective + | syntaxDefn +) parser = Optional(packageDirective) + ZeroOrMore(topLevelStatement) diff --git a/examples/pymicko.py b/examples/pymicko.py index eddfdd3..a5512ea 100644 --- a/examples/pymicko.py +++ b/examples/pymicko.py @@ -20,7 +20,7 @@ from pyparsing import *
from sys import stdin, argv, exit
-#defines debug level
+# defines debug level
# 0 - no debug
# 1 - print parsing results
# 2 - print parsing results and symbol table
@@ -204,56 +204,63 @@ DEBUG = 0 ##########################################################################################
##########################################################################################
+
class Enumerate(dict):
"""C enum emulation (original by Scott David Daniels)"""
+
def __init__(self, names):
for number, name in enumerate(names.split()):
setattr(self, name, number)
self[number] = name
+
class SharedData:
"""Data used in all three main classes"""
- #Possible kinds of symbol table entries
- KINDS = Enumerate("NO_KIND WORKING_REGISTER GLOBAL_VAR FUNCTION PARAMETER LOCAL_VAR CONSTANT")
- #Supported types of functions and variables
+ # Possible kinds of symbol table entries
+ KINDS = Enumerate(
+ "NO_KIND WORKING_REGISTER GLOBAL_VAR FUNCTION PARAMETER LOCAL_VAR CONSTANT"
+ )
+ # Supported types of functions and variables
TYPES = Enumerate("NO_TYPE INT UNSIGNED")
- #bit size of variables
+ # bit size of variables
TYPE_BIT_SIZE = 16
- #min/max values of constants
- MIN_INT = -2 ** (TYPE_BIT_SIZE - 1)
+ # min/max values of constants
+ MIN_INT = -(2 ** (TYPE_BIT_SIZE - 1))
MAX_INT = 2 ** (TYPE_BIT_SIZE - 1) - 1
MAX_UNSIGNED = 2 ** TYPE_BIT_SIZE - 1
- #available working registers (the last one is the register for function's return value!)
+ # available working registers (the last one is the register for function's return value!)
REGISTERS = "%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13".split()
- #register for function's return value
+ # register for function's return value
FUNCTION_REGISTER = len(REGISTERS) - 1
- #the index of last working register
+ # the index of last working register
LAST_WORKING_REGISTER = len(REGISTERS) - 2
- #list of relational operators
+ # list of relational operators
RELATIONAL_OPERATORS = "< > <= >= == !=".split()
def __init__(self):
- #index of the currently parsed function
+ # index of the currently parsed function
self.functon_index = 0
- #name of the currently parsed function
+ # name of the currently parsed function
self.functon_name = 0
- #number of parameters of the currently parsed function
+ # number of parameters of the currently parsed function
self.function_params = 0
- #number of local variables of the currently parsed function
+ # number of local variables of the currently parsed function
self.function_vars = 0
+
##########################################################################################
##########################################################################################
+
class ExceptionSharedData:
"""Class for exception handling data"""
def __init__(self):
- #position in currently parsed text
+ # position in currently parsed text
self.location = 0
- #currently parsed text
+ # currently parsed text
self.text = ""
def setpos(self, location, text):
@@ -261,8 +268,10 @@ class ExceptionSharedData: self.location = location
self.text = text
+
exshared = ExceptionSharedData()
+
class SemanticException(Exception):
"""Exception for semantic errors found during parsing, similar to ParseException.
Introduced because ParseException is used internally in pyparsing and custom
@@ -283,8 +292,10 @@ class SemanticException(Exception): def _get_message(self):
return self._message
+
def _set_message(self, message):
self._message = message
+
message = property(_get_message, _set_message)
def __str__(self):
@@ -297,13 +308,15 @@ class SemanticException(Exception): msg += "\n%s" % self.text
return msg
+
##########################################################################################
##########################################################################################
+
class SymbolTableEntry:
"""Class which represents one symbol table entry."""
- def __init__(self, sname = "", skind = 0, stype = 0, sattr = None, sattr_name = "None"):
+ def __init__(self, sname="", skind=0, stype=0, sattr=None, sattr_name="None"):
"""Initialization of symbol table entry.
sname - symbol name
skind - symbol kind
@@ -325,7 +338,12 @@ class SymbolTableEntry: def attribute_str(self):
"""Returns attribute string (used only for table display)"""
- return "{}={}".format(self.attribute_name, self.attribute) if self.attribute != None else "None"
+ return (
+ "{}={}".format(self.attribute_name, self.attribute)
+ if self.attribute != None
+ else "None"
+ )
+
class SymbolTable:
"""Class for symbol table of microC program"""
@@ -334,10 +352,14 @@ class SymbolTable: """Initialization of the symbol table"""
self.table = []
self.lable_len = 0
- #put working registers in the symbol table
- for reg in range(SharedData.FUNCTION_REGISTER+1):
- self.insert_symbol(SharedData.REGISTERS[reg], SharedData.KINDS.WORKING_REGISTER, SharedData.TYPES.NO_TYPE)
- #shared data
+ # put working registers in the symbol table
+ for reg in range(SharedData.FUNCTION_REGISTER + 1):
+ self.insert_symbol(
+ SharedData.REGISTERS[reg],
+ SharedData.KINDS.WORKING_REGISTER,
+ SharedData.TYPES.NO_TYPE,
+ )
+ # shared data
self.shared = shared
def error(self, text=""):
@@ -351,27 +373,60 @@ class SymbolTable: def display(self):
"""Displays the symbol table content"""
- #Finding the maximum length for each column
+ # Finding the maximum length for each column
sym_name = "Symbol name"
- sym_len = max(max(len(i.name) for i in self.table),len(sym_name))
+ sym_len = max(max(len(i.name) for i in self.table), len(sym_name))
kind_name = "Kind"
- kind_len = max(max(len(SharedData.KINDS[i.kind]) for i in self.table),len(kind_name))
+ kind_len = max(
+ max(len(SharedData.KINDS[i.kind]) for i in self.table), len(kind_name)
+ )
type_name = "Type"
- type_len = max(max(len(SharedData.TYPES[i.type]) for i in self.table),len(type_name))
+ type_len = max(
+ max(len(SharedData.TYPES[i.type]) for i in self.table), len(type_name)
+ )
attr_name = "Attribute"
- attr_len = max(max(len(i.attribute_str()) for i in self.table),len(attr_name))
- #print table header
- print("{0:3s} | {1:^{2}s} | {3:^{4}s} | {5:^{6}s} | {7:^{8}} | {9:s}".format(" No", sym_name, sym_len, kind_name, kind_len, type_name, type_len, attr_name, attr_len, "Parameters"))
- print("-----------------------------" + "-" * (sym_len + kind_len + type_len + attr_len))
- #print symbol table
- for i,sym in enumerate(self.table):
+ attr_len = max(max(len(i.attribute_str()) for i in self.table), len(attr_name))
+ # print table header
+ print(
+ "{0:3s} | {1:^{2}s} | {3:^{4}s} | {5:^{6}s} | {7:^{8}} | {9:s}".format(
+ " No",
+ sym_name,
+ sym_len,
+ kind_name,
+ kind_len,
+ type_name,
+ type_len,
+ attr_name,
+ attr_len,
+ "Parameters",
+ )
+ )
+ print(
+ "-----------------------------"
+ + "-" * (sym_len + kind_len + type_len + attr_len)
+ )
+ # print symbol table
+ for i, sym in enumerate(self.table):
parameters = ""
for p in sym.param_types:
if parameters == "":
parameters = "{}".format(SharedData.TYPES[p])
else:
parameters += ", {}".format(SharedData.TYPES[p])
- print("{0:3d} | {1:^{2}s} | {3:^{4}s} | {5:^{6}s} | {7:^{8}} | ({9})".format(i, sym.name, sym_len, SharedData.KINDS[sym.kind], kind_len, SharedData.TYPES[sym.type], type_len, sym.attribute_str(), attr_len, parameters))
+ print(
+ "{0:3d} | {1:^{2}s} | {3:^{4}s} | {5:^{6}s} | {7:^{8}} | ({9})".format(
+ i,
+ sym.name,
+ sym_len,
+ SharedData.KINDS[sym.kind],
+ kind_len,
+ SharedData.TYPES[sym.type],
+ type_len,
+ sym.attribute_str(),
+ attr_len,
+ parameters,
+ )
+ )
def insert_symbol(self, sname, skind, stype):
"""Inserts new symbol at the end of the symbol table.
@@ -382,7 +437,7 @@ class SymbolTable: """
self.table.append(SymbolTableEntry(sname, skind, stype))
self.table_len = len(self.table)
- return self.table_len-1
+ return self.table_len - 1
def clear_symbols(self, index):
"""Clears all symbols begining with the index to the end of table"""
@@ -392,7 +447,12 @@ class SymbolTable: self.error()
self.table_len = len(self.table)
- def lookup_symbol(self, sname, skind=list(SharedData.KINDS.keys()), stype=list(SharedData.TYPES.keys())):
+ def lookup_symbol(
+ self,
+ sname,
+ skind=list(SharedData.KINDS.keys()),
+ stype=list(SharedData.TYPES.keys()),
+ ):
"""Searches for symbol, from the end to the begining.
Returns symbol index or None
sname - symbol name
@@ -401,7 +461,10 @@ class SymbolTable: """
skind = skind if isinstance(skind, list) else [skind]
stype = stype if isinstance(stype, list) else [stype]
- for i, sym in [[x, self.table[x]] for x in range(len(self.table) - 1, SharedData.LAST_WORKING_REGISTER, -1)]:
+ for i, sym in [
+ [x, self.table[x]]
+ for x in range(len(self.table) - 1, SharedData.LAST_WORKING_REGISTER, -1)
+ ]:
if (sym.name == sname) and (sym.kind in skind) and (sym.type in stype):
return i
return None
@@ -423,26 +486,43 @@ class SymbolTable: def insert_global_var(self, vname, vtype):
"Inserts a new global variable"
- return self.insert_id(vname, SharedData.KINDS.GLOBAL_VAR, [SharedData.KINDS.GLOBAL_VAR, SharedData.KINDS.FUNCTION], vtype)
+ return self.insert_id(
+ vname,
+ SharedData.KINDS.GLOBAL_VAR,
+ [SharedData.KINDS.GLOBAL_VAR, SharedData.KINDS.FUNCTION],
+ vtype,
+ )
def insert_local_var(self, vname, vtype, position):
"Inserts a new local variable"
- index = self.insert_id(vname, SharedData.KINDS.LOCAL_VAR, [SharedData.KINDS.LOCAL_VAR, SharedData.KINDS.PARAMETER], vtype)
+ index = self.insert_id(
+ vname,
+ SharedData.KINDS.LOCAL_VAR,
+ [SharedData.KINDS.LOCAL_VAR, SharedData.KINDS.PARAMETER],
+ vtype,
+ )
self.table[index].attribute = position
def insert_parameter(self, pname, ptype):
"Inserts a new parameter"
- index = self.insert_id(pname, SharedData.KINDS.PARAMETER, SharedData.KINDS.PARAMETER, ptype)
- #set parameter's attribute to it's ordinal number
+ index = self.insert_id(
+ pname, SharedData.KINDS.PARAMETER, SharedData.KINDS.PARAMETER, ptype
+ )
+ # set parameter's attribute to it's ordinal number
self.table[index].set_attribute("Index", self.shared.function_params)
- #set parameter's type in param_types list of a function
+ # set parameter's type in param_types list of a function
self.table[self.shared.function_index].param_types.append(ptype)
return index
def insert_function(self, fname, ftype):
"Inserts a new function"
- index = self.insert_id(fname, SharedData.KINDS.FUNCTION, [SharedData.KINDS.GLOBAL_VAR, SharedData.KINDS.FUNCTION], ftype)
- self.table[index].set_attribute("Params",0)
+ index = self.insert_id(
+ fname,
+ SharedData.KINDS.FUNCTION,
+ [SharedData.KINDS.GLOBAL_VAR, SharedData.KINDS.FUNCTION],
+ ftype,
+ )
+ self.table[index].set_attribute("Params", 0)
return index
def insert_constant(self, cname, ctype):
@@ -454,17 +534,25 @@ class SymbolTable: num = int(cname)
if ctype == SharedData.TYPES.INT:
if (num < SharedData.MIN_INT) or (num > SharedData.MAX_INT):
- raise SemanticException("Integer constant '%s' out of range" % cname)
+ raise SemanticException(
+ "Integer constant '%s' out of range" % cname
+ )
elif ctype == SharedData.TYPES.UNSIGNED:
if (num < 0) or (num > SharedData.MAX_UNSIGNED):
- raise SemanticException("Unsigned constant '%s' out of range" % cname)
+ raise SemanticException(
+ "Unsigned constant '%s' out of range" % cname
+ )
index = self.insert_symbol(cname, SharedData.KINDS.CONSTANT, ctype)
return index
def same_types(self, index1, index2):
"""Returns True if both symbol table elements are of the same type"""
try:
- same = self.table[index1].type == self.table[index2].type != SharedData.TYPES.NO_TYPE
+ same = (
+ self.table[index1].type
+ == self.table[index2].type
+ != SharedData.TYPES.NO_TYPE
+ )
except Exception:
self.error()
return same
@@ -476,7 +564,10 @@ class SymbolTable: argument_number - # of function's argument
"""
try:
- same = self.table[function_index].param_types[argument_number] == self.table[index].type
+ same = (
+ self.table[function_index].param_types[argument_number]
+ == self.table[index].type
+ )
except Exception:
self.error()
return same
@@ -517,45 +608,75 @@ class SymbolTable: except Exception:
self.error()
+
##########################################################################################
##########################################################################################
+
class CodeGenerator:
"""Class for code generation methods."""
- #dictionary of relational operators
- RELATIONAL_DICT = {op:i for i, op in enumerate(SharedData.RELATIONAL_OPERATORS)}
- #conditional jumps for relational operators
- CONDITIONAL_JUMPS = ["JLTS", "JGTS", "JLES", "JGES", "JEQ ", "JNE ",
- "JLTU", "JGTU", "JLEU", "JGEU", "JEQ ", "JNE "]
- #opposite conditional jumps for relational operators
- OPPOSITE_JUMPS = ["JGES", "JLES", "JGTS", "JLTS", "JNE ", "JEQ ",
- "JGEU", "JLEU", "JGTU", "JLTU", "JNE ", "JEQ "]
- #supported operations
- OPERATIONS = {"+" : "ADD", "-" : "SUB", "*" : "MUL", "/" : "DIV"}
- #suffixes for signed and unsigned operations (if no type is specified, unsigned will be assumed)
- OPSIGNS = {SharedData.TYPES.NO_TYPE : "U", SharedData.TYPES.INT : "S", SharedData.TYPES.UNSIGNED : "U"}
- #text at start of data segment
+ # dictionary of relational operators
+ RELATIONAL_DICT = {op: i for i, op in enumerate(SharedData.RELATIONAL_OPERATORS)}
+ # conditional jumps for relational operators
+ CONDITIONAL_JUMPS = [
+ "JLTS",
+ "JGTS",
+ "JLES",
+ "JGES",
+ "JEQ ",
+ "JNE ",
+ "JLTU",
+ "JGTU",
+ "JLEU",
+ "JGEU",
+ "JEQ ",
+ "JNE ",
+ ]
+ # opposite conditional jumps for relational operators
+ OPPOSITE_JUMPS = [
+ "JGES",
+ "JLES",
+ "JGTS",
+ "JLTS",
+ "JNE ",
+ "JEQ ",
+ "JGEU",
+ "JLEU",
+ "JGTU",
+ "JLTU",
+ "JNE ",
+ "JEQ ",
+ ]
+ # supported operations
+ OPERATIONS = {"+": "ADD", "-": "SUB", "*": "MUL", "/": "DIV"}
+ # suffixes for signed and unsigned operations (if no type is specified, unsigned will be assumed)
+ OPSIGNS = {
+ SharedData.TYPES.NO_TYPE: "U",
+ SharedData.TYPES.INT: "S",
+ SharedData.TYPES.UNSIGNED: "U",
+ }
+ # text at start of data segment
DATA_START_TEXT = "#DATA"
- #text at start of code segment
+ # text at start of code segment
CODE_START_TEXT = "#CODE"
def __init__(self, shared, symtab):
- #generated code
+ # generated code
self.code = ""
- #prefix for internal labels
+ # prefix for internal labels
self.internal = "@"
- #suffix for label definition
+ # suffix for label definition
self.definition = ":"
- #list of free working registers
+ # list of free working registers
self.free_registers = list(range(SharedData.FUNCTION_REGISTER, -1, -1))
- #list of used working registers
+ # list of used working registers
self.used_registers = []
- #list of used registers needed when function call is inside of a function call
+ # list of used registers needed when function call is inside of a function call
self.used_registers_stack = []
- #shared data
+ # shared data
self.shared = shared
- #symbol table
+ # symbol table
self.symtab = symtab
def error(self, text):
@@ -564,7 +685,7 @@ class CodeGenerator: """
raise Exception("Compiler error: %s" % text)
- def take_register(self, rtype = SharedData.TYPES.NO_TYPE):
+ def take_register(self, rtype=SharedData.TYPES.NO_TYPE):
"""Reserves one working register and sets its type"""
if len(self.free_registers) == 0:
self.error("no more free registers")
@@ -573,7 +694,7 @@ class CodeGenerator: self.symtab.set_type(reg, rtype)
return reg
- def take_function_register(self, rtype = SharedData.TYPES.NO_TYPE):
+ def take_function_register(self, rtype=SharedData.TYPES.NO_TYPE):
"""Reserves register for function return value and sets its type"""
reg = SharedData.FUNCTION_REGISTER
if reg not in self.free_registers:
@@ -589,7 +710,7 @@ class CodeGenerator: self.error("register %s is not taken" % self.REGISTERS[reg])
self.used_registers.remove(reg)
self.free_registers.append(reg)
- self.free_registers.sort(reverse = True)
+ self.free_registers.sort(reverse=True)
def free_if_register(self, index):
"""If index is a working register, free it, otherwise just return (helper function)"""
@@ -604,20 +725,24 @@ class CodeGenerator: internal - boolean value, adds "@" prefix to label
definition - boolean value, adds ":" suffix to label
"""
- return "{}{}{}".format(self.internal if internal else "", name, self.definition if definition else "")
+ return "{}{}{}".format(
+ self.internal if internal else "",
+ name,
+ self.definition if definition else "",
+ )
def symbol(self, index):
"""Generates symbol name from index"""
- #if index is actually a string, just return it
+ # if index is actually a string, just return it
if isinstance(index, str):
return index
elif (index < 0) or (index >= self.symtab.table_len):
self.error("symbol table index out of range")
sym = self.symtab.table[index]
- #local variables are located at negative offset from frame pointer register
+ # local variables are located at negative offset from frame pointer register
if sym.kind == SharedData.KINDS.LOCAL_VAR:
return "-{}(1:%14)".format(sym.attribute * 4 + 4)
- #parameters are located at positive offset from frame pointer register
+ # parameters are located at positive offset from frame pointer register
elif sym.kind == SharedData.KINDS.PARAMETER:
return "{}(1:%14)".format(8 + sym.attribute * 4)
elif sym.kind == SharedData.KINDS.CONSTANT:
@@ -634,13 +759,13 @@ class CodeGenerator: for reg in used:
self.newline_text("PUSH\t%s" % SharedData.REGISTERS[reg], True)
self.free_registers.extend(used)
- self.free_registers.sort(reverse = True)
+ self.free_registers.sort(reverse=True)
def restore_used_registers(self):
"""Pops all used working registers after function call"""
used = self.used_registers_stack.pop()
self.used_registers = used[:]
- used.sort(reverse = True)
+ used.sort(reverse=True)
for reg in used:
self.newline_text("POP \t%s" % SharedData.REGISTERS[reg], True)
self.free_registers.remove(reg)
@@ -663,7 +788,7 @@ class CodeGenerator: if indent:
self.text("\t\t\t")
- def newline_text(self, text, indent = False):
+ def newline_text(self, text, indent=False):
"""Inserts a newline and text, optionally with indentation (helper function)"""
self.newline(indent)
self.text(text)
@@ -674,7 +799,13 @@ class CodeGenerator: internal - boolean value, adds "@" prefix to label
definition - boolean value, adds ":" suffix to label
"""
- self.newline_text(self.label("{}{}{}".format("@" if internal else "", name, ":" if definition else "")))
+ self.newline_text(
+ self.label(
+ "{}{}{}".format(
+ "@" if internal else "", name, ":" if definition else ""
+ )
+ )
+ )
def global_var(self, name):
"""Inserts a new static (global) variable definition"""
@@ -685,7 +816,7 @@ class CodeGenerator: """Generates an arithmetic instruction mnemonic"""
return self.OPERATIONS[op_name] + self.OPSIGNS[op_type]
- def arithmetic(self, operation, operand1, operand2, operand3 = None):
+ def arithmetic(self, operation, operand1, operand2, operand3=None):
"""Generates an arithmetic instruction
operation - one of supporetd operations
operandX - index in symbol table or text representation of operand
@@ -697,14 +828,26 @@ class CodeGenerator: else:
output_type = None
if isinstance(operand2, int):
- output_type = self.symtab.get_type(operand2) if output_type == None else output_type
+ output_type = (
+ self.symtab.get_type(operand2) if output_type == None else output_type
+ )
self.free_if_register(operand2)
else:
- output_type = SharedData.TYPES.NO_TYPE if output_type == None else output_type
- #if operand3 is not defined, reserve one free register for it
+ output_type = (
+ SharedData.TYPES.NO_TYPE if output_type == None else output_type
+ )
+ # if operand3 is not defined, reserve one free register for it
output = self.take_register(output_type) if operand3 == None else operand3
mnemonic = self.arithmetic_mnemonic(operation, output_type)
- self.newline_text("{}\t{},{},{}".format(mnemonic, self.symbol(operand1), self.symbol(operand2), self.symbol(output)), True)
+ self.newline_text(
+ "{}\t{},{},{}".format(
+ mnemonic,
+ self.symbol(operand1),
+ self.symbol(operand2),
+ self.symbol(output),
+ ),
+ True,
+ )
return output
def relop_code(self, relop, operands_type):
@@ -713,7 +856,11 @@ class CodeGenerator: operands_type - int or unsigned
"""
code = self.RELATIONAL_DICT[relop]
- offset = 0 if operands_type == SharedData.TYPES.INT else len(SharedData.RELATIONAL_OPERATORS)
+ offset = (
+ 0
+ if operands_type == SharedData.TYPES.INT
+ else len(SharedData.RELATIONAL_OPERATORS)
+ )
return code + offset
def jump(self, relcode, opposite, label):
@@ -722,7 +869,11 @@ class CodeGenerator: opposite - generate normal or opposite jump
label - jump label
"""
- jump = self.OPPOSITE_JUMPS[relcode] if opposite else self.CONDITIONAL_JUMPS[relcode]
+ jump = (
+ self.OPPOSITE_JUMPS[relcode]
+ if opposite
+ else self.CONDITIONAL_JUMPS[relcode]
+ )
self.newline_text("{}\t{}".format(jump, label), True)
def unconditional_jump(self, label):
@@ -731,7 +882,7 @@ class CodeGenerator: """
self.newline_text("JMP \t{}".format(label), True)
- def move(self,operand1, operand2):
+ def move(self, operand1, operand2):
"""Generates a move instruction
If the output operand (opernad2) is a working register, sets it's type
operandX - index in symbol table or text representation of operand
@@ -741,7 +892,9 @@ class CodeGenerator: self.free_if_register(operand1)
else:
output_type = SharedData.TYPES.NO_TYPE
- self.newline_text("MOV \t{},{}".format(self.symbol(operand1), self.symbol(operand2)), True)
+ self.newline_text(
+ "MOV \t{},{}".format(self.symbol(operand1), self.symbol(operand2)), True
+ )
if isinstance(operand2, int):
if self.symtab.get_kind(operand2) == SharedData.KINDS.WORKING_REGISTER:
self.symtab.set_type(operand2, output_type)
@@ -761,7 +914,12 @@ class CodeGenerator: typ = self.symtab.get_type(operand1)
self.free_if_register(operand1)
self.free_if_register(operand2)
- self.newline_text("CMP{}\t{},{}".format(self.OPSIGNS[typ], self.symbol(operand1), self.symbol(operand2)), True)
+ self.newline_text(
+ "CMP{}\t{},{}".format(
+ self.OPSIGNS[typ], self.symbol(operand1), self.symbol(operand2)
+ ),
+ True,
+ )
def function_begin(self):
"""Inserts function name label and function frame initialization"""
@@ -772,7 +930,9 @@ class CodeGenerator: def function_body(self):
"""Inserts a local variable initialization and body label"""
if self.shared.function_vars > 0:
- const = self.symtab.insert_constant("0{}".format(self.shared.function_vars * 4), SharedData.TYPES.UNSIGNED)
+ const = self.symtab.insert_constant(
+ "0{}".format(self.shared.function_vars * 4), SharedData.TYPES.UNSIGNED
+ )
self.arithmetic("-", "%15", const, "%15")
self.newline_label(self.shared.function_name + "_body", True, True)
@@ -788,128 +948,204 @@ class CodeGenerator: function - function index in symbol table
arguments - list of arguments (indexes in symbol table)
"""
- #push each argument to stack
+ # push each argument to stack
for arg in arguments:
self.push(self.symbol(arg))
self.free_if_register(arg)
- self.newline_text("CALL\t"+self.symtab.get_name(function), True)
+ self.newline_text("CALL\t" + self.symtab.get_name(function), True)
args = self.symtab.get_attribute(function)
- #generates stack cleanup if function has arguments
+ # generates stack cleanup if function has arguments
if args > 0:
- args_space = self.symtab.insert_constant("{}".format(args * 4), SharedData.TYPES.UNSIGNED)
+ args_space = self.symtab.insert_constant(
+ "{}".format(args * 4), SharedData.TYPES.UNSIGNED
+ )
self.arithmetic("+", "%15", args_space, "%15")
+
##########################################################################################
##########################################################################################
+
class MicroC:
"""Class for microC parser/compiler"""
def __init__(self):
- #Definitions of terminal symbols for microC programming language
- self.tId = Word(alphas+"_",alphanums+"_")
- self.tInteger = Word(nums).setParseAction(lambda x : [x[0], SharedData.TYPES.INT])
- self.tUnsigned = Regex(r"[0-9]+[uU]").setParseAction(lambda x : [x[0][:-1], SharedData.TYPES.UNSIGNED])
- self.tConstant = (self.tUnsigned | self.tInteger).setParseAction(self.constant_action)
- self.tType = Keyword("int").setParseAction(lambda x : SharedData.TYPES.INT) | \
- Keyword("unsigned").setParseAction(lambda x : SharedData.TYPES.UNSIGNED)
+ # Definitions of terminal symbols for microC programming language
+ self.tId = Word(alphas + "_", alphanums + "_")
+ self.tInteger = Word(nums).setParseAction(
+ lambda x: [x[0], SharedData.TYPES.INT]
+ )
+ self.tUnsigned = Regex(r"[0-9]+[uU]").setParseAction(
+ lambda x: [x[0][:-1], SharedData.TYPES.UNSIGNED]
+ )
+ self.tConstant = (self.tUnsigned | self.tInteger).setParseAction(
+ self.constant_action
+ )
+ self.tType = Keyword("int").setParseAction(
+ lambda x: SharedData.TYPES.INT
+ ) | Keyword("unsigned").setParseAction(lambda x: SharedData.TYPES.UNSIGNED)
self.tRelOp = oneOf(SharedData.RELATIONAL_OPERATORS)
self.tMulOp = oneOf("* /")
self.tAddOp = oneOf("+ -")
- #Definitions of rules for global variables
- self.rGlobalVariable = (self.tType("type") + self.tId("name") +
- FollowedBy(";")).setParseAction(self.global_variable_action)
+ # Definitions of rules for global variables
+ self.rGlobalVariable = (
+ self.tType("type") + self.tId("name") + FollowedBy(";")
+ ).setParseAction(self.global_variable_action)
self.rGlobalVariableList = ZeroOrMore(self.rGlobalVariable + Suppress(";"))
- #Definitions of rules for numeric expressions
+ # Definitions of rules for numeric expressions
self.rExp = Forward()
self.rMulExp = Forward()
self.rNumExp = Forward()
- self.rArguments = delimitedList(self.rNumExp("exp").setParseAction(self.argument_action))
- self.rFunctionCall = ((self.tId("name") + FollowedBy("(")).setParseAction(self.function_call_prepare_action) +
- Suppress("(") + Optional(self.rArguments)("args") + Suppress(")")).setParseAction(self.function_call_action)
- self.rExp << (self.rFunctionCall |
- self.tConstant |
- self.tId("name").setParseAction(self.lookup_id_action) |
- Group(Suppress("(") + self.rNumExp + Suppress(")")) |
- Group("+" + self.rExp) |
- Group("-" + self.rExp)).setParseAction(lambda x : x[0])
- self.rMulExp << (self.rExp + ZeroOrMore(self.tMulOp + self.rExp)).setParseAction(self.mulexp_action)
- self.rNumExp << (self.rMulExp + ZeroOrMore(self.tAddOp + self.rMulExp)).setParseAction(self.numexp_action)
-
- #Definitions of rules for logical expressions (these are without parenthesis support)
+ self.rArguments = delimitedList(
+ self.rNumExp("exp").setParseAction(self.argument_action)
+ )
+ self.rFunctionCall = (
+ (self.tId("name") + FollowedBy("(")).setParseAction(
+ self.function_call_prepare_action
+ )
+ + Suppress("(")
+ + Optional(self.rArguments)("args")
+ + Suppress(")")
+ ).setParseAction(self.function_call_action)
+ self.rExp << (
+ self.rFunctionCall
+ | self.tConstant
+ | self.tId("name").setParseAction(self.lookup_id_action)
+ | Group(Suppress("(") + self.rNumExp + Suppress(")"))
+ | Group("+" + self.rExp)
+ | Group("-" + self.rExp)
+ ).setParseAction(lambda x: x[0])
+ self.rMulExp << (
+ self.rExp + ZeroOrMore(self.tMulOp + self.rExp)
+ ).setParseAction(self.mulexp_action)
+ self.rNumExp << (
+ self.rMulExp + ZeroOrMore(self.tAddOp + self.rMulExp)
+ ).setParseAction(self.numexp_action)
+
+ # Definitions of rules for logical expressions (these are without parenthesis support)
self.rAndExp = Forward()
self.rLogExp = Forward()
- self.rRelExp = (self.rNumExp + self.tRelOp + self.rNumExp).setParseAction(self.relexp_action)
- self.rAndExp << (self.rRelExp("exp") + ZeroOrMore(Literal("&&").setParseAction(self.andexp_action) +
- self.rRelExp("exp")).setParseAction(lambda x : self.relexp_code))
- self.rLogExp << (self.rAndExp("exp") + ZeroOrMore(Literal("||").setParseAction(self.logexp_action) +
- self.rAndExp("exp")).setParseAction(lambda x : self.andexp_code))
-
- #Definitions of rules for statements
+ self.rRelExp = (self.rNumExp + self.tRelOp + self.rNumExp).setParseAction(
+ self.relexp_action
+ )
+ self.rAndExp << (
+ self.rRelExp("exp")
+ + ZeroOrMore(
+ Literal("&&").setParseAction(self.andexp_action) + self.rRelExp("exp")
+ ).setParseAction(lambda x: self.relexp_code)
+ )
+ self.rLogExp << (
+ self.rAndExp("exp")
+ + ZeroOrMore(
+ Literal("||").setParseAction(self.logexp_action) + self.rAndExp("exp")
+ ).setParseAction(lambda x: self.andexp_code)
+ )
+
+ # Definitions of rules for statements
self.rStatement = Forward()
self.rStatementList = Forward()
- self.rReturnStatement = (Keyword("return") + self.rNumExp("exp") +
- Suppress(";")).setParseAction(self.return_action)
- self.rAssignmentStatement = (self.tId("var") + Suppress("=") + self.rNumExp("exp") +
- Suppress(";")).setParseAction(self.assignment_action)
+ self.rReturnStatement = (
+ Keyword("return") + self.rNumExp("exp") + Suppress(";")
+ ).setParseAction(self.return_action)
+ self.rAssignmentStatement = (
+ self.tId("var") + Suppress("=") + self.rNumExp("exp") + Suppress(";")
+ ).setParseAction(self.assignment_action)
self.rFunctionCallStatement = self.rFunctionCall + Suppress(";")
- self.rIfStatement = ( (Keyword("if") + FollowedBy("(")).setParseAction(self.if_begin_action) +
- (Suppress("(") + self.rLogExp + Suppress(")")).setParseAction(self.if_body_action) +
- (self.rStatement + Empty()).setParseAction(self.if_else_action) +
- Optional(Keyword("else") + self.rStatement)).setParseAction(self.if_end_action)
- self.rWhileStatement = ( (Keyword("while") + FollowedBy("(")).setParseAction(self.while_begin_action) +
- (Suppress("(") + self.rLogExp + Suppress(")")).setParseAction(self.while_body_action) +
- self.rStatement).setParseAction(self.while_end_action)
- self.rCompoundStatement = Group(Suppress("{") + self.rStatementList + Suppress("}"))
- self.rStatement << (self.rReturnStatement | self.rIfStatement | self.rWhileStatement |
- self.rFunctionCallStatement | self.rAssignmentStatement | self.rCompoundStatement)
+ self.rIfStatement = (
+ (Keyword("if") + FollowedBy("(")).setParseAction(self.if_begin_action)
+ + (Suppress("(") + self.rLogExp + Suppress(")")).setParseAction(
+ self.if_body_action
+ )
+ + (self.rStatement + Empty()).setParseAction(self.if_else_action)
+ + Optional(Keyword("else") + self.rStatement)
+ ).setParseAction(self.if_end_action)
+ self.rWhileStatement = (
+ (Keyword("while") + FollowedBy("(")).setParseAction(self.while_begin_action)
+ + (Suppress("(") + self.rLogExp + Suppress(")")).setParseAction(
+ self.while_body_action
+ )
+ + self.rStatement
+ ).setParseAction(self.while_end_action)
+ self.rCompoundStatement = Group(
+ Suppress("{") + self.rStatementList + Suppress("}")
+ )
+ self.rStatement << (
+ self.rReturnStatement
+ | self.rIfStatement
+ | self.rWhileStatement
+ | self.rFunctionCallStatement
+ | self.rAssignmentStatement
+ | self.rCompoundStatement
+ )
self.rStatementList << ZeroOrMore(self.rStatement)
- self.rLocalVariable = (self.tType("type") + self.tId("name") + FollowedBy(";")).setParseAction(self.local_variable_action)
+ self.rLocalVariable = (
+ self.tType("type") + self.tId("name") + FollowedBy(";")
+ ).setParseAction(self.local_variable_action)
self.rLocalVariableList = ZeroOrMore(self.rLocalVariable + Suppress(";"))
- self.rFunctionBody = Suppress("{") + Optional(self.rLocalVariableList).setParseAction(self.function_body_action) + \
- self.rStatementList + Suppress("}")
- self.rParameter = (self.tType("type") + self.tId("name")).setParseAction(self.parameter_action)
+ self.rFunctionBody = (
+ Suppress("{")
+ + Optional(self.rLocalVariableList).setParseAction(
+ self.function_body_action
+ )
+ + self.rStatementList
+ + Suppress("}")
+ )
+ self.rParameter = (self.tType("type") + self.tId("name")).setParseAction(
+ self.parameter_action
+ )
self.rParameterList = delimitedList(self.rParameter)
- self.rFunction = ( (self.tType("type") + self.tId("name")).setParseAction(self.function_begin_action) +
- Group(Suppress("(") + Optional(self.rParameterList)("params") + Suppress(")") +
- self.rFunctionBody)).setParseAction(self.function_end_action)
+ self.rFunction = (
+ (self.tType("type") + self.tId("name")).setParseAction(
+ self.function_begin_action
+ )
+ + Group(
+ Suppress("(")
+ + Optional(self.rParameterList)("params")
+ + Suppress(")")
+ + self.rFunctionBody
+ )
+ ).setParseAction(self.function_end_action)
self.rFunctionList = OneOrMore(self.rFunction)
- self.rProgram = (Empty().setParseAction(self.data_begin_action) + self.rGlobalVariableList +
- Empty().setParseAction(self.code_begin_action) + self.rFunctionList).setParseAction(self.program_end_action)
-
- #shared data
+ self.rProgram = (
+ Empty().setParseAction(self.data_begin_action)
+ + self.rGlobalVariableList
+ + Empty().setParseAction(self.code_begin_action)
+ + self.rFunctionList
+ ).setParseAction(self.program_end_action)
+
+ # shared data
self.shared = SharedData()
- #symbol table
+ # symbol table
self.symtab = SymbolTable(self.shared)
- #code generator
+ # code generator
self.codegen = CodeGenerator(self.shared, self.symtab)
- #index of the current function call
+ # index of the current function call
self.function_call_index = -1
- #stack for the nested function calls
+ # stack for the nested function calls
self.function_call_stack = []
- #arguments of the current function call
+ # arguments of the current function call
self.function_arguments = []
- #stack for arguments of the nested function calls
+ # stack for arguments of the nested function calls
self.function_arguments_stack = []
- #number of arguments for the curent function call
+ # number of arguments for the curent function call
self.function_arguments_number = -1
- #stack for the number of arguments for the nested function calls
+ # stack for the number of arguments for the nested function calls
self.function_arguments_number_stack = []
- #last relational expression
+ # last relational expression
self.relexp_code = None
- #last and expression
+ # last and expression
self.andexp_code = None
- #label number for "false" internal labels
+ # label number for "false" internal labels
self.false_label_number = -1
- #label number for all other internal labels
+ # label number for all other internal labels
self.label_number = None
- #label stack for nested statements
+ # label stack for nested statements
self.label_stack = []
def warning(self, message, print_location=True):
@@ -925,7 +1161,6 @@ class MicroC: msg += "\n%s" % wtext
print(msg)
-
def data_begin_action(self):
"""Inserts text at start of data segment"""
self.codegen.prepare_data_segment()
@@ -938,9 +1173,11 @@ class MicroC: """Code executed after recognising a global variable"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("GLOBAL_VAR:",var)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
+ print("GLOBAL_VAR:", var)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
index = self.symtab.insert_global_var(var.name, var.type)
self.codegen.global_var(var.name)
return index
@@ -949,10 +1186,14 @@ class MicroC: """Code executed after recognising a local variable"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("LOCAL_VAR:",var, var.name, var.type)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
- index = self.symtab.insert_local_var(var.name, var.type, self.shared.function_vars)
+ print("LOCAL_VAR:", var, var.name, var.type)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
+ index = self.symtab.insert_local_var(
+ var.name, var.type, self.shared.function_vars
+ )
self.shared.function_vars += 1
return index
@@ -960,9 +1201,11 @@ class MicroC: """Code executed after recognising a parameter"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("PARAM:",par)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
+ print("PARAM:", par)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
index = self.symtab.insert_parameter(par.name, par.type)
self.shared.function_params += 1
return index
@@ -971,42 +1214,52 @@ class MicroC: """Code executed after recognising a constant"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("CONST:",const)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
+ print("CONST:", const)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
return self.symtab.insert_constant(const[0], const[1])
def function_begin_action(self, text, loc, fun):
"""Code executed after recognising a function definition (type and function name)"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("FUN_BEGIN:",fun)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
+ print("FUN_BEGIN:", fun)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
self.shared.function_index = self.symtab.insert_function(fun.name, fun.type)
self.shared.function_name = fun.name
self.shared.function_params = 0
self.shared.function_vars = 0
- self.codegen.function_begin();
+ self.codegen.function_begin()
def function_body_action(self, text, loc, fun):
"""Code executed after recognising the beginning of function's body"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("FUN_BODY:",fun)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
+ print("FUN_BODY:", fun)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
self.codegen.function_body()
def function_end_action(self, text, loc, fun):
"""Code executed at the end of function definition"""
if DEBUG > 0:
- print("FUN_END:",fun)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
- #set function's attribute to number of function parameters
- self.symtab.set_attribute(self.shared.function_index, self.shared.function_params)
- #clear local function symbols (but leave function name)
+ print("FUN_END:", fun)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
+ # set function's attribute to number of function parameters
+ self.symtab.set_attribute(
+ self.shared.function_index, self.shared.function_params
+ )
+ # clear local function symbols (but leave function name)
self.symtab.clear_symbols(self.shared.function_index + 1)
self.codegen.function_end()
@@ -1014,27 +1267,40 @@ class MicroC: """Code executed after recognising a return statement"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("RETURN:",ret)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
+ print("RETURN:", ret)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
if not self.symtab.same_types(self.shared.function_index, ret.exp[0]):
raise SemanticException("Incompatible type in return")
- #set register for function's return value to expression value
+ # set register for function's return value to expression value
reg = self.codegen.take_function_register()
self.codegen.move(ret.exp[0], reg)
- #after return statement, register for function's return value is available again
+ # after return statement, register for function's return value is available again
self.codegen.free_register(reg)
- #jump to function's exit
- self.codegen.unconditional_jump(self.codegen.label(self.shared.function_name+"_exit", True))
+ # jump to function's exit
+ self.codegen.unconditional_jump(
+ self.codegen.label(self.shared.function_name + "_exit", True)
+ )
def lookup_id_action(self, text, loc, var):
"""Code executed after recognising an identificator in expression"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("EXP_VAR:",var)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
- var_index = self.symtab.lookup_symbol(var.name, [SharedData.KINDS.GLOBAL_VAR, SharedData.KINDS.PARAMETER, SharedData.KINDS.LOCAL_VAR])
+ print("EXP_VAR:", var)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
+ var_index = self.symtab.lookup_symbol(
+ var.name,
+ [
+ SharedData.KINDS.GLOBAL_VAR,
+ SharedData.KINDS.PARAMETER,
+ SharedData.KINDS.LOCAL_VAR,
+ ],
+ )
if var_index == None:
raise SemanticException("'%s' undefined" % var.name)
return var_index
@@ -1043,10 +1309,19 @@ class MicroC: """Code executed after recognising an assignment statement"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("ASSIGN:",assign)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
- var_index = self.symtab.lookup_symbol(assign.var, [SharedData.KINDS.GLOBAL_VAR, SharedData.KINDS.PARAMETER, SharedData.KINDS.LOCAL_VAR])
+ print("ASSIGN:", assign)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
+ var_index = self.symtab.lookup_symbol(
+ assign.var,
+ [
+ SharedData.KINDS.GLOBAL_VAR,
+ SharedData.KINDS.PARAMETER,
+ SharedData.KINDS.LOCAL_VAR,
+ ],
+ )
if var_index == None:
raise SemanticException("Undefined lvalue '%s' in assignment" % assign.var)
if not self.symtab.same_types(var_index, assign.exp[0]):
@@ -1057,16 +1332,18 @@ class MicroC: """Code executed after recognising a mulexp expression (something *|/ something)"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("MUL_EXP:",mul)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
- #iterate through all multiplications/divisions
+ print("MUL_EXP:", mul)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
+ # iterate through all multiplications/divisions
m = list(mul)
while len(m) > 1:
if not self.symtab.same_types(m[0], m[2]):
raise SemanticException("Invalid opernads to binary '%s'" % m[1])
reg = self.codegen.arithmetic(m[1], m[0], m[2])
- #replace first calculation with it's result
+ # replace first calculation with it's result
m[0:3] = [reg]
return m[0]
@@ -1074,16 +1351,18 @@ class MicroC: """Code executed after recognising a numexp expression (something +|- something)"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("NUM_EXP:",num)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
- #iterate through all additions/substractions
+ print("NUM_EXP:", num)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
+ # iterate through all additions/substractions
n = list(num)
while len(n) > 1:
if not self.symtab.same_types(n[0], n[2]):
raise SemanticException("Invalid opernads to binary '%s'" % n[1])
reg = self.codegen.arithmetic(n[1], n[0], n[2])
- #replace first calculation with it's result
+ # replace first calculation with it's result
n[0:3] = [reg]
return n[0]
@@ -1091,13 +1370,15 @@ class MicroC: """Code executed after recognising a function call (type and function name)"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("FUN_PREP:",fun)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
+ print("FUN_PREP:", fun)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
index = self.symtab.lookup_symbol(fun.name, SharedData.KINDS.FUNCTION)
if index == None:
raise SemanticException("'%s' is not a function" % fun.name)
- #save any previous function call data (for nested function calls)
+ # save any previous function call data (for nested function calls)
self.function_call_stack.append(self.function_call_index)
self.function_call_index = index
self.function_arguments_stack.append(self.function_arguments[:])
@@ -1108,49 +1389,64 @@ class MicroC: """Code executed after recognising each of function's arguments"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("ARGUMENT:",arg.exp)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
+ print("ARGUMENT:", arg.exp)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
arg_ordinal = len(self.function_arguments)
- #check argument's type
- if not self.symtab.same_type_as_argument(arg.exp, self.function_call_index, arg_ordinal):
- raise SemanticException("Incompatible type for argument %d in '%s'" % (arg_ordinal + 1, self.symtab.get_name(self.function_call_index)))
+ # check argument's type
+ if not self.symtab.same_type_as_argument(
+ arg.exp, self.function_call_index, arg_ordinal
+ ):
+ raise SemanticException(
+ "Incompatible type for argument %d in '%s'"
+ % (arg_ordinal + 1, self.symtab.get_name(self.function_call_index))
+ )
self.function_arguments.append(arg.exp)
def function_call_action(self, text, loc, fun):
"""Code executed after recognising the whole function call"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("FUN_CALL:",fun)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
- #check number of arguments
- if len(self.function_arguments) != self.symtab.get_attribute(self.function_call_index):
- raise SemanticException("Wrong number of arguments for function '%s'" % fun.name)
- #arguments should be pushed to stack in reverse order
+ print("FUN_CALL:", fun)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
+ # check number of arguments
+ if len(self.function_arguments) != self.symtab.get_attribute(
+ self.function_call_index
+ ):
+ raise SemanticException(
+ "Wrong number of arguments for function '%s'" % fun.name
+ )
+ # arguments should be pushed to stack in reverse order
self.function_arguments.reverse()
self.codegen.function_call(self.function_call_index, self.function_arguments)
self.codegen.restore_used_registers()
return_type = self.symtab.get_type(self.function_call_index)
- #restore previous function call data
+ # restore previous function call data
self.function_call_index = self.function_call_stack.pop()
self.function_arguments = self.function_arguments_stack.pop()
register = self.codegen.take_register(return_type)
- #move result to a new free register, to allow the next function call
+ # move result to a new free register, to allow the next function call
self.codegen.move(self.codegen.take_function_register(return_type), register)
return register
def relexp_action(self, text, loc, arg):
"""Code executed after recognising a relexp expression (something relop something)"""
if DEBUG > 0:
- print("REL_EXP:",arg)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
+ print("REL_EXP:", arg)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
exshared.setpos(loc, text)
if not self.symtab.same_types(arg[0], arg[2]):
raise SemanticException("Invalid operands for operator '{}'".format(arg[1]))
self.codegen.compare(arg[0], arg[2])
- #return relational operator's code
+ # return relational operator's code
self.relexp_code = self.codegen.relop_code(arg[1], self.symtab.get_type(arg[0]))
return self.relexp_code
@@ -1158,10 +1454,14 @@ class MicroC: """Code executed after recognising a andexp expression (something and something)"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("AND+EXP:",arg)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
- label = self.codegen.label("false{}".format(self.false_label_number), True, False)
+ print("AND+EXP:", arg)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
+ label = self.codegen.label(
+ "false{}".format(self.false_label_number), True, False
+ )
self.codegen.jump(self.relexp_code, True, label)
self.andexp_code = self.relexp_code
return self.andexp_code
@@ -1170,21 +1470,27 @@ class MicroC: """Code executed after recognising logexp expression (something or something)"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("LOG_EXP:",arg)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
+ print("LOG_EXP:", arg)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
label = self.codegen.label("true{}".format(self.label_number), True, False)
self.codegen.jump(self.relexp_code, False, label)
- self.codegen.newline_label("false{}".format(self.false_label_number), True, True)
+ self.codegen.newline_label(
+ "false{}".format(self.false_label_number), True, True
+ )
self.false_label_number += 1
def if_begin_action(self, text, loc, arg):
"""Code executed after recognising an if statement (if keyword)"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("IF_BEGIN:",arg)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
+ print("IF_BEGIN:", arg)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
self.false_label_number += 1
self.label_number = self.false_label_number
self.codegen.newline_label("if{}".format(self.label_number), True, True)
@@ -1193,15 +1499,19 @@ class MicroC: """Code executed after recognising if statement's body"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("IF_BODY:",arg)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
- #generate conditional jump (based on last compare)
- label = self.codegen.label("false{}".format(self.false_label_number), True, False)
+ print("IF_BODY:", arg)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
+ # generate conditional jump (based on last compare)
+ label = self.codegen.label(
+ "false{}".format(self.false_label_number), True, False
+ )
self.codegen.jump(self.relexp_code, True, label)
- #generate 'true' label (executes if condition is satisfied)
+ # generate 'true' label (executes if condition is satisfied)
self.codegen.newline_label("true{}".format(self.label_number), True, True)
- #save label numbers (needed for nested if/while statements)
+ # save label numbers (needed for nested if/while statements)
self.label_stack.append(self.false_label_number)
self.label_stack.append(self.label_number)
@@ -1209,14 +1519,16 @@ class MicroC: """Code executed after recognising if statement's else body"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("IF_ELSE:",arg)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
- #jump to exit after all statements for true condition are executed
+ print("IF_ELSE:", arg)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
+ # jump to exit after all statements for true condition are executed
self.label_number = self.label_stack.pop()
label = self.codegen.label("exit{}".format(self.label_number), True, False)
self.codegen.unconditional_jump(label)
- #generate final 'false' label (executes if condition isn't satisfied)
+ # generate final 'false' label (executes if condition isn't satisfied)
self.codegen.newline_label("false{}".format(self.label_stack.pop()), True, True)
self.label_stack.append(self.label_number)
@@ -1224,18 +1536,22 @@ class MicroC: """Code executed after recognising a whole if statement"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("IF_END:",arg)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
+ print("IF_END:", arg)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
self.codegen.newline_label("exit{}".format(self.label_stack.pop()), True, True)
def while_begin_action(self, text, loc, arg):
"""Code executed after recognising a while statement (while keyword)"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("WHILE_BEGIN:",arg)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
+ print("WHILE_BEGIN:", arg)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
self.false_label_number += 1
self.label_number = self.false_label_number
self.codegen.newline_label("while{}".format(self.label_number), True, True)
@@ -1244,13 +1560,17 @@ class MicroC: """Code executed after recognising while statement's body"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("WHILE_BODY:",arg)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
- #generate conditional jump (based on last compare)
- label = self.codegen.label("false{}".format(self.false_label_number), True, False)
+ print("WHILE_BODY:", arg)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
+ # generate conditional jump (based on last compare)
+ label = self.codegen.label(
+ "false{}".format(self.false_label_number), True, False
+ )
self.codegen.jump(self.relexp_code, True, label)
- #generate 'true' label (executes if condition is satisfied)
+ # generate 'true' label (executes if condition is satisfied)
self.codegen.newline_label("true{}".format(self.label_number), True, True)
self.label_stack.append(self.false_label_number)
self.label_stack.append(self.label_number)
@@ -1259,14 +1579,16 @@ class MicroC: """Code executed after recognising a whole while statement"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("WHILE_END:",arg)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
- #jump to condition checking after while statement body
+ print("WHILE_END:", arg)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
+ # jump to condition checking after while statement body
self.label_number = self.label_stack.pop()
label = self.codegen.label("while{}".format(self.label_number), True, False)
self.codegen.unconditional_jump(label)
- #generate final 'false' label and exit label
+ # generate final 'false' label and exit label
self.codegen.newline_label("false{}".format(self.label_stack.pop()), True, True)
self.codegen.newline_label("exit{}".format(self.label_number), True, True)
@@ -1274,16 +1596,18 @@ class MicroC: """Checks if there is a 'main' function and the type of 'main' function"""
exshared.setpos(loc, text)
if DEBUG > 0:
- print("PROGRAM_END:",arg)
- if DEBUG == 2: self.symtab.display()
- if DEBUG > 2: return
- index = self.symtab.lookup_symbol("main",SharedData.KINDS.FUNCTION)
+ print("PROGRAM_END:", arg)
+ if DEBUG == 2:
+ self.symtab.display()
+ if DEBUG > 2:
+ return
+ index = self.symtab.lookup_symbol("main", SharedData.KINDS.FUNCTION)
if index == None:
raise SemanticException("Undefined reference to 'main'", False)
elif self.symtab.get_type(index) != SharedData.TYPES.INT:
self.warning("Return type of 'main' is not int", False)
- def parse_text(self,text):
+ def parse_text(self, text):
"""Parse string (helper function)"""
try:
return self.rProgram.ignore(cStyleComment).parseString(text, parseAll=True)
@@ -1294,10 +1618,12 @@ class MicroC: print(err)
exit(3)
- def parse_file(self,filename):
+ def parse_file(self, filename):
"""Parse file (helper function)"""
try:
- return self.rProgram.ignore(cStyleComment).parseFile(filename, parseAll=True)
+ return self.rProgram.ignore(cStyleComment).parseFile(
+ filename, parseAll=True
+ )
except SemanticException as err:
print(err)
exit(3)
@@ -1305,10 +1631,11 @@ class MicroC: print(err)
exit(3)
+
##########################################################################################
##########################################################################################
if 0:
- #main program
+ # main program
mc = MicroC()
output_file = "output.asm"
@@ -1322,19 +1649,21 @@ if 0: else:
usage = """Usage: {} [input_file [output_file]]
If output file is omitted, output.asm is used
- If input file is omitted, stdin is used""".format(argv[0])
+ If input file is omitted, stdin is used""".format(
+ argv[0]
+ )
print(usage)
exit(1)
try:
- parse = stdin if input_file == stdin else open(input_file,'r')
+ parse = stdin if input_file == stdin else open(input_file, "r")
except Exception:
print("Input file '%s' open error" % input_file)
exit(2)
mc.parse_file(parse)
- #if you want to see the final symbol table, uncomment next line
- #mc.symtab.display()
+ # if you want to see the final symbol table, uncomment next line
+ # mc.symtab.display()
try:
- out = open(output_file, 'w')
+ out = open(output_file, "w")
out.write(mc.codegen.code)
out.close
except Exception:
diff --git a/examples/pythonGrammarParser.py b/examples/pythonGrammarParser.py index e3685a1..e9d7d94 100644 --- a/examples/pythonGrammarParser.py +++ b/examples/pythonGrammarParser.py @@ -130,35 +130,42 @@ testlist1: test (',' test)* encoding_decl: NAME
"""
+
class SemanticGroup:
- def __init__(self,contents):
+ def __init__(self, contents):
self.contents = contents
while self.contents[-1].__class__ == self.__class__:
self.contents = self.contents[:-1] + self.contents[-1].contents
def __str__(self):
- return "{}({})".format(self.label,
- " ".join([isinstance(c,str) and c or str(c) for c in self.contents]) )
+ return "{}({})".format(
+ self.label,
+ " ".join([isinstance(c, str) and c or str(c) for c in self.contents]),
+ )
+
class OrList(SemanticGroup):
label = "OR"
pass
+
class AndList(SemanticGroup):
label = "AND"
pass
+
class OptionalGroup(SemanticGroup):
label = "OPT"
pass
+
class Atom(SemanticGroup):
- def __init__(self,contents):
+ def __init__(self, contents):
if len(contents) > 1:
self.rep = contents[1]
else:
self.rep = ""
- if isinstance(contents,str):
+ if isinstance(contents, str):
self.contents = contents
else:
self.contents = contents[0]
@@ -166,12 +173,14 @@ class Atom(SemanticGroup): def __str__(self):
return "{}{}".format(self.rep, self.contents)
+
def makeGroupObject(cls):
- def groupAction(s,l,t):
+ def groupAction(s, l, t):
try:
return cls(t[0].asList())
except Exception:
return cls(t)
+
return groupAction
@@ -180,20 +189,27 @@ LPAREN = Suppress("(") RPAREN = Suppress(")")
LBRACK = Suppress("[")
RBRACK = Suppress("]")
-COLON = Suppress(":")
+COLON = Suppress(":")
ALT_OP = Suppress("|")
# bnf grammar
-ident = Word(alphanums+"_")
-bnfToken = Word(alphanums+"_") + ~FollowedBy(":")
+ident = Word(alphanums + "_")
+bnfToken = Word(alphanums + "_") + ~FollowedBy(":")
repSymbol = oneOf("* +")
bnfExpr = Forward()
-optionalTerm = Group(LBRACK + bnfExpr + RBRACK).setParseAction(makeGroupObject(OptionalGroup))
-bnfTerm = ( (bnfToken | quotedString | optionalTerm | ( LPAREN + bnfExpr + RPAREN )) + Optional(repSymbol) ).setParseAction(makeGroupObject(Atom))
+optionalTerm = Group(LBRACK + bnfExpr + RBRACK).setParseAction(
+ makeGroupObject(OptionalGroup)
+)
+bnfTerm = (
+ (bnfToken | quotedString | optionalTerm | (LPAREN + bnfExpr + RPAREN))
+ + Optional(repSymbol)
+).setParseAction(makeGroupObject(Atom))
andList = Group(bnfTerm + OneOrMore(bnfTerm)).setParseAction(makeGroupObject(AndList))
bnfFactor = andList | bnfTerm
-orList = Group( bnfFactor + OneOrMore( ALT_OP + bnfFactor ) ).setParseAction(makeGroupObject(OrList))
-bnfExpr << ( orList | bnfFactor )
+orList = Group(bnfFactor + OneOrMore(ALT_OP + bnfFactor)).setParseAction(
+ makeGroupObject(OrList)
+)
+bnfExpr << (orList | bnfFactor)
bnfLine = ident + COLON + bnfExpr
bnfComment = "#" + restOfLine
@@ -207,14 +223,16 @@ bnfDefs = bnf.parseString(grammar) # correct answer is 78
expected = 78
-assert len(bnfDefs) == expected, \
- "Error, found %d BNF defns, expected %d" % (len(bnfDefs), expected)
+assert len(bnfDefs) == expected, "Error, found %d BNF defns, expected %d" % (
+ len(bnfDefs),
+ expected,
+)
# list out defns in order they were parsed (to verify accuracy of parsing)
-for k,v in bnfDefs:
- print(k,"=",v)
+for k, v in bnfDefs:
+ print(k, "=", v)
print()
# list out parsed grammar defns (demonstrates dictionary access to parsed tokens)
for k in list(bnfDefs.keys()):
- print(k,"=",bnfDefs[k])
+ print(k, "=", bnfDefs[k])
diff --git a/examples/rangeCheck.py b/examples/rangeCheck.py index 66af545..2d1d2c8 100644 --- a/examples/rangeCheck.py +++ b/examples/rangeCheck.py @@ -11,6 +11,7 @@ from pyparsing import Word, nums, Suppress, Optional
from datetime import datetime
+
def ranged_value(expr, minval=None, maxval=None):
# have to specify at least one range boundary
if minval is None and maxval is None:
@@ -19,27 +20,28 @@ def ranged_value(expr, minval=None, maxval=None): # set range testing function and error message depending on
# whether either or both min and max values are given
inRangeCondition = {
- (True, False) : lambda s,l,t : t[0] <= maxval,
- (False, True) : lambda s,l,t : minval <= t[0],
- (False, False) : lambda s,l,t : minval <= t[0] <= maxval,
- }[minval is None, maxval is None]
+ (True, False): lambda s, l, t: t[0] <= maxval,
+ (False, True): lambda s, l, t: minval <= t[0],
+ (False, False): lambda s, l, t: minval <= t[0] <= maxval,
+ }[minval is None, maxval is None]
outOfRangeMessage = {
- (True, False) : "value is greater than %s" % maxval,
- (False, True) : "value is less than %s" % minval,
- (False, False) : "value is not in the range ({} to {})".format(minval,maxval),
- }[minval is None, maxval is None]
+ (True, False): "value is greater than %s" % maxval,
+ (False, True): "value is less than %s" % minval,
+ (False, False): "value is not in the range ({} to {})".format(minval, maxval),
+ }[minval is None, maxval is None]
return expr().addCondition(inRangeCondition, message=outOfRangeMessage)
+
# define the expressions for a date of the form YYYY/MM/DD or YYYY/MM (assumes YYYY/MM/01)
integer = Word(nums).setName("integer")
-integer.setParseAction(lambda t:int(t[0]))
+integer.setParseAction(lambda t: int(t[0]))
month = ranged_value(integer, 1, 12)
day = ranged_value(integer, 1, 31)
year = ranged_value(integer, 2000, None)
-SLASH = Suppress('/')
+SLASH = Suppress("/")
dateExpr = year("year") + SLASH + month("month") + Optional(SLASH + day("day"))
dateExpr.setName("date")
@@ -47,14 +49,16 @@ dateExpr.setName("date") dateExpr.setParseAction(lambda t: datetime(t.year, t.month, t.day or 1).date())
# add range checking on dates
-mindate = datetime(2002,1,1).date()
+mindate = datetime(2002, 1, 1).date()
maxdate = datetime.now().date()
dateExpr = ranged_value(dateExpr, mindate, maxdate)
-dateExpr.runTests("""
+dateExpr.runTests(
+ """
2011/5/8
2001/1/1
2004/2/29
2004/2
- 1999/12/31""")
+ 1999/12/31"""
+)
diff --git a/examples/readJson.py b/examples/readJson.py index f691ea5..f3b6a6f 100644 --- a/examples/readJson.py +++ b/examples/readJson.py @@ -1,14 +1,14 @@ -#~ url = "http://cmsdoc.cern.ch/cms/test/aprom/phedex/dev/gowri/datasvc/tbedi/requestDetails"
-#~ params = {'format':'json'}
-#~ import urllib
-#~ eparams = urllib.urlencode(params)
-#~ import urllib2
-#~ request = urllib2.Request(url,eparams)
-#~ response = urllib2.urlopen(request)
-#~ s = response.read()
-#~ response.close()
+# ~ url = "http://cmsdoc.cern.ch/cms/test/aprom/phedex/dev/gowri/datasvc/tbedi/requestDetails"
+# ~ params = {'format':'json'}
+# ~ import urllib
+# ~ eparams = urllib.urlencode(params)
+# ~ import urllib2
+# ~ request = urllib2.Request(url,eparams)
+# ~ response = urllib2.urlopen(request)
+# ~ s = response.read()
+# ~ response.close()
-#~ print s
+# ~ print s
s = """
{"phedex":{"request":[{"last_update":"1188037561", "numofapproved":"1",
@@ -1903,14 +1903,14 @@ from jsonParser import jsonObject data = jsonObject.parseString(s)
-#~ from pprint import pprint
-#~ pprint( data[0].asList() )
-#~ print
-#~ print data.dump()
+# ~ from pprint import pprint
+# ~ pprint( data[0].asList() )
+# ~ print
+# ~ print data.dump()
print(data.phedex.call_time)
print(data.phedex.instance)
print(data.phedex.request_call)
print(len(data.phedex.request))
for req in data.phedex.request[:10]:
- #~ print req.dump()
+ # ~ print req.dump()
print("-", req.id, req.last_update)
diff --git a/examples/removeLineBreaks.py b/examples/removeLineBreaks.py index 1a77231..03315be 100644 --- a/examples/removeLineBreaks.py +++ b/examples/removeLineBreaks.py @@ -18,11 +18,14 @@ line_end = pp.LineEnd() # define an expression for the body of a line of text - use a predicate condition to # accept only lines with some content. def mustBeNonBlank(t): - return t[0] != '' + return t[0] != "" # could also be written as # return bool(t[0]) -lineBody = pp.SkipTo(line_end).addCondition(mustBeNonBlank, message="line body can't be empty") + +lineBody = pp.SkipTo(line_end).addCondition( + mustBeNonBlank, message="line body can't be empty" +) # now define a line with a trailing lineEnd, to be replaced with a space character textLine = lineBody + line_end().setParseAction(pp.replaceWith(" ")) diff --git a/examples/romanNumerals.py b/examples/romanNumerals.py index 757a925..932daa6 100644 --- a/examples/romanNumerals.py +++ b/examples/romanNumerals.py @@ -5,26 +5,40 @@ import pyparsing as pp + def romanNumeralLiteral(numeralString, value): return pp.Literal(numeralString).setParseAction(pp.replaceWith(value)) -one = romanNumeralLiteral("I", 1) -four = romanNumeralLiteral("IV", 4) -five = romanNumeralLiteral("V", 5) -nine = romanNumeralLiteral("IX", 9) -ten = romanNumeralLiteral("X", 10) -forty = romanNumeralLiteral("XL", 40) -fifty = romanNumeralLiteral("L", 50) -ninety = romanNumeralLiteral("XC", 90) -onehundred = romanNumeralLiteral("C", 100) + +one = romanNumeralLiteral("I", 1) +four = romanNumeralLiteral("IV", 4) +five = romanNumeralLiteral("V", 5) +nine = romanNumeralLiteral("IX", 9) +ten = romanNumeralLiteral("X", 10) +forty = romanNumeralLiteral("XL", 40) +fifty = romanNumeralLiteral("L", 50) +ninety = romanNumeralLiteral("XC", 90) +onehundred = romanNumeralLiteral("C", 100) fourhundred = romanNumeralLiteral("CD", 400) fivehundred = romanNumeralLiteral("D", 500) ninehundred = romanNumeralLiteral("CM", 900) onethousand = romanNumeralLiteral("M", 1000) -numeral = (onethousand | ninehundred | fivehundred | fourhundred - | onehundred | ninety | fifty | forty | ten | nine | five - | four | one).leaveWhitespace() +numeral = ( + onethousand + | ninehundred + | fivehundred + | fourhundred + | onehundred + | ninety + | fifty + | forty + | ten + | nine + | five + | four + | one +).leaveWhitespace() romanNumeral = numeral[1, ...].setParseAction(sum) @@ -52,8 +66,9 @@ def makeRomanNumeral(n): n, ret = addDigits(n, 1, "I", ret) return ret + # make a string of all roman numerals from I to MMMMM -tests = " ".join(makeRomanNumeral(i) for i in range(1, 5000+1)) +tests = " ".join(makeRomanNumeral(i) for i in range(1, 5000 + 1)) # parse each roman numeral, and populate map for validation below roman_int_map = {} @@ -63,17 +78,24 @@ for expected, (t, s, e) in enumerate(romanNumeral.scanString(tests), start=1): print("{} {} {}".format("==>", t, orig)) roman_int_map[orig] = t[0] + def verify_value(s, tokens): expected = roman_int_map[s] if tokens[0] != expected: - raise Exception("incorrect value for {} ({}), expected {}".format(s, tokens[0], expected )) + raise Exception( + "incorrect value for {} ({}), expected {}".format(s, tokens[0], expected) + ) + -romanNumeral.runTests("""\ +romanNumeral.runTests( + """\ XVI XXXIX XIV XIX MCMLXXX MMVI - """, fullDump=False, - postParse=verify_value) + """, + fullDump=False, + postParse=verify_value, +) diff --git a/examples/rosettacode.py b/examples/rosettacode.py index 8a8d5c9..5cbf203 100644 --- a/examples/rosettacode.py +++ b/examples/rosettacode.py @@ -7,7 +7,7 @@ # BNF = """ stmt_list = {stmt} ; - + stmt = ';' | Identifier '=' expr ';' | 'while' paren_expr stmt @@ -16,11 +16,11 @@ BNF = """ | 'putc' paren_expr ';' | '{' stmt_list '}' ; - + paren_expr = '(' expr ')' ; - + prt_list = string | expr {',' String | expr} ; - + expr = and_expr {'||' and_expr} ; and_expr = equality_expr {'&&' equality_expr} ; equality_expr = relational_expr [('==' | '!=') relational_expr] ; @@ -35,28 +35,33 @@ BNF = """ """ import pyparsing as pp + pp.ParserElement.enablePackrat() LBRACE, RBRACE, LPAR, RPAR, SEMI = map(pp.Suppress, "{}();") -EQ = pp.Literal('=') +EQ = pp.Literal("=") -keywords = (WHILE, IF, PRINT, PUTC, ELSE) = map(pp.Keyword, "while if print putc else".split()) +keywords = (WHILE, IF, PRINT, PUTC, ELSE) = map( + pp.Keyword, "while if print putc else".split() +) any_keyword = pp.MatchFirst(keywords) identifier = ~any_keyword + pp.pyparsing_common.identifier integer = pp.pyparsing_common.integer -string = pp.QuotedString('"', convertWhitespaceEscapes=False).setName("quoted string") +string = pp.QuotedString('"', convertWhitespaceEscapes=False).setName("quoted string") char = pp.Regex(r"'\\?.'") -expr = pp.infixNotation(identifier | integer | char, - [ - (pp.oneOf("+ - !"), 1, pp.opAssoc.RIGHT,), - (pp.oneOf("* / %"), 2, pp.opAssoc.LEFT, ), - (pp.oneOf("+ -"), 2, pp.opAssoc.LEFT,), - (pp.oneOf("< <= > >="), 2, pp.opAssoc.LEFT,), - (pp.oneOf("== !="), 2, pp.opAssoc.LEFT,), - (pp.oneOf("&&"), 2, pp.opAssoc.LEFT,), - (pp.oneOf("||"), 2, pp.opAssoc.LEFT,), - ]) +expr = pp.infixNotation( + identifier | integer | char, + [ + (pp.oneOf("+ - !"), 1, pp.opAssoc.RIGHT,), + (pp.oneOf("* / %"), 2, pp.opAssoc.LEFT,), + (pp.oneOf("+ -"), 2, pp.opAssoc.LEFT,), + (pp.oneOf("< <= > >="), 2, pp.opAssoc.LEFT,), + (pp.oneOf("== !="), 2, pp.opAssoc.LEFT,), + (pp.oneOf("&&"), 2, pp.opAssoc.LEFT,), + (pp.oneOf("||"), 2, pp.opAssoc.LEFT,), + ], +) prt_list = pp.Group(pp.delimitedList(string | expr)) paren_expr = pp.Group(LPAR + expr + RPAR) @@ -68,28 +73,29 @@ if_stmt = pp.Group(IF - paren_expr + stmt + pp.Optional(ELSE + stmt)) print_stmt = pp.Group(PRINT - pp.Group(LPAR + prt_list + RPAR) + SEMI) putc_stmt = pp.Group(PUTC - paren_expr + SEMI) stmt_list = pp.Group(LBRACE + stmt[...] + RBRACE) -stmt <<= (pp.Group(SEMI) - | assignment_stmt - | while_stmt - | if_stmt - | print_stmt - | putc_stmt - | stmt_list - ).setName("statement") +stmt <<= ( + pp.Group(SEMI) + | assignment_stmt + | while_stmt + | if_stmt + | print_stmt + | putc_stmt + | stmt_list +).setName("statement") code = stmt[...] code.ignore(pp.cppStyleComment) tests = [ - r''' + r""" count = 1; while (count < 10) { print("count is: ", count, "\n"); count = count + 1; } - ''', - r''' + """, + r""" /* Simple prime number generator */ @@ -110,64 +116,64 @@ tests = [ } } print("Total primes found: ", count, "\n"); - ''', - r''' + """, + r""" /* Hello world */ - print("Hello, World!\n"); - ''', - r''' + print("Hello, World!\n"); + """, + r""" /* Show Ident and Integers */ phoenix_number = 142857; print(phoenix_number, "\n"); - ''', - r''' + """, + r""" /*** test printing, embedded \n and comments with lots of '*' ***/ print(42); print("\nHello World\nGood Bye\nok\n"); print("Print a slash n - \\n.\n"); - ''', - r''' + """, + r""" /* 100 Doors */ i = 1; while (i * i <= 100) { print("door ", i * i, " is open\n"); i = i + 1; } - ''', - r''' + """, + r""" a = (-1 * ((-1 * (5 * 15)) / 10)); print(a, "\n"); b = -a; print(b, "\n"); print(-b, "\n"); print(-(1), "\n"); - ''', - r''' + """, + r""" print(---------------------------------+++5, "\n"); print(((((((((3 + 2) * ((((((2))))))))))))), "\n"); - + if (1) { if (1) { if (1) { if (1) { if (1) { print(15, "\n"); } } } } } - ''', - r''' + """, + r""" /* Compute the gcd of 1071, 1029: 21 */ - + a = 1071; b = 1029; - + while (b != 0) { new_a = b; b = a % b; a = new_a; } print(a); - ''', - r''' + """, + r""" /* 12 factorial is 479001600 */ - + n = 12; result = 1; i = 1; @@ -176,10 +182,10 @@ tests = [ i = i + 1; } print(result); - ''', - r''' + """, + r""" /* fibonacci of 44 is 701408733 */ - + n = 44; i = 1; a = 0; @@ -191,8 +197,8 @@ tests = [ i = i + 1; } print(w, "\n"); - ''', - r''' + """, + r""" /* FizzBuzz */ i = 1; while (i <= 100) { @@ -204,12 +210,12 @@ tests = [ print("Buzz"); else print(i); - + print("\n"); i = i + 1; } - ''', - r''' + """, + r""" /* 99 bottles */ bottles = 99; while (bottles > 0) { @@ -219,8 +225,8 @@ tests = [ bottles = bottles - 1; print(bottles, " bottles of beer on the wall\n\n"); } - ''', - r''' + """, + r""" { /* This is an integer ascii Mandelbrot generator @@ -231,9 +237,9 @@ tests = [ bottom_edge = -300; x_step = 7; y_step = 15; - + max_iter = 200; - + y0 = top_edge; while (y0 > bottom_edge) { x0 = left_edge; @@ -263,10 +269,11 @@ tests = [ y0 = y0 - y_step; } } - ''', + """, ] import sys + sys.setrecursionlimit(2000) for test in tests: diff --git a/examples/scanExamples.py b/examples/scanExamples.py index 4ee62a1..91d0739 100644 --- a/examples/scanExamples.py +++ b/examples/scanExamples.py @@ -5,8 +5,17 @@ #
# Copyright (c) 2004, 2006 Paul McGuire
#
-from pyparsing import Word, alphas, alphanums, Literal, restOfLine, OneOrMore, \
- empty, Suppress, replaceWith
+from pyparsing import (
+ Word,
+ alphas,
+ alphanums,
+ Literal,
+ restOfLine,
+ OneOrMore,
+ empty,
+ Suppress,
+ replaceWith,
+)
# simulate some C++ code
testData = """
@@ -24,10 +33,15 @@ print("Example of an extractor") print("----------------------")
# simple grammar to match #define's
-ident = Word(alphas, alphanums+"_")
-macroDef = Literal("#define") + ident.setResultsName("name") + "=" + restOfLine.setResultsName("value")
-for t,s,e in macroDef.scanString( testData ):
- print(t.name,":", t.value)
+ident = Word(alphas, alphanums + "_")
+macroDef = (
+ Literal("#define")
+ + ident.setResultsName("name")
+ + "="
+ + restOfLine.setResultsName("value")
+)
+for t, s, e in macroDef.scanString(testData):
+ print(t.name, ":", t.value)
# or a quick way to make a dictionary of the names and values
# (return only key and value tokens, and construct dict from key-value pairs)
@@ -43,23 +57,24 @@ print("Examples of a transformer") print("----------------------")
# convert C++ namespaces to mangled C-compatible names
-scopedIdent = ident + OneOrMore( Literal("::").suppress() + ident )
+scopedIdent = ident + OneOrMore(Literal("::").suppress() + ident)
scopedIdent.setParseAction(lambda t: "_".join(t))
print("(replace namespace-scoped names with C-compatible names)")
-print(scopedIdent.transformString( testData ))
+print(scopedIdent.transformString(testData))
# or a crude pre-processor (use parse actions to replace matching text)
-def substituteMacro(s,l,t):
+def substituteMacro(s, l, t):
if t[0] in macros:
return macros[t[0]]
-ident.setParseAction( substituteMacro )
+
+
+ident.setParseAction(substituteMacro)
ident.ignore(macroDef)
print("(simulate #define pre-processor)")
-print(ident.transformString( testData ))
-
+print(ident.transformString(testData))
#################
@@ -70,6 +85,6 @@ from pyparsing import dblQuotedString, LineStart # remove all string macro definitions (after extracting to a string resource table?)
stringMacroDef = Literal("#define") + ident + "=" + dblQuotedString + LineStart()
-stringMacroDef.setParseAction( replaceWith("") )
+stringMacroDef.setParseAction(replaceWith(""))
-print(stringMacroDef.transformString( testData ))
+print(stringMacroDef.transformString(testData))
diff --git a/examples/searchParserAppDemo.py b/examples/searchParserAppDemo.py index d1bf8ba..c608132 100644 --- a/examples/searchParserAppDemo.py +++ b/examples/searchParserAppDemo.py @@ -1,15 +1,24 @@ from searchparser import SearchQueryParser
-products = [ "grape juice", "grape jelly", "orange juice", "orange jujubees",
- "strawberry jam", "prune juice", "prune butter", "orange marmalade",
- "grapefruit juice" ]
+products = [
+ "grape juice",
+ "grape jelly",
+ "orange juice",
+ "orange jujubees",
+ "strawberry jam",
+ "prune juice",
+ "prune butter",
+ "orange marmalade",
+ "grapefruit juice",
+]
+
class FruitSearchParser(SearchQueryParser):
def GetWord(self, word):
- return { p for p in products if p.startswith(word + " ") }
+ return {p for p in products if p.startswith(word + " ")}
def GetWordWildcard(self, word):
- return { p for p in products if p.startswith(word[:-1]) }
+ return {p for p in products if p.startswith(word[:-1])}
def GetQuotes(self, search_string, tmp_result):
result = set()
@@ -17,7 +26,7 @@ class FruitSearchParser(SearchQueryParser): return result
def GetNot(self, not_set):
- return set( products ) - not_set
+ return set(products) - not_set
parser = FruitSearchParser()
@@ -31,4 +40,4 @@ tests = """\ for t in tests:
print(t.strip())
print(parser.Parse(t))
- print('')
+ print("")
diff --git a/examples/searchparser.py b/examples/searchparser.py index 30231b0..4284cc3 100644 --- a/examples/searchparser.py +++ b/examples/searchparser.py @@ -57,19 +57,29 @@ TODO: - ask someone to check my English texts
- add more kinds of wildcards ('*' at the beginning and '*' inside a word)?
"""
-from pyparsing import Word, alphanums, Keyword, Group, Combine, Forward, Suppress, OneOrMore, oneOf
+from pyparsing import (
+ Word,
+ alphanums,
+ Keyword,
+ Group,
+ Combine,
+ Forward,
+ Suppress,
+ OneOrMore,
+ oneOf,
+)
-class SearchQueryParser:
+class SearchQueryParser:
def __init__(self):
self._methods = {
- 'and': self.evaluateAnd,
- 'or': self.evaluateOr,
- 'not': self.evaluateNot,
- 'parenthesis': self.evaluateParenthesis,
- 'quotes': self.evaluateQuotes,
- 'word': self.evaluateWord,
- 'wordwildcard': self.evaluateWordWildcard,
+ "and": self.evaluateAnd,
+ "or": self.evaluateOr,
+ "not": self.evaluateNot,
+ "parenthesis": self.evaluateParenthesis,
+ "quotes": self.evaluateQuotes,
+ "word": self.evaluateWord,
+ "wordwildcard": self.evaluateWordWildcard,
}
self._parser = self.parser()
@@ -90,37 +100,52 @@ class SearchQueryParser: """
operatorOr = Forward()
- operatorWord = Group(Combine(Word(alphanums) + Suppress('*'))).setResultsName('wordwildcard') | \
- Group(Word(alphanums)).setResultsName('word')
+ operatorWord = Group(Combine(Word(alphanums) + Suppress("*"))).setResultsName(
+ "wordwildcard"
+ ) | Group(Word(alphanums)).setResultsName("word")
operatorQuotesContent = Forward()
- operatorQuotesContent << (
- (operatorWord + operatorQuotesContent) | operatorWord
- )
+ operatorQuotesContent << ((operatorWord + operatorQuotesContent) | operatorWord)
- operatorQuotes = Group(
- Suppress('"') + operatorQuotesContent + Suppress('"')
- ).setResultsName("quotes") | operatorWord
+ operatorQuotes = (
+ Group(Suppress('"') + operatorQuotesContent + Suppress('"')).setResultsName(
+ "quotes"
+ )
+ | operatorWord
+ )
- operatorParenthesis = Group(
- Suppress("(") + operatorOr + Suppress(")")
- ).setResultsName("parenthesis") | operatorQuotes
+ operatorParenthesis = (
+ Group(Suppress("(") + operatorOr + Suppress(")")).setResultsName(
+ "parenthesis"
+ )
+ | operatorQuotes
+ )
operatorNot = Forward()
- operatorNot << (Group(
- Suppress(Keyword("not", caseless=True)) + operatorNot
- ).setResultsName("not") | operatorParenthesis)
+ operatorNot << (
+ Group(Suppress(Keyword("not", caseless=True)) + operatorNot).setResultsName(
+ "not"
+ )
+ | operatorParenthesis
+ )
operatorAnd = Forward()
- operatorAnd << (Group(
- operatorNot + Suppress(Keyword("and", caseless=True)) + operatorAnd
- ).setResultsName("and") | Group(
- operatorNot + OneOrMore(~oneOf("and or") + operatorAnd)
- ).setResultsName("and") | operatorNot)
+ operatorAnd << (
+ Group(
+ operatorNot + Suppress(Keyword("and", caseless=True)) + operatorAnd
+ ).setResultsName("and")
+ | Group(
+ operatorNot + OneOrMore(~oneOf("and or") + operatorAnd)
+ ).setResultsName("and")
+ | operatorNot
+ )
- operatorOr << (Group(
- operatorAnd + Suppress(Keyword("or", caseless=True)) + operatorOr
- ).setResultsName("or") | operatorAnd)
+ operatorOr << (
+ Group(
+ operatorAnd + Suppress(Keyword("or", caseless=True)) + operatorOr
+ ).setResultsName("or")
+ | operatorAnd
+ )
return operatorOr.parseString
@@ -151,7 +176,7 @@ class SearchQueryParser: r = self.evaluate(item)
else:
r = r.intersection(self.evaluate(item))
- return self.GetQuotes(' '.join(search_terms), r)
+ return self.GetQuotes(" ".join(search_terms), r)
def evaluateWord(self, argument):
return self.GetWord(argument[0])
@@ -163,7 +188,7 @@ class SearchQueryParser: return self._methods[argument.getName()](argument)
def Parse(self, query):
- #print self._parser(query)[0]
+ # print self._parser(query)[0]
return self.evaluate(self._parser(query)[0])
def GetWord(self, word):
@@ -183,70 +208,71 @@ class ParserTest(SearchQueryParser): """Tests the parser with some search queries
tests containts a dictionary with tests and expected results.
"""
+
tests = {
- 'help': {1, 2, 4, 5},
- 'help or hulp': {1, 2, 3, 4, 5},
- 'help and hulp': {2},
- 'help hulp': {2},
- 'help and hulp or hilp': {2, 3, 4},
- 'help or hulp and hilp': {1, 2, 3, 4, 5},
- 'help or hulp or hilp or halp': {1, 2, 3, 4, 5, 6},
- '(help or hulp) and (hilp or halp)': {3, 4, 5},
- 'help and (hilp or halp)': {4, 5},
- '(help and (hilp or halp)) or hulp': {2, 3, 4, 5},
- 'not help': {3, 6, 7, 8},
- 'not hulp and halp': {5, 6},
- 'not (help and halp)': {1, 2, 3, 4, 6, 7, 8},
+ "help": {1, 2, 4, 5},
+ "help or hulp": {1, 2, 3, 4, 5},
+ "help and hulp": {2},
+ "help hulp": {2},
+ "help and hulp or hilp": {2, 3, 4},
+ "help or hulp and hilp": {1, 2, 3, 4, 5},
+ "help or hulp or hilp or halp": {1, 2, 3, 4, 5, 6},
+ "(help or hulp) and (hilp or halp)": {3, 4, 5},
+ "help and (hilp or halp)": {4, 5},
+ "(help and (hilp or halp)) or hulp": {2, 3, 4, 5},
+ "not help": {3, 6, 7, 8},
+ "not hulp and halp": {5, 6},
+ "not (help and halp)": {1, 2, 3, 4, 6, 7, 8},
'"help me please"': {2},
'"help me please" or hulp': {2, 3},
'"help me please" or (hulp and halp)': {2},
- 'help*': {1, 2, 4, 5, 8},
- 'help or hulp*': {1, 2, 3, 4, 5},
- 'help* and hulp': {2},
- 'help and hulp* or hilp': {2, 3, 4},
- 'help* or hulp or hilp or halp': {1, 2, 3, 4, 5, 6, 8},
- '(help or hulp*) and (hilp* or halp)': {3, 4, 5},
- 'help* and (hilp* or halp*)': {4, 5},
- '(help and (hilp* or halp)) or hulp*': {2, 3, 4, 5},
- 'not help* and halp': {6},
- 'not (help* and helpe*)': {1, 2, 3, 4, 5, 6, 7},
+ "help*": {1, 2, 4, 5, 8},
+ "help or hulp*": {1, 2, 3, 4, 5},
+ "help* and hulp": {2},
+ "help and hulp* or hilp": {2, 3, 4},
+ "help* or hulp or hilp or halp": {1, 2, 3, 4, 5, 6, 8},
+ "(help or hulp*) and (hilp* or halp)": {3, 4, 5},
+ "help* and (hilp* or halp*)": {4, 5},
+ "(help and (hilp* or halp)) or hulp*": {2, 3, 4, 5},
+ "not help* and halp": {6},
+ "not (help* and helpe*)": {1, 2, 3, 4, 5, 6, 7},
'"help* me please"': {2},
'"help* me* please" or hulp*': {2, 3},
'"help me please*" or (hulp and halp)': {2},
'"help me please" not (hulp and halp)': {2},
'"help me please" hulp': {2},
- 'help and hilp and not holp': {4},
- 'help hilp not holp': {4},
- 'help hilp and not holp': {4},
+ "help and hilp and not holp": {4},
+ "help hilp not holp": {4},
+ "help hilp and not holp": {4},
}
docs = {
- 1: 'help',
- 2: 'help me please hulp',
- 3: 'hulp hilp',
- 4: 'help hilp',
- 5: 'halp thinks he needs help',
- 6: 'he needs halp',
- 7: 'nothing',
- 8: 'helper',
+ 1: "help",
+ 2: "help me please hulp",
+ 3: "hulp hilp",
+ 4: "help hilp",
+ 5: "halp thinks he needs help",
+ 6: "he needs halp",
+ 7: "nothing",
+ 8: "helper",
}
index = {
- 'help': {1, 2, 4, 5},
- 'me': {2},
- 'please': {2},
- 'hulp': {2, 3},
- 'hilp': {3, 4},
- 'halp': {5, 6},
- 'thinks': {5},
- 'he': {5, 6},
- 'needs': {5, 6},
- 'nothing': {7},
- 'helper': {8},
+ "help": {1, 2, 4, 5},
+ "me": {2},
+ "please": {2},
+ "hulp": {2, 3},
+ "hilp": {3, 4},
+ "halp": {5, 6},
+ "thinks": {5},
+ "he": {5, 6},
+ "needs": {5, 6},
+ "nothing": {7},
+ "helper": {8},
}
def GetWord(self, word):
- if (word in self.index):
+ if word in self.index:
return self.index[word]
else:
return set()
@@ -254,7 +280,7 @@ class ParserTest(SearchQueryParser): def GetWordWildcard(self, word):
result = set()
for item in list(self.index.keys()):
- if word == item[0:len(word)]:
+ if word == item[0 : len(word)]:
result = result.union(self.index[item])
return result
@@ -275,18 +301,19 @@ class ParserTest(SearchQueryParser): print(item)
r = self.Parse(item)
e = self.tests[item]
- print('Result: %s' % r)
- print('Expect: %s' % e)
+ print("Result: %s" % r)
+ print("Expect: %s" % e)
if e == r:
- print('Test OK')
+ print("Test OK")
else:
all_ok = False
- print('>>>>>>>>>>>>>>>>>>>>>>Test ERROR<<<<<<<<<<<<<<<<<<<<<')
- print('')
+ print(">>>>>>>>>>>>>>>>>>>>>>Test ERROR<<<<<<<<<<<<<<<<<<<<<")
+ print("")
return all_ok
-if __name__=='__main__':
+
+if __name__ == "__main__":
if ParserTest().Test():
- print('All tests OK')
+ print("All tests OK")
else:
- print('One or more tests FAILED')
+ print("One or more tests FAILED")
diff --git a/examples/select_parser.py b/examples/select_parser.py index 7f9273c..723f8b2 100644 --- a/examples/select_parser.py +++ b/examples/select_parser.py @@ -5,29 +5,140 @@ # definition at https://www.sqlite.org/lang_select.html # from pyparsing import * + ParserElement.enablePackrat() -LPAR,RPAR,COMMA = map(Suppress,"(),") -DOT,STAR = map(Literal, ".*") +LPAR, RPAR, COMMA = map(Suppress, "(),") +DOT, STAR = map(Literal, ".*") select_stmt = Forward().setName("select statement") # keywords -(UNION, ALL, AND, INTERSECT, EXCEPT, COLLATE, ASC, DESC, ON, USING, NATURAL, INNER, - CROSS, LEFT, OUTER, JOIN, AS, INDEXED, NOT, SELECT, DISTINCT, FROM, WHERE, GROUP, BY, - HAVING, ORDER, BY, LIMIT, OFFSET, OR) = map(CaselessKeyword, """UNION, ALL, AND, INTERSECT, +( + UNION, + ALL, + AND, + INTERSECT, + EXCEPT, + COLLATE, + ASC, + DESC, + ON, + USING, + NATURAL, + INNER, + CROSS, + LEFT, + OUTER, + JOIN, + AS, + INDEXED, + NOT, + SELECT, + DISTINCT, + FROM, + WHERE, + GROUP, + BY, + HAVING, + ORDER, + BY, + LIMIT, + OFFSET, + OR, +) = map( + CaselessKeyword, + """UNION, ALL, AND, INTERSECT, EXCEPT, COLLATE, ASC, DESC, ON, USING, NATURAL, INNER, CROSS, LEFT, OUTER, JOIN, AS, INDEXED, NOT, SELECT, - DISTINCT, FROM, WHERE, GROUP, BY, HAVING, ORDER, BY, LIMIT, OFFSET, OR""".replace(",","").split()) -(CAST, ISNULL, NOTNULL, NULL, IS, BETWEEN, ELSE, END, CASE, WHEN, THEN, EXISTS, IN, LIKE, GLOB, REGEXP, - MATCH, ESCAPE, CURRENT_TIME, CURRENT_DATE, CURRENT_TIMESTAMP) = map(CaselessKeyword, - """CAST, ISNULL, NOTNULL, NULL, IS, BETWEEN, ELSE, END, CASE, WHEN, THEN, EXISTS, IN, LIKE, GLOB, - REGEXP, MATCH, ESCAPE, CURRENT_TIME, CURRENT_DATE, CURRENT_TIMESTAMP""".replace(",","").split()) -keyword = MatchFirst((UNION, ALL, INTERSECT, EXCEPT, COLLATE, ASC, DESC, ON, USING, NATURAL, INNER, - CROSS, LEFT, OUTER, JOIN, AS, INDEXED, NOT, SELECT, DISTINCT, FROM, WHERE, GROUP, BY, - HAVING, ORDER, BY, LIMIT, OFFSET, CAST, ISNULL, NOTNULL, NULL, IS, BETWEEN, ELSE, END, - CASE, WHEN, THEN, EXISTS, COLLATE, IN, LIKE, GLOB, REGEXP, MATCH, ESCAPE, - CURRENT_TIME, CURRENT_DATE, CURRENT_TIMESTAMP)) - -identifier = ~keyword + Word(alphas, alphanums+"_") + DISTINCT, FROM, WHERE, GROUP, BY, HAVING, ORDER, BY, LIMIT, OFFSET, OR""".replace( + ",", "" + ).split(), +) +( + CAST, + ISNULL, + NOTNULL, + NULL, + IS, + BETWEEN, + ELSE, + END, + CASE, + WHEN, + THEN, + EXISTS, + IN, + LIKE, + GLOB, + REGEXP, + MATCH, + ESCAPE, + CURRENT_TIME, + CURRENT_DATE, + CURRENT_TIMESTAMP, +) = map( + CaselessKeyword, + """CAST, ISNULL, NOTNULL, NULL, IS, BETWEEN, ELSE, END, CASE, WHEN, THEN, EXISTS, IN, LIKE, GLOB, + REGEXP, MATCH, ESCAPE, CURRENT_TIME, CURRENT_DATE, CURRENT_TIMESTAMP""".replace( + ",", "" + ).split(), +) +keyword = MatchFirst( + ( + UNION, + ALL, + INTERSECT, + EXCEPT, + COLLATE, + ASC, + DESC, + ON, + USING, + NATURAL, + INNER, + CROSS, + LEFT, + OUTER, + JOIN, + AS, + INDEXED, + NOT, + SELECT, + DISTINCT, + FROM, + WHERE, + GROUP, + BY, + HAVING, + ORDER, + BY, + LIMIT, + OFFSET, + CAST, + ISNULL, + NOTNULL, + NULL, + IS, + BETWEEN, + ELSE, + END, + CASE, + WHEN, + THEN, + EXISTS, + COLLATE, + IN, + LIKE, + GLOB, + REGEXP, + MATCH, + ESCAPE, + CURRENT_TIME, + CURRENT_DATE, + CURRENT_TIMESTAMP, + ) +) + +identifier = ~keyword + Word(alphas, alphanums + "_") collation_name = identifier.copy() column_name = identifier.copy() column_alias = identifier.copy() @@ -46,87 +157,120 @@ numeric_literal = Regex(r"\d+(\.\d*)?([eE][+-]?\d+)?") string_literal = QuotedString("'") blob_literal = Regex(r"[xX]'[0-9A-Fa-f]+'") literal_value = ( - numeric_literal - | string_literal - | blob_literal - | NULL - | CURRENT_TIME - | CURRENT_DATE - | CURRENT_TIMESTAMP - ) -bind_parameter = ( - Word("?",nums) - | Combine(oneOf(": @ $") + parameter_name) - ) + numeric_literal + | string_literal + | blob_literal + | NULL + | CURRENT_TIME + | CURRENT_DATE + | CURRENT_TIMESTAMP +) +bind_parameter = Word("?", nums) | Combine(oneOf(": @ $") + parameter_name) type_name = oneOf("TEXT REAL INTEGER BLOB NULL") expr_term = ( CAST + LPAR + expr + AS + type_name + RPAR | EXISTS + LPAR + select_stmt + RPAR - | function_name.setName("function_name") + LPAR + Optional(STAR | delimitedList(expr)) + RPAR + | function_name.setName("function_name") + + LPAR + + Optional(STAR | delimitedList(expr)) + + RPAR | literal_value | bind_parameter - | Group(identifier('col_db') + DOT + identifier('col_tab') + DOT + identifier('col')) - | Group(identifier('col_tab') + DOT + identifier('col')) - | Group(identifier('col')) + | Group( + identifier("col_db") + DOT + identifier("col_tab") + DOT + identifier("col") ) + | Group(identifier("col_tab") + DOT + identifier("col")) + | Group(identifier("col")) +) -UNARY,BINARY,TERNARY=1,2,3 -expr << infixNotation(expr_term, +UNARY, BINARY, TERNARY = 1, 2, 3 +expr << infixNotation( + expr_term, [ - (oneOf('- + ~') | NOT, UNARY, opAssoc.RIGHT), - (ISNULL | NOTNULL | NOT + NULL, UNARY, opAssoc.LEFT), - ('||', BINARY, opAssoc.LEFT), - (oneOf('* / %'), BINARY, opAssoc.LEFT), - (oneOf('+ -'), BINARY, opAssoc.LEFT), - (oneOf('<< >> & |'), BINARY, opAssoc.LEFT), - (oneOf('< <= > >='), BINARY, opAssoc.LEFT), - (oneOf('= == != <>') | IS | IN | LIKE | GLOB | MATCH | REGEXP, BINARY, opAssoc.LEFT), - ((BETWEEN,AND), TERNARY, opAssoc.LEFT), - (IN + LPAR + Group(select_stmt | delimitedList(expr)) + RPAR, UNARY, opAssoc.LEFT), - (AND, BINARY, opAssoc.LEFT), - (OR, BINARY, opAssoc.LEFT), - ]) - -compound_operator = (UNION + Optional(ALL) | INTERSECT | EXCEPT) - -ordering_term = Group(expr('order_key') - + Optional(COLLATE + collation_name('collate')) - + Optional(ASC | DESC)('direction')) - -join_constraint = Group(Optional(ON + expr | USING + LPAR + Group(delimitedList(column_name)) + RPAR)) - -join_op = COMMA | Group(Optional(NATURAL) + Optional(INNER | CROSS | LEFT + OUTER | LEFT | OUTER) + JOIN) + (oneOf("- + ~") | NOT, UNARY, opAssoc.RIGHT), + (ISNULL | NOTNULL | NOT + NULL, UNARY, opAssoc.LEFT), + ("||", BINARY, opAssoc.LEFT), + (oneOf("* / %"), BINARY, opAssoc.LEFT), + (oneOf("+ -"), BINARY, opAssoc.LEFT), + (oneOf("<< >> & |"), BINARY, opAssoc.LEFT), + (oneOf("< <= > >="), BINARY, opAssoc.LEFT), + ( + oneOf("= == != <>") | IS | IN | LIKE | GLOB | MATCH | REGEXP, + BINARY, + opAssoc.LEFT, + ), + ((BETWEEN, AND), TERNARY, opAssoc.LEFT), + ( + IN + LPAR + Group(select_stmt | delimitedList(expr)) + RPAR, + UNARY, + opAssoc.LEFT, + ), + (AND, BINARY, opAssoc.LEFT), + (OR, BINARY, opAssoc.LEFT), + ], +) + +compound_operator = UNION + Optional(ALL) | INTERSECT | EXCEPT + +ordering_term = Group( + expr("order_key") + + Optional(COLLATE + collation_name("collate")) + + Optional(ASC | DESC)("direction") +) + +join_constraint = Group( + Optional(ON + expr | USING + LPAR + Group(delimitedList(column_name)) + RPAR) +) + +join_op = COMMA | Group( + Optional(NATURAL) + Optional(INNER | CROSS | LEFT + OUTER | LEFT | OUTER) + JOIN +) join_source = Forward() single_source = ( - Group(database_name("database") + DOT + table_name("table*") - | table_name("table*")) - + Optional(Optional(AS) + table_alias("table_alias*")) - + Optional(INDEXED + BY + index_name("name") | NOT + INDEXED)("index") - | (LPAR + select_stmt + RPAR + Optional(Optional(AS) + table_alias)) - | (LPAR + join_source + RPAR) - ) + Group(database_name("database") + DOT + table_name("table*") | table_name("table*")) + + Optional(Optional(AS) + table_alias("table_alias*")) + + Optional(INDEXED + BY + index_name("name") | NOT + INDEXED)("index") + | (LPAR + select_stmt + RPAR + Optional(Optional(AS) + table_alias)) + | (LPAR + join_source + RPAR) +) -join_source <<= (Group(single_source + OneOrMore(join_op + single_source + join_constraint)) - | single_source) +join_source <<= ( + Group(single_source + OneOrMore(join_op + single_source + join_constraint)) + | single_source +) # result_column = "*" | table_name + "." + "*" | Group(expr + Optional(Optional(AS) + column_alias)) -result_column = Group(STAR('col') - | table_name('col_table') + DOT + STAR('col') - | expr('col') + Optional(Optional(AS) + column_alias('alias')) - ) - -select_core = (SELECT + Optional(DISTINCT | ALL) + Group(delimitedList(result_column))("columns") - + Optional(FROM + join_source("from*")) - + Optional(WHERE + expr("where_expr")) - + Optional(GROUP + BY + Group(delimitedList(ordering_term))("group_by_terms") - + Optional(HAVING + expr("having_expr")))) - -select_stmt << (select_core + ZeroOrMore(compound_operator + select_core) - + Optional(ORDER + BY + Group(delimitedList(ordering_term))("order_by_terms")) - + Optional(LIMIT + (Group(expr + OFFSET + expr) | Group(expr + COMMA + expr) | expr)("limit")) - ) +result_column = Group( + STAR("col") + | table_name("col_table") + DOT + STAR("col") + | expr("col") + Optional(Optional(AS) + column_alias("alias")) +) + +select_core = ( + SELECT + + Optional(DISTINCT | ALL) + + Group(delimitedList(result_column))("columns") + + Optional(FROM + join_source("from*")) + + Optional(WHERE + expr("where_expr")) + + Optional( + GROUP + + BY + + Group(delimitedList(ordering_term))("group_by_terms") + + Optional(HAVING + expr("having_expr")) + ) +) + +select_stmt << ( + select_core + + ZeroOrMore(compound_operator + select_core) + + Optional(ORDER + BY + Group(delimitedList(ordering_term))("order_by_terms")) + + Optional( + LIMIT + + (Group(expr + OFFSET + expr) | Group(expr + COMMA + expr) | expr)("limit") + ) +) tests = """\ select * from xyzzy where z > 100 diff --git a/examples/sexpParser.py b/examples/sexpParser.py index 5c4f14d..2a0f2c7 100644 --- a/examples/sexpParser.py +++ b/examples/sexpParser.py @@ -52,28 +52,40 @@ def verify_length(s, l, t): if t.len is not None: t1len = len(t[1]) if t1len != t.len: - raise pp.ParseFatalException(s, l, "invalid data of length {}, expected {}".format(t1len, t.len)) + raise pp.ParseFatalException( + s, l, "invalid data of length {}, expected {}".format(t1len, t.len) + ) return t[1] # define punctuation literals -LPAR, RPAR, LBRK, RBRK, LBRC, RBRC, VBAR, COLON = (pp.Suppress(c).setName(c) for c in "()[]{}|:") +LPAR, RPAR, LBRK, RBRK, LBRC, RBRC, VBAR, COLON = ( + pp.Suppress(c).setName(c) for c in "()[]{}|:" +) -decimal = pp.Regex(r'-?0|[1-9]\d*').setParseAction(lambda t: int(t[0])) -hexadecimal = ("#" + pp.Word(pp.hexnums)[1, ...] + "#").setParseAction(lambda t: int("".join(t[1:-1]), 16)) +decimal = pp.Regex(r"-?0|[1-9]\d*").setParseAction(lambda t: int(t[0])) +hexadecimal = ("#" + pp.Word(pp.hexnums)[1, ...] + "#").setParseAction( + lambda t: int("".join(t[1:-1]), 16) +) bytes = pp.Word(pp.printables) raw = pp.Group(decimal("len") + COLON + bytes).setParseAction(verify_length) -base64_ = pp.Group(pp.Optional(decimal | hexadecimal, default=None)("len") - + VBAR - + pp.Word(pp.alphanums + "+/=")[1, ...].setParseAction(lambda t: b64decode("".join(t))) - + VBAR - ).setParseAction(verify_length) +base64_ = pp.Group( + pp.Optional(decimal | hexadecimal, default=None)("len") + + VBAR + + pp.Word(pp.alphanums + "+/=")[1, ...].setParseAction( + lambda t: b64decode("".join(t)) + ) + + VBAR +).setParseAction(verify_length) -real = pp.Regex(r"[+-]?\d+\.\d*([eE][+-]?\d+)?").setParseAction(lambda tokens: float(tokens[0])) +real = pp.Regex(r"[+-]?\d+\.\d*([eE][+-]?\d+)?").setParseAction( + lambda tokens: float(tokens[0]) +) token = pp.Word(pp.alphanums + "-./_:*+=!<>") -qString = pp.Group(pp.Optional(decimal, default=None)("len") - + pp.dblQuotedString.setParseAction(pp.removeQuotes) - ).setParseAction(verify_length) +qString = pp.Group( + pp.Optional(decimal, default=None)("len") + + pp.dblQuotedString.setParseAction(pp.removeQuotes) +).setParseAction(verify_length) simpleString = real | base64_ | raw | decimal | token | hexadecimal | qString @@ -151,6 +163,8 @@ test52 = """ """ # Run tests -alltests = [globals()[testname] for testname in sorted(locals()) if testname.startswith("test")] +alltests = [ + globals()[testname] for testname in sorted(locals()) if testname.startswith("test") +] sexp.runTests(alltests, fullDump=False) diff --git a/examples/shapes.py b/examples/shapes.py index b0fe979..5f621b1 100644 --- a/examples/shapes.py +++ b/examples/shapes.py @@ -17,37 +17,45 @@ class Shape: def __str__(self): return "<{}>: {}".format(self.__class__.__name__, vars(self)) + class Square(Shape): def area(self): - return self.side**2 + return self.side ** 2 + class Rectangle(Shape): def area(self): return self.width * self.height + class Circle(Shape): def area(self): - return 3.14159 * self.radius**2 + return 3.14159 * self.radius ** 2 import pyparsing as pp -number = pp.Regex(r'-?\d+(\.\d*)?').setParseAction(lambda t: float(t[0])) +number = pp.Regex(r"-?\d+(\.\d*)?").setParseAction(lambda t: float(t[0])) # Shape expressions: # square : S <centerx> <centery> <side> # rectangle: R <centerx> <centery> <width> <height> # circle : C <centerx> <centery> <diameter> -squareDefn = "S" + number('centerx') + number('centery') + number('side') -rectDefn = "R" + number('centerx') + number('centery') + number('width') + number('height') -circleDefn = "C" + number('centerx') + number('centery') + number('diameter') +squareDefn = "S" + number("centerx") + number("centery") + number("side") +rectDefn = ( + "R" + number("centerx") + number("centery") + number("width") + number("height") +) +circleDefn = "C" + number("centerx") + number("centery") + number("diameter") squareDefn.setParseAction(Square) rectDefn.setParseAction(Rectangle) + def computeRadius(tokens): - tokens['radius'] = tokens.diameter/2.0 + tokens["radius"] = tokens.diameter / 2.0 + + circleDefn.setParseAction(computeRadius, Circle) shapeExpr = squareDefn | rectDefn | circleDefn diff --git a/examples/simpleArith.py b/examples/simpleArith.py index af05373..af3f904 100644 --- a/examples/simpleArith.py +++ b/examples/simpleArith.py @@ -9,15 +9,15 @@ from pyparsing import *
-integer = Word(nums).setParseAction(lambda t:int(t[0]))
-variable = Word(alphas,exact=1)
+integer = Word(nums).setParseAction(lambda t: int(t[0]))
+variable = Word(alphas, exact=1)
operand = integer | variable
-expop = Literal('^')
-signop = oneOf('+ -')
-multop = oneOf('* /')
-plusop = oneOf('+ -')
-factop = Literal('!')
+expop = Literal("^")
+signop = oneOf("+ -")
+multop = oneOf("* /")
+plusop = oneOf("+ -")
+factop = Literal("!")
# To use the infixNotation helper:
# 1. Define the "atom" operand term of the grammar.
@@ -43,24 +43,29 @@ factop = Literal('!') # this expression to parse input strings, or incorporate it
# into a larger, more complex grammar.
#
-expr = infixNotation( operand,
- [("!", 1, opAssoc.LEFT),
- ("^", 2, opAssoc.RIGHT),
- (signop, 1, opAssoc.RIGHT),
- (multop, 2, opAssoc.LEFT),
- (plusop, 2, opAssoc.LEFT),]
- )
+expr = infixNotation(
+ operand,
+ [
+ ("!", 1, opAssoc.LEFT),
+ ("^", 2, opAssoc.RIGHT),
+ (signop, 1, opAssoc.RIGHT),
+ (multop, 2, opAssoc.LEFT),
+ (plusop, 2, opAssoc.LEFT),
+ ],
+)
-test = ["9 + 2 + 3",
- "9 + 2 * 3",
- "(9 + 2) * 3",
- "(9 + -2) * 3",
- "(9 + -2) * 3^2^2",
- "(9! + -2) * 3^2^2",
- "M*X + B",
- "M*(X + B)",
- "1+2*-3^4*5+-+-6",]
+test = [
+ "9 + 2 + 3",
+ "9 + 2 * 3",
+ "(9 + 2) * 3",
+ "(9 + -2) * 3",
+ "(9 + -2) * 3^2^2",
+ "(9! + -2) * 3^2^2",
+ "M*X + B",
+ "M*(X + B)",
+ "1+2*-3^4*5+-+-6",
+]
for t in test:
print(t)
print(expr.parseString(t))
- print('')
+ print("")
diff --git a/examples/simpleBool.py b/examples/simpleBool.py index 81f5049..f47e090 100644 --- a/examples/simpleBool.py +++ b/examples/simpleBool.py @@ -17,84 +17,98 @@ from pyparsing import infixNotation, opAssoc, Keyword, Word, alphas # define classes to be built at parse time, as each matching
# expression type is parsed
class BoolOperand:
- def __init__(self,t):
+ def __init__(self, t):
self.label = t[0]
self.value = eval(t[0])
+
def __bool__(self):
return self.value
+
def __str__(self):
return self.label
+
__repr__ = __str__
class BoolBinOp:
- def __init__(self,t):
+ def __init__(self, t):
self.args = t[0][0::2]
+
def __str__(self):
sep = " %s " % self.reprsymbol
- return "(" + sep.join(map(str,self.args)) + ")"
+ return "(" + sep.join(map(str, self.args)) + ")"
+
def __bool__(self):
return self.evalop(bool(a) for a in self.args)
+
__nonzero__ = __bool__
class BoolAnd(BoolBinOp):
- reprsymbol = '&'
+ reprsymbol = "&"
evalop = all
+
class BoolOr(BoolBinOp):
- reprsymbol = '|'
+ reprsymbol = "|"
evalop = any
+
class BoolNot:
- def __init__(self,t):
+ def __init__(self, t):
self.arg = t[0][1]
+
def __bool__(self):
v = bool(self.arg)
return not v
+
def __str__(self):
return "~" + str(self.arg)
+
__repr__ = __str__
TRUE = Keyword("True")
FALSE = Keyword("False")
-boolOperand = TRUE | FALSE | Word(alphas,max=1)
+boolOperand = TRUE | FALSE | Word(alphas, max=1)
boolOperand.setParseAction(BoolOperand)
# define expression, based on expression operand and
# list of operations in precedence order
-boolExpr = infixNotation( boolOperand,
+boolExpr = infixNotation(
+ boolOperand,
[
- ("not", 1, opAssoc.RIGHT, BoolNot),
- ("and", 2, opAssoc.LEFT, BoolAnd),
- ("or", 2, opAssoc.LEFT, BoolOr),
- ])
+ ("not", 1, opAssoc.RIGHT, BoolNot),
+ ("and", 2, opAssoc.LEFT, BoolAnd),
+ ("or", 2, opAssoc.LEFT, BoolOr),
+ ],
+)
if __name__ == "__main__":
p = True
q = False
r = True
- tests = [("p", True),
- ("q", False),
- ("p and q", False),
- ("p and not q", True),
- ("not not p", True),
- ("not(p and q)", True),
- ("q or not p and r", False),
- ("q or not p or not r", False),
- ("q or not (p and r)", False),
- ("p or q or r", True),
- ("p or q or r and False", True),
- ("(p or q or r) and False", False),
- ]
+ tests = [
+ ("p", True),
+ ("q", False),
+ ("p and q", False),
+ ("p and not q", True),
+ ("not not p", True),
+ ("not(p and q)", True),
+ ("q or not p and r", False),
+ ("q or not p or not r", False),
+ ("q or not (p and r)", False),
+ ("p or q or r", True),
+ ("p or q or r and False", True),
+ ("(p or q or r) and False", False),
+ ]
print("p =", p)
print("q =", q)
print("r =", r)
print()
- for t,expected in tests:
+ for t, expected in tests:
res = boolExpr.parseString(t)[0]
success = "PASS" if bool(res) == expected else "FAIL"
- print (t,'\n', res, '=', bool(res),'\n', success, '\n')
+ print(t, "\n", res, "=", bool(res), "\n", success, "\n")
diff --git a/examples/simpleSQL.py b/examples/simpleSQL.py index 60582cd..a806ad6 100644 --- a/examples/simpleSQL.py +++ b/examples/simpleSQL.py @@ -5,57 +5,75 @@ # # Copyright (c) 2003,2016, Paul McGuire # -from pyparsing import Word, delimitedList, Optional, \ - Group, alphas, alphanums, Forward, oneOf, quotedString, \ - infixNotation, opAssoc, \ - restOfLine, CaselessKeyword, pyparsing_common as ppc +from pyparsing import ( + Word, + delimitedList, + Optional, + Group, + alphas, + alphanums, + Forward, + oneOf, + quotedString, + infixNotation, + opAssoc, + restOfLine, + CaselessKeyword, + pyparsing_common as ppc, +) # define SQL tokens selectStmt = Forward() -SELECT, FROM, WHERE, AND, OR, IN, IS, NOT, NULL = map(CaselessKeyword, - "select from where and or in is not null".split()) +SELECT, FROM, WHERE, AND, OR, IN, IS, NOT, NULL = map( + CaselessKeyword, "select from where and or in is not null".split() +) NOT_NULL = NOT + NULL -ident = Word( alphas, alphanums + "_$" ).setName("identifier") -columnName = delimitedList(ident, ".", combine=True).setName("column name") +ident = Word(alphas, alphanums + "_$").setName("identifier") +columnName = delimitedList(ident, ".", combine=True).setName("column name") columnName.addParseAction(ppc.upcaseTokens) -columnNameList = Group( delimitedList(columnName)) -tableName = delimitedList(ident, ".", combine=True).setName("table name") +columnNameList = Group(delimitedList(columnName)) +tableName = delimitedList(ident, ".", combine=True).setName("table name") tableName.addParseAction(ppc.upcaseTokens) -tableNameList = Group(delimitedList(tableName)) +tableNameList = Group(delimitedList(tableName)) binop = oneOf("= != < > >= <= eq ne lt le gt ge", caseless=True) realNum = ppc.real() intNum = ppc.signed_integer() -columnRval = realNum | intNum | quotedString | columnName # need to add support for alg expressions +columnRval = ( + realNum | intNum | quotedString | columnName +) # need to add support for alg expressions whereCondition = Group( - ( columnName + binop + columnRval ) | - ( columnName + IN + Group("(" + delimitedList( columnRval ) + ")" )) | - ( columnName + IN + Group("(" + selectStmt + ")" )) | - ( columnName + IS + (NULL | NOT_NULL)) - ) + (columnName + binop + columnRval) + | (columnName + IN + Group("(" + delimitedList(columnRval) + ")")) + | (columnName + IN + Group("(" + selectStmt + ")")) + | (columnName + IS + (NULL | NOT_NULL)) +) -whereExpression = infixNotation(whereCondition, - [ - (NOT, 1, opAssoc.RIGHT), - (AND, 2, opAssoc.LEFT), - (OR, 2, opAssoc.LEFT), - ]) +whereExpression = infixNotation( + whereCondition, + [(NOT, 1, opAssoc.RIGHT), (AND, 2, opAssoc.LEFT), (OR, 2, opAssoc.LEFT),], +) # define the grammar -selectStmt <<= (SELECT + ('*' | columnNameList)("columns") + - FROM + tableNameList( "tables" ) + - Optional(Group(WHERE + whereExpression), "")("where")) +selectStmt <<= ( + SELECT + + ("*" | columnNameList)("columns") + + FROM + + tableNameList("tables") + + Optional(Group(WHERE + whereExpression), "")("where") +) simpleSQL = selectStmt # define Oracle comment format, and ignore them oracleSqlComment = "--" + restOfLine -simpleSQL.ignore( oracleSqlComment ) +simpleSQL.ignore(oracleSqlComment) if __name__ == "__main__": - simpleSQL.runTests("""\ + simpleSQL.runTests( + """\ # multiple tables SELECT * from XYZZY, ABC @@ -92,4 +110,5 @@ if __name__ == "__main__": # where clause with comparison operator Select A,b from table1,table2 where table1.id eq table2.id - """) + """ + ) diff --git a/examples/simpleWiki.py b/examples/simpleWiki.py index ca660c5..73c8116 100644 --- a/examples/simpleWiki.py +++ b/examples/simpleWiki.py @@ -8,22 +8,28 @@ Here is a simple Wiki input: Here's a URL to {{Pyparsing's Wiki Page->https://site-closed.wikispaces.com}}
"""
-def convertToHTML(opening,closing):
- def conversionParseAction(s,l,t):
+
+def convertToHTML(opening, closing):
+ def conversionParseAction(s, l, t):
return opening + t[0] + closing
+
return conversionParseAction
-italicized = QuotedString("*").setParseAction(convertToHTML("<I>","</I>"))
-bolded = QuotedString("**").setParseAction(convertToHTML("<B>","</B>"))
-boldItalicized = QuotedString("***").setParseAction(convertToHTML("<B><I>","</I></B>"))
-def convertToHTML_A(s,l,t):
+
+italicized = QuotedString("*").setParseAction(convertToHTML("<I>", "</I>"))
+bolded = QuotedString("**").setParseAction(convertToHTML("<B>", "</B>"))
+boldItalicized = QuotedString("***").setParseAction(convertToHTML("<B><I>", "</I></B>"))
+
+
+def convertToHTML_A(s, l, t):
try:
- text,url=t[0].split("->")
+ text, url = t[0].split("->")
except ValueError:
- raise ParseFatalException(s,l,"invalid URL link reference: " + t[0])
+ raise ParseFatalException(s, l, "invalid URL link reference: " + t[0])
return '<A href="{}">{}</A>'.format(url, text)
-urlRef = QuotedString("{{",endQuoteChar="}}").setParseAction(convertToHTML_A)
+
+urlRef = QuotedString("{{", endQuoteChar="}}").setParseAction(convertToHTML_A)
wikiMarkup = urlRef | boldItalicized | bolded | italicized
diff --git a/examples/sparser.py b/examples/sparser.py index 39758d6..49617ff 100644 --- a/examples/sparser.py +++ b/examples/sparser.py @@ -43,7 +43,7 @@ EXAMPLES: # 675 Mass Ave, Cambridge, MA 02139, USA.
"""
-#===imports======================
+# ===imports======================
import sys
import os
import getopt
@@ -51,57 +51,63 @@ import getopt from pyparsing import *
-#===globals======================
+# ===globals======================
modname = "sparser"
__version__ = "0.1"
-#--option args--
+# --option args--
debug_p = 0
-#opt_b=None #string arg, default is undefined
+# opt_b=None #string arg, default is undefined
-#---positional args, default is empty---
+# ---positional args, default is empty---
pargs = []
-#---other---
+# ---other---
-#===utilities====================
+# ===utilities====================
def msg(txt):
"""Send message to stdout."""
sys.stdout.write(txt)
sys.stdout.flush()
+
def debug(ftn, txt):
"""Used for debugging."""
if debug_p:
sys.stdout.write("{}.{}:{}\n".format(modname, ftn, txt))
sys.stdout.flush()
+
def fatal(ftn, txt):
"""If can't continue."""
msg = "{}.{}:FATAL:{}\n".format(modname, ftn, txt)
raise SystemExit(msg)
+
def usage():
"""Prints the docstring."""
print(__doc__)
-
-#====================================
+# ====================================
class ToInteger(TokenConverter):
"""Converter to make token into an integer."""
- def postParse( self, instring, loc, tokenlist ):
+
+ def postParse(self, instring, loc, tokenlist):
return int(tokenlist[0])
+
class ToFloat(TokenConverter):
"""Converter to make token into a float."""
- def postParse( self, instring, loc, tokenlist ):
+
+ def postParse(self, instring, loc, tokenlist):
return float(tokenlist[0])
+
class ParseFileLineByLine:
"""
Bring data from text files into a program, optionally parsing each line
@@ -133,47 +139,42 @@ class ParseFileLineByLine: '"a"' (append, not supported for .Z files).
"""
- def __init__(self, filename, mode = 'r'):
+ def __init__(self, filename, mode="r"):
"""Opens input file, and if available the definition file. If the
definition file is available __init__ will then create some pyparsing
helper variables. """
- if mode not in ['r', 'w', 'a']:
- raise OSError(0, 'Illegal mode: ' + repr(mode))
+ if mode not in ["r", "w", "a"]:
+ raise OSError(0, "Illegal mode: " + repr(mode))
- if string.find(filename, ':/') > 1: # URL
- if mode == 'w':
+ if string.find(filename, ":/") > 1: # URL
+ if mode == "w":
raise OSError("can't write to a URL")
import urllib.request, urllib.parse, urllib.error
+
self.file = urllib.request.urlopen(filename)
else:
filename = os.path.expanduser(filename)
- if mode == 'r' or mode == 'a':
+ if mode == "r" or mode == "a":
if not os.path.exists(filename):
- raise OSError(2, 'No such file or directory: ' + filename)
+ raise OSError(2, "No such file or directory: " + filename)
filen, file_extension = os.path.splitext(filename)
command_dict = {
- ('.Z', 'r'):
- "self.file = os.popen('uncompress -c ' + filename, mode)",
- ('.gz', 'r'):
- "self.file = gzip.GzipFile(filename, 'rb')",
- ('.bz2', 'r'):
- "self.file = os.popen('bzip2 -dc ' + filename, mode)",
- ('.Z', 'w'):
- "self.file = os.popen('compress > ' + filename, mode)",
- ('.gz', 'w'):
- "self.file = gzip.GzipFile(filename, 'wb')",
- ('.bz2', 'w'):
- "self.file = os.popen('bzip2 > ' + filename, mode)",
- ('.Z', 'a'):
- "raise IOError, (0, 'Can\'t append to .Z files')",
- ('.gz', 'a'):
- "self.file = gzip.GzipFile(filename, 'ab')",
- ('.bz2', 'a'):
- "raise IOError, (0, 'Can\'t append to .bz2 files')",
- }
-
- exec(command_dict.get((file_extension, mode),
- 'self.file = open(filename, mode)'))
+ (".Z", "r"): "self.file = os.popen('uncompress -c ' + filename, mode)",
+ (".gz", "r"): "self.file = gzip.GzipFile(filename, 'rb')",
+ (".bz2", "r"): "self.file = os.popen('bzip2 -dc ' + filename, mode)",
+ (".Z", "w"): "self.file = os.popen('compress > ' + filename, mode)",
+ (".gz", "w"): "self.file = gzip.GzipFile(filename, 'wb')",
+ (".bz2", "w"): "self.file = os.popen('bzip2 > ' + filename, mode)",
+ (".Z", "a"): "raise IOError, (0, 'Can't append to .Z files')",
+ (".gz", "a"): "self.file = gzip.GzipFile(filename, 'ab')",
+ (".bz2", "a"): "raise IOError, (0, 'Can't append to .bz2 files')",
+ }
+
+ exec(
+ command_dict.get(
+ (file_extension, mode), "self.file = open(filename, mode)"
+ )
+ )
self.grammar = None
@@ -209,64 +210,64 @@ class ParseFileLineByLine: decimal_sep = "."
sign = oneOf("+ -")
# part of printables without decimal_sep, +, -
- special_chars = string.replace('!"#$%&\'()*,./:;<=>?@[\\]^_`{|}~',
- decimal_sep, "")
- integer = ToInteger(
- Combine(Optional(sign) +
- Word(nums))).setName("integer")
- positive_integer = ToInteger(
- Combine(Optional("+") +
- Word(nums))).setName("integer")
- negative_integer = ToInteger(
- Combine("-" +
- Word(nums))).setName("integer")
+ special_chars = string.replace(
+ "!\"#$%&'()*,./:;<=>?@[\\]^_`{|}~", decimal_sep, ""
+ )
+ integer = ToInteger(Combine(Optional(sign) + Word(nums))).setName("integer")
+ positive_integer = ToInteger(Combine(Optional("+") + Word(nums))).setName(
+ "integer"
+ )
+ negative_integer = ToInteger(Combine("-" + Word(nums))).setName("integer")
real = ToFloat(
- Combine(Optional(sign) +
- Word(nums) +
- decimal_sep +
- Optional(Word(nums)) +
- Optional(oneOf("E e") +
- Word(nums)))).setName("real")
+ Combine(
+ Optional(sign)
+ + Word(nums)
+ + decimal_sep
+ + Optional(Word(nums))
+ + Optional(oneOf("E e") + Word(nums))
+ )
+ ).setName("real")
positive_real = ToFloat(
- Combine(Optional("+") +
- Word(nums) +
- decimal_sep +
- Optional(Word(nums)) +
- Optional(oneOf("E e") +
- Word(nums)))).setName("real")
+ Combine(
+ Optional("+")
+ + Word(nums)
+ + decimal_sep
+ + Optional(Word(nums))
+ + Optional(oneOf("E e") + Word(nums))
+ )
+ ).setName("real")
negative_real = ToFloat(
- Combine("-" +
- Word(nums) +
- decimal_sep +
- Optional(Word(nums)) +
- Optional(oneOf("E e") +
- Word(nums)))).setName("real")
- qString = ( sglQuotedString | dblQuotedString ).setName("qString")
+ Combine(
+ "-"
+ + Word(nums)
+ + decimal_sep
+ + Optional(Word(nums))
+ + Optional(oneOf("E e") + Word(nums))
+ )
+ ).setName("real")
+ qString = (sglQuotedString | dblQuotedString).setName("qString")
# add other characters we should skip over between interesting fields
integer_junk = Optional(
- Suppress(
- Word(alphas +
- special_chars +
- decimal_sep))).setName("integer_junk")
- real_junk = Optional(
- Suppress(
- Word(alphas +
- special_chars))).setName("real_junk")
+ Suppress(Word(alphas + special_chars + decimal_sep))
+ ).setName("integer_junk")
+ real_junk = Optional(Suppress(Word(alphas + special_chars))).setName(
+ "real_junk"
+ )
qString_junk = SkipTo(qString).setName("qString_junk")
# Now that 'integer', 'real', and 'qString' have been assigned I can
# execute the definition file.
- exec(compile(open(self.parsedef).read(), self.parsedef, 'exec'))
+ exec(compile(open(self.parsedef).read(), self.parsedef, "exec"))
# Build the grammar, combination of the 'integer', 'real, 'qString',
# and '*_junk' variables assigned above in the order specified in the
# definition file.
grammar = []
for nam, expr in parse:
- grammar.append( eval(expr.name + "_junk"))
- grammar.append( expr.setResultsName(nam) )
- self.grammar = And( grammar[1:] + [restOfLine] )
+ grammar.append(eval(expr.name + "_junk"))
+ grammar.append(expr.setResultsName(nam))
+ self.grammar = And(grammar[1:] + [restOfLine])
def __del__(self):
"""Delete (close) the file wrapper."""
@@ -325,7 +326,7 @@ class ParseFileLineByLine: self.file.flush()
-#=============================
+# =============================
def main(pargs):
"""This should only be used for testing. The primary mode of operation is
as an imported library.
@@ -336,28 +337,29 @@ def main(pargs): print(i)
-#-------------------------
-if __name__ == '__main__':
+# -------------------------
+if __name__ == "__main__":
ftn = "main"
- opts, pargs = getopt.getopt(sys.argv[1:], 'hvd',
- ['help', 'version', 'debug', 'bb='])
+ opts, pargs = getopt.getopt(
+ sys.argv[1:], "hvd", ["help", "version", "debug", "bb="]
+ )
for opt in opts:
- if opt[0] == '-h' or opt[0] == '--help':
- print(modname+": version="+__version__)
+ if opt[0] == "-h" or opt[0] == "--help":
+ print(modname + ": version=" + __version__)
usage()
sys.exit(0)
- elif opt[0] == '-v' or opt[0] == '--version':
- print(modname+": version="+__version__)
+ elif opt[0] == "-v" or opt[0] == "--version":
+ print(modname + ": version=" + __version__)
sys.exit(0)
- elif opt[0] == '-d' or opt[0] == '--debug':
+ elif opt[0] == "-d" or opt[0] == "--debug":
debug_p = 1
- elif opt[0] == '--bb':
+ elif opt[0] == "--bb":
opt_b = opt[1]
- #---make the object and run it---
+ # ---make the object and run it---
main(pargs)
-#===Revision Log===
-#Created by mkpythonproj:
-#2006-02-06 Tim Cera
+# ===Revision Log===
+# Created by mkpythonproj:
+# 2006-02-06 Tim Cera
#
diff --git a/examples/sql2dot.py b/examples/sql2dot.py index cd6717c..ca22ed8 100644 --- a/examples/sql2dot.py +++ b/examples/sql2dot.py @@ -47,50 +47,95 @@ alter table only student_registrations (class_id) references classes(class_id);
""".upper()
-from pyparsing import Literal, Word, delimitedList \
- , alphas, alphanums \
- , OneOrMore, ZeroOrMore, CharsNotIn \
- , replaceWith
+from pyparsing import (
+ Literal,
+ Word,
+ delimitedList,
+ alphas,
+ alphanums,
+ OneOrMore,
+ ZeroOrMore,
+ CharsNotIn,
+ replaceWith,
+)
skobki = "(" + ZeroOrMore(CharsNotIn(")")) + ")"
-field_def = OneOrMore(Word(alphas,alphanums+"_\"':-") | skobki)
+field_def = OneOrMore(Word(alphas, alphanums + "_\"':-") | skobki)
+
+
+def field_act(s, loc, tok):
+ return ("<" + tok[0] + "> " + " ".join(tok)).replace('"', '\\"')
-def field_act(s,loc,tok):
- return ("<"+tok[0]+"> " + " ".join(tok)).replace("\"","\\\"")
field_def.setParseAction(field_act)
-field_list_def = delimitedList( field_def )
+field_list_def = delimitedList(field_def)
+
+
def field_list_act(toks):
return " | ".join(toks)
+
field_list_def.setParseAction(field_list_act)
-create_table_def = Literal("CREATE") + "TABLE" + Word(alphas,alphanums+"_").setResultsName("tablename") + \
- "("+field_list_def.setResultsName("columns")+")"+ ";"
+create_table_def = (
+ Literal("CREATE")
+ + "TABLE"
+ + Word(alphas, alphanums + "_").setResultsName("tablename")
+ + "("
+ + field_list_def.setResultsName("columns")
+ + ")"
+ + ";"
+)
+
def create_table_act(toks):
- return """"%(tablename)s" [\n\t label="<%(tablename)s> %(tablename)s | %(columns)s"\n\t shape="record"\n];""" % toks
+ return (
+ """"%(tablename)s" [\n\t label="<%(tablename)s> %(tablename)s | %(columns)s"\n\t shape="record"\n];"""
+ % toks
+ )
+
+
create_table_def.setParseAction(create_table_act)
-add_fkey_def=Literal("ALTER")+"TABLE"+"ONLY" + Word(alphanums+"_").setResultsName("fromtable") + "ADD" \
- + "CONSTRAINT" + Word(alphanums+"_") + "FOREIGN"+"KEY"+"("+Word(alphanums+"_").setResultsName("fromcolumn")+")" \
- +"REFERENCES"+Word(alphanums+"_").setResultsName("totable")+"("+Word(alphanums+"_").setResultsName("tocolumn")+")"+";"
+add_fkey_def = (
+ Literal("ALTER")
+ + "TABLE"
+ + "ONLY"
+ + Word(alphanums + "_").setResultsName("fromtable")
+ + "ADD"
+ + "CONSTRAINT"
+ + Word(alphanums + "_")
+ + "FOREIGN"
+ + "KEY"
+ + "("
+ + Word(alphanums + "_").setResultsName("fromcolumn")
+ + ")"
+ + "REFERENCES"
+ + Word(alphanums + "_").setResultsName("totable")
+ + "("
+ + Word(alphanums + "_").setResultsName("tocolumn")
+ + ")"
+ + ";"
+)
+
def add_fkey_act(toks):
return """ "%(fromtable)s":%(fromcolumn)s -> "%(totable)s":%(tocolumn)s """ % toks
+
+
add_fkey_def.setParseAction(add_fkey_act)
-other_statement_def = ( OneOrMore(CharsNotIn(";") ) + ";")
-other_statement_def.setParseAction( replaceWith("") )
+other_statement_def = OneOrMore(CharsNotIn(";")) + ";"
+other_statement_def.setParseAction(replaceWith(""))
comment_def = "--" + ZeroOrMore(CharsNotIn("\n"))
-comment_def.setParseAction( replaceWith("") )
+comment_def.setParseAction(replaceWith(""))
-statement_def = comment_def | create_table_def | add_fkey_def | other_statement_def
-defs = OneOrMore(statement_def)
+statement_def = comment_def | create_table_def | add_fkey_def | other_statement_def
+defs = OneOrMore(statement_def)
print("""digraph g { graph [ rankdir = "LR" ]; """)
for i in defs.parseString(sampleSQL):
- if i!="":
+ if i != "":
print(i)
print("}")
diff --git a/examples/stackish.py b/examples/stackish.py index d33d4de..f02baf3 100644 --- a/examples/stackish.py +++ b/examples/stackish.py @@ -28,43 +28,66 @@ SPACE White space is basically ignored. This is interesting because since separation character and perform reasonable diffs on two structures.
"""
-from pyparsing import Suppress,Word,nums,alphas,alphanums,Combine,oneOf,\
- Optional,QuotedString,Forward,Group,ZeroOrMore,srange
+from pyparsing import (
+ Suppress,
+ Word,
+ nums,
+ alphas,
+ alphanums,
+ Combine,
+ oneOf,
+ Optional,
+ QuotedString,
+ Forward,
+ Group,
+ ZeroOrMore,
+ srange,
+)
-MARK,UNMARK,AT,COLON,QUOTE = map(Suppress,"[]@:'")
+MARK, UNMARK, AT, COLON, QUOTE = map(Suppress, "[]@:'")
NUMBER = Word(nums)
-NUMBER.setParseAction(lambda t:int(t[0]))
+NUMBER.setParseAction(lambda t: int(t[0]))
FLOAT = Combine(oneOf("+ -") + Word(nums) + "." + Optional(Word(nums)))
-FLOAT.setParseAction(lambda t:float(t[0]))
+FLOAT.setParseAction(lambda t: float(t[0]))
STRING = QuotedString('"', multiline=True)
-WORD = Word(alphas,alphanums+"_:")
+WORD = Word(alphas, alphanums + "_:")
ATTRIBUTE = Combine(AT + WORD)
strBody = Forward()
+
+
def setBodyLength(tokens):
- strBody << Word(srange(r'[\0x00-\0xffff]'), exact=int(tokens[0]))
+ strBody << Word(srange(r"[\0x00-\0xffff]"), exact=int(tokens[0]))
return ""
-BLOB = Combine(QUOTE + Word(nums).setParseAction(setBodyLength) +
- COLON + strBody + QUOTE)
+
+
+BLOB = Combine(
+ QUOTE + Word(nums).setParseAction(setBodyLength) + COLON + strBody + QUOTE
+)
item = Forward()
+
+
def assignUsing(s):
def assignPA(tokens):
if s in tokens:
tokens[tokens[s]] = tokens[0]
del tokens[s]
+
return assignPA
-GROUP = (MARK +
- Group( ZeroOrMore(
- (item +
- Optional(ATTRIBUTE)("attr")
- ).setParseAction(assignUsing("attr"))
- )
- ) +
- ( WORD("name") | UNMARK )
- ).setParseAction(assignUsing("name"))
-item << (NUMBER | FLOAT | STRING | BLOB | GROUP )
+
+
+GROUP = (
+ MARK
+ + Group(
+ ZeroOrMore(
+ (item + Optional(ATTRIBUTE)("attr")).setParseAction(assignUsing("attr"))
+ )
+ )
+ + (WORD("name") | UNMARK)
+).setParseAction(assignUsing("name"))
+item << (NUMBER | FLOAT | STRING | BLOB | GROUP)
tests = """\
[ '10:1234567890' @name 25 @age +0.45 @percentage person:zed
diff --git a/examples/statemachine/documentSignoffDemo.py b/examples/statemachine/documentSignoffDemo.py index 2ca38c8..23c902d 100644 --- a/examples/statemachine/documentSignoffDemo.py +++ b/examples/statemachine/documentSignoffDemo.py @@ -7,7 +7,12 @@ import statemachine import documentsignoffstate -print('\n'.join(t.__name__ for t in documentsignoffstate.DocumentRevisionState.transitions())) +print( + "\n".join( + t.__name__ for t in documentsignoffstate.DocumentRevisionState.transitions() + ) +) + class Document(documentsignoffstate.DocumentRevisionStateMixin): def __init__(self): @@ -27,16 +32,16 @@ def run_demo(): while not isinstance(doc._state, documentsignoffstate.Approved): - print('...submit') + print("...submit") doc.submit() print(doc) print(doc.state.description) - if random.randint(1,10) > 3: - print('...reject') + if random.randint(1, 10) > 3: + print("...reject") doc.reject() else: - print('...approve') + print("...approve") doc.approve() print(doc) @@ -46,5 +51,6 @@ def run_demo(): print(doc) print(doc.state.description) -if __name__ == '__main__': + +if __name__ == "__main__": run_demo() diff --git a/examples/statemachine/documentsignoffstate.pystate b/examples/statemachine/documentsignoffstate.pystate index 04df274..0c8fc61 100644 --- a/examples/statemachine/documentsignoffstate.pystate +++ b/examples/statemachine/documentsignoffstate.pystate @@ -7,7 +7,7 @@ # example using named state transitions # This implements a state model for submitting, -# approving, activating, and purging document +# approving, activating, and purging document # revisions in a document management system. # # The state model looks like: @@ -16,7 +16,7 @@ # | # | (create) # | -# v +# v # Editing ----------------------------------------------+ # | ^ | # | | | @@ -50,7 +50,7 @@ # just an example of a state machine with named transitions. # - + statemachine DocumentRevisionState: New -( create )-> Editing Editing -( cancel )-> Deleted diff --git a/examples/statemachine/libraryBookDemo.py b/examples/statemachine/libraryBookDemo.py index 98f0b2b..61bdd6d 100644 --- a/examples/statemachine/libraryBookDemo.py +++ b/examples/statemachine/libraryBookDemo.py @@ -26,7 +26,11 @@ class RestrictedBook(Book): if user in self._authorized_users: super().checkout() else: - raise Exception("{} could not check out restricted book".format(user if user is not None else "anonymous")) + raise Exception( + "{} could not check out restricted book".format( + user if user is not None else "anonymous" + ) + ) def run_demo(): @@ -41,9 +45,9 @@ def run_demo(): print(book) try: book.checkout() - except Exception as e: # statemachine.InvalidTransitionException: + except Exception as e: # statemachine.InvalidTransitionException: print(e) - print('..cannot check out reserved book') + print("..cannot check out reserved book") book.release() print(book) book.checkout() @@ -58,13 +62,13 @@ def run_demo(): try: restricted_book.checkout(name) except Exception as e: - print('..' + str(e)) + print(".." + str(e)) else: - print('checkout to', name) + print("checkout to", name) print(restricted_book) restricted_book.checkin() print(restricted_book) -if __name__ == '__main__': +if __name__ == "__main__": run_demo() diff --git a/examples/statemachine/statemachine.py b/examples/statemachine/statemachine.py index 4c8ee1d..a5f144e 100644 --- a/examples/statemachine/statemachine.py +++ b/examples/statemachine/statemachine.py @@ -18,7 +18,8 @@ import pyparsing as pp # define basic exception for invalid state transitions - state machine classes will subclass to # define their own specific exception type -class InvalidTransitionException(Exception): pass +class InvalidTransitionException(Exception): + pass ident = pp.Word(pp.alphas + "_", pp.alphanums + "_$") @@ -28,17 +29,30 @@ ident = pp.Word(pp.alphas + "_", pp.alphanums + "_$") def no_keywords_allowed(s, l, t): wd = t[0] return not keyword.iskeyword(wd) -ident.addCondition(no_keywords_allowed, message="cannot use a Python keyword for state or transition identifier") -stateTransition = ident("from_state") + "->" + ident("to_state") -stateMachine = (pp.Keyword("statemachine") + ident("name") + ":" - + pp.OneOrMore(pp.Group(stateTransition))("transitions")) -namedStateTransition = (ident("from_state") - + "-(" + ident("transition") + ")->" - + ident("to_state")) -namedStateMachine = (pp.Keyword("statemachine") + ident("name") + ":" - + pp.OneOrMore(pp.Group(namedStateTransition))("transitions")) +ident.addCondition( + no_keywords_allowed, + message="cannot use a Python keyword for state or transition identifier", +) + +stateTransition = ident("from_state") + "->" + ident("to_state") +stateMachine = ( + pp.Keyword("statemachine") + + ident("name") + + ":" + + pp.OneOrMore(pp.Group(stateTransition))("transitions") +) + +namedStateTransition = ( + ident("from_state") + "-(" + ident("transition") + ")->" + ident("to_state") +) +namedStateMachine = ( + pp.Keyword("statemachine") + + ident("name") + + ":" + + pp.OneOrMore(pp.Group(namedStateTransition))("transitions") +) def expand_state_definition(source, loc, tokens): @@ -58,50 +72,53 @@ def expand_state_definition(source, loc, tokens): # define base class for state classes baseStateClass = tokens.name - statedef.extend([ - "class %s(object):" % baseStateClass, - " def __str__(self):", - " return self.__class__.__name__", - - " @classmethod", - " def states(cls):", - " return list(cls.__subclasses__())", - - " def next_state(self):", - " return self._next_state_class()", - ]) + statedef.extend( + [ + "class %s(object):" % baseStateClass, + " def __str__(self):", + " return self.__class__.__name__", + " @classmethod", + " def states(cls):", + " return list(cls.__subclasses__())", + " def next_state(self):", + " return self._next_state_class()", + ] + ) # define all state classes statedef.extend("class {}({}): pass".format(s, baseStateClass) for s in states) # define state->state transitions - statedef.extend("{}._next_state_class = {}".format(s, fromTo[s]) for s in states if s in fromTo) - - statedef.extend([ - "class {baseStateClass}Mixin:".format(baseStateClass=baseStateClass), - " def __init__(self):", - " self._state = None", - - " def initialize_state(self, init_state):", - " if issubclass(init_state, {baseStateClass}):".format(baseStateClass=baseStateClass), - " init_state = init_state()", - " self._state = init_state", - - " @property", - " def state(self):", - " return self._state", - - " # get behavior/properties from current state", - " def __getattr__(self, attrname):", - " attr = getattr(self._state, attrname)", - " return attr", + statedef.extend( + "{}._next_state_class = {}".format(s, fromTo[s]) for s in states if s in fromTo + ) - " def __str__(self):", - " return '{0}: {1}'.format(self.__class__.__name__, self._state)", - ]) + statedef.extend( + [ + "class {baseStateClass}Mixin:".format(baseStateClass=baseStateClass), + " def __init__(self):", + " self._state = None", + " def initialize_state(self, init_state):", + " if issubclass(init_state, {baseStateClass}):".format( + baseStateClass=baseStateClass + ), + " init_state = init_state()", + " self._state = init_state", + " @property", + " def state(self):", + " return self._state", + " # get behavior/properties from current state", + " def __getattr__(self, attrname):", + " attr = getattr(self._state, attrname)", + " return attr", + " def __str__(self):", + " return '{0}: {1}'.format(self.__class__.__name__, self._state)", + ] + ) return ("\n" + indent).join(statedef) + "\n" + stateMachine.setParseAction(expand_state_definition) @@ -134,102 +151,115 @@ def expand_named_state_definition(source, loc, tokens): fromTo[s] = {} # define state transition class - statedef.extend([ - "class {baseStateClass}Transition:".format(baseStateClass=baseStateClass), - " def __str__(self):", - " return self.transitionName", - ]) statedef.extend( - "{tn_name} = {baseStateClass}Transition()".format(tn_name=tn, - baseStateClass=baseStateClass) - for tn in transitions) - statedef.extend("{tn_name}.transitionName = '{tn_name}'".format(tn_name=tn) - for tn in transitions) + [ + "class {baseStateClass}Transition:".format(baseStateClass=baseStateClass), + " def __str__(self):", + " return self.transitionName", + ] + ) + statedef.extend( + "{tn_name} = {baseStateClass}Transition()".format( + tn_name=tn, baseStateClass=baseStateClass + ) + for tn in transitions + ) + statedef.extend( + "{tn_name}.transitionName = '{tn_name}'".format(tn_name=tn) + for tn in transitions + ) # define base class for state classes - statedef.extend([ - "class %s(object):" % baseStateClass, - " from statemachine import InvalidTransitionException as BaseTransitionException", - " class InvalidTransitionException(BaseTransitionException): pass", - " def __str__(self):", - " return self.__class__.__name__", - - " @classmethod", - " def states(cls):", - " return list(cls.__subclasses__())", - - " @classmethod", - " def next_state(cls, name):", - " try:", - " return cls.tnmap[name]()", - " except KeyError:", - " raise cls.InvalidTransitionException('%s does not support transition %r'% (cls.__name__, name))", - - " def __bad_tn(name):", - " def _fn(cls):", - " raise cls.InvalidTransitionException('%s does not support transition %r'% (cls.__name__, name))", - " _fn.__name__ = name", - " return _fn", - ]) + statedef.extend( + [ + "class %s(object):" % baseStateClass, + " from statemachine import InvalidTransitionException as BaseTransitionException", + " class InvalidTransitionException(BaseTransitionException): pass", + " def __str__(self):", + " return self.__class__.__name__", + " @classmethod", + " def states(cls):", + " return list(cls.__subclasses__())", + " @classmethod", + " def next_state(cls, name):", + " try:", + " return cls.tnmap[name]()", + " except KeyError:", + " raise cls.InvalidTransitionException('%s does not support transition %r'% (cls.__name__, name))", + " def __bad_tn(name):", + " def _fn(cls):", + " raise cls.InvalidTransitionException('%s does not support transition %r'% (cls.__name__, name))", + " _fn.__name__ = name", + " return _fn", + ] + ) # define default 'invalid transition' methods in base class, valid transitions will be implemented in subclasses statedef.extend( " {tn_name} = classmethod(__bad_tn({tn_name!r}))".format(tn_name=tn) - for tn in transitions) + for tn in transitions + ) # define all state classes - statedef.extend("class {}({}): pass".format(s, baseStateClass) - for s in states) + statedef.extend("class {}({}): pass".format(s, baseStateClass) for s in states) # define state transition methods for valid transitions from each state for s in states: trns = list(fromTo[s].items()) # statedef.append("%s.tnmap = {%s}" % (s, ", ".join("%s:%s" % tn for tn in trns))) - statedef.extend("{}.{} = classmethod(lambda cls: {}())".format(s, tn_, to_) - for tn_, to_ in trns) - - statedef.extend([ - "{baseStateClass}.transitions = classmethod(lambda cls: [{transition_class_list}])".format( - baseStateClass=baseStateClass, - transition_class_list = ', '.join("cls.{}".format(tn) for tn in transitions) - ), - "{baseStateClass}.transition_names = [tn.__name__ for tn in {baseStateClass}.transitions()]".format( - baseStateClass=baseStateClass + statedef.extend( + "{}.{} = classmethod(lambda cls: {}())".format(s, tn_, to_) + for tn_, to_ in trns ) - ]) - - # define <state>Mixin class for application classes that delegate to the state - statedef.extend([ - "class {baseStateClass}Mixin:".format(baseStateClass=baseStateClass), - " def __init__(self):", - " self._state = None", - - " def initialize_state(self, init_state):", - " if issubclass(init_state, {baseStateClass}):".format(baseStateClass=baseStateClass), - " init_state = init_state()", - " self._state = init_state", - " @property", - " def state(self):", - " return self._state", - - " # get behavior/properties from current state", - " def __getattr__(self, attrname):", - " attr = getattr(self._state, attrname)", - " return attr", - - " def __str__(self):", - " return '{0}: {1}'.format(self.__class__.__name__, self._state)", + statedef.extend( + [ + "{baseStateClass}.transitions = classmethod(lambda cls: [{transition_class_list}])".format( + baseStateClass=baseStateClass, + transition_class_list=", ".join( + "cls.{}".format(tn) for tn in transitions + ), + ), + "{baseStateClass}.transition_names = [tn.__name__ for tn in {baseStateClass}.transitions()]".format( + baseStateClass=baseStateClass + ), + ] + ) - ]) + # define <state>Mixin class for application classes that delegate to the state + statedef.extend( + [ + "class {baseStateClass}Mixin:".format(baseStateClass=baseStateClass), + " def __init__(self):", + " self._state = None", + " def initialize_state(self, init_state):", + " if issubclass(init_state, {baseStateClass}):".format( + baseStateClass=baseStateClass + ), + " init_state = init_state()", + " self._state = init_state", + " @property", + " def state(self):", + " return self._state", + " # get behavior/properties from current state", + " def __getattr__(self, attrname):", + " attr = getattr(self._state, attrname)", + " return attr", + " def __str__(self):", + " return '{0}: {1}'.format(self.__class__.__name__, self._state)", + ] + ) # define transition methods to be delegated to the _state instance variable statedef.extend( - " def {tn_name}(self): self._state = self._state.{tn_name}()".format(tn_name=tn) + " def {tn_name}(self): self._state = self._state.{tn_name}()".format( + tn_name=tn + ) for tn in transitions ) return ("\n" + indent).join(statedef) + "\n" + namedStateMachine.setParseAction(expand_named_state_definition) @@ -248,15 +278,15 @@ class SuffixImporter: :meth:`register` on your class to actually install it in the appropriate places in :mod:`sys`. """ - scheme = 'suffix' + scheme = "suffix" suffix = None path_entry = None @classmethod def trigger_url(cls): if cls.suffix is None: - raise ValueError('%s.suffix is not set' % cls.__name__) - return 'suffix:%s' % cls.suffix + raise ValueError("%s.suffix is not set" % cls.__name__) + return "suffix:%s" % cls.suffix @classmethod def register(cls): @@ -279,7 +309,7 @@ class SuffixImporter: # it probably isn't even a filesystem path finder = sys.path_importer_cache.get(dirpath) if isinstance(finder, (type(None), importlib.machinery.FileFinder)): - checkpath = os.path.join(dirpath, '{}.{}'.format(fullname, self.suffix)) + checkpath = os.path.join(dirpath, "{}.{}".format(fullname, self.suffix)) yield checkpath def find_module(self, fullname, path=None): @@ -311,25 +341,26 @@ class SuffixImporter: class PystateImporter(SuffixImporter): - suffix = 'pystate' + suffix = "pystate" def process_filedata(self, module, data): # MATT-NOTE: re-worked :func:`get_state_machine` # convert any statemachine expressions - stateMachineExpr = (stateMachine | namedStateMachine).ignore(pp.pythonStyleComment) + stateMachineExpr = (stateMachine | namedStateMachine).ignore( + pp.pythonStyleComment + ) generated_code = stateMachineExpr.transformString(data) - if DEBUG: print(generated_code) + if DEBUG: + print(generated_code) # compile code object from generated code # (strip trailing spaces and tabs, compile doesn't like # dangling whitespace) - COMPILE_MODE = 'exec' + COMPILE_MODE = "exec" - codeobj = compile(generated_code.rstrip(" \t"), - module.__file__, - COMPILE_MODE) + codeobj = compile(generated_code.rstrip(" \t"), module.__file__, COMPILE_MODE) exec(codeobj, module.__dict__) diff --git a/examples/statemachine/vending_machine.py b/examples/statemachine/vending_machine.py index d218608..b5fe524 100644 --- a/examples/statemachine/vending_machine.py +++ b/examples/statemachine/vending_machine.py @@ -22,11 +22,14 @@ statemachine VendingMachineState: """ # convert state machine text to state classes -generated = statemachine.namedStateMachine.transformString(vending_machine_state_description) +generated = statemachine.namedStateMachine.transformString( + vending_machine_state_description +) # print(generated) # exec generated code to define state classes and state mixin exec(generated) + class VendingMachine(VendingMachineStateMixin): def __init__(self): self.initialize_state(Idle) @@ -42,7 +45,7 @@ class VendingMachine(VendingMachineStateMixin): self._pressed = button self.press_digit_button() else: - print('Did not recognize button {!r}'.format(str(button))) + print("Did not recognize button {!r}".format(str(button))) def press_alpha_button(self): try: diff --git a/examples/statemachine/video_demo.py b/examples/statemachine/video_demo.py index fadfb9d..f6dbc8f 100644 --- a/examples/statemachine/video_demo.py +++ b/examples/statemachine/video_demo.py @@ -20,25 +20,29 @@ v = Video("Die Hard.mp4") while True: print(v.state) - cmd = input("Command ({})> ".format('/'.join(videostate.VideoState.transition_names))).lower().strip() + cmd = ( + input("Command ({})> ".format("/".join(videostate.VideoState.transition_names))) + .lower() + .strip() + ) if not cmd: continue - if cmd in ('?', 'h', 'help'): - print('enter a transition {!r}'.format(videostate.VideoState.transition_names)) - print(' q - quit') - print(' ?, h, help - this message') + if cmd in ("?", "h", "help"): + print("enter a transition {!r}".format(videostate.VideoState.transition_names)) + print(" q - quit") + print(" ?, h, help - this message") continue # quitting out - if cmd.startswith('q'): + if cmd.startswith("q"): break # get transition function for given command state_transition_fn = getattr(v, cmd, None) if state_transition_fn is None: - print('???') + print("???") continue # invoke the input transition, handle invalid commands diff --git a/examples/test_bibparse.py b/examples/test_bibparse.py index 12e416e..339fd7c 100644 --- a/examples/test_bibparse.py +++ b/examples/test_bibparse.py @@ -5,158 +5,199 @@ from pyparsing import ParseException from .btpyparse import Macro from . import btpyparse as bp + class TestBibparse(unittest.TestCase): def test_names(self): # check various types of names # All names can contains alphas, but not some special chars - bad_chars = '"#%\'(),={}' - for name_type, dig1f in ((bp.macro_def, False), - (bp.field_name, False), - (bp.entry_type, False), - (bp.cite_key, True)): - if dig1f: # can start with digit - self.assertEqual(name_type.parseString('2t')[0], '2t') + bad_chars = "\"#%'(),={}" + for name_type, dig1f in ( + (bp.macro_def, False), + (bp.field_name, False), + (bp.entry_type, False), + (bp.cite_key, True), + ): + if dig1f: # can start with digit + self.assertEqual(name_type.parseString("2t")[0], "2t") else: - self.assertRaises(ParseException, name_type.parseString, '2t') + self.assertRaises(ParseException, name_type.parseString, "2t") # All of the names cannot contain some characters for char in bad_chars: self.assertRaises(ParseException, name_type.parseString, char) # standard strings all OK - self.assertEqual(name_type.parseString('simple_test')[0], 'simple_test') + self.assertEqual(name_type.parseString("simple_test")[0], "simple_test") # Test macro ref mr = bp.macro_ref # can't start with digit - self.assertRaises(ParseException, mr.parseString, '2t') + self.assertRaises(ParseException, mr.parseString, "2t") for char in bad_chars: self.assertRaises(ParseException, mr.parseString, char) - self.assertEqual(mr.parseString('simple_test')[0].name, 'simple_test') + self.assertEqual(mr.parseString("simple_test")[0].name, "simple_test") def test_numbers(self): - self.assertEqual(bp.number.parseString('1066')[0], '1066') - self.assertEqual(bp.number.parseString('0')[0], '0') - self.assertRaises(ParseException, bp.number.parseString, '-4') - self.assertRaises(ParseException, bp.number.parseString, '+4') - self.assertRaises(ParseException, bp.number.parseString, '.4') + self.assertEqual(bp.number.parseString("1066")[0], "1066") + self.assertEqual(bp.number.parseString("0")[0], "0") + self.assertRaises(ParseException, bp.number.parseString, "-4") + self.assertRaises(ParseException, bp.number.parseString, "+4") + self.assertRaises(ParseException, bp.number.parseString, ".4") # something point something leaves a trailing .4 unmatched - self.assertEqual(bp.number.parseString('0.4')[0], '0') - + self.assertEqual(bp.number.parseString("0.4")[0], "0") def test_parse_string(self): # test string building blocks - self.assertEqual(bp.chars_no_quotecurly.parseString('x')[0], 'x') - self.assertEqual(bp.chars_no_quotecurly.parseString("a string")[0], 'a string') - self.assertEqual(bp.chars_no_quotecurly.parseString('a "string')[0], 'a ') - self.assertEqual(bp.chars_no_curly.parseString('x')[0], 'x') - self.assertEqual(bp.chars_no_curly.parseString("a string")[0], 'a string') - self.assertEqual(bp.chars_no_curly.parseString('a {string')[0], 'a ') - self.assertEqual(bp.chars_no_curly.parseString('a }string')[0], 'a ') + self.assertEqual(bp.chars_no_quotecurly.parseString("x")[0], "x") + self.assertEqual(bp.chars_no_quotecurly.parseString("a string")[0], "a string") + self.assertEqual(bp.chars_no_quotecurly.parseString('a "string')[0], "a ") + self.assertEqual(bp.chars_no_curly.parseString("x")[0], "x") + self.assertEqual(bp.chars_no_curly.parseString("a string")[0], "a string") + self.assertEqual(bp.chars_no_curly.parseString("a {string")[0], "a ") + self.assertEqual(bp.chars_no_curly.parseString("a }string")[0], "a ") # test more general strings together for obj in (bp.curly_string, bp.string, bp.field_value): - self.assertEqual(obj.parseString('{}').asList(), []) + self.assertEqual(obj.parseString("{}").asList(), []) self.assertEqual(obj.parseString('{a "string}')[0], 'a "string') - self.assertEqual(obj.parseString('{a {nested} string}').asList(), - ['a ', ['nested'], ' string']) - self.assertEqual(obj.parseString('{a {double {nested}} string}').asList(), - ['a ', ['double ', ['nested']], ' string']) + self.assertEqual( + obj.parseString("{a {nested} string}").asList(), + ["a ", ["nested"], " string"], + ) + self.assertEqual( + obj.parseString("{a {double {nested}} string}").asList(), + ["a ", ["double ", ["nested"]], " string"], + ) for obj in (bp.quoted_string, bp.string, bp.field_value): self.assertEqual(obj.parseString('""').asList(), []) - self.assertEqual(obj.parseString('"a string"')[0], 'a string') - self.assertEqual(obj.parseString('"a {nested} string"').asList(), - ['a ', ['nested'], ' string']) - self.assertEqual(obj.parseString('"a {double {nested}} string"').asList(), - ['a ', ['double ', ['nested']], ' string']) + self.assertEqual(obj.parseString('"a string"')[0], "a string") + self.assertEqual( + obj.parseString('"a {nested} string"').asList(), + ["a ", ["nested"], " string"], + ) + self.assertEqual( + obj.parseString('"a {double {nested}} string"').asList(), + ["a ", ["double ", ["nested"]], " string"], + ) # check macro def in string - self.assertEqual(bp.string.parseString('someascii')[0], Macro('someascii')) - self.assertRaises(ParseException, bp.string.parseString, '%#= validstring') + self.assertEqual(bp.string.parseString("someascii")[0], Macro("someascii")) + self.assertRaises(ParseException, bp.string.parseString, "%#= validstring") # check number in string - self.assertEqual(bp.string.parseString('1994')[0], '1994') - + self.assertEqual(bp.string.parseString("1994")[0], "1994") def test_parse_field(self): # test field value - hashes included fv = bp.field_value # Macro - self.assertEqual(fv.parseString('aname')[0], Macro('aname')) - self.assertEqual(fv.parseString('ANAME')[0], Macro('aname')) + self.assertEqual(fv.parseString("aname")[0], Macro("aname")) + self.assertEqual(fv.parseString("ANAME")[0], Macro("aname")) # String and macro - self.assertEqual(fv.parseString('aname # "some string"').asList(), - [Macro('aname'), 'some string']) + self.assertEqual( + fv.parseString('aname # "some string"').asList(), + [Macro("aname"), "some string"], + ) # Nested string - self.assertEqual(fv.parseString('aname # {some {string}}').asList(), - [Macro('aname'), 'some ', ['string']]) + self.assertEqual( + fv.parseString("aname # {some {string}}").asList(), + [Macro("aname"), "some ", ["string"]], + ) # String and number - self.assertEqual(fv.parseString('"a string" # 1994').asList(), - ['a string', '1994']) + self.assertEqual( + fv.parseString('"a string" # 1994').asList(), ["a string", "1994"] + ) # String and number and macro - self.assertEqual(fv.parseString('"a string" # 1994 # a_macro').asList(), - ['a string', '1994', Macro('a_macro')]) - + self.assertEqual( + fv.parseString('"a string" # 1994 # a_macro').asList(), + ["a string", "1994", Macro("a_macro")], + ) def test_comments(self): - res = bp.comment.parseString('@Comment{about something}') - self.assertEqual(res.asList(), ['comment', '{about something}']) + res = bp.comment.parseString("@Comment{about something}") + self.assertEqual(res.asList(), ["comment", "{about something}"]) self.assertEqual( - bp.comment.parseString('@COMMENT{about something').asList(), - ['comment', '{about something']) + bp.comment.parseString("@COMMENT{about something").asList(), + ["comment", "{about something"], + ) self.assertEqual( - bp.comment.parseString('@comment(about something').asList(), - ['comment', '(about something']) + bp.comment.parseString("@comment(about something").asList(), + ["comment", "(about something"], + ) self.assertEqual( - bp.comment.parseString('@COMment about something').asList(), - ['comment', ' about something']) - self.assertRaises(ParseException, bp.comment.parseString, - '@commentabout something') - self.assertRaises(ParseException, bp.comment.parseString, - '@comment+about something') - self.assertRaises(ParseException, bp.comment.parseString, - '@comment"about something') - + bp.comment.parseString("@COMment about something").asList(), + ["comment", " about something"], + ) + self.assertRaises( + ParseException, bp.comment.parseString, "@commentabout something" + ) + self.assertRaises( + ParseException, bp.comment.parseString, "@comment+about something" + ) + self.assertRaises( + ParseException, bp.comment.parseString, '@comment"about something' + ) def test_preamble(self): res = bp.preamble.parseString('@preamble{"about something"}') - self.assertEqual(res.asList(), ['preamble', 'about something']) - self.assertEqual(bp.preamble.parseString( - '@PREamble{{about something}}').asList(), - ['preamble', 'about something']) - self.assertEqual(bp.preamble.parseString("""@PREamble{ + self.assertEqual(res.asList(), ["preamble", "about something"]) + self.assertEqual( + bp.preamble.parseString("@PREamble{{about something}}").asList(), + ["preamble", "about something"], + ) + self.assertEqual( + bp.preamble.parseString( + """@PREamble{ {about something} - }""").asList(), - ['preamble', 'about something']) - + }""" + ).asList(), + ["preamble", "about something"], + ) def test_macro(self): res = bp.macro.parseString('@string{ANAME = "about something"}') - self.assertEqual(res.asList(), ['string', 'aname', 'about something']) + self.assertEqual(res.asList(), ["string", "aname", "about something"]) self.assertEqual( - bp.macro.parseString('@string{aname = {about something}}').asList(), - ['string', 'aname', 'about something']) - + bp.macro.parseString("@string{aname = {about something}}").asList(), + ["string", "aname", "about something"], + ) def test_entry(self): txt = """@some_entry{akey, aname = "about something", another={something else}}""" res = bp.entry.parseString(txt) - self.assertEqual(res.asList(), - ['some_entry', 'akey', - ['aname', 'about something'], ['another', 'something else']]) + self.assertEqual( + res.asList(), + [ + "some_entry", + "akey", + ["aname", "about something"], + ["another", "something else"], + ], + ) # Case conversion txt = """@SOME_ENTRY{akey, ANAME = "about something", another={something else}}""" res = bp.entry.parseString(txt) - self.assertEqual(res.asList(), - ['some_entry', 'akey', - ['aname', 'about something'], ['another', 'something else']]) - + self.assertEqual( + res.asList(), + [ + "some_entry", + "akey", + ["aname", "about something"], + ["another", "something else"], + ], + ) def test_bibfile(self): txt = """@some_entry{akey, aname = "about something", another={something else}}""" res = bp.bibfile.parseString(txt) - self.assertEqual(res.asList(), - [['some_entry', 'akey', - ['aname', 'about something'], - ['another', 'something else']]]) - + self.assertEqual( + res.asList(), + [ + [ + "some_entry", + "akey", + ["aname", "about something"], + ["another", "something else"], + ] + ], + ) def test_bib1(self): # First pass whole bib-like tests @@ -186,5 +227,5 @@ class TestBibparse(unittest.TestCase): self.assertEqual(res.asList(), res3) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/examples/urlExtractor.py b/examples/urlExtractor.py index 70835da..2d29200 100644 --- a/examples/urlExtractor.py +++ b/examples/urlExtractor.py @@ -4,11 +4,11 @@ from pyparsing import makeHTMLTags, pyparsing_common as ppc from urllib.request import urlopen import pprint -linkOpenTag, linkCloseTag = makeHTMLTags('a') +linkOpenTag, linkCloseTag = makeHTMLTags("a") linkBody = linkOpenTag.tag_body linkBody.setParseAction(ppc.stripHTMLTags) -linkBody.addParseAction(lambda toks: ' '.join(toks[0].strip().split())) +linkBody.addParseAction(lambda toks: " ".join(toks[0].strip().split())) link = linkOpenTag + linkBody("body") + linkCloseTag.suppress() @@ -24,6 +24,4 @@ for toks, strt, end in link.scanString(htmlText): # Create dictionary from list comprehension, assembled from each pair of tokens returned # from a matched URL. -pprint.pprint( - {toks.body: toks.href for toks, strt, end in link.scanString(htmlText)} - ) +pprint.pprint({toks.body: toks.href for toks, strt, end in link.scanString(htmlText)}) diff --git a/examples/urlExtractorNew.py b/examples/urlExtractorNew.py index 14f6eb1..7a6e54a 100644 --- a/examples/urlExtractorNew.py +++ b/examples/urlExtractorNew.py @@ -27,4 +27,4 @@ for toks, strt, end in link.scanString(htmlText): # from a matched URL. pprint.pprint( {toks.body: toks.startA.href for toks, strt, end in link.scanString(htmlText)} - ) +) diff --git a/examples/verilogParse.py b/examples/verilogParse.py index 2d060e6..8a809d1 100644 --- a/examples/verilogParse.py +++ b/examples/verilogParse.py @@ -67,11 +67,33 @@ import sys __version__ = "1.0.11" -from pyparsing import Literal, Keyword, Word, OneOrMore, ZeroOrMore, \ - Forward, delimitedList, Group, Optional, Combine, alphas, nums, restOfLine, \ - alphanums, dblQuotedString, empty, ParseException, oneOf, \ - StringEnd, FollowedBy, ParserElement, Regex, cppStyleComment +from pyparsing import ( + Literal, + Keyword, + Word, + OneOrMore, + ZeroOrMore, + Forward, + delimitedList, + Group, + Optional, + Combine, + alphas, + nums, + restOfLine, + alphanums, + dblQuotedString, + empty, + ParseException, + oneOf, + StringEnd, + FollowedBy, + ParserElement, + Regex, + cppStyleComment, +) import pyparsing + usePackrat = False packratOn = False @@ -84,184 +106,269 @@ if usePackrat: else: packratOn = True -def dumpTokens(s,l,t): + +def dumpTokens(s, l, t): import pprint - pprint.pprint( t.asList() ) + + pprint.pprint(t.asList()) + verilogbnf = None + + def Verilog_BNF(): global verilogbnf if verilogbnf is None: # compiler directives - compilerDirective = Combine( "`" + \ - oneOf("define undef ifdef else endif default_nettype " - "include resetall timescale unconnected_drive " - "nounconnected_drive celldefine endcelldefine") + \ - restOfLine ).setName("compilerDirective") + compilerDirective = Combine( + "`" + + oneOf( + "define undef ifdef else endif default_nettype " + "include resetall timescale unconnected_drive " + "nounconnected_drive celldefine endcelldefine" + ) + + restOfLine + ).setName("compilerDirective") # primitives - SEMI,COLON,LPAR,RPAR,LBRACE,RBRACE,LBRACK,RBRACK,DOT,COMMA,EQ = map(Literal,";:(){}[].,=") - - identLead = alphas+"$_" - identBody = alphanums+"$_" - identifier1 = Regex( r"\.?["+identLead+"]["+identBody+r"]*(\.["+identLead+"]["+identBody+"]*)*" - ).setName("baseIdent") - identifier2 = Regex(r"\\\S+").setParseAction(lambda t:t[0][1:]).setName("escapedIdent")#.setDebug() + SEMI, COLON, LPAR, RPAR, LBRACE, RBRACE, LBRACK, RBRACK, DOT, COMMA, EQ = map( + Literal, ";:(){}[].,=" + ) + + identLead = alphas + "$_" + identBody = alphanums + "$_" + identifier1 = Regex( + r"\.?[" + + identLead + + "][" + + identBody + + r"]*(\.[" + + identLead + + "][" + + identBody + + "]*)*" + ).setName("baseIdent") + identifier2 = ( + Regex(r"\\\S+").setParseAction(lambda t: t[0][1:]).setName("escapedIdent") + ) # .setDebug() identifier = identifier1 | identifier2 - assert(identifier2 == r'\abc') + assert identifier2 == r"\abc" hexnums = nums + "abcdefABCDEF" + "_?" base = Regex("'[bBoOdDhH]").setName("base") - basedNumber = Combine( Optional( Word(nums + "_") ) + base + Word(hexnums+"xXzZ"), - joinString=" ", adjacent=False ).setName("basedNumber") - #~ number = ( basedNumber | Combine( Word( "+-"+spacedNums, spacedNums ) + - #~ Optional( DOT + Optional( Word( spacedNums ) ) ) + - #~ Optional( e + Word( "+-"+spacedNums, spacedNums ) ) ).setName("numeric") ) - number = ( basedNumber | \ - Regex(r"[+-]?[0-9_]+(\.[0-9_]*)?([Ee][+-]?[0-9_]+)?") \ - ).setName("numeric") - #~ decnums = nums + "_" - #~ octnums = "01234567" + "_" + basedNumber = Combine( + Optional(Word(nums + "_")) + base + Word(hexnums + "xXzZ"), + joinString=" ", + adjacent=False, + ).setName("basedNumber") + # ~ number = ( basedNumber | Combine( Word( "+-"+spacedNums, spacedNums ) + + # ~ Optional( DOT + Optional( Word( spacedNums ) ) ) + + # ~ Optional( e + Word( "+-"+spacedNums, spacedNums ) ) ).setName("numeric") ) + number = ( + basedNumber | Regex(r"[+-]?[0-9_]+(\.[0-9_]*)?([Ee][+-]?[0-9_]+)?") + ).setName("numeric") + # ~ decnums = nums + "_" + # ~ octnums = "01234567" + "_" expr = Forward().setName("expr") - concat = Group( LBRACE + delimitedList( expr ) + RBRACE ) + concat = Group(LBRACE + delimitedList(expr) + RBRACE) multiConcat = Group("{" + expr + concat + "}").setName("multiConcat") - funcCall = Group(identifier + LPAR + Optional( delimitedList( expr ) ) + RPAR).setName("funcCall") + funcCall = Group( + identifier + LPAR + Optional(delimitedList(expr)) + RPAR + ).setName("funcCall") - subscrRef = Group(LBRACK + delimitedList( expr, COLON ) + RBRACK) - subscrIdentifier = Group( identifier + Optional( subscrRef ) ) - #~ scalarConst = "0" | (( FollowedBy('1') + oneOf("1'b0 1'b1 1'bx 1'bX 1'B0 1'B1 1'Bx 1'BX 1") )) + subscrRef = Group(LBRACK + delimitedList(expr, COLON) + RBRACK) + subscrIdentifier = Group(identifier + Optional(subscrRef)) + # ~ scalarConst = "0" | (( FollowedBy('1') + oneOf("1'b0 1'b1 1'bx 1'bX 1'B0 1'B1 1'Bx 1'BX 1") )) scalarConst = Regex("0|1('[Bb][01xX])?") - mintypmaxExpr = Group( expr + COLON + expr + COLON + expr ).setName("mintypmax") + mintypmaxExpr = Group(expr + COLON + expr + COLON + expr).setName("mintypmax") primary = ( - number | - (LPAR + mintypmaxExpr + RPAR ) | - ( LPAR + Group(expr) + RPAR ).setName("nestedExpr") | - multiConcat | - concat | - dblQuotedString | - funcCall | - subscrIdentifier - ) - - unop = oneOf( "+ - ! ~ & ~& | ^| ^ ~^" ).setName("unop") - binop = oneOf( "+ - * / % == != === !== && " - "|| < <= > >= & | ^ ^~ >> << ** <<< >>>" ).setName("binop") + number + | (LPAR + mintypmaxExpr + RPAR) + | (LPAR + Group(expr) + RPAR).setName("nestedExpr") + | multiConcat + | concat + | dblQuotedString + | funcCall + | subscrIdentifier + ) + + unop = oneOf("+ - ! ~ & ~& | ^| ^ ~^").setName("unop") + binop = oneOf( + "+ - * / % == != === !== && " + "|| < <= > >= & | ^ ^~ >> << ** <<< >>>" + ).setName("binop") expr << ( - ( unop + expr ) | # must be first! - ( primary + "?" + expr + COLON + expr ) | - ( primary + Optional( binop + expr ) ) - ) + (unop + expr) + | (primary + "?" + expr + COLON + expr) # must be first! + | (primary + Optional(binop + expr)) + ) lvalue = subscrIdentifier | concat # keywords - if_ = Keyword("if") - else_ = Keyword("else") - edge = Keyword("edge") - posedge = Keyword("posedge") - negedge = Keyword("negedge") - specify = Keyword("specify") + if_ = Keyword("if") + else_ = Keyword("else") + edge = Keyword("edge") + posedge = Keyword("posedge") + negedge = Keyword("negedge") + specify = Keyword("specify") endspecify = Keyword("endspecify") - fork = Keyword("fork") - join = Keyword("join") - begin = Keyword("begin") - end = Keyword("end") - default = Keyword("default") - forever = Keyword("forever") - repeat = Keyword("repeat") - while_ = Keyword("while") - for_ = Keyword("for") - case = oneOf( "case casez casex" ) - endcase = Keyword("endcase") - wait = Keyword("wait") - disable = Keyword("disable") - deassign = Keyword("deassign") - force = Keyword("force") - release = Keyword("release") - assign = Keyword("assign") + fork = Keyword("fork") + join = Keyword("join") + begin = Keyword("begin") + end = Keyword("end") + default = Keyword("default") + forever = Keyword("forever") + repeat = Keyword("repeat") + while_ = Keyword("while") + for_ = Keyword("for") + case = oneOf("case casez casex") + endcase = Keyword("endcase") + wait = Keyword("wait") + disable = Keyword("disable") + deassign = Keyword("deassign") + force = Keyword("force") + release = Keyword("release") + assign = Keyword("assign") eventExpr = Forward() - eventTerm = ( posedge + expr ) | ( negedge + expr ) | expr | ( LPAR + eventExpr + RPAR ) - eventExpr << ( - Group( delimitedList( eventTerm, Keyword("or") ) ) - ) - eventControl = Group( "@" + ( ( LPAR + eventExpr + RPAR ) | identifier | "*" ) ).setName("eventCtrl") - - delayArg = ( number | - Word(alphanums+"$_") | #identifier | - ( LPAR + Group( delimitedList( mintypmaxExpr | expr ) ) + RPAR ) - ).setName("delayArg")#.setDebug() - delay = Group( "#" + delayArg ).setName("delay")#.setDebug() + eventTerm = ( + (posedge + expr) | (negedge + expr) | expr | (LPAR + eventExpr + RPAR) + ) + eventExpr << (Group(delimitedList(eventTerm, Keyword("or")))) + eventControl = Group( + "@" + ((LPAR + eventExpr + RPAR) | identifier | "*") + ).setName("eventCtrl") + + delayArg = ( + number + | Word(alphanums + "$_") + | (LPAR + Group(delimitedList(mintypmaxExpr | expr)) + RPAR) # identifier | + ).setName( + "delayArg" + ) # .setDebug() + delay = Group("#" + delayArg).setName("delay") # .setDebug() delayOrEventControl = delay | eventControl - assgnmt = Group( lvalue + EQ + Optional( delayOrEventControl ) + expr ).setName( "assgnmt" ) - nbAssgnmt = Group(( lvalue + "<=" + Optional( delay ) + expr ) | - ( lvalue + "<=" + Optional( eventControl ) + expr )).setName( "nbassgnmt" ) + assgnmt = Group(lvalue + EQ + Optional(delayOrEventControl) + expr).setName( + "assgnmt" + ) + nbAssgnmt = Group( + (lvalue + "<=" + Optional(delay) + expr) + | (lvalue + "<=" + Optional(eventControl) + expr) + ).setName("nbassgnmt") range = LBRACK + expr + COLON + expr + RBRACK - paramAssgnmt = Group( identifier + EQ + expr ).setName("paramAssgnmt") - parameterDecl = Group( "parameter" + Optional( range ) + delimitedList( paramAssgnmt ) + SEMI).setName("paramDecl") - - inputDecl = Group( "input" + Optional( range ) + delimitedList( identifier ) + SEMI ) - outputDecl = Group( "output" + Optional( range ) + delimitedList( identifier ) + SEMI ) - inoutDecl = Group( "inout" + Optional( range ) + delimitedList( identifier ) + SEMI ) - - regIdentifier = Group( identifier + Optional( LBRACK + expr + COLON + expr + RBRACK ) ) - regDecl = Group( "reg" + Optional("signed") + Optional( range ) + delimitedList( regIdentifier ) + SEMI ).setName("regDecl") - timeDecl = Group( "time" + delimitedList( regIdentifier ) + SEMI ) - integerDecl = Group( "integer" + delimitedList( regIdentifier ) + SEMI ) + paramAssgnmt = Group(identifier + EQ + expr).setName("paramAssgnmt") + parameterDecl = Group( + "parameter" + Optional(range) + delimitedList(paramAssgnmt) + SEMI + ).setName("paramDecl") + + inputDecl = Group("input" + Optional(range) + delimitedList(identifier) + SEMI) + outputDecl = Group( + "output" + Optional(range) + delimitedList(identifier) + SEMI + ) + inoutDecl = Group("inout" + Optional(range) + delimitedList(identifier) + SEMI) + + regIdentifier = Group( + identifier + Optional(LBRACK + expr + COLON + expr + RBRACK) + ) + regDecl = Group( + "reg" + + Optional("signed") + + Optional(range) + + delimitedList(regIdentifier) + + SEMI + ).setName("regDecl") + timeDecl = Group("time" + delimitedList(regIdentifier) + SEMI) + integerDecl = Group("integer" + delimitedList(regIdentifier) + SEMI) strength0 = oneOf("supply0 strong0 pull0 weak0 highz0") strength1 = oneOf("supply1 strong1 pull1 weak1 highz1") - driveStrength = Group( LPAR + ( ( strength0 + COMMA + strength1 ) | - ( strength1 + COMMA + strength0 ) ) + RPAR ).setName("driveStrength") - nettype = oneOf("wire tri tri1 supply0 wand triand tri0 supply1 wor trior trireg") - expandRange = Optional( oneOf("scalared vectored") ) + range - realDecl = Group( "real" + delimitedList( identifier ) + SEMI ) - - eventDecl = Group( "event" + delimitedList( identifier ) + SEMI ) + driveStrength = Group( + LPAR + + ((strength0 + COMMA + strength1) | (strength1 + COMMA + strength0)) + + RPAR + ).setName("driveStrength") + nettype = oneOf( + "wire tri tri1 supply0 wand triand tri0 supply1 wor trior trireg" + ) + expandRange = Optional(oneOf("scalared vectored")) + range + realDecl = Group("real" + delimitedList(identifier) + SEMI) + + eventDecl = Group("event" + delimitedList(identifier) + SEMI) blockDecl = ( - parameterDecl | - regDecl | - integerDecl | - realDecl | - timeDecl | - eventDecl - ) + parameterDecl | regDecl | integerDecl | realDecl | timeDecl | eventDecl + ) - stmt = Forward().setName("stmt")#.setDebug() + stmt = Forward().setName("stmt") # .setDebug() stmtOrNull = stmt | SEMI - caseItem = ( delimitedList( expr ) + COLON + stmtOrNull ) | \ - ( default + Optional(":") + stmtOrNull ) + caseItem = (delimitedList(expr) + COLON + stmtOrNull) | ( + default + Optional(":") + stmtOrNull + ) stmt << Group( - ( begin + Group( ZeroOrMore( stmt ) ) + end ).setName("begin-end") | - ( if_ + Group(LPAR + expr + RPAR) + stmtOrNull + Optional( else_ + stmtOrNull ) ).setName("if") | - ( delayOrEventControl + stmtOrNull ) | - ( case + LPAR + expr + RPAR + OneOrMore( caseItem ) + endcase ) | - ( forever + stmt ) | - ( repeat + LPAR + expr + RPAR + stmt ) | - ( while_ + LPAR + expr + RPAR + stmt ) | - ( for_ + LPAR + assgnmt + SEMI + Group( expr ) + SEMI + assgnmt + RPAR + stmt ) | - ( fork + ZeroOrMore( stmt ) + join ) | - ( fork + COLON + identifier + ZeroOrMore( blockDecl ) + ZeroOrMore( stmt ) + end ) | - ( wait + LPAR + expr + RPAR + stmtOrNull ) | - ( "->" + identifier + SEMI ) | - ( disable + identifier + SEMI ) | - ( assign + assgnmt + SEMI ) | - ( deassign + lvalue + SEMI ) | - ( force + assgnmt + SEMI ) | - ( release + lvalue + SEMI ) | - ( begin + COLON + identifier + ZeroOrMore( blockDecl ) + ZeroOrMore( stmt ) + end ).setName("begin:label-end") | + (begin + Group(ZeroOrMore(stmt)) + end).setName("begin-end") + | ( + if_ + + Group(LPAR + expr + RPAR) + + stmtOrNull + + Optional(else_ + stmtOrNull) + ).setName("if") + | (delayOrEventControl + stmtOrNull) + | (case + LPAR + expr + RPAR + OneOrMore(caseItem) + endcase) + | (forever + stmt) + | (repeat + LPAR + expr + RPAR + stmt) + | (while_ + LPAR + expr + RPAR + stmt) + | ( + for_ + + LPAR + + assgnmt + + SEMI + + Group(expr) + + SEMI + + assgnmt + + RPAR + + stmt + ) + | (fork + ZeroOrMore(stmt) + join) + | ( + fork + + COLON + + identifier + + ZeroOrMore(blockDecl) + + ZeroOrMore(stmt) + + end + ) + | (wait + LPAR + expr + RPAR + stmtOrNull) + | ("->" + identifier + SEMI) + | (disable + identifier + SEMI) + | (assign + assgnmt + SEMI) + | (deassign + lvalue + SEMI) + | (force + assgnmt + SEMI) + | (release + lvalue + SEMI) + | ( + begin + + COLON + + identifier + + ZeroOrMore(blockDecl) + + ZeroOrMore(stmt) + + end + ).setName("begin:label-end") + | # these *have* to go at the end of the list!!! - ( assgnmt + SEMI ) | - ( nbAssgnmt + SEMI ) | - ( Combine( Optional("$") + identifier ) + Optional( LPAR + delimitedList(expr|empty) + RPAR ) + SEMI ) - ).setName("stmtBody") + (assgnmt + SEMI) + | (nbAssgnmt + SEMI) + | ( + Combine(Optional("$") + identifier) + + Optional(LPAR + delimitedList(expr | empty) + RPAR) + + SEMI + ) + ).setName("stmtBody") """ x::=<blocking_assignment> ; x||= <non_blocking_assignment> ; @@ -288,199 +395,359 @@ def Verilog_BNF(): x||= force <assignment> ; x||= release <lvalue> ; """ - alwaysStmt = Group( "always" + Optional(eventControl) + stmt ).setName("alwaysStmt") - initialStmt = Group( "initial" + stmt ).setName("initialStmt") + alwaysStmt = Group("always" + Optional(eventControl) + stmt).setName( + "alwaysStmt" + ) + initialStmt = Group("initial" + stmt).setName("initialStmt") - chargeStrength = Group( LPAR + oneOf( "small medium large" ) + RPAR ).setName("chargeStrength") + chargeStrength = Group(LPAR + oneOf("small medium large") + RPAR).setName( + "chargeStrength" + ) continuousAssign = Group( - assign + Optional( driveStrength ) + Optional( delay ) + delimitedList( assgnmt ) + SEMI - ).setName("continuousAssign") - + assign + + Optional(driveStrength) + + Optional(delay) + + delimitedList(assgnmt) + + SEMI + ).setName("continuousAssign") tfDecl = ( - parameterDecl | - inputDecl | - outputDecl | - inoutDecl | - regDecl | - timeDecl | - integerDecl | - realDecl - ) + parameterDecl + | inputDecl + | outputDecl + | inoutDecl + | regDecl + | timeDecl + | integerDecl + | realDecl + ) functionDecl = Group( - "function" + Optional( range | "integer" | "real" ) + identifier + SEMI + - Group( OneOrMore( tfDecl ) ) + - Group( ZeroOrMore( stmt ) ) + - "endfunction" - ) + "function" + + Optional(range | "integer" | "real") + + identifier + + SEMI + + Group(OneOrMore(tfDecl)) + + Group(ZeroOrMore(stmt)) + + "endfunction" + ) inputOutput = oneOf("input output") - netDecl1Arg = ( nettype + - Optional( expandRange ) + - Optional( delay ) + - Group( delimitedList( ~inputOutput + identifier ) ) ) - netDecl2Arg = ( "trireg" + - Optional( chargeStrength ) + - Optional( expandRange ) + - Optional( delay ) + - Group( delimitedList( ~inputOutput + identifier ) ) ) - netDecl3Arg = ( nettype + - Optional( driveStrength ) + - Optional( expandRange ) + - Optional( delay ) + - Group( delimitedList( assgnmt ) ) ) + netDecl1Arg = ( + nettype + + Optional(expandRange) + + Optional(delay) + + Group(delimitedList(~inputOutput + identifier)) + ) + netDecl2Arg = ( + "trireg" + + Optional(chargeStrength) + + Optional(expandRange) + + Optional(delay) + + Group(delimitedList(~inputOutput + identifier)) + ) + netDecl3Arg = ( + nettype + + Optional(driveStrength) + + Optional(expandRange) + + Optional(delay) + + Group(delimitedList(assgnmt)) + ) netDecl1 = Group(netDecl1Arg + SEMI).setName("netDecl1") netDecl2 = Group(netDecl2Arg + SEMI).setName("netDecl2") netDecl3 = Group(netDecl3Arg + SEMI).setName("netDecl3") - gateType = oneOf("and nand or nor xor xnor buf bufif0 bufif1 " - "not notif0 notif1 pulldown pullup nmos rnmos " - "pmos rpmos cmos rcmos tran rtran tranif0 " - "rtranif0 tranif1 rtranif1" ) - gateInstance = Optional( Group( identifier + Optional( range ) ) ) + \ - LPAR + Group( delimitedList( expr ) ) + RPAR - gateDecl = Group( gateType + - Optional( driveStrength ) + - Optional( delay ) + - delimitedList( gateInstance) + - SEMI ) - - udpInstance = Group( Group( identifier + Optional(range | subscrRef) ) + - LPAR + Group( delimitedList( expr ) ) + RPAR ) - udpInstantiation = Group( identifier - - Optional( driveStrength ) + - Optional( delay ) + - delimitedList( udpInstance ) + - SEMI ).setName("udpInstantiation") - - parameterValueAssignment = Group( Literal("#") + LPAR + Group( delimitedList( expr ) ) + RPAR ) - namedPortConnection = Group( DOT + identifier + LPAR + expr + RPAR ).setName("namedPortConnection")#.setDebug() - assert(r'.\abc (abc )' == namedPortConnection) + gateType = oneOf( + "and nand or nor xor xnor buf bufif0 bufif1 " + "not notif0 notif1 pulldown pullup nmos rnmos " + "pmos rpmos cmos rcmos tran rtran tranif0 " + "rtranif0 tranif1 rtranif1" + ) + gateInstance = ( + Optional(Group(identifier + Optional(range))) + + LPAR + + Group(delimitedList(expr)) + + RPAR + ) + gateDecl = Group( + gateType + + Optional(driveStrength) + + Optional(delay) + + delimitedList(gateInstance) + + SEMI + ) + + udpInstance = Group( + Group(identifier + Optional(range | subscrRef)) + + LPAR + + Group(delimitedList(expr)) + + RPAR + ) + udpInstantiation = Group( + identifier + - Optional(driveStrength) + + Optional(delay) + + delimitedList(udpInstance) + + SEMI + ).setName("udpInstantiation") + + parameterValueAssignment = Group( + Literal("#") + LPAR + Group(delimitedList(expr)) + RPAR + ) + namedPortConnection = Group(DOT + identifier + LPAR + expr + RPAR).setName( + "namedPortConnection" + ) # .setDebug() + assert r".\abc (abc )" == namedPortConnection modulePortConnection = expr | empty - #~ moduleInstance = Group( Group ( identifier + Optional(range) ) + - #~ ( delimitedList( modulePortConnection ) | - #~ delimitedList( namedPortConnection ) ) ) - inst_args = Group( LPAR + (delimitedList( namedPortConnection ) | - delimitedList( modulePortConnection )) + RPAR).setName("inst_args") - moduleInstance = Group( Group ( identifier + Optional(range) ) + inst_args ).setName("moduleInstance")#.setDebug() - - moduleInstantiation = Group( identifier + - Optional( parameterValueAssignment ) + - delimitedList( moduleInstance ).setName("moduleInstanceList") + - SEMI ).setName("moduleInstantiation") - - parameterOverride = Group( "defparam" + delimitedList( paramAssgnmt ) + SEMI ) - task = Group( "task" + identifier + SEMI + - ZeroOrMore( tfDecl ) + - stmtOrNull + - "endtask" ) - - specparamDecl = Group( "specparam" + delimitedList( paramAssgnmt ) + SEMI ) - - pathDescr1 = Group( LPAR + subscrIdentifier + "=>" + subscrIdentifier + RPAR ) - pathDescr2 = Group( LPAR + Group( delimitedList( subscrIdentifier ) ) + "*>" + - Group( delimitedList( subscrIdentifier ) ) + RPAR ) - pathDescr3 = Group( LPAR + Group( delimitedList( subscrIdentifier ) ) + "=>" + - Group( delimitedList( subscrIdentifier ) ) + RPAR ) - pathDelayValue = Group( ( LPAR + Group( delimitedList( mintypmaxExpr | expr ) ) + RPAR ) | - mintypmaxExpr | - expr ) - pathDecl = Group( ( pathDescr1 | pathDescr2 | pathDescr3 ) + EQ + pathDelayValue + SEMI ).setName("pathDecl") + # ~ moduleInstance = Group( Group ( identifier + Optional(range) ) + + # ~ ( delimitedList( modulePortConnection ) | + # ~ delimitedList( namedPortConnection ) ) ) + inst_args = Group( + LPAR + + (delimitedList(namedPortConnection) | delimitedList(modulePortConnection)) + + RPAR + ).setName("inst_args") + moduleInstance = Group(Group(identifier + Optional(range)) + inst_args).setName( + "moduleInstance" + ) # .setDebug() + + moduleInstantiation = Group( + identifier + + Optional(parameterValueAssignment) + + delimitedList(moduleInstance).setName("moduleInstanceList") + + SEMI + ).setName("moduleInstantiation") + + parameterOverride = Group("defparam" + delimitedList(paramAssgnmt) + SEMI) + task = Group( + "task" + identifier + SEMI + ZeroOrMore(tfDecl) + stmtOrNull + "endtask" + ) + + specparamDecl = Group("specparam" + delimitedList(paramAssgnmt) + SEMI) + + pathDescr1 = Group(LPAR + subscrIdentifier + "=>" + subscrIdentifier + RPAR) + pathDescr2 = Group( + LPAR + + Group(delimitedList(subscrIdentifier)) + + "*>" + + Group(delimitedList(subscrIdentifier)) + + RPAR + ) + pathDescr3 = Group( + LPAR + + Group(delimitedList(subscrIdentifier)) + + "=>" + + Group(delimitedList(subscrIdentifier)) + + RPAR + ) + pathDelayValue = Group( + (LPAR + Group(delimitedList(mintypmaxExpr | expr)) + RPAR) + | mintypmaxExpr + | expr + ) + pathDecl = Group( + (pathDescr1 | pathDescr2 | pathDescr3) + EQ + pathDelayValue + SEMI + ).setName("pathDecl") portConditionExpr = Forward() portConditionTerm = Optional(unop) + subscrIdentifier - portConditionExpr << portConditionTerm + Optional( binop + portConditionExpr ) + portConditionExpr << portConditionTerm + Optional(binop + portConditionExpr) polarityOp = oneOf("+ -") levelSensitivePathDecl1 = Group( - if_ + Group(LPAR + portConditionExpr + RPAR) + - subscrIdentifier + Optional( polarityOp ) + "=>" + subscrIdentifier + EQ + - pathDelayValue + - SEMI ) + if_ + + Group(LPAR + portConditionExpr + RPAR) + + subscrIdentifier + + Optional(polarityOp) + + "=>" + + subscrIdentifier + + EQ + + pathDelayValue + + SEMI + ) levelSensitivePathDecl2 = Group( - if_ + Group(LPAR + portConditionExpr + RPAR) + - LPAR + Group( delimitedList( subscrIdentifier ) ) + Optional( polarityOp ) + "*>" + - Group( delimitedList( subscrIdentifier ) ) + RPAR + EQ + - pathDelayValue + - SEMI ) + if_ + + Group(LPAR + portConditionExpr + RPAR) + + LPAR + + Group(delimitedList(subscrIdentifier)) + + Optional(polarityOp) + + "*>" + + Group(delimitedList(subscrIdentifier)) + + RPAR + + EQ + + pathDelayValue + + SEMI + ) levelSensitivePathDecl = levelSensitivePathDecl1 | levelSensitivePathDecl2 edgeIdentifier = posedge | negedge edgeSensitivePathDecl1 = Group( - Optional( if_ + Group(LPAR + expr + RPAR) ) + - LPAR + Optional( edgeIdentifier ) + - subscrIdentifier + "=>" + - LPAR + subscrIdentifier + Optional( polarityOp ) + COLON + expr + RPAR + RPAR + - EQ + - pathDelayValue + - SEMI ) + Optional(if_ + Group(LPAR + expr + RPAR)) + + LPAR + + Optional(edgeIdentifier) + + subscrIdentifier + + "=>" + + LPAR + + subscrIdentifier + + Optional(polarityOp) + + COLON + + expr + + RPAR + + RPAR + + EQ + + pathDelayValue + + SEMI + ) edgeSensitivePathDecl2 = Group( - Optional( if_ + Group(LPAR + expr + RPAR) ) + - LPAR + Optional( edgeIdentifier ) + - subscrIdentifier + "*>" + - LPAR + delimitedList( subscrIdentifier ) + Optional( polarityOp ) + COLON + expr + RPAR + RPAR + - EQ + - pathDelayValue + - SEMI ) + Optional(if_ + Group(LPAR + expr + RPAR)) + + LPAR + + Optional(edgeIdentifier) + + subscrIdentifier + + "*>" + + LPAR + + delimitedList(subscrIdentifier) + + Optional(polarityOp) + + COLON + + expr + + RPAR + + RPAR + + EQ + + pathDelayValue + + SEMI + ) edgeSensitivePathDecl = edgeSensitivePathDecl1 | edgeSensitivePathDecl2 edgeDescr = oneOf("01 10 0x x1 1x x0").setName("edgeDescr") - timCheckEventControl = Group( posedge | negedge | (edge + LBRACK + delimitedList( edgeDescr ) + RBRACK )) + timCheckEventControl = Group( + posedge | negedge | (edge + LBRACK + delimitedList(edgeDescr) + RBRACK) + ) timCheckCond = Forward() timCondBinop = oneOf("== === != !==") - timCheckCondTerm = ( expr + timCondBinop + scalarConst ) | ( Optional("~") + expr ) - timCheckCond << ( ( LPAR + timCheckCond + RPAR ) | timCheckCondTerm ) - timCheckEvent = Group( Optional( timCheckEventControl ) + - subscrIdentifier + - Optional( "&&&" + timCheckCond ) ) + timCheckCondTerm = (expr + timCondBinop + scalarConst) | (Optional("~") + expr) + timCheckCond << ((LPAR + timCheckCond + RPAR) | timCheckCondTerm) + timCheckEvent = Group( + Optional(timCheckEventControl) + + subscrIdentifier + + Optional("&&&" + timCheckCond) + ) timCheckLimit = expr - controlledTimingCheckEvent = Group( timCheckEventControl + subscrIdentifier + - Optional( "&&&" + timCheckCond ) ) + controlledTimingCheckEvent = Group( + timCheckEventControl + subscrIdentifier + Optional("&&&" + timCheckCond) + ) notifyRegister = identifier - systemTimingCheck1 = Group( "$setup" + - LPAR + timCheckEvent + COMMA + timCheckEvent + COMMA + timCheckLimit + - Optional( COMMA + notifyRegister ) + RPAR + - SEMI ) - systemTimingCheck2 = Group( "$hold" + - LPAR + timCheckEvent + COMMA + timCheckEvent + COMMA + timCheckLimit + - Optional( COMMA + notifyRegister ) + RPAR + - SEMI ) - systemTimingCheck3 = Group( "$period" + - LPAR + controlledTimingCheckEvent + COMMA + timCheckLimit + - Optional( COMMA + notifyRegister ) + RPAR + - SEMI ) - systemTimingCheck4 = Group( "$width" + - LPAR + controlledTimingCheckEvent + COMMA + timCheckLimit + - Optional( COMMA + expr + COMMA + notifyRegister ) + RPAR + - SEMI ) - systemTimingCheck5 = Group( "$skew" + - LPAR + timCheckEvent + COMMA + timCheckEvent + COMMA + timCheckLimit + - Optional( COMMA + notifyRegister ) + RPAR + - SEMI ) - systemTimingCheck6 = Group( "$recovery" + - LPAR + controlledTimingCheckEvent + COMMA + timCheckEvent + COMMA + timCheckLimit + - Optional( COMMA + notifyRegister ) + RPAR + - SEMI ) - systemTimingCheck7 = Group( "$setuphold" + - LPAR + timCheckEvent + COMMA + timCheckEvent + COMMA + timCheckLimit + COMMA + timCheckLimit + - Optional( COMMA + notifyRegister ) + RPAR + - SEMI ) - systemTimingCheck = (FollowedBy('$') + ( systemTimingCheck1 | systemTimingCheck2 | systemTimingCheck3 | - systemTimingCheck4 | systemTimingCheck5 | systemTimingCheck6 | systemTimingCheck7 )).setName("systemTimingCheck") - sdpd = if_ + Group(LPAR + expr + RPAR) + \ - ( pathDescr1 | pathDescr2 ) + EQ + pathDelayValue + SEMI - - specifyItem = ~Keyword("endspecify") +( - specparamDecl | - pathDecl | - levelSensitivePathDecl | - edgeSensitivePathDecl | - systemTimingCheck | - sdpd + systemTimingCheck1 = Group( + "$setup" + + LPAR + + timCheckEvent + + COMMA + + timCheckEvent + + COMMA + + timCheckLimit + + Optional(COMMA + notifyRegister) + + RPAR + + SEMI + ) + systemTimingCheck2 = Group( + "$hold" + + LPAR + + timCheckEvent + + COMMA + + timCheckEvent + + COMMA + + timCheckLimit + + Optional(COMMA + notifyRegister) + + RPAR + + SEMI + ) + systemTimingCheck3 = Group( + "$period" + + LPAR + + controlledTimingCheckEvent + + COMMA + + timCheckLimit + + Optional(COMMA + notifyRegister) + + RPAR + + SEMI + ) + systemTimingCheck4 = Group( + "$width" + + LPAR + + controlledTimingCheckEvent + + COMMA + + timCheckLimit + + Optional(COMMA + expr + COMMA + notifyRegister) + + RPAR + + SEMI + ) + systemTimingCheck5 = Group( + "$skew" + + LPAR + + timCheckEvent + + COMMA + + timCheckEvent + + COMMA + + timCheckLimit + + Optional(COMMA + notifyRegister) + + RPAR + + SEMI + ) + systemTimingCheck6 = Group( + "$recovery" + + LPAR + + controlledTimingCheckEvent + + COMMA + + timCheckEvent + + COMMA + + timCheckLimit + + Optional(COMMA + notifyRegister) + + RPAR + + SEMI + ) + systemTimingCheck7 = Group( + "$setuphold" + + LPAR + + timCheckEvent + + COMMA + + timCheckEvent + + COMMA + + timCheckLimit + + COMMA + + timCheckLimit + + Optional(COMMA + notifyRegister) + + RPAR + + SEMI + ) + systemTimingCheck = ( + FollowedBy("$") + + ( + systemTimingCheck1 + | systemTimingCheck2 + | systemTimingCheck3 + | systemTimingCheck4 + | systemTimingCheck5 + | systemTimingCheck6 + | systemTimingCheck7 ) + ).setName("systemTimingCheck") + sdpd = ( + if_ + + Group(LPAR + expr + RPAR) + + (pathDescr1 | pathDescr2) + + EQ + + pathDelayValue + + SEMI + ) + + specifyItem = ~Keyword("endspecify") + ( + specparamDecl + | pathDecl + | levelSensitivePathDecl + | edgeSensitivePathDecl + | systemTimingCheck + | sdpd + ) """ x::= <specparam_declaration> x||= <path_declaration> @@ -489,33 +756,36 @@ def Verilog_BNF(): x||= <system_timing_check> x||= <sdpd> """ - specifyBlock = Group( "specify" + ZeroOrMore( specifyItem ) + "endspecify" ).setName("specifyBlock") + specifyBlock = Group( + "specify" + ZeroOrMore(specifyItem) + "endspecify" + ).setName("specifyBlock") moduleItem = ~Keyword("endmodule") + ( - parameterDecl | - inputDecl | - outputDecl | - inoutDecl | - regDecl | - netDecl3 | - netDecl1 | - netDecl2 | - timeDecl | - integerDecl | - realDecl | - eventDecl | - gateDecl | - parameterOverride | - continuousAssign | - specifyBlock | - initialStmt | - alwaysStmt | - task | - functionDecl | + parameterDecl + | inputDecl + | outputDecl + | inoutDecl + | regDecl + | netDecl3 + | netDecl1 + | netDecl2 + | timeDecl + | integerDecl + | realDecl + | eventDecl + | gateDecl + | parameterOverride + | continuousAssign + | specifyBlock + | initialStmt + | alwaysStmt + | task + | functionDecl + | # these have to be at the end - they start with identifiers - moduleInstantiation | - udpInstantiation - ) + moduleInstantiation + | udpInstantiation + ) """ All possible moduleItems, from Verilog grammar spec x::= <parameter_declaration> x||= <input_declaration> @@ -539,39 +809,55 @@ def Verilog_BNF(): x||= <function> """ portRef = subscrIdentifier - portExpr = portRef | Group( LBRACE + delimitedList( portRef ) + RBRACE ) - port = portExpr | Group( DOT + identifier + LPAR + portExpr + RPAR ) - - moduleHdr = Group ( oneOf("module macromodule") + identifier + - Optional( LPAR + Group( Optional( delimitedList( - Group(oneOf("input output") + - (netDecl1Arg | netDecl2Arg | netDecl3Arg) ) | - port ) ) ) + - RPAR ) + SEMI ).setName("moduleHdr") + portExpr = portRef | Group(LBRACE + delimitedList(portRef) + RBRACE) + port = portExpr | Group(DOT + identifier + LPAR + portExpr + RPAR) + + moduleHdr = Group( + oneOf("module macromodule") + + identifier + + Optional( + LPAR + + Group( + Optional( + delimitedList( + Group( + oneOf("input output") + + (netDecl1Arg | netDecl2Arg | netDecl3Arg) + ) + | port + ) + ) + ) + + RPAR + ) + + SEMI + ).setName("moduleHdr") - module = Group( moduleHdr + - Group( ZeroOrMore( moduleItem ) ) + - "endmodule" ).setName("module")#.setDebug() + module = Group(moduleHdr + Group(ZeroOrMore(moduleItem)) + "endmodule").setName( + "module" + ) # .setDebug() udpDecl = outputDecl | inputDecl | regDecl - #~ udpInitVal = oneOf("1'b0 1'b1 1'bx 1'bX 1'B0 1'B1 1'Bx 1'BX 1 0 x X") + # ~ udpInitVal = oneOf("1'b0 1'b1 1'bx 1'bX 1'B0 1'B1 1'Bx 1'BX 1 0 x X") udpInitVal = (Regex("1'[bB][01xX]") | Regex("[01xX]")).setName("udpInitVal") - udpInitialStmt = Group( "initial" + - identifier + EQ + udpInitVal + SEMI ).setName("udpInitialStmt") + udpInitialStmt = Group("initial" + identifier + EQ + udpInitVal + SEMI).setName( + "udpInitialStmt" + ) levelSymbol = oneOf("0 1 x X ? b B") - levelInputList = Group( OneOrMore( levelSymbol ).setName("levelInpList") ) + levelInputList = Group(OneOrMore(levelSymbol).setName("levelInpList")) outputSymbol = oneOf("0 1 x X") - combEntry = Group( levelInputList + COLON + outputSymbol + SEMI ) + combEntry = Group(levelInputList + COLON + outputSymbol + SEMI) edgeSymbol = oneOf("r R f F p P n N *") - edge = Group( LPAR + levelSymbol + levelSymbol + RPAR ) | \ - Group( edgeSymbol ) - edgeInputList = Group( ZeroOrMore( levelSymbol ) + edge + ZeroOrMore( levelSymbol ) ) + edge = Group(LPAR + levelSymbol + levelSymbol + RPAR) | Group(edgeSymbol) + edgeInputList = Group(ZeroOrMore(levelSymbol) + edge + ZeroOrMore(levelSymbol)) inputList = levelInputList | edgeInputList - seqEntry = Group( inputList + COLON + levelSymbol + COLON + ( outputSymbol | "-" ) + SEMI ).setName("seqEntry") - udpTableDefn = Group( "table" + - OneOrMore( combEntry | seqEntry ) + - "endtable" ).setName("table") + seqEntry = Group( + inputList + COLON + levelSymbol + COLON + (outputSymbol | "-") + SEMI + ).setName("seqEntry") + udpTableDefn = Group( + "table" + OneOrMore(combEntry | seqEntry) + "endtable" + ).setName("table") """ <UDP> @@ -581,49 +867,58 @@ def Verilog_BNF(): <table_definition> endprimitive """ - udp = Group( "primitive" + identifier + - LPAR + Group( delimitedList( identifier ) ) + RPAR + SEMI + - OneOrMore( udpDecl ) + - Optional( udpInitialStmt ) + - udpTableDefn + - "endprimitive" ) - - verilogbnf = OneOrMore( module | udp ) + StringEnd() - - verilogbnf.ignore( cppStyleComment ) - verilogbnf.ignore( compilerDirective ) + udp = Group( + "primitive" + + identifier + + LPAR + + Group(delimitedList(identifier)) + + RPAR + + SEMI + + OneOrMore(udpDecl) + + Optional(udpInitialStmt) + + udpTableDefn + + "endprimitive" + ) + + verilogbnf = OneOrMore(module | udp) + StringEnd() + + verilogbnf.ignore(cppStyleComment) + verilogbnf.ignore(compilerDirective) return verilogbnf -def test( strng ): +def test(strng): tokens = [] try: - tokens = Verilog_BNF().parseString( strng ) + tokens = Verilog_BNF().parseString(strng) except ParseException as err: print(err.line) - print(" "*(err.column-1) + "^") + print(" " * (err.column - 1) + "^") print(err) return tokens -#~ if __name__ == "__main__": +# ~ if __name__ == "__main__": if 0: import pprint + toptest = """ module TOP( in, out ); input [7:0] in; output [5:0] out; COUNT_BITS8 count_bits( .IN( in ), .C( out ) ); endmodule""" - pprint.pprint( test(toptest).asList() ) + pprint.pprint(test(toptest).asList()) else: + def main(): print("Verilog parser test (V %s)" % __version__) print(" - using pyparsing version", pyparsing.__version__) print(" - using Python version", sys.version) - if packratOn: print(" - using packrat parsing") + if packratOn: + print(" - using packrat parsing") print() import os @@ -634,41 +929,41 @@ else: numlines = 0 startTime = time.clock() fileDir = "verilog" - #~ fileDir = "verilog/new" - #~ fileDir = "verilog/new2" - #~ fileDir = "verilog/new3" + # ~ fileDir = "verilog/new" + # ~ fileDir = "verilog/new2" + # ~ fileDir = "verilog/new3" allFiles = [f for f in os.listdir(fileDir) if f.endswith(".v")] - #~ allFiles = [ "list_path_delays_test.v" ] - #~ allFiles = [ "escapedIdent.v" ] - #~ allFiles = filter( lambda f : f.startswith("a") and f.endswith(".v"), os.listdir(fileDir) ) - #~ allFiles = filter( lambda f : f.startswith("c") and f.endswith(".v"), os.listdir(fileDir) ) - #~ allFiles = [ "ff.v" ] + # ~ allFiles = [ "list_path_delays_test.v" ] + # ~ allFiles = [ "escapedIdent.v" ] + # ~ allFiles = filter( lambda f : f.startswith("a") and f.endswith(".v"), os.listdir(fileDir) ) + # ~ allFiles = filter( lambda f : f.startswith("c") and f.endswith(".v"), os.listdir(fileDir) ) + # ~ allFiles = [ "ff.v" ] - pp = pprint.PrettyPrinter( indent=2 ) + pp = pprint.PrettyPrinter(indent=2) totalTime = 0 for vfile in allFiles: gc.collect() - fnam = fileDir + "/"+vfile + fnam = fileDir + "/" + vfile infile = open(fnam) filelines = infile.readlines() infile.close() - print(fnam, len(filelines), end=' ') + print(fnam, len(filelines), end=" ") numlines += len(filelines) teststr = "".join(filelines) time1 = time.clock() - tokens = test( teststr ) + tokens = test(teststr) time2 = time.clock() - elapsed = time2-time1 + elapsed = time2 - time1 totalTime += elapsed - if ( len( tokens ) ): + if len(tokens): print("OK", elapsed) - #~ print "tokens=" - #~ pp.pprint( tokens.asList() ) - #~ print + # ~ print "tokens=" + # ~ pp.pprint( tokens.asList() ) + # ~ print ofnam = fileDir + "/parseOutput/" + vfile + ".parsed.txt" - outfile = open(ofnam,"w") - outfile.write( teststr ) + outfile = open(ofnam, "w") + outfile.write(teststr) outfile.write("\n") outfile.write("\n") outfile.write(pp.pformat(tokens.asList())) @@ -677,12 +972,12 @@ else: else: print("failed", elapsed) failCount += 1 - for i,line in enumerate(filelines,1): - print("%4d: %s" % (i,line.rstrip())) + for i, line in enumerate(filelines, 1): + print("%4d: %s" % (i, line.rstrip())) endTime = time.clock() print("Total parse time:", totalTime) print("Total source lines:", numlines) - print("Average lines/sec:", ( "%.1f" % (float(numlines)/(totalTime+.05 ) ) )) + print("Average lines/sec:", ("%.1f" % (float(numlines) / (totalTime + 0.05)))) if failCount: print("FAIL - %d files failed to parse" % failCount) else: @@ -690,16 +985,16 @@ else: return 0 - #~ from line_profiler import LineProfiler - #~ from pyparsing import ParseResults - #~ lp = LineProfiler(ParseResults.__init__) + # ~ from line_profiler import LineProfiler + # ~ from pyparsing import ParseResults + # ~ lp = LineProfiler(ParseResults.__init__) main() - #~ lp.print_stats() - #~ import hotshot - #~ p = hotshot.Profile("vparse.prof",1,1) - #~ p.start() - #~ main() - #~ p.stop() - #~ p.close() + # ~ lp.print_stats() + # ~ import hotshot + # ~ p = hotshot.Profile("vparse.prof",1,1) + # ~ p.start() + # ~ main() + # ~ p.stop() + # ~ p.close() diff --git a/examples/wordsToNum.py b/examples/wordsToNum.py index 71538ba..99eddab 100644 --- a/examples/wordsToNum.py +++ b/examples/wordsToNum.py @@ -7,73 +7,82 @@ import pyparsing as pp from operator import mul from functools import reduce + def makeLit(s, val): ret = pp.CaselessLiteral(s) return ret.setParseAction(pp.replaceWith(val)) + unitDefinitions = [ - ("zero", 0), - ("oh", 0), - ("zip", 0), - ("zilch", 0), - ("nada", 0), - ("bupkis", 0), - ("one", 1), - ("two", 2), - ("three", 3), - ("four", 4), - ("five", 5), - ("six", 6), - ("seven", 7), - ("eight", 8), - ("nine", 9), - ("ten", 10), - ("eleven", 11), - ("twelve", 12), - ("thirteen", 13), - ("fourteen", 14), - ("fifteen", 15), - ("sixteen", 16), + ("zero", 0), + ("oh", 0), + ("zip", 0), + ("zilch", 0), + ("nada", 0), + ("bupkis", 0), + ("one", 1), + ("two", 2), + ("three", 3), + ("four", 4), + ("five", 5), + ("six", 6), + ("seven", 7), + ("eight", 8), + ("nine", 9), + ("ten", 10), + ("eleven", 11), + ("twelve", 12), + ("thirteen", 13), + ("fourteen", 14), + ("fifteen", 15), + ("sixteen", 16), ("seventeen", 17), - ("eighteen", 18), - ("nineteen", 19), - ] -units = pp.MatchFirst(makeLit(s,v) for s,v in sorted(unitDefinitions, key=lambda d: -len(d[0]))) + ("eighteen", 18), + ("nineteen", 19), +] +units = pp.MatchFirst( + makeLit(s, v) for s, v in sorted(unitDefinitions, key=lambda d: -len(d[0])) +) tensDefinitions = [ - ("ten", 10), - ("twenty", 20), - ("thirty", 30), - ("forty", 40), - ("fourty", 40), # for the spelling-challenged... - ("fifty", 50), - ("sixty", 60), + ("ten", 10), + ("twenty", 20), + ("thirty", 30), + ("forty", 40), + ("fourty", 40), # for the spelling-challenged... + ("fifty", 50), + ("sixty", 60), ("seventy", 70), - ("eighty", 80), - ("ninety", 90), - ] -tens = pp.MatchFirst(makeLit(s,v) for s,v in tensDefinitions) + ("eighty", 80), + ("ninety", 90), +] +tens = pp.MatchFirst(makeLit(s, v) for s, v in tensDefinitions) hundreds = makeLit("hundred", 100) majorDefinitions = [ - ("thousand", int(1e3)), - ("million", int(1e6)), - ("billion", int(1e9)), - ("trillion", int(1e12)), + ("thousand", int(1e3)), + ("million", int(1e6)), + ("billion", int(1e9)), + ("trillion", int(1e12)), ("quadrillion", int(1e15)), ("quintillion", int(1e18)), - ] -mag = pp.MatchFirst(makeLit(s,v) for s,v in majorDefinitions) - -wordprod = lambda t: reduce(mul,t) -numPart = ((((units + pp.Optional(hundreds)).setParseAction(wordprod) - + pp.Optional(tens) - ).setParseAction(sum) - ^ tens) - + pp.Optional(units) - ).setParseAction(sum) -numWords = ((numPart + pp.Optional(mag)).setParseAction(wordprod)[1, ...]).setParseAction(sum) +] +mag = pp.MatchFirst(makeLit(s, v) for s, v in majorDefinitions) + +wordprod = lambda t: reduce(mul, t) +numPart = ( + ( + ( + (units + pp.Optional(hundreds)).setParseAction(wordprod) + pp.Optional(tens) + ).setParseAction(sum) + ^ tens + ) + + pp.Optional(units) +).setParseAction(sum) +numWords = ( + (numPart + pp.Optional(mag)).setParseAction(wordprod)[1, ...] +).setParseAction(sum) numWords.setName("num word parser") numWords.ignore(pp.Literal("-")) @@ -98,13 +107,20 @@ tests = """ """ # use '| ...' to indicate "if omitted, skip to next" logic -test_expr = (numWords('result') | ...) + ',' + (pp.pyparsing_common.integer('expected') | 'None') +test_expr = ( + (numWords("result") | ...) + + "," + + (pp.pyparsing_common.integer("expected") | "None") +) + def verify_result(t): - if '_skipped' in t: - t['pass'] = False - elif 'expected' in t: - t['pass'] = t.result == t.expected + if "_skipped" in t: + t["pass"] = False + elif "expected" in t: + t["pass"] = t.result == t.expected + + test_expr.addParseAction(verify_result) test_expr.runTests(tests) diff --git a/pyparsing.py b/pyparsing.py index eff9170..824b4b1 100644 --- a/pyparsing.py +++ b/pyparsing.py @@ -23,8 +23,7 @@ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__doc__ = \ -""" +__doc__ = """ pyparsing module - Classes and methods to define and execute parsing grammars ============================================================================= @@ -123,6 +122,7 @@ import unittest class __config_flags: """Internal class for defining compatibility and debugging flags""" + _all_names = [] _fixed_names = [] _type_desc = "configuration" @@ -130,8 +130,14 @@ class __config_flags: @classmethod def _set(cls, dname, value): if dname in cls._fixed_names: - warnings.warn("{}.{} {} is {} and cannot be overridden".format(cls.__name__, dname, cls._type_desc, - str(getattr(cls, dname)).upper())) + warnings.warn( + "{}.{} {} is {} and cannot be overridden".format( + cls.__name__, + dname, + cls._type_desc, + str(getattr(cls, dname)).upper(), + ) + ) return if dname in cls._all_names: setattr(cls, dname, value) @@ -141,6 +147,7 @@ class __config_flags: enable = classmethod(lambda cls, name: cls._set(name, True)) disable = classmethod(lambda cls, name: cls._set(name, False)) + class __compat__(__config_flags): """ A cross-version compatibility configuration for pyparsing features that will be @@ -153,15 +160,17 @@ class __compat__(__config_flags): maintained for compatibility, but setting to False no longer restores pre-2.3.1 behavior """ + _type_desc = "compatibility" collect_all_And_tokens = True - _all_names = [__ for __ in locals() if not __.startswith('_')] + _all_names = [__ for __ in locals() if not __.startswith("_")] _fixed_names = """ collect_all_And_tokens """.split() + class __diag__(__config_flags): """ Diagnostic configuration (all default to False) @@ -177,6 +186,7 @@ class __diag__(__config_flags): - enable_debug_on_named_expressions - flag to auto-enable debug on all subsequent calls to ParserElement.setName() """ + _type_desc = "diagnostic" warn_multiple_tokens_in_named_alternation = False @@ -185,7 +195,7 @@ class __diag__(__config_flags): warn_on_multiple_string_args_to_oneof = False enable_debug_on_named_expressions = False - _all_names = [__ for __ in locals() if not __.startswith('_')] + _all_names = [__ for __ in locals() if not __.startswith("_")] _warning_names = [name for name in _all_names if name.startswith("warn")] _debug_names = [name for name in _all_names if name.startswith("enable_debug")] @@ -194,33 +204,128 @@ class __diag__(__config_flags): for name in cls._warning_names: cls.enable(name) + # hide abstract class del __config_flags # ~ sys.stderr.write("testing pyparsing module, version %s, %s\n" % (__version__, __versionTime__)) -__all__ = ['__version__', '__versionTime__', '__author__', '__compat__', '__diag__', - 'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty', - 'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal', - 'PrecededBy', 'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', - 'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException', - 'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException', - 'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', - 'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', 'Char', - 'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col', - 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString', - 'dblSlashComment', 'delimitedList', 'dictOf', 'empty', 'hexnums', - 'htmlComment', 'javaStyleComment', 'line', 'lineEnd', 'lineStart', 'lineno', - 'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral', - 'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'printables', - 'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', - 'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd', - 'stringStart', 'traceParseAction', 'unicodeString', 'withAttribute', - 'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation', 'locatedExpr', 'withClass', - 'CloseMatch', 'tokenMap', 'pyparsing_common', 'pyparsing_unicode', 'unicode_set', - 'conditionAsParseAction', 'pyparsing_test', 're', - ] +__all__ = [ + "__version__", + "__versionTime__", + "__author__", + "__compat__", + "__diag__", + "And", + "CaselessKeyword", + "CaselessLiteral", + "CharsNotIn", + "Combine", + "Dict", + "Each", + "Empty", + "FollowedBy", + "Forward", + "GoToColumn", + "Group", + "Keyword", + "LineEnd", + "LineStart", + "Literal", + "PrecededBy", + "MatchFirst", + "NoMatch", + "NotAny", + "OneOrMore", + "OnlyOnce", + "Optional", + "Or", + "ParseBaseException", + "ParseElementEnhance", + "ParseException", + "ParseExpression", + "ParseFatalException", + "ParseResults", + "ParseSyntaxException", + "ParserElement", + "QuotedString", + "RecursiveGrammarException", + "Regex", + "SkipTo", + "StringEnd", + "StringStart", + "Suppress", + "Token", + "TokenConverter", + "White", + "Word", + "WordEnd", + "WordStart", + "ZeroOrMore", + "Char", + "alphanums", + "alphas", + "alphas8bit", + "anyCloseTag", + "anyOpenTag", + "cStyleComment", + "col", + "commonHTMLEntity", + "countedArray", + "cppStyleComment", + "dblQuotedString", + "dblSlashComment", + "delimitedList", + "dictOf", + "empty", + "hexnums", + "htmlComment", + "javaStyleComment", + "line", + "lineEnd", + "lineStart", + "lineno", + "makeHTMLTags", + "makeXMLTags", + "matchOnlyAtCol", + "matchPreviousExpr", + "matchPreviousLiteral", + "nestedExpr", + "nullDebugAction", + "nums", + "oneOf", + "opAssoc", + "printables", + "punc8bit", + "pythonStyleComment", + "quotedString", + "removeQuotes", + "replaceHTMLEntity", + "replaceWith", + "restOfLine", + "sglQuotedString", + "srange", + "stringEnd", + "stringStart", + "traceParseAction", + "unicodeString", + "withAttribute", + "indentedBlock", + "originalTextFor", + "ungroup", + "infixNotation", + "locatedExpr", + "withClass", + "CloseMatch", + "tokenMap", + "pyparsing_common", + "pyparsing_unicode", + "unicode_set", + "conditionAsParseAction", + "pyparsing_test", + "re", +] system_version = tuple(sys.version_info)[:3] _MAX_INT = sys.maxsize @@ -261,8 +366,10 @@ def conditionAsParseAction(fn, message=None, fatal=False): return pa + class ParseBaseException(Exception): """base exception class for all parsing runtime exceptions""" + # Performance tuning: we construct a *lot* of these, so keep this # constructor as small and fast as possible def __init__(self, pstr, loc=0, msg=None, elem=None): @@ -302,15 +409,24 @@ class ParseBaseException(Exception): def __str__(self): if self.pstr: if self.loc >= len(self.pstr): - foundstr = ', found end of text' + foundstr = ", found end of text" else: - foundstr = (', found %r' % self.pstr[self.loc:self.loc + 1]).replace(r'\\', '\\') + foundstr = (", found %r" % self.pstr[self.loc : self.loc + 1]).replace( + r"\\", "\\" + ) else: - foundstr = '' - return ("%s%s (at char %d), (line:%d, col:%d)" % - (self.msg, foundstr, self.loc, self.lineno, self.column)) + foundstr = "" + return "%s%s (at char %d), (line:%d, col:%d)" % ( + self.msg, + foundstr, + self.loc, + self.lineno, + self.column, + ) + def __repr__(self): return str(self) + def markInputline(self, markerString=">!<"): """Extracts the exception line from the input string, and marks the location of the exception with a special symbol. @@ -318,12 +434,15 @@ class ParseBaseException(Exception): line_str = self.line line_column = self.column - 1 if markerString: - line_str = "".join((line_str[:line_column], - markerString, line_str[line_column:])) + line_str = "".join( + (line_str[:line_column], markerString, line_str[line_column:]) + ) return line_str.strip() + def __dir__(self): return "lineno col line".split() + dir(type(self)) + class ParseException(ParseBaseException): """ Exception thrown when parse expressions don't match class; @@ -378,7 +497,7 @@ class ParseException(ParseBaseException): ret = [] if isinstance(exc, ParseBaseException): ret.append(exc.line) - ret.append(' ' * (exc.col - 1) + '^') + ret.append(" " * (exc.col - 1) + "^") ret.append("{}: {}".format(type(exc).__name__, exc)) if depth > 0: @@ -387,16 +506,20 @@ class ParseException(ParseBaseException): for i, ff in enumerate(callers[-depth:]): frm = ff[0] - f_self = frm.f_locals.get('self', None) + f_self = frm.f_locals.get("self", None) if isinstance(f_self, ParserElement): - if frm.f_code.co_name not in ('parseImpl', '_parseNoCache'): + if frm.f_code.co_name not in ("parseImpl", "_parseNoCache"): continue if f_self in seen: continue seen.add(f_self) self_type = type(f_self) - ret.append("{}.{} - {}".format(self_type.__module__, self_type.__name__, f_self)) + ret.append( + "{}.{} - {}".format( + self_type.__module__, self_type.__name__, f_self + ) + ) elif f_self is not None: self_type = type(f_self) @@ -404,7 +527,7 @@ class ParseException(ParseBaseException): else: code = frm.f_code - if code.co_name in ('wrapper', '<module>'): + if code.co_name in ("wrapper", "<module>"): continue ret.append("{}".format(code.co_name)) @@ -413,55 +536,66 @@ class ParseException(ParseBaseException): if not depth: break - return '\n'.join(ret) + return "\n".join(ret) class ParseFatalException(ParseBaseException): """user-throwable exception thrown when inconsistent parse content is found; stops all parsing immediately""" + pass + class ParseSyntaxException(ParseFatalException): """just like :class:`ParseFatalException`, but thrown internally when an :class:`ErrorStop<And._ErrorStop>` ('-' operator) indicates that parsing is to stop immediately because an unbacktrackable syntax error has been found. """ + pass -#~ class ReparseException(ParseBaseException): - #~ """Experimental class - parse actions can raise this exception to cause - #~ pyparsing to reparse the input string: - #~ - with a modified input string, and/or - #~ - with a modified start location - #~ Set the values of the ReparseException in the constructor, and raise the - #~ exception in a parse action to cause pyparsing to use the new string/location. - #~ Setting the values as None causes no change to be made. - #~ """ - #~ def __init_( self, newstring, restartLoc ): - #~ self.newParseText = newstring - #~ self.reparseLoc = restartLoc + +# ~ class ReparseException(ParseBaseException): +# ~ """Experimental class - parse actions can raise this exception to cause +# ~ pyparsing to reparse the input string: +# ~ - with a modified input string, and/or +# ~ - with a modified start location +# ~ Set the values of the ReparseException in the constructor, and raise the +# ~ exception in a parse action to cause pyparsing to use the new string/location. +# ~ Setting the values as None causes no change to be made. +# ~ """ +# ~ def __init_( self, newstring, restartLoc ): +# ~ self.newParseText = newstring +# ~ self.reparseLoc = restartLoc + class RecursiveGrammarException(Exception): """exception thrown by :class:`ParserElement.validate` if the grammar could be improperly recursive """ + def __init__(self, parseElementList): self.parseElementTrace = parseElementList def __str__(self): return "RecursiveGrammarException: %s" % self.parseElementTrace + class _ParseResultsWithOffset(object): def __init__(self, p1, p2): self.tup = (p1, p2) + def __getitem__(self, i): return self.tup[i] + def __repr__(self): return repr(self.tup[0]) + def setOffset(self, i): self.tup = (self.tup[0], i) + class ParseResults(object): """Structured parse results, to provide multiple means of access to the parsed data: @@ -505,6 +639,7 @@ class ParseResults(object): - month: 12 - year: 1999 """ + def __new__(cls, toklist=None, name=None, asList=True, modal=True): if isinstance(toklist, ParseResults): return toklist @@ -514,7 +649,9 @@ class ParseResults(object): # Performance tuning: we construct a *lot* of these, so keep this # constructor as small and fast as possible - def __init__(self, toklist=None, name=None, asList=True, modal=True, isinstance=isinstance): + def __init__( + self, toklist=None, name=None, asList=True, modal=True, isinstance=isinstance + ): if self.__doinit: self.__doinit = False self.__name = None @@ -538,14 +675,21 @@ class ParseResults(object): if isinstance(name, int): name = str(name) self.__name = name - if not (isinstance(toklist, (type(None), *str_type, list)) and toklist in (None, '', [])): + if not ( + isinstance(toklist, (type(None), *str_type, list)) + and toklist in (None, "", []) + ): if isinstance(toklist, str_type): toklist = [toklist] if asList: if isinstance(toklist, ParseResults): - self[name] = _ParseResultsWithOffset(ParseResults(toklist.__toklist), 0) + self[name] = _ParseResultsWithOffset( + ParseResults(toklist.__toklist), 0 + ) else: - self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]), 0) + self[name] = _ParseResultsWithOffset( + ParseResults(toklist[0]), 0 + ) self[name].__name = name else: try: @@ -570,7 +714,9 @@ class ParseResults(object): self.__toklist[k] = v sub = v else: - self.__tokdict[k] = self.__tokdict.get(k, list()) + [_ParseResultsWithOffset(v, 0)] + self.__tokdict[k] = self.__tokdict.get(k, list()) + [ + _ParseResultsWithOffset(v, 0) + ] sub = v if isinstance(sub, ParseResults): sub.__parent = wkref(self) @@ -592,7 +738,9 @@ class ParseResults(object): for name, occurrences in self.__tokdict.items(): for j in removed: for k, (value, position) in enumerate(occurrences): - occurrences[k] = _ParseResultsWithOffset(value, position - (position > j)) + occurrences[k] = _ParseResultsWithOffset( + value, position - (position > j) + ) else: del self.__tokdict[i] @@ -603,7 +751,7 @@ class ParseResults(object): return len(self.__toklist) def __bool__(self): - return (not not self.__toklist) + return not not self.__toklist def __iter__(self): return iter(self.__toklist) @@ -665,13 +813,11 @@ class ParseResults(object): if not args: args = [-1] for k, v in kwargs.items(): - if k == 'default': + if k == "default": args = (args[0], v) else: raise TypeError("pop() got an unexpected keyword argument '%s'" % k) - if (isinstance(args[0], int) - or len(args) == 1 - or args[0] in self): + if isinstance(args[0], int) or len(args) == 1 or args[0] in self: index = args[0] ret = self[index] del self[index] @@ -722,7 +868,9 @@ class ParseResults(object): # fixup indices in token dictionary for name, occurrences in self.__tokdict.items(): for k, (value, position) in enumerate(occurrences): - occurrences[k] = _ParseResultsWithOffset(value, position + (position > index)) + occurrences[k] = _ParseResultsWithOffset( + value, position + (position > index) + ) def append(self, item): """ @@ -781,8 +929,11 @@ class ParseResults(object): offset = len(self.__toklist) addoffset = lambda a: offset if a < 0 else a + offset otheritems = other.__tokdict.items() - otherdictitems = [(k, _ParseResultsWithOffset(v[0], addoffset(v[1]))) - for k, vlist in otheritems for v in vlist] + otherdictitems = [ + (k, _ParseResultsWithOffset(v[0], addoffset(v[1]))) + for k, vlist in otheritems + for v in vlist + ] for k, v in otherdictitems: self[k] = v if isinstance(v[0], ParseResults): @@ -804,9 +955,16 @@ class ParseResults(object): return "(%s, %s)" % (repr(self.__toklist), repr(self.__tokdict)) def __str__(self): - return '[' + ', '.join(str(i) if isinstance(i, ParseResults) else repr(i) for i in self.__toklist) + ']' + return ( + "[" + + ", ".join( + str(i) if isinstance(i, ParseResults) else repr(i) + for i in self.__toklist + ) + + "]" + ) - def _asStringList(self, sep=''): + def _asStringList(self, sep=""): out = [] for item in self.__toklist: if out and sep: @@ -832,7 +990,10 @@ class ParseResults(object): result_list = result.asList() print(type(result_list), result_list) # -> <class 'list'> ['sldkj', 'lsdkj', 'sldkj'] """ - return [res.asList() if isinstance(res, ParseResults) else res for res in self.__toklist] + return [ + res.asList() if isinstance(res, ParseResults) else res + for res in self.__toklist + ] def asDict(self): """ @@ -854,6 +1015,7 @@ class ParseResults(object): print(json.dumps(result)) # -> Exception: TypeError: ... is not JSON serializable print(json.dumps(result.asDict())) # -> {"month": "31", "day": "1999", "year": "12"} """ + def to_item(obj): if isinstance(obj, ParseResults): return obj.asDict() if obj.haskeys() else [to_item(v) for v in obj] @@ -902,17 +1064,29 @@ class ParseResults(object): return self.__name elif self.__parent: par = self.__parent() + def lookup(self, sub): - return next((k for k, vlist in par.__tokdict.items() for v, loc in vlist if sub is v), None) + return next( + ( + k + for k, vlist in par.__tokdict.items() + for v, loc in vlist + if sub is v + ), + None, + ) + return lookup(self) if par else None - elif (len(self) == 1 - and len(self.__tokdict) == 1 - and next(iter(self.__tokdict.values()))[0][1] in (0, -1)): + elif ( + len(self) == 1 + and len(self.__tokdict) == 1 + and next(iter(self.__tokdict.values()))[0][1] in (0, -1) + ): return next(iter(self.__tokdict.keys())) else: return None - def dump(self, indent='', full=True, include_list=True, _depth=0): + def dump(self, indent="", full=True, include_list=True, _depth=0): """ Diagnostic method for listing out the contents of a :class:`ParseResults`. Accepts an optional ``indent`` argument so @@ -934,8 +1108,8 @@ class ParseResults(object): - year: 12 """ out = [] - NL = '\n' - out.append(indent + str(self.asList()) if include_list else '') + NL = "\n" + out.append(indent + str(self.asList()) if include_list else "") if full: if self.haskeys(): @@ -943,10 +1117,17 @@ class ParseResults(object): for k, v in items: if out: out.append(NL) - out.append("%s%s- %s: " % (indent, (' ' * _depth), k)) + out.append("%s%s- %s: " % (indent, (" " * _depth), k)) if isinstance(v, ParseResults): if v: - out.append(v.dump(indent=indent, full=full, include_list=include_list, _depth=_depth + 1)) + out.append( + v.dump( + indent=indent, + full=full, + include_list=include_list, + _depth=_depth + 1, + ) + ) else: out.append(str(v)) else: @@ -955,22 +1136,34 @@ class ParseResults(object): v = self for i, vv in enumerate(v): if isinstance(vv, ParseResults): - out.append("\n%s%s[%d]:\n%s%s%s" % (indent, - (' ' * (_depth)), - i, - indent, - (' ' * (_depth + 1)), - vv.dump(indent=indent, - full=full, - include_list=include_list, - _depth=_depth + 1))) + out.append( + "\n%s%s[%d]:\n%s%s%s" + % ( + indent, + (" " * (_depth)), + i, + indent, + (" " * (_depth + 1)), + vv.dump( + indent=indent, + full=full, + include_list=include_list, + _depth=_depth + 1, + ), + ) + ) else: - out.append("\n%s%s[%d]:\n%s%s%s" % (indent, - (' ' * (_depth)), - i, - indent, - (' ' * (_depth + 1)), - str(vv))) + out.append( + "\n%s%s[%d]:\n%s%s%s" + % ( + indent, + (" " * (_depth)), + i, + indent, + (" " * (_depth + 1)), + str(vv), + ) + ) return "".join(out) @@ -1003,11 +1196,15 @@ class ParseResults(object): # add support for pickle protocol def __getstate__(self): - return (self.__toklist, - (self.__tokdict.copy(), - self.__parent is not None and self.__parent() or None, - self.__accumNames, - self.__name)) + return ( + self.__toklist, + ( + self.__tokdict.copy(), + self.__parent is not None and self.__parent() or None, + self.__accumNames, + self.__name, + ), + ) def __setstate__(self, state): self.__toklist = state[0] @@ -1032,6 +1229,7 @@ class ParseResults(object): name-value relations as results names. If an optional 'name' argument is given, a nested ParseResults will be returned """ + def is_iterable(obj): try: iter(obj) @@ -1050,9 +1248,11 @@ class ParseResults(object): ret = cls([ret], name=name) return ret + MutableMapping.register(ParseResults) -def col (loc, strg): + +def col(loc, strg): """Returns current column within a string, counting newlines as line separators. The first column is number 1. @@ -1064,7 +1264,8 @@ def col (loc, strg): location, and line and column positions within the parsed string. """ s = strg - return 1 if 0 < loc < len(s) and s[loc-1] == '\n' else loc - s.rfind("\n", 0, loc) + return 1 if 0 < loc < len(s) and s[loc - 1] == "\n" else loc - s.rfind("\n", 0, loc) + def lineno(loc, strg): """Returns current line number within a string, counting newlines as line separators. @@ -1078,28 +1279,42 @@ def lineno(loc, strg): """ return strg.count("\n", 0, loc) + 1 + def line(loc, strg): """Returns the line of text containing loc within a string, counting newlines as line separators. """ lastCR = strg.rfind("\n", 0, loc) nextCR = strg.find("\n", loc) - return strg[lastCR + 1:nextCR] if nextCR >= 0 else strg[lastCR + 1:] + return strg[lastCR + 1 : nextCR] if nextCR >= 0 else strg[lastCR + 1 :] + def _defaultStartDebugAction(instring, loc, expr): - print(("Match " + str(expr) + " at loc " + str(loc) + "(%d,%d)" % (lineno(loc, instring), col(loc, instring)))) + print( + ( + "Match " + + str(expr) + + " at loc " + + str(loc) + + "(%d,%d)" % (lineno(loc, instring), col(loc, instring)) + ) + ) + def _defaultSuccessDebugAction(instring, startloc, endloc, expr, toks): print("Matched " + str(expr) + " -> " + str(toks.asList())) + def _defaultExceptionDebugAction(instring, loc, expr, exc): print("Exception raised:" + str(exc)) + def nullDebugAction(*args): """'Do-nothing' debug action, to suppress debugging output during parsing.""" pass + def _trim_arity(func, maxargs=2): - 'decorator to trim function calls to match the arity of the target' + "decorator to trim function calls to match the arity of the target" if func in singleArgBuiltins: return lambda s, l, t: func(t) @@ -1113,6 +1328,7 @@ def _trim_arity(func, maxargs=2): offset = -3 if system_version == (3, 5, 0) else -2 frame_summary = traceback.extract_stack(limit=-offset + limit - 1)[offset] return [frame_summary[:2]] + def extract_tb(tb, limit=0): frames = traceback.extract_tb(tb, limit=limit) frame_summary = frames[-1] @@ -1157,8 +1373,7 @@ def _trim_arity(func, maxargs=2): # copy func name to wrapper for sensible debug output func_name = "<parse action>" try: - func_name = getattr(func, '__name__', - getattr(func, '__class__').__name__) + func_name = getattr(func, "__name__", getattr(func, "__class__").__name__) except Exception: func_name = str(func) wrapper.__name__ = func_name @@ -1168,6 +1383,7 @@ def _trim_arity(func, maxargs=2): class ParserElement(object): """Abstract base level parser element class.""" + DEFAULT_WHITE_CHARS = " \n\t\r" verbose_stacktrace = False @@ -1224,17 +1440,17 @@ class ParserElement(object): self.skipWhitespace = True self.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS) self.copyDefaultWhiteChars = True - self.mayReturnEmpty = False # used when checking for left-recursion + self.mayReturnEmpty = False # used when checking for left-recursion self.keepTabs = False self.ignoreExprs = list() self.debug = False self.streamlined = False - self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index + self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index self.errmsg = "" - self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all) + self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all) self.debugActions = (None, None, None) # custom debug actions self.re = None - self.callPreparse = True # used to avoid redundant calls to preParse + self.callPreparse = True # used to avoid redundant calls to preParse self.callDuringTry = False def copy(self): @@ -1320,11 +1536,14 @@ class ParserElement(object): """ if breakFlag: _parseMethod = self._parse + def breaker(instring, loc, doActions=True, callPreParse=True): import pdb + # this call to pdb.set_trace() is intentional, not a checkin error pdb.set_trace() return _parseMethod(instring, loc, doActions, callPreParse) + breaker._originalParseMethod = _parseMethod self._parse = breaker else: @@ -1372,7 +1591,9 @@ class ParserElement(object): # note that integer fields are now ints, not strings date_str.parseString("1999/12/31") # -> [1999, '/', 12, '/', 31] """ - if list(fns) == [None,]: + if list(fns) == [ + None, + ]: self.parseAction = [] else: if not all(callable(fn) for fn in fns): @@ -1414,8 +1635,11 @@ class ParserElement(object): (line:1, col:1) """ for fn in fns: - self.parseAction.append(conditionAsParseAction(fn, message=kwargs.get('message'), - fatal=kwargs.get('fatal', False))) + self.parseAction.append( + conditionAsParseAction( + fn, message=kwargs.get("message"), fatal=kwargs.get("fatal", False) + ) + ) self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) return self @@ -1467,7 +1691,7 @@ class ParserElement(object): # ~ @profile def _parseNoCache(self, instring, loc, doActions=True, callPreParse=True): TRY, MATCH, FAIL = 0, 1, 2 - debugging = (self.debug) # and doActions) + debugging = self.debug # and doActions) if debugging or self.failAction: # ~ print("Match", self, "at loc", loc, "(%d, %d)" % (lineno(loc, instring), col(loc, instring))) @@ -1509,7 +1733,9 @@ class ParserElement(object): tokens = self.postParse(instring, loc, tokens) - retTokens = ParseResults(tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults) + retTokens = ParseResults( + tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults + ) if self.parseAction and (doActions or self.callDuringTry): if debugging: try: @@ -1522,10 +1748,13 @@ class ParserElement(object): raise exc if tokens is not None and tokens is not retTokens: - retTokens = ParseResults(tokens, - self.resultsName, - asList=self.saveAsList and isinstance(tokens, (ParseResults, list)), - modal=self.modalResults) + retTokens = ParseResults( + tokens, + self.resultsName, + asList=self.saveAsList + and isinstance(tokens, (ParseResults, list)), + modal=self.modalResults, + ) except Exception as err: # ~ print "Exception raised in user parse action:", err if self.debugActions[FAIL]: @@ -1541,10 +1770,13 @@ class ParserElement(object): raise exc if tokens is not None and tokens is not retTokens: - retTokens = ParseResults(tokens, - self.resultsName, - asList=self.saveAsList and isinstance(tokens, (ParseResults, list)), - modal=self.modalResults) + retTokens = ParseResults( + tokens, + self.resultsName, + asList=self.saveAsList + and isinstance(tokens, (ParseResults, list)), + modal=self.modalResults, + ) if debugging: # ~ print("Matched", self, "->", retTokens.asList()) if self.debugActions[MATCH]: @@ -1618,7 +1850,9 @@ class ParserElement(object): self.__len__ = types.MethodType(cache_len, self) # argument cache for optimizing repeated calls when backtracking through recursive expressions - packrat_cache = {} # this is set later by enabledPackrat(); this is here so that resetCache() doesn't fail + packrat_cache = ( + {} + ) # this is set later by enabledPackrat(); this is here so that resetCache() doesn't fail packrat_cache_lock = RLock() packrat_cache_stats = [0, 0] @@ -1652,9 +1886,12 @@ class ParserElement(object): @staticmethod def resetCache(): ParserElement.packrat_cache.clear() - ParserElement.packrat_cache_stats[:] = [0] * len(ParserElement.packrat_cache_stats) + ParserElement.packrat_cache_stats[:] = [0] * len( + ParserElement.packrat_cache_stats + ) _packratEnabled = False + @staticmethod def enablePackrat(cache_size_limit=128): """Enables "packrat" parsing, which adds memoizing to the parsing logic. @@ -1732,7 +1969,7 @@ class ParserElement(object): ... pyparsing.ParseException: Expected end of text, found 'b' (at char 5), (line:1, col:6) """ - + ParserElement.resetCache() if not self.streamlined: self.streamline() @@ -1896,7 +2133,9 @@ class ParserElement(object): ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity'] """ try: - return ParseResults([t for t, s, e in self.scanString(instring, maxMatches)]) + return ParseResults( + [t for t, s, e in self.scanString(instring, maxMatches)] + ) except ParseBaseException as exc: if ParserElement.verbose_stacktrace: raise @@ -1962,8 +2201,11 @@ class ParserElement(object): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + warnings.warn( + "Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, + stacklevel=2, + ) return None return And([self, other]) @@ -1977,8 +2219,11 @@ class ParserElement(object): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + warnings.warn( + "Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, + stacklevel=2, + ) return None return other + self @@ -1989,8 +2234,11 @@ class ParserElement(object): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + warnings.warn( + "Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, + stacklevel=2, + ) return None return self + And._ErrorStop() + other @@ -2001,8 +2249,11 @@ class ParserElement(object): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + warnings.warn( + "Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, + stacklevel=2, + ) return None return other - self @@ -2029,7 +2280,7 @@ class ParserElement(object): if other is Ellipsis: other = (0, None) elif isinstance(other, tuple) and other[:1] == (Ellipsis,): - other = ((0, ) + other[1:] + (None,))[:2] + other = ((0,) + other[1:] + (None,))[:2] if isinstance(other, int): minElements, optElements = other, 0 @@ -2049,23 +2300,33 @@ class ParserElement(object): minElements, optElements = other optElements -= minElements else: - raise TypeError("cannot multiply 'ParserElement' and ('%s', '%s') objects", type(other[0]), type(other[1])) + raise TypeError( + "cannot multiply 'ParserElement' and ('%s', '%s') objects", + type(other[0]), + type(other[1]), + ) else: - raise TypeError("cannot multiply 'ParserElement' and '%s' objects", type(other)) + raise TypeError( + "cannot multiply 'ParserElement' and '%s' objects", type(other) + ) if minElements < 0: raise ValueError("cannot multiply ParserElement by negative value") if optElements < 0: - raise ValueError("second tuple value must be greater or equal to first tuple value") + raise ValueError( + "second tuple value must be greater or equal to first tuple value" + ) if minElements == optElements == 0: raise ValueError("cannot multiply ParserElement by 0 or (0, 0)") if optElements: + def makeOptionalList(n): if n > 1: return Optional(self + makeOptionalList(n - 1)) else: return Optional(self) + if minElements: if minElements == 1: ret = self + makeOptionalList(optElements) @@ -2093,8 +2354,11 @@ class ParserElement(object): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + warnings.warn( + "Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, + stacklevel=2, + ) return None return MatchFirst([self, other]) @@ -2105,8 +2369,11 @@ class ParserElement(object): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + warnings.warn( + "Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, + stacklevel=2, + ) return None return other | self @@ -2117,8 +2384,11 @@ class ParserElement(object): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + warnings.warn( + "Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, + stacklevel=2, + ) return None return Or([self, other]) @@ -2129,8 +2399,11 @@ class ParserElement(object): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + warnings.warn( + "Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, + stacklevel=2, + ) return None return other ^ self @@ -2141,8 +2414,11 @@ class ParserElement(object): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + warnings.warn( + "Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, + stacklevel=2, + ) return None return Each([self, other]) @@ -2153,8 +2429,11 @@ class ParserElement(object): if isinstance(other, str_type): other = self._literalStringClass(other) if not isinstance(other, ParserElement): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + warnings.warn( + "Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, + stacklevel=2, + ) return None return other & self @@ -2167,7 +2446,7 @@ class ParserElement(object): def __iter__(self): # must implement __iter__ to override legacy use of sequential access to __getitem__ to # iterate over a sequence - raise TypeError('%r object is not iterable' % self.__class__.__name__) + raise TypeError("%r object is not iterable" % self.__class__.__name__) def __getitem__(self, key): """ @@ -2197,9 +2476,11 @@ class ParserElement(object): key = (key, key) if len(key) > 2: - warnings.warn("only 1 or 2 index arguments supported ({}{})".format(key[:5], - '... [{}]'.format(len(key)) - if len(key) > 5 else '')) + warnings.warn( + "only 1 or 2 index arguments supported ({}{})".format( + key[:5], "... [{}]".format(len(key)) if len(key) > 5 else "" + ) + ) # clip to 2 elements ret = self * tuple(key[:2]) @@ -2287,9 +2568,11 @@ class ParserElement(object): """ Enable display of debugging messages while doing pattern matching. """ - self.debugActions = (startAction or _defaultStartDebugAction, - successAction or _defaultSuccessDebugAction, - exceptionAction or _defaultExceptionDebugAction) + self.debugActions = ( + startAction or _defaultStartDebugAction, + successAction or _defaultSuccessDebugAction, + exceptionAction or _defaultExceptionDebugAction, + ) self.debug = True return self @@ -2331,7 +2614,11 @@ class ParserElement(object): name created for the :class:`Word` expression without calling ``setName`` is ``"W:(ABCD...)"``. """ if flag: - self.setDebugActions(_defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction) + self.setDebugActions( + _defaultStartDebugAction, + _defaultSuccessDebugAction, + _defaultExceptionDebugAction, + ) else: self.debug = False return self @@ -2414,9 +2701,17 @@ class ParserElement(object): except ParseBaseException: return False - def runTests(self, tests, parseAll=True, comment='#', - fullDump=True, printResults=True, failureTests=False, postParse=None, - file=None): + def runTests( + self, + tests, + parseAll=True, + comment="#", + fullDump=True, + printResults=True, + failureTests=False, + postParse=None, + file=None, + ): """ Execute the parse expression on a series of test strings, showing each test, the parsed results or where the parse failed. Quick and easy way to @@ -2521,15 +2816,15 @@ class ParserElement(object): allResults = [] comments = [] success = True - NL = Literal(r'\n').addParseAction(replaceWith('\n')).ignore(quotedString) - BOM = '\ufeff' + NL = Literal(r"\n").addParseAction(replaceWith("\n")).ignore(quotedString) + BOM = "\ufeff" for t in tests: if comment is not None and comment.matches(t, False) or comments and not t: comments.append(t) continue if not t: continue - out = ['\n'.join(comments), t] + out = ["\n".join(comments), t] comments = [] try: # convert newline marks to actual newlines, and strip leading BOM if present @@ -2537,11 +2832,11 @@ class ParserElement(object): result = self.parseString(t, parseAll=parseAll) except ParseBaseException as pe: fatal = "(FATAL)" if isinstance(pe, ParseFatalException) else "" - if '\n' in t: + if "\n" in t: out.append(line(pe.loc, t)) - out.append(' ' * (col(pe.loc, t) - 1) + '^' + fatal) + out.append(" " * (col(pe.loc, t) - 1) + "^" + fatal) else: - out.append(' ' * pe.loc + '^' + fatal) + out.append(" " * pe.loc + "^" + fatal) out.append("FAIL: " + str(pe)) success = success and failureTests result = pe @@ -2563,13 +2858,17 @@ class ParserElement(object): out.append(result.dump()) except Exception as e: out.append(result.dump(full=fullDump)) - out.append("{} failed: {}: {}".format(postParse.__name__, type(e).__name__, e)) + out.append( + "{} failed: {}: {}".format( + postParse.__name__, type(e).__name__, e + ) + ) else: out.append(result.dump(full=fullDump)) - out.append('') + out.append("") if printResults: - print_('\n'.join(out)) + print_("\n".join(out)) allResults.append((t, result)) @@ -2581,7 +2880,7 @@ class _PendingSkip(ParserElement): # once another ParserElement is added, this placeholder will be replaced with a SkipTo def __init__(self, expr, must_skip=False): super().__init__() - self.strRepr = str(expr + Empty()).replace('Empty', '...') + self.strRepr = str(expr + Empty()).replace("Empty", "...") self.name = self.strRepr self.anchor = expr self.must_skip = must_skip @@ -2589,16 +2888,21 @@ class _PendingSkip(ParserElement): def __add__(self, other): skipper = SkipTo(other).setName("...")("_skipped*") if self.must_skip: + def must_skip(t): - if not t._skipped or t._skipped.asList() == ['']: + if not t._skipped or t._skipped.asList() == [""]: del t[0] t.pop("_skipped", None) + def show_skip(t): - if t._skipped.asList()[-1:] == ['']: - skipped = t.pop('_skipped') - t['_skipped'] = 'missing <' + repr(self.anchor) + '>' - return (self.anchor + skipper().addParseAction(must_skip) - | skipper().addParseAction(show_skip)) + other + if t._skipped.asList()[-1:] == [""]: + skipped = t.pop("_skipped") + t["_skipped"] = "missing <" + repr(self.anchor) + ">" + + return ( + self.anchor + skipper().addParseAction(must_skip) + | skipper().addParseAction(show_skip) + ) + other return self.anchor + skipper + other @@ -2606,13 +2910,16 @@ class _PendingSkip(ParserElement): return self.strRepr def parseImpl(self, *args): - raise Exception("use of `...` expression without following SkipTo target expression") + raise Exception( + "use of `...` expression without following SkipTo target expression" + ) class Token(ParserElement): """Abstract :class:`ParserElement` subclass, for defining atomic matching patterns. """ + def __init__(self): super().__init__(savelist=False) @@ -2620,6 +2927,7 @@ class Token(ParserElement): class Empty(Token): """An empty token, will always match. """ + def __init__(self): super().__init__() self.name = "Empty" @@ -2630,6 +2938,7 @@ class Empty(Token): class NoMatch(Token): """A token that will never match. """ + def __init__(self): super().__init__() self.name = "NoMatch" @@ -2655,6 +2964,7 @@ class Literal(Token): For keyword matching (force word break before and after the matched string), use :class:`Keyword` or :class:`CaselessKeyword`. """ + def __init__(self, matchString): super().__init__() self.match = matchString @@ -2662,8 +2972,11 @@ class Literal(Token): try: self.firstMatchChar = matchString[0] except IndexError: - warnings.warn("null string passed to Literal; use Empty() instead", - SyntaxWarning, stacklevel=2) + warnings.warn( + "null string passed to Literal; use Empty() instead", + SyntaxWarning, + stacklevel=2, + ) self.__class__ = Empty self.name = '"%s"' % str(self.match) self.errmsg = "Expected " + self.name @@ -2676,19 +2989,24 @@ class Literal(Token): self.__class__ = _SingleCharLiteral def parseImpl(self, instring, loc, doActions=True): - if instring[loc] == self.firstMatchChar and instring.startswith(self.match, loc): + if instring[loc] == self.firstMatchChar and instring.startswith( + self.match, loc + ): return loc + self.matchLen, self.match raise ParseException(instring, loc, self.errmsg, self) + class _SingleCharLiteral(Literal): def parseImpl(self, instring, loc, doActions=True): if instring[loc] == self.firstMatchChar: return loc + 1, self.match raise ParseException(instring, loc, self.errmsg, self) + _L = Literal ParserElement._literalStringClass = Literal + class Keyword(Token): """Token to exactly match a specified string as a keyword, that is, it must be immediately followed by a non-keyword character. Compare @@ -2714,6 +3032,7 @@ class Keyword(Token): For case-insensitive matching, use :class:`CaselessKeyword`. """ + DEFAULT_KEYWORD_CHARS = alphanums + "_$" def __init__(self, matchString, identChars=None, caseless=False): @@ -2725,8 +3044,11 @@ class Keyword(Token): try: self.firstMatchChar = matchString[0] except IndexError: - warnings.warn("null string passed to Keyword; use Empty() instead", - SyntaxWarning, stacklevel=2) + warnings.warn( + "null string passed to Keyword; use Empty() instead", + SyntaxWarning, + stacklevel=2, + ) self.name = '"%s"' % self.match self.errmsg = "Expected " + self.name self.mayReturnEmpty = False @@ -2739,19 +3061,26 @@ class Keyword(Token): def parseImpl(self, instring, loc, doActions=True): if self.caseless: - if ((instring[loc:loc + self.matchLen].upper() == self.caselessmatch) - and (loc >= len(instring) - self.matchLen - or instring[loc + self.matchLen].upper() not in self.identChars) - and (loc == 0 - or instring[loc - 1].upper() not in self.identChars)): + if ( + (instring[loc : loc + self.matchLen].upper() == self.caselessmatch) + and ( + loc >= len(instring) - self.matchLen + or instring[loc + self.matchLen].upper() not in self.identChars + ) + and (loc == 0 or instring[loc - 1].upper() not in self.identChars) + ): return loc + self.matchLen, self.match else: if instring[loc] == self.firstMatchChar: - if ((self.matchLen == 1 or instring.startswith(self.match, loc)) - and (loc >= len(instring) - self.matchLen - or instring[loc + self.matchLen] not in self.identChars) - and (loc == 0 or instring[loc - 1] not in self.identChars)): + if ( + (self.matchLen == 1 or instring.startswith(self.match, loc)) + and ( + loc >= len(instring) - self.matchLen + or instring[loc + self.matchLen] not in self.identChars + ) + and (loc == 0 or instring[loc - 1] not in self.identChars) + ): return loc + self.matchLen, self.match raise ParseException(instring, loc, self.errmsg, self) @@ -2767,6 +3096,7 @@ class Keyword(Token): """ Keyword.DEFAULT_KEYWORD_CHARS = chars + class CaselessLiteral(Literal): """Token to match a specified string, ignoring case of letters. Note: the matched results will always be in the case of the given @@ -2778,6 +3108,7 @@ class CaselessLiteral(Literal): (Contrast with example for :class:`CaselessKeyword`.) """ + def __init__(self, matchString): super().__init__(matchString.upper()) # Preserve the defining literal. @@ -2786,10 +3117,11 @@ class CaselessLiteral(Literal): self.errmsg = "Expected " + self.name def parseImpl(self, instring, loc, doActions=True): - if instring[loc:loc + self.matchLen].upper() == self.match: + if instring[loc : loc + self.matchLen].upper() == self.match: return loc + self.matchLen, self.returnString raise ParseException(instring, loc, self.errmsg, self) + class CaselessKeyword(Keyword): """ Caseless version of :class:`Keyword`. @@ -2800,9 +3132,11 @@ class CaselessKeyword(Keyword): (Contrast with example for :class:`CaselessLiteral`.) """ + def __init__(self, matchString, identChars=None): super().__init__(matchString, identChars, caseless=True) + class CloseMatch(Token): """A variation on :class:`Literal` which matches "close" matches, that is, strings with at most 'n' mismatching characters. @@ -2836,12 +3170,16 @@ class CloseMatch(Token): patt = CloseMatch("ATCATCGAATGGA", maxMismatches=2) patt.parseString("ATCAXCGAAXGGA") # -> (['ATCAXCGAAXGGA'], {'mismatches': [[4, 9]], 'original': ['ATCATCGAATGGA']}) """ + def __init__(self, match_string, maxMismatches=1): super().__init__() self.name = match_string self.match_string = match_string self.maxMismatches = maxMismatches - self.errmsg = "Expected %r (with up to %d mismatches)" % (self.match_string, self.maxMismatches) + self.errmsg = "Expected %r (with up to %d mismatches)" % ( + self.match_string, + self.maxMismatches, + ) self.mayIndexError = False self.mayReturnEmpty = False @@ -2856,7 +3194,9 @@ class CloseMatch(Token): mismatches = [] maxMismatches = self.maxMismatches - for match_stringloc, s_m in enumerate(zip(instring[loc:maxloc], match_string)): + for match_stringloc, s_m in enumerate( + zip(instring[loc:maxloc], match_string) + ): src, mat = s_m if src != mat: mismatches.append(match_stringloc) @@ -2865,8 +3205,8 @@ class CloseMatch(Token): else: loc = start + match_stringloc + 1 results = ParseResults([instring[start:loc]]) - results['original'] = match_string - results['mismatches'] = mismatches + results["original"] = match_string + results["mismatches"] = mismatches return loc, results raise ParseException(instring, loc, self.errmsg, self) @@ -2925,13 +3265,23 @@ class Word(Token): # any string of non-whitespace characters, except for ',' csv_value = Word(printables, excludeChars=",") """ - def __init__(self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None): + + def __init__( + self, + initChars, + bodyChars=None, + min=1, + max=0, + exact=0, + asKeyword=False, + excludeChars=None, + ): super().__init__() if excludeChars: excludeChars = set(excludeChars) - initChars = ''.join(c for c in initChars if c not in excludeChars) + initChars = "".join(c for c in initChars if c not in excludeChars) if bodyChars: - bodyChars = ''.join(c for c in bodyChars if c not in excludeChars) + bodyChars = "".join(c for c in bodyChars if c not in excludeChars) self.initCharsOrig = initChars self.initChars = set(initChars) if bodyChars: @@ -2944,7 +3294,9 @@ class Word(Token): self.maxSpecified = max > 0 if min < 1: - raise ValueError("cannot specify a minimum length < 1; use Optional(Word()) if zero-length word is permitted") + raise ValueError( + "cannot specify a minimum length < 1; use Optional(Word()) if zero-length word is permitted" + ) self.minLen = min @@ -2962,15 +3314,23 @@ class Word(Token): self.mayIndexError = False self.asKeyword = asKeyword - if ' ' not in self.initCharsOrig + self.bodyCharsOrig and (min == 1 and max == 0 and exact == 0): + if " " not in self.initCharsOrig + self.bodyCharsOrig and ( + min == 1 and max == 0 and exact == 0 + ): if self.bodyCharsOrig == self.initCharsOrig: - self.reString = "[%s]+" % _collapseAndEscapeRegexRangeChars(self.initCharsOrig) + self.reString = "[%s]+" % _collapseAndEscapeRegexRangeChars( + self.initCharsOrig + ) elif len(self.initCharsOrig) == 1: - self.reString = "%s[%s]*" % (re.escape(self.initCharsOrig), - _collapseAndEscapeRegexRangeChars(self.bodyCharsOrig),) + self.reString = "%s[%s]*" % ( + re.escape(self.initCharsOrig), + _collapseAndEscapeRegexRangeChars(self.bodyCharsOrig), + ) else: - self.reString = "[%s][%s]*" % (_collapseAndEscapeRegexRangeChars(self.initCharsOrig), - _collapseAndEscapeRegexRangeChars(self.bodyCharsOrig),) + self.reString = "[%s][%s]*" % ( + _collapseAndEscapeRegexRangeChars(self.initCharsOrig), + _collapseAndEscapeRegexRangeChars(self.bodyCharsOrig), + ) if self.asKeyword: self.reString = r"\b" + self.reString + r"\b" @@ -3001,8 +3361,12 @@ class Word(Token): elif self.maxSpecified and loc < instrlen and instring[loc] in bodychars: throwException = True elif self.asKeyword: - if (start > 0 and instring[start - 1] in bodychars - or loc < instrlen and instring[loc] in bodychars): + if ( + start > 0 + and instring[start - 1] in bodychars + or loc < instrlen + and instring[loc] in bodychars + ): throwException = True if throwException: @@ -3025,12 +3389,16 @@ class Word(Token): return s if self.initCharsOrig != self.bodyCharsOrig: - self.strRepr = "W:(%s, %s)" % (charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig)) + self.strRepr = "W:(%s, %s)" % ( + charsAsStr(self.initCharsOrig), + charsAsStr(self.bodyCharsOrig), + ) else: self.strRepr = "W:(%s)" % charsAsStr(self.initCharsOrig) return self.strRepr + class _WordRegex(Word): def parseImpl(self, instring, loc, doActions=True): result = self.re_match(instring, loc) @@ -3046,8 +3414,11 @@ class Char(_WordRegex): when defining a match of any single character in a string of characters. """ + def __init__(self, charset, asKeyword=False, excludeChars=None): - super().__init__(charset, exact=1, asKeyword=asKeyword, excludeChars=excludeChars) + super().__init__( + charset, exact=1, asKeyword=asKeyword, excludeChars=excludeChars + ) self.reString = "[%s]" % _collapseAndEscapeRegexRangeChars(self.initChars) if asKeyword: self.reString = r"\b%s\b" % self.reString @@ -3088,6 +3459,7 @@ class Regex(Token): parser = pp.Word(pp.nums) """ + def __init__(self, pattern, flags=0, asGroupList=False, asMatch=False): """The parameters ``pattern`` and ``flags`` are passed to the ``re.compile()`` function as-is. See the Python @@ -3098,8 +3470,11 @@ class Regex(Token): if isinstance(pattern, str_type): if not pattern: - warnings.warn("null string passed to Regex; use Empty() instead", - SyntaxWarning, stacklevel=2) + warnings.warn( + "null string passed to Regex; use Empty() instead", + SyntaxWarning, + stacklevel=2, + ) self.pattern = pattern self.flags = flags @@ -3108,17 +3483,22 @@ class Regex(Token): self.re = re.compile(self.pattern, self.flags) self.reString = self.pattern except sre_constants.error: - warnings.warn("invalid pattern (%s) passed to Regex" % pattern, - SyntaxWarning, stacklevel=2) + warnings.warn( + "invalid pattern (%s) passed to Regex" % pattern, + SyntaxWarning, + stacklevel=2, + ) raise - elif hasattr(pattern, 'pattern') and hasattr(pattern, 'match'): + elif hasattr(pattern, "pattern") and hasattr(pattern, "match"): self.re = pattern self.pattern = self.reString = pattern.pattern self.flags = flags else: - raise TypeError("Regex may only be constructed with a string or a compiled RE object") + raise TypeError( + "Regex may only be constructed with a string or a compiled RE object" + ) self.re_match = self.re.match @@ -3187,23 +3567,34 @@ class Regex(Token): # prints "<h1>main title</h1>" """ if self.asGroupList: - warnings.warn("cannot use sub() with Regex(asGroupList=True)", - SyntaxWarning, stacklevel=2) + warnings.warn( + "cannot use sub() with Regex(asGroupList=True)", + SyntaxWarning, + stacklevel=2, + ) raise SyntaxError() if self.asMatch and callable(repl): - warnings.warn("cannot use sub() with a callable with Regex(asMatch=True)", - SyntaxWarning, stacklevel=2) + warnings.warn( + "cannot use sub() with a callable with Regex(asMatch=True)", + SyntaxWarning, + stacklevel=2, + ) raise SyntaxError() if self.asMatch: + def pa(tokens): return tokens[0].expand(repl) + else: + def pa(tokens): return self.re.sub(repl, tokens[0]) + return self.addParseAction(pa) + class QuotedString(Token): r""" Token for matching strings that are delimited by quoting characters. @@ -3243,14 +3634,25 @@ class QuotedString(Token): [['This is the "quote"']] [['This is the quote with "embedded" quotes']] """ - def __init__(self, quoteChar, escChar=None, escQuote=None, multiline=False, - unquoteResults=True, endQuoteChar=None, convertWhitespaceEscapes=True): + + def __init__( + self, + quoteChar, + escChar=None, + escQuote=None, + multiline=False, + unquoteResults=True, + endQuoteChar=None, + convertWhitespaceEscapes=True, + ): super().__init__() # remove white space from quote chars - wont work anyway quoteChar = quoteChar.strip() if not quoteChar: - warnings.warn("quoteChar cannot be the empty string", SyntaxWarning, stacklevel=2) + warnings.warn( + "quoteChar cannot be the empty string", SyntaxWarning, stacklevel=2 + ) raise SyntaxError() if endQuoteChar is None: @@ -3258,7 +3660,11 @@ class QuotedString(Token): else: endQuoteChar = endQuoteChar.strip() if not endQuoteChar: - warnings.warn("endQuoteChar cannot be the empty string", SyntaxWarning, stacklevel=2) + warnings.warn( + "endQuoteChar cannot be the empty string", + SyntaxWarning, + stacklevel=2, + ) raise SyntaxError() self.quoteChar = quoteChar @@ -3273,34 +3679,49 @@ class QuotedString(Token): if multiline: self.flags = re.MULTILINE | re.DOTALL - self.pattern = r'%s(?:[^%s%s]' % (re.escape(self.quoteChar), - _escapeRegexRangeChars(self.endQuoteChar[0]), - (escChar is not None and _escapeRegexRangeChars(escChar) or '')) + self.pattern = r"%s(?:[^%s%s]" % ( + re.escape(self.quoteChar), + _escapeRegexRangeChars(self.endQuoteChar[0]), + (escChar is not None and _escapeRegexRangeChars(escChar) or ""), + ) else: self.flags = 0 - self.pattern = r'%s(?:[^%s\n\r%s]' % (re.escape(self.quoteChar), - _escapeRegexRangeChars(self.endQuoteChar[0]), - (escChar is not None and _escapeRegexRangeChars(escChar) or '')) + self.pattern = r"%s(?:[^%s\n\r%s]" % ( + re.escape(self.quoteChar), + _escapeRegexRangeChars(self.endQuoteChar[0]), + (escChar is not None and _escapeRegexRangeChars(escChar) or ""), + ) if len(self.endQuoteChar) > 1: self.pattern += ( - '|(?:' + ')|(?:'.join("%s[^%s]" % (re.escape(self.endQuoteChar[:i]), - _escapeRegexRangeChars(self.endQuoteChar[i])) - for i in range(len(self.endQuoteChar) - 1, 0, -1)) + ')') + "|(?:" + + ")|(?:".join( + "%s[^%s]" + % ( + re.escape(self.endQuoteChar[:i]), + _escapeRegexRangeChars(self.endQuoteChar[i]), + ) + for i in range(len(self.endQuoteChar) - 1, 0, -1) + ) + + ")" + ) if escQuote: - self.pattern += (r'|(?:%s)' % re.escape(escQuote)) + self.pattern += r"|(?:%s)" % re.escape(escQuote) if escChar: - self.pattern += (r'|(?:%s.)' % re.escape(escChar)) + self.pattern += r"|(?:%s.)" % re.escape(escChar) self.escCharReplacePattern = re.escape(self.escChar) + "(.)" - self.pattern += (r')*%s' % re.escape(self.endQuoteChar)) + self.pattern += r")*%s" % re.escape(self.endQuoteChar) try: self.re = re.compile(self.pattern, self.flags) self.reString = self.pattern self.re_match = self.re.match except sre_constants.error: - warnings.warn("invalid pattern (%s) passed to Regex" % self.pattern, - SyntaxWarning, stacklevel=2) + warnings.warn( + "invalid pattern (%s) passed to Regex" % self.pattern, + SyntaxWarning, + stacklevel=2, + ) raise self.name = str(self) @@ -3309,7 +3730,11 @@ class QuotedString(Token): self.mayReturnEmpty = True def parseImpl(self, instring, loc, doActions=True): - result = instring[loc] == self.firstQuoteChar and self.re_match(instring, loc) or None + result = ( + instring[loc] == self.firstQuoteChar + and self.re_match(instring, loc) + or None + ) if not result: raise ParseException(instring, loc, self.errmsg, self) @@ -3319,16 +3744,16 @@ class QuotedString(Token): if self.unquoteResults: # strip off quotes - ret = ret[self.quoteCharLen: -self.endQuoteCharLen] + ret = ret[self.quoteCharLen : -self.endQuoteCharLen] if isinstance(ret, str_type): # replace escaped whitespace - if '\\' in ret and self.convertWhitespaceEscapes: + if "\\" in ret and self.convertWhitespaceEscapes: ws_map = { - r'\t': '\t', - r'\n': '\n', - r'\f': '\f', - r'\r': '\r', + r"\t": "\t", + r"\n": "\n", + r"\f": "\f", + r"\r": "\r", } for wslit, wschar in ws_map.items(): ret = ret.replace(wslit, wschar) @@ -3350,7 +3775,10 @@ class QuotedString(Token): pass if self.strRepr is None: - self.strRepr = "quoted string, starting with %s ending with %s" % (self.quoteChar, self.endQuoteChar) + self.strRepr = "quoted string, starting with %s ending with %s" % ( + self.quoteChar, + self.endQuoteChar, + ) return self.strRepr @@ -3375,14 +3803,17 @@ class CharsNotIn(Token): ['dkls', 'lsdkjf', 's12 34', '@!#', '213'] """ + def __init__(self, notChars, min=1, max=0, exact=0): super().__init__() self.skipWhitespace = False self.notChars = notChars if min < 1: - raise ValueError("cannot specify a minimum length < 1; use " - "Optional(CharsNotIn()) if zero-length char group is permitted") + raise ValueError( + "cannot specify a minimum length < 1; use " + "Optional(CharsNotIn()) if zero-length char group is permitted" + ) self.minLen = min @@ -3397,7 +3828,7 @@ class CharsNotIn(Token): self.name = str(self) self.errmsg = "Expected " + self.name - self.mayReturnEmpty = (self.minLen == 0) + self.mayReturnEmpty = self.minLen == 0 self.mayIndexError = False def parseImpl(self, instring, loc, doActions=True): @@ -3430,6 +3861,7 @@ class CharsNotIn(Token): return self.strRepr + class White(Token): """Special matching class for matching whitespace. Normally, whitespace is ignored by pyparsing grammars. This class is included @@ -3439,38 +3871,42 @@ class White(Token): ``max``, and ``exact`` arguments, as defined for the :class:`Word` class. """ + whiteStrs = { - ' ' : '<SP>', - '\t': '<TAB>', - '\n': '<LF>', - '\r': '<CR>', - '\f': '<FF>', - 'u\00A0': '<NBSP>', - 'u\1680': '<OGHAM_SPACE_MARK>', - 'u\180E': '<MONGOLIAN_VOWEL_SEPARATOR>', - 'u\2000': '<EN_QUAD>', - 'u\2001': '<EM_QUAD>', - 'u\2002': '<EN_SPACE>', - 'u\2003': '<EM_SPACE>', - 'u\2004': '<THREE-PER-EM_SPACE>', - 'u\2005': '<FOUR-PER-EM_SPACE>', - 'u\2006': '<SIX-PER-EM_SPACE>', - 'u\2007': '<FIGURE_SPACE>', - 'u\2008': '<PUNCTUATION_SPACE>', - 'u\2009': '<THIN_SPACE>', - 'u\200A': '<HAIR_SPACE>', - 'u\200B': '<ZERO_WIDTH_SPACE>', - 'u\202F': '<NNBSP>', - 'u\205F': '<MMSP>', - 'u\3000': '<IDEOGRAPHIC_SPACE>', - } + " ": "<SP>", + "\t": "<TAB>", + "\n": "<LF>", + "\r": "<CR>", + "\f": "<FF>", + "u\00A0": "<NBSP>", + "u\1680": "<OGHAM_SPACE_MARK>", + "u\180E": "<MONGOLIAN_VOWEL_SEPARATOR>", + "u\2000": "<EN_QUAD>", + "u\2001": "<EM_QUAD>", + "u\2002": "<EN_SPACE>", + "u\2003": "<EM_SPACE>", + "u\2004": "<THREE-PER-EM_SPACE>", + "u\2005": "<FOUR-PER-EM_SPACE>", + "u\2006": "<SIX-PER-EM_SPACE>", + "u\2007": "<FIGURE_SPACE>", + "u\2008": "<PUNCTUATION_SPACE>", + "u\2009": "<THIN_SPACE>", + "u\200A": "<HAIR_SPACE>", + "u\200B": "<ZERO_WIDTH_SPACE>", + "u\202F": "<NNBSP>", + "u\205F": "<MMSP>", + "u\3000": "<IDEOGRAPHIC_SPACE>", + } + def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0): super().__init__() self.matchWhite = ws - self.setWhitespaceChars("".join(c for c in self.whiteChars if c not in self.matchWhite), - copy_defaults=True) + self.setWhitespaceChars( + "".join(c for c in self.whiteChars if c not in self.matchWhite), + copy_defaults=True, + ) # ~ self.leaveWhitespace() - self.name = ("".join(White.whiteStrs[c] for c in self.matchWhite)) + self.name = "".join(White.whiteStrs[c] for c in self.matchWhite) self.mayReturnEmpty = True self.errmsg = "Expected " + self.name @@ -3508,10 +3944,12 @@ class _PositionToken(Token): self.mayReturnEmpty = True self.mayIndexError = False + class GoToColumn(_PositionToken): """Token to advance to a specific column of input text; useful for tabular report scraping. """ + def __init__(self, colno): super().__init__() self.col = colno @@ -3521,7 +3959,11 @@ class GoToColumn(_PositionToken): instrlen = len(instring) if self.ignoreExprs: loc = self._skipIgnorables(instring, loc) - while loc < instrlen and instring[loc].isspace() and col(loc, instring) != self.col: + while ( + loc < instrlen + and instring[loc].isspace() + and col(loc, instring) != self.col + ): loc += 1 return loc @@ -3530,7 +3972,7 @@ class GoToColumn(_PositionToken): if thiscol > self.col: raise ParseException(instring, loc, "Text not in expected column", self) newloc = loc + self.col - thiscol - ret = instring[loc: newloc] + ret = instring[loc:newloc] return newloc, ret @@ -3556,6 +3998,7 @@ class LineStart(_PositionToken): ['AAA', ' and this line'] """ + def __init__(self): super().__init__() self.errmsg = "Expected start of line" @@ -3565,14 +4008,17 @@ class LineStart(_PositionToken): return loc, [] raise ParseException(instring, loc, self.errmsg, self) + class LineEnd(_PositionToken): """Matches if current position is at the end of a line within the parse string """ + def __init__(self): super().__init__() - self.setWhitespaceChars(ParserElement.DEFAULT_WHITE_CHARS.replace("\n", ""), - copy_defaults=False) + self.setWhitespaceChars( + ParserElement.DEFAULT_WHITE_CHARS.replace("\n", ""), copy_defaults=False + ) self.errmsg = "Expected end of line" def parseImpl(self, instring, loc, doActions=True): @@ -3586,10 +4032,12 @@ class LineEnd(_PositionToken): else: raise ParseException(instring, loc, self.errmsg, self) + class StringStart(_PositionToken): """Matches if current position is at the beginning of the parse string """ + def __init__(self): super().__init__() self.errmsg = "Expected start of text" @@ -3601,9 +4049,11 @@ class StringStart(_PositionToken): raise ParseException(instring, loc, self.errmsg, self) return loc, [] + class StringEnd(_PositionToken): """Matches if current position is at the end of the parse string """ + def __init__(self): super().__init__() self.errmsg = "Expected end of text" @@ -3618,6 +4068,7 @@ class StringEnd(_PositionToken): else: raise ParseException(instring, loc, self.errmsg, self) + class WordStart(_PositionToken): """Matches if the current position is at the beginning of a Word, and is not preceded by any character in a given set of @@ -3627,6 +4078,7 @@ class WordStart(_PositionToken): the beginning of the string being parsed, or at the beginning of a line. """ + def __init__(self, wordChars=printables): super().__init__() self.wordChars = set(wordChars) @@ -3634,11 +4086,14 @@ class WordStart(_PositionToken): def parseImpl(self, instring, loc, doActions=True): if loc != 0: - if (instring[loc - 1] in self.wordChars - or instring[loc] not in self.wordChars): + if ( + instring[loc - 1] in self.wordChars + or instring[loc] not in self.wordChars + ): raise ParseException(instring, loc, self.errmsg, self) return loc, [] + class WordEnd(_PositionToken): """Matches if the current position is at the end of a Word, and is not followed by any character in a given set of ``wordChars`` @@ -3647,6 +4102,7 @@ class WordEnd(_PositionToken): will also match at the end of the string being parsed, or at the end of a line. """ + def __init__(self, wordChars=printables): super().__init__() self.wordChars = set(wordChars) @@ -3656,8 +4112,10 @@ class WordEnd(_PositionToken): def parseImpl(self, instring, loc, doActions=True): instrlen = len(instring) if instrlen > 0 and loc < instrlen: - if (instring[loc] in self.wordChars or - instring[loc - 1] not in self.wordChars): + if ( + instring[loc] in self.wordChars + or instring[loc - 1] not in self.wordChars + ): raise ParseException(instring, loc, self.errmsg, self) return loc, [] @@ -3666,6 +4124,7 @@ class ParseExpression(ParserElement): """Abstract subclass of ParserElement, for combining and post-processing parsed tokens. """ + def __init__(self, exprs, savelist=False): super().__init__(savelist) if isinstance(exprs, _generatorType): @@ -3679,7 +4138,10 @@ class ParseExpression(ParserElement): exprs = list(exprs) # if sequence of strings provided, wrap with Literal if any(isinstance(expr, str_type) for expr in exprs): - exprs = (self._literalStringClass(e) if isinstance(e, str_type) else e for e in exprs) + exprs = ( + self._literalStringClass(e) if isinstance(e, str_type) else e + for e in exprs + ) self.exprs = list(exprs) else: try: @@ -3735,24 +4197,28 @@ class ParseExpression(ParserElement): # (likewise for Or's and MatchFirst's) if len(self.exprs) == 2: other = self.exprs[0] - if (isinstance(other, self.__class__) - and not other.parseAction - and other.resultsName is None - and not other.debug): + if ( + isinstance(other, self.__class__) + and not other.parseAction + and other.resultsName is None + and not other.debug + ): self.exprs = other.exprs[:] + [self.exprs[1]] self.strRepr = None self.mayReturnEmpty |= other.mayReturnEmpty - self.mayIndexError |= other.mayIndexError + self.mayIndexError |= other.mayIndexError other = self.exprs[-1] - if (isinstance(other, self.__class__) - and not other.parseAction - and other.resultsName is None - and not other.debug): + if ( + isinstance(other, self.__class__) + and not other.parseAction + and other.resultsName is None + and not other.debug + ): self.exprs = self.exprs[:-1] + other.exprs[:] self.strRepr = None self.mayReturnEmpty |= other.mayReturnEmpty - self.mayIndexError |= other.mayIndexError + self.mayIndexError |= other.mayIndexError self.errmsg = "Expected " + str(self) @@ -3773,12 +4239,16 @@ class ParseExpression(ParserElement): if __diag__.warn_ungrouped_named_tokens_in_collection: for e in self.exprs: if isinstance(e, ParserElement) and e.resultsName: - warnings.warn("{}: setting results name {!r} on {} expression " - "collides with {!r} on contained expression".format("warn_ungrouped_named_tokens_in_collection", - name, - type(self).__name__, - e.resultsName), - stacklevel=3) + warnings.warn( + "{}: setting results name {!r} on {} expression " + "collides with {!r} on contained expression".format( + "warn_ungrouped_named_tokens_in_collection", + name, + type(self).__name__, + e.resultsName, + ), + stacklevel=3, + ) return super()._setResultsName(name, listAllMatches) @@ -3804,7 +4274,7 @@ class And(ParseExpression): class _ErrorStop(Empty): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.name = '-' + self.name = "-" self.leaveWhitespace() def __init__(self, exprs, savelist=True): @@ -3816,27 +4286,37 @@ class And(ParseExpression): skipto_arg = (Empty() + exprs[i + 1]).exprs[-1] tmp.append(SkipTo(skipto_arg)("_skipped*")) else: - raise Exception("cannot construct And with sequence ending in ...") + raise Exception( + "cannot construct And with sequence ending in ..." + ) else: tmp.append(expr) exprs[:] = tmp super().__init__(exprs, savelist) self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) - self.setWhitespaceChars(self.exprs[0].whiteChars, - copy_defaults=self.exprs[0].copyDefaultWhiteChars) + self.setWhitespaceChars( + self.exprs[0].whiteChars, copy_defaults=self.exprs[0].copyDefaultWhiteChars + ) self.skipWhitespace = self.exprs[0].skipWhitespace self.callPreparse = True def streamline(self): # collapse any _PendingSkip's if self.exprs: - if any(isinstance(e, ParseExpression) and e.exprs and isinstance(e.exprs[-1], _PendingSkip) - for e in self.exprs[:-1]): + if any( + isinstance(e, ParseExpression) + and e.exprs + and isinstance(e.exprs[-1], _PendingSkip) + for e in self.exprs[:-1] + ): for i, e in enumerate(self.exprs[:-1]): if e is None: continue - if (isinstance(e, ParseExpression) - and e.exprs and isinstance(e.exprs[-1], _PendingSkip)): + if ( + isinstance(e, ParseExpression) + and e.exprs + and isinstance(e.exprs[-1], _PendingSkip) + ): e.exprs[-1] = e.exprs[-1] + self.exprs[i + 1] self.exprs[i + 1] = None self.exprs = [e for e in self.exprs if e is not None] @@ -3848,7 +4328,9 @@ class And(ParseExpression): def parseImpl(self, instring, loc, doActions=True): # pass False as last arg to _parse for first element, since we already # pre-parsed the string as part of our And pre-parsing - loc, resultlist = self.exprs[0]._parse(instring, loc, doActions, callPreParse=False) + loc, resultlist = self.exprs[0]._parse( + instring, loc, doActions, callPreParse=False + ) errorStop = False for e in self.exprs[1:]: if isinstance(e, And._ErrorStop): @@ -3863,7 +4345,9 @@ class And(ParseExpression): pe.__traceback__ = None raise ParseSyntaxException._from_exception(pe) except IndexError: - raise ParseSyntaxException(instring, len(instring), self.errmsg, self) + raise ParseSyntaxException( + instring, len(instring), self.errmsg, self + ) else: loc, exprtokens = e._parse(instring, loc, doActions) if exprtokens or exprtokens.haskeys(): @@ -3909,6 +4393,7 @@ class Or(ParseExpression): [['123'], ['3.1416'], ['789']] """ + def __init__(self, exprs, savelist=False): super().__init__(exprs, savelist) if self.exprs: @@ -3943,7 +4428,9 @@ class Or(ParseExpression): maxExcLoc = err.loc except IndexError: if len(instring) > maxExcLoc: - maxException = ParseException(instring, len(instring), e.errmsg, self) + maxException = ParseException( + instring, len(instring), e.errmsg, self + ) maxExcLoc = len(instring) else: # save match among all matches, to retry longest to shortest @@ -3995,8 +4482,9 @@ class Or(ParseExpression): maxException.msg = self.errmsg raise maxException else: - raise ParseException(instring, loc, "no defined alternatives to match", self) - + raise ParseException( + instring, loc, "no defined alternatives to match", self + ) def __ixor__(self, other): if isinstance(other, str_type): @@ -4020,11 +4508,16 @@ class Or(ParseExpression): def _setResultsName(self, name, listAllMatches=False): if __diag__.warn_multiple_tokens_in_named_alternation: if any(isinstance(e, And) for e in self.exprs): - warnings.warn("{}: setting results name {!r} on {} expression " - "will return a list of all parsed tokens in an And alternative, " - "in prior versions only the first token was returned".format( - "warn_multiple_tokens_in_named_alternation", name, type(self).__name__), - stacklevel=3) + warnings.warn( + "{}: setting results name {!r} on {} expression " + "will return a list of all parsed tokens in an And alternative, " + "in prior versions only the first token was returned".format( + "warn_multiple_tokens_in_named_alternation", + name, + type(self).__name__, + ), + stacklevel=3, + ) return super()._setResultsName(name, listAllMatches) @@ -4046,6 +4539,7 @@ class MatchFirst(ParseExpression): number = Combine(Word(nums) + '.' + Word(nums)) | Word(nums) print(number.searchString("123 3.1416 789")) # Better -> [['123'], ['3.1416'], ['789']] """ + def __init__(self, exprs, savelist=False): super().__init__(exprs, savelist) if self.exprs: @@ -4077,7 +4571,9 @@ class MatchFirst(ParseExpression): maxExcLoc = err.loc except IndexError: if len(instring) > maxExcLoc: - maxException = ParseException(instring, len(instring), e.errmsg, self) + maxException = ParseException( + instring, len(instring), e.errmsg, self + ) maxExcLoc = len(instring) # only got here if no expression matched, raise exception for match that made it the furthest @@ -4093,7 +4589,9 @@ class MatchFirst(ParseExpression): maxException.msg = self.errmsg raise maxException else: - raise ParseException(instring, loc, "no defined alternatives to match", self) + raise ParseException( + instring, loc, "no defined alternatives to match", self + ) def __ior__(self, other): if isinstance(other, str_type): @@ -4117,11 +4615,16 @@ class MatchFirst(ParseExpression): def _setResultsName(self, name, listAllMatches=False): if __diag__.warn_multiple_tokens_in_named_alternation: if any(isinstance(e, And) for e in self.exprs): - warnings.warn("{}: setting results name {!r} on {} expression " - "may only return a single token for an And alternative, " - "in future will return the full list of tokens".format( - "warn_multiple_tokens_in_named_alternation", name, type(self).__name__), - stacklevel=3) + warnings.warn( + "{}: setting results name {!r} on {} expression " + "may only return a single token for an And alternative, " + "in future will return the full list of tokens".format( + "warn_multiple_tokens_in_named_alternation", + name, + type(self).__name__, + ), + stacklevel=3, + ) return super()._setResultsName(name, listAllMatches) @@ -4183,6 +4686,7 @@ class Each(ParseExpression): - shape: TRIANGLE - size: 20 """ + def __init__(self, exprs, savelist=True): super().__init__(exprs, savelist) self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) @@ -4197,18 +4701,32 @@ class Each(ParseExpression): def parseImpl(self, instring, loc, doActions=True): if self.initExprGroups: - self.opt1map = dict((id(e.expr), e) for e in self.exprs if isinstance(e, Optional)) + self.opt1map = dict( + (id(e.expr), e) for e in self.exprs if isinstance(e, Optional) + ) opt1 = [e.expr for e in self.exprs if isinstance(e, Optional)] - opt2 = [e for e in self.exprs if e.mayReturnEmpty and not isinstance(e, Optional)] + opt2 = [ + e + for e in self.exprs + if e.mayReturnEmpty and not isinstance(e, Optional) + ] self.optionals = opt1 + opt2 - self.multioptionals = [e.expr for e in self.exprs if isinstance(e, ZeroOrMore)] - self.multirequired = [e.expr for e in self.exprs if isinstance(e, OneOrMore)] - self.required = [e for e in self.exprs if not isinstance(e, (Optional, ZeroOrMore, OneOrMore))] + self.multioptionals = [ + e.expr for e in self.exprs if isinstance(e, ZeroOrMore) + ] + self.multirequired = [ + e.expr for e in self.exprs if isinstance(e, OneOrMore) + ] + self.required = [ + e + for e in self.exprs + if not isinstance(e, (Optional, ZeroOrMore, OneOrMore)) + ] self.required += self.multirequired self.initExprGroups = False tmpLoc = loc tmpReqd = self.required[:] - tmpOpt = self.optionals[:] + tmpOpt = self.optionals[:] matchOrder = [] keepMatching = True @@ -4248,10 +4766,14 @@ class Each(ParseExpression): if tmpReqd: missing = ", ".join(str(e) for e in tmpReqd) - raise ParseException(instring, loc, "Missing one or more required elements (%s)" % missing) + raise ParseException( + instring, loc, "Missing one or more required elements (%s)" % missing + ) # add any unmatched Optionals, in case they have default values defined - matchOrder += [e for e in self.exprs if isinstance(e, Optional) and e.expr in tmpOpt] + matchOrder += [ + e for e in self.exprs if isinstance(e, Optional) and e.expr in tmpOpt + ] resultlist = [] for e in matchOrder: @@ -4280,6 +4802,7 @@ class ParseElementEnhance(ParserElement): """Abstract subclass of :class:`ParserElement`, for combining and post-processing parsed tokens. """ + def __init__(self, expr, savelist=False): super().__init__(savelist) if isinstance(expr, str_type): @@ -4294,7 +4817,9 @@ class ParseElementEnhance(ParserElement): if expr is not None: self.mayIndexError = expr.mayIndexError self.mayReturnEmpty = expr.mayReturnEmpty - self.setWhitespaceChars(expr.whiteChars, copy_defaults=expr.copyDefaultWhiteChars) + self.setWhitespaceChars( + expr.whiteChars, copy_defaults=expr.copyDefaultWhiteChars + ) self.skipWhitespace = expr.skipWhitespace self.saveAsList = expr.saveAsList self.callPreparse = expr.callPreparse @@ -4379,6 +4904,7 @@ class FollowedBy(ParseElementEnhance): [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']] """ + def __init__(self, expr): super().__init__(expr) self.mayReturnEmpty = True @@ -4420,6 +4946,7 @@ class PrecededBy(ParseElementEnhance): str_var = PrecededBy("$") + pyparsing_common.identifier """ + def __init__(self, expr, retreat=None): super().__init__(expr) self.expr = self.expr().leaveWhitespace() @@ -4452,12 +4979,14 @@ class PrecededBy(ParseElementEnhance): else: # retreat specified a maximum lookbehind window, iterate test_expr = self.expr + StringEnd() - instring_slice = instring[max(0, loc - self.retreat):loc] + instring_slice = instring[max(0, loc - self.retreat) : loc] last_expr = ParseException(instring, loc, self.errmsg) - for offset in range(1, min(loc, self.retreat + 1)+1): + for offset in range(1, min(loc, self.retreat + 1) + 1): try: # print('trying', offset, instring_slice, repr(instring_slice[loc - offset:])) - _, ret = test_expr._parse(instring_slice, len(instring_slice) - offset) + _, ret = test_expr._parse( + instring_slice, len(instring_slice) - offset + ) except ParseBaseException as pbe: last_expr = pbe else: @@ -4490,10 +5019,13 @@ class NotAny(ParseElementEnhance): # integers that are followed by "." are actually floats integer = Word(nums) + ~Char(".") """ + def __init__(self, expr): super().__init__(expr) # ~ self.leaveWhitespace() - self.skipWhitespace = False # do NOT use self.leaveWhitespace(), don't want to propagate to exprs + self.skipWhitespace = ( + False # do NOT use self.leaveWhitespace(), don't want to propagate to exprs + ) self.mayReturnEmpty = True self.errmsg = "Found unwanted token, " + str(self.expr) @@ -4511,6 +5043,7 @@ class NotAny(ParseElementEnhance): return self.strRepr + class _MultipleMatch(ParseElementEnhance): def __init__(self, expr, stopOn=None): super().__init__(expr) @@ -4539,7 +5072,7 @@ class _MultipleMatch(ParseElementEnhance): try_not_ender(instring, loc) loc, tokens = self_expr_parse(instring, loc, doActions, callPreParse=False) try: - hasIgnoreExprs = (not not self.ignoreExprs) + hasIgnoreExprs = not not self.ignoreExprs while 1: if check_ender: try_not_ender(instring, loc) @@ -4557,14 +5090,18 @@ class _MultipleMatch(ParseElementEnhance): def _setResultsName(self, name, listAllMatches=False): if __diag__.warn_ungrouped_named_tokens_in_collection: - for e in [self.expr] + getattr(self.expr, 'exprs', []): + for e in [self.expr] + getattr(self.expr, "exprs", []): if isinstance(e, ParserElement) and e.resultsName: - warnings.warn("{}: setting results name {!r} on {} expression " - "collides with {!r} on contained expression".format("warn_ungrouped_named_tokens_in_collection", - name, - type(self).__name__, - e.resultsName), - stacklevel=3) + warnings.warn( + "{}: setting results name {!r} on {} expression " + "collides with {!r} on contained expression".format( + "warn_ungrouped_named_tokens_in_collection", + name, + type(self).__name__, + e.resultsName, + ), + stacklevel=3, + ) return super()._setResultsName(name, listAllMatches) @@ -4604,6 +5141,7 @@ class OneOrMore(_MultipleMatch): return self.strRepr + class ZeroOrMore(_MultipleMatch): """Optional repetition of zero or more of the given expression. @@ -4615,6 +5153,7 @@ class ZeroOrMore(_MultipleMatch): Example: similar to :class:`OneOrMore` """ + def __init__(self, expr, stopOn=None): super().__init__(expr, stopOn=stopOn) self.mayReturnEmpty = True @@ -4638,9 +5177,11 @@ class ZeroOrMore(_MultipleMatch): class _NullToken(object): def __bool__(self): return False + def __str__(self): return "" + class Optional(ParseElementEnhance): """Optional matching of the given expression. @@ -4678,6 +5219,7 @@ class Optional(ParseElementEnhance): ^ FAIL: Expected end of text (at char 5), (line:1, col:6) """ + __optionalNotMatched = _NullToken() def __init__(self, expr, default=__optionalNotMatched): @@ -4709,6 +5251,7 @@ class Optional(ParseElementEnhance): return self.strRepr + class SkipTo(ParseElementEnhance): """Token for skipping over all undefined text until the matched expression is found. @@ -4767,6 +5310,7 @@ class SkipTo(ParseElementEnhance): - issue_num: 79 - sev: Minor """ + def __init__(self, other, include=False, ignore=None, failOn=None): super().__init__(other) self.ignoreExpr = ignore @@ -4785,8 +5329,12 @@ class SkipTo(ParseElementEnhance): instrlen = len(instring) expr = self.expr expr_parse = self.expr._parse - self_failOn_canParseNext = self.failOn.canParseNext if self.failOn is not None else None - self_ignoreExpr_tryParse = self.ignoreExpr.tryParse if self.ignoreExpr is not None else None + self_failOn_canParseNext = ( + self.failOn.canParseNext if self.failOn is not None else None + ) + self_ignoreExpr_tryParse = ( + self.ignoreExpr.tryParse if self.ignoreExpr is not None else None + ) tmploc = loc while tmploc <= instrlen: @@ -4827,6 +5375,7 @@ class SkipTo(ParseElementEnhance): return loc, skipresult + class Forward(ParseElementEnhance): """Forward declaration of an expression to be defined later - used for recursive grammars, such as algebraic infix notation. @@ -4854,6 +5403,7 @@ class Forward(ParseElementEnhance): See :class:`ParseResults.pprint` for an example of a recursive parser created using ``Forward``. """ + def __init__(self, other=None): super().__init__(other, savelist=False) @@ -4864,7 +5414,9 @@ class Forward(ParseElementEnhance): self.strRepr = None self.mayIndexError = self.expr.mayIndexError self.mayReturnEmpty = self.expr.mayReturnEmpty - self.setWhitespaceChars(self.expr.whiteChars, copy_defaults=self.expr.copyDefaultWhiteChars) + self.setWhitespaceChars( + self.expr.whiteChars, copy_defaults=self.expr.copyDefaultWhiteChars + ) self.skipWhitespace = self.expr.skipWhitespace self.saveAsList = self.expr.saveAsList self.ignoreExprs.extend(self.expr.ignoreExprs) @@ -4904,7 +5456,7 @@ class Forward(ParseElementEnhance): self.strRepr = ": ..." # Use the string representation of main expression. - retString = '...' + retString = "..." try: if self.expr is not None: retString = str(self.expr)[:1000] @@ -4925,22 +5477,27 @@ class Forward(ParseElementEnhance): def _setResultsName(self, name, listAllMatches=False): if __diag__.warn_name_set_on_empty_Forward: if self.expr is None: - warnings.warn("{}: setting results name {!r} on {} expression " - "that has no contained expression".format("warn_name_set_on_empty_Forward", - name, - type(self).__name__), - stacklevel=3) + warnings.warn( + "{}: setting results name {!r} on {} expression " + "that has no contained expression".format( + "warn_name_set_on_empty_Forward", name, type(self).__name__ + ), + stacklevel=3, + ) return super()._setResultsName(name, listAllMatches) + class TokenConverter(ParseElementEnhance): """ Abstract subclass of :class:`ParseExpression`, for converting parsed results. """ + def __init__(self, expr, savelist=False): super().__init__(expr) # , savelist) self.saveAsList = False + class Combine(TokenConverter): """Converter to concatenate all matching tokens to a single string. By default, the matching patterns must also be contiguous in the @@ -4959,6 +5516,7 @@ class Combine(TokenConverter): # no match when there are internal spaces print(real.parseString('3. 1416')) # -> Exception: Expected W:(0123...) """ + def __init__(self, expr, joinString="", adjacent=True): super().__init__(expr) # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself @@ -4979,13 +5537,16 @@ class Combine(TokenConverter): def postParse(self, instring, loc, tokenlist): retToks = tokenlist.copy() del retToks[:] - retToks += ParseResults(["".join(tokenlist._asStringList(self.joinString))], modal=self.modalResults) + retToks += ParseResults( + ["".join(tokenlist._asStringList(self.joinString))], modal=self.modalResults + ) if self.resultsName and retToks.haskeys(): return [retToks] else: return retToks + class Group(TokenConverter): """Converter to return the matched tokens as a list - useful for returning tokens of :class:`ZeroOrMore` and :class:`OneOrMore` expressions. @@ -5001,6 +5562,7 @@ class Group(TokenConverter): func = ident + Group(Optional(delimitedList(term))) print(func.parseString("fn a, b, 100")) # -> ['fn', ['a', 'b', '100']] """ + def __init__(self, expr): super().__init__(expr) self.saveAsList = True @@ -5008,6 +5570,7 @@ class Group(TokenConverter): def postParse(self, instring, loc, tokenlist): return [tokenlist] + class Dict(TokenConverter): """Converter to return a repetitive expression as a list, but also as a dictionary. Each element can also be referenced using the first @@ -5047,6 +5610,7 @@ class Dict(TokenConverter): See more examples at :class:`ParseResults` of accessing fields by results name. """ + def __init__(self, expr): super().__init__(expr) self.saveAsList = True @@ -5065,7 +5629,9 @@ class Dict(TokenConverter): else: dictvalue = tok.copy() # ParseResults(i) del dictvalue[0] - if len(dictvalue) != 1 or (isinstance(dictvalue, ParseResults) and dictvalue.haskeys()): + if len(dictvalue) != 1 or ( + isinstance(dictvalue, ParseResults) and dictvalue.haskeys() + ): tokenlist[ikey] = _ParseResultsWithOffset(dictvalue, i) else: tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0], i) @@ -5098,6 +5664,7 @@ class Suppress(TokenConverter): (See also :class:`delimitedList`.) """ + def postParse(self, instring, loc, tokenlist): return [] @@ -5108,18 +5675,22 @@ class Suppress(TokenConverter): class OnlyOnce(object): """Wrapper for parse actions, to ensure they are only called once. """ + def __init__(self, methodCall): self.callable = _trim_arity(methodCall) self.called = False + def __call__(self, s, l, t): if not self.called: results = self.callable(s, l, t) self.called = True return results raise ParseException(s, l, "") + def reset(self): self.called = False + def traceParseAction(f): """Decorator for debugging parse actions. @@ -5146,12 +5717,15 @@ def traceParseAction(f): ['dfjkls'] """ f = _trim_arity(f) + def z(*paArgs): thisFunc = f.__name__ s, l, t = paArgs[-3:] if len(paArgs) > 3: - thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc - sys.stderr.write(">>entering %s(line: '%s', %d, %r)\n" % (thisFunc, line(l, s), l, t)) + thisFunc = paArgs[0].__class__.__name__ + "." + thisFunc + sys.stderr.write( + ">>entering %s(line: '%s', %d, %r)\n" % (thisFunc, line(l, s), l, t) + ) try: ret = f(*paArgs) except Exception as exc: @@ -5159,12 +5733,14 @@ def traceParseAction(f): raise sys.stderr.write("<<leaving %s (ret: %r)\n" % (thisFunc, ret)) return ret + try: z.__name__ = f.__name__ except AttributeError: pass return z + # # global helpers # @@ -5189,6 +5765,7 @@ def delimitedList(expr, delim=",", combine=False): else: return (expr + ZeroOrMore(Suppress(delim) + expr)).setName(dlName) + def countedArray(expr, intExpr=None): """Helper to define a counted list of expressions. @@ -5213,17 +5790,20 @@ def countedArray(expr, intExpr=None): countedArray(Word(alphas), intExpr=binaryConstant).parseString('10 ab cd ef') # -> ['ab', 'cd'] """ arrayExpr = Forward() + def countFieldParseAction(s, l, t): n = t[0] arrayExpr << (n and Group(And([expr] * n)) or Group(empty)) return [] + if intExpr is None: intExpr = Word(nums).setParseAction(lambda t: int(t[0])) else: intExpr = intExpr.copy() intExpr.setName("arrayLen") intExpr.addParseAction(countFieldParseAction, callDuringTry=True) - return (intExpr + arrayExpr).setName('(len) ' + str(expr) + '...') + return (intExpr + arrayExpr).setName("(len) " + str(expr) + "...") + def _flatten(L): ret = [] @@ -5234,6 +5814,7 @@ def _flatten(L): ret.append(i) return ret + def matchPreviousLiteral(expr): """Helper to define an expression that is indirectly defined from the tokens matched in a previous expression, that is, it looks for @@ -5250,6 +5831,7 @@ def matchPreviousLiteral(expr): enabled. """ rep = Forward() + def copyTokenToRepeater(s, l, t): if t: if len(t) == 1: @@ -5260,10 +5842,12 @@ def matchPreviousLiteral(expr): rep << And(Literal(tt) for tt in tflat) else: rep << Empty() + expr.addParseAction(copyTokenToRepeater, callDuringTry=True) - rep.setName('(prev) ' + str(expr)) + rep.setName("(prev) " + str(expr)) return rep + def matchPreviousExpr(expr): """Helper to define an expression that is indirectly defined from the tokens matched in a previous expression, that is, it looks for @@ -5282,17 +5866,22 @@ def matchPreviousExpr(expr): rep = Forward() e2 = expr.copy() rep <<= e2 + def copyTokenToRepeater(s, l, t): matchTokens = _flatten(t.asList()) + def mustMatchTheseTokens(s, l, t): theseTokens = _flatten(t.asList()) if theseTokens != matchTokens: - raise ParseException('', 0, '') + raise ParseException("", 0, "") + rep.setParseAction(mustMatchTheseTokens, callDuringTry=True) + expr.addParseAction(copyTokenToRepeater, callDuringTry=True) - rep.setName('(prev) ' + str(expr)) + rep.setName("(prev) " + str(expr)) return rep + def _escapeRegexRangeChars(s): # ~ escape these chars: ^-] for c in r"\^-]": @@ -5301,6 +5890,7 @@ def _escapeRegexRangeChars(s): s = s.replace("\t", r"\t") return str(s) + def _collapseAndEscapeRegexRangeChars(s): def is_consecutive(c): c_int = ord(c) @@ -5314,7 +5904,7 @@ def _collapseAndEscapeRegexRangeChars(s): is_consecutive.value = -1 def escape_re_range_char(c): - return '\\' + c if c in r"\^-]" else c + return "\\" + c if c in r"\^-]" else c ret = [] for _, chars in itertools.groupby(sorted(s), key=is_consecutive): @@ -5324,9 +5914,10 @@ def _collapseAndEscapeRegexRangeChars(s): if first == last: ret.append(escape_re_range_char(first)) else: - ret.append("{}-{}".format(escape_re_range_char(first), - escape_re_range_char(last))) - return ''.join(ret) + ret.append( + "{}-{}".format(escape_re_range_char(first), escape_re_range_char(last)) + ) + return "".join(ret) def oneOf(strs, caseless=False, useRegex=True, asKeyword=False): @@ -5362,16 +5953,19 @@ def oneOf(strs, caseless=False, useRegex=True, asKeyword=False): [['B', '=', '12'], ['AA', '=', '23'], ['B', '<=', 'AA'], ['AA', '>', '12']] """ if isinstance(caseless, str_type): - warnings.warn("More than one string argument passed to oneOf, pass " - "choices as a list or space-delimited string", stacklevel=2) + warnings.warn( + "More than one string argument passed to oneOf, pass " + "choices as a list or space-delimited string", + stacklevel=2, + ) if caseless: - isequal = (lambda a, b: a.upper() == b.upper()) - masks = (lambda a, b: b.upper().startswith(a.upper())) + isequal = lambda a, b: a.upper() == b.upper() + masks = lambda a, b: b.upper().startswith(a.upper()) parseElementClass = CaselessKeyword if asKeyword else CaselessLiteral else: - isequal = (lambda a, b: a == b) - masks = (lambda a, b: b.startswith(a)) + isequal = lambda a, b: a == b + masks = lambda a, b: b.startswith(a) parseElementClass = Keyword if asKeyword else Literal symbols = [] @@ -5380,8 +5974,11 @@ def oneOf(strs, caseless=False, useRegex=True, asKeyword=False): elif isinstance(strs, Iterable): symbols = list(strs) else: - warnings.warn("Invalid argument to oneOf, expected string or iterable", - SyntaxWarning, stacklevel=2) + warnings.warn( + "Invalid argument to oneOf, expected string or iterable", + SyntaxWarning, + stacklevel=2, + ) if not symbols: return NoMatch() @@ -5391,7 +5988,7 @@ def oneOf(strs, caseless=False, useRegex=True, asKeyword=False): i = 0 while i < len(symbols) - 1: cur = symbols[i] - for j, other in enumerate(symbols[i + 1:]): + for j, other in enumerate(symbols[i + 1 :]): if isequal(other, cur): del symbols[i + j + 1] break @@ -5406,15 +6003,25 @@ def oneOf(strs, caseless=False, useRegex=True, asKeyword=False): # ~ print(strs, "->", "|".join([_escapeRegexChars(sym) for sym in symbols])) try: if len(symbols) == len("".join(symbols)): - return Regex("[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols)).setName(' | '.join(symbols)) + return Regex( + "[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols) + ).setName(" | ".join(symbols)) else: - return Regex("|".join(re.escape(sym) for sym in symbols)).setName(' | '.join(symbols)) + return Regex("|".join(re.escape(sym) for sym in symbols)).setName( + " | ".join(symbols) + ) except Exception: - warnings.warn("Exception creating Regex for oneOf, building MatchFirst", - SyntaxWarning, stacklevel=2) + warnings.warn( + "Exception creating Regex for oneOf, building MatchFirst", + SyntaxWarning, + stacklevel=2, + ) # last resort, just use MatchFirst - return MatchFirst(parseElementClass(sym) for sym in symbols).setName(' | '.join(symbols)) + return MatchFirst(parseElementClass(sym) for sym in symbols).setName( + " | ".join(symbols) + ) + def dictOf(key, value): """Helper to easily and clearly define a dictionary by specifying @@ -5455,6 +6062,7 @@ def dictOf(key, value): """ return Dict(OneOrMore(Group(key + value))) + def originalTextFor(expr, asString=True): """Helper to return the original, untokenized text for a given expression. Useful to restore the parsed fields of an HTML start @@ -5489,20 +6097,24 @@ def originalTextFor(expr, asString=True): endlocMarker.callPreparse = False matchExpr = locMarker("_original_start") + expr + endlocMarker("_original_end") if asString: - extractText = lambda s, l, t: s[t._original_start: t._original_end] + extractText = lambda s, l, t: s[t._original_start : t._original_end] else: + def extractText(s, l, t): - t[:] = [s[t.pop('_original_start'):t.pop('_original_end')]] + t[:] = [s[t.pop("_original_start") : t.pop("_original_end")]] + matchExpr.setParseAction(extractText) matchExpr.ignoreExprs = expr.ignoreExprs return matchExpr + def ungroup(expr): """Helper to undo pyparsing's default grouping of And expressions, even if all but one are non-empty. """ return TokenConverter(expr).addParseAction(lambda t: t[0]) + def locatedExpr(expr): """Helper to decorate a returned token with its starting and ending locations in the input string. @@ -5529,22 +6141,40 @@ def locatedExpr(expr): [[18, 'lkkjj', 23]] """ locator = Empty().setParseAction(lambda s, l, t: l) - return Group(locator("locn_start") + expr("value") + locator.copy().leaveWhitespace()("locn_end")) + return Group( + locator("locn_start") + + expr("value") + + locator.copy().leaveWhitespace()("locn_end") + ) # convenience constants for positional expressions -empty = Empty().setName("empty") -lineStart = LineStart().setName("lineStart") -lineEnd = LineEnd().setName("lineEnd") +empty = Empty().setName("empty") +lineStart = LineStart().setName("lineStart") +lineEnd = LineEnd().setName("lineEnd") stringStart = StringStart().setName("stringStart") -stringEnd = StringEnd().setName("stringEnd") - -_escapedPunc = Word(_bslash, r"\[]-*.$+^?()~ ", exact=2).setParseAction(lambda s, l, t: t[0][1]) -_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s, l, t: chr(int(t[0].lstrip(r'\0x'), 16))) -_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s, l, t: chr(int(t[0][1:], 8))) -_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r'\]', exact=1) +stringEnd = StringEnd().setName("stringEnd") + +_escapedPunc = Word(_bslash, r"\[]-*.$+^?()~ ", exact=2).setParseAction( + lambda s, l, t: t[0][1] +) +_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction( + lambda s, l, t: chr(int(t[0].lstrip(r"\0x"), 16)) +) +_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction( + lambda s, l, t: chr(int(t[0][1:], 8)) +) +_singleChar = ( + _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r"\]", exact=1) +) _charRange = Group(_singleChar + Suppress("-") + _singleChar) -_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group(OneOrMore(_charRange | _singleChar)).setResultsName("body") + "]" +_reBracketExpr = ( + Literal("[") + + Optional("^").setResultsName("negate") + + Group(OneOrMore(_charRange | _singleChar)).setResultsName("body") + + "]" +) + def srange(s): r"""Helper to easily define string ranges for use in Word @@ -5572,21 +6202,29 @@ def srange(s): - any combination of the above (``'aeiouy'``, ``'a-zA-Z0-9_$'``, etc.) """ - _expanded = lambda p: p if not isinstance(p, ParseResults) else ''.join(chr(c) for c in range(ord(p[0]), ord(p[1]) + 1)) + _expanded = ( + lambda p: p + if not isinstance(p, ParseResults) + else "".join(chr(c) for c in range(ord(p[0]), ord(p[1]) + 1)) + ) try: return "".join(_expanded(part) for part in _reBracketExpr.parseString(s).body) except Exception: return "" + def matchOnlyAtCol(n): """Helper method for defining parse actions that require matching at a specific column in the input text. """ + def verifyCol(strg, locn, toks): if col(locn, strg) != n: raise ParseException(strg, locn, "matched token not at column %d" % n) + return verifyCol + def replaceWith(replStr): """Helper method for common parse actions that simply return a literal value. Especially useful when used with @@ -5602,6 +6240,7 @@ def replaceWith(replStr): """ return lambda s, l, t: [replStr] + def removeQuotes(s, l, t): """Helper parse action for removing quotation marks from parsed quoted strings. @@ -5617,6 +6256,7 @@ def removeQuotes(s, l, t): """ return t[0][1:-1] + def tokenMap(func, *args): """Helper to define a parse action by mapping a function to all elements of a ParseResults list. If any additional args are passed, @@ -5653,21 +6293,20 @@ def tokenMap(func, *args): now is the winter of our discontent made glorious summer by this sun of york ['Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York'] """ + def pa(s, l, t): return [func(tokn, *args) for tokn in t] try: - func_name = getattr(func, '__name__', - getattr(func, '__class__').__name__) + func_name = getattr(func, "__name__", getattr(func, "__class__").__name__) except Exception: func_name = str(func) pa.__name__ = func_name return pa -def _makeTags(tagStr, xml, - suppress_LT=Suppress("<"), - suppress_GT=Suppress(">")): + +def _makeTags(tagStr, xml, suppress_LT=Suppress("<"), suppress_GT=Suppress(">")): """Internal helper to construct opening and closing tag expressions, given a tag name""" if isinstance(tagStr, str_type): resname = tagStr @@ -5678,30 +6317,53 @@ def _makeTags(tagStr, xml, tagAttrName = Word(alphas, alphanums + "_-:") if xml: tagAttrValue = dblQuotedString.copy().setParseAction(removeQuotes) - openTag = (suppress_LT - + tagStr("tag") - + Dict(ZeroOrMore(Group(tagAttrName + Suppress("=") + tagAttrValue))) - + Optional("/", default=[False])("empty").setParseAction(lambda s, l, t: t[0] == '/') - + suppress_GT) + openTag = ( + suppress_LT + + tagStr("tag") + + Dict(ZeroOrMore(Group(tagAttrName + Suppress("=") + tagAttrValue))) + + Optional("/", default=[False])("empty").setParseAction( + lambda s, l, t: t[0] == "/" + ) + + suppress_GT + ) else: - tagAttrValue = quotedString.copy().setParseAction(removeQuotes) | Word(printables, excludeChars=">") - openTag = (suppress_LT - + tagStr("tag") - + Dict(ZeroOrMore(Group(tagAttrName.setParseAction(lambda t: t[0].lower()) - + Optional(Suppress("=") + tagAttrValue)))) - + Optional("/", default=[False])("empty").setParseAction(lambda s, l, t: t[0] == '/') - + suppress_GT) + tagAttrValue = quotedString.copy().setParseAction(removeQuotes) | Word( + printables, excludeChars=">" + ) + openTag = ( + suppress_LT + + tagStr("tag") + + Dict( + ZeroOrMore( + Group( + tagAttrName.setParseAction(lambda t: t[0].lower()) + + Optional(Suppress("=") + tagAttrValue) + ) + ) + ) + + Optional("/", default=[False])("empty").setParseAction( + lambda s, l, t: t[0] == "/" + ) + + suppress_GT + ) closeTag = Combine(_L("</") + tagStr + ">", adjacent=False) openTag.setName("<%s>" % resname) # add start<tagname> results name in parse action now that ungrouped names are not reported at two levels - openTag.addParseAction(lambda t: t.__setitem__("start" + "".join(resname.replace(":", " ").title().split()), t.copy())) - closeTag = closeTag("end" + "".join(resname.replace(":", " ").title().split())).setName("</%s>" % resname) + openTag.addParseAction( + lambda t: t.__setitem__( + "start" + "".join(resname.replace(":", " ").title().split()), t.copy() + ) + ) + closeTag = closeTag( + "end" + "".join(resname.replace(":", " ").title().split()) + ).setName("</%s>" % resname) openTag.tag = resname closeTag.tag = resname openTag.tag_body = SkipTo(closeTag()) return openTag, closeTag + def makeHTMLTags(tagStr): """Helper to construct opening and closing tag expressions for HTML, given a tag name. Matches tags in either upper or lower case, @@ -5726,6 +6388,7 @@ def makeHTMLTags(tagStr): """ return _makeTags(tagStr, False) + def makeXMLTags(tagStr): """Helper to construct opening and closing tag expressions for XML, given a tag name. Matches tags only in the given upper/lower case. @@ -5734,6 +6397,7 @@ def makeXMLTags(tagStr): """ return _makeTags(tagStr, True) + def withAttribute(*args, **attrDict): """Helper to create a validating parse action to be used with start tags created with :class:`makeXMLTags` or @@ -5795,17 +6459,26 @@ def withAttribute(*args, **attrDict): else: attrs = attrDict.items() attrs = [(k, v) for k, v in attrs] + def pa(s, l, tokens): for attrName, attrValue in attrs: if attrName not in tokens: raise ParseException(s, l, "no matching attribute " + attrName) if attrValue != withAttribute.ANY_VALUE and tokens[attrName] != attrValue: - raise ParseException(s, l, "attribute '%s' has value '%s', must be '%s'" % - (attrName, tokens[attrName], attrValue)) + raise ParseException( + s, + l, + "attribute '%s' has value '%s', must be '%s'" + % (attrName, tokens[attrName], attrValue), + ) + return pa + + withAttribute.ANY_VALUE = object() -def withClass(classname, namespace=''): + +def withClass(classname, namespace=""): """Simplified version of :class:`withAttribute` when matching on a div class - made difficult because ``class`` is a reserved word in Python. @@ -5843,11 +6516,13 @@ def withClass(classname, namespace=''): classattr = "%s:class" % namespace if namespace else "class" return withAttribute(**{classattr: classname}) + opAssoc = types.SimpleNamespace() opAssoc.LEFT = object() opAssoc.RIGHT = object() -def infixNotation(baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')')): + +def infixNotation(baseExpr, opList, lpar=Suppress("("), rpar=Suppress(")")): """Helper method for constructing grammars of expressions made up of operators working in a precedence hierarchy. Operators may be unary or binary, left- or right-associative. Parse actions can also be @@ -5927,12 +6602,13 @@ def infixNotation(baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')')): ret = Forward() lastExpr = baseExpr | (lpar + ret + rpar) for i, operDef in enumerate(opList): - opExpr, arity, rightLeftAssoc, pa = (operDef + (None, ))[:4] + opExpr, arity, rightLeftAssoc, pa = (operDef + (None,))[:4] termName = "%s term" % opExpr if arity < 3 else "%s%s term" % opExpr if arity == 3: if opExpr is None or len(opExpr) != 2: raise ValueError( - "if numterms=3, opExpr must be a tuple or list of two expressions") + "if numterms=3, opExpr must be a tuple or list of two expressions" + ) opExpr1, opExpr2 = opExpr thisExpr = Forward().setName(termName) if rightLeftAssoc == opAssoc.LEFT: @@ -5940,14 +6616,21 @@ def infixNotation(baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')')): matchExpr = _FB(lastExpr + opExpr) + Group(lastExpr + OneOrMore(opExpr)) elif arity == 2: if opExpr is not None: - matchExpr = _FB(lastExpr + opExpr + lastExpr) + Group(lastExpr + OneOrMore(opExpr + lastExpr)) + matchExpr = _FB(lastExpr + opExpr + lastExpr) + Group( + lastExpr + OneOrMore(opExpr + lastExpr) + ) else: - matchExpr = _FB(lastExpr + lastExpr) + Group(lastExpr + OneOrMore(lastExpr)) + matchExpr = _FB(lastExpr + lastExpr) + Group( + lastExpr + OneOrMore(lastExpr) + ) elif arity == 3: - matchExpr = (_FB(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) - + Group(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr)) + matchExpr = _FB( + lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr + ) + Group(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) else: - raise ValueError("operator must be unary (1), binary (2), or ternary (3)") + raise ValueError( + "operator must be unary (1), binary (2), or ternary (3)" + ) elif rightLeftAssoc == opAssoc.RIGHT: if arity == 1: # try to avoid LR with this extra test @@ -5956,14 +6639,21 @@ def infixNotation(baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')')): matchExpr = _FB(opExpr.expr + thisExpr) + Group(opExpr + thisExpr) elif arity == 2: if opExpr is not None: - matchExpr = _FB(lastExpr + opExpr + thisExpr) + Group(lastExpr + OneOrMore(opExpr + thisExpr)) + matchExpr = _FB(lastExpr + opExpr + thisExpr) + Group( + lastExpr + OneOrMore(opExpr + thisExpr) + ) else: - matchExpr = _FB(lastExpr + thisExpr) + Group(lastExpr + OneOrMore(thisExpr)) + matchExpr = _FB(lastExpr + thisExpr) + Group( + lastExpr + OneOrMore(thisExpr) + ) elif arity == 3: - matchExpr = (_FB(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) - + Group(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr)) + matchExpr = _FB( + lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr + ) + Group(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) else: - raise ValueError("operator must be unary (1), binary (2), or ternary (3)") + raise ValueError( + "operator must be unary (1), binary (2), or ternary (3)" + ) else: raise ValueError("operator must indicate right or left associativity") if pa: @@ -5971,16 +6661,24 @@ def infixNotation(baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')')): matchExpr.setParseAction(*pa) else: matchExpr.setParseAction(pa) - thisExpr <<= (matchExpr.setName(termName) | lastExpr) + thisExpr <<= matchExpr.setName(termName) | lastExpr lastExpr = thisExpr ret <<= lastExpr return ret -dblQuotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"').setName("string enclosed in double quotes") -sglQuotedString = Combine(Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").setName("string enclosed in single quotes") -quotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"' - | Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").setName("quotedString using single or double quotes") -unicodeString = Combine(_L('u') + quotedString.copy()).setName("unicode string literal") + +dblQuotedString = Combine( + Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"' +).setName("string enclosed in double quotes") +sglQuotedString = Combine( + Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'" +).setName("string enclosed in single quotes") +quotedString = Combine( + Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"' + | Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'" +).setName("quotedString using single or double quotes") +unicodeString = Combine(_L("u") + quotedString.copy()).setName("unicode string literal") + def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()): """Helper method for defining nested lists enclosed in opening and @@ -6053,39 +6751,52 @@ def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.cop if isinstance(opener, str_type) and isinstance(closer, str_type): if len(opener) == 1 and len(closer) == 1: if ignoreExpr is not None: - content = (Combine(OneOrMore(~ignoreExpr - + CharsNotIn(opener - + closer - + ParserElement.DEFAULT_WHITE_CHARS, exact=1) - ) - ).setParseAction(lambda t: t[0].strip())) + content = Combine( + OneOrMore( + ~ignoreExpr + + CharsNotIn( + opener + closer + ParserElement.DEFAULT_WHITE_CHARS, + exact=1, + ) + ) + ).setParseAction(lambda t: t[0].strip()) else: - content = (empty.copy() + CharsNotIn(opener - + closer - + ParserElement.DEFAULT_WHITE_CHARS - ).setParseAction(lambda t: t[0].strip())) + content = empty.copy() + CharsNotIn( + opener + closer + ParserElement.DEFAULT_WHITE_CHARS + ).setParseAction(lambda t: t[0].strip()) else: if ignoreExpr is not None: - content = (Combine(OneOrMore(~ignoreExpr - + ~Literal(opener) - + ~Literal(closer) - + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS, exact=1)) - ).setParseAction(lambda t: t[0].strip())) + content = Combine( + OneOrMore( + ~ignoreExpr + + ~Literal(opener) + + ~Literal(closer) + + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS, exact=1) + ) + ).setParseAction(lambda t: t[0].strip()) else: - content = (Combine(OneOrMore(~Literal(opener) - + ~Literal(closer) - + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS, exact=1)) - ).setParseAction(lambda t: t[0].strip())) + content = Combine( + OneOrMore( + ~Literal(opener) + + ~Literal(closer) + + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS, exact=1) + ) + ).setParseAction(lambda t: t[0].strip()) else: - raise ValueError("opening and closing arguments must be strings if no content expression is given") + raise ValueError( + "opening and closing arguments must be strings if no content expression is given" + ) ret = Forward() if ignoreExpr is not None: - ret <<= Group(Suppress(opener) + ZeroOrMore(ignoreExpr | ret | content) + Suppress(closer)) + ret <<= Group( + Suppress(opener) + ZeroOrMore(ignoreExpr | ret | content) + Suppress(closer) + ) else: - ret <<= Group(Suppress(opener) + ZeroOrMore(ret | content) + Suppress(closer)) - ret.setName('nested %s%s expression' % (opener, closer)) + ret <<= Group(Suppress(opener) + ZeroOrMore(ret | content) + Suppress(closer)) + ret.setName("nested %s%s expression" % (opener, closer)) return ret + def indentedBlock(blockStatementExpr, indentStack, indent=True, backup_stacks=[]): """Helper method for defining space-delimited indentation blocks, such as those used to define block statements in Python source code. @@ -6173,7 +6884,8 @@ def indentedBlock(blockStatementExpr, indentStack, indent=True, backup_stacks=[] indentStack[:] = backup_stacks[-1] def checkPeerIndent(s, l, t): - if l >= len(s): return + if l >= len(s): + return curCol = col(l, s) if curCol != indentStack[-1]: if curCol > indentStack[-1]: @@ -6188,45 +6900,62 @@ def indentedBlock(blockStatementExpr, indentStack, indent=True, backup_stacks=[] raise ParseException(s, l, "not a subentry") def checkUnindent(s, l, t): - if l >= len(s): return + if l >= len(s): + return curCol = col(l, s) - if not(indentStack and curCol in indentStack): + if not (indentStack and curCol in indentStack): raise ParseException(s, l, "not an unindent") if curCol < indentStack[-1]: indentStack.pop() NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress()) - INDENT = (Empty() + Empty().setParseAction(checkSubIndent)).setName('INDENT') - PEER = Empty().setParseAction(checkPeerIndent).setName('') - UNDENT = Empty().setParseAction(checkUnindent).setName('UNINDENT') + INDENT = (Empty() + Empty().setParseAction(checkSubIndent)).setName("INDENT") + PEER = Empty().setParseAction(checkPeerIndent).setName("") + UNDENT = Empty().setParseAction(checkUnindent).setName("UNINDENT") if indent: - smExpr = Group(Optional(NL) - + INDENT - + OneOrMore(PEER + Group(blockStatementExpr) + Optional(NL)) - + UNDENT) + smExpr = Group( + Optional(NL) + + INDENT + + OneOrMore(PEER + Group(blockStatementExpr) + Optional(NL)) + + UNDENT + ) else: - smExpr = Group(Optional(NL) - + OneOrMore(PEER + Group(blockStatementExpr) + Optional(NL)) - + Optional(UNDENT)) + smExpr = Group( + Optional(NL) + + OneOrMore(PEER + Group(blockStatementExpr) + Optional(NL)) + + Optional(UNDENT) + ) # add a parse action to remove backup_stack from list of backups - smExpr.addParseAction(lambda: backup_stacks.pop(-1) and None if backup_stacks else None) + smExpr.addParseAction( + lambda: backup_stacks.pop(-1) and None if backup_stacks else None + ) smExpr.setFailAction(lambda a, b, c, d: reset_stack()) blockStatementExpr.ignore(_bslash + LineEnd()) - return smExpr.setName('indented block') + return smExpr.setName("indented block") + alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]") punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]") -anyOpenTag, anyCloseTag = makeHTMLTags(Word(alphas, alphanums + "_:").setName('any tag')) -_htmlEntityMap = dict(zip("gt lt amp nbsp quot apos".split(), '><& "\'')) -commonHTMLEntity = Regex('&(?P<entity>' + '|'.join(_htmlEntityMap.keys()) +");").setName("common HTML entity") +anyOpenTag, anyCloseTag = makeHTMLTags( + Word(alphas, alphanums + "_:").setName("any tag") +) +_htmlEntityMap = dict(zip("gt lt amp nbsp quot apos".split(), "><& \"'")) +commonHTMLEntity = Regex( + "&(?P<entity>" + "|".join(_htmlEntityMap.keys()) + ");" +).setName("common HTML entity") + + def replaceHTMLEntity(t): """Helper parser action to replace common HTML entities with their special characters""" return _htmlEntityMap.get(t.entity) + # it's easy to get these comment structures wrong - they're very common, so may as well make them available -cStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/').setName("C style comment") +cStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + "*/").setName( + "C style comment" +) "Comment of the form ``/* ... */``" htmlComment = Regex(r"<!--[\s\S]*?-->").setName("HTML comment") @@ -6236,7 +6965,9 @@ restOfLine = Regex(r".*").leaveWhitespace().setName("rest of line") dblSlashComment = Regex(r"//(?:\\\n|[^\n])*").setName("// comment") "Comment of the form ``// ... (to end of line)``" -cppStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/' | dblSlashComment).setName("C++ style comment") +cppStyleComment = Combine( + Regex(r"/\*(?:[^*]|\*(?!/))*") + "*/" | dblSlashComment +).setName("C++ style comment") "Comment of either form :class:`cStyleComment` or :class:`dblSlashComment`" javaStyleComment = cppStyleComment @@ -6407,21 +7138,37 @@ class pyparsing_common: hex_integer = Word(hexnums).setName("hex integer").setParseAction(tokenMap(int, 16)) """expression that parses a hexadecimal integer, returns an int""" - signed_integer = Regex(r'[+-]?\d+').setName("signed integer").setParseAction(convertToInteger) + signed_integer = ( + Regex(r"[+-]?\d+").setName("signed integer").setParseAction(convertToInteger) + ) """expression that parses an integer with optional leading sign, returns an int""" - fraction = (signed_integer().setParseAction(convertToFloat) + '/' + signed_integer().setParseAction(convertToFloat)).setName("fraction") + fraction = ( + signed_integer().setParseAction(convertToFloat) + + "/" + + signed_integer().setParseAction(convertToFloat) + ).setName("fraction") """fractional expression of an integer divided by an integer, returns a float""" - fraction.addParseAction(lambda t: t[0]/t[-1]) + fraction.addParseAction(lambda t: t[0] / t[-1]) - mixed_integer = (fraction | signed_integer + Optional(Optional('-').suppress() + fraction)).setName("fraction or mixed integer-fraction") + mixed_integer = ( + fraction | signed_integer + Optional(Optional("-").suppress() + fraction) + ).setName("fraction or mixed integer-fraction") """mixed integer of the form 'integer - fraction', with optional leading integer, returns float""" mixed_integer.addParseAction(sum) - real = Regex(r'[+-]?(:?\d+\.\d*|\.\d+)').setName("real number").setParseAction(convertToFloat) + real = ( + Regex(r"[+-]?(:?\d+\.\d*|\.\d+)") + .setName("real number") + .setParseAction(convertToFloat) + ) """expression that parses a floating point number and returns a float""" - sci_real = Regex(r'[+-]?(:?\d+(:?[eE][+-]?\d+)|(:?\d+\.\d*|\.\d+)(:?[eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat) + sci_real = ( + Regex(r"[+-]?(:?\d+(:?[eE][+-]?\d+)|(:?\d+\.\d*|\.\d+)(:?[eE][+-]?\d+)?)") + .setName("real number with scientific notation") + .setParseAction(convertToFloat) + ) """expression that parses a floating point number with optional scientific notation and returns a float""" @@ -6429,27 +7176,44 @@ class pyparsing_common: number = (sci_real | real | signed_integer).streamline() """any numeric expression, returns the corresponding Python type""" - fnumber = Regex(r'[+-]?\d+\.?\d*([eE][+-]?\d+)?').setName("fnumber").setParseAction(convertToFloat) + fnumber = ( + Regex(r"[+-]?\d+\.?\d*([eE][+-]?\d+)?") + .setName("fnumber") + .setParseAction(convertToFloat) + ) """any int or real number, returned as float""" - identifier = Word(alphas + '_', alphanums + '_').setName("identifier") + identifier = Word(alphas + "_", alphanums + "_").setName("identifier") """typical code identifier (leading alpha or '_', followed by 0 or more alphas, nums, or '_')""" - ipv4_address = Regex(r'(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}').setName("IPv4 address") + ipv4_address = Regex( + r"(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}" + ).setName("IPv4 address") "IPv4 address (``0.0.0.0 - 255.255.255.255``)" - _ipv6_part = Regex(r'[0-9a-fA-F]{1,4}').setName("hex_integer") - _full_ipv6_address = (_ipv6_part + (':' + _ipv6_part) * 7).setName("full IPv6 address") - _short_ipv6_address = (Optional(_ipv6_part + (':' + _ipv6_part) * (0, 6)) - + "::" - + Optional(_ipv6_part + (':' + _ipv6_part) * (0, 6)) - ).setName("short IPv6 address") - _short_ipv6_address.addCondition(lambda t: sum(1 for tt in t if pyparsing_common._ipv6_part.matches(tt)) < 8) + _ipv6_part = Regex(r"[0-9a-fA-F]{1,4}").setName("hex_integer") + _full_ipv6_address = (_ipv6_part + (":" + _ipv6_part) * 7).setName( + "full IPv6 address" + ) + _short_ipv6_address = ( + Optional(_ipv6_part + (":" + _ipv6_part) * (0, 6)) + + "::" + + Optional(_ipv6_part + (":" + _ipv6_part) * (0, 6)) + ).setName("short IPv6 address") + _short_ipv6_address.addCondition( + lambda t: sum(1 for tt in t if pyparsing_common._ipv6_part.matches(tt)) < 8 + ) _mixed_ipv6_address = ("::ffff:" + ipv4_address).setName("mixed IPv6 address") - ipv6_address = Combine((_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).setName("IPv6 address")).setName("IPv6 address") + ipv6_address = Combine( + (_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).setName( + "IPv6 address" + ) + ).setName("IPv6 address") "IPv6 address (long, short, or mixed form)" - mac_address = Regex(r'[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}').setName("MAC address") + mac_address = Regex( + r"[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}" + ).setName("MAC address") "MAC address xx:xx:xx:xx:xx (may also have '-' or '.' delimiters)" @staticmethod @@ -6470,11 +7234,13 @@ class pyparsing_common: [datetime.date(1999, 12, 31)] """ + def cvt_fn(s, l, t): try: return datetime.strptime(t[0], fmt).date() except ValueError as ve: raise ParseException(s, l, str(ve)) + return cvt_fn @staticmethod @@ -6495,23 +7261,30 @@ class pyparsing_common: [datetime.datetime(1999, 12, 31, 23, 59, 59, 999000)] """ + def cvt_fn(s, l, t): try: return datetime.strptime(t[0], fmt) except ValueError as ve: raise ParseException(s, l, str(ve)) + return cvt_fn - iso8601_date = Regex(r'(?P<year>\d{4})(?:-(?P<month>\d\d)(?:-(?P<day>\d\d))?)?').setName("ISO8601 date") + iso8601_date = Regex( + r"(?P<year>\d{4})(?:-(?P<month>\d\d)(?:-(?P<day>\d\d))?)?" + ).setName("ISO8601 date") "ISO8601 date (``yyyy-mm-dd``)" - iso8601_datetime = Regex(r'(?P<year>\d{4})-(?P<month>\d\d)-(?P<day>\d\d)[T ](?P<hour>\d\d):(?P<minute>\d\d)(:(?P<second>\d\d(\.\d*)?)?)?(?P<tz>Z|[+-]\d\d:?\d\d)?').setName("ISO8601 datetime") + iso8601_datetime = Regex( + r"(?P<year>\d{4})-(?P<month>\d\d)-(?P<day>\d\d)[T ](?P<hour>\d\d):(?P<minute>\d\d)(:(?P<second>\d\d(\.\d*)?)?)?(?P<tz>Z|[+-]\d\d:?\d\d)?" + ).setName("ISO8601 datetime") "ISO8601 datetime (``yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)``) - trailing seconds, milliseconds, and timezone optional; accepts separating ``'T'`` or ``' '``" - uuid = Regex(r'[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}').setName("UUID") + uuid = Regex(r"[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}").setName("UUID") "UUID (``xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx``)" _html_stripper = anyOpenTag.suppress() | anyCloseTag.suppress() + @staticmethod def stripHTMLTags(s, l, tokens): """Parse action to remove HTML tags from web page HTML source @@ -6530,13 +7303,21 @@ class pyparsing_common: """ return pyparsing_common._html_stripper.transformString(tokens[0]) - _commasepitem = Combine(OneOrMore(~Literal(",") - + ~LineEnd() - + Word(printables, excludeChars=',') - + Optional(White(" \t") + ~FollowedBy(LineEnd() | ','))) - ).streamline().setName("commaItem") - comma_separated_list = delimitedList(Optional(quotedString.copy() | _commasepitem, default='') - ).setName("comma separated list") + _commasepitem = ( + Combine( + OneOrMore( + ~Literal(",") + + ~LineEnd() + + Word(printables, excludeChars=",") + + Optional(White(" \t") + ~FollowedBy(LineEnd() | ",")) + ) + ) + .streamline() + .setName("commaItem") + ) + comma_separated_list = delimitedList( + Optional(quotedString.copy() | _commasepitem, default="") + ).setName("comma separated list") """Predefined expression of 1 or more printable words or quoted strin gs, separated by commas.""" upcaseTokens = staticmethod(tokenMap(lambda t: t.upper())) @@ -6555,8 +7336,10 @@ class _lazyclassproperty(object): def __get__(self, obj, cls): if cls is None: cls = type(obj) - if not hasattr(cls, '_intern') or any(cls._intern is getattr(superclass, '_intern', []) - for superclass in cls.__mro__[1:]): + if not hasattr(cls, "_intern") or any( + cls._intern is getattr(superclass, "_intern", []) + for superclass in cls.__mro__[1:] + ): cls._intern = {} attrname = self.fn.__name__ if attrname not in cls._intern: @@ -6578,6 +7361,7 @@ class unicode_set(object): class CJK(Chinese, Japanese, Korean): pass """ + _ranges = [] @classmethod @@ -6593,17 +7377,17 @@ class unicode_set(object): @_lazyclassproperty def printables(cls): "all non-whitespace characters in this range" - return ''.join(filterfalse(str.isspace, cls._get_chars_for_ranges())) + return "".join(filterfalse(str.isspace, cls._get_chars_for_ranges())) @_lazyclassproperty def alphas(cls): "all alphabetic characters in this range" - return ''.join(filter(str.isalpha, cls._get_chars_for_ranges())) + return "".join(filter(str.isalpha, cls._get_chars_for_ranges())) @_lazyclassproperty def nums(cls): "all numeric digit characters in this range" - return ''.join(filter(str.isdigit, cls._get_chars_for_ranges())) + return "".join(filter(str.isdigit, cls._get_chars_for_ranges())) @_lazyclassproperty def alphanums(cls): @@ -6615,35 +7399,60 @@ class pyparsing_unicode(unicode_set): """ A namespace class for defining common language unicode_sets. """ + _ranges = [(32, sys.maxunicode)] class Latin1(unicode_set): "Unicode set for Latin-1 Unicode Character Range" - _ranges = [(0x0020, 0x007e), (0x00a0, 0x00ff),] + _ranges = [ + (0x0020, 0x007E), + (0x00A0, 0x00FF), + ] class LatinA(unicode_set): "Unicode set for Latin-A Unicode Character Range" - _ranges = [(0x0100, 0x017f),] + _ranges = [ + (0x0100, 0x017F), + ] class LatinB(unicode_set): "Unicode set for Latin-B Unicode Character Range" - _ranges = [(0x0180, 0x024f),] + _ranges = [ + (0x0180, 0x024F), + ] class Greek(unicode_set): "Unicode set for Greek Unicode Character Ranges" _ranges = [ - (0x0370, 0x03ff), (0x1f00, 0x1f15), (0x1f18, 0x1f1d), (0x1f20, 0x1f45), (0x1f48, 0x1f4d), - (0x1f50, 0x1f57), (0x1f59,), (0x1f5b,), (0x1f5d,), (0x1f5f, 0x1f7d), (0x1f80, 0x1fb4), (0x1fb6, 0x1fc4), - (0x1fc6, 0x1fd3), (0x1fd6, 0x1fdb), (0x1fdd, 0x1fef), (0x1ff2, 0x1ff4), (0x1ff6, 0x1ffe), + (0x0370, 0x03FF), + (0x1F00, 0x1F15), + (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), + (0x1F48, 0x1F4D), + (0x1F50, 0x1F57), + (0x1F59,), + (0x1F5B,), + (0x1F5D,), + (0x1F5F, 0x1F7D), + (0x1F80, 0x1FB4), + (0x1FB6, 0x1FC4), + (0x1FC6, 0x1FD3), + (0x1FD6, 0x1FDB), + (0x1FDD, 0x1FEF), + (0x1FF2, 0x1FF4), + (0x1FF6, 0x1FFE), ] class Cyrillic(unicode_set): "Unicode set for Cyrillic Unicode Character Range" - _ranges = [(0x0400, 0x04ff)] + _ranges = [(0x0400, 0x04FF)] class Chinese(unicode_set): "Unicode set for Chinese Unicode Character Range" - _ranges = [(0x4e00, 0x9fff), (0x3000, 0x303f),] + _ranges = [ + (0x4E00, 0x9FFF), + (0x3000, 0x303F), + ] class Japanese(unicode_set): "Unicode set for Japanese Unicode Character Range, combining Kanji, Hiragana, and Katakana ranges" @@ -6651,19 +7460,33 @@ class pyparsing_unicode(unicode_set): class Kanji(unicode_set): "Unicode set for Kanji Unicode Character Range" - _ranges = [(0x4E00, 0x9Fbf), (0x3000, 0x303f),] + _ranges = [ + (0x4E00, 0x9FBF), + (0x3000, 0x303F), + ] class Hiragana(unicode_set): "Unicode set for Hiragana Unicode Character Range" - _ranges = [(0x3040, 0x309f),] + _ranges = [ + (0x3040, 0x309F), + ] class Katakana(unicode_set): "Unicode set for Katakana Unicode Character Range" - _ranges = [(0x30a0, 0x30ff),] + _ranges = [ + (0x30A0, 0x30FF), + ] class Korean(unicode_set): "Unicode set for Korean Unicode Character Range" - _ranges = [(0xac00, 0xd7af), (0x1100, 0x11ff), (0x3130, 0x318f), (0xa960, 0xa97f), (0xd7b0, 0xd7ff), (0x3000, 0x303f),] + _ranges = [ + (0xAC00, 0xD7AF), + (0x1100, 0x11FF), + (0x3130, 0x318F), + (0xA960, 0xA97F), + (0xD7B0, 0xD7FF), + (0x3000, 0x303F), + ] class CJK(Chinese, Japanese, Korean): "Unicode set for combined Chinese, Japanese, and Korean (CJK) Unicode Character Range" @@ -6671,23 +7494,35 @@ class pyparsing_unicode(unicode_set): class Thai(unicode_set): "Unicode set for Thai Unicode Character Range" - _ranges = [(0x0e01, 0x0e3a), (0x0e3f, 0x0e5b),] + _ranges = [ + (0x0E01, 0x0E3A), + (0x0E3F, 0x0E5B), + ] class Arabic(unicode_set): "Unicode set for Arabic Unicode Character Range" - _ranges = [(0x0600, 0x061b), (0x061e, 0x06ff), (0x0700, 0x077f),] + _ranges = [ + (0x0600, 0x061B), + (0x061E, 0x06FF), + (0x0700, 0x077F), + ] class Hebrew(unicode_set): "Unicode set for Hebrew Unicode Character Range" - _ranges = [(0x0590, 0x05ff),] + _ranges = [ + (0x0590, 0x05FF), + ] class Devanagari(unicode_set): "Unicode set for Devanagari Unicode Character Range" - _ranges = [(0x0900, 0x097f), (0xa8e0, 0xa8ff)] + _ranges = [(0x0900, 0x097F), (0xA8E0, 0xA8FF)] -pyparsing_unicode.Japanese._ranges = (pyparsing_unicode.Japanese.Kanji._ranges - + pyparsing_unicode.Japanese.Hiragana._ranges - + pyparsing_unicode.Japanese.Katakana._ranges) + +pyparsing_unicode.Japanese._ranges = ( + pyparsing_unicode.Japanese.Kanji._ranges + + pyparsing_unicode.Japanese.Hiragana._ranges + + pyparsing_unicode.Japanese.Katakana._ranges +) # define ranges in language character sets pyparsing_unicode.العربية = pyparsing_unicode.Arabic @@ -6708,6 +7543,7 @@ class pyparsing_test: """ namespace class for classes useful in writing unit tests """ + class reset_pyparsing_context: """ Context manager to be used when writing unit tests that modify pyparsing config values: @@ -6730,6 +7566,7 @@ class pyparsing_test: # after exiting context manager, literals are converted to Literal expressions again """ + def __init__(self): self._save_context = {} @@ -6740,30 +7577,41 @@ class pyparsing_test: self.restore() def save(self): - self._save_context['default_whitespace'] = ParserElement.DEFAULT_WHITE_CHARS - self._save_context['default_keyword_chars'] = Keyword.DEFAULT_KEYWORD_CHARS - self._save_context['literal_string_class'] = ParserElement._literalStringClass - self._save_context['packrat_enabled'] = ParserElement._packratEnabled - self._save_context['packrat_parse'] = ParserElement._parse - self._save_context['__diag__'] = {name: getattr(__diag__, name) for name in __diag__._all_names} - self._save_context['__compat__'] = {"collect_all_And_tokens": __compat__.collect_all_And_tokens} + self._save_context["default_whitespace"] = ParserElement.DEFAULT_WHITE_CHARS + self._save_context["default_keyword_chars"] = Keyword.DEFAULT_KEYWORD_CHARS + self._save_context[ + "literal_string_class" + ] = ParserElement._literalStringClass + self._save_context["packrat_enabled"] = ParserElement._packratEnabled + self._save_context["packrat_parse"] = ParserElement._parse + self._save_context["__diag__"] = { + name: getattr(__diag__, name) for name in __diag__._all_names + } + self._save_context["__compat__"] = { + "collect_all_And_tokens": __compat__.collect_all_And_tokens + } return self def restore(self): # restore pyparsing global state to what was saved - ParserElement.setDefaultWhitespaceChars(self._save_context['default_whitespace']) - Keyword.DEFAULT_KEYWORD_CHARS = self._save_context['default_keyword_chars'] - ParserElement.inlineLiteralsUsing(self._save_context['literal_string_class']) - ParserElement._packratEnabled = self._save_context['packrat_enabled'] - ParserElement._parse = self._save_context['packrat_parse'] - for name, value in self._save_context['__diag__'].items(): + ParserElement.setDefaultWhitespaceChars( + self._save_context["default_whitespace"] + ) + Keyword.DEFAULT_KEYWORD_CHARS = self._save_context["default_keyword_chars"] + ParserElement.inlineLiteralsUsing( + self._save_context["literal_string_class"] + ) + ParserElement._packratEnabled = self._save_context["packrat_enabled"] + ParserElement._parse = self._save_context["packrat_parse"] + for name, value in self._save_context["__diag__"].items(): (__diag__.enable if value else __diag__.disable)(name) - for name, value in self._save_context['__compat__'].items(): + for name, value in self._save_context["__compat__"].items(): setattr(__compat__, name, value) class TestParseResultsAsserts(unittest.TestCase): - - def assertParseResultsEquals(self, result, expected_list=None, expected_dict=None, msg=None): + def assertParseResultsEquals( + self, result, expected_list=None, expected_dict=None, msg=None + ): """ Unit test assertion to compare a ParseResults object with an optional expected_list, and compare any defined results names with an optional expected_dict. @@ -6773,7 +7621,9 @@ class pyparsing_test: if expected_dict is not None: self.assertEqual(expected_dict, result.asDict(), msg=msg) - def assertParseAndCheckList(self, expr, test_string, expected_list, msg=None, verbose=True): + def assertParseAndCheckList( + self, expr, test_string, expected_list, msg=None, verbose=True + ): """ Convenience wrapper assert to test a parser element and input string, and assert that the resulting ParseResults.asList() is equal to the expected_list. @@ -6783,7 +7633,9 @@ class pyparsing_test: print(result.dump()) self.assertParseResultsEquals(result, expected_list=expected_list, msg=msg) - def assertParseAndCheckDict(self, expr, test_string, expected_dict, msg=None, verbose=True): + def assertParseAndCheckDict( + self, expr, test_string, expected_dict, msg=None, verbose=True + ): """ Convenience wrapper assert to test a parser element and input string, and assert that the resulting ParseResults.asDict() is equal to the expected_dict. @@ -6793,7 +7645,9 @@ class pyparsing_test: print(result.dump()) self.assertParseResultsEquals(result, expected_dict=expected_dict, msg=msg) - def assertRunTestResults(self, run_tests_report, expected_parse_results=None, msg=None): + def assertRunTestResults( + self, run_tests_report, expected_parse_results=None, msg=None + ): """ Unit test assertion to evaluate output of ParserElement.runTests(). If a list of list-dict tuples is given as the expected_parse_results argument, then these are zipped @@ -6806,31 +7660,53 @@ class pyparsing_test: run_test_success, run_test_results = run_tests_report if expected_parse_results is not None: - merged = [(*rpt, expected) for rpt, expected in zip(run_test_results, expected_parse_results)] + merged = [ + (*rpt, expected) + for rpt, expected in zip(run_test_results, expected_parse_results) + ] for test_string, result, expected in merged: # expected should be a tuple containing a list and/or a dict or an exception, # and optional failure message string # an empty tuple will skip any result validation - fail_msg = next((exp for exp in expected if isinstance(exp, str)), None) - expected_exception = next((exp for exp in expected - if isinstance(exp, type) and issubclass(exp, Exception)), None) + fail_msg = next( + (exp for exp in expected if isinstance(exp, str)), None + ) + expected_exception = next( + ( + exp + for exp in expected + if isinstance(exp, type) and issubclass(exp, Exception) + ), + None, + ) if expected_exception is not None: - with self.assertRaises(expected_exception=expected_exception, msg=fail_msg or msg): + with self.assertRaises( + expected_exception=expected_exception, msg=fail_msg or msg + ): if isinstance(result, Exception): raise result else: - expected_list = next((exp for exp in expected if isinstance(exp, list)), None) - expected_dict = next((exp for exp in expected if isinstance(exp, dict)), None) + expected_list = next( + (exp for exp in expected if isinstance(exp, list)), None + ) + expected_dict = next( + (exp for exp in expected if isinstance(exp, dict)), None + ) if (expected_list, expected_dict) != (None, None): - self.assertParseResultsEquals(result, - expected_list=expected_list, expected_dict=expected_dict, - msg=fail_msg or msg) + self.assertParseResultsEquals( + result, + expected_list=expected_list, + expected_dict=expected_dict, + msg=fail_msg or msg, + ) else: # warning here maybe? print("no validation for {!r}".format(test_string)) # do this last, in case some specific test results can be reported instead - self.assertTrue(run_test_success, msg=msg if msg is not None else "failed runTests") + self.assertTrue( + run_test_success, msg=msg if msg is not None else "failed runTests" + ) @contextmanager def assertRaisesParseException(self, exc_type=ParseException, msg=None): @@ -6840,28 +7716,41 @@ class pyparsing_test: # build list of built-in expressions, for future reference if a global default value # gets updated -_builtin_exprs = [v for v in itertools.chain(vars().values(), vars(pyparsing_common).values()) - if isinstance(v, ParserElement)] +_builtin_exprs = [ + v + for v in itertools.chain(vars().values(), vars(pyparsing_common).values()) + if isinstance(v, ParserElement) +] if __name__ == "__main__": - selectToken = CaselessLiteral("select") + selectToken = CaselessLiteral("select") fromToken = CaselessLiteral("from") ident = Word(alphas, alphanums + "_$") - columnName = delimitedList(ident, ".", combine=True).setParseAction(pyparsing_common.upcaseTokens) + columnName = delimitedList(ident, ".", combine=True).setParseAction( + pyparsing_common.upcaseTokens + ) columnNameList = Group(delimitedList(columnName)).setName("columns") - columnSpec = ('*' | columnNameList) + columnSpec = "*" | columnNameList - tableName = delimitedList(ident, ".", combine=True).setParseAction(pyparsing_common.upcaseTokens) + tableName = delimitedList(ident, ".", combine=True).setParseAction( + pyparsing_common.upcaseTokens + ) tableNameList = Group(delimitedList(tableName)).setName("tables") - simpleSQL = selectToken("command") + columnSpec("columns") + fromToken + tableNameList("tables") + simpleSQL = ( + selectToken("command") + + columnSpec("columns") + + fromToken + + tableNameList("tables") + ) # demo runTests method, including embedded comments in test string - simpleSQL.runTests(""" + simpleSQL.runTests( + """ # '*' as column list and dotted table name select * from SYS.XYZZY @@ -6883,34 +7772,44 @@ if __name__ == "__main__": # invalid column name - should fail Select ^^^ frox Sys.dual - """) + """ + ) - pyparsing_common.number.runTests(""" + pyparsing_common.number.runTests( + """ 100 -100 +100 3.14159 6.02e23 1e-12 - """) + """ + ) # any int or real number, returned as float - pyparsing_common.fnumber.runTests(""" + pyparsing_common.fnumber.runTests( + """ 100 -100 +100 3.14159 6.02e23 1e-12 - """) + """ + ) - pyparsing_common.hex_integer.runTests(""" + pyparsing_common.hex_integer.runTests( + """ 100 FF - """) + """ + ) import uuid + pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID)) - pyparsing_common.uuid.runTests(""" + pyparsing_common.uuid.runTests( + """ 12345678-1234-5678-1234-567812345678 - """) + """ + ) @@ -8,34 +8,36 @@ except ImportError: from distutils.core import setup from pyparsing import __version__ as pyparsing_version, __doc__ as pyparsing_description -modules = ["pyparsing",] +modules = [ + "pyparsing", +] -setup(# Distribution meta-data - name = "pyparsing", - version = pyparsing_version, - description = "Python parsing module", - long_description = pyparsing_description, - author = "Paul McGuire", - author_email = "ptmcg@users.sourceforge.net", - url = "https://github.com/pyparsing/pyparsing/", - download_url = "https://pypi.org/project/pyparsing/", - license = "MIT License", - py_modules = modules, - python_requires='>=3.5', +setup( # Distribution meta-data + name="pyparsing", + version=pyparsing_version, + description="Python parsing module", + long_description=pyparsing_description, + author="Paul McGuire", + author_email="ptmcg@users.sourceforge.net", + url="https://github.com/pyparsing/pyparsing/", + download_url="https://pypi.org/project/pyparsing/", + license="MIT License", + py_modules=modules, + python_requires=">=3.5", classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'License :: OSI Approved :: MIT License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - ] - ) + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + ], +) diff --git a/tests/test_examples.py b/tests/test_examples.py index c58640f..077c617 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -6,9 +6,8 @@ import unittest class TestExamples(unittest.TestCase): - def _run(self, name): - import_module('examples.'+name) + import_module("examples." + name) def test_numerics(self): self._run("numerics") @@ -30,4 +29,3 @@ class TestExamples(unittest.TestCase): def test_eval_arith(self): self._run("eval_arith") - diff --git a/tests/test_simple_unit.py b/tests/test_simple_unit.py index fb5f56d..6caf7f6 100644 --- a/tests/test_simple_unit.py +++ b/tests/test_simple_unit.py @@ -16,9 +16,11 @@ ppt = pp.pyparsing_test TestParseResultsAsserts = ppt.TestParseResultsAsserts # Test spec data class for specifying simple pyparsing test cases -PpTestSpec = namedtuple("PpTestSpec", "desc expr text parse_fn " - "expected_list expected_dict expected_fail_locn") -PpTestSpec.__new__.__defaults__ = ('', pp.Empty(), '', 'parseString', None, None, None) +PpTestSpec = namedtuple( + "PpTestSpec", + "desc expr text parse_fn " "expected_list expected_dict expected_fail_locn", +) +PpTestSpec.__new__.__defaults__ = ("", pp.Empty(), "", "parseString", None, None, None) class PyparsingExpressionTestCase(TestParseResultsAsserts): @@ -27,7 +29,9 @@ class PyparsingExpressionTestCase(TestParseResultsAsserts): given text strings. Subclasses must define a class attribute 'tests' which is a list of PpTestSpec instances. """ + tests = [] + def runTest(self): if self.__class__ is PyparsingExpressionTestCase: return @@ -42,26 +46,30 @@ class PyparsingExpressionTestCase(TestParseResultsAsserts): # the location against an expected value with self.subTest(test_spec=test_spec): test_spec.expr.streamline() - print("\n{} - {}({})".format(test_spec.desc, - type(test_spec.expr).__name__, - test_spec.expr)) + print( + "\n{} - {}({})".format( + test_spec.desc, type(test_spec.expr).__name__, test_spec.expr + ) + ) parsefn = getattr(test_spec.expr, test_spec.parse_fn) if test_spec.expected_fail_locn is None: # expect success result = parsefn(test_spec.text) - if test_spec.parse_fn == 'parseString': + if test_spec.parse_fn == "parseString": print(result.dump()) # compare results against given list and/or dict - self.assertParseResultsEquals(result, - expected_list=test_spec.expected_list, - expected_dict=test_spec.expected_dict) - elif test_spec.parse_fn == 'transformString': + self.assertParseResultsEquals( + result, + expected_list=test_spec.expected_list, + expected_dict=test_spec.expected_dict, + ) + elif test_spec.parse_fn == "transformString": print(result) # compare results against given list and/or dict if test_spec.expected_list is not None: self.assertEqual([result], test_spec.expected_list) - elif test_spec.parse_fn == 'searchString': + elif test_spec.parse_fn == "searchString": print(result) # compare results against given list and/or dict if test_spec.expected_list is not None: @@ -79,366 +87,490 @@ class PyparsingExpressionTestCase(TestParseResultsAsserts): # =========== TEST DEFINITIONS START HERE ============== + class TestLiteral(PyparsingExpressionTestCase): tests = [ PpTestSpec( - desc = "Simple match", - expr = pp.Literal("xyz"), - text = "xyz", - expected_list = ["xyz"], + desc="Simple match", + expr=pp.Literal("xyz"), + text="xyz", + expected_list=["xyz"], ), PpTestSpec( - desc = "Simple match after skipping whitespace", - expr = pp.Literal("xyz"), - text = " xyz", - expected_list = ["xyz"], + desc="Simple match after skipping whitespace", + expr=pp.Literal("xyz"), + text=" xyz", + expected_list=["xyz"], ), PpTestSpec( - desc = "Simple fail - parse an empty string", - expr = pp.Literal("xyz"), - text = "", - expected_fail_locn = 0, + desc="Simple fail - parse an empty string", + expr=pp.Literal("xyz"), + text="", + expected_fail_locn=0, ), PpTestSpec( - desc = "Simple fail - parse a mismatching string", - expr = pp.Literal("xyz"), - text = "xyu", - expected_fail_locn = 0, + desc="Simple fail - parse a mismatching string", + expr=pp.Literal("xyz"), + text="xyu", + expected_fail_locn=0, ), PpTestSpec( - desc = "Simple fail - parse a partially matching string", - expr = pp.Literal("xyz"), - text = "xy", - expected_fail_locn = 0, + desc="Simple fail - parse a partially matching string", + expr=pp.Literal("xyz"), + text="xy", + expected_fail_locn=0, ), PpTestSpec( - desc = "Fail - parse a partially matching string by matching individual letters", - expr = pp.Literal("x") + pp.Literal("y") + pp.Literal("z"), - text = "xy", - expected_fail_locn = 2, + desc="Fail - parse a partially matching string by matching individual letters", + expr=pp.Literal("x") + pp.Literal("y") + pp.Literal("z"), + text="xy", + expected_fail_locn=2, ), ] + class TestCaselessLiteral(PyparsingExpressionTestCase): tests = [ PpTestSpec( - desc = "Match colors, converting to consistent case", - expr = (pp.CaselessLiteral("RED") - | pp.CaselessLiteral("GREEN") - | pp.CaselessLiteral("BLUE"))[...], - text = "red Green BluE blue GREEN green rEd", - expected_list = ['RED', 'GREEN', 'BLUE', 'BLUE', 'GREEN', 'GREEN', 'RED'], + desc="Match colors, converting to consistent case", + expr=( + pp.CaselessLiteral("RED") + | pp.CaselessLiteral("GREEN") + | pp.CaselessLiteral("BLUE") + )[...], + text="red Green BluE blue GREEN green rEd", + expected_list=["RED", "GREEN", "BLUE", "BLUE", "GREEN", "GREEN", "RED"], ), ] + class TestWord(PyparsingExpressionTestCase): tests = [ PpTestSpec( - desc = "Simple Word match", - expr = pp.Word("xy"), - text = "xxyxxyy", - expected_list = ["xxyxxyy"], + desc="Simple Word match", + expr=pp.Word("xy"), + text="xxyxxyy", + expected_list=["xxyxxyy"], ), PpTestSpec( - desc = "Simple Word match of two separate Words", - expr = pp.Word("x") + pp.Word("y"), - text = "xxxxxyy", - expected_list = ["xxxxx", "yy"], + desc="Simple Word match of two separate Words", + expr=pp.Word("x") + pp.Word("y"), + text="xxxxxyy", + expected_list=["xxxxx", "yy"], ), PpTestSpec( - desc = "Simple Word match of two separate Words - implicitly skips whitespace", - expr = pp.Word("x") + pp.Word("y"), - text = "xxxxx yy", - expected_list = ["xxxxx", "yy"], + desc="Simple Word match of two separate Words - implicitly skips whitespace", + expr=pp.Word("x") + pp.Word("y"), + text="xxxxx yy", + expected_list=["xxxxx", "yy"], ), ] + class TestCombine(PyparsingExpressionTestCase): tests = [ PpTestSpec( desc="Parsing real numbers - fail, parsed numbers are in pieces", - expr=(pp.Word(pp.nums) + '.' + pp.Word(pp.nums))[...], + expr=(pp.Word(pp.nums) + "." + pp.Word(pp.nums))[...], text="1.2 2.3 3.1416 98.6", - expected_list=['1', '.', '2', '2', '.', '3', '3', '.', '1416', '98', '.', '6'], + expected_list=[ + "1", + ".", + "2", + "2", + ".", + "3", + "3", + ".", + "1416", + "98", + ".", + "6", + ], ), PpTestSpec( desc="Parsing real numbers - better, use Combine to combine multiple tokens into one", - expr=pp.Combine(pp.Word(pp.nums) + '.' + pp.Word(pp.nums))[...], + expr=pp.Combine(pp.Word(pp.nums) + "." + pp.Word(pp.nums))[...], text="1.2 2.3 3.1416 98.6", - expected_list=['1.2', '2.3', '3.1416', '98.6'], + expected_list=["1.2", "2.3", "3.1416", "98.6"], ), ] + class TestRepetition(PyparsingExpressionTestCase): tests = [ PpTestSpec( - desc = "Match several words", - expr = (pp.Word("x") | pp.Word("y"))[...], - text = "xxyxxyyxxyxyxxxy", - expected_list = ['xx', 'y', 'xx', 'yy', 'xx', 'y', 'x', 'y', 'xxx', 'y'], - ), - PpTestSpec( - desc = "Match several words, skipping whitespace", - expr = (pp.Word("x") | pp.Word("y"))[...], - text = "x x y xxy yxx y xyx xxy", - expected_list = ['x', 'x', 'y', 'xx', 'y', 'y', 'xx', 'y', 'x', 'y', 'x', 'xx', 'y'], - ), - PpTestSpec( - desc = "Match several words, skipping whitespace (old style)", - expr = pp.OneOrMore(pp.Word("x") | pp.Word("y")), - text = "x x y xxy yxx y xyx xxy", - expected_list = ['x', 'x', 'y', 'xx', 'y', 'y', 'xx', 'y', 'x', 'y', 'x', 'xx', 'y'], - ), - PpTestSpec( - desc = "Match words and numbers - show use of results names to collect types of tokens", - expr = (pp.Word(pp.alphas)("alpha*") - | pp.pyparsing_common.integer("int*"))[...], - text = "sdlfj23084ksdfs08234kjsdlfkjd0934", - expected_list = ['sdlfj', 23084, 'ksdfs', 8234, 'kjsdlfkjd', 934], - expected_dict = { 'alpha': ['sdlfj', 'ksdfs', 'kjsdlfkjd'], 'int': [23084, 8234, 934] } - ), - PpTestSpec( - desc = "Using delimitedList (comma is the default delimiter)", - expr = pp.delimitedList(pp.Word(pp.alphas)), - text = "xxyx,xy,y,xxyx,yxx, xy", - expected_list = ['xxyx', 'xy', 'y', 'xxyx', 'yxx', 'xy'], - ), - PpTestSpec( - desc = "Using delimitedList, with ':' delimiter", - expr = pp.delimitedList(pp.Word(pp.hexnums, exact=2), delim=':', combine=True), - text = "0A:4B:73:21:FE:76", - expected_list = ['0A:4B:73:21:FE:76'], + desc="Match several words", + expr=(pp.Word("x") | pp.Word("y"))[...], + text="xxyxxyyxxyxyxxxy", + expected_list=["xx", "y", "xx", "yy", "xx", "y", "x", "y", "xxx", "y"], + ), + PpTestSpec( + desc="Match several words, skipping whitespace", + expr=(pp.Word("x") | pp.Word("y"))[...], + text="x x y xxy yxx y xyx xxy", + expected_list=[ + "x", + "x", + "y", + "xx", + "y", + "y", + "xx", + "y", + "x", + "y", + "x", + "xx", + "y", + ], + ), + PpTestSpec( + desc="Match several words, skipping whitespace (old style)", + expr=pp.OneOrMore(pp.Word("x") | pp.Word("y")), + text="x x y xxy yxx y xyx xxy", + expected_list=[ + "x", + "x", + "y", + "xx", + "y", + "y", + "xx", + "y", + "x", + "y", + "x", + "xx", + "y", + ], + ), + PpTestSpec( + desc="Match words and numbers - show use of results names to collect types of tokens", + expr=(pp.Word(pp.alphas)("alpha*") | pp.pyparsing_common.integer("int*"))[ + ... + ], + text="sdlfj23084ksdfs08234kjsdlfkjd0934", + expected_list=["sdlfj", 23084, "ksdfs", 8234, "kjsdlfkjd", 934], + expected_dict={ + "alpha": ["sdlfj", "ksdfs", "kjsdlfkjd"], + "int": [23084, 8234, 934], + }, + ), + PpTestSpec( + desc="Using delimitedList (comma is the default delimiter)", + expr=pp.delimitedList(pp.Word(pp.alphas)), + text="xxyx,xy,y,xxyx,yxx, xy", + expected_list=["xxyx", "xy", "y", "xxyx", "yxx", "xy"], + ), + PpTestSpec( + desc="Using delimitedList, with ':' delimiter", + expr=pp.delimitedList( + pp.Word(pp.hexnums, exact=2), delim=":", combine=True + ), + text="0A:4B:73:21:FE:76", + expected_list=["0A:4B:73:21:FE:76"], ), ] + class TestResultsName(PyparsingExpressionTestCase): tests = [ PpTestSpec( - desc = "Match with results name", - expr = pp.Literal("xyz").setResultsName("value"), - text = "xyz", - expected_dict = {'value': 'xyz'}, - expected_list = ['xyz'], + desc="Match with results name", + expr=pp.Literal("xyz").setResultsName("value"), + text="xyz", + expected_dict={"value": "xyz"}, + expected_list=["xyz"], ), PpTestSpec( - desc = "Match with results name - using naming short-cut", - expr = pp.Literal("xyz")("value"), - text = "xyz", - expected_dict = {'value': 'xyz'}, - expected_list = ['xyz'], + desc="Match with results name - using naming short-cut", + expr=pp.Literal("xyz")("value"), + text="xyz", + expected_dict={"value": "xyz"}, + expected_list=["xyz"], ), PpTestSpec( - desc = "Define multiple results names", - expr = pp.Word(pp.alphas, pp.alphanums)("key") + '=' + pp.pyparsing_common.integer("value"), - text = "range=5280", - expected_dict = {'key': 'range', 'value': 5280}, - expected_list = ['range', '=', 5280], + desc="Define multiple results names", + expr=pp.Word(pp.alphas, pp.alphanums)("key") + + "=" + + pp.pyparsing_common.integer("value"), + text="range=5280", + expected_dict={"key": "range", "value": 5280}, + expected_list=["range", "=", 5280], ), ] + class TestGroups(PyparsingExpressionTestCase): - EQ = pp.Suppress('=') + EQ = pp.Suppress("=") tests = [ PpTestSpec( - desc = "Define multiple results names in groups", - expr = pp.Group(pp.Word(pp.alphas)("key") - + EQ - + pp.pyparsing_common.number("value"))[...], - text = "range=5280 long=-138.52 lat=46.91", - expected_list = [['range', 5280], ['long', -138.52], ['lat', 46.91]], - ), - PpTestSpec( - desc = "Define multiple results names in groups - use Dict to define results names using parsed keys", - expr = pp.Dict(pp.Group(pp.Word(pp.alphas) - + EQ - + pp.pyparsing_common.number)[...]), - text = "range=5280 long=-138.52 lat=46.91", - expected_list = [['range', 5280], ['long', -138.52], ['lat', 46.91]], - expected_dict = {'lat': 46.91, 'long': -138.52, 'range': 5280} - ), - PpTestSpec( - desc = "Define multiple value types", - expr = pp.Dict(pp.Group(pp.Word(pp.alphas) - + EQ - + (pp.pyparsing_common.number | pp.oneOf("True False") | pp.QuotedString("'")) - )[...] - ), - text = "long=-122.47 lat=37.82 public=True name='Golden Gate Bridge'", - expected_list = [['long', -122.47], ['lat', 37.82], ['public', 'True'], ['name', 'Golden Gate Bridge']], - expected_dict = {'long': -122.47, 'lat': 37.82, 'public': 'True', 'name': 'Golden Gate Bridge'} + desc="Define multiple results names in groups", + expr=pp.Group( + pp.Word(pp.alphas)("key") + EQ + pp.pyparsing_common.number("value") + )[...], + text="range=5280 long=-138.52 lat=46.91", + expected_list=[["range", 5280], ["long", -138.52], ["lat", 46.91]], + ), + PpTestSpec( + desc="Define multiple results names in groups - use Dict to define results names using parsed keys", + expr=pp.Dict( + pp.Group(pp.Word(pp.alphas) + EQ + pp.pyparsing_common.number)[...] + ), + text="range=5280 long=-138.52 lat=46.91", + expected_list=[["range", 5280], ["long", -138.52], ["lat", 46.91]], + expected_dict={"lat": 46.91, "long": -138.52, "range": 5280}, + ), + PpTestSpec( + desc="Define multiple value types", + expr=pp.Dict( + pp.Group( + pp.Word(pp.alphas) + + EQ + + ( + pp.pyparsing_common.number + | pp.oneOf("True False") + | pp.QuotedString("'") + ) + )[...] + ), + text="long=-122.47 lat=37.82 public=True name='Golden Gate Bridge'", + expected_list=[ + ["long", -122.47], + ["lat", 37.82], + ["public", "True"], + ["name", "Golden Gate Bridge"], + ], + expected_dict={ + "long": -122.47, + "lat": 37.82, + "public": "True", + "name": "Golden Gate Bridge", + }, ), ] + class TestParseAction(PyparsingExpressionTestCase): tests = [ PpTestSpec( desc="Parsing real numbers - use parse action to convert to float at parse time", - expr=pp.Combine(pp.Word(pp.nums) + '.' + pp.Word(pp.nums)).addParseAction(lambda t: float(t[0]))[...], + expr=pp.Combine(pp.Word(pp.nums) + "." + pp.Word(pp.nums)).addParseAction( + lambda t: float(t[0]) + )[...], text="1.2 2.3 3.1416 98.6", - expected_list= [1.2, 2.3, 3.1416, 98.6], # note, these are now floats, not strs + expected_list=[ + 1.2, + 2.3, + 3.1416, + 98.6, + ], # note, these are now floats, not strs ), PpTestSpec( - desc = "Match with numeric string converted to int", - expr = pp.Word("0123456789").addParseAction(lambda t: int(t[0])), - text = "12345", - expected_list = [12345], # note - result is type int, not str + desc="Match with numeric string converted to int", + expr=pp.Word("0123456789").addParseAction(lambda t: int(t[0])), + text="12345", + expected_list=[12345], # note - result is type int, not str ), PpTestSpec( - desc = "Use two parse actions to convert numeric string, then convert to datetime", - expr = pp.Word(pp.nums).addParseAction(lambda t: int(t[0]), - lambda t: datetime.utcfromtimestamp(t[0])), - text = "1537415628", - expected_list = [datetime(2018, 9, 20, 3, 53, 48)], + desc="Use two parse actions to convert numeric string, then convert to datetime", + expr=pp.Word(pp.nums).addParseAction( + lambda t: int(t[0]), lambda t: datetime.utcfromtimestamp(t[0]) + ), + text="1537415628", + expected_list=[datetime(2018, 9, 20, 3, 53, 48)], ), PpTestSpec( - desc = "Use tokenMap for parse actions that operate on a single-length token", - expr = pp.Word(pp.nums).addParseAction(pp.tokenMap(int), - pp.tokenMap(datetime.utcfromtimestamp)), - text = "1537415628", - expected_list = [datetime(2018, 9, 20, 3, 53, 48)], + desc="Use tokenMap for parse actions that operate on a single-length token", + expr=pp.Word(pp.nums).addParseAction( + pp.tokenMap(int), pp.tokenMap(datetime.utcfromtimestamp) + ), + text="1537415628", + expected_list=[datetime(2018, 9, 20, 3, 53, 48)], ), PpTestSpec( - desc = "Using a built-in function that takes a sequence of strs as a parse action", - expr = pp.Word(pp.hexnums, exact=2)[...].addParseAction(':'.join), - text = "0A4B7321FE76", - expected_list = ['0A:4B:73:21:FE:76'], + desc="Using a built-in function that takes a sequence of strs as a parse action", + expr=pp.Word(pp.hexnums, exact=2)[...].addParseAction(":".join), + text="0A4B7321FE76", + expected_list=["0A:4B:73:21:FE:76"], ), PpTestSpec( - desc = "Using a built-in function that takes a sequence of strs as a parse action", - expr = pp.Word(pp.hexnums, exact=2)[...].addParseAction(sorted), - text = "0A4B7321FE76", - expected_list = ['0A', '21', '4B', '73', '76', 'FE'], + desc="Using a built-in function that takes a sequence of strs as a parse action", + expr=pp.Word(pp.hexnums, exact=2)[...].addParseAction(sorted), + text="0A4B7321FE76", + expected_list=["0A", "21", "4B", "73", "76", "FE"], ), ] + class TestResultsModifyingParseAction(PyparsingExpressionTestCase): def compute_stats_parse_action(t): # by the time this parse action is called, parsed numeric words # have been converted to ints by a previous parse action, so # they can be treated as ints - t['sum'] = sum(t) - t['ave'] = sum(t) / len(t) - t['min'] = min(t) - t['max'] = max(t) + t["sum"] = sum(t) + t["ave"] = sum(t) / len(t) + t["min"] = min(t) + t["max"] = max(t) tests = [ PpTestSpec( - desc = "A parse action that adds new key-values", - expr = pp.pyparsing_common.integer[...].addParseAction(compute_stats_parse_action), - text = "27 1 14 22 89", - expected_list = [27, 1, 14, 22, 89], - expected_dict = {'ave': 30.6, 'max': 89, 'min': 1, 'sum': 153} + desc="A parse action that adds new key-values", + expr=pp.pyparsing_common.integer[...].addParseAction( + compute_stats_parse_action + ), + text="27 1 14 22 89", + expected_list=[27, 1, 14, 22, 89], + expected_dict={"ave": 30.6, "max": 89, "min": 1, "sum": 153}, ), ] + class TestRegex(PyparsingExpressionTestCase): tests = [ PpTestSpec( desc="Parsing real numbers - using Regex instead of Combine", - expr=pp.Regex(r'\d+\.\d+').addParseAction(lambda t: float(t[0]))[...], + expr=pp.Regex(r"\d+\.\d+").addParseAction(lambda t: float(t[0]))[...], text="1.2 2.3 3.1416 98.6", - expected_list=[1.2, 2.3, 3.1416, 98.6], # note, these are now floats, not strs + expected_list=[ + 1.2, + 2.3, + 3.1416, + 98.6, + ], # note, these are now floats, not strs ), ] + class TestParseCondition(PyparsingExpressionTestCase): tests = [ PpTestSpec( - desc = "Define a condition to only match numeric values that are multiples of 7", - expr = pp.Word(pp.nums).addCondition(lambda t: int(t[0]) % 7 == 0)[...], - text = "14 35 77 12 28", - expected_list = ['14', '35', '77'], + desc="Define a condition to only match numeric values that are multiples of 7", + expr=pp.Word(pp.nums).addCondition(lambda t: int(t[0]) % 7 == 0)[...], + text="14 35 77 12 28", + expected_list=["14", "35", "77"], ), PpTestSpec( - desc = "Separate conversion to int and condition into separate parse action/conditions", - expr = pp.Word(pp.nums).addParseAction(lambda t: int(t[0])) - .addCondition(lambda t: t[0] % 7 == 0)[...], - text = "14 35 77 12 28", - expected_list = [14, 35, 77], + desc="Separate conversion to int and condition into separate parse action/conditions", + expr=pp.Word(pp.nums) + .addParseAction(lambda t: int(t[0])) + .addCondition(lambda t: t[0] % 7 == 0)[...], + text="14 35 77 12 28", + expected_list=[14, 35, 77], ), ] + class TestTransformStringUsingParseActions(PyparsingExpressionTestCase): markup_convert_map = { - '*' : 'B', - '_' : 'U', - '/' : 'I', + "*": "B", + "_": "U", + "/": "I", } + def markup_convert(t): - htmltag = TestTransformStringUsingParseActions.markup_convert_map[t.markup_symbol] + htmltag = TestTransformStringUsingParseActions.markup_convert_map[ + t.markup_symbol + ] return "<{}>{}</{}>".format(htmltag, t.body, htmltag) tests = [ PpTestSpec( - desc = "Use transformString to convert simple markup to HTML", - expr = (pp.oneOf(markup_convert_map)('markup_symbol') - + "(" + pp.CharsNotIn(")")('body') + ")").addParseAction(markup_convert), - text = "Show in *(bold), _(underscore), or /(italic) type", - expected_list = ['Show in <B>bold</B>, <U>underscore</U>, or <I>italic</I> type'], - parse_fn = 'transformString', + desc="Use transformString to convert simple markup to HTML", + expr=( + pp.oneOf(markup_convert_map)("markup_symbol") + + "(" + + pp.CharsNotIn(")")("body") + + ")" + ).addParseAction(markup_convert), + text="Show in *(bold), _(underscore), or /(italic) type", + expected_list=[ + "Show in <B>bold</B>, <U>underscore</U>, or <I>italic</I> type" + ], + parse_fn="transformString", ), ] + class TestCommonHelperExpressions(PyparsingExpressionTestCase): tests = [ PpTestSpec( - desc = "A comma-delimited list of words", - expr = pp.delimitedList(pp.Word(pp.alphas)), - text = "this, that, blah,foo, bar", - expected_list = ['this', 'that', 'blah', 'foo', 'bar'], - ), - PpTestSpec( - desc = "A counted array of words", - expr = pp.countedArray(pp.Word('ab'))[...], - text = "2 aaa bbb 0 3 abab bbaa abbab", - expected_list = [['aaa', 'bbb'], [], ['abab', 'bbaa', 'abbab']], - ), - PpTestSpec( - desc = "skipping comments with ignore", - expr = (pp.pyparsing_common.identifier('lhs') - + '=' - + pp.pyparsing_common.fnumber('rhs')).ignore(pp.cppStyleComment), - text = "abc_100 = /* value to be tested */ 3.1416", - expected_list = ['abc_100', '=', 3.1416], - expected_dict = {'lhs': 'abc_100', 'rhs': 3.1416}, - ), - PpTestSpec( - desc = "some pre-defined expressions in pyparsing_common, and building a dotted identifier with delimted_list", - expr = (pp.pyparsing_common.number("id_num") - + pp.delimitedList(pp.pyparsing_common.identifier, '.', combine=True)("name") - + pp.pyparsing_common.ipv4_address("ip_address") - ), - text = "1001 www.google.com 192.168.10.199", - expected_list = [1001, 'www.google.com', '192.168.10.199'], - expected_dict = {'id_num': 1001, 'name': 'www.google.com', 'ip_address': '192.168.10.199'}, - ), - PpTestSpec( - desc = "using oneOf (shortcut for Literal('a') | Literal('b') | Literal('c'))", - expr = pp.oneOf("a b c")[...], - text = "a b a b b a c c a b b", - expected_list = ['a', 'b', 'a', 'b', 'b', 'a', 'c', 'c', 'a', 'b', 'b'], - ), - PpTestSpec( - desc = "parsing nested parentheses", - expr = pp.nestedExpr(), - text = "(a b (c) d (e f g ()))", - expected_list = [['a', 'b', ['c'], 'd', ['e', 'f', 'g', []]]], - ), - PpTestSpec( - desc = "parsing nested braces", - expr = (pp.Keyword('if') - + pp.nestedExpr()('condition') - + pp.nestedExpr('{', '}')('body')), - text = 'if ((x == y) || !z) {printf("{}");}', - expected_list = ['if', [['x', '==', 'y'], '||', '!z'], ['printf(', '"{}"', ');']], - expected_dict = {'condition': [[['x', '==', 'y'], '||', '!z']], - 'body': [['printf(', '"{}"', ');']]}, + desc="A comma-delimited list of words", + expr=pp.delimitedList(pp.Word(pp.alphas)), + text="this, that, blah,foo, bar", + expected_list=["this", "that", "blah", "foo", "bar"], + ), + PpTestSpec( + desc="A counted array of words", + expr=pp.countedArray(pp.Word("ab"))[...], + text="2 aaa bbb 0 3 abab bbaa abbab", + expected_list=[["aaa", "bbb"], [], ["abab", "bbaa", "abbab"]], + ), + PpTestSpec( + desc="skipping comments with ignore", + expr=( + pp.pyparsing_common.identifier("lhs") + + "=" + + pp.pyparsing_common.fnumber("rhs") + ).ignore(pp.cppStyleComment), + text="abc_100 = /* value to be tested */ 3.1416", + expected_list=["abc_100", "=", 3.1416], + expected_dict={"lhs": "abc_100", "rhs": 3.1416}, + ), + PpTestSpec( + desc="some pre-defined expressions in pyparsing_common, and building a dotted identifier with delimted_list", + expr=( + pp.pyparsing_common.number("id_num") + + pp.delimitedList(pp.pyparsing_common.identifier, ".", combine=True)( + "name" + ) + + pp.pyparsing_common.ipv4_address("ip_address") + ), + text="1001 www.google.com 192.168.10.199", + expected_list=[1001, "www.google.com", "192.168.10.199"], + expected_dict={ + "id_num": 1001, + "name": "www.google.com", + "ip_address": "192.168.10.199", + }, + ), + PpTestSpec( + desc="using oneOf (shortcut for Literal('a') | Literal('b') | Literal('c'))", + expr=pp.oneOf("a b c")[...], + text="a b a b b a c c a b b", + expected_list=["a", "b", "a", "b", "b", "a", "c", "c", "a", "b", "b"], + ), + PpTestSpec( + desc="parsing nested parentheses", + expr=pp.nestedExpr(), + text="(a b (c) d (e f g ()))", + expected_list=[["a", "b", ["c"], "d", ["e", "f", "g", []]]], + ), + PpTestSpec( + desc="parsing nested braces", + expr=( + pp.Keyword("if") + + pp.nestedExpr()("condition") + + pp.nestedExpr("{", "}")("body") + ), + text='if ((x == y) || !z) {printf("{}");}', + expected_list=[ + "if", + [["x", "==", "y"], "||", "!z"], + ["printf(", '"{}"', ");"], + ], + expected_dict={ + "condition": [[["x", "==", "y"], "||", "!z"]], + "body": [["printf(", '"{}"', ");"]], + }, ), ] def _get_decl_line_no(cls): import inspect + return inspect.getsourcelines(cls)[1] @@ -456,7 +588,7 @@ suite = unittest.TestSuite(cls() for cls in test_case_classes) # ============ MAIN ================ -if __name__ == '__main__': +if __name__ == "__main__": result = unittest.TextTestRunner().run(suite) diff --git a/tests/test_unit.py b/tests/test_unit.py index 94ffa2a..495fd0d 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -22,18 +22,21 @@ ppt = pp.pyparsing_test TestParseResultsAsserts = ppt.TestParseResultsAsserts # see which Python implementation we are running -CPYTHON_ENV = (sys.platform == "win32") -IRON_PYTHON_ENV = (sys.platform == "cli") +CPYTHON_ENV = sys.platform == "win32" +IRON_PYTHON_ENV = sys.platform == "cli" JYTHON_ENV = sys.platform.startswith("java") VERBOSE = True # simple utility for flattening nested lists def flatten(L): - if type(L) is not list: return [L] - if L == []: return L + if type(L) is not list: + return [L] + if L == []: + return L return flatten(L[0]) + flatten(L[1:]) + """ class ParseTest(TestCase): def setUp(self): @@ -46,6 +49,7 @@ class ParseTest(TestCase): pass """ + class resetting: def __init__(self, *args): ob = args[0] @@ -64,8 +68,16 @@ class resetting: class Test1_PyparsingTestInit(TestCase): def runTest(self): - from pyparsing import __version__ as pyparsingVersion, __versionTime__ as pyparsingVersionTime - print("Beginning test of pyparsing, version", pyparsingVersion, pyparsingVersionTime) + from pyparsing import ( + __version__ as pyparsingVersion, + __versionTime__ as pyparsingVersionTime, + ) + + print( + "Beginning test of pyparsing, version", + pyparsingVersion, + pyparsingVersionTime, + ) print("Python version", sys.version) @@ -82,21 +94,33 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): try: pp.dblQuotedString.copyDefaultWhiteChars = False pp.ParserElement.setDefaultWhitespaceChars(" \t") - self.assertEqual(set(pp.sglQuotedString.whiteChars), set(" \t"), - "setDefaultWhitespaceChars did not update sglQuotedString") - self.assertEqual(set(pp.dblQuotedString.whiteChars), set(prev_default_whitespace_chars), - "setDefaultWhitespaceChars updated dblQuotedString but should not") + self.assertEqual( + set(pp.sglQuotedString.whiteChars), + set(" \t"), + "setDefaultWhitespaceChars did not update sglQuotedString", + ) + self.assertEqual( + set(pp.dblQuotedString.whiteChars), + set(prev_default_whitespace_chars), + "setDefaultWhitespaceChars updated dblQuotedString but should not", + ) finally: pp.dblQuotedString.copyDefaultWhiteChars = True pp.ParserElement.setDefaultWhitespaceChars(prev_default_whitespace_chars) - self.assertEqual(set(pp.dblQuotedString.whiteChars), set(prev_default_whitespace_chars), - "setDefaultWhitespaceChars updated dblQuotedString") + self.assertEqual( + set(pp.dblQuotedString.whiteChars), + set(prev_default_whitespace_chars), + "setDefaultWhitespaceChars updated dblQuotedString", + ) with ppt.reset_pyparsing_context(): pp.ParserElement.setDefaultWhitespaceChars(" \t") - self.assertNotEqual(set(pp.dblQuotedString.whiteChars), set(prev_default_whitespace_chars), - "setDefaultWhitespaceChars updated dblQuotedString but should not") + self.assertNotEqual( + set(pp.dblQuotedString.whiteChars), + set(prev_default_whitespace_chars), + "setDefaultWhitespaceChars updated dblQuotedString but should not", + ) EOL = pp.LineEnd().suppress().setName("EOL") @@ -127,10 +151,15 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): parsed_program = program.parseString(test) print(parsed_program.dump()) - self.assertEqual(len(parsed_program), 3, "failed to apply new whitespace chars to existing builtins") + self.assertEqual( + len(parsed_program), + 3, + "failed to apply new whitespace chars to existing builtins", + ) def testUpdateDefaultWhitespace2(self): import pyparsing as pp + ppc = pp.pyparsing_common with ppt.reset_pyparsing_context(): @@ -145,7 +174,7 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): for expr, test_str in expr_tests: parser = pp.Group(expr[1, ...] + pp.Optional(NL))[1, ...] - test_string = '\n'.join([test_str]*3) + test_string = "\n".join([test_str] * 3) result = parser.parseString(test_string, parseAll=True) print(result.dump()) self.assertEqual(len(result), 1, "failed {!r}".format(test_string)) @@ -154,7 +183,7 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): for expr, test_str in expr_tests: parser = pp.Group(expr[1, ...] + pp.Optional(NL))[1, ...] - test_string = '\n'.join([test_str]*3) + test_string = "\n".join([test_str] * 3) result = parser.parseString(test_string, parseAll=True) print(result.dump()) self.assertEqual(len(result), 3, "failed {!r}".format(test_string)) @@ -163,7 +192,7 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): for expr, test_str in expr_tests: parser = pp.Group(expr[1, ...] + pp.Optional(NL))[1, ...] - test_string = '\n'.join([test_str]*3) + test_string = "\n".join([test_str] * 3) result = parser.parseString(test_string, parseAll=True) print(result.dump()) self.assertEqual(len(result), 1, "failed {!r}".format(test_string)) @@ -171,6 +200,7 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): def testParseFourFn(self): import examples.fourFn as fourFn import math + def test(s, ans): fourFn.exprStack[:] = [] results = fourFn.BNF().parseString(s) @@ -179,7 +209,10 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): except Exception: self.assertIsNone(ans, "exception raised for expression {!r}".format(s)) else: - self.assertTrue(resultValue == ans, "failed to evaluate {}, got {:f}".format(s, resultValue)) + self.assertTrue( + resultValue == ans, + "failed to evaluate {}, got {:f}".format(s, resultValue), + ) print(s, "->", resultValue) test("9", 9) @@ -198,7 +231,7 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): test("PI*PI/10", math.pi * math.pi / 10) test("PI^2", math.pi ** 2) test("round(PI^2)", round(math.pi ** 2)) - test("6.02E23 * 8.048", 6.02E23 * 8.048) + test("6.02E23 * 8.048", 6.02e23 * 8.048) test("e / 3", math.e / 3) test("sin(PI/2)", math.sin(math.pi / 2)) test("10+sin(PI/4)^2", 10 + math.sin(math.pi / 4) ** 2) @@ -233,13 +266,20 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): try: sqlToks = flatten(simpleSQL.simpleSQL.parseString(s).asList()) print(s, sqlToks, len(sqlToks)) - self.assertEqual(len(sqlToks), numToks, - "invalid parsed tokens, expected {}, found {} ({})".format(numToks, - len(sqlToks), - sqlToks)) + self.assertEqual( + len(sqlToks), + numToks, + "invalid parsed tokens, expected {}, found {} ({})".format( + numToks, len(sqlToks), sqlToks + ), + ) except ParseException as e: if errloc >= 0: - self.assertEqual(e.loc, errloc, "expected error at {}, found at {}".format(errloc, e.loc)) + self.assertEqual( + e.loc, + errloc, + "expected error at {}, found at {}".format(errloc, e.loc), + ) test("SELECT * from XYZZY, ABC", 6) test("select * from SYS.XYZZY", 5) @@ -252,208 +292,398 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): test("Select", 0, 6) test("Select &&& frox Sys.dual", 0, 7) test("Select A from Sys.dual where a in ('RED','GREEN','BLUE')", 12) - test("Select A from Sys.dual where a in ('RED','GREEN','BLUE') and b in (10,20,30)", 20) - test("Select A,b from table1,table2 where table1.id eq table2.id -- test out comparison operators", 10) + test( + "Select A from Sys.dual where a in ('RED','GREEN','BLUE') and b in (10,20,30)", + 20, + ) + test( + "Select A,b from table1,table2 where table1.id eq table2.id -- test out comparison operators", + 10, + ) def testParseConfigFile(self): from examples import configParse def test(fnam, numToks, resCheckList): - print("Parsing", fnam, "...", end=' ') + print("Parsing", fnam, "...", end=" ") with open(fnam) as infile: iniFileLines = "\n".join(infile.read().splitlines()) iniData = configParse.inifile_BNF().parseString(iniFileLines) print(len(flatten(iniData.asList()))) print(list(iniData.keys())) - self.assertEqual(len(flatten(iniData.asList())), numToks, "file %s not parsed correctly" % fnam) + self.assertEqual( + len(flatten(iniData.asList())), + numToks, + "file %s not parsed correctly" % fnam, + ) for chk in resCheckList: var = iniData - for attr in chk[0].split('.'): + for attr in chk[0].split("."): var = getattr(var, attr) print(chk[0], var, chk[1]) - self.assertEqual(var, chk[1], - "ParseConfigFileTest: failed to parse ini {!r} as expected {}, found {}".format(chk[0], - chk[1], - var)) + self.assertEqual( + var, + chk[1], + "ParseConfigFileTest: failed to parse ini {!r} as expected {}, found {}".format( + chk[0], chk[1], var + ), + ) print("OK") - test("tests/karthik.ini", 23, - [ ("users.K", "8"), - ("users.mod_scheme", "'QPSK'"), - ("users.Na", "K+2") ] - ) - test("examples/Setup.ini", 125, - [ ("Startup.audioinf", "M3i"), - ("Languages.key1", "0x0003"), - ("test.foo", "bar") ]) + test( + "tests/karthik.ini", + 23, + [("users.K", "8"), ("users.mod_scheme", "'QPSK'"), ("users.Na", "K+2")], + ) + test( + "examples/Setup.ini", + 125, + [ + ("Startup.audioinf", "M3i"), + ("Languages.key1", "0x0003"), + ("test.foo", "bar"), + ], + ) def testParseJSONData(self): expected = [ - [['glossary', - [['title', 'example glossary'], - ['GlossDiv', - [['title', 'S'], - ['GlossList', - [[['ID', 'SGML'], - ['SortAs', 'SGML'], - ['GlossTerm', 'Standard Generalized Markup Language'], - ['Acronym', 'SGML'], - ['LargestPrimeLessThan100', 97], - ['AvogadroNumber', 6.02e+23], - ['EvenPrimesGreaterThan2', None], - ['PrimesLessThan10', [2, 3, 5, 7]], - ['WMDsFound', False], - ['IraqAlQaedaConnections', None], - ['Abbrev', 'ISO 8879:1986'], - ['GlossDef', - 'A meta-markup language, used to create markup languages such as ' - 'DocBook.'], - ['GlossSeeAlso', ['GML', 'XML', 'markup']], - ['EmptyDict', []], - ['EmptyList', [[]]]]]]]]] - ]] - , - [['menu', - [['id', 'file'], - ['value', 'File:'], - ['popup', - [['menuitem', - [[['value', 'New'], ['onclick', 'CreateNewDoc()']], - [['value', 'Open'], ['onclick', 'OpenDoc()']], - [['value', 'Close'], ['onclick', 'CloseDoc()']]]]]]]]] - , - [['widget', - [['debug', 'on'], - ['window', - [['title', 'Sample Konfabulator Widget'], - ['name', 'main_window'], - ['width', 500], - ['height', 500]]], - ['image', - [['src', 'Images/Sun.png'], - ['name', 'sun1'], - ['hOffset', 250], - ['vOffset', 250], - ['alignment', 'center']]], - ['text', - [['data', 'Click Here'], - ['size', 36], - ['style', 'bold'], - ['name', 'text1'], - ['hOffset', 250], - ['vOffset', 100], - ['alignment', 'center'], - ['onMouseUp', 'sun1.opacity = (sun1.opacity / 100) * 90;']]]]]] - , - [['web-app', - [['servlet', - [[['servlet-name', 'cofaxCDS'], - ['servlet-class', 'org.cofax.cds.CDSServlet'], - ['init-param', - [['configGlossary:installationAt', 'Philadelphia, PA'], - ['configGlossary:adminEmail', 'ksm@pobox.com'], - ['configGlossary:poweredBy', 'Cofax'], - ['configGlossary:poweredByIcon', '/images/cofax.gif'], - ['configGlossary:staticPath', '/content/static'], - ['templateProcessorClass', 'org.cofax.WysiwygTemplate'], - ['templateLoaderClass', 'org.cofax.FilesTemplateLoader'], - ['templatePath', 'templates'], - ['templateOverridePath', ''], - ['defaultListTemplate', 'listTemplate.htm'], - ['defaultFileTemplate', 'articleTemplate.htm'], - ['useJSP', False], - ['jspListTemplate', 'listTemplate.jsp'], - ['jspFileTemplate', 'articleTemplate.jsp'], - ['cachePackageTagsTrack', 200], - ['cachePackageTagsStore', 200], - ['cachePackageTagsRefresh', 60], - ['cacheTemplatesTrack', 100], - ['cacheTemplatesStore', 50], - ['cacheTemplatesRefresh', 15], - ['cachePagesTrack', 200], - ['cachePagesStore', 100], - ['cachePagesRefresh', 10], - ['cachePagesDirtyRead', 10], - ['searchEngineListTemplate', 'forSearchEnginesList.htm'], - ['searchEngineFileTemplate', 'forSearchEngines.htm'], - ['searchEngineRobotsDb', 'WEB-INF/robots.db'], - ['useDataStore', True], - ['dataStoreClass', 'org.cofax.SqlDataStore'], - ['redirectionClass', 'org.cofax.SqlRedirection'], - ['dataStoreName', 'cofax'], - ['dataStoreDriver', 'com.microsoft.jdbc.sqlserver.SQLServerDriver'], - ['dataStoreUrl', - 'jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon'], - ['dataStoreUser', 'sa'], - ['dataStorePassword', 'dataStoreTestQuery'], - ['dataStoreTestQuery', "SET NOCOUNT ON;select test='test';"], - ['dataStoreLogFile', '/usr/local/tomcat/logs/datastore.log'], - ['dataStoreInitConns', 10], - ['dataStoreMaxConns', 100], - ['dataStoreConnUsageLimit', 100], - ['dataStoreLogLevel', 'debug'], - ['maxUrlLength', 500]]]], - [['servlet-name', 'cofaxEmail'], - ['servlet-class', 'org.cofax.cds.EmailServlet'], - ['init-param', [['mailHost', 'mail1'], ['mailHostOverride', 'mail2']]]], - [['servlet-name', 'cofaxAdmin'], - ['servlet-class', 'org.cofax.cds.AdminServlet']], - [['servlet-name', 'fileServlet'], - ['servlet-class', 'org.cofax.cds.FileServlet']], - [['servlet-name', 'cofaxTools'], - ['servlet-class', 'org.cofax.cms.CofaxToolsServlet'], - ['init-param', - [['templatePath', 'toolstemplates/'], - ['log', 1], - ['logLocation', '/usr/local/tomcat/logs/CofaxTools.log'], - ['logMaxSize', ''], - ['dataLog', 1], - ['dataLogLocation', '/usr/local/tomcat/logs/dataLog.log'], - ['dataLogMaxSize', ''], - ['removePageCache', '/content/admin/remove?cache=pages&id='], - ['removeTemplateCache', '/content/admin/remove?cache=templates&id='], - ['fileTransferFolder', - '/usr/local/tomcat/webapps/content/fileTransferFolder'], - ['lookInContext', 1], - ['adminGroupID', 4], - ['betaServer', True]]]]]], - ['servlet-mapping', - [['cofaxCDS', '/'], - ['cofaxEmail', '/cofaxutil/aemail/*'], - ['cofaxAdmin', '/admin/*'], - ['fileServlet', '/static/*'], - ['cofaxTools', '/tools/*']]], - ['taglib', - [['taglib-uri', 'cofax.tld'], - ['taglib-location', '/WEB-INF/tlds/cofax.tld']]]]]] - , - [['menu', - [['header', 'SVG Viewer'], - ['items', - [[['id', 'Open']], - [['id', 'OpenNew'], ['label', 'Open New']], - None, - [['id', 'ZoomIn'], ['label', 'Zoom In']], - [['id', 'ZoomOut'], ['label', 'Zoom Out']], - [['id', 'OriginalView'], ['label', 'Original View']], - None, - [['id', 'Quality']], - [['id', 'Pause']], - [['id', 'Mute']], - None, - [['id', 'Find'], ['label', 'Find...']], - [['id', 'FindAgain'], ['label', 'Find Again']], - [['id', 'Copy']], - [['id', 'CopyAgain'], ['label', 'Copy Again']], - [['id', 'CopySVG'], ['label', 'Copy SVG']], - [['id', 'ViewSVG'], ['label', 'View SVG']], - [['id', 'ViewSource'], ['label', 'View Source']], - [['id', 'SaveAs'], ['label', 'Save As']], - None, - [['id', 'Help']], - [['id', 'About'], ['label', 'About Adobe CVG Viewer...']]]]]]] - , - ] + [ + [ + "glossary", + [ + ["title", "example glossary"], + [ + "GlossDiv", + [ + ["title", "S"], + [ + "GlossList", + [ + [ + ["ID", "SGML"], + ["SortAs", "SGML"], + [ + "GlossTerm", + "Standard Generalized Markup Language", + ], + ["Acronym", "SGML"], + ["LargestPrimeLessThan100", 97], + ["AvogadroNumber", 6.02e23], + ["EvenPrimesGreaterThan2", None], + ["PrimesLessThan10", [2, 3, 5, 7]], + ["WMDsFound", False], + ["IraqAlQaedaConnections", None], + ["Abbrev", "ISO 8879:1986"], + [ + "GlossDef", + "A meta-markup language, used to create markup languages such as " + "DocBook.", + ], + ["GlossSeeAlso", ["GML", "XML", "markup"]], + ["EmptyDict", []], + ["EmptyList", [[]]], + ] + ], + ], + ], + ], + ], + ] + ], + [ + [ + "menu", + [ + ["id", "file"], + ["value", "File:"], + [ + "popup", + [ + [ + "menuitem", + [ + [ + ["value", "New"], + ["onclick", "CreateNewDoc()"], + ], + [["value", "Open"], ["onclick", "OpenDoc()"]], + [["value", "Close"], ["onclick", "CloseDoc()"]], + ], + ] + ], + ], + ], + ] + ], + [ + [ + "widget", + [ + ["debug", "on"], + [ + "window", + [ + ["title", "Sample Konfabulator Widget"], + ["name", "main_window"], + ["width", 500], + ["height", 500], + ], + ], + [ + "image", + [ + ["src", "Images/Sun.png"], + ["name", "sun1"], + ["hOffset", 250], + ["vOffset", 250], + ["alignment", "center"], + ], + ], + [ + "text", + [ + ["data", "Click Here"], + ["size", 36], + ["style", "bold"], + ["name", "text1"], + ["hOffset", 250], + ["vOffset", 100], + ["alignment", "center"], + [ + "onMouseUp", + "sun1.opacity = (sun1.opacity / 100) * 90;", + ], + ], + ], + ], + ] + ], + [ + [ + "web-app", + [ + [ + "servlet", + [ + [ + ["servlet-name", "cofaxCDS"], + ["servlet-class", "org.cofax.cds.CDSServlet"], + [ + "init-param", + [ + [ + "configGlossary:installationAt", + "Philadelphia, PA", + ], + [ + "configGlossary:adminEmail", + "ksm@pobox.com", + ], + ["configGlossary:poweredBy", "Cofax"], + [ + "configGlossary:poweredByIcon", + "/images/cofax.gif", + ], + [ + "configGlossary:staticPath", + "/content/static", + ], + [ + "templateProcessorClass", + "org.cofax.WysiwygTemplate", + ], + [ + "templateLoaderClass", + "org.cofax.FilesTemplateLoader", + ], + ["templatePath", "templates"], + ["templateOverridePath", ""], + ["defaultListTemplate", "listTemplate.htm"], + [ + "defaultFileTemplate", + "articleTemplate.htm", + ], + ["useJSP", False], + ["jspListTemplate", "listTemplate.jsp"], + ["jspFileTemplate", "articleTemplate.jsp"], + ["cachePackageTagsTrack", 200], + ["cachePackageTagsStore", 200], + ["cachePackageTagsRefresh", 60], + ["cacheTemplatesTrack", 100], + ["cacheTemplatesStore", 50], + ["cacheTemplatesRefresh", 15], + ["cachePagesTrack", 200], + ["cachePagesStore", 100], + ["cachePagesRefresh", 10], + ["cachePagesDirtyRead", 10], + [ + "searchEngineListTemplate", + "forSearchEnginesList.htm", + ], + [ + "searchEngineFileTemplate", + "forSearchEngines.htm", + ], + [ + "searchEngineRobotsDb", + "WEB-INF/robots.db", + ], + ["useDataStore", True], + [ + "dataStoreClass", + "org.cofax.SqlDataStore", + ], + [ + "redirectionClass", + "org.cofax.SqlRedirection", + ], + ["dataStoreName", "cofax"], + [ + "dataStoreDriver", + "com.microsoft.jdbc.sqlserver.SQLServerDriver", + ], + [ + "dataStoreUrl", + "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon", + ], + ["dataStoreUser", "sa"], + ["dataStorePassword", "dataStoreTestQuery"], + [ + "dataStoreTestQuery", + "SET NOCOUNT ON;select test='test';", + ], + [ + "dataStoreLogFile", + "/usr/local/tomcat/logs/datastore.log", + ], + ["dataStoreInitConns", 10], + ["dataStoreMaxConns", 100], + ["dataStoreConnUsageLimit", 100], + ["dataStoreLogLevel", "debug"], + ["maxUrlLength", 500], + ], + ], + ], + [ + ["servlet-name", "cofaxEmail"], + ["servlet-class", "org.cofax.cds.EmailServlet"], + [ + "init-param", + [ + ["mailHost", "mail1"], + ["mailHostOverride", "mail2"], + ], + ], + ], + [ + ["servlet-name", "cofaxAdmin"], + ["servlet-class", "org.cofax.cds.AdminServlet"], + ], + [ + ["servlet-name", "fileServlet"], + ["servlet-class", "org.cofax.cds.FileServlet"], + ], + [ + ["servlet-name", "cofaxTools"], + [ + "servlet-class", + "org.cofax.cms.CofaxToolsServlet", + ], + [ + "init-param", + [ + ["templatePath", "toolstemplates/"], + ["log", 1], + [ + "logLocation", + "/usr/local/tomcat/logs/CofaxTools.log", + ], + ["logMaxSize", ""], + ["dataLog", 1], + [ + "dataLogLocation", + "/usr/local/tomcat/logs/dataLog.log", + ], + ["dataLogMaxSize", ""], + [ + "removePageCache", + "/content/admin/remove?cache=pages&id=", + ], + [ + "removeTemplateCache", + "/content/admin/remove?cache=templates&id=", + ], + [ + "fileTransferFolder", + "/usr/local/tomcat/webapps/content/fileTransferFolder", + ], + ["lookInContext", 1], + ["adminGroupID", 4], + ["betaServer", True], + ], + ], + ], + ], + ], + [ + "servlet-mapping", + [ + ["cofaxCDS", "/"], + ["cofaxEmail", "/cofaxutil/aemail/*"], + ["cofaxAdmin", "/admin/*"], + ["fileServlet", "/static/*"], + ["cofaxTools", "/tools/*"], + ], + ], + [ + "taglib", + [ + ["taglib-uri", "cofax.tld"], + ["taglib-location", "/WEB-INF/tlds/cofax.tld"], + ], + ], + ], + ] + ], + [ + [ + "menu", + [ + ["header", "SVG Viewer"], + [ + "items", + [ + [["id", "Open"]], + [["id", "OpenNew"], ["label", "Open New"]], + None, + [["id", "ZoomIn"], ["label", "Zoom In"]], + [["id", "ZoomOut"], ["label", "Zoom Out"]], + [["id", "OriginalView"], ["label", "Original View"]], + None, + [["id", "Quality"]], + [["id", "Pause"]], + [["id", "Mute"]], + None, + [["id", "Find"], ["label", "Find..."]], + [["id", "FindAgain"], ["label", "Find Again"]], + [["id", "Copy"]], + [["id", "CopyAgain"], ["label", "Copy Again"]], + [["id", "CopySVG"], ["label", "Copy SVG"]], + [["id", "ViewSVG"], ["label", "View SVG"]], + [["id", "ViewSource"], ["label", "View Source"]], + [["id", "SaveAs"], ["label", "Save As"]], + None, + [["id", "Help"]], + [ + ["id", "About"], + ["label", "About Adobe CVG Viewer..."], + ], + ], + ], + ], + ] + ], + ] for t, exp in zip((test1, test2, test3, test4, test5), expected): result = jsonObject.parseString(t) @@ -470,32 +700,40 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): "John Doe, 123 Main St., Cleveland, Ohio", "Jane Doe, 456 St. James St., Los Angeles , California ", "", - ] + ] testVals = [ - [(3, '100.2'), (4, ''), (5, '3')], - [(2, 'j k'), (3, 'm')], - [(0, "'Hello, World'"), (2, 'g'), (3, '')], - [(0, 'John Doe'), (1, '123 Main St.'), (2, 'Cleveland'), (3, 'Ohio')], - [(0, 'Jane Doe'), (1, '456 St. James St.'), (2, 'Los Angeles'), (3, 'California')] - ] + [(3, "100.2"), (4, ""), (5, "3")], + [(2, "j k"), (3, "m")], + [(0, "'Hello, World'"), (2, "g"), (3, "")], + [(0, "John Doe"), (1, "123 Main St."), (2, "Cleveland"), (3, "Ohio")], + [ + (0, "Jane Doe"), + (1, "456 St. James St."), + (2, "Los Angeles"), + (3, "California"), + ], + ] for line, tests in zip(testData, testVals): - print("Parsing: %r ->" % line, end=' ') + print("Parsing: %r ->" % line, end=" ") results = ppc.comma_separated_list.parseString(line) print(results) for t in tests: - if not(len(results) > t[0] and results[t[0]] == t[1]): + if not (len(results) > t[0] and results[t[0]] == t[1]): print("$$$", results.dump()) print("$$$", results[0]) - self.assertTrue(len(results) > t[0] and results[t[0]] == t[1], - "failed on %s, item %d s/b '%s', got '%s'" % (line, t[0], t[1], str(results.asList()))) + self.assertTrue( + len(results) > t[0] and results[t[0]] == t[1], + "failed on %s, item %d s/b '%s', got '%s'" + % (line, t[0], t[1], str(results.asList())), + ) def testParseEBNF(self): from examples import ebnf from pyparsing import Word, quotedString, alphas, nums - print('Constructing EBNF parser with pyparsing...') + print("Constructing EBNF parser with pyparsing...") - grammar = ''' + grammar = """ syntax = (syntax_rule), {(syntax_rule)}; syntax_rule = meta_identifier, '=', definitions_list, ';'; definitions_list = single_definition, {'|', single_definition}; @@ -513,26 +751,31 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): meta_identifier = letter, {letter | digit}; integer = digit, {digit}; *) - ''' + """ table = {} - table['terminal_string'] = quotedString - table['meta_identifier'] = Word(alphas + "_", alphas + "_" + nums) - table['integer'] = Word(nums) + table["terminal_string"] = quotedString + table["meta_identifier"] = Word(alphas + "_", alphas + "_" + nums) + table["integer"] = Word(nums) - print('Parsing EBNF grammar with EBNF parser...') + print("Parsing EBNF grammar with EBNF parser...") parsers = ebnf.parse(grammar, table) - ebnf_parser = parsers['syntax'] + ebnf_parser = parsers["syntax"] print("-", "\n- ".join(parsers.keys())) - self.assertEqual(len(list(parsers.keys())), 13, "failed to construct syntax grammar") + self.assertEqual( + len(list(parsers.keys())), 13, "failed to construct syntax grammar" + ) - print('Parsing EBNF grammar with generated EBNF parser...') + print("Parsing EBNF grammar with generated EBNF parser...") parsed_chars = ebnf_parser.parseString(grammar) parsed_char_len = len(parsed_chars) print("],\n".join(str(parsed_chars.asList()).split("],"))) - self.assertEqual(len(flatten(parsed_chars.asList())), 98, "failed to tokenize grammar correctly") - + self.assertEqual( + len(flatten(parsed_chars.asList())), + 98, + "failed to tokenize grammar correctly", + ) def testParseIDL(self): from examples import idlParse @@ -546,14 +789,28 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): tokens.pprint() tokens = flatten(tokens.asList()) print(len(tokens)) - self.assertEqual(len(tokens), numToks, "error matching IDL string, {} -> {}".format(strng, str(tokens))) + self.assertEqual( + len(tokens), + numToks, + "error matching IDL string, {} -> {}".format(strng, str(tokens)), + ) except ParseException as err: print(err.line) - print(" " * (err.column-1) + "^") + print(" " * (err.column - 1) + "^") print(err) - self.assertEqual(numToks, 0, "unexpected ParseException while parsing {}, {}".format(strng, str(err))) - self.assertEqual(err.loc, errloc, - "expected ParseException at %d, found exception at %d" % (errloc, err.loc)) + self.assertEqual( + numToks, + 0, + "unexpected ParseException while parsing {}, {}".format( + strng, str(err) + ), + ) + self.assertEqual( + err.loc, + errloc, + "expected ParseException at %d, found exception at %d" + % (errloc, err.loc), + ) test( """ @@ -569,8 +826,9 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): stringSeqSeq method2(in string arg1, inout long arg2, inout long arg3); string method3(); }; - """, 59 - ) + """, + 59, + ) test( """ /* @@ -590,8 +848,9 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): stringSeqSeq method2(in string arg1, inout long arg2, inout long arg3); string method3(); }; - """, 59 - ) + """, + 59, + ) test( r""" const string test="Test String\n"; @@ -609,8 +868,9 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): { void method1(in string arg1, inout long arg2); }; - """, 60 - ) + """, + 60, + ) test( """ module Test1 @@ -626,8 +886,10 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): raises (TestException); }; }; - """, 0, 56 - ) + """, + 0, + 56, + ) test( """ module Test1 @@ -638,14 +900,16 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): }; }; - """, 13 - ) + """, + 13, + ) def testParseVerilog(self): pass def testScanString(self): from pyparsing import Word, Combine, Suppress, CharsNotIn, nums, StringEnd + testdata = """ <table border="0" cellpadding="3" cellspacing="3" frame="" width="90%"> <tr align="left" valign="top"> @@ -684,14 +948,31 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): ipAddress = Combine(integer + "." + integer + "." + integer + "." + integer) tdStart = Suppress("<td>") tdEnd = Suppress("</td>") - timeServerPattern = (tdStart + ipAddress("ipAddr") + tdEnd - + tdStart + CharsNotIn("<")("loc") + tdEnd) - servers = [srvr.ipAddr for srvr, startloc, endloc in timeServerPattern.scanString(testdata)] + timeServerPattern = ( + tdStart + + ipAddress("ipAddr") + + tdEnd + + tdStart + + CharsNotIn("<")("loc") + + tdEnd + ) + servers = [ + srvr.ipAddr + for srvr, startloc, endloc in timeServerPattern.scanString(testdata) + ] print(servers) - self.assertEqual(servers, - ['129.6.15.28', '129.6.15.29', '132.163.4.101', '132.163.4.102', '132.163.4.103'], - "failed scanString()") + self.assertEqual( + servers, + [ + "129.6.15.28", + "129.6.15.29", + "132.163.4.101", + "132.163.4.102", + "132.163.4.103", + ], + "failed scanString()", + ) # test for stringEnd detection in scanString foundStringEnds = [r for r in StringEnd().scanString("xyzzy")] @@ -699,9 +980,14 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): self.assertTrue(foundStringEnds, "Failed to find StringEnd in scanString") def testQuotedStrings(self): - from pyparsing import sglQuotedString, dblQuotedString, quotedString, QuotedString - testData = \ - """ + from pyparsing import ( + sglQuotedString, + dblQuotedString, + quotedString, + QuotedString, + ) + + testData = """ 'a valid single quoted string' 'an invalid single quoted string because it spans lines' @@ -711,81 +997,125 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): """ print(testData) - sglStrings = [(t[0], b, e) for (t, b, e) in sglQuotedString.scanString(testData)] + sglStrings = [ + (t[0], b, e) for (t, b, e) in sglQuotedString.scanString(testData) + ] print(sglStrings) - self.assertTrue(len(sglStrings) == 1 and (sglStrings[0][1] == 17 and sglStrings[0][2] == 47), - "single quoted string failure") - - dblStrings = [(t[0], b, e) for (t, b, e) in dblQuotedString.scanString(testData)] + self.assertTrue( + len(sglStrings) == 1 + and (sglStrings[0][1] == 17 and sglStrings[0][2] == 47), + "single quoted string failure", + ) + + dblStrings = [ + (t[0], b, e) for (t, b, e) in dblQuotedString.scanString(testData) + ] print(dblStrings) - self.assertTrue(len(dblStrings) == 1 and (dblStrings[0][1] == 154 and dblStrings[0][2] == 184), - "double quoted string failure") + self.assertTrue( + len(dblStrings) == 1 + and (dblStrings[0][1] == 154 and dblStrings[0][2] == 184), + "double quoted string failure", + ) allStrings = [(t[0], b, e) for (t, b, e) in quotedString.scanString(testData)] print(allStrings) - self.assertTrue(len(allStrings) == 2 - and (allStrings[0][1] == 17 - and allStrings[0][2] == 47) - and (allStrings[1][1] == 154 - and allStrings[1][2] == 184), - "quoted string failure") - - escapedQuoteTest = \ - r""" + self.assertTrue( + len(allStrings) == 2 + and (allStrings[0][1] == 17 and allStrings[0][2] == 47) + and (allStrings[1][1] == 154 and allStrings[1][2] == 184), + "quoted string failure", + ) + + escapedQuoteTest = r""" 'This string has an escaped (\') quote character' "This string has an escaped (\") quote character" """ - sglStrings = [(t[0], b, e) for (t, b, e) in sglQuotedString.scanString(escapedQuoteTest)] + sglStrings = [ + (t[0], b, e) for (t, b, e) in sglQuotedString.scanString(escapedQuoteTest) + ] print(sglStrings) - self.assertTrue(len(sglStrings) == 1 and (sglStrings[0][1] == 17 and sglStrings[0][2] == 66), - "single quoted string escaped quote failure (%s)" % str(sglStrings[0])) - - dblStrings = [(t[0], b, e) for (t, b, e) in dblQuotedString.scanString(escapedQuoteTest)] + self.assertTrue( + len(sglStrings) == 1 + and (sglStrings[0][1] == 17 and sglStrings[0][2] == 66), + "single quoted string escaped quote failure (%s)" % str(sglStrings[0]), + ) + + dblStrings = [ + (t[0], b, e) for (t, b, e) in dblQuotedString.scanString(escapedQuoteTest) + ] print(dblStrings) - self.assertTrue(len(dblStrings) == 1 and (dblStrings[0][1] == 83 and dblStrings[0][2] == 132), - "double quoted string escaped quote failure (%s)" % str(dblStrings[0])) - - allStrings = [(t[0], b, e) for (t, b, e) in quotedString.scanString(escapedQuoteTest)] + self.assertTrue( + len(dblStrings) == 1 + and (dblStrings[0][1] == 83 and dblStrings[0][2] == 132), + "double quoted string escaped quote failure (%s)" % str(dblStrings[0]), + ) + + allStrings = [ + (t[0], b, e) for (t, b, e) in quotedString.scanString(escapedQuoteTest) + ] print(allStrings) - self.assertTrue(len(allStrings) == 2 - and (allStrings[0][1] == 17 - and allStrings[0][2] == 66 - and allStrings[1][1] == 83 - and allStrings[1][2] == 132), - "quoted string escaped quote failure (%s)" % ([str(s[0]) for s in allStrings])) - - dblQuoteTest = \ - r""" + self.assertTrue( + len(allStrings) == 2 + and ( + allStrings[0][1] == 17 + and allStrings[0][2] == 66 + and allStrings[1][1] == 83 + and allStrings[1][2] == 132 + ), + "quoted string escaped quote failure (%s)" + % ([str(s[0]) for s in allStrings]), + ) + + dblQuoteTest = r""" 'This string has an doubled ('') quote character' "This string has an doubled ("") quote character" """ - sglStrings = [(t[0], b, e) for (t, b, e) in sglQuotedString.scanString(dblQuoteTest)] + sglStrings = [ + (t[0], b, e) for (t, b, e) in sglQuotedString.scanString(dblQuoteTest) + ] print(sglStrings) - self.assertTrue(len(sglStrings) == 1 and (sglStrings[0][1] == 17 and sglStrings[0][2] == 66), - "single quoted string escaped quote failure (%s)" % str(sglStrings[0])) - dblStrings = [(t[0], b, e) for (t, b, e) in dblQuotedString.scanString(dblQuoteTest)] + self.assertTrue( + len(sglStrings) == 1 + and (sglStrings[0][1] == 17 and sglStrings[0][2] == 66), + "single quoted string escaped quote failure (%s)" % str(sglStrings[0]), + ) + dblStrings = [ + (t[0], b, e) for (t, b, e) in dblQuotedString.scanString(dblQuoteTest) + ] print(dblStrings) - self.assertTrue(len(dblStrings) == 1 and (dblStrings[0][1] == 83 and dblStrings[0][2] == 132), - "double quoted string escaped quote failure (%s)" % str(dblStrings[0])) - allStrings = [(t[0], b, e) for (t, b, e) in quotedString.scanString(dblQuoteTest)] + self.assertTrue( + len(dblStrings) == 1 + and (dblStrings[0][1] == 83 and dblStrings[0][2] == 132), + "double quoted string escaped quote failure (%s)" % str(dblStrings[0]), + ) + allStrings = [ + (t[0], b, e) for (t, b, e) in quotedString.scanString(dblQuoteTest) + ] print(allStrings) - self.assertTrue(len(allStrings) == 2 - and (allStrings[0][1] == 17 - and allStrings[0][2] == 66 - and allStrings[1][1] == 83 - and allStrings[1][2] == 132), - "quoted string escaped quote failure (%s)" % ([str(s[0]) for s in allStrings])) - - print("testing catastrophic RE backtracking in implementation of dblQuotedString") + self.assertTrue( + len(allStrings) == 2 + and ( + allStrings[0][1] == 17 + and allStrings[0][2] == 66 + and allStrings[1][1] == 83 + and allStrings[1][2] == 132 + ), + "quoted string escaped quote failure (%s)" + % ([str(s[0]) for s in allStrings]), + ) + + print( + "testing catastrophic RE backtracking in implementation of dblQuotedString" + ) for expr, test_string in [ - (dblQuotedString, '"' + '\\xff' * 500), - (sglQuotedString, "'" + '\\xff' * 500), - (quotedString, '"' + '\\xff' * 500), - (quotedString, "'" + '\\xff' * 500), - (QuotedString('"'), '"' + '\\xff' * 500), - (QuotedString("'"), "'" + '\\xff' * 500), - ]: + (dblQuotedString, '"' + "\\xff" * 500), + (sglQuotedString, "'" + "\\xff" * 500), + (quotedString, '"' + "\\xff" * 500), + (quotedString, "'" + "\\xff" * 500), + (QuotedString('"'), '"' + "\\xff" * 500), + (QuotedString("'"), "'" + "\\xff" * 500), + ]: expr.parseString(test_string + test_string[0]) try: expr.parseString(test_string) @@ -801,19 +1131,28 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): caseless2 = oneOf("d a b c Aa B A C", caseless=True) caseless2str = str(caseless2) print(caseless2str) - self.assertEqual(caseless1str.upper(), caseless2str.upper(), "oneOf not handling caseless option properly") - self.assertNotEqual(caseless1str, caseless2str, "Caseless option properly sorted") + self.assertEqual( + caseless1str.upper(), + caseless2str.upper(), + "oneOf not handling caseless option properly", + ) + self.assertNotEqual( + caseless1str, caseless2str, "Caseless option properly sorted" + ) res = caseless1[...].parseString("AAaaAaaA") print(res) self.assertEqual(len(res), 4, "caseless1 oneOf failed") - self.assertEqual("".join(res), "aA" * 4, "caseless1 CaselessLiteral return failed") + self.assertEqual( + "".join(res), "aA" * 4, "caseless1 CaselessLiteral return failed" + ) - res =caseless2[...].parseString("AAaaAaaA") + res = caseless2[...].parseString("AAaaAaaA") print(res) self.assertEqual(len(res), 4, "caseless2 oneOf failed") - self.assertEqual("".join(res), "Aa" * 4, "caseless1 CaselessLiteral return failed") - + self.assertEqual( + "".join(res), "Aa" * 4, "caseless1 CaselessLiteral return failed" + ) def testCommentParser(self): @@ -831,9 +1170,14 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): ablsjdflj */ """ - foundLines = [pp.lineno(s, testdata) - for t, s, e in pp.cStyleComment.scanString(testdata)] - self.assertEqual(foundLines, list(range(11))[2:], "only found C comments on lines " + str(foundLines)) + foundLines = [ + pp.lineno(s, testdata) for t, s, e in pp.cStyleComment.scanString(testdata) + ] + self.assertEqual( + foundLines, + list(range(11))[2:], + "only found C comments on lines " + str(foundLines), + ) testdata = """ <!-- --> <!--- ---> @@ -849,9 +1193,14 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): ablsjdflj --> """ - foundLines = [pp.lineno(s, testdata) - for t, s, e in pp.htmlComment.scanString(testdata)] - self.assertEqual(foundLines, list(range(11))[2:], "only found HTML comments on lines " + str(foundLines)) + foundLines = [ + pp.lineno(s, testdata) for t, s, e in pp.htmlComment.scanString(testdata) + ] + self.assertEqual( + foundLines, + list(range(11))[2:], + "only found HTML comments on lines " + str(foundLines), + ) # test C++ single line comments that have line terminated with '\' (should continue comment to following line) testSource = r""" @@ -860,8 +1209,11 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): still comment 2 // comment 3 """ - self.assertEqual(len(pp.cppStyleComment.searchString(testSource)[1][0]), 41, - r"failed to match single-line comment with '\' at EOL") + self.assertEqual( + len(pp.cppStyleComment.searchString(testSource)[1][0]), + 41, + r"failed to match single-line comment with '\' at EOL", + ) def testParseExpressionResults(self): from pyparsing import Word, alphas, OneOrMore, Optional, Group @@ -875,16 +1227,18 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): words = Group(OneOrMore(~a + word)).setName("words") - phrase = (words("Head") - + Group(a + Optional(b + Optional(c)))("ABC") - + words("Tail")) + phrase = ( + words("Head") + Group(a + Optional(b + Optional(c)))("ABC") + words("Tail") + ) results = phrase.parseString("xavier yeti alpha beta charlie will beaver") print(results, results.Head, results.ABC, results.Tail) for key, ln in [("Head", 2), ("ABC", 3), ("Tail", 2)]: - self.assertEqual(len(results[key]), ln, - "expected %d elements in %s, found %s" % (ln, key, str(results[key]))) - + self.assertEqual( + len(results[key]), + ln, + "expected %d elements in %s, found %s" % (ln, key, str(results[key])), + ) def testParseKeyword(self): from pyparsing import Literal, Keyword @@ -894,24 +1248,28 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): def test(s, litShouldPass, kwShouldPass): print("Test", s) - print("Match Literal", end=' ') + print("Match Literal", end=" ") try: print(lit.parseString(s)) except Exception: print("failed") if litShouldPass: - self.assertTrue(False, "Literal failed to match %s, should have" % s) + self.assertTrue( + False, "Literal failed to match %s, should have" % s + ) else: if not litShouldPass: self.assertTrue(False, "Literal matched %s, should not have" % s) - print("Match Keyword", end=' ') + print("Match Keyword", end=" ") try: print(kw.parseString(s)) except Exception: print("failed") if kwShouldPass: - self.assertTrue(False, "Keyword failed to match %s, should have" % s) + self.assertTrue( + False, "Keyword failed to match %s, should have" % s + ) else: if not kwShouldPass: self.assertTrue(False, "Keyword matched %s, should not have" % s) @@ -926,47 +1284,62 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): test("If(OnlyIfOnly)", False, True) test("iF (OnlyIf Only)", False, True) - - def testParseExpressionResultsAccumulate(self): from pyparsing import Word, delimitedList, Combine, alphas, nums - num=Word(nums).setName("num")("base10*") - hexnum=Combine("0x"+ Word(nums)).setName("hexnum")("hex*") + num = Word(nums).setName("num")("base10*") + hexnum = Combine("0x" + Word(nums)).setName("hexnum")("hex*") name = Word(alphas).setName("word")("word*") - list_of_num=delimitedList(hexnum | num | name, ",") + list_of_num = delimitedList(hexnum | num | name, ",") - tokens = list_of_num.parseString('1, 0x2, 3, 0x4, aaa') + tokens = list_of_num.parseString("1, 0x2, 3, 0x4, aaa") print(tokens.dump()) - self.assertParseResultsEquals(tokens, - expected_list=['1', '0x2', '3', '0x4', 'aaa'], - expected_dict={'base10': ['1', '3'], - 'hex': ['0x2', '0x4'], - 'word': ['aaa']}, - ) - - from pyparsing import Literal, Word, nums, Group, Dict, alphas, \ - quotedString, oneOf, delimitedList, removeQuotes, alphanums + self.assertParseResultsEquals( + tokens, + expected_list=["1", "0x2", "3", "0x4", "aaa"], + expected_dict={ + "base10": ["1", "3"], + "hex": ["0x2", "0x4"], + "word": ["aaa"], + }, + ) + + from pyparsing import ( + Literal, + Word, + nums, + Group, + Dict, + alphas, + quotedString, + oneOf, + delimitedList, + removeQuotes, + alphanums, + ) lbrack = Literal("(").suppress() rbrack = Literal(")").suppress() integer = Word(nums).setName("int") variable = Word(alphas, max=1).setName("variable") - relation_body_item = variable | integer | quotedString.copy().setParseAction(removeQuotes) + relation_body_item = ( + variable | integer | quotedString.copy().setParseAction(removeQuotes) + ) relation_name = Word(alphas + "_", alphanums + "_") relation_body = lbrack + Group(delimitedList(relation_body_item)) + rbrack Goal = Dict(Group(relation_name + relation_body)) Comparison_Predicate = Group(variable + oneOf("< >") + integer)("pred*") Query = Goal("head") + ":-" + delimitedList(Goal | Comparison_Predicate) - test="""Q(x,y,z):-Bloo(x,"Mitsis",y),Foo(y,z,1243),y>28,x<12,x>3""" + test = """Q(x,y,z):-Bloo(x,"Mitsis",y),Foo(y,z,1243),y>28,x<12,x>3""" queryRes = Query.parseString(test) print(queryRes.dump()) - self.assertParseResultsEquals(queryRes.pred, - expected_list=[['y', '>', '28'], ['x', '<', '12'], ['x', '>', '3']], - msg="Incorrect list for attribute pred, %s" % str(queryRes.pred.asList()) - ) + self.assertParseResultsEquals( + queryRes.pred, + expected_list=[["y", ">", "28"], ["x", "<", "12"], ["x", ">", "3"]], + msg="Incorrect list for attribute pred, %s" % str(queryRes.pred.asList()), + ) def testReStringRange(self): testCases = ( @@ -990,8 +1363,8 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): (r"[A-]"), (r"[-A]"), (r"[\x21]"), - (r"[а-яА-ЯёЁA-Z$_\041α-ω]") - ) + (r"[а-яА-ЯёЁA-Z$_\041α-ω]"), + ) expectedResults = ( "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "A", @@ -1014,108 +1387,169 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): "-A", "!", "абвгдежзийклмнопрстуфхцчшщъыьэюяАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯёЁABCDEFGHIJKLMNOPQRSTUVWXYZ$_!αβγδεζηθικλμνξοπρςστυφχψω", - ) + ) for test in zip(testCases, expectedResults): t, exp = test res = pp.srange(t) - #print(t, "->", res) - self.assertEqual(res, exp, "srange error, srange({!r})->'{!r}', expected '{!r}'".format(t, res, exp)) + # print(t, "->", res) + self.assertEqual( + res, + exp, + "srange error, srange({!r})->'{!r}', expected '{!r}'".format( + t, res, exp + ), + ) def testSkipToParserTests(self): from pyparsing import Literal, SkipTo, cStyleComment, ParseBaseException - thingToFind = Literal('working') - testExpr = SkipTo(Literal(';'), include=True, ignore=cStyleComment) + thingToFind + thingToFind = Literal("working") + testExpr = ( + SkipTo(Literal(";"), include=True, ignore=cStyleComment) + thingToFind + ) - def tryToParse (someText, fail_expected=False): + def tryToParse(someText, fail_expected=False): try: print(testExpr.parseString(someText)) - self.assertFalse(fail_expected, "expected failure but no exception raised") + self.assertFalse( + fail_expected, "expected failure but no exception raised" + ) except Exception as e: print("Exception {} while parsing string {}".format(e, repr(someText))) - self.assertTrue(fail_expected and isinstance(e, ParseBaseException), - "Exception {} while parsing string {}".format(e, repr(someText))) + self.assertTrue( + fail_expected and isinstance(e, ParseBaseException), + "Exception {} while parsing string {}".format(e, repr(someText)), + ) # This first test works, as the SkipTo expression is immediately following the ignore expression (cStyleComment) - tryToParse('some text /* comment with ; in */; working') + tryToParse("some text /* comment with ; in */; working") # This second test previously failed, as there is text following the ignore expression, and before the SkipTo expression. - tryToParse('some text /* comment with ; in */some other stuff; working') + tryToParse("some text /* comment with ; in */some other stuff; working") # tests for optional failOn argument - testExpr = SkipTo(Literal(';'), include=True, ignore=cStyleComment, failOn='other') + thingToFind - tryToParse('some text /* comment with ; in */; working') - tryToParse('some text /* comment with ; in */some other stuff; working', fail_expected=True) + testExpr = ( + SkipTo(Literal(";"), include=True, ignore=cStyleComment, failOn="other") + + thingToFind + ) + tryToParse("some text /* comment with ; in */; working") + tryToParse( + "some text /* comment with ; in */some other stuff; working", + fail_expected=True, + ) # test that we correctly create named results text = "prefixDATAsuffix" data = Literal("DATA") suffix = Literal("suffix") - expr = SkipTo(data + suffix)('prefix') + data + suffix + expr = SkipTo(data + suffix)("prefix") + data + suffix result = expr.parseString(text) - self.assertTrue(isinstance(result.prefix, str), "SkipTo created with wrong saveAsList attribute") + self.assertTrue( + isinstance(result.prefix, str), + "SkipTo created with wrong saveAsList attribute", + ) from pyparsing import Literal, And, Word, alphas, nums + alpha_word = (~Literal("end") + Word(alphas, asKeyword=True)).setName("alpha") num_word = Word(nums, asKeyword=True).setName("int") def test(expr, test_string, expected_list, expected_dict): if (expected_list, expected_dict) == (None, None): - with self.assertRaises(Exception, msg="{} failed to parse {!r}".format(expr, test_string)): + with self.assertRaises( + Exception, msg="{} failed to parse {!r}".format(expr, test_string) + ): expr.parseString(test_string) else: result = expr.parseString(test_string) - self.assertParseResultsEquals(result, expected_list=expected_list, expected_dict=expected_dict) + self.assertParseResultsEquals( + result, expected_list=expected_list, expected_dict=expected_dict + ) # ellipses for SkipTo e = ... + Literal("end") - test(e, "start 123 end", ['start 123 ', 'end'], {'_skipped': ['start 123 ']}) + test(e, "start 123 end", ["start 123 ", "end"], {"_skipped": ["start 123 "]}) e = Literal("start") + ... + Literal("end") - test(e, "start 123 end", ['start', '123 ', 'end'], {'_skipped': ['123 ']}) + test(e, "start 123 end", ["start", "123 ", "end"], {"_skipped": ["123 "]}) e = Literal("start") + ... test(e, "start 123 end", None, None) e = And(["start", ..., "end"]) - test(e, "start 123 end", ['start', '123 ', 'end'], {'_skipped': ['123 ']}) + test(e, "start 123 end", ["start", "123 ", "end"], {"_skipped": ["123 "]}) e = And([..., "end"]) - test(e, "start 123 end", ['start 123 ', 'end'], {'_skipped': ['start 123 ']}) + test(e, "start 123 end", ["start 123 ", "end"], {"_skipped": ["start 123 "]}) e = "start" + (num_word | ...) + "end" - test(e, "start 456 end", ['start', '456', 'end'], {}) - test(e, "start 123 456 end", ['start', '123', '456 ', 'end'], {'_skipped': ['456 ']}) - test(e, "start end", ['start', '', 'end'], {'_skipped': ['missing <int>']}) + test(e, "start 456 end", ["start", "456", "end"], {}) + test( + e, + "start 123 456 end", + ["start", "123", "456 ", "end"], + {"_skipped": ["456 "]}, + ) + test(e, "start end", ["start", "", "end"], {"_skipped": ["missing <int>"]}) # e = define_expr('"start" + (num_word | ...)("inner") + "end"') # test(e, "start 456 end", ['start', '456', 'end'], {'inner': '456'}) e = "start" + (alpha_word[...] & num_word[...] | ...) + "end" - test(e, "start 456 red end", ['start', '456', 'red', 'end'], {}) - test(e, "start red 456 end", ['start', 'red', '456', 'end'], {}) - test(e, "start 456 red + end", ['start', '456', 'red', '+ ', 'end'], {'_skipped': ['+ ']}) - test(e, "start red end", ['start', 'red', 'end'], {}) - test(e, "start 456 end", ['start', '456', 'end'], {}) - test(e, "start end", ['start', 'end'], {}) - test(e, "start 456 + end", ['start', '456', '+ ', 'end'], {'_skipped': ['+ ']}) + test(e, "start 456 red end", ["start", "456", "red", "end"], {}) + test(e, "start red 456 end", ["start", "red", "456", "end"], {}) + test( + e, + "start 456 red + end", + ["start", "456", "red", "+ ", "end"], + {"_skipped": ["+ "]}, + ) + test(e, "start red end", ["start", "red", "end"], {}) + test(e, "start 456 end", ["start", "456", "end"], {}) + test(e, "start end", ["start", "end"], {}) + test(e, "start 456 + end", ["start", "456", "+ ", "end"], {"_skipped": ["+ "]}) e = "start" + (alpha_word[1, ...] & num_word[1, ...] | ...) + "end" - test(e, "start 456 red end", ['start', '456', 'red', 'end'], {}) - test(e, "start red 456 end", ['start', 'red', '456', 'end'], {}) - test(e, "start 456 red + end", ['start', '456', 'red', '+ ', 'end'], {'_skipped': ['+ ']}) - test(e, "start red end", ['start', 'red ', 'end'], {'_skipped': ['red ']}) - test(e, "start 456 end", ['start', '456 ', 'end'], {'_skipped': ['456 ']}) - test(e, "start end", ['start', '', 'end'], {'_skipped': ['missing <{{alpha}... & {int}...}>']}) - test(e, "start 456 + end", ['start', '456 + ', 'end'], {'_skipped': ['456 + ']}) + test(e, "start 456 red end", ["start", "456", "red", "end"], {}) + test(e, "start red 456 end", ["start", "red", "456", "end"], {}) + test( + e, + "start 456 red + end", + ["start", "456", "red", "+ ", "end"], + {"_skipped": ["+ "]}, + ) + test(e, "start red end", ["start", "red ", "end"], {"_skipped": ["red "]}) + test(e, "start 456 end", ["start", "456 ", "end"], {"_skipped": ["456 "]}) + test( + e, + "start end", + ["start", "", "end"], + {"_skipped": ["missing <{{alpha}... & {int}...}>"]}, + ) + test(e, "start 456 + end", ["start", "456 + ", "end"], {"_skipped": ["456 + "]}) e = "start" + (alpha_word | ...) + (num_word | ...) + "end" - test(e, "start red 456 end", ['start', 'red', '456', 'end'], {}) - test(e, "start red end", ['start', 'red', '', 'end'], {'_skipped': ['missing <int>']}) - test(e, "start end", ['start', '', '', 'end'], {'_skipped': ['missing <alpha>', 'missing <int>']}) + test(e, "start red 456 end", ["start", "red", "456", "end"], {}) + test( + e, + "start red end", + ["start", "red", "", "end"], + {"_skipped": ["missing <int>"]}, + ) + test( + e, + "start end", + ["start", "", "", "end"], + {"_skipped": ["missing <alpha>", "missing <int>"]}, + ) e = Literal("start") + ... + "+" + ... + "end" - test(e, "start red + 456 end", ['start', 'red ', '+', '456 ', 'end'], {'_skipped': ['red ', '456 ']}) + test( + e, + "start red + 456 end", + ["start", "red ", "+", "456 ", "end"], + {"_skipped": ["red ", "456 "]}, + ) def testEllipsisRepetion(self): import pyparsing as pp @@ -1168,59 +1602,74 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): label = pp.Word(pp.alphas) val = pp.pyparsing_common.integer() - parser = label('label') + pp.ZeroOrMore(val)('values') + parser = label("label") + pp.ZeroOrMore(val)("values") - _, results = parser.runTests(""" + _, results = parser.runTests( + """ a 1 b 1 2 3 c - """) + """ + ) expected = [ - (['a', 1], {'label': 'a', 'values': [1]}), - (['b', 1, 2, 3], {'label': 'b', 'values': [1, 2, 3]}), - (['c'], {'label': 'c', 'values': []}), + (["a", 1], {"label": "a", "values": [1]}), + (["b", 1, 2, 3], {"label": "b", "values": [1, 2, 3]}), + (["c"], {"label": "c", "values": []}), ] for obs, exp in zip(results, expected): test, result = obs exp_list, exp_dict = exp - self.assertParseResultsEquals(result, expected_list=exp_list, expected_dict=exp_dict) + self.assertParseResultsEquals( + result, expected_list=exp_list, expected_dict=exp_dict + ) - parser = label('label') + val[...]('values') + parser = label("label") + val[...]("values") - _, results = parser.runTests(""" + _, results = parser.runTests( + """ a 1 b 1 2 3 c - """) + """ + ) expected = [ - (['a', 1], {'label': 'a', 'values': [1]}), - (['b', 1, 2, 3], {'label': 'b', 'values': [1, 2, 3]}), - (['c'], {'label': 'c', 'values': []}), + (["a", 1], {"label": "a", "values": [1]}), + (["b", 1, 2, 3], {"label": "b", "values": [1, 2, 3]}), + (["c"], {"label": "c", "values": []}), ] for obs, exp in zip(results, expected): test, result = obs exp_list, exp_dict = exp - self.assertParseResultsEquals(result, expected_list=exp_list, expected_dict=exp_dict) + self.assertParseResultsEquals( + result, expected_list=exp_list, expected_dict=exp_dict + ) - pt = pp.Group(val('x') + pp.Suppress(',') + val('y')) - parser = label('label') + pt[...]("points") - _, results = parser.runTests(""" + pt = pp.Group(val("x") + pp.Suppress(",") + val("y")) + parser = label("label") + pt[...]("points") + _, results = parser.runTests( + """ a 1,1 b 1,1 2,2 3,3 c - """) + """ + ) expected = [ - (['a', [1, 1]], {'label': 'a', 'points': [{'x': 1, 'y': 1}]}), - (['b', [1, 1], [2, 2], [3, 3]], {'label': 'b', 'points': [{'x': 1, 'y': 1}, - {'x': 2, 'y': 2}, - {'x': 3, 'y': 3}]}), - (['c'], {'label': 'c', 'points': []}), + (["a", [1, 1]], {"label": "a", "points": [{"x": 1, "y": 1}]}), + ( + ["b", [1, 1], [2, 2], [3, 3]], + { + "label": "b", + "points": [{"x": 1, "y": 1}, {"x": 2, "y": 2}, {"x": 3, "y": 3}], + }, + ), + (["c"], {"label": "c", "points": []}), ] for obs, exp in zip(results, expected): test, result = obs exp_list, exp_dict = exp - self.assertParseResultsEquals(result, expected_list=exp_list, expected_dict=exp_dict) - + self.assertParseResultsEquals( + result, expected_list=exp_list, expected_dict=exp_dict + ) def testCustomQuotes(self): from pyparsing import QuotedString @@ -1233,41 +1682,55 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): sdlfjs ^^^==sdf\:j=lz::--djf: sl=^^=kfsjf sdlfjs ==sdf\:j=ls::--djf: sl==kfsjf^^^ """ - colonQuotes = QuotedString(':', '\\', '::') - dashQuotes = QuotedString('-', '\\', '--') - hatQuotes = QuotedString('^', '\\') - hatQuotes1 = QuotedString('^', '\\', '^^') - dblEqQuotes = QuotedString('==', '\\') + colonQuotes = QuotedString(":", "\\", "::") + dashQuotes = QuotedString("-", "\\", "--") + hatQuotes = QuotedString("^", "\\") + hatQuotes1 = QuotedString("^", "\\", "^^") + dblEqQuotes = QuotedString("==", "\\") def test(quoteExpr, expected): print(quoteExpr.pattern) print(quoteExpr.searchString(testString)) print(quoteExpr.searchString(testString)[0][0]) print(expected) - self.assertEqual(quoteExpr.searchString(testString)[0][0], - expected, - "failed to match {}, expected '{}', got '{}'".format(quoteExpr, expected, - quoteExpr.searchString(testString)[0])) + self.assertEqual( + quoteExpr.searchString(testString)[0][0], + expected, + "failed to match {}, expected '{}', got '{}'".format( + quoteExpr, expected, quoteExpr.searchString(testString)[0] + ), + ) print() test(colonQuotes, r"sdf:jls:djf") - test(dashQuotes, r"sdf:jls::-djf: sl") - test(hatQuotes, r"sdf:jls") - test(hatQuotes1, r"sdf:jls^--djf") + test(dashQuotes, r"sdf:jls::-djf: sl") + test(hatQuotes, r"sdf:jls") + test(hatQuotes1, r"sdf:jls^--djf") test(dblEqQuotes, r"sdf:j=ls::--djf: sl") - test(QuotedString(':::'), 'jls::--djf: sl') - test(QuotedString('==', endQuoteChar='--'), r'sdf\:j=lz::') - test(QuotedString('^^^', multiline=True), r"""==sdf\:j=lz::--djf: sl=^^=kfsjf - sdlfjs ==sdf\:j=ls::--djf: sl==kfsjf""") + test(QuotedString(":::"), "jls::--djf: sl") + test(QuotedString("==", endQuoteChar="--"), r"sdf\:j=lz::") + test( + QuotedString("^^^", multiline=True), + r"""==sdf\:j=lz::--djf: sl=^^=kfsjf + sdlfjs ==sdf\:j=ls::--djf: sl==kfsjf""", + ) try: - bad1 = QuotedString('', '\\') + bad1 = QuotedString("", "\\") except SyntaxError as se: pass else: - self.assertTrue(False, "failed to raise SyntaxError with empty quote string") + self.assertTrue( + False, "failed to raise SyntaxError with empty quote string" + ) def testRepeater(self): - from pyparsing import matchPreviousLiteral, matchPreviousExpr, Word, nums, ParserElement + from pyparsing import ( + matchPreviousLiteral, + matchPreviousExpr, + Word, + nums, + ParserElement, + ) if ParserElement._packratEnabled: print("skipping this test, not compatible with packratting") @@ -1294,7 +1757,11 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): found = True if not found: print("No literal match in", tst) - self.assertEqual(found, result, "Failed repeater for test: {}, matching {}".format(tst, str(seq))) + self.assertEqual( + found, + result, + "Failed repeater for test: {}, matching {}".format(tst, str(seq)), + ) print() # retest using matchPreviousExpr instead of matchPreviousLiteral @@ -1305,7 +1772,7 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): ("abc12abc", True), ("abc12cba", False), ("abc12abcdef", False), - ] + ] for tst, result in tests: found = False @@ -1314,7 +1781,11 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): found = True if not found: print("No expression match in", tst) - self.assertEqual(found, result, "Failed repeater for test: {}, matching {}".format(tst, str(seq))) + self.assertEqual( + found, + result, + "Failed repeater for test: {}, matching {}".format(tst, str(seq)), + ) print() @@ -1332,7 +1803,7 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): ("abc12abc:abc12abc", True), ("abc12cba:abc12abc", False), ("abc12abc:abc12abcdef", False), - ] + ] for tst, result in tests: found = False @@ -1342,7 +1813,11 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): break if not found: print("No expression match in", tst) - self.assertEqual(found, result, "Failed repeater for test: {}, matching {}".format(tst, str(seq))) + self.assertEqual( + found, + result, + "Failed repeater for test: {}, matching {}".format(tst, str(seq)), + ) print() eFirst = Word(nums) @@ -1352,7 +1827,7 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): tests = [ ("1:1A", True), ("1:10", False), - ] + ] for tst, result in tests: found = False @@ -1361,19 +1836,25 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): found = True if not found: print("No match in", tst) - self.assertEqual(found, result, "Failed repeater for test: {}, matching {}".format(tst, str(seq))) + self.assertEqual( + found, + result, + "Failed repeater for test: {}, matching {}".format(tst, str(seq)), + ) def testRecursiveCombine(self): from pyparsing import Forward, Word, alphas, nums, Optional, Combine testInput = "myc(114)r(11)dd" - Stream=Forward() + Stream = Forward() Stream << Optional(Word(alphas)) + Optional("(" + Word(nums) + ")" + Stream) - expected = [''.join(Stream.parseString(testInput))] + expected = ["".join(Stream.parseString(testInput))] print(expected) - Stream=Forward() - Stream << Combine(Optional(Word(alphas)) + Optional("(" + Word(nums) + ")" + Stream)) + Stream = Forward() + Stream << Combine( + Optional(Word(alphas)) + Optional("(" + Word(nums) + ")" + Stream) + ) testVal = Stream.parseString(testInput) print(testVal) @@ -1383,35 +1864,40 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): from pyparsing import Word, nums, alphas, Literal, oneOf, infixNotation, opAssoc import ast - integer = Word(nums).setParseAction(lambda t:int(t[0])) + integer = Word(nums).setParseAction(lambda t: int(t[0])) variable = Word(alphas, exact=1) operand = integer | variable - expop = Literal('^') - signop = oneOf('+ -') - multop = oneOf('* /') - plusop = oneOf('+ -') - factop = Literal('!') - - expr = infixNotation(operand, - [(factop, 1, opAssoc.LEFT), - (expop, 2, opAssoc.RIGHT), - (signop, 1, opAssoc.RIGHT), - (multop, 2, opAssoc.LEFT), - (plusop, 2, opAssoc.LEFT),] - ) - - test = ["9 + 2 + 3", - "9 + 2 * 3", - "(9 + 2) * 3", - "(9 + -2) * 3", - "(9 + --2) * 3", - "(9 + -2) * 3^2^2", - "(9! + -2) * 3^2^2", - "M*X + B", - "M*(X + B)", - "1+2*-3^4*5+-+-6", - "3!!"] + expop = Literal("^") + signop = oneOf("+ -") + multop = oneOf("* /") + plusop = oneOf("+ -") + factop = Literal("!") + + expr = infixNotation( + operand, + [ + (factop, 1, opAssoc.LEFT), + (expop, 2, opAssoc.RIGHT), + (signop, 1, opAssoc.RIGHT), + (multop, 2, opAssoc.LEFT), + (plusop, 2, opAssoc.LEFT), + ], + ) + + test = [ + "9 + 2 + 3", + "9 + 2 * 3", + "(9 + 2) * 3", + "(9 + -2) * 3", + "(9 + --2) * 3", + "(9 + -2) * 3^2^2", + "(9! + -2) * 3^2^2", + "M*X + B", + "M*(X + B)", + "1+2*-3^4*5+-+-6", + "3!!", + ] expected = """[[9, '+', 2, '+', 3]] [[9, '+', [2, '*', 3]]] [[[9, '+', 2], '*', 3]] @@ -1422,30 +1908,39 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): [[['M', '*', 'X'], '+', 'B']] [['M', '*', ['X', '+', 'B']]] [[1, '+', [2, '*', ['-', [3, '^', 4]], '*', 5], '+', ['-', ['+', ['-', 6]]]]] - [[3, '!', '!']]""".split('\n') + [[3, '!', '!']]""".split( + "\n" + ) expected = [ast.literal_eval(x.strip()) for x in expected] for test_str, exp_list in zip(test, expected): result = expr.parseString(test_str) print(test_str, "->", exp_list, "got", result) - self.assertParseResultsEquals(result, expected_list=exp_list, - msg="mismatched results for infixNotation: got %s, expected %s" % - (result.asList(), exp_list)) + self.assertParseResultsEquals( + result, + expected_list=exp_list, + msg="mismatched results for infixNotation: got %s, expected %s" + % (result.asList(), exp_list), + ) def testInfixNotationGrammarTest2(self): from pyparsing import infixNotation, Word, alphas, oneOf, opAssoc - boolVars = { "True":True, "False":False } + boolVars = {"True": True, "False": False} + class BoolOperand: - reprsymbol = '' + reprsymbol = "" + def __init__(self, t): self.args = t[0][0::2] + def __str__(self): sep = " %s " % self.reprsymbol return "(" + sep.join(map(str, self.args)) + ")" class BoolAnd(BoolOperand): - reprsymbol = '&' + reprsymbol = "&" + def __bool__(self): for a in self.args: if isinstance(a, str): @@ -1457,7 +1952,8 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): return True class BoolOr(BoolOperand): - reprsymbol = '|' + reprsymbol = "|" + def __bool__(self): for a in self.args: if isinstance(a, str): @@ -1471,8 +1967,10 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): class BoolNot(BoolOperand): def __init__(self, t): self.arg = t[0][1] + def __str__(self): return "~" + str(self.arg) + def __bool__(self): if isinstance(self.arg, str): v = boolVars[self.arg] @@ -1481,22 +1979,25 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): return not v boolOperand = Word(alphas, max=1) | oneOf("True False") - boolExpr = infixNotation(boolOperand, + boolExpr = infixNotation( + boolOperand, [ - ("not", 1, opAssoc.RIGHT, BoolNot), - ("and", 2, opAssoc.LEFT, BoolAnd), - ("or", 2, opAssoc.LEFT, BoolOr), - ]) - test = ["p and not q", - "not not p", - "not(p and q)", - "q or not p and r", - "q or not p or not r", - "q or not (p and r)", - "p or q or r", - "p or q or r and False", - "(p or q or r) and False", - ] + ("not", 1, opAssoc.RIGHT, BoolNot), + ("and", 2, opAssoc.LEFT, BoolAnd), + ("or", 2, opAssoc.LEFT, BoolOr), + ], + ) + test = [ + "p and not q", + "not not p", + "not(p and q)", + "q or not p and r", + "q or not p or not r", + "q or not (p and r)", + "p or q or r", + "p or q or r and False", + "(p or q or r) and False", + ] boolVars["p"] = True boolVars["q"] = False @@ -1507,11 +2008,10 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): print() for t in test: res = boolExpr.parseString(t) - print(t, '\n', res[0], '=', bool(res[0]), '\n') + print(t, "\n", res[0], "=", bool(res[0]), "\n") expected = eval(t, {}, boolVars) self.assertEquals(expected, bool(res[0])) - def testInfixNotationGrammarTest3(self): from pyparsing import infixNotation, Word, alphas, oneOf, opAssoc, nums, Literal @@ -1530,20 +2030,22 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): variable = Word(alphas, exact=1) operand = integer | variable - expop = Literal('^') - signop = oneOf('+ -') - multop = oneOf('* /') - plusop = oneOf('+ -') - factop = Literal('!') + expop = Literal("^") + signop = oneOf("+ -") + multop = oneOf("* /") + plusop = oneOf("+ -") + factop = Literal("!") - expr = infixNotation(operand, + expr = infixNotation( + operand, [ - ("!", 1, opAssoc.LEFT), - ("^", 2, opAssoc.LEFT), - (signop, 1, opAssoc.RIGHT), - (multop, 2, opAssoc.LEFT), - (plusop, 2, opAssoc.LEFT), - ]) + ("!", 1, opAssoc.LEFT), + ("^", 2, opAssoc.LEFT), + (signop, 1, opAssoc.RIGHT), + (multop, 2, opAssoc.LEFT), + (plusop, 2, opAssoc.LEFT), + ], + ) test = ["9"] for t in test: @@ -1563,16 +2065,20 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): ops = [ (supLiteral("!"), 1, pp.opAssoc.RIGHT, lambda s, l, t: ["!", t[0][0]]), (pp.oneOf("= !="), 2, pp.opAssoc.LEFT,), - (supLiteral("&"), 2, pp.opAssoc.LEFT, lambda s, l, t: ["&", t[0]]), - (supLiteral("|"), 2, pp.opAssoc.LEFT, lambda s, l, t: ["|", t[0]])] + (supLiteral("&"), 2, pp.opAssoc.LEFT, lambda s, l, t: ["&", t[0]]), + (supLiteral("|"), 2, pp.opAssoc.LEFT, lambda s, l, t: ["|", t[0]]), + ] return pp.infixNotation(atom, ops) f = booleanExpr(word) + pp.StringEnd() tests = [ - ("bar = foo", [['bar', '=', 'foo']]), - ("bar = foo & baz = fee", ['&', [['bar', '=', 'foo'], ['baz', '=', 'fee']]]), - ] + ("bar = foo", [["bar", "=", "foo"]]), + ( + "bar = foo & baz = fee", + ["&", [["bar", "=", "foo"], ["baz", "=", "fee"]]], + ), + ] for test, expected in tests: print(test) results = f.parseString(test) @@ -1581,12 +2087,18 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): print() def testInfixNotationGrammarTest5(self): - from pyparsing import infixNotation, opAssoc, pyparsing_common as ppc, Literal, oneOf - - expop = Literal('**') - signop = oneOf('+ -') - multop = oneOf('* /') - plusop = oneOf('+ -') + from pyparsing import ( + infixNotation, + opAssoc, + pyparsing_common as ppc, + Literal, + oneOf, + ) + + expop = Literal("**") + signop = oneOf("+ -") + multop = oneOf("* /") + plusop = oneOf("+ -") class ExprNode: def __init__(self, tokens): @@ -1601,7 +2113,7 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): class SignOp(ExprNode): def eval(self): - mult = {'+': 1, '-': -1}[self.tokens[0]] + mult = {"+": 1, "-": -1}[self.tokens[0]] return mult * self.tokens[1].eval() class BinOp(ExprNode): @@ -1612,24 +2124,28 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): return ret class ExpOp(BinOp): - opn_map = {'**': lambda a, b: b ** a} + opn_map = {"**": lambda a, b: b ** a} class MultOp(BinOp): import operator - opn_map = {'*': operator.mul, '/': operator.truediv} + + opn_map = {"*": operator.mul, "/": operator.truediv} class AddOp(BinOp): import operator - opn_map = {'+': operator.add, '-': operator.sub} + + opn_map = {"+": operator.add, "-": operator.sub} operand = ppc.number().setParseAction(NumberNode) - expr = infixNotation(operand, - [ - (expop, 2, opAssoc.LEFT, (lambda pr: [pr[0][::-1]], ExpOp)), - (signop, 1, opAssoc.RIGHT, SignOp), - (multop, 2, opAssoc.LEFT, MultOp), - (plusop, 2, opAssoc.LEFT, AddOp), - ]) + expr = infixNotation( + operand, + [ + (expop, 2, opAssoc.LEFT, (lambda pr: [pr[0][::-1]], ExpOp)), + (signop, 1, opAssoc.RIGHT, SignOp), + (multop, 2, opAssoc.LEFT, MultOp), + (plusop, 2, opAssoc.LEFT, AddOp), + ], + ) tests = """\ 2+7 @@ -1646,9 +2162,13 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): parsed = expr.parseString(t) eval_value = parsed[0].eval() - self.assertEqual(eval_value, eval(t), - "Error evaluating {!r}, expected {!r}, got {!r}".format(t, eval(t), eval_value)) - + self.assertEqual( + eval_value, + eval(t), + "Error evaluating {!r}, expected {!r}, got {!r}".format( + t, eval(t), eval_value + ), + ) def testParseResultsPickle(self): from pyparsing import makeHTMLTags, ParseResults @@ -1674,8 +2194,11 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): print(newresult.dump()) print() - self.assertEqual(result.dump(), newresult.dump(), - "Error pickling ParseResults object (protocol=%d)" % protocol) + self.assertEqual( + result.dump(), + newresult.dump(), + "Error pickling ParseResults object (protocol=%d)" % protocol, + ) # test 2 import pyparsing as pp @@ -1688,7 +2211,7 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): greeting = salutation + pp.Suppress(comma) + greetee + pp.Suppress(endpunc) greeting.setParseAction(PickleTest_Greeting) - string = 'Good morning, Miss Crabtree!' + string = "Good morning, Miss Crabtree!" result = greeting.parseString(string) @@ -1702,8 +2225,13 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): else: newresult = pickle.loads(pickleString) print(newresult.dump()) - self.assertEqual(newresult.dump(), result.dump(), - "failed to pickle/unpickle ParseResults: expected {!r}, got {!r}".format(result, newresult)) + self.assertEqual( + newresult.dump(), + result.dump(), + "failed to pickle/unpickle ParseResults: expected {!r}, got {!r}".format( + result, newresult + ), + ) def testParseResultsWithNamedTuple(self): @@ -1715,10 +2243,12 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): res = expr.parseString("A") print(repr(res)) print(res.Achar) - self.assertParseResultsEquals(res, expected_dict={'Achar': ('A', 'Z')}, - msg="Failed accessing named results containing a tuple, " - "got {!r}".format(res.Achar)) - + self.assertParseResultsEquals( + res, + expected_dict={"Achar": ("A", "Z")}, + msg="Failed accessing named results containing a tuple, " + "got {!r}".format(res.Achar), + ) def testParseHTMLTags(self): test = """ @@ -1732,11 +2262,11 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): results = [ ("startBody", False, "", ""), ("startBody", False, "#00FFCC", ""), - ("startBody", True, "#00FFAA", ""), + ("startBody", True, "#00FFAA", ""), ("startBody", False, "#00FFBB", "black"), ("startBody", True, "", ""), ("endBody", False, "", ""), - ] + ] bodyStart, bodyEnd = pp.makeHTMLTags("BODY") resIter = iter(results) @@ -1746,20 +2276,34 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): print(t.dump()) if "startBody" in t: - self.assertEqual(bool(t.empty), expectedEmpty, - "expected {} token, got {}".format(expectedEmpty and "empty" or "not empty", - t.empty and "empty" or "not empty")) - self.assertEqual(t.bgcolor, expectedBG, - "failed to match BGCOLOR, expected {}, got {}".format(expectedBG, t.bgcolor)) - self.assertEqual(t.fgcolor, expectedFG, - "failed to match FGCOLOR, expected {}, got {}".format(expectedFG, t.bgcolor)) + self.assertEqual( + bool(t.empty), + expectedEmpty, + "expected {} token, got {}".format( + expectedEmpty and "empty" or "not empty", + t.empty and "empty" or "not empty", + ), + ) + self.assertEqual( + t.bgcolor, + expectedBG, + "failed to match BGCOLOR, expected {}, got {}".format( + expectedBG, t.bgcolor + ), + ) + self.assertEqual( + t.fgcolor, + expectedFG, + "failed to match FGCOLOR, expected {}, got {}".format( + expectedFG, t.bgcolor + ), + ) elif "endBody" in t: print("end tag") pass else: print("BAD!!!") - def testUpcaseDowncaseUnicode(self): import pyparsing as pp @@ -1767,12 +2311,15 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): from pyparsing import pyparsing_common as ppc import sys - a = '\u00bfC\u00f3mo esta usted?' + a = "\u00bfC\u00f3mo esta usted?" if not JYTHON_ENV: ualphas = ppu.alphas else: - ualphas = "".join(chr(i) for i in list(range(0xd800)) + list(range(0xe000, sys.maxunicode)) - if chr(i).isalpha()) + ualphas = "".join( + chr(i) + for i in list(range(0xD800)) + list(range(0xE000, sys.maxunicode)) + if chr(i).isalpha() + ) uword = pp.Word(ualphas).setParseAction(ppc.upcaseTokens) print = lambda *args: None @@ -1782,110 +2329,183 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): print(uword.searchString(a)) - kw = pp.Keyword('mykey', caseless=True).setParseAction(ppc.upcaseTokens)('rname') - ret = kw.parseString('mykey') + kw = pp.Keyword("mykey", caseless=True).setParseAction(ppc.upcaseTokens)( + "rname" + ) + ret = kw.parseString("mykey") print(ret.rname) - self.assertEqual(ret.rname, 'MYKEY', "failed to upcase with named result (pyparsing_common)") - - kw = pp.Keyword('MYKEY', caseless=True).setParseAction(ppc.downcaseTokens)('rname') - ret = kw.parseString('mykey') + self.assertEqual( + ret.rname, "MYKEY", "failed to upcase with named result (pyparsing_common)" + ) + + kw = pp.Keyword("MYKEY", caseless=True).setParseAction(ppc.downcaseTokens)( + "rname" + ) + ret = kw.parseString("mykey") print(ret.rname) - self.assertEqual(ret.rname, 'mykey', "failed to upcase with named result") + self.assertEqual(ret.rname, "mykey", "failed to upcase with named result") if not IRON_PYTHON_ENV: - #test html data + # test html data html = "<TR class=maintxt bgColor=#ffffff> \ <TD vAlign=top>Производитель, модель</TD> \ <TD vAlign=top><STRONG>BenQ-Siemens CF61</STRONG></TD> \ - "#.decode('utf-8') + " # .decode('utf-8') # 'Manufacturer, model - text_manuf = 'Производитель, модель' + text_manuf = "Производитель, модель" manufacturer = pp.Literal(text_manuf) td_start, td_end = pp.makeHTMLTags("td") - manuf_body = td_start.suppress() + manufacturer + pp.SkipTo(td_end)("cells*") + td_end.suppress() + manuf_body = ( + td_start.suppress() + + manufacturer + + pp.SkipTo(td_end)("cells*") + + td_end.suppress() + ) - #~ manuf_body.setDebug() + # ~ manuf_body.setDebug() - #~ for tokens in manuf_body.scanString(html): - #~ print(tokens) + # ~ for tokens in manuf_body.scanString(html): + # ~ print(tokens) def testParseUsingRegex(self): import re - signedInt = pp.Regex(r'[-+][0-9]+') - unsignedInt = pp.Regex(r'[0-9]+') + signedInt = pp.Regex(r"[-+][0-9]+") + unsignedInt = pp.Regex(r"[0-9]+") simpleString = pp.Regex(r'("[^\"]*")|(\'[^\']*\')') namedGrouping = pp.Regex(r'("(?P<content>[^\"]*)")') - compiledRE = pp.Regex(re.compile(r'[A-Z]+')) + compiledRE = pp.Regex(re.compile(r"[A-Z]+")) - def testMatch (expression, instring, shouldPass, expectedString=None): + def testMatch(expression, instring, shouldPass, expectedString=None): if shouldPass: try: result = expression.parseString(instring) - print('{} correctly matched {}'.format(repr(expression), repr(instring))) + print( + "{} correctly matched {}".format( + repr(expression), repr(instring) + ) + ) if expectedString != result[0]: - print('\tbut failed to match the pattern as expected:') - print('\tproduced %s instead of %s' % \ - (repr(result[0]), repr(expectedString))) + print("\tbut failed to match the pattern as expected:") + print( + "\tproduced %s instead of %s" + % (repr(result[0]), repr(expectedString)) + ) return True except pp.ParseException: - print('%s incorrectly failed to match %s' % \ - (repr(expression), repr(instring))) + print( + "%s incorrectly failed to match %s" + % (repr(expression), repr(instring)) + ) else: try: result = expression.parseString(instring) - print('{} incorrectly matched {}'.format(repr(expression), repr(instring))) - print('\tproduced %s as a result' % repr(result[0])) + print( + "{} incorrectly matched {}".format( + repr(expression), repr(instring) + ) + ) + print("\tproduced %s as a result" % repr(result[0])) except pp.ParseException: - print('%s correctly failed to match %s' % \ - (repr(expression), repr(instring))) + print( + "%s correctly failed to match %s" + % (repr(expression), repr(instring)) + ) return True return False # These should fail - self.assertTrue(testMatch(signedInt, '1234 foo', False), "Re: (1) passed, expected fail") - self.assertTrue(testMatch(signedInt, ' +foo', False), "Re: (2) passed, expected fail") - self.assertTrue(testMatch(unsignedInt, 'abc', False), "Re: (3) passed, expected fail") - self.assertTrue(testMatch(unsignedInt, '+123 foo', False), "Re: (4) passed, expected fail") - self.assertTrue(testMatch(simpleString, 'foo', False), "Re: (5) passed, expected fail") - self.assertTrue(testMatch(simpleString, '"foo bar\'', False), "Re: (6) passed, expected fail") - self.assertTrue(testMatch(simpleString, '\'foo bar"', False), "Re: (7) passed, expected fail") + self.assertTrue( + testMatch(signedInt, "1234 foo", False), "Re: (1) passed, expected fail" + ) + self.assertTrue( + testMatch(signedInt, " +foo", False), "Re: (2) passed, expected fail" + ) + self.assertTrue( + testMatch(unsignedInt, "abc", False), "Re: (3) passed, expected fail" + ) + self.assertTrue( + testMatch(unsignedInt, "+123 foo", False), "Re: (4) passed, expected fail" + ) + self.assertTrue( + testMatch(simpleString, "foo", False), "Re: (5) passed, expected fail" + ) + self.assertTrue( + testMatch(simpleString, "\"foo bar'", False), + "Re: (6) passed, expected fail", + ) + self.assertTrue( + testMatch(simpleString, "'foo bar\"", False), + "Re: (7) passed, expected fail", + ) # These should pass - self.assertTrue(testMatch(signedInt, ' +123', True, '+123'), "Re: (8) failed, expected pass") - self.assertTrue(testMatch(signedInt, '+123', True, '+123'), "Re: (9) failed, expected pass") - self.assertTrue(testMatch(signedInt, '+123 foo', True, '+123'), "Re: (10) failed, expected pass") - self.assertTrue(testMatch(signedInt, '-0 foo', True, '-0'), "Re: (11) failed, expected pass") - self.assertTrue(testMatch(unsignedInt, '123 foo', True, '123'), "Re: (12) failed, expected pass") - self.assertTrue(testMatch(unsignedInt, '0 foo', True, '0'), "Re: (13) failed, expected pass") - self.assertTrue(testMatch(simpleString, '"foo"', True, '"foo"'), "Re: (14) failed, expected pass") - self.assertTrue(testMatch(simpleString, "'foo bar' baz", True, "'foo bar'"), "Re: (15) failed, expected pass") - - self.assertTrue(testMatch(compiledRE, 'blah', False), "Re: (16) passed, expected fail") - self.assertTrue(testMatch(compiledRE, 'BLAH', True, 'BLAH'), "Re: (17) failed, expected pass") - - self.assertTrue(testMatch(namedGrouping, '"foo bar" baz', True, '"foo bar"'), "Re: (16) failed, expected pass") + self.assertTrue( + testMatch(signedInt, " +123", True, "+123"), + "Re: (8) failed, expected pass", + ) + self.assertTrue( + testMatch(signedInt, "+123", True, "+123"), "Re: (9) failed, expected pass" + ) + self.assertTrue( + testMatch(signedInt, "+123 foo", True, "+123"), + "Re: (10) failed, expected pass", + ) + self.assertTrue( + testMatch(signedInt, "-0 foo", True, "-0"), "Re: (11) failed, expected pass" + ) + self.assertTrue( + testMatch(unsignedInt, "123 foo", True, "123"), + "Re: (12) failed, expected pass", + ) + self.assertTrue( + testMatch(unsignedInt, "0 foo", True, "0"), "Re: (13) failed, expected pass" + ) + self.assertTrue( + testMatch(simpleString, '"foo"', True, '"foo"'), + "Re: (14) failed, expected pass", + ) + self.assertTrue( + testMatch(simpleString, "'foo bar' baz", True, "'foo bar'"), + "Re: (15) failed, expected pass", + ) + + self.assertTrue( + testMatch(compiledRE, "blah", False), "Re: (16) passed, expected fail" + ) + self.assertTrue( + testMatch(compiledRE, "BLAH", True, "BLAH"), + "Re: (17) failed, expected pass", + ) + + self.assertTrue( + testMatch(namedGrouping, '"foo bar" baz', True, '"foo bar"'), + "Re: (16) failed, expected pass", + ) ret = namedGrouping.parseString('"zork" blah') print(ret) print(list(ret.items())) print(ret.content) - self.assertEqual(ret.content, 'zork', "named group lookup failed") - self.assertEqual(ret[0], simpleString.parseString('"zork" blah')[0], - "Regex not properly returning ParseResults for named vs. unnamed groups") + self.assertEqual(ret.content, "zork", "named group lookup failed") + self.assertEqual( + ret[0], + simpleString.parseString('"zork" blah')[0], + "Regex not properly returning ParseResults for named vs. unnamed groups", + ) try: - #~ print "lets try an invalid RE" - invRe = pp.Regex('("[^\"]*")|(\'[^\']*\'') + # ~ print "lets try an invalid RE" + invRe = pp.Regex("(\"[^\"]*\")|('[^']*'") except Exception as e: - print("successfully rejected an invalid RE:", end=' ') + print("successfully rejected an invalid RE:", end=" ") print(e) else: self.assertTrue(False, "failed to reject invalid RE") - invRe = pp.Regex('') + invRe = pp.Regex("") def testRegexAsType(self): import pyparsing as pp @@ -1898,19 +2518,30 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): result = expr.parseString(test_str) print(result.dump()) print(expected_group_list) - self.assertParseResultsEquals(result, expected_list=expected_group_list, - msg="incorrect group list returned by Regex)") + self.assertParseResultsEquals( + result, + expected_list=expected_group_list, + msg="incorrect group list returned by Regex)", + ) print("return as re.match instance") - expr = pp.Regex(r"\w+ (?P<num1>\d+) (?P<num2>\d+) (?P<last_word>\w+)", asMatch=True) + expr = pp.Regex( + r"\w+ (?P<num1>\d+) (?P<num2>\d+) (?P<last_word>\w+)", asMatch=True + ) result = expr.parseString(test_str) print(result.dump()) print(result[0].groups()) print(expected_group_list) - self.assertEqual(result[0].groupdict(), {'num1': '123', 'num2': '456', 'last_word': 'lsdfkj'}, - 'invalid group dict from Regex(asMatch=True)') - self.assertEqual(result[0].groups(), expected_group_list[0], - "incorrect group list returned by Regex(asMatch)") + self.assertEqual( + result[0].groupdict(), + {"num1": "123", "num2": "456", "last_word": "lsdfkj"}, + "invalid group dict from Regex(asMatch=True)", + ) + self.assertEqual( + result[0].groups(), + expected_group_list[0], + "incorrect group list returned by Regex(asMatch)", + ) def testRegexSub(self): import pyparsing as pp @@ -1919,55 +2550,81 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): expr = pp.Regex(r"<title>").sub("'Richard III'") result = expr.transformString("This is the title: <title>") print(result) - self.assertEqual(result, "This is the title: 'Richard III'", "incorrect Regex.sub result with simple string") + self.assertEqual( + result, + "This is the title: 'Richard III'", + "incorrect Regex.sub result with simple string", + ) print("test sub with re string") expr = pp.Regex(r"([Hh]\d):\s*(.*)").sub(r"<\1>\2</\1>") - result = expr.transformString("h1: This is the main heading\nh2: This is the sub-heading") + result = expr.transformString( + "h1: This is the main heading\nh2: This is the sub-heading" + ) print(result) - self.assertEqual(result, '<h1>This is the main heading</h1>\n<h2>This is the sub-heading</h2>', - "incorrect Regex.sub result with re string") + self.assertEqual( + result, + "<h1>This is the main heading</h1>\n<h2>This is the sub-heading</h2>", + "incorrect Regex.sub result with re string", + ) print("test sub with re string (Regex returns re.match)") expr = pp.Regex(r"([Hh]\d):\s*(.*)", asMatch=True).sub(r"<\1>\2</\1>") - result = expr.transformString("h1: This is the main heading\nh2: This is the sub-heading") + result = expr.transformString( + "h1: This is the main heading\nh2: This is the sub-heading" + ) print(result) - self.assertEqual(result, '<h1>This is the main heading</h1>\n<h2>This is the sub-heading</h2>', - "incorrect Regex.sub result with re string") + self.assertEqual( + result, + "<h1>This is the main heading</h1>\n<h2>This is the sub-heading</h2>", + "incorrect Regex.sub result with re string", + ) print("test sub with callable that return str") expr = pp.Regex(r"<(.*?)>").sub(lambda m: m.group(1).upper()) result = expr.transformString("I want this in upcase: <what? what?>") print(result) - self.assertEqual(result, 'I want this in upcase: WHAT? WHAT?', "incorrect Regex.sub result with callable") + self.assertEqual( + result, + "I want this in upcase: WHAT? WHAT?", + "incorrect Regex.sub result with callable", + ) try: expr = pp.Regex(r"<(.*?)>", asMatch=True).sub(lambda m: m.group(1).upper()) except SyntaxError: pass else: - self.assertTrue(False, "failed to warn using a Regex.sub(callable) with asMatch=True") + self.assertTrue( + False, "failed to warn using a Regex.sub(callable) with asMatch=True" + ) try: - expr = pp.Regex(r"<(.*?)>", asGroupList=True).sub(lambda m: m.group(1).upper()) + expr = pp.Regex(r"<(.*?)>", asGroupList=True).sub( + lambda m: m.group(1).upper() + ) except SyntaxError: pass else: - self.assertTrue(False, "failed to warn using a Regex.sub() with asGroupList=True") + self.assertTrue( + False, "failed to warn using a Regex.sub() with asGroupList=True" + ) try: expr = pp.Regex(r"<(.*?)>", asGroupList=True).sub("") except SyntaxError: pass else: - self.assertTrue(False, "failed to warn using a Regex.sub() with asGroupList=True") + self.assertTrue( + False, "failed to warn using a Regex.sub() with asGroupList=True" + ) def testPrecededBy(self): import pyparsing as pp num = pp.Word(pp.nums).setParseAction(lambda t: int(t[0])) interesting_num = pp.PrecededBy(pp.Char("abc")("prefix*")) + num - semi_interesting_num = pp.PrecededBy('_') + num + semi_interesting_num = pp.PrecededBy("_") + num crazy_num = pp.PrecededBy(pp.Word("^", "$%^")("prefix*"), 10) + num boring_num = ~pp.PrecededBy(pp.Char("abc_$%^" + pp.nums)) + num very_boring_num = pp.PrecededBy(pp.WordStart()) + num @@ -1976,24 +2633,24 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): s = "c384 b8324 _9293874 _293 404 $%^$^%$2939" print(s) for expr, expected_list, expected_dict in [ - (interesting_num, [384, 8324], {'prefix': ['c', 'b']}), + (interesting_num, [384, 8324], {"prefix": ["c", "b"]}), (semi_interesting_num, [9293874, 293], {}), (boring_num, [404], {}), - (crazy_num, [2939], {'prefix': ['^%$']}), + (crazy_num, [2939], {"prefix": ["^%$"]}), (finicky_num, [2939], {}), (very_boring_num, [404], {}), - ]: + ]: # print(expr.searchString(s)) result = sum(expr.searchString(s)) print(result.dump()) self.assertParseResultsEquals(result, expected_list, expected_dict) # infinite loop test - from Issue #127 - string_test = 'notworking' + string_test = "notworking" # negs = pp.Or(['not', 'un'])('negs') - negs_pb = pp.PrecededBy('not', retreat=100)('negs_lb') + negs_pb = pp.PrecededBy("not", retreat=100)("negs_lb") # negs_pb = pp.PrecededBy(negs, retreat=100)('negs_lb') - pattern = (negs_pb + pp.Literal('working'))('main') + pattern = (negs_pb + pp.Literal("working"))("main") results = pattern.searchString(string_test) try: @@ -2015,9 +2672,11 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): print(testString) print(r) - self.assertParseResultsEquals(r, expected_list=[[5, 7], [0, 1, 2, 3, 4, 5], [], [5, 4, 3]]) + self.assertParseResultsEquals( + r, expected_list=[[5, 7], [0, 1, 2, 3, 4, 5], [], [5, 4, 3]] + ) -# addresses bug raised by Ralf Vosseler + # addresses bug raised by Ralf Vosseler def testCountedArrayTest2(self): from pyparsing import Word, nums, OneOrMore, countedArray @@ -2031,10 +2690,13 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): print(testString) print(r) - self.assertParseResultsEquals(r, expected_list=[[5, 7], [0, 1, 2, 3, 4, 5], [], [5, 4, 3]]) + self.assertParseResultsEquals( + r, expected_list=[[5, 7], [0, 1, 2, 3, 4, 5], [], [5, 4, 3]] + ) def testCountedArrayTest3(self): from pyparsing import Word, nums, OneOrMore, countedArray, alphas + int_chars = "_" + alphas array_counter = Word(int_chars).setParseAction(lambda t: int_chars.index(t[0])) @@ -2048,7 +2710,9 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): print(testString) print(r) - self.assertParseResultsEquals(r, expected_list=[[5, 7], [0, 1, 2, 3, 4, 5], [], [5, 4, 3]]) + self.assertParseResultsEquals( + r, expected_list=[[5, 7], [0, 1, 2, 3, 4, 5], [], [5, 4, 3]] + ) def testLineStart(self): import pyparsing as pp @@ -2062,7 +2726,7 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): AAA... BBB """, - ] + ] fail_tests = [ """\ AAA... @@ -2074,10 +2738,16 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): ] # cleanup test strings - pass_tests = ['\n'.join(s.lstrip() for s in t.splitlines()).replace('.', ' ') for t in pass_tests] - fail_tests = ['\n'.join(s.lstrip() for s in t.splitlines()).replace('.', ' ') for t in fail_tests] + pass_tests = [ + "\n".join(s.lstrip() for s in t.splitlines()).replace(".", " ") + for t in pass_tests + ] + fail_tests = [ + "\n".join(s.lstrip() for s in t.splitlines()).replace(".", " ") + for t in fail_tests + ] - test_patt = pp.Word('A') - pp.LineStart() + pp.Word('B') + test_patt = pp.Word("A") - pp.LineStart() + pp.Word("B") print(test_patt.streamline()) success = test_patt.runTests(pass_tests)[0] self.assertTrue(success, "failed LineStart passing tests (1)") @@ -2086,10 +2756,10 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): self.assertTrue(success, "failed LineStart failure mode tests (1)") with ppt.reset_pyparsing_context(): - print(r'no \n in default whitespace chars') - pp.ParserElement.setDefaultWhitespaceChars(' ') + print(r"no \n in default whitespace chars") + pp.ParserElement.setDefaultWhitespaceChars(" ") - test_patt = pp.Word('A') - pp.LineStart() + pp.Word('B') + test_patt = pp.Word("A") - pp.LineStart() + pp.Word("B") print(test_patt.streamline()) # should fail the pass tests too, since \n is no longer valid whitespace and we aren't parsing for it success = test_patt.runTests(pass_tests, failureTests=True)[0] @@ -2098,7 +2768,13 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): success = test_patt.runTests(fail_tests, failureTests=True)[0] self.assertTrue(success, "failed LineStart failure mode tests (2)") - test_patt = pp.Word('A') - pp.LineEnd().suppress() + pp.LineStart() + pp.Word('B') + pp.LineEnd().suppress() + test_patt = ( + pp.Word("A") + - pp.LineEnd().suppress() + + pp.LineStart() + + pp.Word("B") + + pp.LineEnd().suppress() + ) print(test_patt.streamline()) success = test_patt.runTests(pass_tests)[0] self.assertTrue(success, "failed LineStart passing tests (3)") @@ -2117,152 +2793,208 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): """ from textwrap import dedent + test = dedent(test) print(test) - for t, s, e in (pp.LineStart() + 'AAA').scanString(test): + for t, s, e in (pp.LineStart() + "AAA").scanString(test): print(s, e, pp.lineno(s, test), pp.line(s, test), ord(test[s])) print() - self.assertEqual(test[s], 'A', 'failed LineStart with insignificant newlines') + self.assertEqual( + test[s], "A", "failed LineStart with insignificant newlines" + ) with ppt.reset_pyparsing_context(): - pp.ParserElement.setDefaultWhitespaceChars(' ') - for t, s, e in (pp.LineStart() + 'AAA').scanString(test): + pp.ParserElement.setDefaultWhitespaceChars(" ") + for t, s, e in (pp.LineStart() + "AAA").scanString(test): print(s, e, pp.lineno(s, test), pp.line(s, test), ord(test[s])) print() - self.assertEqual(test[s], 'A', 'failed LineStart with insignificant newlines') - + self.assertEqual( + test[s], "A", "failed LineStart with insignificant newlines" + ) def testLineAndStringEnd(self): - from pyparsing import OneOrMore, lineEnd, alphanums, Word, stringEnd, delimitedList, SkipTo + from pyparsing import ( + OneOrMore, + lineEnd, + alphanums, + Word, + stringEnd, + delimitedList, + SkipTo, + ) NLs = OneOrMore(lineEnd) bnf1 = delimitedList(Word(alphanums).leaveWhitespace(), NLs) bnf2 = Word(alphanums) + stringEnd bnf3 = Word(alphanums) + SkipTo(stringEnd) tests = [ - ("testA\ntestB\ntestC\n", ['testA', 'testB', 'testC']), - ("testD\ntestE\ntestF", ['testD', 'testE', 'testF']), - ("a", ['a']), - ] + ("testA\ntestB\ntestC\n", ["testA", "testB", "testC"]), + ("testD\ntestE\ntestF", ["testD", "testE", "testF"]), + ("a", ["a"]), + ] for test, expected in tests: res1 = bnf1.parseString(test) - print(res1, '=?', expected) - self.assertParseResultsEquals(res1, expected_list=expected, - msg="Failed lineEnd/stringEnd test (1): " + repr(test)+ " -> " + str(res1)) + print(res1, "=?", expected) + self.assertParseResultsEquals( + res1, + expected_list=expected, + msg="Failed lineEnd/stringEnd test (1): " + + repr(test) + + " -> " + + str(res1), + ) res2 = bnf2.searchString(test)[0] - print(res2, '=?', expected[-1:]) - self.assertParseResultsEquals(res2, expected_list=expected[-1:], - msg="Failed lineEnd/stringEnd test (2): " + repr(test)+ " -> " + str(res2)) + print(res2, "=?", expected[-1:]) + self.assertParseResultsEquals( + res2, + expected_list=expected[-1:], + msg="Failed lineEnd/stringEnd test (2): " + + repr(test) + + " -> " + + str(res2), + ) res3 = bnf3.parseString(test) first = res3[0] rest = res3[1] - #~ print res3.dump() - print(repr(rest), '=?', repr(test[len(first) + 1:])) - self.assertEqual(rest, test[len(first) + 1:], - "Failed lineEnd/stringEnd test (3): " + repr(test)+ " -> " + str(res3.asList())) + # ~ print res3.dump() + print(repr(rest), "=?", repr(test[len(first) + 1 :])) + self.assertEqual( + rest, + test[len(first) + 1 :], + "Failed lineEnd/stringEnd test (3): " + + repr(test) + + " -> " + + str(res3.asList()), + ) print() from pyparsing import Regex import re - k = Regex(r'a+', flags=re.S + re.M) + k = Regex(r"a+", flags=re.S + re.M) k = k.parseWithTabs() k = k.leaveWhitespace() tests = [ - (r'aaa', ['aaa']), - (r'\naaa', None), - (r'a\naa', None), - (r'aaa\n', None), - ] + (r"aaa", ["aaa"]), + (r"\naaa", None), + (r"a\naa", None), + (r"aaa\n", None), + ] for i, (src, expected) in enumerate(tests): - print(i, repr(src).replace('\\\\', '\\'), end=' ') + print(i, repr(src).replace("\\\\", "\\"), end=" ") if expected is None: with self.assertRaisesParseException(): k.parseString(src, parseAll=True) else: res = k.parseString(src, parseAll=True) - self.assertParseResultsEquals(res, expected, msg="Failed on parseAll=True test %d" % i) + self.assertParseResultsEquals( + res, expected, msg="Failed on parseAll=True test %d" % i + ) def testVariableParseActionArgs(self): pa3 = lambda s, l, t: t pa2 = lambda l, t: t pa1 = lambda t: t - pa0 = lambda : None + pa0 = lambda: None + class Callable3: def __call__(self, s, l, t): return t + class Callable2: def __call__(self, l, t): return t + class Callable1: def __call__(self, t): return t + class Callable0: def __call__(self): return + class CallableS3: - #~ @staticmethod + # ~ @staticmethod def __call__(s, l, t): return t - __call__=staticmethod(__call__) + + __call__ = staticmethod(__call__) + class CallableS2: - #~ @staticmethod + # ~ @staticmethod def __call__(l, t): return t - __call__=staticmethod(__call__) + + __call__ = staticmethod(__call__) + class CallableS1: - #~ @staticmethod + # ~ @staticmethod def __call__(t): return t - __call__=staticmethod(__call__) + + __call__ = staticmethod(__call__) + class CallableS0: - #~ @staticmethod + # ~ @staticmethod def __call__(): return - __call__=staticmethod(__call__) + + __call__ = staticmethod(__call__) + class CallableC3: - #~ @classmethod + # ~ @classmethod def __call__(cls, s, l, t): return t - __call__=classmethod(__call__) + + __call__ = classmethod(__call__) + class CallableC2: - #~ @classmethod + # ~ @classmethod def __call__(cls, l, t): return t - __call__=classmethod(__call__) + + __call__ = classmethod(__call__) + class CallableC1: - #~ @classmethod + # ~ @classmethod def __call__(cls, t): return t - __call__=classmethod(__call__) + + __call__ = classmethod(__call__) + class CallableC0: - #~ @classmethod + # ~ @classmethod def __call__(cls): return - __call__=classmethod(__call__) + + __call__ = classmethod(__call__) class parseActionHolder: - #~ @staticmethod + # ~ @staticmethod def pa3(s, l, t): return t - pa3=staticmethod(pa3) - #~ @staticmethod + + pa3 = staticmethod(pa3) + # ~ @staticmethod def pa2(l, t): return t - pa2=staticmethod(pa2) - #~ @staticmethod + + pa2 = staticmethod(pa2) + # ~ @staticmethod def pa1(t): return t - pa1=staticmethod(pa1) - #~ @staticmethod + + pa1 = staticmethod(pa1) + # ~ @staticmethod def pa0(): return - pa0=staticmethod(pa0) + + pa0 = staticmethod(pa0) def paArgs(*args): print(args) @@ -2271,6 +3003,7 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): class ClassAsPA0: def __init__(self): pass + def __str__(self): return "A" @@ -2278,18 +3011,21 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): def __init__(self, t): print("making a ClassAsPA1") self.t = t + def __str__(self): return self.t[0] class ClassAsPA2: def __init__(self, l, t): self.t = t + def __str__(self): return self.t[0] class ClassAsPA3: def __init__(self, s, l, t): self.t = t + def __str__(self): return self.t[0] @@ -2297,8 +3033,9 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): def __new__(cls, *args): print("make a ClassAsPAStarNew", args) return tuple.__new__(cls, *args[2].asList()) + def __str__(self): - return ''.join(self) + return "".join(self) from pyparsing import Literal, OneOrMore @@ -2325,13 +3062,38 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): U = Literal("U").setParseAction(parseActionHolder.pa0) V = Literal("V") - gg = OneOrMore(A | C | D | E | F | G | H | - I | J | K | L | M | N | O | P | Q | R | S | U | V | B | T) + gg = OneOrMore( + A + | C + | D + | E + | F + | G + | H + | I + | J + | K + | L + | M + | N + | O + | P + | Q + | R + | S + | U + | V + | B + | T + ) testString = "VUTSRQPONMLKJIHGFEDCBA" res = gg.parseString(testString) print(res) - self.assertParseResultsEquals(res, expected_list=list(testString), - msg="Failed to parse using variable length parse actions") + self.assertParseResultsEquals( + res, + expected_list=list(testString), + msg="Failed to parse using variable length parse actions", + ) A = Literal("A").setParseAction(ClassAsPA0) B = Literal("B").setParseAction(ClassAsPA1) @@ -2339,14 +3101,39 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): D = Literal("D").setParseAction(ClassAsPA3) E = Literal("E").setParseAction(ClassAsPAStarNew) - gg = OneOrMore(A | B | C | D | E | F | G | H | - I | J | K | L | M | N | O | P | Q | R | S | T | U | V) + gg = OneOrMore( + A + | B + | C + | D + | E + | F + | G + | H + | I + | J + | K + | L + | M + | N + | O + | P + | Q + | R + | S + | T + | U + | V + ) testString = "VUTSRQPONMLKJIHGFEDCBA" res = gg.parseString(testString) print(list(map(str, res))) - self.assertEqual(list(map(str, res)), list(testString), - "Failed to parse using variable length parse actions " - "using class constructors as parse actions") + self.assertEqual( + list(map(str, res)), + list(testString), + "Failed to parse using variable length parse actions " + "using class constructors as parse actions", + ) def testSingleArgException(self): from pyparsing import ParseBaseException, ParseFatalException @@ -2359,7 +3146,9 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): except ParseBaseException as pbe: print("Received expected exception:", pbe) raisedMsg = pbe.msg - self.assertEqual(raisedMsg, testMessage, "Failed to get correct exception message") + self.assertEqual( + raisedMsg, testMessage, "Failed to get correct exception message" + ) def testOriginalTextFor(self): from pyparsing import makeHTMLTags, originalTextFor @@ -2367,68 +3156,107 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): def rfn(t): return "%s:%d" % (t.src, len("".join(t))) - makeHTMLStartTag = lambda tag: originalTextFor(makeHTMLTags(tag)[0], asString=False) + makeHTMLStartTag = lambda tag: originalTextFor( + makeHTMLTags(tag)[0], asString=False + ) # use the lambda, Luke - start = makeHTMLStartTag('IMG') + start = makeHTMLStartTag("IMG") # don't replace our fancy parse action with rfn, # append rfn to the list of parse actions start.addParseAction(rfn) - text = '''_<img src="images/cal.png" - alt="cal image" width="16" height="15">_''' + text = """_<img src="images/cal.png" + alt="cal image" width="16" height="15">_""" s = start.transformString(text) if VERBOSE: print(s) - self.assertTrue(s.startswith("_images/cal.png:"), "failed to preserve input s properly") - self.assertTrue(s.endswith("77_"), "failed to return full original text properly") + self.assertTrue( + s.startswith("_images/cal.png:"), "failed to preserve input s properly" + ) + self.assertTrue( + s.endswith("77_"), "failed to return full original text properly" + ) tag_fields = makeHTMLStartTag("IMG").searchString(text)[0] if VERBOSE: print(sorted(tag_fields.keys())) - self.assertEqual(sorted(tag_fields.keys()), - ['alt', 'empty', 'height', 'src', 'startImg', 'tag', 'width'], - 'failed to preserve results names in originalTextFor') + self.assertEqual( + sorted(tag_fields.keys()), + ["alt", "empty", "height", "src", "startImg", "tag", "width"], + "failed to preserve results names in originalTextFor", + ) def testPackratParsingCacheCopy(self): - from pyparsing import Word, nums, delimitedList, Literal, Optional, alphas, alphanums, empty + from pyparsing import ( + Word, + nums, + delimitedList, + Literal, + Optional, + alphas, + alphanums, + empty, + ) integer = Word(nums).setName("integer") - id = Word(alphas + '_', alphanums + '_') - simpleType = Literal('int'); - arrayType= simpleType + ('[' + delimitedList(integer) + ']')[...] + id = Word(alphas + "_", alphanums + "_") + simpleType = Literal("int") + arrayType = simpleType + ("[" + delimitedList(integer) + "]")[...] varType = arrayType | simpleType - varDec = varType + delimitedList(id + Optional('=' + integer)) + ';' + varDec = varType + delimitedList(id + Optional("=" + integer)) + ";" - codeBlock = Literal('{}') + codeBlock = Literal("{}") - funcDef = Optional(varType | 'void') + id + '(' + (delimitedList(varType + id)|'void'|empty) + ')' + codeBlock + funcDef = ( + Optional(varType | "void") + + id + + "(" + + (delimitedList(varType + id) | "void" | empty) + + ")" + + codeBlock + ) program = varDec | funcDef - input = 'int f(){}' + input = "int f(){}" results = program.parseString(input) print("Parsed '{}' as {}".format(input, results.asList())) - self.assertEqual(results.asList(), ['int', 'f', '(', ')', '{}'], "Error in packrat parsing") + self.assertEqual( + results.asList(), ["int", "f", "(", ")", "{}"], "Error in packrat parsing" + ) def testPackratParsingCacheCopyTest2(self): - from pyparsing import Keyword, Word, Suppress, Forward, Optional, delimitedList, Group + from pyparsing import ( + Keyword, + Word, + Suppress, + Forward, + Optional, + delimitedList, + Group, + ) DO, AA = list(map(Keyword, "DO AA".split())) LPAR, RPAR = list(map(Suppress, "()")) identifier = ~AA + Word("Z") function_name = identifier.copy() - #~ function_name = ~AA + Word("Z") #identifier.copy() + # ~ function_name = ~AA + Word("Z") #identifier.copy() expr = Forward().setName("expr") - expr << (Group(function_name + LPAR + Optional(delimitedList(expr)) + RPAR).setName("functionCall") | - identifier.setName("ident")#.setDebug()#.setBreak() - ) + expr << ( + Group(function_name + LPAR + Optional(delimitedList(expr)) + RPAR).setName( + "functionCall" + ) + | identifier.setName("ident") # .setDebug()#.setBreak() + ) stmt = DO + Group(delimitedList(identifier + ".*" | expr)) result = stmt.parseString("DO Z") print(result.asList()) - self.assertEqual(len(result[1]), 1, "packrat parsing is duplicating And term exprs") + self.assertEqual( + len(result[1]), 1, "packrat parsing is duplicating And term exprs" + ) def testParseResultsDel(self): from pyparsing import OneOrMore, Word, alphas, nums @@ -2441,10 +3269,16 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): del res[1] del res["words"] print(res.dump()) - self.assertEqual(res[1], 'ABC', "failed to delete 0'th element correctly") - self.assertEqual(res.ints.asList(), origInts, "updated named attributes, should have updated list only") + self.assertEqual(res[1], "ABC", "failed to delete 0'th element correctly") + self.assertEqual( + res.ints.asList(), + origInts, + "updated named attributes, should have updated list only", + ) self.assertEqual(res.words, "", "failed to update named attribute correctly") - self.assertEqual(res[-1], 'DEF', "updated list, should have updated named attributes only") + self.assertEqual( + res[-1], "DEF", "updated list, should have updated named attributes only" + ) def testWithAttributeParseAction(self): """ @@ -2472,26 +3306,38 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): expr = tagStart + Word(nums)("value") + tagEnd - expected = ([['a', ['b', 'x'], False, '2', '</a>'], - ['a', ['b', 'x'], False, '3', '</a>']], - [['a', ['b', 'x'], False, '2', '</a>'], - ['a', ['b', 'x'], False, '3', '</a>']], - [['a', ['class', 'boo'], False, '8', '</a>']], - ) + expected = ( + [ + ["a", ["b", "x"], False, "2", "</a>"], + ["a", ["b", "x"], False, "3", "</a>"], + ], + [ + ["a", ["b", "x"], False, "2", "</a>"], + ["a", ["b", "x"], False, "3", "</a>"], + ], + [["a", ["class", "boo"], False, "8", "</a>"]], + ) - for attrib, exp in zip([ - withAttribute(b="x"), - #withAttribute(B="x"), - withAttribute(("b", "x")), - #withAttribute(("B", "x")), - withClass("boo"), - ], expected): + for attrib, exp in zip( + [ + withAttribute(b="x"), + # withAttribute(B="x"), + withAttribute(("b", "x")), + # withAttribute(("B", "x")), + withClass("boo"), + ], + expected, + ): tagStart.setParseAction(attrib) result = expr.searchString(data) print(result.dump()) - self.assertEqual(result.asList(), exp, "Failed test, expected {}, got {}".format(expected, result.asList())) + self.assertEqual( + result.asList(), + exp, + "Failed test, expected {}, got {}".format(expected, result.asList()), + ) def testNestedExpressions(self): """ @@ -2509,41 +3355,59 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): """ from pyparsing import nestedExpr, Literal, Regex, restOfLine, quotedString - #All defaults. Straight out of the example script. Also, qualifies for - #the bonus: note the fact that (Z | (E^F) & D) is not parsed :-). + # All defaults. Straight out of the example script. Also, qualifies for + # the bonus: note the fact that (Z | (E^F) & D) is not parsed :-). # Tests for bug fixed in 1.4.10 print("Test defaults:") teststring = "((ax + by)*C) (Z | (E^F) & D)" expr = nestedExpr() - expected = [[['ax', '+', 'by'], '*C']] + expected = [[["ax", "+", "by"], "*C"]] result = expr.parseString(teststring) print(result.dump()) - self.assertEqual(result.asList(), expected, "Defaults didn't work. That's a bad sign. Expected: {}, got: {}".format(expected, result)) - - #Going through non-defaults, one by one; trying to think of anything - #odd that might not be properly handled. - - #Change opener + self.assertEqual( + result.asList(), + expected, + "Defaults didn't work. That's a bad sign. Expected: {}, got: {}".format( + expected, result + ), + ) + + # Going through non-defaults, one by one; trying to think of anything + # odd that might not be properly handled. + + # Change opener print("\nNon-default opener") opener = "[" teststring = "[[ ax + by)*C)" - expected = [[['ax', '+', 'by'], '*C']] + expected = [[["ax", "+", "by"], "*C"]] expr = nestedExpr("[") result = expr.parseString(teststring) print(result.dump()) - self.assertEqual(result.asList(), expected, "Non-default opener didn't work. Expected: {}, got: {}".format(expected, result)) - - #Change closer + self.assertEqual( + result.asList(), + expected, + "Non-default opener didn't work. Expected: {}, got: {}".format( + expected, result + ), + ) + + # Change closer print("\nNon-default closer") teststring = "((ax + by]*C]" - expected = [[['ax', '+', 'by'], '*C']] + expected = [[["ax", "+", "by"], "*C"]] expr = nestedExpr(closer="]") result = expr.parseString(teststring) print(result.dump()) - self.assertEqual(result.asList(), expected, "Non-default closer didn't work. Expected: {}, got: {}".format(expected, result)) + self.assertEqual( + result.asList(), + expected, + "Non-default closer didn't work. Expected: {}, got: {}".format( + expected, result + ), + ) # #Multicharacter opener, closer # opener = "bar" @@ -2551,59 +3415,90 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): print("\nLiteral expressions for opener and closer") opener, closer = list(map(Literal, "bar baz".split())) - expr = nestedExpr(opener, closer, - content=Regex(r"([^b ]|b(?!a)|ba(?![rz]))+")) + expr = nestedExpr(opener, closer, content=Regex(r"([^b ]|b(?!a)|ba(?![rz]))+")) teststring = "barbar ax + bybaz*Cbaz" - expected = [[['ax', '+', 'by'], '*C']] + expected = [[["ax", "+", "by"], "*C"]] # expr = nestedExpr(opener, closer) result = expr.parseString(teststring) print(result.dump()) - self.assertEqual(result.asList(), expected, "Multicharacter opener and closer didn't work. Expected: {}, got: {}".format(expected, result)) - - #Lisp-ish comments + self.assertEqual( + result.asList(), + expected, + "Multicharacter opener and closer didn't work. Expected: {}, got: {}".format( + expected, result + ), + ) + + # Lisp-ish comments print("\nUse ignore expression (1)") comment = Regex(r";;.*") - teststring = \ - """ + teststring = """ (let ((greeting "Hello, world!")) ;;(foo bar (display greeting)) """ - expected = [['let', [['greeting', '"Hello,', 'world!"']], ';;(foo bar',\ - ['display', 'greeting']]] + expected = [ + [ + "let", + [["greeting", '"Hello,', 'world!"']], + ";;(foo bar", + ["display", "greeting"], + ] + ] expr = nestedExpr(ignoreExpr=comment) result = expr.parseString(teststring) print(result.dump()) - self.assertEqual(result.asList(), expected , "Lisp-ish comments (\";; <...> $\") didn't work. Expected: {}, got: {}".format(expected, result)) - - - #Lisp-ish comments, using a standard bit of pyparsing, and an Or. + self.assertEqual( + result.asList(), + expected, + 'Lisp-ish comments (";; <...> $") didn\'t work. Expected: {}, got: {}'.format( + expected, result + ), + ) + + # Lisp-ish comments, using a standard bit of pyparsing, and an Or. print("\nUse ignore expression (2)") - comment = ';;' + restOfLine + comment = ";;" + restOfLine - teststring = \ - """ + teststring = """ (let ((greeting "Hello, )world!")) ;;(foo bar (display greeting)) """ - expected = [['let', [['greeting', '"Hello, )world!"']], ';;', '(foo bar', - ['display', 'greeting']]] + expected = [ + [ + "let", + [["greeting", '"Hello, )world!"']], + ";;", + "(foo bar", + ["display", "greeting"], + ] + ] expr = nestedExpr(ignoreExpr=(comment ^ quotedString)) result = expr.parseString(teststring) print(result.dump()) - self.assertEqual(result.asList(), expected , - "Lisp-ish comments (\";; <...> $\") and quoted strings didn't work. Expected: {}, got: {}".format(expected, result)) + self.assertEqual( + result.asList(), + expected, + 'Lisp-ish comments (";; <...> $") and quoted strings didn\'t work. Expected: {}, got: {}'.format( + expected, result + ), + ) def testWordExclude(self): from pyparsing import Word, printables + allButPunc = Word(printables, excludeChars=".,:;-_!?") test = "Hello, Mr. Ed, it's Wilbur!" result = allButPunc.searchString(test).asList() print(result) - self.assertEqual(result, [['Hello'], ['Mr'], ['Ed'], ["it's"], ['Wilbur']], "failed WordExcludeTest") + self.assertEqual( + result, + [["Hello"], ["Mr"], ["Ed"], ["it's"], ["Wilbur"]], + "failed WordExcludeTest", + ) def testParseAll(self): from pyparsing import Word, cppStyleComment @@ -2615,15 +3510,23 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): ("AAAAA", True, True), ("AAABB", False, True), ("AAABB", True, False), - ] + ] for s, parseAllFlag, shouldSucceed in tests: try: - print("'{}' parseAll={} (shouldSucceed={})".format(s, parseAllFlag, shouldSucceed)) + print( + "'{}' parseAll={} (shouldSucceed={})".format( + s, parseAllFlag, shouldSucceed + ) + ) testExpr.parseString(s, parseAll=parseAllFlag) - self.assertTrue(shouldSucceed, "successfully parsed when should have failed") + self.assertTrue( + shouldSucceed, "successfully parsed when should have failed" + ) except ParseException as pe: print(pe.explain(pe)) - self.assertFalse(shouldSucceed, "failed to parse when should have succeeded") + self.assertFalse( + shouldSucceed, "failed to parse when should have succeeded" + ) # add test for trailing comments testExpr.ignore(cppStyleComment) @@ -2633,39 +3536,62 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): ("AAAAA //blah", True, True), ("AAABB //blah", False, True), ("AAABB //blah", True, False), - ] + ] for s, parseAllFlag, shouldSucceed in tests: try: - print("'{}' parseAll={} (shouldSucceed={})".format(s, parseAllFlag, shouldSucceed)) + print( + "'{}' parseAll={} (shouldSucceed={})".format( + s, parseAllFlag, shouldSucceed + ) + ) testExpr.parseString(s, parseAll=parseAllFlag) - self.assertTrue(shouldSucceed, "successfully parsed when should have failed") + self.assertTrue( + shouldSucceed, "successfully parsed when should have failed" + ) except ParseException as pe: print(pe.explain(pe)) - self.assertFalse(shouldSucceed, "failed to parse when should have succeeded") + self.assertFalse( + shouldSucceed, "failed to parse when should have succeeded" + ) # add test with very long expression string # testExpr = pp.MatchFirst([pp.Literal(c) for c in pp.printables if c != 'B'])[1, ...] - anything_but_an_f = pp.OneOrMore(pp.MatchFirst([pp.Literal(c) for c in pp.printables if c != 'f'])) - testExpr = pp.Word('012') + anything_but_an_f + anything_but_an_f = pp.OneOrMore( + pp.MatchFirst([pp.Literal(c) for c in pp.printables if c != "f"]) + ) + testExpr = pp.Word("012") + anything_but_an_f tests = [ ("00aab", False, True), ("00aab", True, True), ("00aaf", False, True), ("00aaf", True, False), - ] + ] for s, parseAllFlag, shouldSucceed in tests: try: - print("'{}' parseAll={} (shouldSucceed={})".format(s, parseAllFlag, shouldSucceed)) + print( + "'{}' parseAll={} (shouldSucceed={})".format( + s, parseAllFlag, shouldSucceed + ) + ) testExpr.parseString(s, parseAll=parseAllFlag) - self.assertTrue(shouldSucceed, "successfully parsed when should have failed") + self.assertTrue( + shouldSucceed, "successfully parsed when should have failed" + ) except ParseException as pe: print(pe.explain(pe)) - self.assertFalse(shouldSucceed, "failed to parse when should have succeeded") - + self.assertFalse( + shouldSucceed, "failed to parse when should have succeeded" + ) def testGreedyQuotedStrings(self): - from pyparsing import QuotedString, sglQuotedString, dblQuotedString, quotedString, delimitedList + from pyparsing import ( + QuotedString, + sglQuotedString, + dblQuotedString, + quotedString, + delimitedList, + ) src = """\ "string1", "strin""g2" @@ -2673,29 +3599,41 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): ^string1^, ^string2^ <string1>, <string2>""" - testExprs = (sglQuotedString, dblQuotedString, quotedString, - QuotedString('"', escQuote='""'), QuotedString("'", escQuote="''"), - QuotedString("^"), QuotedString("<", endQuoteChar=">")) + testExprs = ( + sglQuotedString, + dblQuotedString, + quotedString, + QuotedString('"', escQuote='""'), + QuotedString("'", escQuote="''"), + QuotedString("^"), + QuotedString("<", endQuoteChar=">"), + ) for expr in testExprs: strs = delimitedList(expr).searchString(src) print(strs) - self.assertTrue(bool(strs), "no matches found for test expression '%s'" % expr) + self.assertTrue( + bool(strs), "no matches found for test expression '%s'" % expr + ) for lst in strs: - self.assertEqual(len(lst), 2, "invalid match found for test expression '%s'" % expr) + self.assertEqual( + len(lst), 2, "invalid match found for test expression '%s'" % expr + ) from pyparsing import alphas, nums, Word + src = """'ms1',1,0,'2009-12-22','2009-12-22 10:41:22') ON DUPLICATE KEY UPDATE sent_count = sent_count + 1, mtime = '2009-12-22 10:41:22';""" - tok_sql_quoted_value = ( - QuotedString("'", "\\", "''", True, False) ^ - QuotedString('"', "\\", '""', True, False)) + tok_sql_quoted_value = QuotedString( + "'", "\\", "''", True, False + ) ^ QuotedString('"', "\\", '""', True, False) tok_sql_computed_value = Word(nums) tok_sql_identifier = Word(alphas) val = tok_sql_quoted_value | tok_sql_computed_value | tok_sql_identifier vals = delimitedList(val) print(vals.parseString(src)) - self.assertEqual(len(vals.parseString(src)), 5, "error in greedy quote escaping") - + self.assertEqual( + len(vals.parseString(src)), 5, "error in greedy quote escaping" + ) def testWordBoundaryExpressions(self): from pyparsing import WordEnd, WordStart, oneOf @@ -2720,39 +3658,50 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): tests.append("\n".join(tests)) expectedResult = [ - [['D', 'G'], ['A'], ['C', 'F'], ['I'], ['E'], ['A', 'I']], - [['J', 'M', 'P'], [], ['L', 'R'], ['O'], [], ['O']], - [['S', 'V'], ['Y'], ['X', 'Z'], ['U'], [], ['U', 'Y']], - [['D', 'G', 'J', 'M', 'P', 'S', 'V'], - ['A', 'Y'], - ['C', 'F', 'L', 'R', 'X', 'Z'], - ['I', 'O', 'U'], - ['E'], - ['A', 'I', 'O', 'U', 'Y']], - ] + [["D", "G"], ["A"], ["C", "F"], ["I"], ["E"], ["A", "I"]], + [["J", "M", "P"], [], ["L", "R"], ["O"], [], ["O"]], + [["S", "V"], ["Y"], ["X", "Z"], ["U"], [], ["U", "Y"]], + [ + ["D", "G", "J", "M", "P", "S", "V"], + ["A", "Y"], + ["C", "F", "L", "R", "X", "Z"], + ["I", "O", "U"], + ["E"], + ["A", "I", "O", "U", "Y"], + ], + ] for t, expected in zip(tests, expectedResult): print(t) - results = [flatten(e.searchString(t).asList()) for e in [ - leadingConsonant, - leadingVowel, - trailingConsonant, - trailingVowel, - internalVowel, - bnf, - ]] + results = [ + flatten(e.searchString(t).asList()) + for e in [ + leadingConsonant, + leadingVowel, + trailingConsonant, + trailingVowel, + internalVowel, + bnf, + ] + ] print(results) print() - self.assertEqual(results, expected, "Failed WordBoundaryTest, expected {}, got {}".format(expected, results)) + self.assertEqual( + results, + expected, + "Failed WordBoundaryTest, expected {}, got {}".format( + expected, results + ), + ) def testRequiredEach(self): from pyparsing import Keyword - parser = Keyword('bam') & Keyword('boo') + parser = Keyword("bam") & Keyword("boo") try: - res1 = parser.parseString('bam boo') + res1 = parser.parseString("bam boo") print(res1.asList()) - res2 = parser.parseString('boo bam') + res2 = parser.parseString("boo bam") print(res2.asList()) except ParseException: failed = True @@ -2760,38 +3709,55 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): failed = False self.assertFalse(failed, "invalid logic in Each") - self.assertEqual(set(res1), set(res2), "Failed RequiredEachTest, expected " - + str(res1.asList()) + " and " + str(res2.asList()) - + "to contain same words in any order") + self.assertEqual( + set(res1), + set(res2), + "Failed RequiredEachTest, expected " + + str(res1.asList()) + + " and " + + str(res2.asList()) + + "to contain same words in any order", + ) def testOptionalEachTest1(self): from pyparsing import Optional, Keyword the_input = "Major Tal Weiss" - parser1 = (Optional('Tal') + Optional('Weiss')) & Keyword('Major') - parser2 = Optional(Optional('Tal') + Optional('Weiss')) & Keyword('Major') + parser1 = (Optional("Tal") + Optional("Weiss")) & Keyword("Major") + parser2 = Optional(Optional("Tal") + Optional("Weiss")) & Keyword("Major") p1res = parser1.parseString(the_input) p2res = parser2.parseString(the_input) - self.assertEqual(p1res.asList(), p2res.asList(), - "Each failed to match with nested Optionals, " - + str(p1res.asList()) + " should match " + str(p2res.asList())) + self.assertEqual( + p1res.asList(), + p2res.asList(), + "Each failed to match with nested Optionals, " + + str(p1res.asList()) + + " should match " + + str(p2res.asList()), + ) def testOptionalEachTest2(self): from pyparsing import Word, alphanums, OneOrMore, Group, Regex, Optional - word = Word(alphanums + '_').setName("word") - with_stmt = 'with' + OneOrMore(Group(word('key') + '=' + word('value')))('overrides') - using_stmt = 'using' + Regex('id-[0-9a-f]{8}')('id') - modifiers = Optional(with_stmt('with_stmt')) & Optional(using_stmt('using_stmt')) + word = Word(alphanums + "_").setName("word") + with_stmt = "with" + OneOrMore(Group(word("key") + "=" + word("value")))( + "overrides" + ) + using_stmt = "using" + Regex("id-[0-9a-f]{8}")("id") + modifiers = Optional(with_stmt("with_stmt")) & Optional( + using_stmt("using_stmt") + ) self.assertEqual(modifiers, "with foo=bar bing=baz using id-deadbeef") - self.assertNotEqual(modifiers, "with foo=bar bing=baz using id-deadbeef using id-feedfeed") + self.assertNotEqual( + modifiers, "with foo=bar bing=baz using id-deadbeef using id-feedfeed" + ) def testOptionalEachTest3(self): from pyparsing import Literal, Suppress - foo = Literal('foo') - bar = Literal('bar') + foo = Literal("foo") + bar = Literal("bar") openBrace = Suppress(Literal("{")) closeBrace = Suppress(Literal("}")) @@ -2807,51 +3773,74 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): if not test: continue result = exp.parseString(test) - print(test, '->', result.asList()) - self.assertEqual(result.asList(), test.strip("{}").split(), "failed to parse Each expression %r" % test) + print(test, "->", result.asList()) + self.assertEqual( + result.asList(), + test.strip("{}").split(), + "failed to parse Each expression %r" % test, + ) print(result.dump()) try: result = exp.parseString("{bar}") - self.assertTrue(False, "failed to raise exception when required element is missing") + self.assertTrue( + False, "failed to raise exception when required element is missing" + ) except ParseException as pe: pass def testOptionalEachTest4(self): from pyparsing import pyparsing_common, Group - expr = ((~pyparsing_common.iso8601_date + pyparsing_common.integer("id")) - & (Group(pyparsing_common.iso8601_date)("date*")[...])) + expr = (~pyparsing_common.iso8601_date + pyparsing_common.integer("id")) & ( + Group(pyparsing_common.iso8601_date)("date*")[...] + ) - expr.runTests(""" + expr.runTests( + """ 1999-12-31 100 2001-01-01 42 - """) - + """ + ) def testEachWithParseFatalException(self): import pyparsing as pp + ppc = pp.pyparsing_common - option_expr = pp.Keyword('options') - '(' + ppc.integer + ')' - step_expr1 = pp.Keyword('step') - '(' + ppc.integer + ")" - step_expr2 = pp.Keyword('step') - '(' + ppc.integer + "Z" + ")" + option_expr = pp.Keyword("options") - "(" + ppc.integer + ")" + step_expr1 = pp.Keyword("step") - "(" + ppc.integer + ")" + step_expr2 = pp.Keyword("step") - "(" + ppc.integer + "Z" + ")" step_expr = step_expr1 ^ step_expr2 parser = option_expr & step_expr[...] tests = [ - ("options(100) step(A)", "Expected integer, found 'A' (at char 18), (line:1, col:19)"), - ("step(A) options(100)", "Expected integer, found 'A' (at char 5), (line:1, col:6)"), - ("options(100) step(100A)", """Expected "Z", found 'A' (at char 21), (line:1, col:22)"""), - ("options(100) step(22) step(100ZA)", - """Expected ")", found 'A' (at char 31), (line:1, col:32)"""), + ( + "options(100) step(A)", + "Expected integer, found 'A' (at char 18), (line:1, col:19)", + ), + ( + "step(A) options(100)", + "Expected integer, found 'A' (at char 5), (line:1, col:6)", + ), + ( + "options(100) step(100A)", + """Expected "Z", found 'A' (at char 21), (line:1, col:22)""", + ), + ( + "options(100) step(22) step(100ZA)", + """Expected ")", found 'A' (at char 31), (line:1, col:32)""", + ), ] test_lookup = dict(tests) success, output = parser.runTests((t[0] for t in tests), failureTests=True) for test_str, result in output: - self.assertEqual(test_lookup[test_str], str(result), - "incorrect exception raised for test string {!r}".format(test_str)) + self.assertEqual( + test_lookup[test_str], + str(result), + "incorrect exception raised for test string {!r}".format(test_str), + ) def testSumParseResults(self): @@ -2866,14 +3855,25 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): res4 = "ID:PARI12345678 DOB: INFO: I am cool" from pyparsing import Regex, Word, alphanums, restOfLine + dob_ref = "DOB" + Regex(r"\d{2}-\d{2}-\d{4}")("dob") id_ref = "ID" + Word(alphanums, exact=12)("id") info_ref = "-" + restOfLine("info") person_data = dob_ref | id_ref | info_ref - tests = (samplestr1, samplestr2, samplestr3, samplestr4,) - results = (res1, res2, res3, res4,) + tests = ( + samplestr1, + samplestr2, + samplestr3, + samplestr4, + ) + results = ( + res1, + res2, + res3, + res4, + ) for test, expected in zip(tests, results): person = sum(person_data.searchString(test)) result = "ID:{} DOB:{} INFO:{}".format(person.id, person.dob, person.info) @@ -2883,14 +3883,20 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): for pd in person_data.searchString(test): print(pd.dump()) print() - self.assertEqual(expected, result, - "Failed to parse '{}' correctly, \nexpected '{}', got '{}'".format(test, expected, result)) + self.assertEqual( + expected, + result, + "Failed to parse '{}' correctly, \nexpected '{}', got '{}'".format( + test, expected, result + ), + ) def testMarkInputLine(self): samplestr1 = "DOB 100-10-2010;more garbage\nID PARI12345678;more garbage" from pyparsing import Regex + dob_ref = "DOB" + Regex(r"\d{2}-\d{2}-\d{4}")("dob") try: @@ -2898,9 +3904,15 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): except ParseException as pe: outstr = pe.markInputline() print(outstr) - self.assertEqual(outstr, "DOB >!<100-10-2010;more garbage", "did not properly create marked input line") + self.assertEqual( + outstr, + "DOB >!<100-10-2010;more garbage", + "did not properly create marked input line", + ) else: - self.assertEqual(False, "test construction failed - should have raised an exception") + self.assertEqual( + False, "test construction failed - should have raised an exception" + ) def testLocatedExpr(self): @@ -2908,12 +3920,16 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): samplestr1 = "DOB 10-10-2010;more garbage;ID PARI12345678 ;more garbage" from pyparsing import Word, alphanums, locatedExpr + id_ref = locatedExpr("ID" + Word(alphanums, exact=12)("id")) res = id_ref.searchString(samplestr1)[0][0] print(res.dump()) - self.assertEqual(samplestr1[res.locn_start:res.locn_end], 'ID PARI12345678', "incorrect location calculation") - + self.assertEqual( + samplestr1[res.locn_start : res.locn_end], + "ID PARI12345678", + "incorrect location calculation", + ) def testPop(self): from pyparsing import Word, alphas, nums @@ -2923,11 +3939,11 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): result = patt.parseString(source) tests = [ - (0, 'AAA', ['123', '456', '789', '234']), - (None, '234', ['123', '456', '789']), - ('name', 'AAA', ['123', '456', '789']), - (-1, '789', ['123', '456']), - ] + (0, "AAA", ["123", "456", "789", "234"]), + (None, "234", ["123", "456", "789"]), + ("name", "AAA", ["123", "456", "789"]), + (-1, "789", ["123", "456"]), + ] for test in tests: idx, val, remaining = test if idx is not None: @@ -2937,20 +3953,38 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): print("EXP:", val, remaining) print("GOT:", ret, result.asList()) print(ret, result.asList()) - self.assertEqual(ret, val, "wrong value returned, got {!r}, expected {!r}".format(ret, val)) - self.assertEqual(remaining, result.asList(), - "list is in wrong state after pop, got {!r}, expected {!r}".format(result.asList(), remaining)) + self.assertEqual( + ret, + val, + "wrong value returned, got {!r}, expected {!r}".format(ret, val), + ) + self.assertEqual( + remaining, + result.asList(), + "list is in wrong state after pop, got {!r}, expected {!r}".format( + result.asList(), remaining + ), + ) print() prevlist = result.asList() - ret = result.pop('name', default="noname") + ret = result.pop("name", default="noname") print(ret) print(result.asList()) - self.assertEqual(ret, "noname", - "default value not successfully returned, got {!r}, expected {!r}".format(ret, "noname")) - self.assertEqual(result.asList(), prevlist, - "list is in wrong state after pop, got {!r}, expected {!r}".format(result.asList(), remaining)) - + self.assertEqual( + ret, + "noname", + "default value not successfully returned, got {!r}, expected {!r}".format( + ret, "noname" + ), + ) + self.assertEqual( + result.asList(), + prevlist, + "list is in wrong state after pop, got {!r}, expected {!r}".format( + result.asList(), remaining + ), + ) def testAddCondition(self): from pyparsing import Word, nums, Suppress, ParseFatalException @@ -2962,26 +3996,40 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): result = numParser.searchString("1 2 3 4 5 6 7 8 9 10") print(result.asList()) - self.assertEqual(result.asList(), [[7], [9]], "failed to properly process conditions") + self.assertEqual( + result.asList(), [[7], [9]], "failed to properly process conditions" + ) numParser = Word(nums) numParser.addParseAction(lambda s, l, t: int(t[0])) - rangeParser = (numParser("from_") + Suppress('-') + numParser("to")) + rangeParser = numParser("from_") + Suppress("-") + numParser("to") result = rangeParser.searchString("1-4 2-4 4-3 5 6 7 8 9 10") print(result.asList()) - self.assertEqual(result.asList(), [[1, 4], [2, 4], [4, 3]], "failed to properly process conditions") - - rangeParser.addCondition(lambda t: t.to > t.from_, message="from must be <= to", fatal=False) + self.assertEqual( + result.asList(), + [[1, 4], [2, 4], [4, 3]], + "failed to properly process conditions", + ) + + rangeParser.addCondition( + lambda t: t.to > t.from_, message="from must be <= to", fatal=False + ) result = rangeParser.searchString("1-4 2-4 4-3 5 6 7 8 9 10") print(result.asList()) - self.assertEqual(result.asList(), [[1, 4], [2, 4]], "failed to properly process conditions") - - rangeParser = (numParser("from_") + Suppress('-') + numParser("to")) - rangeParser.addCondition(lambda t: t.to > t.from_, message="from must be <= to", fatal=True) + self.assertEqual( + result.asList(), [[1, 4], [2, 4]], "failed to properly process conditions" + ) + + rangeParser = numParser("from_") + Suppress("-") + numParser("to") + rangeParser.addCondition( + lambda t: t.to > t.from_, message="from must be <= to", fatal=True + ) try: result = rangeParser.searchString("1-4 2-4 4-3 5 6 7 8 9 10") - self.assertTrue(False, "failed to interrupt parsing on fatal condition failure") + self.assertTrue( + False, "failed to interrupt parsing on fatal condition failure" + ) except ParseFatalException: print("detected fatal condition") @@ -2996,72 +4044,96 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): raise pp.ParseException("signalling invalid token") return token - a = pp.Word("de").setName("Word")#.setDebug() - b = pp.Literal("def").setName("Literal").setParseAction(validate)#.setDebug() - c = pp.Literal("d").setName("d")#.setDebug() + a = pp.Word("de").setName("Word") # .setDebug() + b = pp.Literal("def").setName("Literal").setParseAction(validate) # .setDebug() + c = pp.Literal("d").setName("d") # .setDebug() # The "Literal" expressions's ParseAction is not executed directly after syntactically # detecting the "Literal" Expression but only after the Or-decision has been made # (which is too late)... try: result = (a ^ b ^ c).parseString("def") - self.assertEqual(result.asList(), ['de'], "failed to select longest match, chose %s" % result) + self.assertEqual( + result.asList(), + ["de"], + "failed to select longest match, chose %s" % result, + ) except ParseException: failed = True else: failed = False - self.assertFalse(failed, "invalid logic in Or, fails on longest match with exception in parse action") + self.assertFalse( + failed, + "invalid logic in Or, fails on longest match with exception in parse action", + ) # from issue #93 - word = pp.Word(pp.alphas).setName('word') - word_1 = pp.Word(pp.alphas).setName('word_1').addCondition(lambda t: len(t[0]) == 1) + word = pp.Word(pp.alphas).setName("word") + word_1 = ( + pp.Word(pp.alphas).setName("word_1").addCondition(lambda t: len(t[0]) == 1) + ) a = word + (word_1 + word ^ word) b = word * 3 c = a ^ b c.streamline() print(c) - test_string = 'foo bar temp' + test_string = "foo bar temp" result = c.parseString(test_string) - print(test_string, '->', result.asList()) - - self.assertEqual(result.asList(), test_string.split(), "failed to match longest choice") + print(test_string, "->", result.asList()) + self.assertEqual( + result.asList(), test_string.split(), "failed to match longest choice" + ) def testEachWithOptionalWithResultsName(self): from pyparsing import Optional - result = (Optional('foo')('one') & Optional('bar')('two')).parseString('bar foo') + result = (Optional("foo")("one") & Optional("bar")("two")).parseString( + "bar foo" + ) print(result.dump()) - self.assertEqual(sorted(result.keys()), ['one', 'two']) + self.assertEqual(sorted(result.keys()), ["one", "two"]) def testUnicodeExpression(self): from pyparsing import Literal, ParseException - z = 'a' | Literal('\u1111') + z = "a" | Literal("\u1111") z.streamline() try: - z.parseString('b') + z.parseString("b") except ParseException as pe: - self.assertEqual(pe.msg, r'''Expected {"a" | "ᄑ"}''', - "Invalid error message raised, got %r" % pe.msg) + self.assertEqual( + pe.msg, + r"""Expected {"a" | "ᄑ"}""", + "Invalid error message raised, got %r" % pe.msg, + ) def testSetName(self): - from pyparsing import (oneOf, infixNotation, Word, nums, opAssoc, delimitedList, countedArray, - nestedExpr, makeHTMLTags, anyOpenTag, anyCloseTag, commonHTMLEntity, replaceHTMLEntity, - Forward) + from pyparsing import ( + oneOf, + infixNotation, + Word, + nums, + opAssoc, + delimitedList, + countedArray, + nestedExpr, + makeHTMLTags, + anyOpenTag, + anyCloseTag, + commonHTMLEntity, + replaceHTMLEntity, + Forward, + ) a = oneOf("a b c") b = oneOf("d e f") - arith_expr = infixNotation(Word(nums), - [ - (oneOf('* /'), 2, opAssoc.LEFT), - (oneOf('+ -'), 2, opAssoc.LEFT), - ]) - arith_expr2 = infixNotation(Word(nums), - [ - (('?', ':'), 3, opAssoc.LEFT), - ]) + arith_expr = infixNotation( + Word(nums), + [(oneOf("* /"), 2, opAssoc.LEFT), (oneOf("+ -"), 2, opAssoc.LEFT),], + ) + arith_expr2 = infixNotation(Word(nums), [(("?", ":"), 3, opAssoc.LEFT),]) recursive = Forward() recursive <<= a + (b + recursive)[...] @@ -3077,13 +4149,17 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): delimitedList(Word(nums).setName("int")), countedArray(Word(nums).setName("int")), nestedExpr(), - makeHTMLTags('Z'), + makeHTMLTags("Z"), (anyOpenTag, anyCloseTag), commonHTMLEntity, - commonHTMLEntity.setParseAction(replaceHTMLEntity).transformString("lsdjkf <lsdjkf>&'"&xyzzy;"), - ] + commonHTMLEntity.setParseAction(replaceHTMLEntity).transformString( + "lsdjkf <lsdjkf>&'"&xyzzy;" + ), + ] - expected = map(str.strip, """\ + expected = map( + str.strip, + """\ a | b | c d | e | f {a | b | c | d | e | f} @@ -3098,22 +4174,31 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): (<Z>, </Z>) (<any tag>, </any tag>) common HTML entity - lsdjkf <lsdjkf>&'"&xyzzy;""".splitlines()) + lsdjkf <lsdjkf>&'"&xyzzy;""".splitlines(), + ) for t, e in zip(tests, expected): tname = str(t) print(tname) - self.assertEqual(tname, e, "expression name mismatch, expected {} got {}".format(e, tname)) + self.assertEqual( + tname, + e, + "expression name mismatch, expected {} got {}".format(e, tname), + ) def testTrimArityExceptionMasking(self): from pyparsing import Word invalid_message = "<lambda>() missing 1 required positional argument: 't'" try: - Word('a').setParseAction(lambda t: t[0] + 1).parseString('aaa') + Word("a").setParseAction(lambda t: t[0] + 1).parseString("aaa") except Exception as e: exc_msg = str(e) - self.assertNotEqual(exc_msg, invalid_message, "failed to catch TypeError thrown in _trim_arity") + self.assertNotEqual( + exc_msg, + invalid_message, + "failed to catch TypeError thrown in _trim_arity", + ) def testTrimArityExceptionMaskingTest2(self): # construct deep call tree @@ -3126,11 +4211,14 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): invalid_message = "<lambda>() missing 1 required positional argument: 't'" try: - Word('a').setParseAction(lambda t: t[0] + 1).parseString('aaa') + Word("a").setParseAction(lambda t: t[0] + 1).parseString("aaa") except Exception as e: exc_msg = str(e) - self.assertNotEqual(exc_msg, invalid_message, "failed to catch TypeError thrown in _trim_arity") - + self.assertNotEqual( + exc_msg, + invalid_message, + "failed to catch TypeError thrown in _trim_arity", + ) def B(): A() @@ -3161,59 +4249,96 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): K() - def testClearParseActions(self): import pyparsing as pp + ppc = pp.pyparsing_common realnum = ppc.real() - self.assertEqual(realnum.parseString("3.14159")[0], 3.14159, "failed basic real number parsing") + self.assertEqual( + realnum.parseString("3.14159")[0], + 3.14159, + "failed basic real number parsing", + ) # clear parse action that converts to float realnum.setParseAction(None) - self.assertEqual(realnum.parseString("3.14159")[0], "3.14159", "failed clearing parse action") + self.assertEqual( + realnum.parseString("3.14159")[0], "3.14159", "failed clearing parse action" + ) # add a new parse action that tests if a '.' is prsent - realnum.addParseAction(lambda t: '.' in t[0]) - self.assertEqual(realnum.parseString("3.14159")[0], True, - "failed setting new parse action after clearing parse action") + realnum.addParseAction(lambda t: "." in t[0]) + self.assertEqual( + realnum.parseString("3.14159")[0], + True, + "failed setting new parse action after clearing parse action", + ) def testOneOrMoreStop(self): - from pyparsing import (Word, OneOrMore, alphas, Keyword, CaselessKeyword, - nums, alphanums) + from pyparsing import ( + Word, + OneOrMore, + alphas, + Keyword, + CaselessKeyword, + nums, + alphanums, + ) test = "BEGIN aaa bbb ccc END" - BEGIN, END = map(Keyword, "BEGIN,END".split(',')) + BEGIN, END = map(Keyword, "BEGIN,END".split(",")) body_word = Word(alphas).setName("word") for ender in (END, "END", CaselessKeyword("END")): expr = BEGIN + OneOrMore(body_word, stopOn=ender) + END - self.assertEqual(test, expr, "Did not successfully stop on ending expression %r" % ender) + self.assertEqual( + test, expr, "Did not successfully stop on ending expression %r" % ender + ) expr = BEGIN + body_word[...].stopOn(ender) + END - self.assertEqual(test, expr, "Did not successfully stop on ending expression %r" % ender) + self.assertEqual( + test, expr, "Did not successfully stop on ending expression %r" % ender + ) - number = Word(nums + ',.()').setName("number with optional commas") - parser= (OneOrMore(Word(alphanums + '-/.'), stopOn=number)('id').setParseAction(' '.join) - + number('data')) - result = parser.parseString(' XXX Y/123 1,234.567890') - self.assertEqual(result.asList(), ['XXX Y/123', '1,234.567890'], - "Did not successfully stop on ending expression %r" % number) + number = Word(nums + ",.()").setName("number with optional commas") + parser = OneOrMore(Word(alphanums + "-/."), stopOn=number)("id").setParseAction( + " ".join + ) + number("data") + result = parser.parseString(" XXX Y/123 1,234.567890") + self.assertEqual( + result.asList(), + ["XXX Y/123", "1,234.567890"], + "Did not successfully stop on ending expression %r" % number, + ) def testZeroOrMoreStop(self): - from pyparsing import (Word, ZeroOrMore, alphas, Keyword, CaselessKeyword) + from pyparsing import Word, ZeroOrMore, alphas, Keyword, CaselessKeyword test = "BEGIN END" - BEGIN, END = map(Keyword, "BEGIN,END".split(',')) + BEGIN, END = map(Keyword, "BEGIN,END".split(",")) body_word = Word(alphas).setName("word") for ender in (END, "END", CaselessKeyword("END")): expr = BEGIN + ZeroOrMore(body_word, stopOn=ender) + END - self.assertEqual(test, expr, "Did not successfully stop on ending expression %r" % ender) + self.assertEqual( + test, expr, "Did not successfully stop on ending expression %r" % ender + ) expr = BEGIN + body_word[0, ...].stopOn(ender) + END - self.assertEqual(test, expr, "Did not successfully stop on ending expression %r" % ender) + self.assertEqual( + test, expr, "Did not successfully stop on ending expression %r" % ender + ) def testNestedAsDict(self): - from pyparsing import Literal, Forward, alphanums, Group, delimitedList, Dict, Word, Optional + from pyparsing import ( + Literal, + Forward, + alphanums, + Group, + delimitedList, + Dict, + Word, + Optional, + ) equals = Literal("=").suppress() lbracket = Literal("[").suppress() @@ -3221,30 +4346,39 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): lbrace = Literal("{").suppress() rbrace = Literal("}").suppress() - value_dict = Forward() - value_list = Forward() - value_string = Word(alphanums + "@. ") + value_dict = Forward() + value_list = Forward() + value_string = Word(alphanums + "@. ") - value = value_list ^ value_dict ^ value_string - values = Group(delimitedList(value, ",")) - #~ values = delimitedList(value, ",").setParseAction(lambda toks: [toks.asList()]) + value = value_list ^ value_dict ^ value_string + values = Group(delimitedList(value, ",")) + # ~ values = delimitedList(value, ",").setParseAction(lambda toks: [toks.asList()]) - value_list << lbracket + values + rbracket + value_list << lbracket + values + rbracket - identifier = Word(alphanums + "_.") + identifier = Word(alphanums + "_.") - assignment = Group(identifier + equals + Optional(value)) - assignments = Dict(delimitedList(assignment, ';')) - value_dict << lbrace + assignments + rbrace + assignment = Group(identifier + equals + Optional(value)) + assignments = Dict(delimitedList(assignment, ";")) + value_dict << lbrace + assignments + rbrace response = assignments - rsp = 'username=goat; errors={username=[already taken, too short]}; empty_field=' + rsp = ( + "username=goat; errors={username=[already taken, too short]}; empty_field=" + ) result_dict = response.parseString(rsp).asDict() print(result_dict) - self.assertEqual(result_dict['username'], 'goat', "failed to process string in ParseResults correctly") - self.assertEqual(result_dict['errors']['username'], ['already taken', 'too short'], - "failed to process nested ParseResults correctly") + self.assertEqual( + result_dict["username"], + "goat", + "failed to process string in ParseResults correctly", + ) + self.assertEqual( + result_dict["errors"]["username"], + ["already taken", "too short"], + "failed to process nested ParseResults correctly", + ) def testTraceParseActionDecorator(self): from pyparsing import traceParseAction, Word, nums @@ -3265,9 +4399,13 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): def testRunTests(self): from pyparsing import Word, nums, delimitedList - integer = Word(nums).setParseAction(lambda t : int(t[0])) - intrange = integer("start") + '-' + integer("end") - intrange.addCondition(lambda t: t.end > t.start, message="invalid range, start must be <= end", fatal=True) + integer = Word(nums).setParseAction(lambda t: int(t[0])) + intrange = integer("start") + "-" + integer("end") + intrange.addCondition( + lambda t: t.end > t.start, + message="invalid range, start must be <= end", + fatal=True, + ) intrange.addParseAction(lambda t: list(range(t.start, t.end + 1))) indices = delimitedList(intrange | integer) @@ -3284,7 +4422,7 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): expectedResults = [ [1, 2, 3, 4, 6, 8, 9, 10, 16], [11], - ] + ] for res, expected in zip(results, expectedResults): print(res[1].asList()) print(expected) @@ -3301,57 +4439,74 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): import pyparsing as pp integer = pp.pyparsing_common.integer - fraction = integer('numerator') + '/' + integer('denominator') + fraction = integer("numerator") + "/" + integer("denominator") accum = [] + def eval_fraction(test, result): accum.append((test, result.asList())) return "eval: {}".format(result.numerator / result.denominator) - success = fraction.runTests("""\ + success = fraction.runTests( + """\ 1/2 1/0 - """, postParse=eval_fraction)[0] + """, + postParse=eval_fraction, + )[0] print(success) self.assertTrue(success, "failed to parse fractions in RunTestsPostParse") - expected_accum = [('1/2', [1, '/', 2]), ('1/0', [1, '/', 0])] - self.assertEqual(accum, expected_accum, "failed to call postParse method during runTests") + expected_accum = [("1/2", [1, "/", 2]), ("1/0", [1, "/", 0])] + self.assertEqual( + accum, expected_accum, "failed to call postParse method during runTests" + ) def testCommonExpressions(self): from pyparsing import pyparsing_common import ast - success = pyparsing_common.mac_address.runTests(""" + success = pyparsing_common.mac_address.runTests( + """ AA:BB:CC:DD:EE:FF AA.BB.CC.DD.EE.FF AA-BB-CC-DD-EE-FF - """)[0] + """ + )[0] self.assertTrue(success, "error in parsing valid MAC address") - success = pyparsing_common.mac_address.runTests(""" + success = pyparsing_common.mac_address.runTests( + """ # mixed delimiters AA.BB:CC:DD:EE:FF - """, failureTests=True)[0] + """, + failureTests=True, + )[0] self.assertTrue(success, "error in detecting invalid mac address") - success = pyparsing_common.ipv4_address.runTests(""" + success = pyparsing_common.ipv4_address.runTests( + """ 0.0.0.0 1.1.1.1 127.0.0.1 1.10.100.199 255.255.255.255 - """)[0] + """ + )[0] self.assertTrue(success, "error in parsing valid IPv4 address") - success = pyparsing_common.ipv4_address.runTests(""" + success = pyparsing_common.ipv4_address.runTests( + """ # out of range value 256.255.255.255 - """, failureTests=True)[0] + """, + failureTests=True, + )[0] self.assertTrue(success, "error in detecting invalid IPv4 address") - success = pyparsing_common.ipv6_address.runTests(""" + success = pyparsing_common.ipv6_address.runTests( + """ 2001:0db8:85a3:0000:0000:8a2e:0370:7334 2134::1234:4567:2468:1236:2444:2106 0:0:0:0:0:0:A00:1 @@ -3366,94 +4521,132 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): # ipv4 compatibility form ::ffff:192.168.0.1 - """)[0] + """ + )[0] self.assertTrue(success, "error in parsing valid IPv6 address") - success = pyparsing_common.ipv6_address.runTests(""" + success = pyparsing_common.ipv6_address.runTests( + """ # too few values 1080:0:0:0:8:800:200C # too many ::'s, only 1 allowed 2134::1234:4567::2444:2106 - """, failureTests=True)[0] + """, + failureTests=True, + )[0] self.assertTrue(success, "error in detecting invalid IPv6 address") - success = pyparsing_common.number.runTests(""" + success = pyparsing_common.number.runTests( + """ 100 -100 +100 3.14159 6.02e23 1e-12 - """)[0] + """ + )[0] self.assertTrue(success, "error in parsing valid numerics") - success = pyparsing_common.sci_real.runTests(""" + success = pyparsing_common.sci_real.runTests( + """ 1e12 -1e12 3.14159 6.02e23 - """)[0] + """ + )[0] self.assertTrue(success, "error in parsing valid scientific notation reals") # any int or real number, returned as float - success = pyparsing_common.fnumber.runTests(""" + success = pyparsing_common.fnumber.runTests( + """ 100 -100 +100 3.14159 6.02e23 1e-12 - """)[0] + """ + )[0] self.assertTrue(success, "error in parsing valid numerics") - success, results = pyparsing_common.iso8601_date.runTests(""" + success, results = pyparsing_common.iso8601_date.runTests( + """ 1997 1997-07 1997-07-16 - """) + """ + ) self.assertTrue(success, "error in parsing valid iso8601_date") expected = [ - ('1997', None, None), - ('1997', '07', None), - ('1997', '07', '16'), + ("1997", None, None), + ("1997", "07", None), + ("1997", "07", "16"), ] for r, exp in zip(results, expected): - self.assertTrue((r[1].year, r[1].month, r[1].day,) == exp, "failed to parse date into fields") + self.assertTrue( + (r[1].year, r[1].month, r[1].day,) == exp, + "failed to parse date into fields", + ) - success, results = pyparsing_common.iso8601_date().addParseAction(pyparsing_common.convertToDate()).runTests(""" + success, results = ( + pyparsing_common.iso8601_date() + .addParseAction(pyparsing_common.convertToDate()) + .runTests( + """ 1997-07-16 - """) - self.assertTrue(success, "error in parsing valid iso8601_date with parse action") + """ + ) + ) + self.assertTrue( + success, "error in parsing valid iso8601_date with parse action" + ) self.assertTrue(results[0][1][0] == datetime.date(1997, 7, 16)) - success, results = pyparsing_common.iso8601_datetime.runTests(""" + success, results = pyparsing_common.iso8601_datetime.runTests( + """ 1997-07-16T19:20+01:00 1997-07-16T19:20:30+01:00 1997-07-16T19:20:30.45Z 1997-07-16 19:20:30.45 - """) + """ + ) self.assertTrue(success, "error in parsing valid iso8601_datetime") - success, results = pyparsing_common.iso8601_datetime().addParseAction(pyparsing_common.convertToDatetime()).runTests(""" + success, results = ( + pyparsing_common.iso8601_datetime() + .addParseAction(pyparsing_common.convertToDatetime()) + .runTests( + """ 1997-07-16T19:20:30.45 - """) + """ + ) + ) self.assertTrue(success, "error in parsing valid iso8601_datetime") - self.assertTrue(results[0][1][0] == datetime.datetime(1997, 7, 16, 19, 20, 30, 450000)) + self.assertTrue( + results[0][1][0] == datetime.datetime(1997, 7, 16, 19, 20, 30, 450000) + ) - success = pyparsing_common.uuid.runTests(""" + success = pyparsing_common.uuid.runTests( + """ 123e4567-e89b-12d3-a456-426655440000 - """)[0] + """ + )[0] self.assertTrue(success, "failed to parse valid uuid") - success = pyparsing_common.fraction.runTests(""" + success = pyparsing_common.fraction.runTests( + """ 1/2 -15/16 -3/-4 - """)[0] + """ + )[0] self.assertTrue(success, "failed to parse valid fraction") - success = pyparsing_common.mixed_integer.runTests(""" + success = pyparsing_common.mixed_integer.runTests( + """ 1/2 -15/16 -3/-4 @@ -3461,25 +4654,40 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): 2 -15/16 0 -3/-4 12 - """)[0] + """ + )[0] self.assertTrue(success, "failed to parse valid mixed integer") - success, results = pyparsing_common.number.runTests(""" + success, results = pyparsing_common.number.runTests( + """ 100 -3 1.732 -3.14159 - 6.02e23""") + 6.02e23""" + ) self.assertTrue(success, "failed to parse numerics") for test, result in results: expected = ast.literal_eval(test) - self.assertEqual(result[0], expected, "numeric parse failed (wrong value) ({} should be {})".format(result[0], expected)) - self.assertEqual(type(result[0]), type(expected), "numeric parse failed (wrong type) ({} should be {})".format(type(result[0]), type(expected))) - + self.assertEqual( + result[0], + expected, + "numeric parse failed (wrong value) ({} should be {})".format( + result[0], expected + ), + ) + self.assertEqual( + type(result[0]), + type(expected), + "numeric parse failed (wrong type) ({} should be {})".format( + type(result[0]), type(expected) + ), + ) def testNumericExpressions(self): import pyparsing as pp + ppc = pp.pyparsing_common # disable parse actions that do type conversion so we don't accidentally trigger @@ -3491,26 +4699,36 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): from itertools import product def make_tests(): - leading_sign = ['+', '-', ''] - leading_digit = ['0', ''] - dot = ['.', ''] - decimal_digit = ['1', ''] - e = ['e', 'E', ''] - e_sign = ['+', '-', ''] - e_int = ['22', ''] - stray = ['9', '.', ''] + leading_sign = ["+", "-", ""] + leading_digit = ["0", ""] + dot = [".", ""] + decimal_digit = ["1", ""] + e = ["e", "E", ""] + e_sign = ["+", "-", ""] + e_int = ["22", ""] + stray = ["9", ".", ""] seen = set() - seen.add('') - for parts in product(leading_sign, stray, leading_digit, dot, decimal_digit, stray, e, e_sign, e_int, - stray): - parts_str = ''.join(parts).strip() + seen.add("") + for parts in product( + leading_sign, + stray, + leading_digit, + dot, + decimal_digit, + stray, + e, + e_sign, + e_int, + stray, + ): + parts_str = "".join(parts).strip() if parts_str in seen: continue seen.add(parts_str) yield parts_str - print(len(seen)-1, "tests produced") + print(len(seen) - 1, "tests produced") # collect tests into valid/invalid sets, depending on whether they evaluate to valid Python floats or ints valid_ints = set() @@ -3522,16 +4740,16 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): # check which strings parse as valid floats or ints, and store in related valid or invalid test sets for test_str in make_tests(): - if '.' in test_str or 'e' in test_str.lower(): + if "." in test_str or "e" in test_str.lower(): try: float(test_str) except ValueError: invalid_sci_reals.add(test_str) - if 'e' not in test_str.lower(): + if "e" not in test_str.lower(): invalid_reals.add(test_str) else: valid_sci_reals.add(test_str) - if 'e' not in test_str.lower(): + if "e" not in test_str.lower(): valid_reals.add(test_str) try: @@ -3543,12 +4761,20 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): # now try all the test sets against their respective expressions all_pass = True - suppress_results = {'printResults': False} - for expr, tests, is_fail, fn in zip([real, sci_real, signed_integer] * 2, - [valid_reals, valid_sci_reals, valid_ints, - invalid_reals, invalid_sci_reals, invalid_ints], - [False, False, False, True, True, True], - [float, float, int] * 2): + suppress_results = {"printResults": False} + for expr, tests, is_fail, fn in zip( + [real, sci_real, signed_integer] * 2, + [ + valid_reals, + valid_sci_reals, + valid_ints, + invalid_reals, + invalid_sci_reals, + invalid_ints, + ], + [False, False, False, True, True, True], + [float, float, int] * 2, + ): # # success, test_results = expr.runTests(sorted(tests, key=len), failureTests=is_fail, **suppress_results) # filter_result_fn = (lambda r: isinstance(r, Exception), @@ -3576,8 +4802,11 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): if not is_fail: print(t, "should not fail but did") success = False - print(expr, ('FAIL', 'PASS')[success], "{}valid tests ({})".format('in' if is_fail else '', - len(tests),)) + print( + expr, + ("FAIL", "PASS")[success], + "{}valid tests ({})".format("in" if is_fail else "", len(tests),), + ) all_pass = all_pass and success self.assertTrue(all_pass, "failed one or more numeric tests") @@ -3586,9 +4815,11 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): from pyparsing import tokenMap, Word, hexnums, OneOrMore parser = OneOrMore(Word(hexnums)).setParseAction(tokenMap(int, 16)) - success, report = parser.runTests(""" + success, report = parser.runTests( + """ 00 11 22 aa FF 0a 0d 1a - """) + """ + ) # WAS: # self.assertTrue(success, "failed to parse hex integers") # print(results) @@ -3601,15 +4832,15 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): # if I hadn't unpacked the return from runTests, I could have just passed it directly, # instead of reconstituting as a tuple - self.assertRunTestResults((success, report), - [ - ([0, 17, 34, 170, 255, 10, 13, 26], "tokenMap parse action failed"), - ], - msg="failed to parse hex integers") - + self.assertRunTestResults( + (success, report), + [([0, 17, 34, 170, 255, 10, 13, 26], "tokenMap parse action failed"),], + msg="failed to parse hex integers", + ) def testParseFile(self): from pyparsing import pyparsing_common, OneOrMore + s = """ 123 456 789 """ @@ -3619,12 +4850,17 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): results = OneOrMore(integer).parseFile(input_file) print(results) - results = OneOrMore(integer).parseFile('tests/parsefiletest_input_file.txt') + results = OneOrMore(integer).parseFile("tests/parsefiletest_input_file.txt") print(results) - def testHTMLStripper(self): - from pyparsing import pyparsing_common, originalTextFor, OneOrMore, Word, printables + from pyparsing import ( + pyparsing_common, + originalTextFor, + OneOrMore, + Word, + printables, + ) sample = """ <html> @@ -3635,17 +4871,16 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): read_everything.addParseAction(pyparsing_common.stripHTMLTags) result = read_everything.parseString(sample) - self.assertEqual(result[0].strip(), 'Here is some sample HTML text.') + self.assertEqual(result[0].strip(), "Here is some sample HTML text.") def testExprSplitter(self): from pyparsing import Literal, quotedString, pythonStyleComment, Empty - expr = Literal(';') + Empty() + expr = Literal(";") + Empty() expr.ignore(quotedString) expr.ignore(pythonStyleComment) - sample = """ def main(): this_semi_does_nothing(); @@ -3671,74 +4906,115 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): return self.bar """ expected = [ - [' this_semi_does_nothing()', ''], - [' neither_does_this_but_there_are_spaces_afterward()', ''], - [' a = "a;b"', 'return a # this is a comment; it has a semicolon!'], - [' z=1000', 'b("; in quotes")', 'c=200', 'return z'], + [" this_semi_does_nothing()", ""], + [" neither_does_this_but_there_are_spaces_afterward()", ""], + [ + ' a = "a;b"', + "return a # this is a comment; it has a semicolon!", + ], + [" z=1000", 'b("; in quotes")', "c=200", "return z"], [" return ';'"], [" '''a docstring; with a semicolon'''"], - [' a = 10', 'b = 11', 'c = 12'], - [' # this comment; has several; semicolons'], - [' x = 12', 'return x # so; does; this; one'], - [' x = 15', '', '', 'y += x', 'return y'], - ] + [" a = 10", "b = 11", "c = 12"], + [" # this comment; has several; semicolons"], + [" x = 12", "return x # so; does; this; one"], + [" x = 15", "", "", "y += x", "return y"], + ] exp_iter = iter(expected) - for line in filter(lambda ll: ';' in ll, sample.splitlines()): - print(str(list(expr.split(line))) + ',') - self.assertEqual(list(expr.split(line)), next(exp_iter), "invalid split on expression") + for line in filter(lambda ll: ";" in ll, sample.splitlines()): + print(str(list(expr.split(line))) + ",") + self.assertEqual( + list(expr.split(line)), next(exp_iter), "invalid split on expression" + ) print() expected = [ - [' this_semi_does_nothing()', ';', ''], - [' neither_does_this_but_there_are_spaces_afterward()', ';', ''], - [' a = "a;b"', ';', 'return a # this is a comment; it has a semicolon!'], - [' z=1000', ';', 'b("; in quotes")', ';', 'c=200', ';', 'return z'], + [" this_semi_does_nothing()", ";", ""], + [" neither_does_this_but_there_are_spaces_afterward()", ";", ""], + [ + ' a = "a;b"', + ";", + "return a # this is a comment; it has a semicolon!", + ], + [ + " z=1000", + ";", + 'b("; in quotes")', + ";", + "c=200", + ";", + "return z", + ], [" return ';'"], [" '''a docstring; with a semicolon'''"], - [' a = 10', ';', 'b = 11', ';', 'c = 12'], - [' # this comment; has several; semicolons'], - [' x = 12', ';', 'return x # so; does; this; one'], - [' x = 15', ';', '', ';', '', ';', 'y += x', ';', 'return y'], - ] + [" a = 10", ";", "b = 11", ";", "c = 12"], + [" # this comment; has several; semicolons"], + [" x = 12", ";", "return x # so; does; this; one"], + [ + " x = 15", + ";", + "", + ";", + "", + ";", + "y += x", + ";", + "return y", + ], + ] exp_iter = iter(expected) - for line in filter(lambda ll: ';' in ll, sample.splitlines()): - print(str(list(expr.split(line, includeSeparators=True))) + ',') - self.assertEqual(list(expr.split(line, includeSeparators=True)), next(exp_iter), - "invalid split on expression") + for line in filter(lambda ll: ";" in ll, sample.splitlines()): + print(str(list(expr.split(line, includeSeparators=True))) + ",") + self.assertEqual( + list(expr.split(line, includeSeparators=True)), + next(exp_iter), + "invalid split on expression", + ) print() - expected = [ - [' this_semi_does_nothing()', ''], - [' neither_does_this_but_there_are_spaces_afterward()', ''], - [' a = "a;b"', 'return a # this is a comment; it has a semicolon!'], - [' z=1000', 'b("; in quotes"); c=200;return z'], - [' a = 10', 'b = 11; c = 12'], - [' x = 12', 'return x # so; does; this; one'], - [' x = 15', ';; y += x; return y'], - ] + [" this_semi_does_nothing()", ""], + [" neither_does_this_but_there_are_spaces_afterward()", ""], + [ + ' a = "a;b"', + "return a # this is a comment; it has a semicolon!", + ], + [" z=1000", 'b("; in quotes"); c=200;return z'], + [" a = 10", "b = 11; c = 12"], + [" x = 12", "return x # so; does; this; one"], + [" x = 15", ";; y += x; return y"], + ] exp_iter = iter(expected) for line in sample.splitlines(): pieces = list(expr.split(line, maxsplit=1)) - print(str(pieces) + ',') + print(str(pieces) + ",") if len(pieces) == 2: exp = next(exp_iter) - self.assertEqual(pieces, exp, "invalid split on expression with maxSplits=1") + self.assertEqual( + pieces, exp, "invalid split on expression with maxSplits=1" + ) elif len(pieces) == 1: - self.assertEqual(len(expr.searchString(line)), 0, "invalid split with maxSplits=1 when expr not present") + self.assertEqual( + len(expr.searchString(line)), + 0, + "invalid split with maxSplits=1 when expr not present", + ) else: print("\n>>> " + line) - self.assertTrue(False, "invalid split on expression with maxSplits=1, corner case") + self.assertTrue( + False, "invalid split on expression with maxSplits=1, corner case" + ) def testParseFatalException(self): from pyparsing import Word, nums, ParseFatalException - with self.assertRaisesParseException(exc_type=ParseFatalException, - msg="failed to raise ErrorStop exception"): + with self.assertRaisesParseException( + exc_type=ParseFatalException, msg="failed to raise ErrorStop exception" + ): expr = "ZZZ" - Word(nums) expr.parseString("ZZZ bad") @@ -3758,16 +5034,26 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): def testInlineLiteralsUsing(self): - from pyparsing import ParserElement, Suppress, Literal, CaselessLiteral, Word, alphas, oneOf, CaselessKeyword, nums + from pyparsing import ( + ParserElement, + Suppress, + Literal, + CaselessLiteral, + Word, + alphas, + oneOf, + CaselessKeyword, + nums, + ) wd = Word(alphas) ParserElement.inlineLiteralsUsing(Suppress) - result = (wd + ',' + wd + oneOf("! . ?")).parseString("Hello, World!") + result = (wd + "," + wd + oneOf("! . ?")).parseString("Hello, World!") self.assertEqual(len(result), 3, "inlineLiteralsUsing(Suppress) failed!") ParserElement.inlineLiteralsUsing(Literal) - result = (wd + ',' + wd + oneOf("! . ?")).parseString("Hello, World!") + result = (wd + "," + wd + oneOf("! . ?")).parseString("Hello, World!") self.assertEqual(len(result), 4, "inlineLiteralsUsing(Literal) failed!") ParserElement.inlineLiteralsUsing(CaselessKeyword) @@ -3775,140 +5061,190 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): # result = ("SELECT" + wd + "FROM" + wd).parseString("select color from colors") # self.assertEqual(result.asList(), "SELECT color FROM colors".split(), # "inlineLiteralsUsing(CaselessKeyword) failed!") - self.assertParseAndCheckList("SELECT" + wd + "FROM" + wd, - "select color from colors", - expected_list=['SELECT', 'color', 'FROM', 'colors'], - msg="inlineLiteralsUsing(CaselessKeyword) failed!") + self.assertParseAndCheckList( + "SELECT" + wd + "FROM" + wd, + "select color from colors", + expected_list=["SELECT", "color", "FROM", "colors"], + msg="inlineLiteralsUsing(CaselessKeyword) failed!", + ) ParserElement.inlineLiteralsUsing(CaselessLiteral) # result = ("SELECT" + wd + "FROM" + wd).parseString("select color from colors") # self.assertEqual(result.asList(), "SELECT color FROM colors".split(), # "inlineLiteralsUsing(CaselessLiteral) failed!") - self.assertParseAndCheckList("SELECT" + wd + "FROM" + wd, - "select color from colors", - expected_list=['SELECT', 'color', 'FROM', 'colors'], - msg="inlineLiteralsUsing(CaselessLiteral) failed!") + self.assertParseAndCheckList( + "SELECT" + wd + "FROM" + wd, + "select color from colors", + expected_list=["SELECT", "color", "FROM", "colors"], + msg="inlineLiteralsUsing(CaselessLiteral) failed!", + ) integer = Word(nums) ParserElement.inlineLiteralsUsing(Literal) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + "/" + integer("month") + "/" + integer("day") # result = date_str.parseString("1999/12/31") # self.assertEqual(result.asList(), ['1999', '/', '12', '/', '31'], "inlineLiteralsUsing(example 1) failed!") - self.assertParseAndCheckList(date_str, "1999/12/31", expected_list=['1999', '/', '12', '/', '31'], - msg="inlineLiteralsUsing(example 1) failed!") + self.assertParseAndCheckList( + date_str, + "1999/12/31", + expected_list=["1999", "/", "12", "/", "31"], + msg="inlineLiteralsUsing(example 1) failed!", + ) # change to Suppress ParserElement.inlineLiteralsUsing(Suppress) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + "/" + integer("month") + "/" + integer("day") # result = date_str.parseString("1999/12/31") # -> ['1999', '12', '31'] # self.assertEqual(result.asList(), ['1999', '12', '31'], "inlineLiteralsUsing(example 2) failed!") - self.assertParseAndCheckList(date_str, "1999/12/31", expected_list=['1999', '12', '31'], - msg="inlineLiteralsUsing(example 2) failed!") + self.assertParseAndCheckList( + date_str, + "1999/12/31", + expected_list=["1999", "12", "31"], + msg="inlineLiteralsUsing(example 2) failed!", + ) def testCloseMatch(self): import pyparsing as pp searchseq = pp.CloseMatch("ATCATCGAATGGA", 2) - _, results = searchseq.runTests(""" + _, results = searchseq.runTests( + """ ATCATCGAATGGA XTCATCGAATGGX ATCATCGAAXGGA ATCAXXGAATGGA ATCAXXGAATGXA ATCAXXGAATGG - """) - expected = ( - [], - [0, 12], - [9], - [4, 5], - None, - None - ) + """ + ) + expected = ([], [0, 12], [9], [4, 5], None, None) for r, exp in zip(results, expected): if exp is not None: - self.assertEquals(r[1].mismatches, exp, - "fail CloseMatch between {!r} and {!r}".format(searchseq.match_string, r[0])) - print(r[0], 'exc: %s' % r[1] if exp is None and isinstance(r[1], Exception) - else ("no match", "match")[r[1].mismatches == exp]) + self.assertEquals( + r[1].mismatches, + exp, + "fail CloseMatch between {!r} and {!r}".format( + searchseq.match_string, r[0] + ), + ) + print( + r[0], + "exc: %s" % r[1] + if exp is None and isinstance(r[1], Exception) + else ("no match", "match")[r[1].mismatches == exp], + ) def testDefaultKeywordChars(self): import pyparsing as pp - with self.assertRaisesParseException(msg="failed to fail matching keyword using updated keyword chars"): + with self.assertRaisesParseException( + msg="failed to fail matching keyword using updated keyword chars" + ): pp.Keyword("start").parseString("start1000") try: pp.Keyword("start", identChars=pp.alphas).parseString("start1000") except pp.ParseException: - self.assertTrue(False, "failed to match keyword using updated keyword chars") + self.assertTrue( + False, "failed to match keyword using updated keyword chars" + ) with ppt.reset_pyparsing_context(): pp.Keyword.setDefaultKeywordChars(pp.alphas) try: pp.Keyword("start").parseString("start1000") except pp.ParseException: - self.assertTrue(False, "failed to match keyword using updated keyword chars") + self.assertTrue( + False, "failed to match keyword using updated keyword chars" + ) - with self.assertRaisesParseException(msg="failed to fail matching keyword using updated keyword chars"): + with self.assertRaisesParseException( + msg="failed to fail matching keyword using updated keyword chars" + ): pp.CaselessKeyword("START").parseString("start1000") try: pp.CaselessKeyword("START", identChars=pp.alphas).parseString("start1000") except pp.ParseException: - self.assertTrue(False, "failed to match keyword using updated keyword chars") + self.assertTrue( + False, "failed to match keyword using updated keyword chars" + ) with ppt.reset_pyparsing_context(): pp.Keyword.setDefaultKeywordChars(pp.alphas) try: pp.CaselessKeyword("START").parseString("start1000") except pp.ParseException: - self.assertTrue(False, "failed to match keyword using updated keyword chars") + self.assertTrue( + False, "failed to match keyword using updated keyword chars" + ) def testCol(self): test = "*\n* \n* ALF\n*\n" initials = [c for i, c in enumerate(test) if pp.col(i, test) == 1] print(initials) - self.assertTrue(len(initials) == 4 and all(c == '*' for c in initials), 'fail col test') + self.assertTrue( + len(initials) == 4 and all(c == "*" for c in initials), "fail col test" + ) def testLiteralException(self): import pyparsing as pp - for cls in (pp.Literal, pp.CaselessLiteral, pp.Keyword, pp.CaselessKeyword, - pp.Word, pp.Regex): - expr = cls('xyz')#.setName('{}_expr'.format(cls.__name__.lower())) + for cls in ( + pp.Literal, + pp.CaselessLiteral, + pp.Keyword, + pp.CaselessKeyword, + pp.Word, + pp.Regex, + ): + expr = cls("xyz") # .setName('{}_expr'.format(cls.__name__.lower())) try: - expr.parseString(' ') + expr.parseString(" ") except Exception as e: print(cls.__name__, str(e)) - self.assertTrue(isinstance(e, pp.ParseBaseException), - "class {} raised wrong exception type {}".format(cls.__name__, type(e).__name__)) + self.assertTrue( + isinstance(e, pp.ParseBaseException), + "class {} raised wrong exception type {}".format( + cls.__name__, type(e).__name__ + ), + ) def testParseActionException(self): import pyparsing as pp import traceback number = pp.Word(pp.nums) + def number_action(): - raise IndexError # this is the important line! + raise IndexError # this is the important line! number.setParseAction(number_action) - symbol = pp.Word('abcd', max=1) + symbol = pp.Word("abcd", max=1) expr = number | symbol try: - expr.parseString('1 + 2') + expr.parseString("1 + 2") except Exception as e: print_traceback = True try: - self.assertTrue(hasattr(e, '__cause__'), "no __cause__ attribute in the raised exception") - self.assertTrue(e.__cause__ is not None, "__cause__ not propagated to outer exception") - self.assertTrue(type(e.__cause__) == IndexError, "__cause__ references wrong exception") + self.assertTrue( + hasattr(e, "__cause__"), + "no __cause__ attribute in the raised exception", + ) + self.assertTrue( + e.__cause__ is not None, + "__cause__ not propagated to outer exception", + ) + self.assertTrue( + type(e.__cause__) == IndexError, + "__cause__ references wrong exception", + ) print_traceback = False finally: if print_traceback: @@ -3920,21 +5256,26 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): def testParseActionNesting(self): vals = pp.OneOrMore(pp.pyparsing_common.integer)("int_values") + def add_total(tokens): - tokens['total'] = sum(tokens) + tokens["total"] = sum(tokens) return tokens + vals.addParseAction(add_total) results = vals.parseString("244 23 13 2343") print(results.dump()) - self.assertParseResultsEquals(results, expected_dict={'int_values': [244, 23, 13, 2343], 'total': 2623}, - msg="noop parse action changed ParseResults structure") - - name = pp.Word(pp.alphas)('name') - score = pp.Word(pp.nums + '.')('score') + self.assertParseResultsEquals( + results, + expected_dict={"int_values": [244, 23, 13, 2343], "total": 2623}, + msg="noop parse action changed ParseResults structure", + ) + + name = pp.Word(pp.alphas)("name") + score = pp.Word(pp.nums + ".")("score") nameScore = pp.Group(name + score) - line1 = nameScore('Rider') + line1 = nameScore("Rider") - result1 = line1.parseString('Mauney 46.5') + result1 = line1.parseString("Mauney 46.5") print("### before parse action is added ###") print("result1.dump():\n" + result1.dump() + "\n") @@ -3942,26 +5283,35 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): line1.setParseAction(lambda t: t) - result1 = line1.parseString('Mauney 46.5') + result1 = line1.parseString("Mauney 46.5") after_pa_dict = result1.asDict() print("### after parse action was added ###") print("result1.dump():\n" + result1.dump() + "\n") - self.assertEqual(before_pa_dict, after_pa_dict, "noop parse action changed ParseResults structure") + self.assertEqual( + before_pa_dict, + after_pa_dict, + "noop parse action changed ParseResults structure", + ) def testParseResultsNameBelowUngroupedName(self): import pyparsing as pp rule_num = pp.Regex("[0-9]+")("LIT_NUM*") - list_num = pp.Group(pp.Literal("[")("START_LIST") - + pp.delimitedList(rule_num)("LIST_VALUES") - + pp.Literal("]")("END_LIST"))("LIST") + list_num = pp.Group( + pp.Literal("[")("START_LIST") + + pp.delimitedList(rule_num)("LIST_VALUES") + + pp.Literal("]")("END_LIST") + )("LIST") test_string = "[ 1,2,3,4,5,6 ]" list_num.runTests(test_string) U = list_num.parseString(test_string) - self.assertTrue("LIT_NUM" not in U.LIST.LIST_VALUES, "results name retained as sub in ungrouped named result") + self.assertTrue( + "LIT_NUM" not in U.LIST.LIST_VALUES, + "results name retained as sub in ungrouped named result", + ) def testParseResultsNamesInGroupWithDict(self): import pyparsing as pp @@ -3971,31 +5321,44 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): value = ppc.integer() lat = ppc.real() long = ppc.real() - EQ = pp.Suppress('=') + EQ = pp.Suppress("=") - data = lat("lat") + long("long") + pp.Dict(pp.OneOrMore(pp.Group(key + EQ + value))) + data = ( + lat("lat") + + long("long") + + pp.Dict(pp.OneOrMore(pp.Group(key + EQ + value))) + ) site = pp.QuotedString('"')("name") + pp.Group(data)("data") test_string = '"Golden Gate Bridge" 37.819722 -122.478611 height=746 span=4200' site.runTests(test_string) - a, aEnd = pp.makeHTMLTags('a') + a, aEnd = pp.makeHTMLTags("a") attrs = a.parseString("<a href='blah'>") print(attrs.dump()) - self.assertParseResultsEquals(attrs, - expected_dict = {'startA': {'href': 'blah', 'tag': 'a', 'empty': False}, - 'href': 'blah', 'tag': 'a', 'empty': False}) - + self.assertParseResultsEquals( + attrs, + expected_dict={ + "startA": {"href": "blah", "tag": "a", "empty": False}, + "href": "blah", + "tag": "a", + "empty": False, + }, + ) def testFollowedBy(self): import pyparsing as pp from pyparsing import pyparsing_common as ppc + expr = pp.Word(pp.alphas)("item") + pp.FollowedBy(ppc.integer("qty")) result = expr.parseString("balloon 99") print(result.dump()) - self.assertTrue('qty' in result, "failed to capture results name in FollowedBy") - self.assertEqual(result.asDict(), {'item': 'balloon', 'qty': 99}, - "invalid results name structure from FollowedBy") + self.assertTrue("qty" in result, "failed to capture results name in FollowedBy") + self.assertEqual( + result.asDict(), + {"item": "balloon", "qty": 99}, + "invalid results name structure from FollowedBy", + ) def testSetBreak(self): """ @@ -4004,16 +5367,19 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): Temporarily monkeypatches pdb.set_trace. """ was_called = False + def mock_set_trace(): nonlocal was_called was_called = True import pyparsing as pp + wd = pp.Word(pp.alphas) wd.setBreak() print("Before parsing with setBreak:", was_called) import pdb + with ppt.reset_pyparsing_context(): pdb.set_trace = mock_set_trace wd.parseString("ABC") @@ -4023,6 +5389,7 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): def testUnicodeTests(self): import pyparsing as pp + ppu = pp.pyparsing_unicode ppc = pp.pyparsing_common @@ -4031,58 +5398,72 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): katakana_printables = ppu.Japanese.Katakana.printables hiragana_printables = ppu.Japanese.Hiragana.printables japanese_printables = ppu.Japanese.printables - self.assertEqual(set(japanese_printables), set(kanji_printables - + katakana_printables - + hiragana_printables), - "failed to construct ranges by merging Japanese types") + self.assertEqual( + set(japanese_printables), + set(kanji_printables + katakana_printables + hiragana_printables), + "failed to construct ranges by merging Japanese types", + ) # verify proper merging of ranges using multiple inheritance cjk_printables = ppu.CJK.printables - self.assertEqual(len(cjk_printables), len(set(cjk_printables)), - "CJK contains duplicate characters - all should be unique") + self.assertEqual( + len(cjk_printables), + len(set(cjk_printables)), + "CJK contains duplicate characters - all should be unique", + ) chinese_printables = ppu.Chinese.printables korean_printables = ppu.Korean.printables - print(len(cjk_printables), len(set(chinese_printables - + korean_printables - + japanese_printables))) + print( + len(cjk_printables), + len(set(chinese_printables + korean_printables + japanese_printables)), + ) - self.assertEqual(len(cjk_printables), len(set(chinese_printables - + korean_printables - + japanese_printables)), - "failed to construct ranges by merging Chinese, Japanese and Korean") + self.assertEqual( + len(cjk_printables), + len(set(chinese_printables + korean_printables + japanese_printables)), + "failed to construct ranges by merging Chinese, Japanese and Korean", + ) alphas = ppu.Greek.alphas - greet = pp.Word(alphas) + ',' + pp.Word(alphas) + '!' + greet = pp.Word(alphas) + "," + pp.Word(alphas) + "!" # input string hello = "Καλημέρα, κόσμε!" result = greet.parseString(hello) print(result) - self.assertParseResultsEquals(result, - expected_list=['Καλημέρα', ',', 'κόσμε', '!'], - msg="Failed to parse Greek 'Hello, World!' using " - "pyparsing_unicode.Greek.alphas") + self.assertParseResultsEquals( + result, + expected_list=["Καλημέρα", ",", "κόσμε", "!"], + msg="Failed to parse Greek 'Hello, World!' using " + "pyparsing_unicode.Greek.alphas", + ) # define a custom unicode range using multiple inheritance class Turkish_set(ppu.Latin1, ppu.LatinA): pass - self.assertEqual(set(Turkish_set.printables), - set(ppu.Latin1.printables + ppu.LatinA.printables), - "failed to construct ranges by merging Latin1 and LatinA (printables)") + self.assertEqual( + set(Turkish_set.printables), + set(ppu.Latin1.printables + ppu.LatinA.printables), + "failed to construct ranges by merging Latin1 and LatinA (printables)", + ) - self.assertEqual(set(Turkish_set.alphas), - set(ppu.Latin1.alphas + ppu.LatinA.alphas), - "failed to construct ranges by merging Latin1 and LatinA (alphas)") + self.assertEqual( + set(Turkish_set.alphas), + set(ppu.Latin1.alphas + ppu.LatinA.alphas), + "failed to construct ranges by merging Latin1 and LatinA (alphas)", + ) - self.assertEqual(set(Turkish_set.nums), - set(ppu.Latin1.nums + ppu.LatinA.nums), - "failed to construct ranges by merging Latin1 and LatinA (nums)") + self.assertEqual( + set(Turkish_set.nums), + set(ppu.Latin1.nums + ppu.LatinA.nums), + "failed to construct ranges by merging Latin1 and LatinA (nums)", + ) key = pp.Word(Turkish_set.alphas) value = ppc.integer | pp.Word(Turkish_set.alphas, Turkish_set.alphanums) - EQ = pp.Suppress('=') + EQ = pp.Suppress("=") key_value = key + EQ + value sample = """\ @@ -4092,17 +5473,30 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): result = pp.Dict(pp.OneOrMore(pp.Group(key_value))).parseString(sample) print(result.dump()) - self.assertParseResultsEquals(result, - expected_dict={'şehir': 'İzmir', 'ülke': 'Türkiye', 'nüfus': 4279677}, - msg="Failed to parse Turkish key-value pairs") - + self.assertParseResultsEquals( + result, + expected_dict={"şehir": "İzmir", "ülke": "Türkiye", "nüfus": 4279677}, + msg="Failed to parse Turkish key-value pairs", + ) # Make sure example in indentedBlock docstring actually works! def testIndentedBlockExample(self): from textwrap import dedent - from pyparsing import (Word, alphas, alphanums, indentedBlock, Optional, delimitedList, Group, Forward, - nums, OneOrMore) - data = dedent(''' + from pyparsing import ( + Word, + alphas, + alphanums, + indentedBlock, + Optional, + delimitedList, + Group, + Forward, + nums, + OneOrMore, + ) + + data = dedent( + """ def A(z): A1 B = 100 @@ -4121,13 +5515,19 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): def spam(x,y): def eggs(z): pass - ''') + """ + ) indentStack = [1] stmt = Forward() identifier = Word(alphas, alphanums) - funcDecl = ("def" + identifier + Group("(" + Optional(delimitedList(identifier)) + ")") + ":") + funcDecl = ( + "def" + + identifier + + Group("(" + Optional(delimitedList(identifier)) + ")") + + ":" + ) func_body = indentedBlock(stmt, indentStack) funcDef = Group(funcDecl + func_body) @@ -4141,33 +5541,53 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): parseTree = module_body.parseString(data) parseTree.pprint() - self.assertEqual(parseTree.asList(), - [['def', - 'A', - ['(', 'z', ')'], - ':', - [['A1'], [['B', '=', '100']], [['G', '=', 'A2']], ['A2'], ['A3']]], - 'B', - ['def', - 'BB', - ['(', 'a', 'b', 'c', ')'], - ':', - [['BB1'], [['def', 'BBA', ['(', ')'], ':', [['bba1'], ['bba2'], ['bba3']]]]]], - 'C', - 'D', - ['def', - 'spam', - ['(', 'x', 'y', ')'], - ':', - [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]], - "Failed indentedBlock example" - ) + self.assertEqual( + parseTree.asList(), + [ + [ + "def", + "A", + ["(", "z", ")"], + ":", + [["A1"], [["B", "=", "100"]], [["G", "=", "A2"]], ["A2"], ["A3"]], + ], + "B", + [ + "def", + "BB", + ["(", "a", "b", "c", ")"], + ":", + [ + ["BB1"], + [ + [ + "def", + "BBA", + ["(", ")"], + ":", + [["bba1"], ["bba2"], ["bba3"]], + ] + ], + ], + ], + "C", + "D", + [ + "def", + "spam", + ["(", "x", "y", ")"], + ":", + [[["def", "eggs", ["(", "z", ")"], ":", [["pass"]]]]], + ], + ], + "Failed indentedBlock example", + ) def testIndentedBlock(self): # parse pseudo-yaml indented text import textwrap - EQ = pp.Suppress('=') + EQ = pp.Suppress("=") stack = [1] key = pp.pyparsing_common.identifier value = pp.Forward() @@ -4191,15 +5611,23 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): result = parser.parseString(text) print(result.dump()) - self.assertEqual(result.a, 100, "invalid indented block result") - self.assertEqual(result.c.c1, 200, "invalid indented block result") + self.assertEqual(result.a, 100, "invalid indented block result") + self.assertEqual(result.c.c1, 200, "invalid indented block result") self.assertEqual(result.c.c2.c21, 999, "invalid indented block result") - # exercise indentedBlock with example posted in issue #87 def testIndentedBlockTest2(self): from textwrap import dedent - from pyparsing import Word, alphas, alphanums, Suppress, Forward, indentedBlock, Literal, OneOrMore + from pyparsing import ( + Word, + alphas, + alphanums, + Suppress, + Forward, + indentedBlock, + Literal, + OneOrMore, + ) indent_stack = [1] @@ -4209,7 +5637,7 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): suite = indentedBlock(stmt, indent_stack) body = key + suite - pattern = (Word(alphas) + Suppress("(") + Word(alphas) + Suppress(")")) + pattern = Word(alphas) + Suppress("(") + Word(alphas) + Suppress(")") stmt << pattern def key_parse_action(toks): @@ -4217,7 +5645,7 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): key.setParseAction(key_parse_action) header = Suppress("[") + Literal("test") + Suppress("]") - content = (header - OneOrMore(indentedBlock(body, indent_stack, False))) + content = header - OneOrMore(indentedBlock(body, indent_stack, False)) contents = Forward() suites = indentedBlock(content, indent_stack) @@ -4227,46 +5655,49 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): parser = OneOrMore(contents) - sample = dedent(""" + sample = dedent( + """ extra: [test] - one0: + one0: two (three) four0: five (seven) extra: [test] - one1: + one1: two (three) four1: five (seven) - """) + """ + ) success, _ = parser.runTests([sample]) self.assertTrue(success, "Failed indentedBlock test for issue #87") - sample2 = dedent(""" + sample2 = dedent( + """ extra: [test] - one: + one: two (three) four: five (seven) extra: [test] - one: + one: two (three) four: five (seven) - + [test] - one: + one: two (three) four: five (seven) [test] - eight: + eight: nine (ten) eleven: twelve (thirteen) @@ -4275,7 +5706,8 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): fifteen (sixteen) seventeen: eighteen (nineteen) - """) + """ + ) del indent_stack[1:] success, _ = parser.runTests([sample2]) @@ -4288,193 +5720,257 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): """ stack = [1] block = pp.Forward() - body = pp.indentedBlock(pp.Literal('A') ^ block, indentStack=stack, indent=True) - block <<= pp.Literal('block:') + body + body = pp.indentedBlock( + pp.Literal("A") ^ block, indentStack=stack, indent=True + ) + block <<= pp.Literal("block:") + body return block from textwrap import dedent # This input string is a perfect match for the parser, so a single match is found p1 = get_parser() - r1 = list(p1.scanString(dedent("""\ + r1 = list( + p1.scanString( + dedent( + """\ block: A - """))) + """ + ) + ) + ) self.assertEqual(len(r1), 1) # This input string is a perfect match for the parser, except for the letter B instead of A, so this will fail (and should) p2 = get_parser() - r2 = list(p2.scanString(dedent("""\ + r2 = list( + p2.scanString( + dedent( + """\ block: B - """))) + """ + ) + ) + ) self.assertEqual(len(r2), 0) # This input string contains both string A and string B, and it finds one match (as it should) p3 = get_parser() - r3 = list(p3.scanString(dedent("""\ + r3 = list( + p3.scanString( + dedent( + """\ block: A block: B - """))) + """ + ) + ) + ) self.assertEqual(len(r3), 1) # This input string contains both string A and string B, but in a different order. p4 = get_parser() - r4 = list(p4.scanString(dedent("""\ + r4 = list( + p4.scanString( + dedent( + """\ block: B block: A - """))) + """ + ) + ) + ) self.assertEqual(len(r4), 1) # This is the same as case 3, but with nesting p5 = get_parser() - r5 = list(p5.scanString(dedent("""\ + r5 = list( + p5.scanString( + dedent( + """\ block: block: A block: block: B - """))) + """ + ) + ) + ) self.assertEqual(len(r5), 1) # This is the same as case 4, but with nesting p6 = get_parser() - r6 = list(p6.scanString(dedent("""\ + r6 = list( + p6.scanString( + dedent( + """\ block: block: B block: block: A - """))) + """ + ) + ) + ) self.assertEqual(len(r6), 1) - def testInvalidDiagSetting(self): import pyparsing as pp - with self.assertRaises(ValueError, msg="failed to raise exception when setting non-existent __diag__"): + with self.assertRaises( + ValueError, + msg="failed to raise exception when setting non-existent __diag__", + ): pp.__diag__.enable("xyzzy") - with self.assertWarns(UserWarning, msg="failed to warn disabling 'collect_all_And_tokens"): + with self.assertWarns( + UserWarning, msg="failed to warn disabling 'collect_all_And_tokens" + ): pp.__compat__.disable("collect_all_And_tokens") - def testParseResultsWithNameMatchFirst(self): import pyparsing as pp - expr_a = pp.Literal('not') + pp.Literal('the') + pp.Literal('bird') - expr_b = pp.Literal('the') + pp.Literal('bird') - expr = (expr_a | expr_b)('rexp') - success, report = expr.runTests("""\ + expr_a = pp.Literal("not") + pp.Literal("the") + pp.Literal("bird") + expr_b = pp.Literal("the") + pp.Literal("bird") + expr = (expr_a | expr_b)("rexp") + + success, report = expr.runTests( + """\ not the bird the bird - """) + """ + ) results = [rpt[1] for rpt in report] - self.assertParseResultsEquals(results[0], - ['not', 'the', 'bird'], - {'rexp': ['not', 'the', 'bird']}) - self.assertParseResultsEquals(results[1], - ['the', 'bird'], - {'rexp': ['the', 'bird']}) + self.assertParseResultsEquals( + results[0], ["not", "the", "bird"], {"rexp": ["not", "the", "bird"]} + ) + self.assertParseResultsEquals( + results[1], ["the", "bird"], {"rexp": ["the", "bird"]} + ) # test compatibility mode, no longer restoring pre-2.3.1 behavior with ppt.reset_pyparsing_context(): pp.__compat__.collect_all_And_tokens = False pp.__diag__.enable("warn_multiple_tokens_in_named_alternation") - expr_a = pp.Literal('not') + pp.Literal('the') + pp.Literal('bird') - expr_b = pp.Literal('the') + pp.Literal('bird') - with self.assertWarns(UserWarning, msg="failed to warn of And within alternation"): - expr = (expr_a | expr_b)('rexp') - - success, report = expr.runTests(""" + expr_a = pp.Literal("not") + pp.Literal("the") + pp.Literal("bird") + expr_b = pp.Literal("the") + pp.Literal("bird") + with self.assertWarns( + UserWarning, msg="failed to warn of And within alternation" + ): + expr = (expr_a | expr_b)("rexp") + + success, report = expr.runTests( + """ not the bird the bird - """) + """ + ) results = [rpt[1] for rpt in report] - self.assertParseResultsEquals(results[0], - ['not', 'the', 'bird'], - {'rexp': ['not', 'the', 'bird']}) - self.assertParseResultsEquals(results[1], - ['the', 'bird'], - {'rexp': ['the', 'bird']}) - + self.assertParseResultsEquals( + results[0], ["not", "the", "bird"], {"rexp": ["not", "the", "bird"]} + ) + self.assertParseResultsEquals( + results[1], ["the", "bird"], {"rexp": ["the", "bird"]} + ) def testParseResultsWithNameOr(self): import pyparsing as pp - expr_a = pp.Literal('not') + pp.Literal('the') + pp.Literal('bird') - expr_b = pp.Literal('the') + pp.Literal('bird') - expr = (expr_a ^ expr_b)('rexp') - expr.runTests("""\ + + expr_a = pp.Literal("not") + pp.Literal("the") + pp.Literal("bird") + expr_b = pp.Literal("the") + pp.Literal("bird") + expr = (expr_a ^ expr_b)("rexp") + expr.runTests( + """\ not the bird the bird - """) - result = expr.parseString('not the bird') - self.assertParseResultsEquals(result, - ['not', 'the', 'bird'], - {'rexp': ['not', 'the', 'bird']}) - result = expr.parseString('the bird') - self.assertParseResultsEquals(result, - ['the', 'bird'], - {'rexp': ['the', 'bird']}) - - expr = (expr_a | expr_b)('rexp') - expr.runTests("""\ + """ + ) + result = expr.parseString("not the bird") + self.assertParseResultsEquals( + result, ["not", "the", "bird"], {"rexp": ["not", "the", "bird"]} + ) + result = expr.parseString("the bird") + self.assertParseResultsEquals( + result, ["the", "bird"], {"rexp": ["the", "bird"]} + ) + + expr = (expr_a | expr_b)("rexp") + expr.runTests( + """\ not the bird the bird - """) - result = expr.parseString('not the bird') - self.assertParseResultsEquals(result, - ['not', 'the', 'bird'], - {'rexp': - ['not', 'the', 'bird']}) - result = expr.parseString('the bird') - self.assertParseResultsEquals(result, - ['the', 'bird'], - {'rexp': ['the', 'bird']}) + """ + ) + result = expr.parseString("not the bird") + self.assertParseResultsEquals( + result, ["not", "the", "bird"], {"rexp": ["not", "the", "bird"]} + ) + result = expr.parseString("the bird") + self.assertParseResultsEquals( + result, ["the", "bird"], {"rexp": ["the", "bird"]} + ) # test compatibility mode, no longer restoring pre-2.3.1 behavior with ppt.reset_pyparsing_context(): pp.__compat__.collect_all_And_tokens = False pp.__diag__.enable("warn_multiple_tokens_in_named_alternation") - expr_a = pp.Literal('not') + pp.Literal('the') + pp.Literal('bird') - expr_b = pp.Literal('the') + pp.Literal('bird') + expr_a = pp.Literal("not") + pp.Literal("the") + pp.Literal("bird") + expr_b = pp.Literal("the") + pp.Literal("bird") - with self.assertWarns(UserWarning, msg="failed to warn of And within alternation"): - expr = (expr_a ^ expr_b)('rexp') + with self.assertWarns( + UserWarning, msg="failed to warn of And within alternation" + ): + expr = (expr_a ^ expr_b)("rexp") - expr.runTests("""\ + expr.runTests( + """\ not the bird the bird - """) - self.assertEqual(list(expr.parseString('not the bird')['rexp']), 'not the bird'.split()) - self.assertEqual(list(expr.parseString('the bird')['rexp']), 'the bird'.split()) - + """ + ) + self.assertEqual( + list(expr.parseString("not the bird")["rexp"]), "not the bird".split() + ) + self.assertEqual( + list(expr.parseString("the bird")["rexp"]), "the bird".split() + ) def testEmptyDictDoesNotRaiseException(self): import pyparsing as pp key = pp.Word(pp.alphas) value = pp.Word(pp.nums) - EQ = pp.Suppress('=') + EQ = pp.Suppress("=") key_value_dict = pp.dictOf(key, EQ + value) - print(key_value_dict.parseString("""\ + print( + key_value_dict.parseString( + """\ a = 10 b = 20 - """).dump()) + """ + ).dump() + ) try: print(key_value_dict.parseString("").dump()) except pp.ParseException as pe: print(pp.ParseException.explain(pe)) else: - self.assertTrue(False, "failed to raise exception when matching empty string") + self.assertTrue( + False, "failed to raise exception when matching empty string" + ) def testExplainException(self): import pyparsing as pp @@ -4510,47 +6006,55 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): print(pp.ParseException.explain(exc)) raise - def testCaselessKeywordVsKeywordCaseless(self): import pyparsing as pp - frule = pp.Keyword('t', caseless=True) + pp.Keyword('yes', caseless=True) - crule = pp.CaselessKeyword('t') + pp.CaselessKeyword('yes') + frule = pp.Keyword("t", caseless=True) + pp.Keyword("yes", caseless=True) + crule = pp.CaselessKeyword("t") + pp.CaselessKeyword("yes") - flist = frule.searchString('not yes').asList() + flist = frule.searchString("not yes").asList() print(flist) - clist = crule.searchString('not yes').asList() + clist = crule.searchString("not yes").asList() print(clist) - self.assertEqual(flist, clist, "CaselessKeyword not working the same as Keyword(caseless=True)") - + self.assertEqual( + flist, + clist, + "CaselessKeyword not working the same as Keyword(caseless=True)", + ) def testOneOfKeywords(self): import pyparsing as pp literal_expr = pp.oneOf("a b c") - success, _ = literal_expr[...].runTests(""" + success, _ = literal_expr[...].runTests( + """ # literal oneOf tests a b c a a a abc - """) + """ + ) self.assertTrue(success, "failed literal oneOf matching") keyword_expr = pp.oneOf("a b c", asKeyword=True) - success, _ = keyword_expr[...].runTests(""" + success, _ = keyword_expr[...].runTests( + """ # keyword oneOf tests a b c a a a - """) + """ + ) self.assertTrue(success, "failed keyword oneOf matching") - success, _ = keyword_expr[...].runTests(""" + success, _ = keyword_expr[...].runTests( + """ # keyword oneOf failure tests abc - """, failureTests=True) + """, + failureTests=True, + ) self.assertTrue(success, "failed keyword oneOf failure tests") - def testWarnUngroupedNamedTokens(self): """ - warn_ungrouped_named_tokens_in_collection - flag to enable warnings when a results @@ -4558,19 +6062,22 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): have results names (default=True) """ import pyparsing as pp + ppc = pp.pyparsing_common with ppt.reset_pyparsing_context(): pp.__diag__.enable("warn_ungrouped_named_tokens_in_collection") - COMMA = pp.Suppress(',').setName("comma") - coord = (ppc.integer('x') + COMMA + ppc.integer('y')) + COMMA = pp.Suppress(",").setName("comma") + coord = ppc.integer("x") + COMMA + ppc.integer("y") # this should emit a warning - with self.assertWarns(UserWarning, msg="failed to warn with named repetition of" - " ungrouped named expressions"): - path = coord[...].setResultsName('path') - + with self.assertWarns( + UserWarning, + msg="failed to warn with named repetition of" + " ungrouped named expressions", + ): + path = coord[...].setResultsName("path") def testWarnNameSetOnEmptyForward(self): """ @@ -4584,10 +6091,12 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): base = pp.Forward() - with self.assertWarns(UserWarning, msg="failed to warn when naming an empty Forward expression"): + with self.assertWarns( + UserWarning, + msg="failed to warn when naming an empty Forward expression", + ): base("x") - def testWarnOnMultipleStringArgsToOneOf(self): """ - warn_on_multiple_string_args_to_oneof - flag to enable warnings whan oneOf is @@ -4598,9 +6107,11 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): with ppt.reset_pyparsing_context(): pp.__diag__.enable("warn_on_multiple_string_args_to_oneof") - with self.assertWarns(UserWarning, msg="failed to warn when incorrectly calling oneOf(string, string)"): - a = pp.oneOf('A', 'B') - + with self.assertWarns( + UserWarning, + msg="failed to warn when incorrectly calling oneOf(string, string)", + ): + a = pp.oneOf("A", "B") def testEnableDebugOnNamedExpressions(self): """ @@ -4613,16 +6124,17 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): with ppt.reset_pyparsing_context(): test_stdout = StringIO() - with resetting(sys, 'stdout', 'stderr'): + with resetting(sys, "stdout", "stderr"): sys.stdout = test_stdout sys.stderr = test_stdout pp.__diag__.enable("enable_debug_on_named_expressions") - integer = pp.Word(pp.nums).setName('integer') + integer = pp.Word(pp.nums).setName("integer") integer[...].parseString("1 2 3") - expected_debug_output = textwrap.dedent("""\ + expected_debug_output = textwrap.dedent( + """\ Match integer at loc 0(1,1) Matched integer -> ['1'] Match integer at loc 1(1,2) @@ -4631,17 +6143,20 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): Matched integer -> ['3'] Match integer at loc 5(1,6) Exception raised:Expected integer, found end of text (at char 5), (line:1, col:6) - """) + """ + ) output = test_stdout.getvalue() print(output) - self.assertEquals(output, - expected_debug_output, - "failed to auto-enable debug on named expressions " - "using enable_debug_on_named_expressions") - + self.assertEquals( + output, + expected_debug_output, + "failed to auto-enable debug on named expressions " + "using enable_debug_on_named_expressions", + ) def testUndesirableButCommonPractices(self): import pyparsing as pp + ppc = pp.pyparsing_common # While these are valid constructs, and they are not encouraged @@ -4651,29 +6166,36 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): # Even though they are not encouraged, we shouldn't break them. # Create an And using a list of expressions instead of using '+' operator - expr = pp.And([pp.Word('abc'), pp.Word('123')]) - expr.runTests(""" + expr = pp.And([pp.Word("abc"), pp.Word("123")]) + expr.runTests( + """ aaa 333 b 1 ababab 32123 - """) + """ + ) # Passing a single expression to a ParseExpression, when it really wants a sequence expr = pp.Or(pp.Or(ppc.integer)) - expr.runTests(""" + expr.runTests( + """ 123 456 abc - """) + """ + ) def testEnableWarnDiags(self): import pyparsing as pp import pprint def filtered_vars(var_dict): - dunders = [nm for nm in var_dict if nm.startswith('__')] - return {k: v for k, v in var_dict.items() - if isinstance(v, bool) and k not in dunders} + dunders = [nm for nm in var_dict if nm.startswith("__")] + return { + k: v + for k, v in var_dict.items() + if isinstance(v, bool) and k not in dunders + } pprint.pprint(filtered_vars(vars(pp.__diag__)), width=30) @@ -4682,7 +6204,10 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): # make sure they are off by default for diag_name in warn_names: - self.assertFalse(getattr(pp.__diag__, diag_name), "__diag__.{} not set to True".format(diag_name)) + self.assertFalse( + getattr(pp.__diag__, diag_name), + "__diag__.{} not set to True".format(diag_name), + ) with ppt.reset_pyparsing_context(): # enable all warn_* diag_names @@ -4691,27 +6216,50 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): # make sure they are on after being enabled for diag_name in warn_names: - self.assertTrue(getattr(pp.__diag__, diag_name), "__diag__.{} not set to True".format(diag_name)) + self.assertTrue( + getattr(pp.__diag__, diag_name), + "__diag__.{} not set to True".format(diag_name), + ) # non-warn diag_names must be enabled individually for diag_name in other_names: - self.assertFalse(getattr(pp.__diag__, diag_name), "__diag__.{} not set to True".format(diag_name)) + self.assertFalse( + getattr(pp.__diag__, diag_name), + "__diag__.{} not set to True".format(diag_name), + ) # make sure they are off after AutoReset for diag_name in warn_names: - self.assertFalse(getattr(pp.__diag__, diag_name), "__diag__.{} not set to True".format(diag_name)) - + self.assertFalse( + getattr(pp.__diag__, diag_name), + "__diag__.{} not set to True".format(diag_name), + ) def testWordInternalReRanges(self): import pyparsing as pp import random import re - self.assertEqual(pp.Word(pp.printables).reString, "[!-~]+", "failed to generate correct internal re") - self.assertEqual(pp.Word(pp.alphanums).reString, "[0-9A-Za-z]+", "failed to generate correct internal re") - self.assertEqual(pp.Word(pp.pyparsing_unicode.Latin1.printables).reString, "[!-~¡-ÿ]+", - "failed to generate correct internal re") - self.assertEqual(pp.Word(pp.alphas8bit).reString, "[À-ÖØ-öø-ÿ]+", "failed to generate correct internal re") + self.assertEqual( + pp.Word(pp.printables).reString, + "[!-~]+", + "failed to generate correct internal re", + ) + self.assertEqual( + pp.Word(pp.alphanums).reString, + "[0-9A-Za-z]+", + "failed to generate correct internal re", + ) + self.assertEqual( + pp.Word(pp.pyparsing_unicode.Latin1.printables).reString, + "[!-~¡-ÿ]+", + "failed to generate correct internal re", + ) + self.assertEqual( + pp.Word(pp.alphas8bit).reString, + "[À-ÖØ-öø-ÿ]+", + "failed to generate correct internal re", + ) esc_chars = r"\^-]" esc_chars2 = r"*+.?[" @@ -4720,61 +6268,149 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): next_char = chr(ord(esc_char) + 1) prev_char = chr(ord(esc_char) - 1) esc_word = pp.Word(esc_char + next_char) - expected = r"[{}{}-{}{}]+".format('\\' if esc_char in esc_chars else '', esc_char, - '\\' if next_char in esc_chars else '', next_char) - print("Testing escape char: {} -> {} re: '{}')".format(esc_char, esc_word, esc_word.reString)) - self.assertEqual(esc_word.reString, expected, "failed to generate correct internal re") - test_string = ''.join(random.choice([esc_char, next_char]) for __ in range(16)) - print("Match '{}' -> {}".format(test_string, test_string == esc_word.parseString(test_string)[0])) - self.assertEqual(esc_word.parseString(test_string)[0], test_string, - "Word using escaped range char failed to parse") + expected = r"[{}{}-{}{}]+".format( + "\\" if esc_char in esc_chars else "", + esc_char, + "\\" if next_char in esc_chars else "", + next_char, + ) + print( + "Testing escape char: {} -> {} re: '{}')".format( + esc_char, esc_word, esc_word.reString + ) + ) + self.assertEqual( + esc_word.reString, expected, "failed to generate correct internal re" + ) + test_string = "".join( + random.choice([esc_char, next_char]) for __ in range(16) + ) + print( + "Match '{}' -> {}".format( + test_string, test_string == esc_word.parseString(test_string)[0] + ) + ) + self.assertEqual( + esc_word.parseString(test_string)[0], + test_string, + "Word using escaped range char failed to parse", + ) # test escape char as last character in range esc_word = pp.Word(prev_char + esc_char) - expected = r"[{}{}-{}{}]+".format('\\' if prev_char in esc_chars else '', prev_char, - '\\' if esc_char in esc_chars else '', esc_char) - print("Testing escape char: {} -> {} re: '{}')".format(esc_char, esc_word, esc_word.reString)) - self.assertEqual(esc_word.reString, expected, "failed to generate correct internal re") - test_string = ''.join(random.choice([esc_char, prev_char]) for __ in range(16)) - print("Match '{}' -> {}".format(test_string, test_string == esc_word.parseString(test_string)[0])) - self.assertEqual(esc_word.parseString(test_string)[0], test_string, - "Word using escaped range char failed to parse") + expected = r"[{}{}-{}{}]+".format( + "\\" if prev_char in esc_chars else "", + prev_char, + "\\" if esc_char in esc_chars else "", + esc_char, + ) + print( + "Testing escape char: {} -> {} re: '{}')".format( + esc_char, esc_word, esc_word.reString + ) + ) + self.assertEqual( + esc_word.reString, expected, "failed to generate correct internal re" + ) + test_string = "".join( + random.choice([esc_char, prev_char]) for __ in range(16) + ) + print( + "Match '{}' -> {}".format( + test_string, test_string == esc_word.parseString(test_string)[0] + ) + ) + self.assertEqual( + esc_word.parseString(test_string)[0], + test_string, + "Word using escaped range char failed to parse", + ) # test escape char as first character in range next_char = chr(ord(esc_char) + 1) prev_char = chr(ord(esc_char) - 1) esc_word = pp.Word(esc_char + next_char) - expected = r"[{}{}-{}{}]+".format('\\' if esc_char in esc_chars else '', esc_char, - '\\' if next_char in esc_chars else '', next_char) - print("Testing escape char: {} -> {} re: '{}')".format(esc_char, esc_word, esc_word.reString)) - self.assertEqual(esc_word.reString, expected, "failed to generate correct internal re") - test_string = ''.join(random.choice([esc_char, next_char]) for __ in range(16)) - print("Match '{}' -> {}".format(test_string, test_string == esc_word.parseString(test_string)[0])) - self.assertEqual(esc_word.parseString(test_string)[0], test_string, - "Word using escaped range char failed to parse") + expected = r"[{}{}-{}{}]+".format( + "\\" if esc_char in esc_chars else "", + esc_char, + "\\" if next_char in esc_chars else "", + next_char, + ) + print( + "Testing escape char: {} -> {} re: '{}')".format( + esc_char, esc_word, esc_word.reString + ) + ) + self.assertEqual( + esc_word.reString, expected, "failed to generate correct internal re" + ) + test_string = "".join( + random.choice([esc_char, next_char]) for __ in range(16) + ) + print( + "Match '{}' -> {}".format( + test_string, test_string == esc_word.parseString(test_string)[0] + ) + ) + self.assertEqual( + esc_word.parseString(test_string)[0], + test_string, + "Word using escaped range char failed to parse", + ) # test escape char as only character in range esc_word = pp.Word(esc_char + esc_char, pp.alphas.upper()) - expected = r"[{}{}][A-Z]*".format('\\' if esc_char in esc_chars else '', esc_char) - print("Testing escape char: {} -> {} re: '{}')".format(esc_char, esc_word, esc_word.reString)) - self.assertEqual(esc_word.reString, expected, "failed to generate correct internal re") - test_string = esc_char + ''.join(random.choice(pp.alphas.upper()) for __ in range(16)) - print("Match '{}' -> {}".format(test_string, test_string == esc_word.parseString(test_string)[0])) - self.assertEqual(esc_word.parseString(test_string)[0], test_string, - "Word using escaped range char failed to parse") + expected = r"[{}{}][A-Z]*".format( + "\\" if esc_char in esc_chars else "", esc_char + ) + print( + "Testing escape char: {} -> {} re: '{}')".format( + esc_char, esc_word, esc_word.reString + ) + ) + self.assertEqual( + esc_word.reString, expected, "failed to generate correct internal re" + ) + test_string = esc_char + "".join( + random.choice(pp.alphas.upper()) for __ in range(16) + ) + print( + "Match '{}' -> {}".format( + test_string, test_string == esc_word.parseString(test_string)[0] + ) + ) + self.assertEqual( + esc_word.parseString(test_string)[0], + test_string, + "Word using escaped range char failed to parse", + ) # test escape char as only character esc_word = pp.Word(esc_char, pp.alphas.upper()) expected = r"{}[A-Z]*".format(re.escape(esc_char)) - print("Testing escape char: {} -> {} re: '{}')".format(esc_char, esc_word, esc_word.reString)) - self.assertEqual(esc_word.reString, expected, "failed to generate correct internal re") - test_string = esc_char + ''.join(random.choice(pp.alphas.upper()) for __ in range(16)) - print("Match '{}' -> {}".format(test_string, test_string == esc_word.parseString(test_string)[0])) - self.assertEqual(esc_word.parseString(test_string)[0], test_string, - "Word using escaped range char failed to parse") + print( + "Testing escape char: {} -> {} re: '{}')".format( + esc_char, esc_word, esc_word.reString + ) + ) + self.assertEqual( + esc_word.reString, expected, "failed to generate correct internal re" + ) + test_string = esc_char + "".join( + random.choice(pp.alphas.upper()) for __ in range(16) + ) + print( + "Match '{}' -> {}".format( + test_string, test_string == esc_word.parseString(test_string)[0] + ) + ) + self.assertEqual( + esc_word.parseString(test_string)[0], + test_string, + "Word using escaped range char failed to parse", + ) print() - def testMiscellaneousParserTests(self): runtests = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" @@ -4787,31 +6423,45 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): try: test1 = pp.oneOf("a b c d a") except RuntimeError: - self.assertTrue(False, "still have infinite loop in oneOf with duplicate symbols (string input)") + self.assertTrue( + False, + "still have infinite loop in oneOf with duplicate symbols (string input)", + ) print("verify oneOf handles generator input") try: test1 = pp.oneOf(c for c in "a b c d a" if not c.isspace()) except RuntimeError: - self.assertTrue(False, "still have infinite loop in oneOf with duplicate symbols (generator input)") + self.assertTrue( + False, + "still have infinite loop in oneOf with duplicate symbols (generator input)", + ) print("verify oneOf handles list input") try: test1 = pp.oneOf("a b c d a".split()) except RuntimeError: - self.assertTrue(False, "still have infinite loop in oneOf with duplicate symbols (list input)") + self.assertTrue( + False, + "still have infinite loop in oneOf with duplicate symbols (list input)", + ) print("verify oneOf handles set input") try: test1 = pp.oneOf(set("a b c d a")) except RuntimeError: - self.assertTrue(False, "still have infinite loop in oneOf with duplicate symbols (set input)") + self.assertTrue( + False, + "still have infinite loop in oneOf with duplicate symbols (set input)", + ) # test MatchFirst bugfix if "B" in runtests: print("verify MatchFirst iterates properly") results = pp.quotedString.parseString("'this is a single quoted string'") - self.assertTrue(len(results) > 0, "MatchFirst error - not iterating over all choices") + self.assertTrue( + len(results) > 0, "MatchFirst error - not iterating over all choices" + ) # verify streamline of subexpressions if "C" in runtests: @@ -4832,32 +6482,41 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): testGrammar.parseString("AC") except pp.ParseException as pe: print(pe.pstr, "->", pe) - self.assertTrue(False, "error in Optional matching of string %s" % pe.pstr) + self.assertTrue( + False, "error in Optional matching of string %s" % pe.pstr + ) # test return of furthest exception if "E" in runtests: - testGrammar = (pp.Literal("A") | - (pp.Optional("B") + pp.Literal("C")) | - pp.Literal("D")) + testGrammar = ( + pp.Literal("A") | (pp.Optional("B") + pp.Literal("C")) | pp.Literal("D") + ) try: testGrammar.parseString("BC") testGrammar.parseString("BD") except pp.ParseException as pe: print(pe.pstr, "->", pe) self.assertEqual(pe.pstr, "BD", "wrong test string failed to parse") - self.assertEqual(pe.loc, 1, "error in Optional matching, pe.loc=" + str(pe.loc)) + self.assertEqual( + pe.loc, 1, "error in Optional matching, pe.loc=" + str(pe.loc) + ) # test validate if "F" in runtests: print("verify behavior of validate()") + def testValidation(grmr, gnam, isValid): try: grmr.streamline() grmr.validate() - self.assertTrue(isValid, "validate() accepted invalid grammar " + gnam) + self.assertTrue( + isValid, "validate() accepted invalid grammar " + gnam + ) except pp.RecursiveGrammarException as e: print(grmr) - self.assertFalse(isValid, "validate() rejected valid grammar " + gnam) + self.assertFalse( + isValid, "validate() rejected valid grammar " + gnam + ) fwd = pp.Forward() g1 = pp.OneOrMore((pp.Literal("A") + "B" + "C") | fwd) @@ -4894,15 +6553,21 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): names.append(None) print(teststring) print(names) - self.assertEqual(names, [None, 'B', 'B', 'A', 'B', 'B', 'A', 'B', None, 'B', 'A'], - "failure in getting names for tokens") + self.assertEqual( + names, + [None, "B", "B", "A", "B", "B", "A", "B", None, "B", "A"], + "failure in getting names for tokens", + ) from pyparsing import Keyword, Word, alphas, OneOrMore + IF, AND, BUT = map(Keyword, "if and but".split()) ident = ~(IF | AND | BUT) + Word(alphas)("non-key") scanner = OneOrMore(IF | AND | BUT | ident) + def getNameTester(s, l, t): print(t, t.getName()) + ident.addParseAction(getNameTester) scanner.parseString("lsjd sldkjf IF Saslkj AND lsdjf") @@ -4914,7 +6579,9 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): print(res.dump()) print(res.get("A", "A not found")) print(res.get("D", "!D")) - self.assertEqual(res.get("A", "A not found"), "aaa", "get on existing key failed") + self.assertEqual( + res.get("A", "A not found"), "aaa", "get on existing key failed" + ) self.assertEqual(res.get("D", "!D"), "!D", "get on missing key failed") if "I" in runtests: @@ -4926,7 +6593,9 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): # test creating Literal with empty string if "J" in runtests: print('verify non-fatal usage of Literal("")') - with self.assertWarns(SyntaxWarning, msg="failed to warn use of empty string for Literal"): + with self.assertWarns( + SyntaxWarning, msg="failed to warn use of empty string for Literal" + ): e = pp.Literal("") try: e.parseString("SLJFD") @@ -4935,20 +6604,32 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): # test line() behavior when starting at 0 and the opening line is an \n if "K" in runtests: - print('verify correct line() behavior when first line is empty string') - self.assertEqual(pp.line(0, "\nabc\ndef\n"), '', "Error in line() with empty first line in text") + print("verify correct line() behavior when first line is empty string") + self.assertEqual( + pp.line(0, "\nabc\ndef\n"), + "", + "Error in line() with empty first line in text", + ) txt = "\nabc\ndef\n" results = [pp.line(i, txt) for i in range(len(txt))] - self.assertEqual(results, ['', 'abc', 'abc', 'abc', 'abc', 'def', 'def', 'def', 'def'], - "Error in line() with empty first line in text") + self.assertEqual( + results, + ["", "abc", "abc", "abc", "abc", "def", "def", "def", "def"], + "Error in line() with empty first line in text", + ) txt = "abc\ndef\n" results = [pp.line(i, txt) for i in range(len(txt))] - self.assertEqual(results, ['abc', 'abc', 'abc', 'abc', 'def', 'def', 'def', 'def'], - "Error in line() with non-empty first line in text") + self.assertEqual( + results, + ["abc", "abc", "abc", "abc", "def", "def", "def", "def"], + "Error in line() with non-empty first line in text", + ) # test bugfix with repeated tokens when packrat parsing enabled if "L" in runtests: - print('verify behavior with repeated tokens when packrat parsing is enabled') + print( + "verify behavior with repeated tokens when packrat parsing is enabled" + ) a = pp.Literal("a") b = pp.Literal("b") c = pp.Literal("c") @@ -4958,35 +6639,81 @@ class Test2_WithoutPackrat(TestParseResultsAsserts): aba = a + b + a grammar = abb | abc | aba - self.assertEqual(''.join(grammar.parseString("aba")), 'aba', "Packrat ABA failure!") + self.assertEqual( + "".join(grammar.parseString("aba")), "aba", "Packrat ABA failure!" + ) if "M" in runtests: - print('verify behavior of setResultsName with OneOrMore and ZeroOrMore') - - stmt = pp.Keyword('test') - print(stmt[...]('tests').parseString('test test').tests) - print(stmt[1, ...]('tests').parseString('test test').tests) - print(pp.Optional(stmt[1, ...]('tests')).parseString('test test').tests) - print(pp.Optional(stmt[1, ...])('tests').parseString('test test').tests) - print(pp.Optional(pp.delimitedList(stmt))('tests').parseString('test,test').tests) - self.assertEqual(len(stmt[...]('tests').parseString('test test').tests), 2, "ZeroOrMore failure with setResultsName") - self.assertEqual(len(stmt[1, ...]('tests').parseString('test test').tests), 2, "OneOrMore failure with setResultsName") - self.assertEqual(len(pp.Optional(stmt[1, ...]('tests')).parseString('test test').tests), 2, "OneOrMore failure with setResultsName") - self.assertEqual(len(pp.Optional(pp.delimitedList(stmt))('tests').parseString('test,test').tests), 2, "delimitedList failure with setResultsName") - self.assertEqual(len((stmt * 2)('tests').parseString('test test').tests), 2, "multiplied(1) failure with setResultsName") - self.assertEqual(len(stmt[..., 2]('tests').parseString('test test').tests), 2, "multiplied(2) failure with setResultsName") - self.assertEqual(len(stmt[1, ...]('tests').parseString('test test').tests), 2, "multipled(3) failure with setResultsName") - self.assertEqual(len(stmt[2, ...]('tests').parseString('test test').tests), 2, "multipled(3) failure with setResultsName") - - -class PickleTest_Greeting(): + print("verify behavior of setResultsName with OneOrMore and ZeroOrMore") + + stmt = pp.Keyword("test") + print(stmt[...]("tests").parseString("test test").tests) + print(stmt[1, ...]("tests").parseString("test test").tests) + print(pp.Optional(stmt[1, ...]("tests")).parseString("test test").tests) + print(pp.Optional(stmt[1, ...])("tests").parseString("test test").tests) + print( + pp.Optional(pp.delimitedList(stmt))("tests") + .parseString("test,test") + .tests + ) + self.assertEqual( + len(stmt[...]("tests").parseString("test test").tests), + 2, + "ZeroOrMore failure with setResultsName", + ) + self.assertEqual( + len(stmt[1, ...]("tests").parseString("test test").tests), + 2, + "OneOrMore failure with setResultsName", + ) + self.assertEqual( + len(pp.Optional(stmt[1, ...]("tests")).parseString("test test").tests), + 2, + "OneOrMore failure with setResultsName", + ) + self.assertEqual( + len( + pp.Optional(pp.delimitedList(stmt))("tests") + .parseString("test,test") + .tests + ), + 2, + "delimitedList failure with setResultsName", + ) + self.assertEqual( + len((stmt * 2)("tests").parseString("test test").tests), + 2, + "multiplied(1) failure with setResultsName", + ) + self.assertEqual( + len(stmt[..., 2]("tests").parseString("test test").tests), + 2, + "multiplied(2) failure with setResultsName", + ) + self.assertEqual( + len(stmt[1, ...]("tests").parseString("test test").tests), + 2, + "multipled(3) failure with setResultsName", + ) + self.assertEqual( + len(stmt[2, ...]("tests").parseString("test test").tests), + 2, + "multipled(3) failure with setResultsName", + ) + + +class PickleTest_Greeting: def __init__(self, toks): self.salutation = toks[0] self.greetee = toks[1] def __repr__(self): - return "{}: {{{}}}".format(self.__class__.__name__, - ', '.join('{!r}: {!r}'.format(k, getattr(self, k)) for k in sorted(self.__dict__))) + return "{}: {{{}}}".format( + self.__class__.__name__, + ", ".join( + "{!r}: {!r}".format(k, getattr(self, k)) for k in sorted(self.__dict__) + ), + ) class Test3_EnablePackratParsing(TestCase): @@ -4997,14 +6724,9 @@ class Test3_EnablePackratParsing(TestCase): Test2_WithoutPackrat.suite_context = ppt.reset_pyparsing_context() Test2_WithoutPackrat.suite_context.save() -Test4_WithPackrat = type( - "Test4_WithPackrat", - (Test2_WithoutPackrat,), - {} -) + +Test4_WithPackrat = type("Test4_WithPackrat", (Test2_WithoutPackrat,), {}) Test2_WithoutPackrat.suite_context = ppt.reset_pyparsing_context() Test2_WithoutPackrat.suite_context.save() - - @@ -1,7 +1,13 @@ [tox] -envlist=py{35,36,37,38,py3} +envlist = + black + py{35,36,37,38,py3} [testenv] deps=coverage commands= coverage run --parallel --branch -m unittest + +[testenv:black] +deps = black +commands = {envbindir}/black --target-version py35 --check --diff . diff --git a/update_pyparsing_timestamp.py b/update_pyparsing_timestamp.py index 841b8ec..09233fa 100644 --- a/update_pyparsing_timestamp.py +++ b/update_pyparsing_timestamp.py @@ -3,15 +3,15 @@ from datetime import datetime nw = datetime.utcnow()
nowstring = '"%s"' % (nw.strftime("%d %b %Y %X")[:-3] + " UTC")
-print (nowstring)
+print(nowstring)
quoted_time = quotedString()
quoted_time.setParseAction(lambda: nowstring)
version_time = "__versionTime__ = " + quoted_time
-with open('pyparsing.py', encoding='utf-8') as oldpp:
+with open("pyparsing.py", encoding="utf-8") as oldpp:
orig_code = oldpp.read()
new_code = version_time.transformString(orig_code)
-with open('pyparsing.py','w', encoding='utf-8') as newpp:
+with open("pyparsing.py", "w", encoding="utf-8") as newpp:
newpp.write(new_code)
|