Updated runTests to call postParse before dumping parsed results; added nested_markup.py and updated delta_time.py (renamed from deltaTime.py) examples

3 files changed, 366 insertions, 208 deletions

diff --git a/examples/deltaTime.py b/examples/deltaTime.py
deleted file mode 100644
index 2fa8769..0000000
--- a/examples/deltaTime.py
+++ /dev/null
@@ -1,208 +0,0 @@
-# deltaTime.py

-#

-# Parser to convert a conversational time reference such as "in a minute" or

-# "noon tomorrow" and convert it to a Python datetime.  The returned

-# ParseResults object contains the results name "timeOffset" containing

-# the timedelta, and "calculatedTime" containing the computed time relative

-# to datetime.now().

-#

-# Copyright 2010, by Paul McGuire

-#

-

-from datetime import datetime, timedelta

-from pyparsing import *

-import calendar

-

-__all__ = ["nlTimeExpression"]

-

-# string conversion parse actions

-def convertToTimedelta(toks):

-    unit = toks.timeunit.lower().rstrip("s")

-    td = {

-        'week'    : timedelta(7),

-        'day'    : timedelta(1),

-        'hour'   : timedelta(0,0,0,0,0,1),

-        'minute' : timedelta(0,0,0,0,1),

-        'second' : timedelta(0,1),

-        }[unit]

-    if toks.qty:

-        td *= int(toks.qty)

-    if toks.dir:

-        td *= toks.dir

-    toks["timeOffset"] = td

-

-def convertToDay(toks):

-    now = datetime.now()

-    if "wkdayRef" in toks:

-        todaynum = now.weekday()

-        daynames = [n.lower() for n in calendar.day_name]

-        nameddaynum = daynames.index(toks.wkdayRef.day.lower())

-        if toks.wkdayRef.dir > 0:

-            daydiff = (nameddaynum + 7 - todaynum) % 7

-        else:

-            daydiff = -((todaynum + 7 - nameddaynum) % 7)

-        toks["absTime"] = datetime(now.year, now.month, now.day)+timedelta(daydiff)

-    else:

-        name = toks.name.lower()

-        toks["absTime"] = {

-            "now"       : now,

-            "today"     : datetime(now.year, now.month, now.day),

-            "yesterday" : datetime(now.year, now.month, now.day)+timedelta(-1),

-            "tomorrow"  : datetime(now.year, now.month, now.day)+timedelta(+1),

-            }[name]

-

-def convertToAbsTime(toks):

-    now = datetime.now()

-    if "dayRef" in toks:

-        day = toks.dayRef.absTime

-        day = datetime(day.year, day.month, day.day)

-    else:

-        day = datetime(now.year, now.month, now.day)

-    if "timeOfDay" in toks:

-        if isinstance(toks.timeOfDay,str):

-            timeOfDay = {

-                "now"      : timedelta(0, (now.hour*60+now.minute)*60+now.second, now.microsecond),

-                "noon"     : timedelta(0,0,0,0,0,12),

-                "midnight" : timedelta(),

-                }[toks.timeOfDay]

-        else:

-            hhmmss = toks.timeparts

-            if hhmmss.miltime:

-                hh,mm = hhmmss.miltime

-                ss = 0

-            else:

-                hh,mm,ss = (hhmmss.HH % 12), hhmmss.MM, hhmmss.SS

-                if not mm: mm = 0

-                if not ss: ss = 0

-                if toks.timeOfDay.ampm == 'pm':

-                    hh += 12

-            timeOfDay = timedelta(0, (hh*60+mm)*60+ss, 0)

-    else:

-        timeOfDay = timedelta(0, (now.hour*60+now.minute)*60+now.second, now.microsecond)

-    toks["absTime"] = day + timeOfDay

-

-def calculateTime(toks):

-    if toks.absTime:

-        absTime = toks.absTime

-    else:

-        absTime = datetime.now()

-    if toks.timeOffset:

-        absTime += toks.timeOffset

-    toks["calculatedTime"] = absTime

-

-# grammar definitions

-CL = CaselessLiteral

-today, tomorrow, yesterday, noon, midnight, now = map( CL,

-    "today tomorrow yesterday noon midnight now".split())

-plural = lambda s : Combine(CL(s) + Optional(CL("s")))

-week, day, hour, minute, second = map( plural,

-    "week day hour minute second".split())

-am = CL("am")

-pm = CL("pm")

-COLON = Suppress(':')

-

-# are these actually operators?

-in_ = CL("in").setParseAction(replaceWith(1))

-from_ = CL("from").setParseAction(replaceWith(1))

-before = CL("before").setParseAction(replaceWith(-1))

-after = CL("after").setParseAction(replaceWith(1))

-ago = CL("ago").setParseAction(replaceWith(-1))

-next_ = CL("next").setParseAction(replaceWith(1))

-last_ = CL("last").setParseAction(replaceWith(-1))

-at_ = CL("at")

-on_ = CL("on")

-

-couple = (Optional(CL("a")) + CL("couple") + Optional(CL("of"))).setParseAction(replaceWith(2))

-a_qty = CL("a").setParseAction(replaceWith(1))

-integer = Word(nums).setParseAction(lambda t:int(t[0]))

-int4 = Group(Word(nums,exact=4).setParseAction(lambda t: [int(t[0][:2]),int(t[0][2:])] ))

-def fill_timefields(t):

-    t[0]['HH'] = t[0][0]

-    t[0]['MM'] = t[0][1]

-    t[0]['ampm'] = ('am','pm')[t[0].HH >= 12]

-int4.addParseAction(fill_timefields)

-qty = integer | couple | a_qty

-dayName = oneOf( list(calendar.day_name) )

-

-dayOffset = (qty("qty") + (week | day)("timeunit"))

-dayFwdBack = (from_ + now.suppress() | ago)("dir")

-weekdayRef = (Optional(next_ | last_,1)("dir") + dayName("day"))

-dayRef = Optional( (dayOffset + (before | after | from_)("dir") ).setParseAction(convertToTimedelta) ) + \

-            ((yesterday | today | tomorrow)("name")|

-             weekdayRef("wkdayRef")).setParseAction(convertToDay)

-todayRef = (dayOffset + dayFwdBack).setParseAction(convertToTimedelta) | \

-            (in_("dir") + qty("qty") + day("timeunit")).setParseAction(convertToTimedelta)

-

-dayTimeSpec = dayRef | todayRef

-dayTimeSpec.setParseAction(calculateTime)

-

-relativeTimeUnit = (week | day | hour | minute | second)

-

-timespec = Group(ungroup(int4) |

-                 integer("HH") +

-                 ungroup(Optional(COLON + integer,[0]))("MM") +

-                 ungroup(Optional(COLON + integer,[0]))("SS") +

-                 (am | pm)("ampm")

-                 )

-

-absTimeSpec = ((noon | midnight | now | timespec("timeparts"))("timeOfDay") +

-                Optional(on_) + Optional(dayRef)("dayRef") |

-                dayRef("dayRef") + at_ +

-                (noon | midnight | now | timespec("timeparts"))("timeOfDay"))

-absTimeSpec.setParseAction(convertToAbsTime,calculateTime)

-

-relTimeSpec = qty("qty") + relativeTimeUnit("timeunit") + \

-                (from_ | before | after)("dir") + \

-                Optional(at_) + \

-                absTimeSpec("absTime") | \

-              qty("qty") + relativeTimeUnit("timeunit") + ago("dir") | \

-              in_ + qty("qty") + relativeTimeUnit("timeunit")

-relTimeSpec.setParseAction(convertToTimedelta,calculateTime)

-

-nlTimeExpression = (absTimeSpec + Optional(dayTimeSpec) |

-                    dayTimeSpec + Optional(Optional(at_) + absTimeSpec) |

-                    relTimeSpec + Optional(absTimeSpec))

-

-if __name__ == "__main__":

-    # test grammar

-    tests = """\

-    today

-    tomorrow

-    yesterday

-    in a couple of days

-    a couple of days from now

-    a couple of days from today

-    in a day

-    3 days ago

-    3 days from now

-    a day ago

-    in 2 weeks

-    in 3 days at 5pm

-    now

-    10 minutes ago

-    10 minutes from now

-    in 10 minutes

-    in a minute

-    in a couple of minutes

-    20 seconds ago

-    in 30 seconds

-    20 seconds before noon

-    20 seconds before noon tomorrow

-    noon

-    midnight

-    noon tomorrow

-    6am tomorrow

-    0800 yesterday

-    12:15 AM today

-    3pm 2 days from today

-    a week from today

-    a week from now

-    3 weeks ago

-    noon next Sunday

-    noon Sunday

-    noon last Sunday

-    2pm next Sunday

-    next Sunday at 2pm"""

-

-    print("(relative to %s)" % datetime.now())

-    nlTimeExpression.runTests(tests)

diff --git a/examples/delta_time.py b/examples/delta_time.py
new file mode 100644
index 0000000..5c6ceb7
--- /dev/null
+++ b/examples/delta_time.py
@@ -0,0 +1,318 @@
+# deltaTime.py
+#
+# Parser to convert a conversational time reference such as "in a minute" or
+# "noon tomorrow" and convert it to a Python datetime. The returned
+# ParseResults object contains
+#   - original - the original time expression string
+#   - computed_dt - the Python datetime representing the computed time
+#   - relative_to - the reference "now" time
+#   - time_offset - the difference between the reference time and the computed time
+#
+# BNF:
+#     time_and_day ::= time_reference [day_reference] | day_reference 'at' absolute_time_of_day
+#     day_reference ::= absolute_day_reference | relative_day_reference
+#     absolute_day_reference ::= 'today' | 'tomorrow' | 'yesterday' | ('next' | 'last') weekday_name
+#     relative_day_reference ::= 'in' qty day_units
+#                                | qty day_units 'ago'
+#                                | 'qty day_units ('from' | 'before' | 'after') absolute_day_reference
+#     day_units ::= 'days' | 'weeks'
+#
+#     time_reference ::= absolute_time_of_day | relative_time_reference
+#     relative_time_reference ::= qty time_units ('from' | 'before' | 'after') absolute_time_of_day
+#                                 | qty time_units 'ago'
+#                                 | 'in' qty time_units
+#     time_units ::= 'hours' | 'minutes' | 'seconds'
+#     absolute_time_of_day ::= 'noon' | 'midnight' | 'now' | absolute_time
+#     absolute_time ::=  24hour_time | hour ("o'clock" | ':' minute) ('AM'|'PM')
+#
+#     qty ::= integer | integer_words | 'a couple of' | 'a' | 'the'
+#
+# Copyright 2010, 2019 by Paul McGuire
+#
+
+from datetime import datetime, time, timedelta
+import pyparsing as pp
+import calendar
+
+__all__ = ["time_expression"]
+
+# basic grammar definitions
+def make_integer_word_expr(int_name, int_value):
+    return pp.CaselessKeyword(int_name).addParseAction(pp.replaceWith(int_value))
+integer_word = pp.Or(make_integer_word_expr(int_str, int_value)
+                     for int_value, int_str
+                     in enumerate("one two three four five six seven eight nine ten"
+                                  " eleven twelve thirteen fourteen fifteen sixteen"
+                                  " seventeen eighteen nineteen twenty".split(), start=1))
+integer = pp.pyparsing_common.integer | integer_word
+
+CK = pp.CaselessKeyword
+CL = pp.CaselessLiteral
+today, tomorrow, yesterday, noon, midnight, now = map(CK, "today tomorrow yesterday noon midnight now".split())
+def plural(s):
+    return CK(s) | CK(s + 's').addParseAction(pp.replaceWith(s))
+week, day, hour, minute, second = map(plural, "week day hour minute second".split())
+am = CL("am")
+pm = CL("pm")
+COLON = pp.Suppress(':')
+
+in_ = CK("in").setParseAction(pp.replaceWith(1))
+from_ = CK("from").setParseAction(pp.replaceWith(1))
+before = CK("before").setParseAction(pp.replaceWith(-1))
+after = CK("after").setParseAction(pp.replaceWith(1))
+ago = CK("ago").setParseAction(pp.replaceWith(-1))
+next_ = CK("next").setParseAction(pp.replaceWith(1))
+last_ = CK("last").setParseAction(pp.replaceWith(-1))
+at_ = CK("at")
+on_ = CK("on")
+
+couple = (pp.Optional(CK("a")) + CK("couple") + pp.Optional(CK("of"))).setParseAction(pp.replaceWith(2))
+a_qty = CK("a").setParseAction(pp.replaceWith(1))
+the_qty = CK("the").setParseAction(pp.replaceWith(1))
+qty = integer | couple | a_qty | the_qty
+time_ref_present = pp.Empty().addParseAction(pp.replaceWith(True))('time_ref_present')
+
+def fill_24hr_time_fields(t):
+    t['HH'] = t[0]
+    t['MM'] = t[1]
+    t['SS'] = 0
+    t['ampm'] = ('am','pm')[t.HH >= 12]
+
+def fill_default_time_fields(t):
+    for fld in 'HH MM SS'.split():
+        if fld not in t:
+            t[fld] = 0
+
+weekday_name_list = list(calendar.day_name)
+weekday_name = pp.oneOf(weekday_name_list)
+
+_24hour_time = pp.Word(pp.nums, exact=4).addParseAction(lambda t: [int(t[0][:2]),int(t[0][2:])],
+                                                        fill_24hr_time_fields)
+_24hour_time.setName("0000 time")
+ampm = am | pm
+timespec = (integer("HH")
+                    + pp.Optional(CK("o'clock")
+                       |
+                       COLON + integer("MM")
+                       + pp.Optional(COLON + integer("SS"))
+                       )
+                   + (am | pm)("ampm")
+                 ).addParseAction(fill_default_time_fields)
+absolute_time = _24hour_time | timespec
+
+absolute_time_of_day = noon | midnight | now | absolute_time
+
+def add_computed_time(t):
+    if t[0] in 'now noon midnight'.split():
+        t['computed_time'] = {'now': datetime.now().time().replace(microsecond=0),
+                              'noon': time(hour=12),
+                              'midnight': time()}[t[0]]
+    else:
+        t['HH'] = {'am': int(t['HH']) % 12,
+                   'pm': int(t['HH']) % 12 + 12}[t.ampm]
+        t['computed_time'] = time(hour=t.HH, minute=t.MM, second=t.SS)
+
+absolute_time_of_day.addParseAction(add_computed_time)
+
+
+#     relative_time_reference ::= qty time_units ('from' | 'before' | 'after') absolute_time_of_day
+#                                 | qty time_units 'ago'
+#                                 | 'in' qty time_units
+time_units = hour | minute | second
+relative_time_reference = (qty('qty') + time_units('units') + ago('dir')
+                           | qty('qty') + time_units('units')
+                                 + (from_ | before | after)('dir')
+                                 + pp.Group(absolute_time_of_day)('ref_time')
+                           | in_('dir') + qty('qty') + time_units('units')
+                           )
+
+def compute_relative_time(t):
+    if 'ref_time' not in t:
+        t['ref_time'] = datetime.now().time().replace(microsecond=0)
+    else:
+        t['ref_time'] = t.ref_time.computed_time
+    delta_seconds = {'hour': 3600,
+                     'minute': 60,
+                     'second': 1}[t.units] * t.qty[0]
+    t['time_delta'] = timedelta(seconds=t.dir * delta_seconds)
+
+relative_time_reference.addParseAction(compute_relative_time)
+
+time_reference = absolute_time_of_day | relative_time_reference
+def add_default_time_ref_fields(t):
+    if 'time_delta' not in t:
+        t['time_delta'] = timedelta()
+time_reference.addParseAction(add_default_time_ref_fields)
+
+#     absolute_day_reference ::= 'today' | 'tomorrow' | 'yesterday' | ('next' | 'last') weekday_name
+#     day_units ::= 'days' | 'weeks'
+
+day_units = day | week
+weekday_reference = pp.Optional(next_ | last_, 1)('dir') + weekday_name('day_name')
+
+def convert_abs_day_reference_to_date(t):
+    now = datetime.now().replace(microsecond=0)
+
+    # handle day reference by weekday name
+    if 'day_name' in t:
+        todaynum = now.weekday()
+        daynames = [n.lower() for n in weekday_name_list]
+        nameddaynum = daynames.index(t.day_name.lower())
+        # compute difference in days - if current weekday name is referenced, then
+        # computed 0 offset is changed to 7
+        if t.dir > 0:
+            daydiff = (nameddaynum + 7 - todaynum) % 7 or 7
+        else:
+            daydiff = -((todaynum + 7 - nameddaynum) % 7 or 7)
+        t["abs_date"] = datetime(now.year, now.month, now.day)+timedelta(daydiff)
+    else:
+        name = t[0]
+        t["abs_date"] = {
+            "now"       : now,
+            "today"     : datetime(now.year, now.month, now.day),
+            "yesterday" : datetime(now.year, now.month, now.day)+timedelta(days=-1),
+            "tomorrow"  : datetime(now.year, now.month, now.day)+timedelta(days=+1),
+            }[name]
+
+absolute_day_reference = today | tomorrow | yesterday | now + time_ref_present | weekday_reference
+absolute_day_reference.addParseAction(convert_abs_day_reference_to_date)
+
+
+#     relative_day_reference ::=  'in' qty day_units
+#                                   | qty day_units 'ago'
+#                                   | 'qty day_units ('from' | 'before' | 'after') absolute_day_reference
+relative_day_reference = (in_('dir') + qty('qty') + day_units('units')
+                          | qty('qty') + day_units('units') + ago('dir')
+                          | qty('qty') + day_units('units') + (from_ | before | after)('dir')
+                            + absolute_day_reference('ref_day')
+                          )
+
+def compute_relative_date(t):
+    now = datetime.now().replace(microsecond=0)
+    if 'ref_day' in t:
+        t['computed_date'] = t.ref_day
+    else:
+        t['computed_date'] = now.date()
+    day_diff = t.dir * t.qty[0] * {'week': 7, 'day': 1}[t.units]
+    t['date_delta'] = timedelta(days=day_diff)
+relative_day_reference.addParseAction(compute_relative_date)
+
+# combine expressions for absolute and relative day references
+day_reference = relative_day_reference | absolute_day_reference
+def add_default_date_fields(t):
+    if 'date_delta' not in t:
+        t['date_delta'] = timedelta()
+day_reference.addParseAction(add_default_date_fields)
+
+# combine date and time expressions into single overall parser
+time_and_day = (time_reference + time_ref_present + pp.Optional(pp.Optional(on_) + day_reference)
+               | day_reference + pp.Optional(at_ + absolute_time_of_day + time_ref_present)
+                )
+
+# parse actions for total time_and_day expression
+def save_original_string(s, l, t):
+    # save original input string and reference time
+    t['original'] = ' '.join(s.strip().split())
+    t['relative_to'] = datetime.now().replace(microsecond=0)
+
+def compute_timestamp(t):
+    # accumulate values from parsed time and day subexpressions - fill in defaults for omitted parts
+    now = datetime.now().replace(microsecond=0)
+    if 'computed_time' not in t:
+        t['computed_time'] = t.ref_time or now.time()
+    if 'abs_date' not in t:
+        t['abs_date'] = now
+
+    # roll up all fields and apply any time or day deltas
+    t['computed_dt'] = (
+        t.abs_date.replace(hour=t.computed_time.hour, minute=t.computed_time.minute, second=t.computed_time.second)
+        + (t.time_delta or timedelta(0))
+        + (t.date_delta or timedelta(0))
+    )
+
+    # if time just given in terms of day expressions, zero out time fields
+    if not t.time_ref_present:
+        t['computed_dt'] = t.computed_dt.replace(hour=0, minute=0, second=0)
+
+    # add time_offset fields
+    t['time_offset'] = t.computed_dt - t.relative_to
+
+def remove_temp_keys(t):
+    # strip out keys that are just used internally
+    all_keys = list(t.keys())
+    for k in all_keys:
+        if k not in ('computed_dt', 'original', 'relative_to', 'time_offset'):
+            del t[k]
+
+time_and_day.addParseAction(save_original_string, compute_timestamp, remove_temp_keys)
+
+
+time_expression = time_and_day
+
+
+if __name__ == "__main__":
+    # test grammar
+    tests = """\
+        today
+        tomorrow
+        yesterday
+        the day before yesterday
+        the day after tomorrow
+        2 weeks after today
+        in a couple of days
+        a couple of days from now
+        a couple of days from today
+        in a day
+        3 days ago
+        3 days from now
+        a day ago
+        in 2 weeks
+        in 3 days at 5pm
+        now
+        10 minutes ago
+        10 minutes from now
+        in 10 minutes
+        in a minute
+        in a couple of minutes
+        20 seconds ago
+        in 30 seconds
+        20 seconds before noon
+        ten seconds before noon tomorrow
+        noon
+        midnight
+        noon tomorrow
+        6am tomorrow
+        0800 yesterday
+        12:15 AM today
+        3pm 2 days from today
+        a week from today
+        a week from now
+        three weeks ago
+        noon next Sunday
+        noon Sunday
+        noon last Sunday
+        2pm next Sunday
+        next Sunday at 2pm
+        last Sunday at 2pm
+    """
+
+    expected = {
+        'now' : timedelta(0),
+        '10 minutes ago': timedelta(minutes=-10),
+        '10 minutes from now': timedelta(minutes=10),
+        'in 10 minutes': timedelta(minutes=10),
+        'in a minute': timedelta(minutes=1),
+        'in a couple of minutes': timedelta(minutes=2),
+        '20 seconds ago': timedelta(seconds=-20),
+        'in 30 seconds': timedelta(seconds=30),
+        'a week from now': timedelta(days=7),
+    }
+    def verify_offset(instring, parsed):
+        if instring in expected:
+            if parsed.time_offset == expected[instring]:
+                parsed['verify_offset'] = 'PASS'
+            else:
+                parsed['verify_offset'] = 'FAIL'
+
+    print("(relative to %s)" % datetime.now())
+    time_expression.runTests(tests, postParse=verify_offset)
diff --git a/examples/nested_markup.py b/examples/nested_markup.py
new file mode 100644
index 0000000..40267e6
--- /dev/null
+++ b/examples/nested_markup.py
@@ -0,0 +1,48 @@
+#
+# nested_markup.py
+#
+# Example markup parser to recursively transform nested markup directives.
+#
+# Copyright 2019, Paul McGuire
+#
+import pyparsing as pp
+
+wiki_markup = pp.Forward()
+
+# a method that will construct and return a parse action that will
+# do the proper wrapping in opening and closing HTML, and recursively call
+# wiki_markup.transformString on the markup body text
+def convert_markup_to_html(opening,closing):
+    def conversionParseAction(s, l, t):
+        return opening + wiki_markup.transformString(t[1][1:-1]) + closing
+    return conversionParseAction
+
+# use a nestedExpr with originalTextFor to parse nested braces, but return the
+# parsed text as a single string containing the outermost nested braces instead
+# of a nested list of parsed tokens
+markup_body = pp.originalTextFor(pp.nestedExpr('{', '}'))
+italicized = ('ital' + markup_body).setParseAction(convert_markup_to_html("<I>", "</I>"))
+bolded = ('bold' + markup_body).setParseAction(convert_markup_to_html("<B>", "</B>"))
+
+# another markup and parse action to parse links - again using transform string
+# to recursively parse any markup in the link text
+def convert_link_to_html(s, l, t):
+    t['link_text'] = wiki_markup.transformString(t['link_text'])
+    return '<A href="{url}">{link_text}</A>'.format_map(t)
+urlRef = ('link'
+          + '{' + pp.SkipTo('->')('link_text') + '->' + pp.SkipTo('}')('url') + '}'
+          ).setParseAction(convert_link_to_html)
+
+# now inject all the markup bits as possible markup expressions
+wiki_markup <<= urlRef | italicized | bolded
+
+# try it out!
+wiki_input = """
+Here is a simple Wiki input:
+
+  ital{This is in italics}.
+  bold{This is in bold}!
+  bold{This is in ital{bold italics}! But this is just bold.}
+  Here's a URL to link{Pyparsing's bold{Wiki Page}!->https://github.com/pyparsing/pyparsing/wiki}
+"""
+print(wiki_markup.transformString(wiki_input))