From 8835f465fa94f114dcf865429c0410821d365dae Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 21 Mar 2020 01:42:50 -0400 Subject: bpo-38945: UU Encoding: Don't let newline in filename corrupt the output format (GH-17418) (GH-17444) (#17445) (cherry picked from commit a62ad4730c9b575f140f24074656c0257c86a09a) Co-authored-by: Matthew Rollings <1211162+stealthcopter@users.noreply.github.com> --- Lib/encodings/uu_codec.py | 4 ++++ Lib/test/test_uu.py | 9 +++++++++ Lib/uu.py | 7 +++++++ .../next/Security/2019-12-01-22-44-40.bpo-38945.ztmNXc.rst | 1 + 4 files changed, 21 insertions(+) create mode 100644 Misc/NEWS.d/next/Security/2019-12-01-22-44-40.bpo-38945.ztmNXc.rst diff --git a/Lib/encodings/uu_codec.py b/Lib/encodings/uu_codec.py index 2a5728fb5b..4e58c62fe9 100644 --- a/Lib/encodings/uu_codec.py +++ b/Lib/encodings/uu_codec.py @@ -20,6 +20,10 @@ def uu_encode(input, errors='strict', filename='', mode=0o666): read = infile.read write = outfile.write + # Remove newline chars from filename + filename = filename.replace('\n','\\n') + filename = filename.replace('\r','\\r') + # Encode write(('begin %o %s\n' % (mode & 0o777, filename)).encode('ascii')) chunk = read(45) diff --git a/Lib/test/test_uu.py b/Lib/test/test_uu.py index 25fffbf993..4ba0ed8d0f 100644 --- a/Lib/test/test_uu.py +++ b/Lib/test/test_uu.py @@ -115,6 +115,15 @@ class UUTest(unittest.TestCase): decoded = codecs.decode(encodedtext, "uu_codec") self.assertEqual(decoded, plaintext) + def test_newlines_escaped(self): + # Test newlines are escaped with uu.encode + inp = io.BytesIO(plaintext) + out = io.BytesIO() + filename = "test.txt\n\roverflow.txt" + safefilename = b"test.txt\\n\\roverflow.txt" + uu.encode(inp, out, filename) + self.assertIn(safefilename, out.getvalue()) + class UUStdIOTest(unittest.TestCase): def setUp(self): diff --git a/Lib/uu.py b/Lib/uu.py index d68d29374a..3a8c31cff0 100755 --- a/Lib/uu.py +++ b/Lib/uu.py @@ -73,6 +73,13 @@ def encode(in_file, out_file, name=None, mode=None): name = '-' if mode is None: mode = 0o666 + + # + # Remove newline chars from name + # + name = name.replace('\n','\\n') + name = name.replace('\r','\\r') + # # Write the data # diff --git a/Misc/NEWS.d/next/Security/2019-12-01-22-44-40.bpo-38945.ztmNXc.rst b/Misc/NEWS.d/next/Security/2019-12-01-22-44-40.bpo-38945.ztmNXc.rst new file mode 100644 index 0000000000..1bf6ed567b --- /dev/null +++ b/Misc/NEWS.d/next/Security/2019-12-01-22-44-40.bpo-38945.ztmNXc.rst @@ -0,0 +1 @@ +Newline characters have been escaped when performing uu encoding to prevent them from overflowing into to content section of the encoded file. This prevents malicious or accidental modification of data during the decoding process. \ No newline at end of file -- cgit v1.2.1 From ed07522a5faa3101f68be8e4b8369310f60860f8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 3 Apr 2020 01:09:58 +0200 Subject: bpo-40156: Copy Codecov configuration from master (#19309) Disable "Codevov patch" job on pull requests. --- .github/codecov.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/codecov.yml b/.github/codecov.yml index dc21321d0b..ea504f4867 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -5,7 +5,7 @@ codecov: comment: off ignore: - "Doc/**/*" - - "Misc/*" + - "Misc/**/*" - "Mac/**/*" - "PC/**/*" - "PCbuild/**/*" @@ -13,18 +13,12 @@ ignore: - "Grammar/*" coverage: precision: 2 - range: - - 70.0 - - 100.0 + range: 70...90 round: down status: changes: off project: off - patch: - default: - target: 100% - only_pulls: true - threshold: 0.05 + patch: off parsers: gcov: branch_detection: -- cgit v1.2.1 From 55a6a16a46239a71b635584e532feb8b17ae7fdf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 3 Apr 2020 03:37:32 +0200 Subject: bpo-38804: Fix REDoS in http.cookiejar (GH-17157) (#17344) The regex http.cookiejar.LOOSE_HTTP_DATE_RE was vulnerable to regular expression denial of service (REDoS). LOOSE_HTTP_DATE_RE.match is called when using http.cookiejar.CookieJar to parse Set-Cookie headers returned by a server. Processing a response from a malicious HTTP server can lead to extreme CPU usage and execution will be blocked for a long time. The regex contained multiple overlapping \s* capture groups. Ignoring the ?-optional capture groups the regex could be simplified to \d+-\w+-\d+(\s*\s*\s*)$ Therefore, a long sequence of spaces can trigger bad performance. Matching a malicious string such as LOOSE_HTTP_DATE_RE.match("1-c-1" + (" " * 2000) + "!") caused catastrophic backtracking. The fix removes ambiguity about which \s* should match a particular space. You can create a malicious server which responds with Set-Cookie headers to attack all python programs which access it e.g. from http.server import BaseHTTPRequestHandler, HTTPServer def make_set_cookie_value(n_spaces): spaces = " " * n_spaces expiry = f"1-c-1{spaces}!" return f"b;Expires={expiry}" class Handler(BaseHTTPRequestHandler): def do_GET(self): self.log_request(204) self.send_response_only(204) # Don't bother sending Server and Date n_spaces = ( int(self.path[1:]) # Can GET e.g. /100 to test shorter sequences if len(self.path) > 1 else 65506 # Max header line length 65536 ) value = make_set_cookie_value(n_spaces) for i in range(99): # Not necessary, but we can have up to 100 header lines self.send_header("Set-Cookie", value) self.end_headers() if __name__ == "__main__": HTTPServer(("", 44020), Handler).serve_forever() This server returns 99 Set-Cookie headers. Each has 65506 spaces. Extracting the cookies will pretty much never complete. Vulnerable client using the example at the bottom of https://docs.python.org/3/library/http.cookiejar.html : import http.cookiejar, urllib.request cj = http.cookiejar.CookieJar() opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj)) r = opener.open("http://localhost:44020/") The popular requests library was also vulnerable without any additional options (as it uses http.cookiejar by default): import requests requests.get("http://localhost:44020/") * Regression test for http.cookiejar REDoS If we regress, this test will take a very long time. * Improve performance of http.cookiejar.ISO_DATE_RE A string like "444444" + (" " * 2000) + "A" could cause poor performance due to the 2 overlapping \s* groups, although this is not as serious as the REDoS in LOOSE_HTTP_DATE_RE was. (cherry picked from commit 1b779bfb8593739b11cbb988ef82a883ec9d077e) Co-authored-by: bcaller --- Lib/http/cookiejar.py | 18 ++++++++++++------ Lib/test/test_http_cookiejar.py | 13 +++++++++++++ Misc/ACKS | 1 + .../Security/2019-11-15-00-54-42.bpo-38804.vjbM8V.rst | 1 + 4 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2019-11-15-00-54-42.bpo-38804.vjbM8V.rst diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index c6b9d8c011..afed5bc93c 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -216,10 +216,14 @@ LOOSE_HTTP_DATE_RE = re.compile( (?::(\d\d))? # optional seconds )? # optional clock \s* - ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone + (?: + ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+) # timezone + \s* + )? + (?: + \(\w+\) # ASCII representation of timezone in parens. \s* - (?:\(\w+\))? # ASCII representation of timezone in parens. - \s*$""", re.X | re.ASCII) + )?$""", re.X | re.ASCII) def http2time(text): """Returns time in seconds since epoch of time represented by a string. @@ -289,9 +293,11 @@ ISO_DATE_RE = re.compile( (?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional) )? # optional clock \s* - ([-+]?\d\d?:?(:?\d\d)? - |Z|z)? # timezone (Z is "zero meridian", i.e. GMT) - \s*$""", re.X | re. ASCII) + (?: + ([-+]?\d\d?:?(:?\d\d)? + |Z|z) # timezone (Z is "zero meridian", i.e. GMT) + \s* + )?$""", re.X | re. ASCII) def iso2time(text): """ As for http2time, but parses the ISO 8601 formats: diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index 767b0fd137..218edeb9e8 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -122,6 +122,13 @@ class DateTimeTests(unittest.TestCase): "http2time(%s) is not None\n" "http2time(test) %s" % (test, http2time(test))) + def test_http2time_redos_regression_actually_completes(self): + # LOOSE_HTTP_DATE_RE was vulnerable to malicious input which caused catastrophic backtracking (REDoS). + # If we regress to cubic complexity, this test will take a very long time to succeed. + # If fixed, it should complete within a fraction of a second. + http2time("01 Jan 1970{}00:00:00 GMT!".format(" " * 10 ** 5)) + http2time("01 Jan 1970 00:00:00{}GMT!".format(" " * 10 ** 5)) + def test_iso2time(self): def parse_date(text): return time.gmtime(iso2time(text))[:6] @@ -181,6 +188,12 @@ class DateTimeTests(unittest.TestCase): "iso2time(%s) is not None\n" "iso2time(test) %s" % (test, iso2time(test))) + def test_iso2time_performance_regression(self): + # If ISO_DATE_RE regresses to quadratic complexity, this test will take a very long time to succeed. + # If fixed, it should complete within a fraction of a second. + iso2time('1994-02-03{}14:15:29 -0100!'.format(' '*10**6)) + iso2time('1994-02-03 14:15:29{}-0100!'.format(' '*10**6)) + class HeaderTests(unittest.TestCase): diff --git a/Misc/ACKS b/Misc/ACKS index 72c5d740bd..778dce1b0a 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -227,6 +227,7 @@ Zach Byrne Vedran Čačić Nicolas Cadou Jp Calderone +Ben Caller Arnaud Calmettes Daniel Calvelo Tony Campbell diff --git a/Misc/NEWS.d/next/Security/2019-11-15-00-54-42.bpo-38804.vjbM8V.rst b/Misc/NEWS.d/next/Security/2019-11-15-00-54-42.bpo-38804.vjbM8V.rst new file mode 100644 index 0000000000..1f45142d9f --- /dev/null +++ b/Misc/NEWS.d/next/Security/2019-11-15-00-54-42.bpo-38804.vjbM8V.rst @@ -0,0 +1 @@ +Fixes a ReDoS vulnerability in :mod:`http.cookiejar`. Patch by Ben Caller. -- cgit v1.2.1