From f6d5c35f43c4172348da4efaabf83395ce0b7853 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 18 May 2020 09:43:32 -0400 Subject: GitHub release automation --- ci/github_releases.py | 142 ++++++++++++++++++++++++++++++++++++++++++++++++ ci/parse_relnotes.py | 115 +++++++++++++++++++++++++++++++++++++++ ci/tidelift_relnotes.py | 50 +++++++++++++++++ ci/upload_relnotes.py | 122 ----------------------------------------- 4 files changed, 307 insertions(+), 122 deletions(-) create mode 100644 ci/github_releases.py create mode 100644 ci/parse_relnotes.py create mode 100644 ci/tidelift_relnotes.py delete mode 100644 ci/upload_relnotes.py (limited to 'ci') diff --git a/ci/github_releases.py b/ci/github_releases.py new file mode 100644 index 00000000..408a2ab8 --- /dev/null +++ b/ci/github_releases.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +""" +Upload release notes into GitHub releases. +""" + +import json +import shlex +import subprocess +import sys + +import pkg_resources +import requests + + +RELEASES_URL = "https://api.github.com/repos/{repo}/releases" + +def run_command(cmd): + """ + Run a command line (with no shell). + + Returns a tuple: + bool: true if the command succeeded. + str: the output of the command. + + """ + proc = subprocess.run( + shlex.split(cmd), + shell=False, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + output = proc.stdout.decode("utf-8") + succeeded = proc.returncode == 0 + return succeeded, output + +def does_tag_exist(tag_name): + """ + Does `tag_name` exist as a tag in git? + """ + return run_command(f"git rev-parse --verify {tag_name}")[0] + +def check_ok(resp): + """ + Check that the Requests response object was successful. + + Raise an exception if not. + """ + if not resp: + print(f"text: {resp.text!r}") + resp.raise_for_status() + +def github_paginated(session, url): + """ + Get all the results from a paginated GitHub url. + """ + while True: + print(f"GETTING: {url}") + resp = session.get(url) + check_ok(resp) + yield from resp.json() + if 'Link' not in resp.headers: + break + links = resp.headers['link'].split(",") + next_link = next((link for link in links if 'rel="next"' in link), None) + if not next_link: + break + url = next_link.split(";")[0].strip(" <>") + +def get_releases(session, repo): + """ + Get all the releases from a name/project repo. + + Returns: + A dict mapping tag names to release dictionaries. + """ + url = RELEASES_URL.format(repo=repo) + "?per_page=100" + releases = { r['tag_name']: r for r in github_paginated(session, url) } + return releases + +def release_for_relnote(relnote): + """ + Turn a release note dict into the data needed by GitHub for a release. + """ + tag = f"coverage-{relnote['version']}" + return { + "tag_name": tag, + "name": tag, + "body": relnote["text"], + "draft": False, + "prerelease": relnote["prerelease"], + } + +def create_release(session, repo, relnote): + """ + Create a new GitHub release. + """ + print(f"Creating {relnote['version']}") + data = release_for_relnote(relnote) + resp = session.post(RELEASES_URL.format(repo=repo), json=data) + check_ok(resp) + +def update_release(session, url, relnote): + """ + Update an existing GitHub release. + """ + print(f"Updating {relnote['version']}") + data = release_for_relnote(relnote) + resp = session.patch(url, json=data) + check_ok(resp) + +def update_github_releases(json_filename, repo): + """ + Read the json file, and create or update releases in GitHub. + """ + gh_session = requests.Session() + releases = get_releases(gh_session, repo) + if 0: # if you need to delete all the releases! + for release in releases.values(): + print(release["tag_name"]) + resp = gh_session.delete(release["url"]) + check_ok(resp) + return + + with open(json_filename) as jf: + relnotes = json.load(jf) + relnotes.sort(key=lambda rel: pkg_resources.parse_version(rel["version"])) + for relnote in relnotes: + tag = "coverage-" + relnote["version"] + if not does_tag_exist(tag): + continue + exists = tag in releases + if not exists: + create_release(gh_session, repo, relnote) + else: + release = releases[tag] + if release["body"] != relnote["text"]: + url = release["url"] + update_release(gh_session, url, relnote) + +if __name__ == "__main__": + update_github_releases(*sys.argv[1:]) # pylint: disable=no-value-for-parameter diff --git a/ci/parse_relnotes.py b/ci/parse_relnotes.py new file mode 100644 index 00000000..d19e6d60 --- /dev/null +++ b/ci/parse_relnotes.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +""" +Parse CHANGES.md into a JSON structure. + +Run with two arguments: the .md file to parse, and the JSON file to write: + + python parse_relnotes.py CHANGES.md relnotes.json + +Every section that has something that looks like a version number in it will +be recorded as the release notes for that version. + +""" + +import json +import re +import sys + + +class TextChunkBuffer: + """Hold onto text chunks until needed.""" + def __init__(self): + self.buffer = [] + + def append(self, text): + """Add `text` to the buffer.""" + self.buffer.append(text) + + def clear(self): + """Clear the buffer.""" + self.buffer = [] + + def flush(self): + """Produce a ("text", text) tuple if there's anything here.""" + buffered = "".join(self.buffer).strip() + if buffered: + yield ("text", buffered) + self.clear() + + +def parse_md(lines): + """Parse markdown lines, producing (type, text) chunks.""" + buffer = TextChunkBuffer() + + for line in lines: + header_match = re.search(r"^(#+) (.+)$", line) + is_header = bool(header_match) + if is_header: + yield from buffer.flush() + hashes, text = header_match.groups() + yield (f"h{len(hashes)}", text) + else: + buffer.append(line) + + yield from buffer.flush() + + +def sections(parsed_data): + """Convert a stream of parsed tokens into sections with text and notes. + + Yields a stream of: + ('h-level', 'header text', 'text') + + """ + header = None + text = [] + for ttype, ttext in parsed_data: + if ttype.startswith('h'): + if header: + yield (*header, "\n".join(text)) + text = [] + header = (ttype, ttext) + elif ttype == "text": + text.append(ttext) + else: + raise Exception(f"Don't know ttype {ttype!r}") + yield (*header, "\n".join(text)) + + +def refind(regex, text): + """Find a regex in some text, and return the matched text, or None.""" + m = re.search(regex, text) + if m: + return m.group() + else: + return None + +def relnotes(mdlines): + r"""Yield (version, text) pairs from markdown lines. + + Each tuple is a separate version mentioned in the release notes. + + A version is any section with \d\.\d in the header text. + + """ + for _, htext, text in sections(parse_md(mdlines)): + version = refind(r"\d+\.\d[^ ]*", htext) + if version: + prerelease = any(c in version for c in "abc") + when = refind(r"\d+-\d+-\d+", htext) + yield { + "version": version, + "text": text, + "prerelease": prerelease, + "when": when, + } + +def parse(md_filename, json_filename): + """Main function: parse markdown and write JSON.""" + with open(md_filename) as mf: + markdown = mf.read() + with open(json_filename, "w") as jf: + json.dump(list(relnotes(markdown.splitlines(True))), jf, indent=4) + +if __name__ == "__main__": + parse(*sys.argv[1:]) # pylint: disable=no-value-for-parameter diff --git a/ci/tidelift_relnotes.py b/ci/tidelift_relnotes.py new file mode 100644 index 00000000..bc3a37d4 --- /dev/null +++ b/ci/tidelift_relnotes.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +""" +Upload release notes from a JSON file to Tidelift as Markdown chunks + +Put your Tidelift API token in a file called tidelift.token alongside this +program, for example: + + user/n3IwOpxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxc2ZwE4 + +Run with two arguments: the JSON file of release notes, and the Tidelift +package name: + + python tidelift_relnotes.py relnotes.json pypi/coverage + +Every section that has something that looks like a version number in it will +be uploaded as the release notes for that version. + +""" + +import json +import os.path +import sys + +import requests + + +def update_release_note(package, version, text): + """Update the release notes for one version of a package.""" + url = f"https://api.tidelift.com/external-api/lifting/{package}/release-notes/{version}" + token_file = os.path.join(os.path.dirname(__file__), "tidelift.token") + with open(token_file) as ftoken: + token = ftoken.read().strip() + headers = { + "Authorization": f"Bearer: {token}", + } + req_args = dict(url=url, data=text.encode('utf8'), headers=headers) + result = requests.post(**req_args) + if result.status_code == 409: + result = requests.put(**req_args) + print(f"{version}: {result.status_code}") + +def upload(json_filename, package): + """Main function: parse markdown and upload to Tidelift.""" + with open(json_filename) as jf: + relnotes = json.load(jf) + for relnote in relnotes: + update_release_note(package, relnote["version"], relnote["text"]) + +if __name__ == "__main__": + upload(*sys.argv[1:]) # pylint: disable=no-value-for-parameter diff --git a/ci/upload_relnotes.py b/ci/upload_relnotes.py deleted file mode 100644 index 630f4d0a..00000000 --- a/ci/upload_relnotes.py +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env python3 -""" -Upload CHANGES.md to Tidelift as Markdown chunks - -Put your Tidelift API token in a file called tidelift.token alongside this -program, for example: - - user/n3IwOpxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxc2ZwE4 - -Run with two arguments: the .md file to parse, and the Tidelift package name: - - python upload_relnotes.py CHANGES.md pypi/coverage - -Every section that has something that looks like a version number in it will -be uploaded as the release notes for that version. - -""" - -import os.path -import re -import sys - -import requests - -class TextChunkBuffer: - """Hold onto text chunks until needed.""" - def __init__(self): - self.buffer = [] - - def append(self, text): - """Add `text` to the buffer.""" - self.buffer.append(text) - - def clear(self): - """Clear the buffer.""" - self.buffer = [] - - def flush(self): - """Produce a ("text", text) tuple if there's anything here.""" - buffered = "".join(self.buffer).strip() - if buffered: - yield ("text", buffered) - self.clear() - - -def parse_md(lines): - """Parse markdown lines, producing (type, text) chunks.""" - buffer = TextChunkBuffer() - - for line in lines: - header_match = re.search(r"^(#+) (.+)$", line) - is_header = bool(header_match) - if is_header: - yield from buffer.flush() - hashes, text = header_match.groups() - yield (f"h{len(hashes)}", text) - else: - buffer.append(line) - - yield from buffer.flush() - - -def sections(parsed_data): - """Convert a stream of parsed tokens into sections with text and notes. - - Yields a stream of: - ('h-level', 'header text', 'text') - - """ - header = None - text = [] - for ttype, ttext in parsed_data: - if ttype.startswith('h'): - if header: - yield (*header, "\n".join(text)) - text = [] - header = (ttype, ttext) - elif ttype == "text": - text.append(ttext) - else: - raise Exception(f"Don't know ttype {ttype!r}") - yield (*header, "\n".join(text)) - - -def relnotes(mdlines): - r"""Yield (version, text) pairs from markdown lines. - - Each tuple is a separate version mentioned in the release notes. - - A version is any section with \d\.\d in the header text. - - """ - for _, htext, text in sections(parse_md(mdlines)): - m_version = re.search(r"\d+\.\d[^ ]*", htext) - if m_version: - version = m_version.group() - yield version, text - -def update_release_note(package, version, text): - """Update the release notes for one version of a package.""" - url = f"https://api.tidelift.com/external-api/lifting/{package}/release-notes/{version}" - token_file = os.path.join(os.path.dirname(__file__), "tidelift.token") - with open(token_file) as ftoken: - token = ftoken.read().strip() - headers = { - "Authorization": f"Bearer: {token}", - } - req_args = dict(url=url, data=text.encode('utf8'), headers=headers) - result = requests.post(**req_args) - if result.status_code == 409: - result = requests.put(**req_args) - print(f"{version}: {result.status_code}") - -def parse_and_upload(md_filename, package): - """Main function: parse markdown and upload to Tidelift.""" - with open(md_filename) as f: - markdown = f.read() - for version, text in relnotes(markdown.splitlines(True)): - update_release_note(package, version, text) - -if __name__ == "__main__": - parse_and_upload(*sys.argv[1:]) # pylint: disable=no-value-for-parameter -- cgit v1.2.1