GitHub release automation

3 files changed, 223 insertions, 38 deletions

diff --git a/ci/github_releases.py b/ci/github_releases.py
new file mode 100644
index 00000000..408a2ab8
--- /dev/null
+++ b/ci/github_releases.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+"""
+Upload release notes into GitHub releases.
+"""
+
+import json
+import shlex
+import subprocess
+import sys
+
+import pkg_resources
+import requests
+
+
+RELEASES_URL = "https://api.github.com/repos/{repo}/releases"
+
+def run_command(cmd):
+    """
+    Run a command line (with no shell).
+
+    Returns a tuple:
+        bool: true if the command succeeded.
+        str: the output of the command.
+
+    """
+    proc = subprocess.run(
+        shlex.split(cmd),
+        shell=False,
+        check=False,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+    output = proc.stdout.decode("utf-8")
+    succeeded = proc.returncode == 0
+    return succeeded, output
+
+def does_tag_exist(tag_name):
+    """
+    Does `tag_name` exist as a tag in git?
+    """
+    return run_command(f"git rev-parse --verify {tag_name}")[0]
+
+def check_ok(resp):
+    """
+    Check that the Requests response object was successful.
+
+    Raise an exception if not.
+    """
+    if not resp:
+        print(f"text: {resp.text!r}")
+        resp.raise_for_status()
+
+def github_paginated(session, url):
+    """
+    Get all the results from a paginated GitHub url.
+    """
+    while True:
+        print(f"GETTING: {url}")
+        resp = session.get(url)
+        check_ok(resp)
+        yield from resp.json()
+        if 'Link' not in resp.headers:
+            break
+        links = resp.headers['link'].split(",")
+        next_link = next((link for link in links if 'rel="next"' in link), None)
+        if not next_link:
+            break
+        url = next_link.split(";")[0].strip(" <>")
+
+def get_releases(session, repo):
+    """
+    Get all the releases from a name/project repo.
+
+    Returns:
+        A dict mapping tag names to release dictionaries.
+    """
+    url = RELEASES_URL.format(repo=repo) + "?per_page=100"
+    releases = { r['tag_name']: r for r in github_paginated(session, url) }
+    return releases
+
+def release_for_relnote(relnote):
+    """
+    Turn a release note dict into the data needed by GitHub for a release.
+    """
+    tag = f"coverage-{relnote['version']}"
+    return {
+        "tag_name": tag,
+        "name": tag,
+        "body": relnote["text"],
+        "draft": False,
+        "prerelease": relnote["prerelease"],
+    }
+
+def create_release(session, repo, relnote):
+    """
+    Create a new GitHub release.
+    """
+    print(f"Creating {relnote['version']}")
+    data = release_for_relnote(relnote)
+    resp = session.post(RELEASES_URL.format(repo=repo), json=data)
+    check_ok(resp)
+
+def update_release(session, url, relnote):
+    """
+    Update an existing GitHub release.
+    """
+    print(f"Updating {relnote['version']}")
+    data = release_for_relnote(relnote)
+    resp = session.patch(url, json=data)
+    check_ok(resp)
+
+def update_github_releases(json_filename, repo):
+    """
+    Read the json file, and create or update releases in GitHub.
+    """
+    gh_session = requests.Session()
+    releases = get_releases(gh_session, repo)
+    if 0:   # if you need to delete all the releases!
+        for release in releases.values():
+            print(release["tag_name"])
+            resp = gh_session.delete(release["url"])
+            check_ok(resp)
+        return
+
+    with open(json_filename) as jf:
+        relnotes = json.load(jf)
+    relnotes.sort(key=lambda rel: pkg_resources.parse_version(rel["version"]))
+    for relnote in relnotes:
+        tag = "coverage-" + relnote["version"]
+        if not does_tag_exist(tag):
+            continue
+        exists = tag in releases
+        if not exists:
+            create_release(gh_session, repo, relnote)
+        else:
+            release = releases[tag]
+            if release["body"] != relnote["text"]:
+                url = release["url"]
+                update_release(gh_session, url, relnote)
+
+if __name__ == "__main__":
+    update_github_releases(*sys.argv[1:])   # pylint: disable=no-value-for-parameter
diff --git a/ci/upload_relnotes.py b/ci/parse_relnotes.py
index 630f4d0a..d19e6d60 100644
--- a/ci/upload_relnotes.py
+++ b/ci/parse_relnotes.py
@@ -1,26 +1,20 @@
 #!/usr/bin/env python3
 """
-Upload CHANGES.md to Tidelift as Markdown chunks
+Parse CHANGES.md into a JSON structure.
 
-Put your Tidelift API token in a file called tidelift.token alongside this
-program, for example:
+Run with two arguments: the .md file to parse, and the JSON file to write:
 
-    user/n3IwOpxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxc2ZwE4
-
-Run with two arguments: the .md file to parse, and the Tidelift package name:
-
-	python upload_relnotes.py CHANGES.md pypi/coverage
+	python parse_relnotes.py CHANGES.md relnotes.json
 
 Every section that has something that looks like a version number in it will
-be uploaded as the release notes for that version.
+be recorded as the release notes for that version.
 
 """
 
-import os.path
+import json
 import re
 import sys
 
-import requests
 
 class TextChunkBuffer:
     """Hold onto text chunks until needed."""
@@ -82,6 +76,14 @@ def sections(parsed_data):
     yield (*header, "\n".join(text))
 
 
+def refind(regex, text):
+    """Find a regex in some text, and return the matched text, or None."""
+    m = re.search(regex, text)
+    if m:
+        return m.group()
+    else:
+        return None
+
 def relnotes(mdlines):
     r"""Yield (version, text) pairs from markdown lines.
 
@@ -91,32 +93,23 @@ def relnotes(mdlines):
 
     """
     for _, htext, text in sections(parse_md(mdlines)):
-        m_version = re.search(r"\d+\.\d[^ ]*", htext)
-        if m_version:
-            version = m_version.group()
-            yield version, text
-
-def update_release_note(package, version, text):
-    """Update the release notes for one version of a package."""
-    url = f"https://api.tidelift.com/external-api/lifting/{package}/release-notes/{version}"
-    token_file = os.path.join(os.path.dirname(__file__), "tidelift.token")
-    with open(token_file) as ftoken:
-        token = ftoken.read().strip()
-    headers = {
-        "Authorization": f"Bearer: {token}",
-    }
-    req_args = dict(url=url, data=text.encode('utf8'), headers=headers)
-    result = requests.post(**req_args)
-    if result.status_code == 409:
-        result = requests.put(**req_args)
-    print(f"{version}: {result.status_code}")
-
-def parse_and_upload(md_filename, package):
-    """Main function: parse markdown and upload to Tidelift."""
-    with open(md_filename) as f:
-        markdown = f.read()
-    for version, text in relnotes(markdown.splitlines(True)):
-        update_release_note(package, version, text)
+        version = refind(r"\d+\.\d[^ ]*", htext)
+        if version:
+            prerelease = any(c in version for c in "abc")
+            when = refind(r"\d+-\d+-\d+", htext)
+            yield {
+                "version": version,
+                "text": text,
+                "prerelease": prerelease,
+                "when": when,
+            }
+
+def parse(md_filename, json_filename):
+    """Main function: parse markdown and write JSON."""
+    with open(md_filename) as mf:
+        markdown = mf.read()
+    with open(json_filename, "w") as jf:
+        json.dump(list(relnotes(markdown.splitlines(True))), jf, indent=4)
 
 if __name__ == "__main__":
-    parse_and_upload(*sys.argv[1:])       # pylint: disable=no-value-for-parameter
+    parse(*sys.argv[1:])       # pylint: disable=no-value-for-parameter
diff --git a/ci/tidelift_relnotes.py b/ci/tidelift_relnotes.py
new file mode 100644
index 00000000..bc3a37d4
--- /dev/null
+++ b/ci/tidelift_relnotes.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+"""
+Upload release notes from a JSON file to Tidelift as Markdown chunks
+
+Put your Tidelift API token in a file called tidelift.token alongside this
+program, for example:
+
+    user/n3IwOpxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxc2ZwE4
+
+Run with two arguments: the JSON file of release notes, and the Tidelift
+package name:
+
+	python tidelift_relnotes.py relnotes.json pypi/coverage
+
+Every section that has something that looks like a version number in it will
+be uploaded as the release notes for that version.
+
+"""
+
+import json
+import os.path
+import sys
+
+import requests
+
+
+def update_release_note(package, version, text):
+    """Update the release notes for one version of a package."""
+    url = f"https://api.tidelift.com/external-api/lifting/{package}/release-notes/{version}"
+    token_file = os.path.join(os.path.dirname(__file__), "tidelift.token")
+    with open(token_file) as ftoken:
+        token = ftoken.read().strip()
+    headers = {
+        "Authorization": f"Bearer: {token}",
+    }
+    req_args = dict(url=url, data=text.encode('utf8'), headers=headers)
+    result = requests.post(**req_args)
+    if result.status_code == 409:
+        result = requests.put(**req_args)
+    print(f"{version}: {result.status_code}")
+
+def upload(json_filename, package):
+    """Main function: parse markdown and upload to Tidelift."""
+    with open(json_filename) as jf:
+        relnotes = json.load(jf)
+    for relnote in relnotes:
+        update_release_note(package, relnote["version"], relnote["text"])
+
+if __name__ == "__main__":
+    upload(*sys.argv[1:])           # pylint: disable=no-value-for-parameter