diff options
| -rw-r--r-- | Makefile | 24 | ||||
| -rw-r--r-- | ci/github_releases.py | 142 | ||||
| -rw-r--r-- | ci/parse_relnotes.py (renamed from ci/upload_relnotes.py) | 69 | ||||
| -rw-r--r-- | ci/tidelift_relnotes.py | 50 | ||||
| -rw-r--r-- | howto.txt | 4 |
5 files changed, 244 insertions, 45 deletions
@@ -133,17 +133,17 @@ WEBHOME = ~/web/stellated/ WEBSAMPLE = $(WEBHOME)/files/sample_coverage_html WEBSAMPLEBETA = $(WEBHOME)/files/sample_coverage_html_beta -docreqs: +$(DOCPYTHON): tox -q -e doc --notest -dochtml: docreqs ## Build the docs HTML output. +dochtml: $(DOCBIN) ## Build the docs HTML output. $(DOCBIN)/python doc/check_copied_from.py doc/*.rst $(SPHINXBUILD) -b html doc doc/_build/html docdev: dochtml ## Build docs, and auto-watch for changes. PATH=$(DOCBIN):$(PATH) $(SPHINXAUTOBUILD) -b html doc doc/_build/html -docspell: docreqs +docspell: $(DOCBIN) $(SPHINXBUILD) -b spelling doc doc/_spell publish: @@ -156,7 +156,19 @@ publishbeta: mkdir -p $(WEBSAMPLEBETA) cp doc/sample_html_beta/*.* $(WEBSAMPLEBETA) -upload_relnotes: docreqs ## Upload parsed release notes to Tidelift. +CHANGES_MD = /tmp/rst_rst/changes.md +RELNOTES_JSON = /tmp/relnotes.json + +$(CHANGES_MD): CHANGES.rst $(DOCBIN) $(SPHINXBUILD) -b rst doc /tmp/rst_rst - pandoc -frst -tmarkdown_strict --atx-headers /tmp/rst_rst/changes.rst > /tmp/rst_rst/changes.md - python ci/upload_relnotes.py /tmp/rst_rst/changes.md pypi/coverage + pandoc -frst -tmarkdown_strict --atx-headers --wrap=none /tmp/rst_rst/changes.rst > $(CHANGES_MD) + +relnotes_json: $(RELNOTES_JSON) +$(RELNOTES_JSON): $(CHANGES_MD) + $(DOCBIN)/python ci/parse_relnotes.py /tmp/rst_rst/changes.md $(RELNOTES_JSON) + +tidelift_relnotes: $(RELNOTES_JSON) ## Upload parsed release notes to Tidelift. + $(DOCBIN)/python ci/tidelift_relnotes.py $(RELNOTES_JSON) pypi/coverage + +github_releases: $(RELNOTES_JSON) ## Update GitHub releases. + $(DOCBIN)/python ci/github_releases.py $(RELNOTES_JSON) nedbat/coveragepy diff --git a/ci/github_releases.py b/ci/github_releases.py new file mode 100644 index 00000000..408a2ab8 --- /dev/null +++ b/ci/github_releases.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +""" +Upload release notes into GitHub releases. +""" + +import json +import shlex +import subprocess +import sys + +import pkg_resources +import requests + + +RELEASES_URL = "https://api.github.com/repos/{repo}/releases" + +def run_command(cmd): + """ + Run a command line (with no shell). + + Returns a tuple: + bool: true if the command succeeded. + str: the output of the command. + + """ + proc = subprocess.run( + shlex.split(cmd), + shell=False, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + output = proc.stdout.decode("utf-8") + succeeded = proc.returncode == 0 + return succeeded, output + +def does_tag_exist(tag_name): + """ + Does `tag_name` exist as a tag in git? + """ + return run_command(f"git rev-parse --verify {tag_name}")[0] + +def check_ok(resp): + """ + Check that the Requests response object was successful. + + Raise an exception if not. + """ + if not resp: + print(f"text: {resp.text!r}") + resp.raise_for_status() + +def github_paginated(session, url): + """ + Get all the results from a paginated GitHub url. + """ + while True: + print(f"GETTING: {url}") + resp = session.get(url) + check_ok(resp) + yield from resp.json() + if 'Link' not in resp.headers: + break + links = resp.headers['link'].split(",") + next_link = next((link for link in links if 'rel="next"' in link), None) + if not next_link: + break + url = next_link.split(";")[0].strip(" <>") + +def get_releases(session, repo): + """ + Get all the releases from a name/project repo. + + Returns: + A dict mapping tag names to release dictionaries. + """ + url = RELEASES_URL.format(repo=repo) + "?per_page=100" + releases = { r['tag_name']: r for r in github_paginated(session, url) } + return releases + +def release_for_relnote(relnote): + """ + Turn a release note dict into the data needed by GitHub for a release. + """ + tag = f"coverage-{relnote['version']}" + return { + "tag_name": tag, + "name": tag, + "body": relnote["text"], + "draft": False, + "prerelease": relnote["prerelease"], + } + +def create_release(session, repo, relnote): + """ + Create a new GitHub release. + """ + print(f"Creating {relnote['version']}") + data = release_for_relnote(relnote) + resp = session.post(RELEASES_URL.format(repo=repo), json=data) + check_ok(resp) + +def update_release(session, url, relnote): + """ + Update an existing GitHub release. + """ + print(f"Updating {relnote['version']}") + data = release_for_relnote(relnote) + resp = session.patch(url, json=data) + check_ok(resp) + +def update_github_releases(json_filename, repo): + """ + Read the json file, and create or update releases in GitHub. + """ + gh_session = requests.Session() + releases = get_releases(gh_session, repo) + if 0: # if you need to delete all the releases! + for release in releases.values(): + print(release["tag_name"]) + resp = gh_session.delete(release["url"]) + check_ok(resp) + return + + with open(json_filename) as jf: + relnotes = json.load(jf) + relnotes.sort(key=lambda rel: pkg_resources.parse_version(rel["version"])) + for relnote in relnotes: + tag = "coverage-" + relnote["version"] + if not does_tag_exist(tag): + continue + exists = tag in releases + if not exists: + create_release(gh_session, repo, relnote) + else: + release = releases[tag] + if release["body"] != relnote["text"]: + url = release["url"] + update_release(gh_session, url, relnote) + +if __name__ == "__main__": + update_github_releases(*sys.argv[1:]) # pylint: disable=no-value-for-parameter diff --git a/ci/upload_relnotes.py b/ci/parse_relnotes.py index 630f4d0a..d19e6d60 100644 --- a/ci/upload_relnotes.py +++ b/ci/parse_relnotes.py @@ -1,26 +1,20 @@ #!/usr/bin/env python3 """ -Upload CHANGES.md to Tidelift as Markdown chunks +Parse CHANGES.md into a JSON structure. -Put your Tidelift API token in a file called tidelift.token alongside this -program, for example: +Run with two arguments: the .md file to parse, and the JSON file to write: - user/n3IwOpxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxc2ZwE4 - -Run with two arguments: the .md file to parse, and the Tidelift package name: - - python upload_relnotes.py CHANGES.md pypi/coverage + python parse_relnotes.py CHANGES.md relnotes.json Every section that has something that looks like a version number in it will -be uploaded as the release notes for that version. +be recorded as the release notes for that version. """ -import os.path +import json import re import sys -import requests class TextChunkBuffer: """Hold onto text chunks until needed.""" @@ -82,6 +76,14 @@ def sections(parsed_data): yield (*header, "\n".join(text)) +def refind(regex, text): + """Find a regex in some text, and return the matched text, or None.""" + m = re.search(regex, text) + if m: + return m.group() + else: + return None + def relnotes(mdlines): r"""Yield (version, text) pairs from markdown lines. @@ -91,32 +93,23 @@ def relnotes(mdlines): """ for _, htext, text in sections(parse_md(mdlines)): - m_version = re.search(r"\d+\.\d[^ ]*", htext) - if m_version: - version = m_version.group() - yield version, text - -def update_release_note(package, version, text): - """Update the release notes for one version of a package.""" - url = f"https://api.tidelift.com/external-api/lifting/{package}/release-notes/{version}" - token_file = os.path.join(os.path.dirname(__file__), "tidelift.token") - with open(token_file) as ftoken: - token = ftoken.read().strip() - headers = { - "Authorization": f"Bearer: {token}", - } - req_args = dict(url=url, data=text.encode('utf8'), headers=headers) - result = requests.post(**req_args) - if result.status_code == 409: - result = requests.put(**req_args) - print(f"{version}: {result.status_code}") - -def parse_and_upload(md_filename, package): - """Main function: parse markdown and upload to Tidelift.""" - with open(md_filename) as f: - markdown = f.read() - for version, text in relnotes(markdown.splitlines(True)): - update_release_note(package, version, text) + version = refind(r"\d+\.\d[^ ]*", htext) + if version: + prerelease = any(c in version for c in "abc") + when = refind(r"\d+-\d+-\d+", htext) + yield { + "version": version, + "text": text, + "prerelease": prerelease, + "when": when, + } + +def parse(md_filename, json_filename): + """Main function: parse markdown and write JSON.""" + with open(md_filename) as mf: + markdown = mf.read() + with open(json_filename, "w") as jf: + json.dump(list(relnotes(markdown.splitlines(True))), jf, indent=4) if __name__ == "__main__": - parse_and_upload(*sys.argv[1:]) # pylint: disable=no-value-for-parameter + parse(*sys.argv[1:]) # pylint: disable=no-value-for-parameter diff --git a/ci/tidelift_relnotes.py b/ci/tidelift_relnotes.py new file mode 100644 index 00000000..bc3a37d4 --- /dev/null +++ b/ci/tidelift_relnotes.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +""" +Upload release notes from a JSON file to Tidelift as Markdown chunks + +Put your Tidelift API token in a file called tidelift.token alongside this +program, for example: + + user/n3IwOpxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxc2ZwE4 + +Run with two arguments: the JSON file of release notes, and the Tidelift +package name: + + python tidelift_relnotes.py relnotes.json pypi/coverage + +Every section that has something that looks like a version number in it will +be uploaded as the release notes for that version. + +""" + +import json +import os.path +import sys + +import requests + + +def update_release_note(package, version, text): + """Update the release notes for one version of a package.""" + url = f"https://api.tidelift.com/external-api/lifting/{package}/release-notes/{version}" + token_file = os.path.join(os.path.dirname(__file__), "tidelift.token") + with open(token_file) as ftoken: + token = ftoken.read().strip() + headers = { + "Authorization": f"Bearer: {token}", + } + req_args = dict(url=url, data=text.encode('utf8'), headers=headers) + result = requests.post(**req_args) + if result.status_code == 409: + result = requests.put(**req_args) + print(f"{version}: {result.status_code}") + +def upload(json_filename, package): + """Main function: parse markdown and upload to Tidelift.""" + with open(json_filename) as jf: + relnotes = json.load(jf) + for relnote in relnotes: + update_release_note(package, relnote["version"], relnote["text"]) + +if __name__ == "__main__": + upload(*sys.argv[1:]) # pylint: disable=no-value-for-parameter @@ -74,7 +74,9 @@ - add an "Unreleased" section to the top. - git push - Update Tidelift: - - make upload_relnotes + - make tidelift_relnotes +- Update GitHub releases: + - make github_releases - Update readthedocs - IF NOT PRE-RELEASE: - update git "stable" branch to point to latest release |