Skip to content

Commit f6d5c35

Browse files
nedbatnedbat
committed
GitHub release automation
1 parent 7f8dd68 commit f6d5c35

File tree

5 files changed

+244
-45
lines changed

5 files changed

+244
-45
lines changed

‎Makefile

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -133,17 +133,17 @@ WEBHOME = ~/web/stellated/
133133
WEBSAMPLE = $(WEBHOME)/files/sample_coverage_html
134134
WEBSAMPLEBETA = $(WEBHOME)/files/sample_coverage_html_beta
135135

136-
docreqs:
136+
$(DOCPYTHON):
137137
tox -q -e doc --notest
138138

139-
dochtml: docreqs ## Build the docs HTML output.
139+
dochtml: $(DOCBIN) ## Build the docs HTML output.
140140
$(DOCBIN)/python doc/check_copied_from.py doc/*.rst
141141
$(SPHINXBUILD) -b html doc doc/_build/html
142142

143143
docdev: dochtml ## Build docs, and auto-watch for changes.
144144
PATH=$(DOCBIN):$(PATH) $(SPHINXAUTOBUILD) -b html doc doc/_build/html
145145

146-
docspell: docreqs
146+
docspell: $(DOCBIN)
147147
$(SPHINXBUILD) -b spelling doc doc/_spell
148148

149149
publish:
@@ -156,7 +156,19 @@ publishbeta:
156156
mkdir -p $(WEBSAMPLEBETA)
157157
cp doc/sample_html_beta/*.* $(WEBSAMPLEBETA)
158158

159-
upload_relnotes: docreqs ## Upload parsed release notes to Tidelift.
159+
CHANGES_MD = /tmp/rst_rst/changes.md
160+
RELNOTES_JSON = /tmp/relnotes.json
161+
162+
$(CHANGES_MD): CHANGES.rst $(DOCBIN)
160163
$(SPHINXBUILD) -b rst doc /tmp/rst_rst
161-
pandoc -frst -tmarkdown_strict --atx-headers /tmp/rst_rst/changes.rst > /tmp/rst_rst/changes.md
162-
python ci/upload_relnotes.py /tmp/rst_rst/changes.md pypi/coverage
164+
pandoc -frst -tmarkdown_strict --atx-headers --wrap=none /tmp/rst_rst/changes.rst > $(CHANGES_MD)
165+
166+
relnotes_json: $(RELNOTES_JSON)
167+
$(RELNOTES_JSON): $(CHANGES_MD)
168+
$(DOCBIN)/python ci/parse_relnotes.py /tmp/rst_rst/changes.md $(RELNOTES_JSON)
169+
170+
tidelift_relnotes: $(RELNOTES_JSON) ## Upload parsed release notes to Tidelift.
171+
$(DOCBIN)/python ci/tidelift_relnotes.py $(RELNOTES_JSON) pypi/coverage
172+
173+
github_releases: $(RELNOTES_JSON) ## Update GitHub releases.
174+
$(DOCBIN)/python ci/github_releases.py $(RELNOTES_JSON) nedbat/coveragepy

‎ci/github_releases.py

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Upload release notes into GitHub releases.
4+
"""
5+
6+
import json
7+
import shlex
8+
import subprocess
9+
import sys
10+
11+
import pkg_resources
12+
import requests
13+
14+
15+
RELEASES_URL = "https://api.github.com/repos/{repo}/releases"
16+
17+
def run_command(cmd):
18+
"""
19+
Run a command line (with no shell).
20+
21+
Returns a tuple:
22+
bool: true if the command succeeded.
23+
str: the output of the command.
24+
25+
"""
26+
proc = subprocess.run(
27+
shlex.split(cmd),
28+
shell=False,
29+
check=False,
30+
stdout=subprocess.PIPE,
31+
stderr=subprocess.STDOUT,
32+
)
33+
output = proc.stdout.decode("utf-8")
34+
succeeded = proc.returncode == 0
35+
return succeeded, output
36+
37+
def does_tag_exist(tag_name):
38+
"""
39+
Does `tag_name` exist as a tag in git?
40+
"""
41+
return run_command(f"git rev-parse --verify {tag_name}")[0]
42+
43+
def check_ok(resp):
44+
"""
45+
Check that the Requests response object was successful.
46+
47+
Raise an exception if not.
48+
"""
49+
if not resp:
50+
print(f"text: {resp.text!r}")
51+
resp.raise_for_status()
52+
53+
def github_paginated(session, url):
54+
"""
55+
Get all the results from a paginated GitHub url.
56+
"""
57+
while True:
58+
print(f"GETTING: {url}")
59+
resp = session.get(url)
60+
check_ok(resp)
61+
yield from resp.json()
62+
if 'Link' not in resp.headers:
63+
break
64+
links = resp.headers['link'].split(",")
65+
next_link = next((link for link in links if 'rel="next"' in link), None)
66+
if not next_link:
67+
break
68+
url = next_link.split(";")[0].strip(" <>")
69+
70+
def get_releases(session, repo):
71+
"""
72+
Get all the releases from a name/project repo.
73+
74+
Returns:
75+
A dict mapping tag names to release dictionaries.
76+
"""
77+
url = RELEASES_URL.format(repo=repo) + "?per_page=100"
78+
releases = { r['tag_name']: r for r in github_paginated(session, url) }
79+
return releases
80+
81+
def release_for_relnote(relnote):
82+
"""
83+
Turn a release note dict into the data needed by GitHub for a release.
84+
"""
85+
tag = f"coverage-{relnote['version']}"
86+
return {
87+
"tag_name": tag,
88+
"name": tag,
89+
"body": relnote["text"],
90+
"draft": False,
91+
"prerelease": relnote["prerelease"],
92+
}
93+
94+
def create_release(session, repo, relnote):
95+
"""
96+
Create a new GitHub release.
97+
"""
98+
print(f"Creating {relnote['version']}")
99+
data = release_for_relnote(relnote)
100+
resp = session.post(RELEASES_URL.format(repo=repo), json=data)
101+
check_ok(resp)
102+
103+
def update_release(session, url, relnote):
104+
"""
105+
Update an existing GitHub release.
106+
"""
107+
print(f"Updating {relnote['version']}")
108+
data = release_for_relnote(relnote)
109+
resp = session.patch(url, json=data)
110+
check_ok(resp)
111+
112+
def update_github_releases(json_filename, repo):
113+
"""
114+
Read the json file, and create or update releases in GitHub.
115+
"""
116+
gh_session = requests.Session()
117+
releases = get_releases(gh_session, repo)
118+
if 0: # if you need to delete all the releases!
119+
for release in releases.values():
120+
print(release["tag_name"])
121+
resp = gh_session.delete(release["url"])
122+
check_ok(resp)
123+
return
124+
125+
with open(json_filename) as jf:
126+
relnotes = json.load(jf)
127+
relnotes.sort(key=lambda rel: pkg_resources.parse_version(rel["version"]))
128+
for relnote in relnotes:
129+
tag = "coverage-" + relnote["version"]
130+
if not does_tag_exist(tag):
131+
continue
132+
exists = tag in releases
133+
if not exists:
134+
create_release(gh_session, repo, relnote)
135+
else:
136+
release = releases[tag]
137+
if release["body"] != relnote["text"]:
138+
url = release["url"]
139+
update_release(gh_session, url, relnote)
140+
141+
if __name__ == "__main__":
142+
update_github_releases(*sys.argv[1:]) # pylint: disable=no-value-for-parameter

‎ci/upload_relnotes.py renamed to ‎ci/parse_relnotes.py

Lines changed: 31 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,20 @@
11
#!/usr/bin/env python3
22
"""
3-
Upload CHANGES.md to Tidelift as Markdown chunks
3+
Parse CHANGES.md into a JSON structure.
44
5-
Put your Tidelift API token in a file called tidelift.token alongside this
6-
program, for example:
5+
Run with two arguments: the .md file to parse, and the JSON file to write:
76
8-
user/n3IwOpxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxc2ZwE4
9-
10-
Run with two arguments: the .md file to parse, and the Tidelift package name:
11-
12-
python upload_relnotes.py CHANGES.md pypi/coverage
7+
python parse_relnotes.py CHANGES.md relnotes.json
138
149
Every section that has something that looks like a version number in it will
15-
be uploaded as the release notes for that version.
10+
be recorded as the release notes for that version.
1611
1712
"""
1813

19-
import os.path
14+
import json
2015
import re
2116
import sys
2217

23-
import requests
2418

2519
class TextChunkBuffer:
2620
"""Hold onto text chunks until needed."""
@@ -82,6 +76,14 @@ def sections(parsed_data):
8276
yield (*header, "\n".join(text))
8377

8478

79+
def refind(regex, text):
80+
"""Find a regex in some text, and return the matched text, or None."""
81+
m = re.search(regex, text)
82+
if m:
83+
return m.group()
84+
else:
85+
return None
86+
8587
def relnotes(mdlines):
8688
r"""Yield (version, text) pairs from markdown lines.
8789
@@ -91,32 +93,23 @@ def relnotes(mdlines):
9193
9294
"""
9395
for _, htext, text in sections(parse_md(mdlines)):
94-
m_version = re.search(r"\d+\.\d[^ ]*", htext)
95-
if m_version:
96-
version = m_version.group()
97-
yield version, text
98-
99-
def update_release_note(package, version, text):
100-
"""Update the release notes for one version of a package."""
101-
url = f"https://api.tidelift.com/external-api/lifting/{package}/release-notes/{version}"
102-
token_file = os.path.join(os.path.dirname(__file__), "tidelift.token")
103-
with open(token_file) as ftoken:
104-
token = ftoken.read().strip()
105-
headers = {
106-
"Authorization": f"Bearer: {token}",
107-
}
108-
req_args = dict(url=url, data=text.encode('utf8'), headers=headers)
109-
result = requests.post(**req_args)
110-
if result.status_code == 409:
111-
result = requests.put(**req_args)
112-
print(f"{version}: {result.status_code}")
113-
114-
def parse_and_upload(md_filename, package):
115-
"""Main function: parse markdown and upload to Tidelift."""
116-
with open(md_filename) as f:
117-
markdown = f.read()
118-
for version, text in relnotes(markdown.splitlines(True)):
119-
update_release_note(package, version, text)
96+
version = refind(r"\d+\.\d[^ ]*", htext)
97+
if version:
98+
prerelease = any(c in version for c in "abc")
99+
when = refind(r"\d+-\d+-\d+", htext)
100+
yield {
101+
"version": version,
102+
"text": text,
103+
"prerelease": prerelease,
104+
"when": when,
105+
}
106+
107+
def parse(md_filename, json_filename):
108+
"""Main function: parse markdown and write JSON."""
109+
with open(md_filename) as mf:
110+
markdown = mf.read()
111+
with open(json_filename, "w") as jf:
112+
json.dump(list(relnotes(markdown.splitlines(True))), jf, indent=4)
120113

121114
if __name__ == "__main__":
122-
parse_and_upload(*sys.argv[1:]) # pylint: disable=no-value-for-parameter
115+
parse(*sys.argv[1:]) # pylint: disable=no-value-for-parameter

‎ci/tidelift_relnotes.py

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Upload release notes from a JSON file to Tidelift as Markdown chunks
4+
5+
Put your Tidelift API token in a file called tidelift.token alongside this
6+
program, for example:
7+
8+
user/n3IwOpxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxc2ZwE4
9+
10+
Run with two arguments: the JSON file of release notes, and the Tidelift
11+
package name:
12+
13+
python tidelift_relnotes.py relnotes.json pypi/coverage
14+
15+
Every section that has something that looks like a version number in it will
16+
be uploaded as the release notes for that version.
17+
18+
"""
19+
20+
import json
21+
import os.path
22+
import sys
23+
24+
import requests
25+
26+
27+
def update_release_note(package, version, text):
28+
"""Update the release notes for one version of a package."""
29+
url = f"https://api.tidelift.com/external-api/lifting/{package}/release-notes/{version}"
30+
token_file = os.path.join(os.path.dirname(__file__), "tidelift.token")
31+
with open(token_file) as ftoken:
32+
token = ftoken.read().strip()
33+
headers = {
34+
"Authorization": f"Bearer: {token}",
35+
}
36+
req_args = dict(url=url, data=text.encode('utf8'), headers=headers)
37+
result = requests.post(**req_args)
38+
if result.status_code == 409:
39+
result = requests.put(**req_args)
40+
print(f"{version}: {result.status_code}")
41+
42+
def upload(json_filename, package):
43+
"""Main function: parse markdown and upload to Tidelift."""
44+
with open(json_filename) as jf:
45+
relnotes = json.load(jf)
46+
for relnote in relnotes:
47+
update_release_note(package, relnote["version"], relnote["text"])
48+
49+
if __name__ == "__main__":
50+
upload(*sys.argv[1:]) # pylint: disable=no-value-for-parameter

‎howto.txt

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,9 @@
7474
- add an "Unreleased" section to the top.
7575
- git push
7676
- Update Tidelift:
77-
- make upload_relnotes
77+
- make tidelift_relnotes
78+
- Update GitHub releases:
79+
- make github_releases
7880
- Update readthedocs
7981
- IF NOT PRE-RELEASE:
8082
- update git "stable" branch to point to latest release

0 commit comments

Comments
 (0)