Release management using CircleCI (#3498)

* Release management using CircleCI

* Changelog updated

scripts/release_management/README.md

@@ -0,0 +1,65 @@
+# Release management scripts
+
+## Overview
+The scripts in this folder are used for release management in CircleCI. Although the scripts are fully configurable using input parameters,
+the default settings were modified to accommodate CircleCI execution.
+
+# Build scripts
+These scripts help during the build process. They prepare the release files.
+
+## bump-semver.py
+Bumps the semantic version of the input `--version`. Versions are expected in vMAJOR.MINOR.PATCH format or vMAJOR.MINOR format.
+
+In vMAJOR.MINOR format, the result will be patch version 0 of that version, for example `v1.2 -> v1.2.0`.
+
+In vMAJOR.MINOR.PATCH format, the result will be a bumped PATCH version, for example `v1.2.3 -> v1.2.4`.
+
+If the PATCH number contains letters, it is considered a development version, in which case, the result is the non-development version of that number.
+The patch number will not be bumped, only the "-dev" or similar additional text will be removed. For example: `v1.2.6-rc1 -> v1.2.6`.
+
+## zip-file.py
+Specialized ZIP command for release management. Special features:
+1. Uses Python ZIP libaries, so the `zip` command does not need to be installed.
+1. Can only zip one file.
+1. Optionally gets file version, Go OS and architecture.
+1. By default all inputs and output is formatted exactly how CircleCI needs it.
+
+By default, the command will try to ZIP the file at `build/tendermint_${GOOS}_${GOARCH}`.
+This can be changed with the `--file` input parameter.
+
+By default, the command will output the ZIP file to `build/tendermint_${CIRCLE_TAG}_${GOOS}_${GOARCH}.zip`.
+This can be changed with the `--destination` (folder), `--version`, `--goos` and `--goarch` input parameters respectively.
+
+## sha-files.py
+Specialized `shasum` command for release management. Special features:
+1. Reads all ZIP files in the given folder.
+1. By default all inputs and output is formatted exactly how CircleCI needs it.
+
+By default, the command will look up all ZIP files in the `build/` folder.
+
+By default, the command will output results into the `build/SHA256SUMS` file.
+
+# GitHub management
+Uploading build results to GitHub requires at least these steps:
+1. Create a new release on GitHub with content
+2. Upload all binaries to the release
+3. Publish the release
+The below scripts help with these steps.
+
+## github-draft.py
+Creates a GitHub release and fills the content with the CHANGELOG.md link. The version number can be changed by the `--version` parameter.
+
+By default, the command will use the tendermint/tendermint organization/repo, which can be changed using the `--org` and `--repo` parameters.
+
+By default, the command will get the version number from the `${CIRCLE_TAG}` variable.
+
+Returns the GitHub release ID.
+
+## github-upload.py
+Upload a file to a GitHub release. The release is defined by the mandatory `--id` (release ID) input parameter.
+
+By default, the command will upload the file `/tmp/workspace/tendermint_${CIRCLE_TAG}_${GOOS}_${GOARCH}.zip`. This can be changed by the `--file` input parameter.
+
+## github-publish.py
+Publish a GitHub release. The release is defined by the mandatory `--id` (release ID) input parameter.
+

scripts/release_management/bump-semver.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+# Bump the release number of a semantic version number and print it. --version is required.
+# Version is
+# - vA.B.C, in which case vA.B.C+1 will be returned
+# - vA.B.C-devorwhatnot in which case vA.B.C will be returned
+# - vA.B in which case vA.B.0 will be returned
+
+import re
+import argparse
+
+
+def semver(ver):
+  if re.match('v[0-9]+\.[0-9]+',ver) is None:
+    ver="v0.0"
+    #raise argparse.ArgumentTypeError('--version must be a semantic version number with major, minor and patch numbers')
+  return ver
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--version", help="Version number to bump, e.g.: v1.0.0", required=True, type=semver)
+  args = parser.parse_args()
+
+  found = re.match('(v[0-9]+\.[0-9]+)(\.(.+))?', args.version)
+  majorminorprefix = found.group(1)
+  patch = found.group(3)
+  if patch is None:
+    patch = "0-new"
+
+  if re.match('[0-9]+$',patch) is None:
+    patchfound = re.match('([0-9]+)',patch)
+    patch = int(patchfound.group(1))
+  else:
+    patch = int(patch) + 1
+
+  print("{0}.{1}".format(majorminorprefix, patch))

scripts/release_management/github-draft.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+
+# Create a draft release on GitHub. By default in the tendermint/tendermint repo.
+# Optimized for CircleCI
+
+import argparse
+import httplib
+import json
+import os
+from base64 import b64encode
+
+def request(org, repo, data):
+  user_and_pass = b64encode(b"{0}:{1}".format(os.environ['GITHUB_USERNAME'], os.environ['GITHUB_TOKEN'])).decode("ascii")
+  headers = {
+    'User-Agent': 'tenderbot',
+    'Accept': 'application/vnd.github.v3+json',
+    'Authorization': 'Basic %s' % user_and_pass
+  }
+  
+  conn = httplib.HTTPSConnection('api.github.com', timeout=5)
+  conn.request('POST', '/repos/{0}/{1}/releases'.format(org,repo), data, headers)
+  response = conn.getresponse()
+  if response.status < 200 or response.status > 299:
+    print("{0}: {1}".format(response.status, response.reason))
+    conn.close()
+    raise IOError(response.reason)
+  responsedata = response.read()
+  conn.close()
+  return json.loads(responsedata)
+
+
+def create_draft(org,repo,version):
+  draft = {
+    'tag_name': version,
+    'target_commitish': 'master',
+    'name': '{0} (WARNING: ALPHA SOFTWARE)'.format(version),
+    'body': '<a href=https://github.com/{0}/{1}/blob/master/CHANGELOG.md#{2}>https://github.com/{0}/{1}/blob/master/CHANGELOG.md#{2}</a>'.format(org,repo,version.replace('v','').replace('.','')),
+    'draft': True,
+    'prerelease': False
+  }
+  data=json.dumps(draft)
+  return request(org, repo, data)
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--org", default="tendermint", help="GitHub organization")
+  parser.add_argument("--repo", default="tendermint", help="GitHub repository")
+  parser.add_argument("--version", default=os.environ.get('CIRCLE_TAG'), help="Version number for binary, e.g.: v1.0.0")
+  args = parser.parse_args()
+
+  if not os.environ.has_key('GITHUB_USERNAME'):
+    raise parser.error('environment variable GITHUB_USERNAME is required')
+
+  if not os.environ.has_key('GITHUB_TOKEN'):
+    raise parser.error('environment variable GITHUB_TOKEN is required')
+
+  release = create_draft(args.org,args.repo,args.version)
+
+  print(release["id"])
+

scripts/release_management/github-openpr.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+
+# Open a PR against the develop branch. --branch required.
+# Optimized for CircleCI
+
+import json
+import os
+import argparse
+import httplib
+from base64 import b64encode
+
+
+def request(org, repo, data):
+  user_and_pass = b64encode(b"{0}:{1}".format(os.environ['GITHUB_USERNAME'], os.environ['GITHUB_TOKEN'])).decode("ascii")
+  headers = {
+    'User-Agent': 'tenderbot',
+    'Accept': 'application/vnd.github.v3+json',
+    'Authorization': 'Basic %s' % user_and_pass
+  }
+
+  conn = httplib.HTTPSConnection('api.github.com', timeout=5)
+  conn.request('POST', '/repos/{0}/{1}/pulls'.format(org,repo), data, headers)
+  response = conn.getresponse()
+  if response.status < 200 or response.status > 299:
+    print(response)
+    conn.close()
+    raise IOError(response.reason)
+  responsedata = response.read()
+  conn.close()
+  return json.loads(responsedata)
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--org", default="tendermint", help="GitHub organization. Defaults to tendermint.")
+  parser.add_argument("--repo", default="tendermint", help="GitHub repository. Defaults to tendermint.")
+  parser.add_argument("--head", help="The name of the branch where your changes are implemented.", required=True)
+  parser.add_argument("--base", help="The name of the branch you want the changes pulled into.", required=True)
+  parser.add_argument("--title", default="Security release {0}".format(os.environ.get('CIRCLE_TAG')), help="The title of the pull request.")
+  args = parser.parse_args()
+
+  if not os.environ.has_key('GITHUB_USERNAME'):
+    raise parser.error('GITHUB_USERNAME not set.')
+
+  if not os.environ.has_key('GITHUB_TOKEN'):
+    raise parser.error('GITHUB_TOKEN not set.')
+
+  if os.environ.get('CIRCLE_TAG') is None:
+    raise parser.error('CIRCLE_TAG not set.')
+
+  result = request(args.org, args.repo, data=json.dumps({'title':"{0}".format(args.title),'head':"{0}".format(args.head),'base':"{0}".format(args.base),'body':"<Please fill in details.>"}))
+  print(result['html_url'])

scripts/release_management/github-public-newbranch.bash

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+# github-public-newbranch.bash - create public branch from the security repository
+
+set -euo pipefail
+
+# Create new branch
+BRANCH="${CIRCLE_TAG:-v0.0.0}-security-`date -u +%Y%m%d%H%M%S`"
+# Check if the patch release exist already as a branch
+if [ -n "`git branch | grep '${BRANCH}'`" ]; then
+  echo "WARNING: Branch ${BRANCH} already exists."
+else
+  echo "Creating branch ${BRANCH}."
+  git branch "${BRANCH}"
+fi
+
+# ... and check it out
+git checkout "${BRANCH}"
+
+# Add entry to public repository
+git remote add tendermint-origin git@github.com:tendermint/tendermint.git
+
+# Push branch and tag to public repository
+git push tendermint-origin
+git push tendermint-origin --tags
+
+# Create a PR from the public branch to the assumed release branch in public (release branch has to exist)
+python -u scripts/release_management/github-openpr.py --head "${BRANCH}" --base "${BRANCH:%.*}"

scripts/release_management/github-publish.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+
+# Publish an existing GitHub draft release. --id required.
+# Optimized for CircleCI
+
+import json
+import os
+import argparse
+import httplib
+from base64 import b64encode
+
+
+def request(org, repo, id, data):
+  user_and_pass = b64encode(b"{0}:{1}".format(os.environ['GITHUB_USERNAME'], os.environ['GITHUB_TOKEN'])).decode("ascii")
+  headers = {
+    'User-Agent': 'tenderbot',
+    'Accept': 'application/vnd.github.v3+json',
+    'Authorization': 'Basic %s' % user_and_pass
+  }
+
+  conn = httplib.HTTPSConnection('api.github.com', timeout=5)
+  conn.request('POST', '/repos/{0}/{1}/releases/{2}'.format(org,repo,id), data, headers)
+  response = conn.getresponse()
+  if response.status < 200 or response.status > 299:
+    print(response)
+    conn.close()
+    raise IOError(response.reason)
+  responsedata = response.read()
+  conn.close()
+  return json.loads(responsedata)
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--org", default="tendermint", help="GitHub organization")
+  parser.add_argument("--repo", default="tendermint", help="GitHub repository")
+  parser.add_argument("--id", help="GitHub release ID", required=True, type=int)
+  parser.add_argument("--version", default=os.environ.get('CIRCLE_TAG'), help="Version number for the release, e.g.: v1.0.0")
+  args = parser.parse_args()
+
+  if not os.environ.has_key('GITHUB_USERNAME'):
+    raise parser.error('GITHUB_USERNAME not set.')
+
+  if not os.environ.has_key('GITHUB_TOKEN'):
+    raise parser.error('GITHUB_TOKEN not set.')
+
+  try:
+    result = request(args.org, args.repo, args.id, data=json.dumps({'draft':False,'tag_name':"{0}".format(args.version)}))
+  except IOError as e:
+    print(e)
+    result = request(args.org, args.repo, args.id, data=json.dumps({'draft':False,'tag_name':"{0}-autorelease".format(args.version)}))
+
+  print(result['name'])

scripts/release_management/github-upload.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+
+# Upload a file to a GitHub draft release. --id and --file are required.
+# Optimized for CircleCI
+
+import json
+import os
+import re
+import argparse
+import mimetypes
+import httplib
+from base64 import b64encode
+
+
+def request(baseurl, path, mimetype, mimeencoding, data):
+  user_and_pass = b64encode(b"{0}:{1}".format(os.environ['GITHUB_USERNAME'], os.environ['GITHUB_TOKEN'])).decode("ascii")
+  
+  headers = {
+    'User-Agent': 'tenderbot',
+    'Accept': 'application/vnd.github.v3.raw+json',
+    'Authorization': 'Basic %s' % user_and_pass,
+    'Content-Type': mimetype,
+    'Content-Encoding': mimeencoding
+  }
+
+  conn = httplib.HTTPSConnection(baseurl, timeout=5)
+  conn.request('POST', path, data, headers)
+  response = conn.getresponse()
+  if response.status < 200 or response.status > 299:
+    print(response)
+    conn.close()
+    raise IOError(response.reason)
+  responsedata = response.read()
+  conn.close()
+  return json.loads(responsedata)
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--id", help="GitHub release ID", required=True, type=int)
+  parser.add_argument("--file", default="/tmp/workspace/tendermint_{0}_{1}_{2}.zip".format(os.environ.get('CIRCLE_TAG'),os.environ.get('GOOS'),os.environ.get('GOARCH')), help="File to upload")
+  parser.add_argument("--return-id-only", help="Return only the release ID after upload to GitHub.", action='store_true')
+  args = parser.parse_args()
+
+  if not os.environ.has_key('GITHUB_USERNAME'):
+    raise parser.error('GITHUB_USERNAME not set.')
+
+  if not os.environ.has_key('GITHUB_TOKEN'):
+    raise parser.error('GITHUB_TOKEN not set.')
+
+  mimetypes.init()
+  filename = os.path.basename(args.file)
+  mimetype,mimeencoding = mimetypes.guess_type(filename, strict=False)
+  if mimetype is None:
+    mimetype = 'application/zip'
+  if mimeencoding is None:
+    mimeencoding = 'utf8'
+
+  with open(args.file,'rb') as f:
+    asset = f.read()
+
+  result = request('uploads.github.com', '/repos/tendermint/tendermint/releases/{0}/assets?name={1}'.format(args.id, filename), mimetype, mimeencoding, asset)
+
+  if args.return_id_only:
+    print(result['id'])
+  else:
+    print(result['browser_download_url'])
+

scripts/release_management/sha-files.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+# Create SHA256 summaries from all ZIP files in a folder
+# Optimized for CircleCI
+
+import re
+import os
+import argparse
+import zipfile
+import hashlib
+
+
+BLOCKSIZE = 65536
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--folder", default="/tmp/workspace", help="Folder to look for, for ZIP files")
+  parser.add_argument("--shafile", default="/tmp/workspace/SHA256SUMS", help="SHA256 summaries File")
+  args = parser.parse_args()
+
+  for filename in os.listdir(args.folder):
+    if re.search('\.zip$',filename) is None:
+      continue
+    if not os.path.isfile(os.path.join(args.folder, filename)):
+      continue
+    with open(args.shafile,'a+') as shafile:
+      hasher = hashlib.sha256()
+      with open(os.path.join(args.folder, filename),'r') as f:
+        buf = f.read(BLOCKSIZE)
+        while len(buf) > 0:
+          hasher.update(buf)
+          buf = f.read(BLOCKSIZE)
+      shafile.write("{0} {1}\n".format(hasher.hexdigest(),filename))  
+

scripts/release_management/zip-file.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+
+# ZIP one file as "tendermint" into a ZIP like tendermint_VERSION_OS_ARCH.zip
+# Use environment variables CIRCLE_TAG, GOOS and GOARCH for easy input parameters.
+# Optimized for CircleCI
+
+import os
+import argparse
+import zipfile
+import hashlib
+
+
+BLOCKSIZE = 65536
+
+
+def zip_asset(file,destination,arcname,version,goos,goarch):
+  filename = os.path.basename(file)
+  output = "{0}/{1}_{2}_{3}_{4}.zip".format(destination,arcname,version,goos,goarch)
+
+  with zipfile.ZipFile(output,'w') as f:
+    f.write(filename=file,arcname=arcname)
+    f.comment=filename
+  return output
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--file", default="build/tendermint_{0}_{1}".format(os.environ.get('GOOS'),os.environ.get('GOARCH')), help="File to zip")
+  parser.add_argument("--destination", default="build", help="Destination folder for files")
+  parser.add_argument("--version", default=os.environ.get('CIRCLE_TAG'), help="Version number for binary, e.g.: v1.0.0")
+  parser.add_argument("--goos", default=os.environ.get('GOOS'), help="GOOS parameter")
+  parser.add_argument("--goarch", default=os.environ.get('GOARCH'), help="GOARCH parameter")
+  args = parser.parse_args()
+
+  if args.version is None:
+    raise parser.error("argument --version is required")
+  if args.goos is None:
+    raise parser.error("argument --goos is required")
+  if args.goarch is None:
+    raise parser.error("argument --goarch is required")
+
+  file = zip_asset(args.file,args.destination,"tendermint",args.version,args.goos,args.goarch)
+  print(file)
+