diff --git a/.github/scripts/auto-backport.py b/.github/scripts/auto-backport.py new file mode 100755 index 0000000000..b89df98ce9 --- /dev/null +++ b/.github/scripts/auto-backport.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 + +import argparse +import os +import re +import sys +import tempfile +import logging + +from github import Github, GithubException +from git import Repo, GitCommandError + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +try: + github_token = os.environ["GITHUB_TOKEN"] +except KeyError: + print("Please set the 'GITHUB_TOKEN' environment variable") + sys.exit(1) + + +def is_pull_request(): + return '--pull-request' in sys.argv[1:] + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--repo', type=str, required=True, help='Github repository name') + parser.add_argument('--base-branch', type=str, default='refs/heads/master', help='Base branch') + parser.add_argument('--commits', default=None, type=str, help='Range of promoted commits.') + parser.add_argument('--pull-request', type=int, help='Pull request number to be backported') + parser.add_argument('--head-commit', type=str, required=is_pull_request(), help='The HEAD of target branch after the pull request specified by --pull-request is merged') + return parser.parse_args() + + +def create_pull_request(repo, new_branch_name, base_branch_name, pr, backport_pr_title, commits, is_draft=False): + pr_body = f'{pr.body}\n\n' + for commit in commits: + pr_body += f'- (cherry picked from commit {commit})\n\n' + pr_body += f'Parent PR: #{pr.number}' + if is_draft: + new_branch_name = f'{pr.user.login}:{new_branch_name}' + try: + backport_pr = repo.create_pull( + title=backport_pr_title, + body=pr_body, + head=new_branch_name, + base=base_branch_name, + draft=is_draft + ) + logging.info(f"Pull request created: {backport_pr.html_url}") + backport_pr.add_to_assignees(pr.user) + logging.info(f"Assigned PR to original author: {pr.user}") + return backport_pr + except GithubException as e: + if 'A pull request already exists' in str(e): + logging.warning(f'A pull request already exists for {pr.user}:{new_branch_name}') + else: + logging.error(f'Failed to create PR: {e}') + + +def get_pr_commits(repo, pr, stable_branch, start_commit=None): + commits = [] + if pr.merged: + merge_commit = repo.get_commit(pr.merge_commit_sha) + if len(merge_commit.parents) > 1: # Check if this merge commit includes multiple commits + commits.append(pr.merge_commit_sha) + else: + if start_commit: + promoted_commits = repo.compare(start_commit, stable_branch).commits + else: + promoted_commits = repo.get_commits(sha=stable_branch) + for commit in pr.get_commits(): + for promoted_commit in promoted_commits: + commit_title = commit.commit.message.splitlines()[0] + # In Scylla-pkg and scylla-dtest, for example, + # we don't create a merge commit for a PR with multiple commits, + # according to the GitHub API, the last commit will be the merge commit, + # which is not what we need when backporting (we need all the commits). + # So here, we are validating the correct SHA for each commit so we can cherry-pick + if promoted_commit.commit.message.startswith(commit_title): + commits.append(promoted_commit.sha) + + elif pr.state == 'closed': + events = pr.get_issue_events() + for event in events: + if event.event == 'closed': + commits.append(event.commit_id) + return commits + + +def backport(repo, pr, version, commits, backport_base_branch, user): + with (tempfile.TemporaryDirectory() as local_repo_path): + try: + new_branch_name = f'backport/{pr.number}/to-{version}' + backport_pr_title = f'[Backport {version}] {pr.title}' + repo_local = Repo.clone_from(f'https://{user.login}:{github_token}@github.com/{repo.full_name}.git', local_repo_path, branch=backport_base_branch) + repo_local.git.checkout(b=new_branch_name) + try: + fork_repo = pr.user.get_repo(repo.name) + except Exception as e: + print(f"Error retrieving repository: {e}") + # Since Scylla core repo was modified a few years ago to ScyllaDB, + # some developers may have forks based on the original name `scylla` + print(f"{pr.user.login} fork repo is not {repo.full_name}, trying {repo.organization.login}/scylla") + fork_repo = pr.user.get_repo('scylla') + fork_repo_url = f'https://{user.login}:{github_token}@github.com/{fork_repo.full_name}.git' + repo_local.create_remote('fork', fork_repo_url) + remote = 'origin' + is_draft = False + for commit in commits: + try: + repo_local.git.cherry_pick(commit, '-m1', '-x') + except GitCommandError as e: + logging.warning(f'Cherry-pick conflict on commit {commit}: {e}') + remote = 'fork' + is_draft = True + repo_local.git.add(A=True) + repo_local.git.cherry_pick('--continue') + repo_local.git.push(remote, new_branch_name, force=True) + create_pull_request(repo, new_branch_name, backport_base_branch, pr, backport_pr_title, commits, + is_draft=is_draft) + + except GitCommandError as e: + logging.warning(f"GitCommandError: {e}") + + +def create_pr_comment_and_remove_label(pr): + comment_body = f':warning: @{pr.user.login} PR body does not contain a valid reference to an issue ' + comment_body += ' based on [linking-a-pull-request-to-an-issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword)' + comment_body += ' and can not be backported\n\n' + comment_body += 'The following labels were removed:\n' + labels = pr.get_labels() + pattern = re.compile(r"backport/\d+\.\d+$") + for label in labels: + if pattern.match(label.name): + print(f"Removing label: {label.name}") + comment_body += f'- {label.name}\n' + pr.remove_from_labels(label) + comment_body += f'\nPlease add the relevant backport labels after PR body is fixed' + pr.create_issue_comment(comment_body) + + +def main(): + args = parse_args() + base_branch = args.base_branch.split('/')[2] + promoted_label = 'promoted-to-master' + repo_name = args.repo + if 'scylla-enterprise' in args.repo: + promoted_label = 'promoted-to-enterprise' + stable_branch = base_branch + backport_branch = 'branch-' + + backport_label_pattern = re.compile(r'backport/\d+\.\d+$') + + g = Github(github_token) + repo = g.get_repo(repo_name) + user = g.get_user() + closed_prs = [] + start_commit = None + + if args.commits: + start_commit, end_commit = args.commits.split('..') + commits = repo.compare(start_commit, end_commit).commits + for commit in commits: + for pr in commit.get_pulls(): + closed_prs.append(pr) + if args.pull_request: + start_commit = args.head_commit + pr = repo.get_pull(args.pull_request) + closed_prs = [pr] + + for pr in closed_prs: + labels = [label.name for label in pr.labels] + backport_labels = [label for label in labels if backport_label_pattern.match(label)] + if promoted_label not in labels: + print(f'no {promoted_label} label: {pr.number}') + continue + if not backport_labels: + print(f'no backport label: {pr.number}') + continue + commits = get_pr_commits(repo, pr, stable_branch, start_commit) + logging.info(f"Found PR #{pr.number} with commit {commits} and the following labels: {backport_labels}") + for backport_label in backport_labels: + version = backport_label.replace('backport/', '') + backport_base_branch = backport_label.replace('backport/', backport_branch) + backport(repo, pr, version, commits, backport_base_branch, user) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/label_promoted_commits.py b/.github/scripts/label_promoted_commits.py index 4d4fed5df2..80e60dfcb4 100755 --- a/.github/scripts/label_promoted_commits.py +++ b/.github/scripts/label_promoted_commits.py @@ -16,13 +16,8 @@ def parser(): parser = argparse.ArgumentParser() parser.add_argument('--repository', type=str, required=True, help='Github repository name (e.g., scylladb/scylladb)') - parser.add_argument('--commit_before_merge', type=str, required=True, help='Git commit ID to start labeling from (' - 'newest commit).') - parser.add_argument('--commit_after_merge', type=str, required=True, - help='Git commit ID to end labeling at (oldest ' - 'commit, exclusive).') - parser.add_argument('--update_issue', type=bool, default=False, help='Set True to update issues when backport was ' - 'done') + parser.add_argument('--commits', type=str, required=True, help='Range of promoted commits.') + parser.add_argument('--label', type=str, default='promoted-to-master', help='Label to use') parser.add_argument('--ref', type=str, required=True, help='PR target branch') return parser.parse_args() @@ -53,10 +48,11 @@ def main(): target_branch = re.search(r'branch-(\d+\.\d+)', args.ref) g = Github(github_token) repo = g.get_repo(args.repository, lazy=False) - commits = repo.compare(head=args.commit_after_merge, base=args.commit_before_merge) + start_commit, end_commit = args.commits.split('..') + commits = repo.compare(start_commit, end_commit).commits processed_prs = set() # Print commit information - for commit in commits.commits: + for commit in commits: print(f'Commit sha is: {commit.sha}') match = pr_pattern.search(commit.commit.message) if match: @@ -66,13 +62,13 @@ def main(): if target_branch: pr = repo.get_pull(pr_number) branch_name = target_branch[1] - refs_pr = re.findall(r'Refs (?:#|https.*?)(\d+)', pr.body) + refs_pr = re.findall(r'Parent PR: (?:#|https.*?)(\d+)', pr.body) if refs_pr: print(f'branch-{target_branch.group(1)}, pr number is: {pr_number}') # 1. change the backport label of the parent PR to note that - # we've merge the corresponding backport PR + # we've merged the corresponding backport PR # 2. close the backport PR and leave a comment on it to note - # that it has been merged with a certain git commit, + # that it has been merged with a certain git commit. ref_pr_number = refs_pr[0] mark_backport_done(repo, ref_pr_number, branch_name) comment = f'Closed via {commit.sha}' diff --git a/.github/workflows/add-label-when-promoted.yaml b/.github/workflows/add-label-when-promoted.yaml index 7fa8bc8f7f..f4870496f5 100644 --- a/.github/workflows/add-label-when-promoted.yaml +++ b/.github/workflows/add-label-when-promoted.yaml @@ -5,9 +5,10 @@ on: branches: - master - branch-*.* - -env: - DEFAULT_BRANCH: 'master' + - enterprise + pull_request_target: + types: [labeled] + branches: [master, next, enterprise] jobs: check-commit: @@ -20,17 +21,51 @@ jobs: env: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" + - name: Set Default Branch + id: set_branch + run: | + if [[ "${{ github.repository }}" == *enterprise* ]]; then + echo "DEFAULT_BRANCH=enterprise" >> $GITHUB_ENV + else + echo "DEFAULT_BRANCH=master" >> $GITHUB_ENV + fi - name: Checkout repository uses: actions/checkout@v4 with: repository: ${{ github.repository }} ref: ${{ env.DEFAULT_BRANCH }} + token: ${{ secrets.AUTO_BACKPORT_TOKEN }} fetch-depth: 0 # Fetch all history for all tags and branches - + - name: Set up Git identity + run: | + git config --global user.name "GitHub Action" + git config --global user.email "action@github.com" + git config --global merge.conflictstyle diff3 - name: Install dependencies - run: sudo apt-get install -y python3-github - + run: sudo apt-get install -y python3-github python3-git - name: Run python script + if: github.event_name == 'push' env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: python .github/scripts/label_promoted_commits.py --commit_before_merge ${{ github.event.before }} --commit_after_merge ${{ github.event.after }} --repository ${{ github.repository }} --ref ${{ github.ref }} + GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }} + run: python .github/scripts/label_promoted_commits.py --commits ${{ github.event.before }}..${{ github.sha }} --repository ${{ github.repository }} --ref ${{ github.ref }} + - name: Run auto-backport.py when promotion completed + if: github.event_name == 'push' + env: + GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }} + run: python .github/scripts/auto-backport.py --repo ${{ github.repository }} --base-branch ${{ github.ref }} --commits ${{ github.event.before }}..${{ github.sha }} + - name: Check if label starts with 'backport/' and contains digits + id: check_label + run: | + label_name="${{ github.event.label.name }}" + if [[ "$label_name" =~ ^backport/[0-9]+\.[0-9]+$ ]]; then + echo "Label matches backport/X.X pattern." + echo "backport_label=true" >> $GITHUB_OUTPUT + else + echo "Label does not match the required pattern." + echo "backport_label=false" >> $GITHUB_OUTPUT + fi + - name: Run auto-backport.py when label was added + if: github.event_name == 'pull_request_target' && steps.check_label.outputs.backport_label == 'true' && (github.event.pull_request.state == 'closed' && github.event.pull_request.merged == true) + env: + GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }} + run: python .github/scripts/auto-backport.py --repo ${{ github.repository }} --base-branch ${{ github.ref }} --pull-request ${{ github.event.pull_request.number }} --head-commit ${{ github.event.pull_request.base.sha }}