dist: systemd: use default KillMode

before this change, we specify the KillMode of the scylla-service
service unit explicitly to "process". according to
according to
https://www.freedesktop.org/software/systemd/man/latest/systemd.kill.html,

> If set to process, only the main process itself is killed (not recommended!).

and the document suggests use "control-group" over "process".
but scylla server is not a multi-process server, it is a multi-threaded
server. so it should not make any difference even if we switch to
the recommended "control-group".

in the light that we've been seeing "defunct" scylla process after
stopping the scylla service using systemd. we are wondering if we should
try to change the `KillMode` to "control-group", which is the default
value of this setting.

in this change, we just drop the setting so that the systemd stops the
service by stopping all processes in the control group of this unit
are stopped.

Fixes scylladb/scylladb#21507

Signed-off-by: Kefu Chai <kefu.chai@scylladb.com>

Closes scylladb/scylladb#21508

(cherry picked from commit 961a53f716)

Closes scylladb/scylladb#23177

.gitattributes

@@ -1,3 +1,4 @@
 *.cc diff=cpp
 *.hh diff=cpp
 *.svg binary
+docs/_static/api/js/* binary

.github/actions/setup-build/action.yaml

@@ -1,84 +0,0 @@
-name: setup-build-env
-description: Setup Building Environment
-inputs:
-  install_clang_tool:
-    description: 'install clang-tool'
-    required: false
-    default: false
-    type: boolean
-  install_clang_tidy:
-    description: 'install clang-tidy'
-    required: false
-    default: false
-    type: boolean
-
-# use the stable branch
-# should be the same as the one used by the compositing workflow
-env:
-  CLANG_VERSION: 18
-
-runs:
-  using: 'composite'
-  steps:
-    - name: Add scylla-ppa repo
-      shell: bash
-      run: |
-        sudo add-apt-repository ppa:scylladb/ppa
-
-    - name: Add clang apt repo
-      if: ${{ inputs.install_clang_tool || inputs.install_clang_tidy }}
-      shell: bash
-      run: |
-        sudo apt-get install -y curl
-        curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc >/dev/null
-        repo_component=llvm-toolchain-jammy
-        # use the development branch if $CLANG_VERSION is empty
-        if [ -n "$CLANG_VERSION" ]; then
-            repo_component+=-$CLANG_VERSION
-        fi
-        echo "deb http://apt.llvm.org/jammy/ $repo_component main" | sudo tee -a /etc/apt/sources.list.d/llvm.list
-        sudo apt-get update
-
-    - name: Install clang-tools
-      if: ${{ inputs.install_clang_tools }}
-      shell: bash
-      run: |
-        sudo apt-get install -y clang-tools-$CLANG_VERSION
-
-    - name: Install clang-tidy
-      if: ${{ inputs.install_clang_tidy }}
-      shell: bash
-      run: |
-        sudo apt-get install -y clang-tidy-$CLANG_VERSION
-
-    - name: Install GCC-12
-      # ubuntu:jammy comes with GCC-11. and libstdc++-11 fails to compile
-      # scylla which defines value type of std::unordered_map in .cc
-      shell: bash
-      run: |
-        sudo add-apt-repository -y ppa:ubuntu-toolchain-r/ppa
-        sudo apt-get install -y libstdc++-12-dev
-
-    - name: Install more build dependencies
-      shell: bash
-      run: |
-        # - do not install java dependencies, which is not only not necessary,
-        #   and they include "python", which is not EOL and not available.
-        # - replace "scylla-libthrift010" with "libthrift-dev". because
-        #   scylla-libthrift010 : Depends: libssl1.0.0 (>= 1.0.1) but it is not installable
-        # - we don't perform tests, so minio is not necessary.
-        sed -i.orig \
-          -e '/tools\/.*\/install-dependencies.sh/d' \
-          -e 's/scylla-libthrift010-dev/libthrift-dev/' \
-          -e 's/(minio_download_jobs)/(true)/' \
-          ./install-dependencies.sh
-        sudo ./install-dependencies.sh
-        mv ./install-dependencies.sh{.orig,}
-        # for ld.lld
-        sudo apt-get install -y lld-18
-
-    - name: Install {fmt} using cooking.sh
-      shell: bash
-      run: |
-        sudo apt-get remove -y libfmt-dev
-        seastar/cooking.sh -d build-fmt -p cooking -i fmt

.github/clang-include-cleaner.json

@@ -0,0 +1,20 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "clang-include-cleaner",
+            "severity": "error",
+            "pattern": [
+                {
+                    "regexp": "^([^\\-\\+].*)$",
+                    "file": 1
+                },
+                {
+                    "regexp": "^(-\\s+[^\\s]+)\\s+@Line:(\\d+)$",
+                    "line": 2,
+                    "message": 1,
+                    "loop": true
+                }
+            ]
+        }
+    ]
+}

.github/clang-matcher.json

@@ -1,7 +1,7 @@
 {
     "problemMatcher": [
         {
-            "owner": "clang-tidy",
+            "owner": "clang",
             "pattern": [
                 {
                     "regexp": "^([^:]+):(\\d+):(\\d+):\\s+(warning|error):\\s+(.*?)\\s+\\[(.*?)\\]$",

.github/mergify.yml

@@ -65,3 +65,28 @@ pull_request_rules:
           - branch-5.4
         assignees:
           - "{{ author }}"
+  - name: Automate backport pull request 6.0
+    conditions:
+      - or:
+        - closed
+        - merged
+      - or:
+          - base=master
+          - base=next
+      - label=backport/6.0 # The PR must have this label to trigger the backport
+      - label=promoted-to-master
+    actions:
+      copy:
+        title: "[Backport 6.0] {{ title }}"
+        body: |
+          {{ body }}
+
+          {% for c in commits %}
+          (cherry picked from commit {{ c.sha }})
+          {% endfor %}
+
+           Refs #{{number}}
+        branches:
+          - branch-6.0
+        assignees:
+          - "{{ author }}"

.github/pull_request_template.md

@@ -0,0 +1 @@
+**Please replace this line with justification for the backport/\* labels added to this PR**

.github/scripts/auto-backport.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import re
+import sys
+import tempfile
+import logging
+
+from github import Github, GithubException
+from git import Repo, GitCommandError
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+try:
+    github_token = os.environ["GITHUB_TOKEN"]
+except KeyError:
+    print("Please set the 'GITHUB_TOKEN' environment variable")
+    sys.exit(1)
+
+
+def is_pull_request():
+    return '--pull-request' in sys.argv[1:]
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--repo', type=str, required=True, help='Github repository name')
+    parser.add_argument('--base-branch', type=str, default='refs/heads/master', help='Base branch')
+    parser.add_argument('--commits', default=None, type=str, help='Range of promoted commits.')
+    parser.add_argument('--pull-request', type=int, help='Pull request number to be backported')
+    parser.add_argument('--head-commit', type=str, required=is_pull_request(), help='The HEAD of target branch after the pull request specified by --pull-request is merged')
+    return parser.parse_args()
+
+
+def create_pull_request(repo, new_branch_name, base_branch_name, pr, backport_pr_title, commits, is_draft=False):
+    pr_body = f'{pr.body}\n\n'
+    for commit in commits:
+        pr_body += f'- (cherry picked from commit {commit})\n\n'
+    pr_body += f'Parent PR: #{pr.number}'
+    try:
+        backport_pr = repo.create_pull(
+            title=backport_pr_title,
+            body=pr_body,
+            head=f'scylladbbot:{new_branch_name}',
+            base=base_branch_name,
+            draft=is_draft
+        )
+        logging.info(f"Pull request created: {backport_pr.html_url}")
+        backport_pr.add_to_assignees(pr.user)
+        if is_draft:
+            backport_pr.add_to_labels("conflicts")
+            pr_comment = f"@{pr.user} - This PR was marked as draft because it has conflicts\n"
+            pr_comment += "Please resolve them and mark this PR as ready for review"
+            backport_pr.create_issue_comment(pr_comment)
+        logging.info(f"Assigned PR to original author: {pr.user}")
+        return backport_pr
+    except GithubException as e:
+        if 'A pull request already exists' in str(e):
+            logging.warning(f'A pull request already exists for {pr.user}:{new_branch_name}')
+        else:
+            logging.error(f'Failed to create PR: {e}')
+
+
+def get_pr_commits(repo, pr, stable_branch, start_commit=None):
+    commits = []
+    if pr.merged:
+        merge_commit = repo.get_commit(pr.merge_commit_sha)
+        if len(merge_commit.parents) > 1:  # Check if this merge commit includes multiple commits
+            commits.append(pr.merge_commit_sha)
+        else:
+            if start_commit:
+                promoted_commits = repo.compare(start_commit, stable_branch).commits
+            else:
+                promoted_commits = repo.get_commits(sha=stable_branch)
+            for commit in pr.get_commits():
+                for promoted_commit in promoted_commits:
+                    commit_title = commit.commit.message.splitlines()[0]
+                    # In Scylla-pkg and scylla-dtest, for example,
+                    # we don't create a merge commit for a PR with multiple commits,
+                    # according to the GitHub API, the last commit will be the merge commit,
+                    # which is not what we need when backporting (we need all the commits).
+                    # So here, we are validating the correct SHA for each commit so we can cherry-pick
+                    if promoted_commit.commit.message.startswith(commit_title):
+                        commits.append(promoted_commit.sha)
+
+    elif pr.state == 'closed':
+        events = pr.get_issue_events()
+        for event in events:
+            if event.event == 'closed':
+                commits.append(event.commit_id)
+    return commits
+
+
+def create_pr_comment_and_remove_label(pr, comment_body):
+    labels = pr.get_labels()
+    pattern = re.compile(r"backport/\d+\.\d+$")
+    for label in labels:
+        if pattern.match(label.name):
+            print(f"Removing label: {label.name}")
+            comment_body += f'- {label.name}\n'
+            pr.remove_from_labels(label)
+    pr.create_issue_comment(comment_body)
+
+
+def backport(repo, pr, version, commits, backport_base_branch):
+    new_branch_name = f'backport/{pr.number}/to-{version}'
+    backport_pr_title = f'[Backport {version}] {pr.title}'
+    repo_url = f'https://scylladbbot:{github_token}@github.com/{repo.full_name}.git'
+    fork_repo = f'https://scylladbbot:{github_token}@github.com/scylladbbot/{repo.name}.git'
+    with (tempfile.TemporaryDirectory() as local_repo_path):
+        try:
+            repo_local = Repo.clone_from(repo_url, local_repo_path, branch=backport_base_branch)
+            repo_local.git.checkout(b=new_branch_name)
+            is_draft = False
+            for commit in commits:
+                try:
+                    repo_local.git.cherry_pick(commit, '-m1', '-x')
+                except GitCommandError as e:
+                    logging.warning(f'Cherry-pick conflict on commit {commit}: {e}')
+                    is_draft = True
+                    repo_local.git.add(A=True)
+                    repo_local.git.cherry_pick('--continue')
+            if not repo.private and not repo.has_in_collaborators(pr.user.login):
+                repo.add_to_collaborators(pr.user.login, permission="push")
+                comment = f':warning:  @{pr.user.login} you have been added as collaborator to scylladbbot fork '
+                comment += f'Please check your inbox and approve the invitation, once it is done, please add the backport labels again'
+                create_pr_comment_and_remove_label(pr, comment)
+                return
+            repo_local.git.push(fork_repo, new_branch_name, force=True)
+            create_pull_request(repo, new_branch_name, backport_base_branch, pr, backport_pr_title, commits,
+                                is_draft=is_draft)
+
+        except GitCommandError as e:
+            logging.warning(f"GitCommandError: {e}")
+
+
+def main():
+    args = parse_args()
+    base_branch = args.base_branch.split('/')[2]
+    promoted_label = 'promoted-to-master'
+    repo_name = args.repo
+    if 'scylla-enterprise' in args.repo:
+        promoted_label = 'promoted-to-enterprise'
+    stable_branch = base_branch
+    backport_branch = 'branch-'
+
+    backport_label_pattern = re.compile(r'backport/\d+\.\d+$')
+
+    g = Github(github_token)
+    repo = g.get_repo(repo_name)
+    closed_prs = []
+    start_commit = None
+
+    if args.commits:
+        start_commit, end_commit = args.commits.split('..')
+        commits = repo.compare(start_commit, end_commit).commits
+        for commit in commits:
+            match = re.search(rf"Closes .*#([0-9]+)", commit.commit.message, re.IGNORECASE)
+            if match:
+                pr_number = int(match.group(1))
+                pr = repo.get_pull(pr_number)
+                closed_prs.append(pr)
+    if args.pull_request:
+        start_commit = args.head_commit
+        pr = repo.get_pull(args.pull_request)
+        closed_prs = [pr]
+
+    for pr in closed_prs:
+        labels = [label.name for label in pr.labels]
+        backport_labels = [label for label in labels if backport_label_pattern.match(label)]
+        if promoted_label not in labels:
+            print(f'no {promoted_label} label: {pr.number}')
+            continue
+        if not backport_labels:
+            print(f'no backport label: {pr.number}')
+            continue
+        commits = get_pr_commits(repo, pr, stable_branch, start_commit)
+        logging.info(f"Found PR #{pr.number} with commit {commits} and the following labels: {backport_labels}")
+        for backport_label in backport_labels:
+            version = backport_label.replace('backport/', '')
+            backport_base_branch = backport_label.replace('backport/', backport_branch)
+            backport(repo, pr, version, commits, backport_base_branch)
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/label_promoted_commits.py

@@ -16,13 +16,8 @@ def parser():
     parser = argparse.ArgumentParser()
     parser.add_argument('--repository', type=str, required=True,
                         help='Github repository name (e.g., scylladb/scylladb)')
-    parser.add_argument('--commit_before_merge', type=str, required=True, help='Git commit ID to start labeling from ('
-                                                                               'newest commit).')
-    parser.add_argument('--commit_after_merge', type=str, required=True,
-                        help='Git commit ID to end labeling at (oldest '
-                             'commit, exclusive).')
-    parser.add_argument('--update_issue', type=bool, default=False, help='Set True to update issues when backport was '
-                                                                         'done')
+    parser.add_argument('--commits', type=str, required=True, help='Range of promoted commits.')
+    parser.add_argument('--label', type=str, default='promoted-to-master', help='Label to use')
     parser.add_argument('--ref', type=str, required=True, help='PR target branch')
     return parser.parse_args()
 
@@ -53,10 +48,11 @@ def main():
     target_branch = re.search(r'branch-(\d+\.\d+)', args.ref)
     g = Github(github_token)
     repo = g.get_repo(args.repository, lazy=False)
-    commits = repo.compare(head=args.commit_after_merge, base=args.commit_before_merge)
+    start_commit, end_commit = args.commits.split('..')
+    commits = repo.compare(start_commit, end_commit).commits
     processed_prs = set()
     # Print commit information
-    for commit in commits.commits:
+    for commit in commits:
         print(f'Commit sha is: {commit.sha}')
         match = pr_pattern.search(commit.commit.message)
         if match:
@@ -66,13 +62,13 @@ def main():
             if target_branch:
                 pr = repo.get_pull(pr_number)
                 branch_name = target_branch[1]
-                refs_pr = re.findall(r'Refs (?:#|https.*?)(\d+)', pr.body)
+                refs_pr = re.findall(r'Parent PR: (?:#|https.*?)(\d+)', pr.body)
                 if refs_pr:
                     print(f'branch-{target_branch.group(1)}, pr number is: {pr_number}')
                     # 1. change the backport label of the parent PR to note that
-                    #    we've merge the corresponding backport PR
+                    #    we've merged the corresponding backport PR
                     # 2. close the backport PR and leave a comment on it to note
-                    #    that it has been merged with a certain git commit,
+                    #    that it has been merged with a certain git commit.
                     ref_pr_number = refs_pr[0]
                     mark_backport_done(repo, ref_pr_number, branch_name)
                     comment = f'Closed via {commit.sha}'

.github/workflows/add-label-when-promoted.yaml

@@ -5,9 +5,10 @@ on:
     branches:
       - master
       - branch-*.*
-
-env:
-  DEFAULT_BRANCH: 'master'
+      - enterprise
+    pull_request_target:
+      types: [labeled]
+      branches: [master, next, enterprise]
 
 jobs:
   check-commit:
@@ -16,17 +17,55 @@ jobs:
       pull-requests: write
       issues: write
     steps:
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - name: Set Default Branch
+        id: set_branch
+        run: |
+          if [[ "${{ github.repository }}" == *enterprise* ]]; then
+            echo "DEFAULT_BRANCH=enterprise" >> $GITHUB_ENV
+          else
+            echo "DEFAULT_BRANCH=master" >> $GITHUB_ENV
+          fi
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           repository: ${{ github.repository }}
           ref: ${{ env.DEFAULT_BRANCH }}
+          token: ${{ secrets.AUTO_BACKPORT_TOKEN }}
           fetch-depth: 0  # Fetch all history for all tags and branches
-
+      - name: Set up Git identity
+        run: |
+          git config --global user.name "GitHub Action"
+          git config --global user.email "action@github.com"
+          git config --global merge.conflictstyle diff3
       - name: Install dependencies
-        run: sudo apt-get install -y python3-github
-
+        run: sudo apt-get install -y python3-github python3-git
       - name: Run python script
+        if: github.event_name == 'push'
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: python .github/scripts/label_promoted_commits.py --commit_before_merge ${{ github.event.before }} --commit_after_merge ${{ github.event.after }} --repository ${{ github.repository }} --ref ${{ github.ref }}
+          GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }}
+        run: python .github/scripts/label_promoted_commits.py  --commits ${{ github.event.before }}..${{ github.sha }} --repository ${{ github.repository }} --ref ${{ github.ref }}
+      - name: Run auto-backport.py when promotion completed
+        if: ${{ github.event_name == 'push' && github.ref == format('refs/heads/{0}', env.DEFAULT_BRANCH) }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }}
+        run: python .github/scripts/auto-backport.py --repo ${{ github.repository }} --base-branch ${{ github.ref }} --commits ${{ github.event.before }}..${{ github.sha }}
+      - name: Check if label starts with 'backport/' and contains digits
+        id: check_label
+        run: |
+          label_name="${{ github.event.label.name }}"
+          if [[ "$label_name" =~ ^backport/[0-9]+\.[0-9]+$ ]]; then
+            echo "Label matches backport/X.X pattern."
+            echo "backport_label=true" >> $GITHUB_OUTPUT
+          else
+            echo "Label does not match the required pattern."
+            echo "backport_label=false" >> $GITHUB_OUTPUT
+          fi
+      - name: Run auto-backport.py when label was added
+        if: ${{ github.event_name == 'pull_request_target' && steps.check_label.outputs.backport_label == 'true' && github.event.pull_request.state == 'closed' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }}
+        run: python .github/scripts/auto-backport.py --repo ${{ github.repository }} --base-branch ${{ github.ref }} --pull-request ${{ github.event.pull_request.number }} --head-commit ${{ github.event.pull_request.base.sha }}

.github/workflows/build-scylla.yaml

@@ -0,0 +1,35 @@
+name: Build Scylla
+
+on:
+  workflow_call:
+    inputs:
+      build_mode:
+        description: 'the build mode'
+        type: string
+        required: true
+    outputs:
+      md5sum:
+        description: 'the md5sum for scylla executable'
+        value: ${{ jobs.build.outputs.md5sum }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    # be consistent with tools/toolchain/image
+    container: scylladb/scylla-toolchain:fedora-40-20240621
+    outputs:
+      md5sum: ${{ steps.checksum.outputs.md5sum }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Generate the building system
+        run: |
+          git config --global --add safe.directory $GITHUB_WORKSPACE
+          ./configure.py --mode ${{ inputs.build_mode }} --with scylla
+      - run: |
+          ninja build/${{ inputs.build_mode }}/scylla
+      - id: checksum
+        run: |
+          checksum=$(md5sum build/${{ inputs.build_mode }}/scylla | cut -c -32)
+          echo "md5sum=$checksum" >> $GITHUB_OUTPUT

.github/workflows/clang-nightly.yaml

@@ -0,0 +1,65 @@
+name: clang-nightly
+
+on:
+  schedule:
+    # only at 5AM Saturday
+    - cron: '0 5 * * SAT'
+
+env:
+  # use the development branch explicitly
+  CLANG_VERSION: 19
+  BUILD_DIR: build
+
+permissions: {}
+
+# cancel the in-progress run upon a repush
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  clang-dev:
+    name: Build with clang nightly
+    runs-on: ubuntu-latest
+    container: fedora:40
+    strategy:
+      matrix:
+        build_type:
+          - Debug
+          - RelWithDebInfo
+          - Dev
+    steps:
+      - run: |
+          sudo dnf -y install git
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Install build dependencies
+        run: |
+          # use the copr repo for llvm snapshot builds, see
+          # https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/llvm-snapshots/
+          sudo dnf -y install 'dnf-command(copr)'
+          sudo dnf copr enable -y @fedora-llvm-team/llvm-snapshots
+          # do not install java dependencies, which is not only not used here
+          sed -i.orig \
+            -e '/tools\/.*\/install-dependencies.sh/d' \
+            -e 's/(minio_download_jobs)/(true)/' \
+            ./install-dependencies.sh
+          sudo ./install-dependencies.sh
+          sudo dnf -y install lld
+      - name: Generate the building system
+        run: |
+          cmake                                         \
+            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+            -DCMAKE_C_COMPILER=clang-$CLANG_VERSION     \
+            -DCMAKE_CXX_COMPILER=clang++-$CLANG_VERSION \
+            -G Ninja                                    \
+            -B $BUILD_DIR                               \
+            -S .
+      # see https://github.com/actions/toolkit/blob/main/docs/problem-matchers.md
+      - run: |
+          echo "::add-matcher::.github/clang-matcher.json"
+      - run: |
+          cmake --build $BUILD_DIR --target scylla
+      - run: |
+          echo "::remove-matcher owner=clang::"

.github/workflows/clang-tidy.yaml

@@ -15,8 +15,6 @@ on:
     - cron: '0 5 * * SAT'
 
 env:
-  # use the stable branch
-  CLANG_VERSION: 18
   BUILD_TYPE: RelWithDebInfo
   BUILD_DIR: build
   CLANG_TIDY_CHECKS: '-*,bugprone-use-after-move'
@@ -29,35 +27,41 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  read-toolchain:
+    uses: ./.github/workflows/read-toolchain.yaml
   clang-tidy:
     name: Run clang-tidy
+    needs:
+      - read-toolchain
     runs-on: ubuntu-latest
+    container: ${{ needs.read-toolchain.outputs.image }}
     steps:
+      - env:
+          IMAGE: ${{ needs.read-toolchain.image }}
+        run: |
+          echo ${{ needs.read-toolchain.image }}
       - uses: actions/checkout@v4
         with:
           submodules: true
-      - uses: ./.github/actions/setup-build
-        with:
-          install_clang_tidy: true
+      - run: |
+          sudo dnf -y install clang-tools-extra
       - name: Generate the building system
         run: |
           cmake                                         \
             -DCMAKE_BUILD_TYPE=$BUILD_TYPE              \
-            -DCMAKE_C_COMPILER=clang-$CLANG_VERSION     \
-            -DScylla_USE_LINKER=ld.lld-$CLANG_VERSION   \
-            -DCMAKE_CXX_COMPILER=clang++-$CLANG_VERSION \
+            -DCMAKE_C_COMPILER=clang                    \
+            -DScylla_USE_LINKER=ld.lld                  \
+            -DCMAKE_CXX_COMPILER=clang++                \
             -DCMAKE_EXPORT_COMPILE_COMMANDS=ON          \
-            -DCMAKE_CXX_CLANG_TIDY="clang-tidy-$CLANG_VERSION;--checks=$CLANG_TIDY_CHECKS" \
-            -DCMAKE_CXX_FLAGS=-DFMT_HEADER_ONLY         \
-            -DCMAKE_PREFIX_PATH=$PWD/cooking            \
+            -DCMAKE_CXX_CLANG_TIDY="clang-tidy;--checks=$CLANG_TIDY_CHECKS" \
             -G Ninja                                    \
             -B $BUILD_DIR                               \
             -S .
       # see https://github.com/actions/toolkit/blob/main/docs/problem-matchers.md
       - run: |
-          echo "::add-matcher::.github/clang-tidy-matcher.json"
+          echo "::add-matcher::.github/clang-matcher.json"
       - name: Build with clang-tidy enabled
         run: |
           cmake --build $BUILD_DIR --target scylla
       - run: |
-          echo "::remove-matcher owner=clang-tidy::"
+          echo "::remove-matcher owner=clang::"

.github/workflows/codespell.yaml

@@ -14,4 +14,4 @@ jobs:
         with:
           only_warn: 1
           ignore_words_list: "ans,datas,fo,ser,ue,crate,nd,reenable,strat,stap,te,raison"
-          skip: "./.git,./build,./tools,*.js,*.thrift,*.lock,./test,./licenses,./redis/lolwut.cc,*.svg"
+          skip: "./.git,./build,./tools,*.js,*.lock,./test,./licenses,./redis/lolwut.cc,*.svg"

.github/workflows/iwyu.yaml

@@ -0,0 +1,80 @@
+name: iwyu
+
+on:
+  pull_request:
+    branches:
+      - master
+
+env:
+  BUILD_TYPE: RelWithDebInfo
+  BUILD_DIR: build
+  CLEANER_OUTPUT_PATH: build/clang-include-cleaner.log
+  CLEANER_DIRS: test/unit exceptions alternator api auth cdc compaction
+
+permissions: {}
+
+# cancel the in-progress run upon a repush
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  read-toolchain:
+    uses: ./.github/workflows/read-toolchain.yaml
+  clang-include-cleaner:
+    name: "Analyze #includes in source files"
+    needs:
+      - read-toolchain
+    runs-on: ubuntu-latest
+    container: ${{ needs.read-toolchain.outputs.image }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - run: |
+          sudo dnf -y install clang-tools-extra
+      - name: Generate compilation database
+        run: |
+          cmake                                         \
+            -DCMAKE_BUILD_TYPE=$BUILD_TYPE              \
+            -DCMAKE_C_COMPILER=clang                    \
+            -DCMAKE_CXX_COMPILER=clang++                \
+            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON          \
+            -G Ninja                                    \
+            -B $BUILD_DIR                               \
+            -S .
+      - name: Build headers
+        run: |
+          swagger_targets=''
+          for f in api/api-doc/*.json; do
+            if test "${f#*.}" = json; then
+              name=$(basename "$f" .json)
+              if test $name != swagger20_header; then
+                swagger_targets+=" scylla_swagger_gen_$name"
+              fi
+            fi
+          done
+          cmake                                         \
+            --build build                               \
+             --target seastar_http_request_parser       \
+             --target idl-sources                       \
+             --target $swagger_targets
+      - run: |
+          echo "::add-matcher::.github/clang-include-cleaner.json"
+      - name: clang-include-cleaner
+        run: |
+          for d in $CLEANER_DIRS; do
+            find $d -name '*.cc' -o -name '*.hh'          \
+              -exec echo {} \;                            \
+              -exec clang-include-cleaner                 \
+                --ignore-headers=seastarx.hh              \
+                --print=changes                           \
+                -p $BUILD_DIR                             \
+                {} \; | tee --append $CLEANER_OUTPUT_PATH
+          done
+      - run: |
+          echo "::remove-matcher owner=clang-include-cleaner::"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: Logs (clang-include-cleaner)
+          path: "./${{ env.CLEANER_OUTPUT_PATH }}"

.github/workflows/read-toolchain.yaml

@@ -0,0 +1,23 @@
+name: Read Toolchain
+
+on:
+  workflow_call:
+    outputs:
+      image:
+        description: "the toolchain docker image"
+        value: ${{ jobs.read-toolchain.outputs.image }}
+
+jobs:
+  read-toolchain:
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.read.outputs.image }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: tools/toolchain/image
+          sparse-checkout-cone-mode: false
+      - id: read
+        run: |
+          image=$(cat tools/toolchain/image)
+          echo "image=$image" >> $GITHUB_OUTPUT

.github/workflows/reproducible-build.yaml

@@ -0,0 +1,34 @@
+name: Check Reproducible Build
+
+on:
+  schedule:
+    # 5AM every friday
+    - cron: '0 5 * * FRI'
+
+permissions: {}
+
+env:
+  BUILD_MODE: release
+jobs:
+  build-a:
+    uses: ./.github/workflows/build-scylla.yaml
+    with:
+      build_mode: release
+  build-b:
+    uses: ./.github/workflows/build-scylla.yaml
+    with:
+      build_mode: release
+  compare-checksum:
+    runs-on: ubuntu-latest
+    needs:
+      - build-a
+      - build-b
+    steps:
+      - env:
+          CHECKSUM_A: ${{needs.build-a.outputs.md5sum}}
+          CHECKSUM_B: ${{needs.build-b.outputs.md5sum}}
+        run: |
+          if [ $CHECKSUM_A != $CHECKSUM_B ]; then                             \
+            echo "::error::mismatched checksums: $CHECKSUM_A != $CHECKSUM_B"; \
+            exit 1;                                                           \
+          fi

.github/workflows/seastar.yaml

@@ -0,0 +1,50 @@
+name: Build with the latest Seastar
+
+on:
+  schedule:
+    # 5AM everyday
+    - cron: '0 5 * * *'
+
+permissions: {}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  BUILD_DIR: build
+
+jobs:
+  build-with-the-latest-seastar:
+    runs-on: ubuntu-latest
+    # be consistent with tools/toolchain/image
+    container: scylladb/scylla-toolchain:fedora-40-20240621
+    strategy:
+      matrix:
+        build_type:
+          - Debug
+          - RelWithDebInfo
+          - Dev
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - run: |
+          rm -rf seastar
+      - uses: actions/checkout@v4
+        with:
+          repository: scylladb/seastar
+          submodules: true
+          path: seastar
+      - name: Generate the building system
+        run: |
+          git config --global --add safe.directory $GITHUB_WORKSPACE
+          cmake                                         \
+            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+            -DCMAKE_C_COMPILER=clang                    \
+            -DCMAKE_CXX_COMPILER=clang++                \
+            -G Ninja                                    \
+            -B $BUILD_DIR                               \
+            -S .
+      - run: |
+          cmake --build $BUILD_DIR --target scylla

.github/workflows/sync-labels.yaml

@@ -16,6 +16,10 @@ jobs:
       pull-requests: write
       issues: write
     steps:
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
@@ -33,7 +37,7 @@ jobs:
         run: python .github/scripts/sync_labels.py --repo ${{ github.repository }} --number ${{ github.event.number }} --action ${{ github.event.action }}
 
       - name: Pull request labeled or unlabeled event
-        if: github.event_name == 'pull_request' && startsWith(github.event.label.name, 'backport/')
+        if: github.event_name == 'pull_request_target' && startsWith(github.event.label.name, 'backport/')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: python .github/scripts/sync_labels.py --repo ${{ github.repository }} --number ${{ github.event.number }} --action ${{ github.event.action }} --label ${{ github.event.label.name }}

.gitignore

@@ -3,6 +3,7 @@
 .settings
 build
 build.ninja
+build.ninja.new
 cscope.*
 /debian/
 dist/ami/files/*.rpm

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "seastar"]
 	path = seastar
-	url = ../seastar
+	url = ../scylla-seastar
 	ignore = dirty
 [submodule "swagger-ui"]
 	path = swagger-ui

CMakeLists.txt

@@ -44,12 +44,14 @@ endif()
 
 include(limit_jobs)
 # Configure Seastar compile options to align with Scylla
-set(CMAKE_CXX_STANDARD "20" CACHE INTERNAL "")
+set(CMAKE_CXX_STANDARD "23" CACHE INTERNAL "")
 set(CMAKE_CXX_EXTENSIONS ON CACHE INTERNAL "")
+set(CMAKE_CXX_SCAN_FOR_MODULES OFF CACHE INTERNAL "")
 set(CMAKE_CXX_VISIBILITY_PRESET hidden)
 
 set(Seastar_TESTING ON CACHE BOOL "" FORCE)
 set(Seastar_API_LEVEL 7 CACHE STRING "" FORCE)
+set(Seastar_DEPRECATED_OSTREAM_FORMATTERS OFF CACHE BOOL "" FORCE)
 set(Seastar_APPS ON CACHE BOOL "" FORCE)
 set(Seastar_EXCLUDE_APPS_FROM_ALL ON CACHE BOOL "" FORCE)
 set(Seastar_EXCLUDE_TESTS_FROM_ALL ON CACHE BOOL "" FORCE)
@@ -98,8 +100,8 @@ find_package(libdeflate REQUIRED)
 find_package(libxcrypt REQUIRED)
 find_package(Snappy REQUIRED)
 find_package(RapidJSON REQUIRED)
-find_package(Thrift REQUIRED)
 find_package(xxHash REQUIRED)
+find_package(zstd REQUIRED)
 
 set(scylla_gen_build_dir "${CMAKE_BINARY_DIR}/gen")
 file(MAKE_DIRECTORY "${scylla_gen_build_dir}")
@@ -107,6 +109,14 @@ file(MAKE_DIRECTORY "${scylla_gen_build_dir}")
 include(add_version_library)
 generate_scylla_version()
 
+add_library(scylla-zstd STATIC
+    zstd.cc)
+target_link_libraries(scylla-zstd
+  PRIVATE
+    db
+    Seastar::seastar
+    zstd::libzstd)
+
 add_library(scylla-main STATIC)
 target_sources(scylla-main
   PRIVATE
@@ -145,10 +155,10 @@ target_sources(scylla-main
     timeout_config.cc
     unimplemented.cc
     validation.cc
-    vint-serialization.cc
-    zstd.cc)
+    vint-serialization.cc)
 target_link_libraries(scylla-main
   PRIVATE
+    "$<LINK_LIBRARY:WHOLE_ARCHIVE,scylla-zstd>"
     db
     absl::headers
     absl::btree
@@ -196,7 +206,6 @@ add_subdirectory(dht)
 add_subdirectory(gms)
 add_subdirectory(idl)
 add_subdirectory(index)
-add_subdirectory(interface)
 add_subdirectory(lang)
 add_subdirectory(locator)
 add_subdirectory(message)
@@ -214,7 +223,6 @@ add_subdirectory(service)
 add_subdirectory(sstables)
 add_subdirectory(streaming)
 add_subdirectory(test)
-add_subdirectory(thrift)
 add_subdirectory(tools)
 add_subdirectory(tracing)
 add_subdirectory(transport)
@@ -255,7 +263,6 @@ target_link_libraries(scylla PRIVATE
     sstables
     streaming
     test-perf
-    thrift
     tools
     tracing
     transport

HACKING.md

@@ -199,7 +199,7 @@ The `scylla.yaml` file in the repository by default writes all database data to
 
 Scylla has a number of requirements for the file-system and operating system to operate ideally and at peak performance. However, during development, these requirements can be relaxed with the `--developer-mode` flag.
 
-Additionally, when running on under-powered platforms like portable laptops, the `--overprovisined` flag is useful.
+Additionally, when running on under-powered platforms like portable laptops, the `--overprovisioned` flag is useful.
 
 On a development machine, one might run Scylla as
 

README.md

@@ -65,11 +65,13 @@ $ ./tools/toolchain/dbuild ./build/release/scylla --help
 
 ## Testing
 
+[![Build with the latest Seastar](https://github.com/scylladb/scylladb/actions/workflows/seastar.yaml/badge.svg)](https://github.com/scylladb/scylladb/actions/workflows/seastar.yaml) [![Check Reproducible Build](https://github.com/scylladb/scylladb/actions/workflows/reproducible-build.yaml/badge.svg)](https://github.com/scylladb/scylladb/actions/workflows/reproducible-build.yaml) [![clang-nightly](https://github.com/scylladb/scylladb/actions/workflows/clang-nightly.yaml/badge.svg)](https://github.com/scylladb/scylladb/actions/workflows/clang-nightly.yaml)
+
 See [test.py manual](docs/dev/testing.md).
 
 ## Scylla APIs and compatibility
-By default, Scylla is compatible with Apache Cassandra and its APIs - CQL and
-Thrift. There is also support for the API of Amazon DynamoDB™,
+By default, Scylla is compatible with Apache Cassandra and its API - CQL.
+There is also support for the API of Amazon DynamoDB™,
 which needs to be enabled and configured in order to be used. For more
 information on how to enable the DynamoDB™ API in Scylla,
 and the current compatibility of this feature as well as Scylla-specific extensions, see

SCYLLA-VERSION-GEN

@@ -78,7 +78,7 @@ fi
 
 # Default scylla product/version tags
 PRODUCT=scylla
-VERSION=6.0.5
+VERSION=6.1.6
 
 if test -f version
 then
@@ -104,7 +104,7 @@ else
 fi
 
 if [ -f "$OUTPUT_DIR/SCYLLA-RELEASE-FILE" ]; then
-	GIT_COMMIT_FILE=$(cat "$OUTPUT_DIR/SCYLLA-RELEASE-FILE" |cut -d . -f 3)
+	GIT_COMMIT_FILE=$(cat "$OUTPUT_DIR/SCYLLA-RELEASE-FILE" | rev | cut -d . -f 1 | rev)
 	if [ "$GIT_COMMIT" = "$GIT_COMMIT_FILE" ]; then
 		exit 0
 	fi

alternator/auth.cc

@@ -19,6 +19,7 @@
 #include "alternator/executor.hh"
 #include "cql3/selection/selection.hh"
 #include "cql3/result_set.hh"
+#include "types/types.hh"
 #include <seastar/core/coroutine.hh>
 
 namespace alternator {
@@ -31,11 +32,12 @@ future<std::string> get_key_from_roles(service::storage_proxy& proxy, auth::serv
     dht::partition_range_vector partition_ranges{dht::partition_range(dht::decorate_key(*schema, pk))};
     std::vector<query::clustering_range> bounds{query::clustering_range::make_open_ended_both_sides()};
     const column_definition* salted_hash_col = schema->get_column_definition(bytes("salted_hash"));
-    if (!salted_hash_col) {
+    const column_definition* can_login_col = schema->get_column_definition(bytes("can_login"));
+    if (!salted_hash_col || !can_login_col) {
         co_await coroutine::return_exception(api_error::unrecognized_client(format("Credentials cannot be fetched for: {}", username)));
     }
-    auto selection = cql3::selection::selection::for_columns(schema, {salted_hash_col});
-    auto partition_slice = query::partition_slice(std::move(bounds), {}, query::column_id_vector{salted_hash_col->id}, selection->get_query_options());
+    auto selection = cql3::selection::selection::for_columns(schema, {salted_hash_col, can_login_col});
+    auto partition_slice = query::partition_slice(std::move(bounds), {}, query::column_id_vector{salted_hash_col->id, can_login_col->id}, selection->get_query_options());
     auto command = ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice,
             proxy.get_max_result_size(partition_slice), query::tombstone_limit(proxy.get_tombstone_limit()));
     auto cl = auth::password_authenticator::consistency_for_user(username);
@@ -51,7 +53,14 @@ future<std::string> get_key_from_roles(service::storage_proxy& proxy, auth::serv
     if (result_set->empty()) {
         co_await coroutine::return_exception(api_error::unrecognized_client(format("User not found: {}", username)));
     }
-    const managed_bytes_opt& salted_hash = result_set->rows().front().front(); // We only asked for 1 row and 1 column
+    const auto& result = result_set->rows().front();
+    bool can_login = result[1] && value_cast<bool>(boolean_type->deserialize(*result[1]));
+    if (!can_login) {
+        // This is a valid role name, but has "login=False" so should not be
+        // usable for authentication (see #19735).
+        co_await coroutine::return_exception(api_error::unrecognized_client(format("Role {} has login=false so cannot be used for login", username)));
+    }
+    const managed_bytes_opt& salted_hash = result.front();
     if (!salted_hash) {
         co_await coroutine::return_exception(api_error::unrecognized_client(format("No password found for user: {}", username)));
     }

alternator/controller.cc

@@ -32,8 +32,10 @@ controller::controller(
         sharded<service::memory_limiter>& memory_limiter,
         sharded<auth::service>& auth_service,
         sharded<qos::service_level_controller>& sl_controller,
-        const db::config& config)
-    : _gossiper(gossiper)
+        const db::config& config,
+        seastar::scheduling_group sg)
+    : protocol_server(sg)
+    , _gossiper(gossiper)
     , _proxy(proxy)
     , _mm(mm)
     , _sys_dist_ks(sys_dist_ks)
@@ -62,7 +64,9 @@ std::vector<socket_address> controller::listen_addresses() const {
 }
 
 future<> controller::start_server() {
-    return seastar::async([this] {
+    seastar::thread_attributes attr;
+    attr.sched_group = _sched_group;
+    return seastar::async(std::move(attr), [this] {
         _listen_addresses.clear();
 
         auto preferred = _config.listen_interface_prefer_ipv6() ? std::make_optional(net::inet_address::family::INET6) : std::nullopt;
@@ -156,7 +160,9 @@ future<> controller::stop_server() {
 }
 
 future<> controller::request_stop_server() {
-    return stop_server();
+    return with_scheduling_group(_sched_group, [this] {
+        return stop_server();
+    });
 }
 
 }

alternator/controller.hh

@@ -80,7 +80,8 @@ public:
         sharded<service::memory_limiter>& memory_limiter,
         sharded<auth::service>& auth_service,
         sharded<qos::service_level_controller>& sl_controller,
-        const db::config& config);
+        const db::config& config,
+        seastar::scheduling_group sg);
 
     virtual sstring name() const override;
     virtual sstring protocol() const override;

alternator/executor.cc

@@ -1252,7 +1252,7 @@ future<executor::request_return_type> executor::update_table(client_state& clien
 
         auto schema = builder.build();
 
-        auto m = co_await service::prepare_column_family_update_announcement(p.local(), schema, false,  std::vector<view_ptr>(), group0_guard.write_timestamp());
+        auto m = co_await service::prepare_column_family_update_announcement(p.local(), schema,  std::vector<view_ptr>(), group0_guard.write_timestamp());
 
         co_await mm.announce(std::move(m), std::move(group0_guard), format("alternator-executor: update {} table", tab->cf_name()));
 

alternator/executor.hh

@@ -9,7 +9,6 @@
 #pragma once
 
 #include <seastar/core/future.hh>
-#include <seastar/http/httpd.hh>
 #include "seastarx.hh"
 #include <seastar/json/json_elements.hh>
 #include <seastar/core/sharded.hh>

alternator/expressions.cc

@@ -28,7 +28,7 @@
 
 namespace alternator {
 
-template <typename Func, typename Result = std::result_of_t<Func(expressionsParser&)>>
+template <typename Func, typename Result = std::invoke_result_t<Func, expressionsParser&>>
 static Result do_with_parser(std::string_view input, Func&& f) {
     expressionsLexer::InputStreamType input_stream{
         reinterpret_cast<const ANTLR_UINT8*>(input.data()),
@@ -43,7 +43,7 @@ static Result do_with_parser(std::string_view input, Func&& f) {
     return result;
 }
 
-template <typename Func, typename Result = std::result_of_t<Func(expressionsParser&)>>
+template <typename Func, typename Result = std::invoke_result_t<Func, expressionsParser&>>
 static Result parse(const char* input_name, std::string_view input, Func&& f) {
     if (input.length() > 4096) {
         throw expressions_syntax_error(format("{} expression size {} exceeds allowed maximum 4096.",

alternator/expressions_types.hh

@@ -66,7 +66,6 @@ public:
     std::vector<std::variant<std::string, unsigned>>& operators() {
         return _operators;
     }
-    friend std::ostream& operator<<(std::ostream&, const path&);
 };
 
 // When an expression is first parsed, all constants are references, like

alternator/server.cc

@@ -21,6 +21,8 @@
 #include "utils/rjson.hh"
 #include "auth.hh"
 #include <cctype>
+#include <string_view>
+#include <utility>
 #include "service/storage_proxy.hh"
 #include "gms/gossiper.hh"
 #include "utils/overloaded_functor.hh"
@@ -34,8 +36,6 @@ using reply = http::reply;
 
 namespace alternator {
 
-static constexpr auto TARGET = "X-Amz-Target";
-
 inline std::vector<std::string_view> split(std::string_view text, char separator) {
     std::vector<std::string_view> tokens;
     if (text == "") {
@@ -213,9 +213,11 @@ protected:
         for (auto& ip : local_dc_nodes) {
             // Note that it's not enough for the node to be is_alive() - a
             // node joining the cluster is also "alive" but not responsive to
-            // requests. We need the node to be in normal state. See #19694.
-            if (_gossiper.is_normal(ip)) {
-                rjson::push_back(results, rjson::from_string(fmt::to_string(ip)));
+            // requests. We alive *and* normal. See #19694, #21538.
+            if (_gossiper.is_alive(ip) && _gossiper.is_normal(ip)) {
+                // Use the gossiped broadcast_rpc_address if available instead
+                // of the internal IP address "ip". See discussion in #18711.
+                rjson::push_back(results, rjson::from_string(_gossiper.get_rpc_address(ip)));
             }
         }
         rep->set_status(reply::status_type::ok);
@@ -388,10 +390,10 @@ static tracing::trace_state_ptr maybe_trace_query(service::client_state& client_
 
 future<executor::request_return_type> server::handle_api_request(std::unique_ptr<request> req) {
     _executor._stats.total_operations++;
-    sstring target = req->get_header(TARGET);
-    std::vector<std::string_view> split_target = split(target, '.');
-    //NOTICE(sarna): Target consists of Dynamo API version followed by a dot '.' and operation type (e.g. CreateTable)
-    std::string op = split_target.empty() ? std::string() : std::string(split_target.back());
+    sstring target = req->get_header("X-Amz-Target");
+    // target is DynamoDB API version followed by a dot '.' and operation type (e.g. CreateTable)
+    auto dot = target.find('.');
+    std::string_view op = (dot == sstring::npos) ? std::string_view() : std::string_view(target).substr(dot+1);
     // JSON parsing can allocate up to roughly 2x the size of the raw
     // document, + a couple of bytes for maintenance.
     // TODO: consider the case where req->content_length is missing. Maybe
@@ -637,7 +639,7 @@ future<> server::json_parser::stop() {
 
 const char* api_error::what() const noexcept {
     if (_what_string.empty()) {
-        _what_string = format("{} {}: {}", static_cast<int>(_http_code), _type, _msg);
+        _what_string = format("{} {}: {}", std::to_underlying(_http_code), _type, _msg);
     }
     return _what_string.c_str();
 }

alternator/stats.hh

@@ -11,7 +11,6 @@
 #include <cstdint>
 
 #include <seastar/core/metrics_registration.hh>
-#include "utils/estimated_histogram.hh"
 #include "utils/histogram.hh"
 #include "cql3/stats.hh"
 

alternator/streams.cc

@@ -233,11 +233,8 @@ struct shard_id {
 
     // dynamo specifies shardid as max 65 chars. 
     friend std::ostream& operator<<(std::ostream& os, const shard_id& id) {
-        boost::io::ios_flags_saver fs(os);
-        return os << marker << std::hex  
-            << id.time.time_since_epoch().count()
-            << ':' << id.id.to_bytes()
-            ;
+        fmt::print(os, "{} {:x}:{}", marker, id.time.time_since_epoch().count(), id.id.to_bytes());
+        return os;
     }
 };
 
@@ -779,7 +776,7 @@ struct event_id {
     cdc::stream_id stream;
     utils::UUID timestamp;
 
-    static const auto marker = 'E';
+    static constexpr auto marker = 'E';
 
     event_id(cdc::stream_id s, utils::UUID ts)
         : stream(s)
@@ -787,10 +784,8 @@ struct event_id {
     {}
     
     friend std::ostream& operator<<(std::ostream& os, const event_id& id) {
-        boost::io::ios_flags_saver fs(os);
-        return os << marker << std::hex << id.stream.to_bytes()
-            << ':' << id.timestamp
-            ;
+        fmt::print(os, "{}{}:{}", marker, id.stream.to_bytes(), id.timestamp);
+        return os;
     }
 };
 }

alternator/ttl.cc

@@ -313,7 +313,7 @@ static size_t random_offset(size_t min, size_t max) {
 // this range's primary node is down. For this we need to return not just
 // a list of this node's secondary ranges - but also the primary owner of
 // each of those ranges.
-static std::vector<std::pair<dht::token_range, gms::inet_address>> get_secondary_ranges(
+static future<std::vector<std::pair<dht::token_range, gms::inet_address>>> get_secondary_ranges(
         const locator::effective_replication_map_ptr& erm,
         gms::inet_address ep) {
     const auto& tm = *erm->get_token_metadata_ptr();
@@ -324,6 +324,7 @@ static std::vector<std::pair<dht::token_range, gms::inet_address>> get_secondary
     }
     auto prev_tok = sorted_tokens.back();
     for (const auto& tok : sorted_tokens) {
+        co_await coroutine::maybe_yield();
         inet_address_vector_replica_set eps = erm->get_natural_endpoints(tok);
         if (eps.size() <= 1 || eps[1] != ep) {
             prev_tok = tok;
@@ -351,7 +352,7 @@ static std::vector<std::pair<dht::token_range, gms::inet_address>> get_secondary
         }
         prev_tok = tok;
     }
-    return ret;
+    co_return ret;
 }
 
 
@@ -387,63 +388,63 @@ static std::vector<std::pair<dht::token_range, gms::inet_address>> get_secondary
 //
 // FIXME: Check if this algorithm is safe with tablet migration.
 // https://github.com/scylladb/scylladb/issues/16567
-enum primary_or_secondary_t {primary, secondary};
-template<primary_or_secondary_t primary_or_secondary>
-class token_ranges_owned_by_this_shard {
-    // ranges_holder_primary holds just the primary ranges themselves
-    class ranges_holder_primary {
-        const dht::token_range_vector _token_ranges;
-     public:
-        ranges_holder_primary(const locator::vnode_effective_replication_map_ptr& erm, gms::gossiper& g, gms::inet_address ep)
-            : _token_ranges(erm->get_primary_ranges(ep)) {}
-        std::size_t size() const { return _token_ranges.size(); }
-        const dht::token_range& operator[](std::size_t i) const {
-            return _token_ranges[i];
-        }
-        bool should_skip(std::size_t i) const {
-            return false;
-        }
-    };
-    // ranges_holder<secondary> holds the secondary token ranges plus each
-    // range's primary owner, needed to implement should_skip().
-    class ranges_holder_secondary {
-        std::vector<std::pair<dht::token_range, gms::inet_address>> _token_ranges;
-        gms::gossiper& _gossiper;
-     public:
-        ranges_holder_secondary(const locator::effective_replication_map_ptr& erm, gms::gossiper& g, gms::inet_address ep)
-            : _token_ranges(get_secondary_ranges(erm, ep))
-            , _gossiper(g) {}
-        std::size_t size() const { return _token_ranges.size(); }
-        const dht::token_range& operator[](std::size_t i) const {
-            return _token_ranges[i].first;
-        }
-        // range i should be skipped if its primary owner is alive.
-        bool should_skip(std::size_t i) const {
-            return _gossiper.is_alive(_token_ranges[i].second);
-        }
-    };
 
+// ranges_holder_primary holds just the primary ranges themselves
+class ranges_holder_primary {
+    dht::token_range_vector _token_ranges;
+public:
+    explicit ranges_holder_primary(dht::token_range_vector token_ranges) : _token_ranges(std::move(token_ranges)) {}
+    static future<ranges_holder_primary> make(const locator::vnode_effective_replication_map_ptr& erm, gms::inet_address ep) {
+        co_return ranges_holder_primary(co_await erm->get_primary_ranges(ep));
+    }
+    std::size_t size() const { return _token_ranges.size(); }
+    const dht::token_range& operator[](std::size_t i) const {
+        return _token_ranges[i];
+    }
+    bool should_skip(std::size_t i) const {
+        return false;
+    }
+};
+// ranges_holder<secondary> holds the secondary token ranges plus each
+// range's primary owner, needed to implement should_skip().
+class ranges_holder_secondary {
+    std::vector<std::pair<dht::token_range, gms::inet_address>> _token_ranges;
+    const gms::gossiper& _gossiper;
+public:
+    explicit ranges_holder_secondary(std::vector<std::pair<dht::token_range, gms::inet_address>> token_ranges, const gms::gossiper& g)
+        : _token_ranges(std::move(token_ranges))
+        , _gossiper(g) {}
+    static future<ranges_holder_secondary> make(const locator::effective_replication_map_ptr& erm, gms::inet_address ep, const gms::gossiper& g) {
+        co_return ranges_holder_secondary(co_await get_secondary_ranges(erm, ep), g);
+    }
+    std::size_t size() const { return _token_ranges.size(); }
+    const dht::token_range& operator[](std::size_t i) const {
+        return _token_ranges[i].first;
+    }
+    // range i should be skipped if its primary owner is alive.
+    bool should_skip(std::size_t i) const {
+        return _gossiper.is_alive(_token_ranges[i].second);
+    }
+};
+
+template<class primary_or_secondary_t>
+class token_ranges_owned_by_this_shard {
     schema_ptr _s;
     locator::effective_replication_map_ptr _erm;
     // _token_ranges will contain a list of token ranges owned by this node.
     // We'll further need to split each such range to the pieces owned by
     // the current shard, using _intersecter.
-    using ranges_holder = std::conditional_t<
-            primary_or_secondary == primary_or_secondary_t::primary,
-            ranges_holder_primary,
-            ranges_holder_secondary>;
-    const ranges_holder _token_ranges;
+    const primary_or_secondary_t _token_ranges;
     // NOTICE: _range_idx is used modulo _token_ranges size when accessing
     // the data to ensure that it doesn't go out of bounds
     size_t _range_idx;
     size_t _end_idx;
     std::optional<dht::selective_token_range_sharder> _intersecter;
 public:
-    token_ranges_owned_by_this_shard(replica::database& db, gms::gossiper& g, schema_ptr s)
+    token_ranges_owned_by_this_shard(schema_ptr s, primary_or_secondary_t token_ranges)
         :  _s(s)
         , _erm(s->table().get_effective_replication_map())
-        , _token_ranges(db.find_keyspace(s->ks_name()).get_vnode_effective_replication_map(),
-                g, _erm->get_topology().my_address())
+        , _token_ranges(std::move(token_ranges))
         , _range_idx(random_offset(0, _token_ranges.size() - 1))
         , _end_idx(_range_idx + _token_ranges.size())
     {
@@ -726,7 +727,9 @@ static future<bool> scan_table(
     expiration_stats.scan_table++;
     // FIXME: need to pace the scan, not do it all at once.
     scan_ranges_context scan_ctx{s, proxy, std::move(column_name), std::move(member)};
-    token_ranges_owned_by_this_shard<primary> my_ranges(db.real_database(), gossiper, s);
+    auto erm = db.real_database().find_keyspace(s->ks_name()).get_vnode_effective_replication_map();
+    auto my_address = erm->get_topology().my_address();
+    token_ranges_owned_by_this_shard my_ranges(s, co_await ranges_holder_primary::make(erm, my_address));
     while (std::optional<dht::partition_range> range = my_ranges.next_partition_range()) {
         // Note that because of issue #9167 we need to run a separate
         // query on each partition range, and can't pass several of
@@ -746,7 +749,7 @@ static future<bool> scan_table(
     // by tasking another node to take over scanning of the dead node's primary
     // ranges. What we do here is that this node will also check expiration
     // on its *secondary* ranges - but only those whose primary owner is down.
-    token_ranges_owned_by_this_shard<secondary> my_secondary_ranges(db.real_database(), gossiper, s);
+    token_ranges_owned_by_this_shard my_secondary_ranges(s, co_await ranges_holder_secondary::make(erm, my_address, gossiper));
     while (std::optional<dht::partition_range> range = my_secondary_ranges.next_partition_range()) {
         expiration_stats.secondary_ranges_scanned++;
         dht::partition_range_vector partition_ranges;

api/api-doc/collectd.json

@@ -67,7 +67,7 @@
                "parameters":[
                   {
                      "name":"pluginid",
-                     "description":"The plugin ID, describe the component the metric belongs to. Examples are cache, thrift, etc'. Regex are supported.The plugin ID, describe the component the metric belong to. Examples are: cache, thrift etc'. regex are supported",
+                     "description":"The plugin ID, describe the component the metric belongs to. Examples are cache and alternator, etc'. Regex are supported.",
                      "required":true,
                      "allowMultiple":false,
                      "type":"string",
@@ -199,4 +199,4 @@
          }
       }
    }
-}
+}

api/api-doc/raft.json

@@ -62,6 +62,38 @@
                ]
             }
          ]
+      },
+      {
+         "path": "/raft/read_barrier",
+         "operations": [
+            {
+               "method": "POST",
+               "summary": "Triggers read barrier for the given Raft group to wait for previously committed commands in this group to be applied locally. For example, can be used on group 0 to wait for the node to obtain latest schema changes.",
+               "type": "string",
+               "nickname": "read_barrier",
+               "produces": [
+                  "application/json"
+               ],
+               "parameters": [
+                  {
+                     "name": "group_id",
+                     "description": "The ID of the group. When absent, group0 is used.",
+                     "required": false,
+                     "allowMultiple": false,
+                     "type": "string",
+                     "paramType": "query"
+                  },
+                  {
+                     "name": "timeout",
+                     "description": "Timeout in seconds after which the endpoint returns a failure. If not provided, 60s is used.",
+                     "required": false,
+                     "allowMultiple": false,
+                     "type": "long",
+                     "paramType": "query"
+                  }
+               ]
+            }
+         ]
       }
    ]
 }

api/api-doc/storage_service.json

@@ -1689,33 +1689,11 @@
       {
          "path":"/storage_service/rpc_server",
          "operations":[
-            {
-               "method":"DELETE",
-               "summary":"Allows a user to disable thrift",
-               "type":"void",
-               "nickname":"stop_rpc_server",
-               "produces":[
-                  "application/json"
-               ],
-               "parameters":[
-               ]
-            },
-            {
-               "method":"POST",
-               "summary":"allows a user to re-enable thrift",
-               "type":"void",
-               "nickname":"start_rpc_server",
-               "produces":[
-                  "application/json"
-               ],
-               "parameters":[
-               ]
-            },
             {
                "method":"GET",
                "summary":"Determine if thrift is running",
                "type":"boolean",
-               "nickname":"is_rpc_server_running",
+               "nickname":"is_thrift_server_running",
                "produces":[
                   "application/json"
                ],
@@ -2078,7 +2056,7 @@
          "operations":[
             {
                "method":"POST",
-               "summary":"Enables/Disables tracing for the whole system. Only thrift requests can start tracing currently",
+               "summary":"Enables/Disables tracing for the whole system.",
                "type":"void",
                "nickname":"set_trace_probability",
                "produces":[

api/api-doc/utils.json

@@ -75,7 +75,7 @@
                "items":{
                   "type":"double"
                },
-               "description":"One, five and fifteen mintues rates"
+               "description":"One, five and fifteen minutes rates"
             },
             "mean_rate": {
                "type":"double",

api/api.cc

@@ -71,6 +71,8 @@ future<> set_server_init(http_context& ctx) {
         rb->register_function(r, "error_injection",
             "The error injection API");
         set_error_injection(ctx, r);
+        rb->register_function(r, "storage_proxy",
+                "The storage proxy API");
     });
 }
 
@@ -81,6 +83,10 @@ future<> set_server_config(http_context& ctx, const db::config& cfg) {
     });
 }
 
+future<> unset_server_config(http_context& ctx) {
+    return ctx.http_server.set_routes([&ctx] (routes& r) { unset_config(ctx, r); });
+}
+
 static future<> register_api(http_context& ctx, const sstring& api_name,
         const sstring api_desc,
         std::function<void(http_context& ctx, routes& r)> f) {
@@ -100,12 +106,12 @@ future<> unset_transport_controller(http_context& ctx) {
     return ctx.http_server.set_routes([&ctx] (routes& r) { unset_transport_controller(ctx, r); });
 }
 
-future<> set_rpc_controller(http_context& ctx, thrift_controller& ctl) {
-    return ctx.http_server.set_routes([&ctx, &ctl] (routes& r) { set_rpc_controller(ctx, r, ctl); });
+future<> set_thrift_controller(http_context& ctx) {
+    return ctx.http_server.set_routes([&ctx] (routes& r) { set_thrift_controller(ctx, r); });
 }
 
-future<> unset_rpc_controller(http_context& ctx) {
-    return ctx.http_server.set_routes([&ctx] (routes& r) { unset_rpc_controller(ctx, r); });
+future<> unset_thrift_controller(http_context& ctx) {
+    return ctx.http_server.set_routes([&ctx] (routes& r) { unset_thrift_controller(ctx, r); });
 }
 
 future<> set_server_storage_service(http_context& ctx, sharded<service::storage_service>& ss, service::raft_group0_client& group0_client) {
@@ -118,6 +124,14 @@ future<> unset_server_storage_service(http_context& ctx) {
     return ctx.http_server.set_routes([&ctx] (routes& r) { unset_storage_service(ctx, r); });
 }
 
+future<> set_load_meter(http_context& ctx, service::load_meter& lm) {
+    return ctx.http_server.set_routes([&ctx, &lm] (routes& r) { set_load_meter(ctx, r, lm); });
+}
+
+future<> unset_load_meter(http_context& ctx) {
+    return ctx.http_server.set_routes([&ctx] (routes& r) { unset_load_meter(ctx, r); });
+}
+
 future<> set_server_sstables_loader(http_context& ctx, sharded<sstables_loader>& sst_loader) {
     return ctx.http_server.set_routes([&ctx, &sst_loader] (routes& r) { set_sstables_loader(ctx, r, sst_loader); });
 }
@@ -180,10 +194,21 @@ future<> unset_server_snitch(http_context& ctx) {
 }
 
 future<> set_server_gossip(http_context& ctx, sharded<gms::gossiper>& g) {
-    return register_api(ctx, "gossiper",
+    co_await register_api(ctx, "gossiper",
                 "The gossiper API", [&g] (http_context& ctx, routes& r) {
                     set_gossiper(ctx, r, g.local());
                 });
+    co_await register_api(ctx, "failure_detector",
+                "The failure detector API", [&g] (http_context& ctx, routes& r) {
+                    set_failure_detector(ctx, r, g.local());
+                });
+}
+
+future<> unset_server_gossip(http_context& ctx) {
+    return ctx.http_server.set_routes([&ctx] (routes& r) {
+        unset_gossiper(ctx, r);
+        unset_failure_detector(ctx, r);
+    });
 }
 
 future<> set_server_column_family(http_context& ctx, sharded<db::system_keyspace>& sys_ks) {
@@ -208,10 +233,7 @@ future<> unset_server_messaging_service(http_context& ctx) {
 }
 
 future<> set_server_storage_proxy(http_context& ctx, sharded<service::storage_proxy>& proxy) {
-    return register_api(ctx, "storage_proxy",
-                "The storage proxy API", [&proxy] (http_context& ctx, routes& r) {
-                    set_storage_proxy(ctx, r, proxy);
-                });
+    return ctx.http_server.set_routes([&ctx, &proxy] (routes& r) { set_storage_proxy(ctx, r, proxy); });
 }
 
 future<> unset_server_storage_proxy(http_context& ctx) {
@@ -245,16 +267,6 @@ future<> unset_hinted_handoff(http_context& ctx) {
     return ctx.http_server.set_routes([&ctx] (routes& r) { unset_hinted_handoff(ctx, r); });
 }
 
-future<> set_server_gossip_settle(http_context& ctx, sharded<gms::gossiper>& g) {
-    auto rb = std::make_shared < api_registry_builder > (ctx.api_doc);
-
-    return ctx.http_server.set_routes([rb, &ctx, &g](routes& r) {
-        rb->register_function(r, "failure_detector",
-                "The failure detector API");
-        set_failure_detector(ctx, r, g.local());
-    });
-}
-
 future<> set_server_compaction_manager(http_context& ctx) {
     auto rb = std::make_shared < api_registry_builder > (ctx.api_doc);
 

api/api_init.hh

@@ -46,7 +46,6 @@ class snitch_ptr;
 } // namespace locator
 
 namespace cql_transport { class controller; }
-class thrift_controller;
 namespace db {
 class snapshot_ctl;
 class config;
@@ -77,17 +76,16 @@ struct http_context {
     sstring api_doc;
     httpd::http_server_control http_server;
     distributed<replica::database>& db;
-    service::load_meter& lmeter;
 
-    http_context(distributed<replica::database>& _db,
-            service::load_meter& _lm)
-            : db(_db), lmeter(_lm)
+    http_context(distributed<replica::database>& _db)
+            : db(_db)
     {
     }
 };
 
 future<> set_server_init(http_context& ctx);
 future<> set_server_config(http_context& ctx, const db::config& cfg);
+future<> unset_server_config(http_context& ctx);
 future<> set_server_snitch(http_context& ctx, sharded<locator::snitch_ptr>& snitch);
 future<> unset_server_snitch(http_context& ctx);
 future<> set_server_storage_service(http_context& ctx, sharded<service::storage_service>& ss, service::raft_group0_client&);
@@ -100,8 +98,8 @@ future<> set_server_repair(http_context& ctx, sharded<repair_service>& repair);
 future<> unset_server_repair(http_context& ctx);
 future<> set_transport_controller(http_context& ctx, cql_transport::controller& ctl);
 future<> unset_transport_controller(http_context& ctx);
-future<> set_rpc_controller(http_context& ctx, thrift_controller& ctl);
-future<> unset_rpc_controller(http_context& ctx);
+future<> set_thrift_controller(http_context& ctx);
+future<> unset_thrift_controller(http_context& ctx);
 future<> set_server_authorization_cache(http_context& ctx, sharded<auth::service> &auth_service);
 future<> unset_server_authorization_cache(http_context& ctx);
 future<> set_server_snapshot(http_context& ctx, sharded<db::snapshot_ctl>& snap_ctl);
@@ -109,6 +107,7 @@ future<> unset_server_snapshot(http_context& ctx);
 future<> set_server_token_metadata(http_context& ctx, sharded<locator::shared_token_metadata>& tm);
 future<> unset_server_token_metadata(http_context& ctx);
 future<> set_server_gossip(http_context& ctx, sharded<gms::gossiper>& g);
+future<> unset_server_gossip(http_context& ctx);
 future<> set_server_column_family(http_context& ctx, sharded<db::system_keyspace>& sys_ks);
 future<> unset_server_column_family(http_context& ctx);
 future<> set_server_messaging_service(http_context& ctx, sharded<netw::messaging_service>& ms);
@@ -119,7 +118,6 @@ future<> set_server_stream_manager(http_context& ctx, sharded<streaming::stream_
 future<> unset_server_stream_manager(http_context& ctx);
 future<> set_hinted_handoff(http_context& ctx, sharded<service::storage_proxy>& p);
 future<> unset_hinted_handoff(http_context& ctx);
-future<> set_server_gossip_settle(http_context& ctx, sharded<gms::gossiper>& g);
 future<> set_server_cache(http_context& ctx);
 future<> set_server_compaction_manager(http_context& ctx);
 future<> set_server_done(http_context& ctx);
@@ -131,5 +129,7 @@ future<> set_server_tasks_compaction_module(http_context& ctx, sharded<service::
 future<> unset_server_tasks_compaction_module(http_context& ctx);
 future<> set_server_raft(http_context&, sharded<service::raft_group_registry>&);
 future<> unset_server_raft(http_context&);
+future<> set_load_meter(http_context& ctx, service::load_meter& lm);
+future<> unset_load_meter(http_context& ctx);
 
 }

api/authorization_cache.hh

@@ -8,11 +8,20 @@
 
 #pragma once
 
-#include "api.hh"
+#include <seastar/core/sharded.hh>
+
+namespace seastar::httpd {
+class routes;
+}
+
+namespace auth {
+class service;
+}
 
 namespace api {
 
-void set_authorization_cache(http_context& ctx, httpd::routes& r, sharded<auth::service> &auth_service);
-void unset_authorization_cache(http_context& ctx, httpd::routes& r);
+struct http_context;
+void set_authorization_cache(http_context& ctx, seastar::httpd::routes& r, seastar::sharded<auth::service> &auth_service);
+void unset_authorization_cache(http_context& ctx, seastar::httpd::routes& r);
 
 }

api/cache_service.cc

@@ -7,6 +7,7 @@
  */
 
 #include "cache_service.hh"
+#include "api/api.hh"
 #include "api/api-doc/cache_service.json.hh"
 #include "column_family.hh"
 
@@ -195,9 +196,9 @@ void set_cache_service(http_context& ctx, routes& r) {
         return make_ready_future<json::json_return_type>(0);
     });
 
-    cs::get_row_capacity.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return ctx.db.map_reduce0([](replica::database& db) -> uint64_t {
-            return memory::stats().total_memory();
+    cs::get_row_capacity.set(r, [] (std::unique_ptr<http::request> req) {
+        return seastar::map_reduce(smp::all_cpus(), [] (int cpu) {
+            return make_ready_future<uint64_t>(memory::stats().total_memory());
         }, uint64_t(0), std::plus<uint64_t>()).then([](const int64_t& res) {
             return make_ready_future<json::json_return_type>(res);
         });

api/cache_service.hh

@@ -8,10 +8,13 @@
 
 #pragma once
 
-#include "api.hh"
+namespace seastar::httpd {
+class routes;
+}
 
 namespace api {
 
-void set_cache_service(http_context& ctx, httpd::routes& r);
+struct http_context;
+void set_cache_service(http_context& ctx, seastar::httpd::routes& r);
 
 }

api/column_family.hh

@@ -9,7 +9,6 @@
 #pragma once
 
 #include "replica/database.hh"
-#include <seastar/core/future-util.hh>
 #include <seastar/json/json_elements.hh>
 #include <any>
 #include "api/api_init.hh"

api/config.cc

@@ -6,8 +6,11 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
+#include "api/api.hh"
 #include "api/config.hh"
 #include "api/api-doc/config.json.hh"
+#include "api/api-doc/storage_proxy.json.hh"
+#include "replica/database.hh"
 #include "db/config.hh"
 #include <sstream>
 #include <boost/algorithm/string/replace.hpp>
@@ -15,6 +18,7 @@
 
 namespace api {
 using namespace seastar::httpd;
+namespace sp = httpd::storage_proxy_json;
 
 template<class T>
 json::json_return_type get_json_return_type(const T& val) {
@@ -101,6 +105,102 @@ void set_config(std::shared_ptr < api_registry_builder20 > rb, http_context& ctx
         }
         throw bad_param_exception(sstring("No such config entry: ") + id);
     });
+
+    sp::get_rpc_timeout.set(r, [&cfg](const_req req)  {
+        return cfg.request_timeout_in_ms()/1000.0;
+    });
+
+    sp::set_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
+        //TBD
+        unimplemented();
+        auto enable = req->get_query_param("timeout");
+        return make_ready_future<json::json_return_type>(seastar::json::json_void());
+    });
+
+    sp::get_read_rpc_timeout.set(r, [&cfg](const_req req)  {
+        return cfg.read_request_timeout_in_ms()/1000.0;
+    });
+
+    sp::set_read_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
+        //TBD
+        unimplemented();
+        auto enable = req->get_query_param("timeout");
+        return make_ready_future<json::json_return_type>(seastar::json::json_void());
+    });
+
+    sp::get_write_rpc_timeout.set(r, [&cfg](const_req req)  {
+        return cfg.write_request_timeout_in_ms()/1000.0;
+    });
+
+    sp::set_write_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
+        //TBD
+        unimplemented();
+        auto enable = req->get_query_param("timeout");
+        return make_ready_future<json::json_return_type>(seastar::json::json_void());
+    });
+
+    sp::get_counter_write_rpc_timeout.set(r, [&cfg](const_req req)  {
+        return cfg.counter_write_request_timeout_in_ms()/1000.0;
+    });
+
+    sp::set_counter_write_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
+        //TBD
+        unimplemented();
+        auto enable = req->get_query_param("timeout");
+        return make_ready_future<json::json_return_type>(seastar::json::json_void());
+    });
+
+    sp::get_cas_contention_timeout.set(r, [&cfg](const_req req)  {
+        return cfg.cas_contention_timeout_in_ms()/1000.0;
+    });
+
+    sp::set_cas_contention_timeout.set(r, [](std::unique_ptr<http::request> req)  {
+        //TBD
+        unimplemented();
+        auto enable = req->get_query_param("timeout");
+        return make_ready_future<json::json_return_type>(seastar::json::json_void());
+    });
+
+    sp::get_range_rpc_timeout.set(r, [&cfg](const_req req)  {
+        return cfg.range_request_timeout_in_ms()/1000.0;
+    });
+
+    sp::set_range_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
+        //TBD
+        unimplemented();
+        auto enable = req->get_query_param("timeout");
+        return make_ready_future<json::json_return_type>(seastar::json::json_void());
+    });
+
+    sp::get_truncate_rpc_timeout.set(r, [&cfg](const_req req)  {
+        return cfg.truncate_request_timeout_in_ms()/1000.0;
+    });
+
+    sp::set_truncate_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
+        //TBD
+        unimplemented();
+        auto enable = req->get_query_param("timeout");
+        return make_ready_future<json::json_return_type>(seastar::json::json_void());
+    });
+
+}
+
+void unset_config(http_context& ctx, routes& r) {
+    cs::find_config_id.unset(r);
+    sp::get_rpc_timeout.unset(r);
+    sp::set_rpc_timeout.unset(r);
+    sp::get_read_rpc_timeout.unset(r);
+    sp::set_read_rpc_timeout.unset(r);
+    sp::get_write_rpc_timeout.unset(r);
+    sp::set_write_rpc_timeout.unset(r);
+    sp::get_counter_write_rpc_timeout.unset(r);
+    sp::set_counter_write_rpc_timeout.unset(r);
+    sp::get_cas_contention_timeout.unset(r);
+    sp::set_cas_contention_timeout.unset(r);
+    sp::get_range_rpc_timeout.unset(r);
+    sp::set_range_rpc_timeout.unset(r);
+    sp::get_truncate_rpc_timeout.unset(r);
+    sp::set_truncate_rpc_timeout.unset(r);
 }
 
 }

api/config.hh

@@ -14,4 +14,5 @@
 namespace api {
 
 void set_config(std::shared_ptr<httpd::api_registry_builder20> rb, http_context& ctx, httpd::routes& r, const db::config& cfg, bool first = false);
+void unset_config(http_context& ctx, httpd::routes& r);
 }

api/error_injection.cc

@@ -7,10 +7,8 @@
  */
 
 #include "api/api-doc/error_injection.json.hh"
-#include "api/api.hh"
-
+#include "api/api_init.hh"
 #include <seastar/http/exception.hh>
-#include "log.hh"
 #include "utils/error_injection.hh"
 #include "utils/rjson.hh"
 #include <seastar/core/future-util.hh>

api/failure_detector.cc

@@ -99,5 +99,16 @@ void set_failure_detector(http_context& ctx, routes& r, gms::gossiper& g) {
     });
 }
 
+void unset_failure_detector(http_context& ctx, routes& r) {
+    fd::get_all_endpoint_states.unset(r);
+    fd::get_up_endpoint_count.unset(r);
+    fd::get_down_endpoint_count.unset(r);
+    fd::get_phi_convict_threshold.unset(r);
+    fd::get_simple_states.unset(r);
+    fd::set_phi_convict_threshold.unset(r);
+    fd::get_endpoint_state.unset(r);
+    fd::get_endpoint_phi_values.unset(r);
+}
+
 }
 

api/failure_detector.hh

@@ -19,5 +19,6 @@ class gossiper;
 namespace api {
 
 void set_failure_detector(http_context& ctx, httpd::routes& r, gms::gossiper& g);
+void unset_failure_detector(http_context& ctx, httpd::routes& r);
 
 }

api/gossiper.cc

@@ -71,4 +71,14 @@ void set_gossiper(http_context& ctx, routes& r, gms::gossiper& g) {
     });
 }
 
+void unset_gossiper(http_context& ctx, routes& r) {
+    httpd::gossiper_json::get_down_endpoint.unset(r);
+    httpd::gossiper_json::get_live_endpoint.unset(r);
+    httpd::gossiper_json::get_endpoint_downtime.unset(r);
+    httpd::gossiper_json::get_current_generation_number.unset(r);
+    httpd::gossiper_json::get_current_heart_beat_version.unset(r);
+    httpd::gossiper_json::assassinate_endpoint.unset(r);
+    httpd::gossiper_json::force_remove_endpoint.unset(r);
+}
+
 }

api/gossiper.hh

@@ -19,5 +19,6 @@ class gossiper;
 namespace api {
 
 void set_gossiper(http_context& ctx, httpd::routes& r, gms::gossiper& g);
+void unset_gossiper(http_context& ctx, httpd::routes& r);
 
 }

api/raft.cc

@@ -8,10 +8,10 @@
 
 #include <seastar/core/coroutine.hh>
 
-#include "api/api.hh"
 #include "api/api-doc/raft.json.hh"
 
 #include "service/raft/raft_group_registry.hh"
+#include "log.hh"
 
 using namespace seastar::httpd;
 
@@ -19,34 +19,44 @@ extern logging::logger apilog;
 
 namespace api {
 
+struct http_context;
 namespace r = httpd::raft_json;
 using namespace json;
 
+
+namespace {
+
+::service::raft_timeout get_request_timeout(const http::request& req) {
+    return std::invoke([timeout_str = req.get_query_param("timeout")] {
+        if (timeout_str.empty()) {
+            return ::service::raft_timeout{};
+        }
+        auto dur = std::stoll(timeout_str);
+        if (dur <= 0) {
+            throw bad_param_exception{"Timeout must be a positive number."};
+        }
+        return ::service::raft_timeout{.value = lowres_clock::now() + std::chrono::seconds{dur}};
+    });
+}
+
+}  // namespace
+
+
 void set_raft(http_context&, httpd::routes& r, sharded<service::raft_group_registry>& raft_gr) {
     r::trigger_snapshot.set(r, [&raft_gr] (std::unique_ptr<http::request> req) -> future<json_return_type> {
         raft::group_id gid{utils::UUID{req->get_path_param("group_id")}};
-        auto timeout_dur = std::invoke([timeout_str = req->get_query_param("timeout")] {
-            if (timeout_str.empty()) {
-                return std::chrono::seconds{60};
-            }
-            auto dur = std::stoll(timeout_str);
-            if (dur <= 0) {
-                throw std::runtime_error{"Timeout must be a positive number."};
-            }
-            return std::chrono::seconds{dur};
-        });
+        auto timeout = get_request_timeout(*req);
 
         std::atomic<bool> found_srv{false};
-        co_await raft_gr.invoke_on_all([gid, timeout_dur, &found_srv] (service::raft_group_registry& raft_gr) -> future<> {
-            auto* srv = raft_gr.find_server(gid);
-            if (!srv) {
+        co_await raft_gr.invoke_on_all([gid, timeout, &found_srv] (service::raft_group_registry& raft_gr) -> future<> {
+            if (!raft_gr.find_server(gid)) {
                 co_return;
             }
 
             found_srv = true;
-            abort_on_expiry aoe(lowres_clock::now() + timeout_dur);
             apilog.info("Triggering Raft group {} snapshot", gid);
-            auto result = co_await srv->trigger_snapshot(&aoe.abort_source());
+            auto srv = raft_gr.get_server_with_timeouts(gid);
+            auto result = co_await srv.trigger_snapshot(nullptr, timeout);
             if (result) {
                 apilog.info("New snapshot for Raft group {} created", gid);
             } else {
@@ -55,7 +65,7 @@ void set_raft(http_context&, httpd::routes& r, sharded<service::raft_group_regis
         });
 
         if (!found_srv) {
-            throw std::runtime_error{fmt::format("Server for group ID {} not found", gid)};
+            throw bad_param_exception{fmt::format("Server for group ID {} not found", gid)};
         }
 
         co_return json_void{};
@@ -87,12 +97,40 @@ void set_raft(http_context&, httpd::routes& r, sharded<service::raft_group_regis
 
         co_return json_return_type(leader_id.load().to_sstring());
     });
+    r::read_barrier.set(r, [&raft_gr] (std::unique_ptr<http::request> req) -> future<json_return_type> {
+        auto timeout = get_request_timeout(*req);
+
+        if (!req->query_parameters.contains("group_id")) {
+            // Read barrier on group 0 by default
+            co_await raft_gr.invoke_on(0, [timeout] (service::raft_group_registry& raft_gr) {
+                return raft_gr.group0_with_timeouts().read_barrier(nullptr, timeout);
+            });
+            co_return json_void{};
+        }
+
+        raft::group_id gid{utils::UUID{req->get_query_param("group_id")}};
+
+        std::atomic<bool> found_srv{false};
+        co_await raft_gr.invoke_on_all([gid, timeout, &found_srv] (service::raft_group_registry& raft_gr) {
+            if (!raft_gr.find_server(gid)) {
+                return make_ready_future<>();
+            }
+            found_srv = true;
+            return raft_gr.get_server_with_timeouts(gid).read_barrier(nullptr, timeout);
+        });
+
+        if (!found_srv) {
+            throw bad_param_exception{fmt::format("Server for group ID {} not found", gid)};
+        }
+
+        co_return json_void{};
+    });
 }
 
 void unset_raft(http_context&, httpd::routes& r) {
     r::trigger_snapshot.unset(r);
     r::get_leader_host.unset(r);
+    r::read_barrier.unset(r);
 }
 
 }
-

api/scrub_status.hh

@@ -6,6 +6,8 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
+#pragma once
+
 namespace api {
 
 enum class scrub_status {

api/storage_proxy.cc

@@ -13,7 +13,6 @@
 #include "api/api-doc/utils.json.hh"
 #include "db/config.hh"
 #include "utils/histogram.hh"
-#include "replica/database.hh"
 #include <seastar/core/scheduling_specific.hh>
 
 namespace api {
@@ -259,83 +258,6 @@ void set_storage_proxy(http_context& ctx, routes& r, sharded<service::storage_pr
         return make_ready_future<json::json_return_type>(0);
     });
 
-    sp::get_rpc_timeout.set(r, [&ctx](const_req req)  {
-        return ctx.db.local().get_config().request_timeout_in_ms()/1000.0;
-    });
-
-    sp::set_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
-        //TBD
-        unimplemented();
-        auto enable = req->get_query_param("timeout");
-        return make_ready_future<json::json_return_type>(json_void());
-    });
-
-    sp::get_read_rpc_timeout.set(r, [&ctx](const_req req)  {
-        return ctx.db.local().get_config().read_request_timeout_in_ms()/1000.0;
-    });
-
-    sp::set_read_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
-        //TBD
-        unimplemented();
-        auto enable = req->get_query_param("timeout");
-        return make_ready_future<json::json_return_type>(json_void());
-    });
-
-    sp::get_write_rpc_timeout.set(r, [&ctx](const_req req)  {
-        return ctx.db.local().get_config().write_request_timeout_in_ms()/1000.0;
-    });
-
-    sp::set_write_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
-        //TBD
-        unimplemented();
-        auto enable = req->get_query_param("timeout");
-        return make_ready_future<json::json_return_type>(json_void());
-    });
-
-    sp::get_counter_write_rpc_timeout.set(r, [&ctx](const_req req)  {
-        return ctx.db.local().get_config().counter_write_request_timeout_in_ms()/1000.0;
-    });
-
-    sp::set_counter_write_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
-        //TBD
-        unimplemented();
-        auto enable = req->get_query_param("timeout");
-        return make_ready_future<json::json_return_type>(json_void());
-    });
-
-    sp::get_cas_contention_timeout.set(r, [&ctx](const_req req)  {
-        return ctx.db.local().get_config().cas_contention_timeout_in_ms()/1000.0;
-    });
-
-    sp::set_cas_contention_timeout.set(r, [](std::unique_ptr<http::request> req)  {
-        //TBD
-        unimplemented();
-        auto enable = req->get_query_param("timeout");
-        return make_ready_future<json::json_return_type>(json_void());
-    });
-
-    sp::get_range_rpc_timeout.set(r, [&ctx](const_req req)  {
-        return ctx.db.local().get_config().range_request_timeout_in_ms()/1000.0;
-    });
-
-    sp::set_range_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
-        //TBD
-        unimplemented();
-        auto enable = req->get_query_param("timeout");
-        return make_ready_future<json::json_return_type>(json_void());
-    });
-
-    sp::get_truncate_rpc_timeout.set(r, [&ctx](const_req req)  {
-        return ctx.db.local().get_config().truncate_request_timeout_in_ms()/1000.0;
-    });
-
-    sp::set_truncate_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
-        //TBD
-        unimplemented();
-        auto enable = req->get_query_param("timeout");
-        return make_ready_future<json::json_return_type>(json_void());
-    });
-
     sp::reload_trigger_classes.set(r, [](std::unique_ptr<http::request> req)  {
         //TBD
         unimplemented();
@@ -516,20 +438,6 @@ void unset_storage_proxy(http_context& ctx, routes& r) {
     sp::get_max_hints_in_progress.unset(r);
     sp::set_max_hints_in_progress.unset(r);
     sp::get_hints_in_progress.unset(r);
-    sp::get_rpc_timeout.unset(r);
-    sp::set_rpc_timeout.unset(r);
-    sp::get_read_rpc_timeout.unset(r);
-    sp::set_read_rpc_timeout.unset(r);
-    sp::get_write_rpc_timeout.unset(r);
-    sp::set_write_rpc_timeout.unset(r);
-    sp::get_counter_write_rpc_timeout.unset(r);
-    sp::set_counter_write_rpc_timeout.unset(r);
-    sp::get_cas_contention_timeout.unset(r);
-    sp::set_cas_contention_timeout.unset(r);
-    sp::get_range_rpc_timeout.unset(r);
-    sp::set_range_rpc_timeout.unset(r);
-    sp::get_truncate_rpc_timeout.unset(r);
-    sp::set_truncate_rpc_timeout.unset(r);
     sp::reload_trigger_classes.unset(r);
     sp::get_read_repair_attempted.unset(r);
     sp::get_read_repair_repaired_blocking.unset(r);

api/storage_service.cc

@@ -49,7 +49,6 @@
 #include "db/extensions.hh"
 #include "db/snapshot-ctl.hh"
 #include "transport/controller.hh"
-#include "thrift/controller.hh"
 #include "locator/token_metadata.hh"
 #include "cdc/generation_service.hh"
 #include "locator/abstract_replication_strategy.hh"
@@ -308,21 +307,17 @@ future<scrub_info> parse_scrub_options(const http_context& ctx, sharded<db::snap
 }
 
 void set_transport_controller(http_context& ctx, routes& r, cql_transport::controller& ctl) {
-    ss::start_native_transport.set(r, [&ctx, &ctl](std::unique_ptr<http::request> req) {
+    ss::start_native_transport.set(r, [&ctl](std::unique_ptr<http::request> req) {
         return smp::submit_to(0, [&] {
-            return with_scheduling_group(ctx.db.local().get_statement_scheduling_group(), [&ctl] {
-                return ctl.start_server();
-            });
+            return ctl.start_server();
         }).then([] {
             return make_ready_future<json::json_return_type>(json_void());
         });
     });
 
-    ss::stop_native_transport.set(r, [&ctx, &ctl](std::unique_ptr<http::request> req) {
+    ss::stop_native_transport.set(r, [&ctl](std::unique_ptr<http::request> req) {
         return smp::submit_to(0, [&] {
-            return with_scheduling_group(ctx.db.local().get_statement_scheduling_group(), [&ctl] {
-                return ctl.request_stop_server();
-            });
+            return ctl.request_stop_server();
         }).then([] {
             return make_ready_future<json::json_return_type>(json_void());
         });
@@ -343,40 +338,17 @@ void unset_transport_controller(http_context& ctx, routes& r) {
     ss::is_native_transport_running.unset(r);
 }
 
-void set_rpc_controller(http_context& ctx, routes& r, thrift_controller& ctl) {
-    ss::stop_rpc_server.set(r, [&ctx, &ctl](std::unique_ptr<http::request> req) {
-        return smp::submit_to(0, [&] {
-            return with_scheduling_group(ctx.db.local().get_statement_scheduling_group(), [&ctl] {
-                return ctl.request_stop_server();
-            });
-        }).then([] {
-            return make_ready_future<json::json_return_type>(json_void());
-        });
-    });
-
-    ss::start_rpc_server.set(r, [&ctx, &ctl](std::unique_ptr<http::request> req) {
-        return smp::submit_to(0, [&] {
-            return with_scheduling_group(ctx.db.local().get_statement_scheduling_group(), [&ctl] {
-                return ctl.start_server();
-            });
-        }).then([] {
-            return make_ready_future<json::json_return_type>(json_void());
-        });
-    });
-
-    ss::is_rpc_server_running.set(r, [&ctl] (std::unique_ptr<http::request> req) {
-        return smp::submit_to(0, [&] {
-            return !ctl.listen_addresses().empty();
-        }).then([] (bool running) {
-            return make_ready_future<json::json_return_type>(running);
+// NOTE: preserved only for backward compatibility
+void set_thrift_controller(http_context& ctx, routes& r) {
+    ss::is_thrift_server_running.set(r, [] (std::unique_ptr<http::request> req) {
+        return smp::submit_to(0, [] {
+            return make_ready_future<json::json_return_type>(false);
         });
     });
 }
 
-void unset_rpc_controller(http_context& ctx, routes& r) {
-    ss::stop_rpc_server.unset(r);
-    ss::start_rpc_server.unset(r);
-    ss::is_rpc_server_running.unset(r);
+void unset_thrift_controller(http_context& ctx, routes& r) {
+    ss::is_thrift_server_running.unset(r);
 }
 
 void set_repair(http_context& ctx, routes& r, sharded<repair_service>& repair) {
@@ -710,19 +682,6 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
         return get_cf_stats(ctx, &replica::column_family_stats::live_disk_space_used);
     });
 
-    ss::get_load_map.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return ctx.lmeter.get_load_map().then([] (auto&& load_map) {
-            std::vector<ss::map_string_double> res;
-            for (auto i : load_map) {
-                ss::map_string_double val;
-                val.key = i.first;
-                val.value = i.second;
-                res.push_back(val);
-            }
-            return make_ready_future<json::json_return_type>(res);
-        });
-    });
-
     ss::get_current_generation_number.set(r, [&ss](std::unique_ptr<http::request> req) {
         auto ep = ss.local().get_token_metadata().get_topology().my_address();
         return ss.local().gossiper().get_current_generation_number(ep).then([](gms::generation_type res) {
@@ -1616,7 +1575,6 @@ void unset_storage_service(http_context& ctx, routes& r) {
     ss::get_pending_range_to_endpoint_map.unset(r);
     ss::describe_ring.unset(r);
     ss::get_load.unset(r);
-    ss::get_load_map.unset(r);
     ss::get_current_generation_number.unset(r);
     ss::get_natural_endpoints.unset(r);
     ss::cdc_streams_check_and_repair.unset(r);
@@ -1692,6 +1650,24 @@ void unset_storage_service(http_context& ctx, routes& r) {
     sp::get_schema_versions.unset(r);
 }
 
+void set_load_meter(http_context& ctx, routes& r, service::load_meter& lm) {
+    ss::get_load_map.set(r, [&lm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
+        auto load_map = co_await lm.get_load_map();
+        std::vector<ss::map_string_double> res;
+        for (auto i : load_map) {
+            ss::map_string_double val;
+            val.key = i.first;
+            val.value = i.second;
+            res.push_back(val);
+        }
+        co_return res;
+    });
+}
+
+void unset_load_meter(http_context& ctx, routes& r) {
+    ss::get_load_map.unset(r);
+}
+
 void set_snapshot(http_context& ctx, routes& r, sharded<db::snapshot_ctl>& snap_ctl) {
     ss::get_snapshot_details.set(r, [&snap_ctl](std::unique_ptr<http::request> req) -> future<json::json_return_type> {
         auto result = co_await snap_ctl.local().get_snapshot_details();

api/storage_service.hh

@@ -14,7 +14,6 @@
 #include "db/data_listeners.hh"
 
 namespace cql_transport { class controller; }
-class thrift_controller;
 namespace db {
 class snapshot_ctl;
 namespace view {
@@ -80,10 +79,12 @@ void set_repair(http_context& ctx, httpd::routes& r, sharded<repair_service>& re
 void unset_repair(http_context& ctx, httpd::routes& r);
 void set_transport_controller(http_context& ctx, httpd::routes& r, cql_transport::controller& ctl);
 void unset_transport_controller(http_context& ctx, httpd::routes& r);
-void set_rpc_controller(http_context& ctx, httpd::routes& r, thrift_controller& ctl);
-void unset_rpc_controller(http_context& ctx, httpd::routes& r);
+void set_thrift_controller(http_context& ctx, httpd::routes& r);
+void unset_thrift_controller(http_context& ctx, httpd::routes& r);
 void set_snapshot(http_context& ctx, httpd::routes& r, sharded<db::snapshot_ctl>& snap_ctl);
 void unset_snapshot(http_context& ctx, httpd::routes& r);
+void set_load_meter(http_context& ctx, httpd::routes& r, service::load_meter& lm);
+void unset_load_meter(http_context& ctx, httpd::routes& r);
 seastar::future<json::json_return_type> run_toppartitions_query(db::toppartitions_query& q, http_context &ctx, bool legacy_request = false);
 
 } // namespace api

api/system.hh

@@ -8,10 +8,13 @@
 
 #pragma once
 
-#include "api.hh"
+namespace seastar::httpd {
+class routes;
+}
 
 namespace api {
 
-void set_system(http_context& ctx, httpd::routes& r);
+struct http_context;
+void set_system(http_context& ctx, seastar::httpd::routes& r);
 
 }

api/task_manager.cc

@@ -35,7 +35,6 @@ struct full_task_status {
     tasks::task_manager::task::status task_status;
     std::string type;
     tasks::task_manager::task::progress progress;
-    std::string module;
     tasks::task_id parent_id;
     tasks::is_abortable abortable;
     std::vector<std::string> children_ids;
@@ -102,7 +101,6 @@ future<full_task_status> retrieve_status(const tasks::task_manager::foreign_task
     s.type = task->type();
     s.parent_id = task->get_parent_id();
     s.abortable = task->is_abortable();
-    s.module = task->get_module_name();
     s.progress.completed = progress.completed;
     s.progress.total = progress.total;
     std::vector<std::string> ct = co_await task->get_children().map_each_task<std::string>([] (const tasks::task_manager::foreign_task_ptr& child) {

api/task_manager_test.hh

@@ -11,16 +11,19 @@
 #pragma once
 
 #include <seastar/core/sharded.hh>
-#include "api.hh"
 
 namespace tasks {
 class task_manager;
 }
 
-namespace api {
+namespace seastar::httpd {
+class routes;
+}
 
-void set_task_manager_test(http_context& ctx, httpd::routes& r, sharded<tasks::task_manager>& tm);
-void unset_task_manager_test(http_context& ctx, httpd::routes& r);
+namespace api {
+struct http_context;
+void set_task_manager_test(http_context& ctx, seastar::httpd::routes& r, seastar::sharded<tasks::task_manager>& tm);
+void unset_task_manager_test(http_context& ctx, seastar::httpd::routes& r);
 
 }
 

api/tasks.hh

@@ -8,11 +8,20 @@
 
 #pragma once
 
-#include "api.hh"
-#include "db/config.hh"
+#include <seastar/core/sharded.hh>
+#include "db/snapshot-ctl.hh"
+
+namespace seastar::httpd {
+class routes;
+}
+
+namespace service {
+class storage_service;
+}
 
 namespace api {
 
+struct http_context;
 void set_tasks_compaction_module(http_context& ctx, httpd::routes& r, sharded<service::storage_service>& ss, sharded<db::snapshot_ctl>& snap_ctl);
 void unset_tasks_compaction_module(http_context& ctx, httpd::routes& r);
 

api/token_metadata.hh

@@ -9,13 +9,16 @@
 #pragma once
 
 #include <seastar/core/sharded.hh>
-#include "api/api_init.hh"
+
+namespace seastar::httpd {
+class routes;
+}
 
 namespace locator { class shared_token_metadata; }
 
 namespace api {
-
-void set_token_metadata(http_context& ctx, httpd::routes& r, sharded<locator::shared_token_metadata>& tm);
-void unset_token_metadata(http_context& ctx, httpd::routes& r);
+struct http_context;
+void set_token_metadata(http_context& ctx, seastar::httpd::routes& r, seastar::sharded<locator::shared_token_metadata>& tm);
+void unset_token_metadata(http_context& ctx, seastar::httpd::routes& r);
 
 }

auth/allow_all_authenticator.hh

@@ -59,15 +59,15 @@ public:
         return make_ready_future<authenticated_user>(anonymous_user());
     }
 
-    virtual future<> create(std::string_view, const authentication_options& options) override {
+    virtual future<> create(std::string_view, const authentication_options& options, ::service::group0_batch&) override {
         return make_ready_future();
     }
 
-    virtual future<> alter(std::string_view, const authentication_options& options) override {
+    virtual future<> alter(std::string_view, const authentication_options& options, ::service::group0_batch&) override {
         return make_ready_future();
     }
 
-    virtual future<> drop(std::string_view) override {
+    virtual future<> drop(std::string_view, ::service::group0_batch&) override {
         return make_ready_future();
     }
 

auth/allow_all_authorizer.hh

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "auth/authorizer.hh"
+#include <seastar/core/future.hh>
 
 namespace cql3 {
 class query_processor;
@@ -44,12 +45,12 @@ public:
         return make_ready_future<permission_set>(permissions::ALL);
     }
 
-    virtual future<> grant(std::string_view, permission_set, const resource&) override {
+    virtual future<> grant(std::string_view, permission_set, const resource&, ::service::group0_batch&) override {
         return make_exception_future<>(
                 unsupported_authorization_operation("GRANT operation is not supported by AllowAllAuthorizer"));
     }
 
-    virtual future<> revoke(std::string_view, permission_set, const resource&) override {
+    virtual future<> revoke(std::string_view, permission_set, const resource&, ::service::group0_batch&) override {
         return make_exception_future<>(
                 unsupported_authorization_operation("REVOKE operation is not supported by AllowAllAuthorizer"));
     }
@@ -60,14 +61,12 @@ public:
                         "LIST PERMISSIONS operation is not supported by AllowAllAuthorizer"));
     }
 
-    virtual future<> revoke_all(std::string_view) override {
-        return make_exception_future(
-                unsupported_authorization_operation("REVOKE operation is not supported by AllowAllAuthorizer"));
+    virtual future<> revoke_all(std::string_view, ::service::group0_batch&) override {
+        return make_ready_future();
     }
 
-    virtual future<> revoke_all(const resource&) override {
-        return make_exception_future(
-                unsupported_authorization_operation("REVOKE operation is not supported by AllowAllAuthorizer"));
+    virtual future<> revoke_all(const resource&, ::service::group0_batch&) override {
+        return make_ready_future();
     }
 
     virtual const resource_set& protected_resources() const override {

auth/authenticator.hh

@@ -16,15 +16,15 @@
 #include <optional>
 #include <functional>
 
-#include <seastar/core/enum.hh>
 #include <seastar/core/future.hh>
 #include <seastar/core/sstring.hh>
-#include <seastar/core/shared_ptr.hh>
 
 #include "auth/authentication_options.hh"
 #include "auth/resource.hh"
 #include "auth/sasl_challenge.hh"
 
+#include "service/raft/raft_group0_client.hh"
+
 namespace db {
     class config;
 }
@@ -106,7 +106,7 @@ public:
     ///
     /// The options provided must be a subset of `supported_options()`.
     ///
-    virtual future<> create(std::string_view role_name, const authentication_options& options) = 0;
+    virtual future<> create(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) = 0;
 
     ///
     /// Alter the authentication record of an existing user.
@@ -115,12 +115,12 @@ public:
     ///
     /// Callers must ensure that the specification of `alterable_options()` is adhered to.
     ///
-    virtual future<> alter(std::string_view role_name, const authentication_options& options) = 0;
+    virtual future<> alter(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) = 0;
 
     ///
     /// Delete the authentication record for a user. This will disallow the user from logging in.
     ///
-    virtual future<> drop(std::string_view role_name) = 0;
+    virtual future<> drop(std::string_view role_name, ::service::group0_batch&) = 0;
 
     ///
     /// Query for custom options (those corresponding to \ref authentication_options::options).

auth/authorizer.hh

@@ -16,10 +16,10 @@
 #include <vector>
 
 #include <seastar/core/future.hh>
-#include <seastar/core/shared_ptr.hh>
 
 #include "auth/permission.hh"
 #include "auth/resource.hh"
+#include "service/raft/raft_group0_client.hh"
 #include "seastarx.hh"
 
 namespace auth {
@@ -81,14 +81,14 @@ public:
     ///
     /// \throws \ref unsupported_authorization_operation if granting permissions is not supported.
     ///
-    virtual future<> grant(std::string_view role_name, permission_set, const resource&) = 0;
+    virtual future<> grant(std::string_view role_name, permission_set, const resource&, ::service::group0_batch&) = 0;
 
     ///
     /// Revoke a set of permissions from a role for a particular \ref resource.
     ///
     /// \throws \ref unsupported_authorization_operation if revoking permissions is not supported.
     ///
-    virtual future<> revoke(std::string_view role_name, permission_set, const resource&) = 0;
+    virtual future<> revoke(std::string_view role_name, permission_set, const resource&, ::service::group0_batch&) = 0;
 
     ///
     /// Query for all directly granted permissions.
@@ -102,14 +102,14 @@ public:
     ///
     /// \throws \ref unsupported_authorization_operation if revoking permissions is not supported.
     ///
-    virtual future<> revoke_all(std::string_view role_name) = 0;
+    virtual future<> revoke_all(std::string_view role_name, ::service::group0_batch&) = 0;
 
     ///
     /// Revoke all permissions granted to any role for a particular resource.
     ///
     /// \throws \ref unsupported_authorization_operation if revoking permissions is not supported.
     ///
-    virtual future<> revoke_all(const resource&) = 0;
+    virtual future<> revoke_all(const resource&, ::service::group0_batch&) = 0;
 
     ///
     /// System resources used internally as part of the implementation. These are made inaccessible to users.

auth/certificate_authenticator.cc

@@ -157,16 +157,16 @@ future<auth::authenticated_user> auth::certificate_authenticator::authenticate(c
     throw exceptions::authentication_exception("Cannot authenticate using attribute map");
 }
 
-future<> auth::certificate_authenticator::create(std::string_view role_name, const authentication_options& options) {
+future<> auth::certificate_authenticator::create(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) {
     // TODO: should we keep track of roles/enforce existence? Role manager should deal with this...
     co_return;
 }
 
-future<> auth::certificate_authenticator::alter(std::string_view role_name, const authentication_options& options) {
+future<> auth::certificate_authenticator::alter(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) {
     co_return;
 }
 
-future<> auth::certificate_authenticator::drop(std::string_view role_name) {
+future<> auth::certificate_authenticator::drop(std::string_view role_name, ::service::group0_batch&) {
     co_return;
 }
 

auth/certificate_authenticator.hh

@@ -9,7 +9,6 @@
 
 #pragma once
 
-#include <boost/regex.hpp>
 #include "auth/authenticator.hh"
 
 namespace cql3 {
@@ -47,9 +46,9 @@ public:
     future<authenticated_user> authenticate(const credentials_map& credentials) const override;
     future<std::optional<authenticated_user>> authenticate(session_dn_func) const override;
 
-    future<> create(std::string_view role_name, const authentication_options& options) override;
-    future<> alter(std::string_view role_name, const authentication_options& options) override;
-    future<> drop(std::string_view role_name) override;
+    future<> create(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) override;
+    future<> alter(std::string_view role_name, const authentication_options& options, ::service::group0_batch&) override;
+    future<> drop(std::string_view role_name, ::service::group0_batch&) override;
 
     future<custom_options> query_custom_options(std::string_view role_name) const override;
 

auth/common.cc

@@ -23,7 +23,6 @@
 #include "service/migration_manager.hh"
 #include "service/raft/group0_state_machine.hh"
 #include "timeout_config.hh"
-#include "db/config.hh"
 #include "utils/error_injection.hh"
 
 namespace auth {
@@ -137,7 +136,7 @@ static future<> announce_mutations_with_guard(
 future<> announce_mutations_with_batching(
         ::service::raft_group0_client& group0_client,
         start_operation_func_t start_operation_func,
-        std::function<mutations_generator(api::timestamp_type& t)> gen,
+        std::function<::service::mutations_generator(api::timestamp_type t)> gen,
         seastar::abort_source& as,
         std::optional<::service::raft_timeout> timeout) {
     // account for command's overhead, it's better to use smaller threshold than constantly bounce off the limit
@@ -202,4 +201,17 @@ future<> announce_mutations(
     co_await announce_mutations_with_guard(group0_client, std::move(cmuts), std::move(group0_guard), as, timeout);
 }
 
+future<> collect_mutations(
+        cql3::query_processor& qp,
+        ::service::group0_batch& collector,
+        const sstring query_string,
+        std::vector<data_value_or_unset> values) {
+    auto muts = co_await qp.get_mutations_internal(
+            query_string,
+            internal_distributed_query_state(),
+            collector.write_timestamp(),
+            std::move(values));
+    collector.add_mutations(std::move(muts), format("auth internal statement: {}", query_string));
+}
+
 }

auth/common.hh

@@ -13,12 +13,8 @@
 #include <seastar/core/future.hh>
 #include <seastar/core/abort_source.hh>
 #include <seastar/util/noncopyable_function.hh>
-#include <seastar/core/seastar.hh>
-#include <seastar/core/resource.hh>
 #include <seastar/core/sstring.hh>
-#include <seastar/core/smp.hh>
 
-#include "schema/schema_registry.hh"
 #include "types/types.hh"
 #include "service/raft/raft_group0_client.hh"
 
@@ -85,14 +81,13 @@ future<> create_legacy_metadata_table_if_missing(
 // Use this function when need to perform read before write on a single guard or if
 // you have more than one mutation and potentially exceed single command size limit.
 using start_operation_func_t = std::function<future<::service::group0_guard>(abort_source&)>;
-using mutations_generator = coroutine::experimental::generator<mutation>;
 future<> announce_mutations_with_batching(
         ::service::raft_group0_client& group0_client,
         // since we can operate also in topology coordinator context where we need stronger
         // guarantees than start_operation from group0_client gives we allow to inject custom
         // function here
         start_operation_func_t start_operation_func,
-        std::function<mutations_generator(api::timestamp_type& t)> gen,
+        std::function<::service::mutations_generator(api::timestamp_type t)> gen,
         seastar::abort_source& as,
         std::optional<::service::raft_timeout> timeout);
 
@@ -105,4 +100,10 @@ future<> announce_mutations(
         seastar::abort_source& as,
         std::optional<::service::raft_timeout> timeout);
 
+// Appends mutations to a collector, they will be applied later on all nodes via group0 mechanism.
+future<> collect_mutations(
+        cql3::query_processor& qp,
+        ::service::group0_batch& collector,
+        const sstring query_string,
+        std::vector<data_value_or_unset> values);
 }

auth/default_authorizer.cc

@@ -90,9 +90,10 @@ future<> default_authorizer::migrate_legacy_metadata() {
             return do_with(
                     row.get_as<sstring>("username"),
                     parse_resource(row.get_as<sstring>(RESOURCE_NAME)),
-                    [this, &row](const auto& username, const auto& r) {
+                    ::service::group0_batch::unused(),
+                    [this, &row](const auto& username, const auto& r, auto& mc) {
                 const permission_set perms = permissions::from_strings(row.get_set<sstring>(PERMISSIONS_NAME));
-                return grant(username, perms, r);
+                return grant(username, perms, r, mc);
             });
         }).finally([results] {});
     }).then([] {
@@ -185,7 +186,8 @@ default_authorizer::modify(
         std::string_view role_name,
         permission_set set,
         const resource& resource,
-        std::string_view op) {
+        std::string_view op,
+        ::service::group0_batch& mc) {
     const sstring query = format("UPDATE {}.{} SET {} = {} {} ? WHERE {} = ? AND {} = ?",
             get_auth_ks_name(_qp),
             PERMISSIONS_CF,
@@ -202,17 +204,17 @@ default_authorizer::modify(
                 {permissions::to_strings(set), sstring(role_name), resource.name()},
                 cql3::query_processor::cache_internal::no).discard_result();
     }
-    co_return co_await announce_mutations(_qp, _group0_client, query,
-        {permissions::to_strings(set), sstring(role_name), resource.name()}, _as, ::service::raft_timeout{});
+    co_await collect_mutations(_qp, mc, query,
+            {permissions::to_strings(set), sstring(role_name), resource.name()});
 }
 
 
-future<> default_authorizer::grant(std::string_view role_name, permission_set set, const resource& resource) {
-    return modify(role_name, std::move(set), resource, "+");
+future<> default_authorizer::grant(std::string_view role_name, permission_set set, const resource& resource, ::service::group0_batch& mc) {
+    return modify(role_name, std::move(set), resource, "+", mc);
 }
 
-future<> default_authorizer::revoke(std::string_view role_name, permission_set set, const resource& resource) {
-    return modify(role_name, std::move(set), resource, "-");
+future<> default_authorizer::revoke(std::string_view role_name, permission_set set, const resource& resource, ::service::group0_batch& mc) {
+    return modify(role_name, std::move(set), resource, "-", mc);
 }
 
 future<std::vector<permission_details>> default_authorizer::list_all() const {
@@ -242,7 +244,7 @@ future<std::vector<permission_details>> default_authorizer::list_all() const {
     co_return all_details;
 }
 
-future<> default_authorizer::revoke_all(std::string_view role_name) {
+future<> default_authorizer::revoke_all(std::string_view role_name, ::service::group0_batch& mc) {
     try {
         const sstring query = format("DELETE FROM {}.{} WHERE {} = ?",
                 get_auth_ks_name(_qp),
@@ -256,7 +258,7 @@ future<> default_authorizer::revoke_all(std::string_view role_name) {
                     {sstring(role_name)},
                     cql3::query_processor::cache_internal::no).discard_result();
         } else {
-            co_await announce_mutations(_qp, _group0_client, query, {sstring(role_name)}, _as, ::service::raft_timeout{});
+            co_await collect_mutations(_qp, mc, query, {sstring(role_name)});
         }
     } catch (exceptions::request_execution_exception& e) {
         alogger.warn("CassandraAuthorizer failed to revoke all permissions of {}: {}", role_name, e);
@@ -308,51 +310,88 @@ future<> default_authorizer::revoke_all_legacy(const resource& resource) {
     });
 }
 
-future<> default_authorizer::revoke_all(const resource& resource) {
+future<> default_authorizer::revoke_all(const resource& resource, ::service::group0_batch& mc) {
     if (legacy_mode(_qp)) {
         co_return co_await revoke_all_legacy(resource);
     }
+
+    if (resource.kind() == resource_kind::data &&
+            data_resource_view(resource).is_keyspace()) {
+        revoke_all_keyspace_resources(resource, mc);
+        co_return;
+    }
+
     auto name = resource.name();
-    try {
-        auto gen = [this, name] (api::timestamp_type& t) -> mutations_generator {
-            const sstring query = format("SELECT {} FROM {}.{} WHERE {} = ? ALLOW FILTERING",
-                    ROLE_NAME,
+    auto gen = [this, name] (api::timestamp_type t) -> ::service::mutations_generator {
+        const sstring query = format("SELECT {} FROM {}.{} WHERE {} = ? ALLOW FILTERING",
+                ROLE_NAME,
+                get_auth_ks_name(_qp),
+                PERMISSIONS_CF,
+                RESOURCE_NAME);
+        auto res = co_await _qp.execute_internal(
+                query,
+                db::consistency_level::LOCAL_ONE,
+                {name},
+                cql3::query_processor::cache_internal::no);
+        for (const auto& r : *res) {
+            const sstring query = format("DELETE FROM {}.{} WHERE {} = ? AND {} = ?",
                     get_auth_ks_name(_qp),
                     PERMISSIONS_CF,
+                    ROLE_NAME,
                     RESOURCE_NAME);
-            auto res = co_await _qp.execute_internal(
+            auto muts = co_await _qp.get_mutations_internal(
                     query,
-                    db::consistency_level::LOCAL_ONE,
-                    {name},
-                    cql3::query_processor::cache_internal::no);
-            for (const auto& r : *res) {
-                const sstring query = format("DELETE FROM {}.{} WHERE {} = ? AND {} = ?",
-                        get_auth_ks_name(_qp),
-                        PERMISSIONS_CF,
-                        ROLE_NAME,
-                        RESOURCE_NAME);
-                auto muts = co_await _qp.get_mutations_internal(
-                        query,
-                        internal_distributed_query_state(),
-                        t,
-                        {r.get_as<sstring>(ROLE_NAME), name});
-                if (muts.size() != 1) {
-                    on_internal_error(alogger,
-                        format("expecting single delete mutation, got {}", muts.size()));
-                }
-                co_yield std::move(muts[0]);
+                    internal_distributed_query_state(),
+                    t,
+                    {r.get_as<sstring>(ROLE_NAME), name});
+            if (muts.size() != 1) {
+                on_internal_error(alogger,
+                    format("expecting single delete mutation, got {}", muts.size()));
             }
-        };
-        const auto timeout = ::service::raft_timeout{};
-        co_await announce_mutations_with_batching(
-                _group0_client,
-                [this, timeout](abort_source& as) { return _group0_client.start_operation(as, timeout); },
-                std::move(gen),
-                _as,
-            timeout);
-    } catch (exceptions::request_execution_exception& e) {
-        alogger.warn("CassandraAuthorizer failed to revoke all permissions on {}: {}", name, e);
-    }
+            co_yield std::move(muts[0]);
+        }
+    };
+    mc.add_generator(std::move(gen), "default_authorizer::revoke_all");
+}
+
+void default_authorizer::revoke_all_keyspace_resources(const resource& ks_resource, ::service::group0_batch& mc) {
+    auto ks_name = ks_resource.name();
+    auto gen = [this, ks_name] (api::timestamp_type t) -> ::service::mutations_generator {
+        const sstring query = format("SELECT {}, {} FROM {}.{}",
+                ROLE_NAME,
+                RESOURCE_NAME,
+                get_auth_ks_name(_qp),
+                PERMISSIONS_CF);
+        auto res = co_await _qp.execute_internal(
+                query,
+                db::consistency_level::LOCAL_ONE,
+                {},
+                cql3::query_processor::cache_internal::no);
+        auto ks_prefix = ks_name + "/";
+        for (const auto& r : *res) {
+            auto name = r.get_as<sstring>(RESOURCE_NAME);
+            if (name != ks_name && !name.starts_with(ks_prefix)) {
+                // r doesn't represent resource related to ks_resource
+                continue;
+            }
+            const sstring query = format("DELETE FROM {}.{} WHERE {} = ? AND {} = ?",
+                    get_auth_ks_name(_qp),
+                    PERMISSIONS_CF,
+                    ROLE_NAME,
+                    RESOURCE_NAME);
+            auto muts = co_await _qp.get_mutations_internal(
+                    query,
+                    internal_distributed_query_state(),
+                    t,
+                    {r.get_as<sstring>(ROLE_NAME), name});
+            if (muts.size() != 1) {
+                on_internal_error(alogger,
+                    format("expecting single delete mutation, got {}", muts.size()));
+            }
+            co_yield std::move(muts[0]);
+        }
+    };
+    mc.add_generator(std::move(gen), "default_authorizer::revoke_all_keyspace_resources");
 }
 
 const resource_set& default_authorizer::protected_resources() const {

auth/default_authorizer.hh

@@ -47,15 +47,15 @@ public:
 
     virtual future<permission_set> authorize(const role_or_anonymous&, const resource&) const override;
 
-    virtual future<> grant(std::string_view, permission_set, const resource&) override;
+    virtual future<> grant(std::string_view, permission_set, const resource&, ::service::group0_batch&) override;
 
-    virtual future<> revoke( std::string_view, permission_set, const resource&) override;
+    virtual future<> revoke( std::string_view, permission_set, const resource&, ::service::group0_batch&) override;
 
     virtual future<std::vector<permission_details>> list_all() const override;
 
-    virtual future<> revoke_all(std::string_view) override;
+    virtual future<> revoke_all(std::string_view, ::service::group0_batch&) override;
 
-    virtual future<> revoke_all(const resource&) override;
+    virtual future<> revoke_all(const resource&, ::service::group0_batch&) override;
 
     virtual const resource_set& protected_resources() const override;
 
@@ -70,7 +70,9 @@ private:
 
     future<> migrate_legacy_metadata();
 
-    future<> modify(std::string_view, permission_set, const resource&, std::string_view);
+    future<> modify(std::string_view, permission_set, const resource&, std::string_view, ::service::group0_batch&);
+
+    void revoke_all_keyspace_resources(const resource& ks_resource, ::service::group0_batch& mc);
 };
 
 } /* namespace auth */

auth/maintenance_socket_role_manager.cc

@@ -49,23 +49,23 @@ future<T> operation_not_supported_exception(std::string_view operation) {
         std::runtime_error(format("role manager: {} operation not supported through maintenance socket", operation)));
 }
 
-future<> maintenance_socket_role_manager::create(std::string_view role_name, const role_config&) {
+future<> maintenance_socket_role_manager::create(std::string_view role_name, const role_config&, ::service::group0_batch&) {
     return operation_not_supported_exception("CREATE");
 }
 
-future<> maintenance_socket_role_manager::drop(std::string_view role_name) {
+future<> maintenance_socket_role_manager::drop(std::string_view role_name, ::service::group0_batch& mc) {
     return operation_not_supported_exception("DROP");
 }
 
-future<> maintenance_socket_role_manager::alter(std::string_view role_name, const role_config_update&) {
+future<> maintenance_socket_role_manager::alter(std::string_view role_name, const role_config_update&, ::service::group0_batch&) {
     return operation_not_supported_exception("ALTER");
 }
 
-future<> maintenance_socket_role_manager::grant(std::string_view grantee_name, std::string_view role_name) {
+future<> maintenance_socket_role_manager::grant(std::string_view grantee_name, std::string_view role_name, ::service::group0_batch& mc) {
     return operation_not_supported_exception("GRANT");
 }
 
-future<> maintenance_socket_role_manager::revoke(std::string_view revokee_name, std::string_view role_name) {
+future<> maintenance_socket_role_manager::revoke(std::string_view revokee_name, std::string_view role_name, ::service::group0_batch& mc) {
     return operation_not_supported_exception("REVOKE");
 }
 
@@ -97,11 +97,11 @@ future<role_manager::attribute_vals> maintenance_socket_role_manager::query_attr
     return operation_not_supported_exception<role_manager::attribute_vals>("QUERY ATTRIBUTE");
 }
 
-future<> maintenance_socket_role_manager::set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value) {
+future<> maintenance_socket_role_manager::set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value, ::service::group0_batch& mc) {
     return operation_not_supported_exception("SET ATTRIBUTE");
 }
 
-future<> maintenance_socket_role_manager::remove_attribute(std::string_view role_name, std::string_view attribute_name) {
+future<> maintenance_socket_role_manager::remove_attribute(std::string_view role_name, std::string_view attribute_name, ::service::group0_batch& mc) {
     return operation_not_supported_exception("REMOVE ATTRIBUTE");
 }
 

auth/maintenance_socket_role_manager.hh

@@ -39,15 +39,15 @@ public:
 
     virtual future<> stop() override;
 
-    virtual future<> create(std::string_view role_name, const role_config&) override;
+    virtual future<> create(std::string_view role_name, const role_config&, ::service::group0_batch&) override;
 
-    virtual future<> drop(std::string_view role_name) override;
+    virtual future<> drop(std::string_view role_name, ::service::group0_batch& mc) override;
 
-    virtual future<> alter(std::string_view role_name, const role_config_update&) override;
+    virtual future<> alter(std::string_view role_name, const role_config_update&, ::service::group0_batch&) override;
 
-    virtual future<> grant(std::string_view grantee_name, std::string_view role_name) override;
+    virtual future<> grant(std::string_view grantee_name, std::string_view role_name, ::service::group0_batch& mc) override;
 
-    virtual future<> revoke(std::string_view revokee_name, std::string_view role_name) override;
+    virtual future<> revoke(std::string_view revokee_name, std::string_view role_name, ::service::group0_batch& mc) override;
 
     virtual future<role_set> query_granted(std::string_view grantee_name, recursive_role_query) override;
 
@@ -63,9 +63,9 @@ public:
 
     virtual future<role_manager::attribute_vals> query_attribute_for_all(std::string_view attribute_name) override;
 
-    virtual future<> set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value) override;
+    virtual future<> set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value, ::service::group0_batch& mc) override;
 
-    virtual future<> remove_attribute(std::string_view role_name, std::string_view attribute_name) override;
+    virtual future<> remove_attribute(std::string_view role_name, std::string_view attribute_name, ::service::group0_batch& mc) override;
 };
 
 }

auth/password_authenticator.cc

@@ -257,7 +257,7 @@ future<authenticated_user> password_authenticator::authenticate(
     }
 }
 
-future<> password_authenticator::create(std::string_view role_name, const authentication_options& options) {
+future<> password_authenticator::create(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) {
     if (!options.password) {
         co_return;
     }
@@ -270,12 +270,12 @@ future<> password_authenticator::create(std::string_view role_name, const authen
                 {passwords::hash(*options.password, rng_for_salt), sstring(role_name)},
                 cql3::query_processor::cache_internal::no).discard_result();
     } else {
-        co_await announce_mutations(_qp, _group0_client, query,
-                {passwords::hash(*options.password, rng_for_salt), sstring(role_name)}, _as, ::service::raft_timeout{});
+        co_await collect_mutations(_qp, mc, query,
+                {passwords::hash(*options.password, rng_for_salt), sstring(role_name)});
     }
 }
 
-future<> password_authenticator::alter(std::string_view role_name, const authentication_options& options) {
+future<> password_authenticator::alter(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) {
     if (!options.password) {
         co_return;
     }
@@ -293,12 +293,12 @@ future<> password_authenticator::alter(std::string_view role_name, const authent
                 {passwords::hash(*options.password, rng_for_salt), sstring(role_name)},
                 cql3::query_processor::cache_internal::no).discard_result();
     } else {
-        co_await announce_mutations(_qp, _group0_client, query,
-            {passwords::hash(*options.password, rng_for_salt), sstring(role_name)}, _as, ::service::raft_timeout{});
+        co_await collect_mutations(_qp, mc, query,
+                {passwords::hash(*options.password, rng_for_salt), sstring(role_name)});
     }
 }
 
-future<> password_authenticator::drop(std::string_view name) {
+future<> password_authenticator::drop(std::string_view name, ::service::group0_batch& mc) {
     const sstring query = format("DELETE {} FROM {}.{} WHERE {} = ?",
             SALTED_HASH,
             get_auth_ks_name(_qp),
@@ -311,7 +311,7 @@ future<> password_authenticator::drop(std::string_view name) {
                 {sstring(name)},
                 cql3::query_processor::cache_internal::no).discard_result();
     } else {
-        co_await announce_mutations(_qp, _group0_client, query, {sstring(name)}, _as, ::service::raft_timeout{});
+        co_await collect_mutations(_qp, mc, query, {sstring(name)});
     }
 }
 
@@ -329,7 +329,7 @@ const resource_set& password_authenticator::protected_resources() const {
         credentials_map credentials{};
         credentials[USERNAME_KEY] = sstring(username);
         credentials[PASSWORD_KEY] = sstring(password);
-        return this->authenticate(credentials);
+        return authenticate(credentials);
     });
 }
 

auth/password_authenticator.hh

@@ -64,11 +64,11 @@ public:
 
     virtual future<authenticated_user> authenticate(const credentials_map& credentials) const override;
 
-    virtual future<> create(std::string_view role_name, const authentication_options& options) override;
+    virtual future<> create(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) override;
 
-    virtual future<> alter(std::string_view role_name, const authentication_options& options) override;
+    virtual future<> alter(std::string_view role_name, const authentication_options& options, ::service::group0_batch&) override;
 
-    virtual future<> drop(std::string_view role_name) override;
+    virtual future<> drop(std::string_view role_name, ::service::group0_batch&) override;
 
     virtual future<custom_options> query_custom_options(std::string_view role_name) const override;
 

auth/permissions_cache.hh

@@ -13,8 +13,6 @@
 
 #include <fmt/core.h>
 #include <seastar/core/future.hh>
-#include <seastar/core/shared_ptr.hh>
-#include <seastar/core/sstring.hh>
 
 #include "auth/permission.hh"
 #include "auth/resource.hh"

auth/resource.cc

@@ -19,11 +19,16 @@
 #include <boost/algorithm/string/split.hpp>
 #include <boost/algorithm/string/classification.hpp>
 
+#include "cql3/functions/aggregate_function.hh"
+#include "cql3/functions/user_function.hh"
 #include "cql3/util.hh"
 #include "db/marshal/type_parser.hh"
+#include "log.hh"
 
 namespace auth {
 
+static logging::logger logger("auth_resource");
+
 static const std::unordered_map<resource_kind, std::string_view> roots{
         {resource_kind::data, "data"},
         {resource_kind::role, "roles"},
@@ -223,6 +228,15 @@ static sstring decoded_signature_string(std::string_view encoded_signature) {
             }), ", "));
 }
 
+resource make_functions_resource(const cql3::functions::function& f) {
+    if (!dynamic_cast<const cql3::functions::user_function*>(&f) &&
+            !dynamic_cast<const cql3::functions::aggregate_function*>(&f)) {
+        on_internal_error(logger, "unsuppported function type");
+    }
+    auto&& sig = auth::encode_signature(f.name().name, f.arg_types());
+    return make_functions_resource(f.name().keyspace, sig);
+}
+
 functions_resource_view::functions_resource_view(const resource& r) : _resource(r) {
     if (r._kind != resource_kind::functions) {
         throw resource_kind_mismatch(resource_kind::functions, r._kind);
@@ -282,6 +296,10 @@ std::optional<std::string_view> data_resource_view::keyspace() const {
     return _resource._parts[1];
 }
 
+bool data_resource_view::is_keyspace() const {
+    return _resource._parts.size() == 2;
+}
+
 std::optional<std::string_view> data_resource_view::table() const {
     if (_resource._parts.size() <= 2) {
         return {};

auth/resource.hh

@@ -18,11 +18,11 @@
 #include <unordered_set>
 
 #include <fmt/core.h>
-#include <boost/range/adaptor/transformed.hpp>
 #include <seastar/core/print.hh>
 #include <seastar/core/sstring.hh>
 
 #include "auth/permission.hh"
+#include "cql3/functions/function.hh"
 #include "seastarx.hh"
 #include "utils/hash.hh"
 #include "utils/small_vector.hh"
@@ -166,6 +166,7 @@ public:
     explicit data_resource_view(const resource& r);
 
     std::optional<std::string_view> keyspace() const;
+    bool is_keyspace() const;
 
     std::optional<std::string_view> table() const;
 };
@@ -262,6 +263,8 @@ inline resource make_functions_resource(std::string_view keyspace, std::string_v
     return resource(functions_resource_t{}, keyspace, function_name, function_signature);
 }
 
+resource make_functions_resource(const cql3::functions::function& f);
+
 sstring encode_signature(std::string_view name, std::vector<data_type> args);
 
 std::pair<sstring, std::vector<data_type>> decode_signature(std::string_view encoded_signature);

auth/role_manager.hh

@@ -20,6 +20,7 @@
 #include "auth/resource.hh"
 #include "seastarx.hh"
 #include "exceptions/exceptions.hh"
+#include "service/raft/raft_group0_client.hh"
 
 namespace auth {
 
@@ -107,17 +108,17 @@ public:
     ///
     /// \returns an exceptional future with \ref role_already_exists for a role that has previously been created.
     ///
-    virtual future<> create(std::string_view role_name, const role_config&) = 0;
+    virtual future<> create(std::string_view role_name, const role_config&, ::service::group0_batch&) = 0;
 
     ///
     /// \returns an exceptional future with \ref nonexistant_role if the role does not exist.
     ///
-    virtual future<> drop(std::string_view role_name) = 0;
+    virtual future<> drop(std::string_view role_name, ::service::group0_batch&) = 0;
 
     ///
     /// \returns an exceptional future with \ref nonexistant_role if the role does not exist.
     ///
-    virtual future<> alter(std::string_view role_name, const role_config_update&) = 0;
+    virtual future<> alter(std::string_view role_name, const role_config_update&, ::service::group0_batch&) = 0;
 
     ///
     /// Grant `role_name` to `grantee_name`.
@@ -127,7 +128,7 @@ public:
     /// \returns an exceptional future with \ref role_already_included if granting the role would be redundant, or
     /// create a cycle.
     ///
-    virtual future<> grant(std::string_view grantee_name, std::string_view role_name) = 0;
+    virtual future<> grant(std::string_view grantee_name, std::string_view role_name, ::service::group0_batch& mc) = 0;
 
     ///
     /// Revoke `role_name` from `revokee_name`.
@@ -136,7 +137,7 @@ public:
     ///
     /// \returns an exceptional future with \ref revoke_ungranted_role if the role was not granted.
     ///
-    virtual future<> revoke(std::string_view revokee_name, std::string_view role_name) = 0;
+    virtual future<> revoke(std::string_view revokee_name, std::string_view role_name, ::service::group0_batch& mc) = 0;
 
     ///
     /// \returns an exceptional future with \ref nonexistant_role if the role does not exist.
@@ -170,12 +171,12 @@ public:
     /// Sets `attribute_name` with `attribute_value` for `role_name`.
     /// \returns an exceptional future with nonexistant_role if the role does not exist.
     ///
-    virtual future<> set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value) = 0;
+    virtual future<> set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value, ::service::group0_batch& mc) = 0;
 
     /// Removes `attribute_name` for `role_name`.
     /// \returns an exceptional future with nonexistant_role if the role does not exist.
     /// \note: This is a no-op if the role does not have the named attribute set.
     ///
-    virtual future<> remove_attribute(std::string_view role_name, std::string_view attribute_name) = 0;
+    virtual future<> remove_attribute(std::string_view role_name, std::string_view attribute_name, ::service::group0_batch& mc) = 0;
 };
 }

auth/role_or_anonymous.cc

@@ -10,11 +10,6 @@
 
 namespace auth {
 
-std::ostream& operator<<(std::ostream& os, const role_or_anonymous& mr) {
-    os << mr.name.value_or("<anonymous>");
-    return os;
-}
-
 bool is_anonymous(const role_or_anonymous& mr) noexcept {
     return !mr.name.has_value();
 }

auth/roles-metadata.hh

@@ -8,6 +8,7 @@
 
 #pragma once
 
+#include <optional>
 #include <string_view>
 #include <functional>
 

auth/service.cc

@@ -6,6 +6,7 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
+#include <exception>
 #include <seastar/core/coroutine.hh>
 #include "auth/resource.hh"
 #include "auth/service.hh"
@@ -31,7 +32,9 @@
 #include "log.hh"
 #include "schema/schema_fwd.hh"
 #include <seastar/core/future.hh>
+#include <seastar/coroutine/parallel_for_each.hh>
 #include "service/migration_manager.hh"
+#include "service/raft/raft_group0_client.hh"
 #include "timestamp.hh"
 #include "utils/class_registrator.hh"
 #include "locator/abstract_replication_strategy.hh"
@@ -54,9 +57,10 @@ static logging::logger log("auth_service");
 
 class auth_migration_listener final : public ::service::migration_listener {
     authorizer& _authorizer;
+    cql3::query_processor& _qp;
 
 public:
-    explicit auth_migration_listener(authorizer& a) : _authorizer(a) {
+    explicit auth_migration_listener(authorizer& a, cql3::query_processor& qp) : _authorizer(a),  _qp(qp) {
     }
 
 private:
@@ -76,27 +80,33 @@ private:
     void on_update_tablet_metadata() override {}
 
     void on_drop_keyspace(const sstring& ks_name) override {
+        if (!legacy_mode(_qp)) {
+            // in non legacy path revoke is part of schema change statement execution
+            return;
+        }
         // Do it in the background.
-        (void)_authorizer.revoke_all(
-                auth::make_data_resource(ks_name)).handle_exception_type([](const unsupported_authorization_operation&) {
-            // Nothing.
+        (void)do_with(::service::group0_batch::unused(), [this, &ks_name] (auto& mc) mutable {
+            return _authorizer.revoke_all(auth::make_data_resource(ks_name), mc);
         }).handle_exception([] (std::exception_ptr e) {
             log.error("Unexpected exception while revoking all permissions on dropped keyspace: {}", e);
         });
-        (void)_authorizer.revoke_all(
-            auth::make_functions_resource(ks_name)).handle_exception_type([](const unsupported_authorization_operation&) {
-            // Nothing.
+
+        (void)do_with(::service::group0_batch::unused(), [this, &ks_name] (auto& mc) mutable {
+            return _authorizer.revoke_all(auth::make_functions_resource(ks_name), mc);
         }).handle_exception([] (std::exception_ptr e) {
             log.error("Unexpected exception while revoking all permissions on functions in dropped keyspace: {}", e);
         });
     }
 
     void on_drop_column_family(const sstring& ks_name, const sstring& cf_name) override {
+        if (!legacy_mode(_qp)) {
+            // in non legacy path revoke is part of schema change statement execution
+            return;
+        }
         // Do it in the background.
-        (void)_authorizer.revoke_all(
-                auth::make_data_resource(
-                        ks_name, cf_name)).handle_exception_type([](const unsupported_authorization_operation&) {
-            // Nothing.
+        (void)do_with(::service::group0_batch::unused(), [this, &ks_name, &cf_name] (auto& mc) mutable {
+            return _authorizer.revoke_all(
+                    auth::make_data_resource(ks_name, cf_name), mc);
         }).handle_exception([] (std::exception_ptr e) {
             log.error("Unexpected exception while revoking all permissions on dropped table: {}", e);
         });
@@ -104,17 +114,26 @@ private:
 
     void on_drop_user_type(const sstring& ks_name, const sstring& type_name) override {}
     void on_drop_function(const sstring& ks_name, const sstring& function_name) override {
-        (void)_authorizer.revoke_all(
-            auth::make_functions_resource(ks_name, function_name)).handle_exception_type([](const unsupported_authorization_operation&) {
-            // Nothing.
+        if (!legacy_mode(_qp)) {
+            // in non legacy path revoke is part of schema change statement execution
+            return;
+        }
+        // Do it in the background.
+        (void)do_with(::service::group0_batch::unused(), [this, &ks_name, &function_name] (auto& mc) mutable {
+            return _authorizer.revoke_all(
+                    auth::make_functions_resource(ks_name, function_name), mc);
         }).handle_exception([] (std::exception_ptr e) {
             log.error("Unexpected exception while revoking all permissions on dropped function: {}", e);
         });
     }
     void on_drop_aggregate(const sstring& ks_name, const sstring& aggregate_name) override {
-        (void)_authorizer.revoke_all(
-            auth::make_functions_resource(ks_name, aggregate_name)).handle_exception_type([](const unsupported_authorization_operation&) {
-            // Nothing.
+        if (!legacy_mode(_qp)) {
+            // in non legacy path revoke is part of schema change statement execution
+            return;
+        }
+        (void)do_with(::service::group0_batch::unused(), [this, &ks_name, &aggregate_name] (auto& mc) mutable {
+            return _authorizer.revoke_all(
+                    auth::make_functions_resource(ks_name, aggregate_name), mc);
         }).handle_exception([] (std::exception_ptr e) {
             log.error("Unexpected exception while revoking all permissions on dropped aggregate: {}", e);
         });
@@ -133,6 +152,7 @@ static future<> validate_role_exists(const service& ser, std::string_view role_n
 service::service(
         utils::loading_cache_config c,
         cql3::query_processor& qp,
+        ::service::raft_group0_client& g0,
         ::service::migration_notifier& mn,
         std::unique_ptr<authorizer> z,
         std::unique_ptr<authenticator> a,
@@ -141,11 +161,12 @@ service::service(
             : _loading_cache_config(std::move(c))
             , _permissions_cache(nullptr)
             , _qp(qp)
+            , _group0_client(g0)
             , _mnotifier(mn)
             , _authorizer(std::move(z))
             , _authenticator(std::move(a))
             , _role_manager(std::move(r))
-            , _migration_listener(std::make_unique<auth_migration_listener>(*_authorizer))
+            , _migration_listener(std::make_unique<auth_migration_listener>(*_authorizer, qp))
             , _permissions_cache_cfg_cb([this] (uint32_t) { (void) _permissions_cache_config_action.trigger_later(); })
             , _permissions_cache_config_action([this] { update_cache_config(); return make_ready_future<>(); })
             , _permissions_cache_max_entries_observer(_qp.db().get_config().permissions_cache_max_entries.observe(_permissions_cache_cfg_cb))
@@ -164,6 +185,7 @@ service::service(
             : service(
                       std::move(c),
                       qp,
+                      g0,
                       mn,
                       create_object<authorizer>(sc.authorizer_java_name, qp, g0, mm),
                       create_object<authenticator>(sc.authenticator_java_name, qp, g0, mm),
@@ -221,6 +243,7 @@ future<> service::start(::service::migration_manager& mm, db::system_keyspace& s
 }
 
 future<> service::stop() {
+    _as.request_abort();
     // Only one of the shards has the listener registered, but let's try to
     // unregister on each one just to make sure.
     return _mnotifier.unregister_listener(_migration_listener.get()).then([this] {
@@ -254,56 +277,83 @@ void service::reset_authorization_cache() {
 future<permission_set>
 service::get_uncached_permissions(const role_or_anonymous& maybe_role, const resource& r) const {
     if (is_anonymous(maybe_role)) {
-        return _authorizer->authorize(maybe_role, r);
+        co_return co_await _authorizer->authorize(maybe_role, r);
     }
-
     const std::string_view role_name = *maybe_role.name;
-
-    return has_superuser(role_name).then([this, role_name, &r](bool superuser) {
-        if (superuser) {
-            return make_ready_future<permission_set>(r.applicable_permissions());
-        }
-
-        //
-        // Aggregate the permissions from all granted roles.
-        //
-
-        return do_with(permission_set(), [this, role_name, &r](auto& all_perms) {
-            return get_roles(role_name).then([this, &r, &all_perms](role_set all_roles) {
-                return do_with(std::move(all_roles), [this, &r, &all_perms](const auto& all_roles) {
-                    return parallel_for_each(all_roles, [this, &r, &all_perms](std::string_view role_name) {
-                        return _authorizer->authorize(role_name, r).then([&all_perms](permission_set perms) {
-                            all_perms = permission_set::from_mask(all_perms.mask() | perms.mask());
-                        });
-                    });
-                });
-            }).then([&all_perms] {
-                return all_perms;
-            });
-        });
+    auto all_roles = co_await get_roles(role_name);
+    auto superuser = co_await has_superuser(role_name, all_roles);
+    if (superuser) {
+        co_return r.applicable_permissions();
+    }
+    // Aggregate the permissions from all granted roles.
+    permission_set all_perms;
+    co_await coroutine::parallel_for_each(all_roles, [this, &r, &all_perms](std::string_view role_name) -> future<> {
+        auto perms = co_await _authorizer->authorize(role_name, r);
+        all_perms = permission_set::from_mask(all_perms.mask() | perms.mask());
     });
+    co_return std::move(all_perms);
 }
 
 future<permission_set> service::get_permissions(const role_or_anonymous& maybe_role, const resource& r) const {
     return _permissions_cache->get(maybe_role, r);
 }
 
+future<bool> service::has_superuser(std::string_view role_name, const role_set& roles) const {
+    for (const auto& role : roles) {
+        if (co_await _role_manager->is_superuser(role)) {
+            co_return true;
+        }
+    }
+    co_return false;
+}
+
 future<bool> service::has_superuser(std::string_view role_name) const {
-    return this->get_roles(std::move(role_name)).then([this](role_set roles) {
-        return do_with(std::move(roles), [this](const role_set& roles) {
-            return do_with(false, roles.begin(), [this, &roles](bool& any_super, auto& iter) {
-                return do_until(
-                        [&roles, &any_super, &iter] { return any_super || (iter == roles.end()); },
-                        [this, &any_super, &iter] {
-                    return _role_manager->is_superuser(*iter++).then([&any_super](bool super) {
-                        any_super = super;
-                    });
-                }).then([&any_super] {
-                    return any_super;
-                });
-            });
-        });
-    });
+    auto roles = co_await get_roles(role_name);
+    co_return co_await has_superuser(role_name, roles);
+}
+
+static void validate_authentication_options_are_supported(
+        const authentication_options& options,
+        const authentication_option_set& supported) {
+    const auto check = [&supported](authentication_option k) {
+        if (!supported.contains(k)) {
+            throw unsupported_authentication_option(k);
+        }
+    };
+
+    if (options.password) {
+        check(authentication_option::password);
+    }
+
+    if (options.options) {
+        check(authentication_option::options);
+    }
+}
+
+future<> service::create_role(std::string_view name,
+        const role_config& config,
+        const authentication_options& options,
+        ::service::group0_batch& mc) const {
+    co_await underlying_role_manager().create(name, config, mc);
+    if (!auth::any_authentication_options(options)) {
+        co_return;
+    }
+    std::exception_ptr ep;
+    try {
+        validate_authentication_options_are_supported(options,
+                underlying_authenticator().supported_options());
+        co_await underlying_authenticator().create(name, options, mc);
+    } catch (...) {
+        ep = std::current_exception();
+    }
+    if (ep) {
+        // Rollback only in legacy mode as normally mutations won't be
+        // applied in case exception is raised
+        if (legacy_mode(_qp)) {
+            co_await underlying_role_manager().drop(name, mc);
+        }
+        std::rethrow_exception(std::move(ep));
+    }
 }
 
 future<role_set> service::get_roles(std::string_view role_name) const {
@@ -360,7 +410,7 @@ future<bool> service::exists(const resource& r) const {
                 return make_ready_future<bool>(db.has_keyspace(sstring(*keyspace)));
             }
             auto [name, function_args] = auth::decode_signature(*function_signature);
-            return make_ready_future<bool>(cql3::functions::functions::find(db::functions::function_name{sstring(*keyspace), name}, function_args));
+            return make_ready_future<bool>(cql3::functions::instance().find(db::functions::function_name{sstring(*keyspace), name}, function_args));
         }
     }
 
@@ -394,15 +444,6 @@ future<permission_set> get_permissions(const service& ser, const authenticated_u
     });
 }
 
-bool is_enforcing(const service& ser)  {
-    const bool enforcing_authorizer = ser.underlying_authorizer().qualified_java_name() != allow_all_authorizer_name;
-
-    const bool enforcing_authenticator = ser.underlying_authenticator().qualified_java_name()
-            != allow_all_authenticator_name;
-
-    return enforcing_authorizer || enforcing_authenticator;
-}
-
 bool is_protected(const service& ser, command_desc cmd) noexcept {
     if (cmd.type_ == command_desc::type::ALTER_WITH_OPTS) {
         return false; // Table attributes are OK to modify; see #7057.
@@ -412,84 +453,45 @@ bool is_protected(const service& ser, command_desc cmd) noexcept {
             || ser.underlying_authorizer().protected_resources().contains(cmd.resource);
 }
 
-static void validate_authentication_options_are_supported(
-        const authentication_options& options,
-        const authentication_option_set& supported) {
-    const auto check = [&supported](authentication_option k) {
-        if (!supported.contains(k)) {
-            throw unsupported_authentication_option(k);
-        }
-    };
-
-    if (options.password) {
-        check(authentication_option::password);
-    }
-
-    if (options.options) {
-        check(authentication_option::options);
-    }
-}
-
-
 future<> create_role(
         const service& ser,
         std::string_view name,
         const role_config& config,
-        const authentication_options& options) {
-    return ser.underlying_role_manager().create(name, config).then([&ser, name, &options] {
-        if (!auth::any_authentication_options(options)) {
-            return make_ready_future<>();
-        }
-
-        return futurize_invoke(
-                &validate_authentication_options_are_supported,
-                options,
-                ser.underlying_authenticator().supported_options()).then([&ser, name, &options] {
-            return ser.underlying_authenticator().create(name, options);
-        }).handle_exception([&ser, name](std::exception_ptr ep) {
-            // Roll-back.
-            return ser.underlying_role_manager().drop(name).then([ep = std::move(ep)] {
-                std::rethrow_exception(ep);
-            });
-        });
-    });
+        const authentication_options& options,
+        ::service::group0_batch& mc) {
+    return ser.create_role(name, config, options, mc);
 }
 
 future<> alter_role(
         const service& ser,
         std::string_view name,
         const role_config_update& config_update,
-        const authentication_options& options) {
-    return ser.underlying_role_manager().alter(name, config_update).then([&ser, name, &options] {
-        if (!any_authentication_options(options)) {
-            return make_ready_future<>();
-        }
-
-        return futurize_invoke(
-                &validate_authentication_options_are_supported,
-                options,
-                ser.underlying_authenticator().supported_options()).then([&ser, name, &options] {
-            return ser.underlying_authenticator().alter(name, options);
-        });
-    });
+        const authentication_options& options,
+        ::service::group0_batch& mc) {
+    co_await ser.underlying_role_manager().alter(name, config_update, mc);
+    if (!any_authentication_options(options)) {
+        co_return;
+    }
+    validate_authentication_options_are_supported(options,
+            ser.underlying_authenticator().supported_options());
+    co_await ser.underlying_authenticator().alter(name, options, mc);
 }
 
-future<> drop_role(const service& ser, std::string_view name) {
-    return do_with(make_role_resource(name), [&ser, name](const resource& r) {
-        auto& a = ser.underlying_authorizer();
+future<> drop_role(const service& ser, std::string_view name, ::service::group0_batch& mc) {
+    auto& a = ser.underlying_authorizer();
+    auto r = make_role_resource(name);
+    co_await a.revoke_all(name, mc);
+    co_await a.revoke_all(r, mc);
+    co_await ser.underlying_authenticator().drop(name, mc);
+    co_await ser.underlying_role_manager().drop(name, mc);
+}
 
-        return when_all_succeed(
-                a.revoke_all(name),
-                a.revoke_all(r))
-                    .discard_result()
-                    .handle_exception_type([](const unsupported_authorization_operation&) {
-            // Nothing.
-        });
-    }).then([&ser, name] {
-        return ser.underlying_authenticator().drop(name);
-    }).then([&ser, name] {
-        return ser.underlying_role_manager().drop(name);
-    });
+future<> grant_role(const service& ser, std::string_view grantee_name, std::string_view role_name, ::service::group0_batch& mc) {
+    return ser.underlying_role_manager().grant(grantee_name, role_name, mc);
+}
+
+future<> revoke_role(const service& ser, std::string_view revokee_name, std::string_view role_name, ::service::group0_batch& mc) {
+    return ser.underlying_role_manager().revoke(revokee_name, role_name, mc);
 }
 
 future<bool> has_role(const service& ser, std::string_view grantee, std::string_view name) {
@@ -507,35 +509,47 @@ future<bool> has_role(const service& ser, const authenticated_user& u, std::stri
     return has_role(ser, *u.name, name);
 }
 
+future<> set_attribute(const service& ser, std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value, ::service::group0_batch& mc) {
+    return ser.underlying_role_manager().set_attribute(role_name, attribute_name, attribute_value, mc);
+}
+
+future<> remove_attribute(const service& ser, std::string_view role_name, std::string_view attribute_name, ::service::group0_batch& mc) {
+    return ser.underlying_role_manager().remove_attribute(role_name, attribute_name, mc);
+}
+
 future<> grant_permissions(
         const service& ser,
         std::string_view role_name,
         permission_set perms,
-        const resource& r) {
-    return validate_role_exists(ser, role_name).then([&ser, role_name, perms, &r] {
-        return ser.underlying_authorizer().grant(role_name, perms, r);
-    });
+        const resource& r,
+        ::service::group0_batch& mc) {
+    co_await validate_role_exists(ser, role_name);
+    co_await ser.underlying_authorizer().grant(role_name, perms, r, mc);
 }
 
-future<> grant_applicable_permissions(const service& ser, std::string_view role_name, const resource& r) {
-    return grant_permissions(ser, role_name, r.applicable_permissions(), r);
+future<> grant_applicable_permissions(const service& ser, std::string_view role_name, const resource& r, ::service::group0_batch& mc) {
+    return grant_permissions(ser, role_name, r.applicable_permissions(), r, mc);
 }
-future<> grant_applicable_permissions(const service& ser, const authenticated_user& u, const resource& r) {
+
+future<> grant_applicable_permissions(const service& ser, const authenticated_user& u, const resource& r, ::service::group0_batch& mc) {
     if (is_anonymous(u)) {
         return make_ready_future<>();
     }
-
-    return grant_applicable_permissions(ser, *u.name, r);
+    return grant_applicable_permissions(ser, *u.name, r, mc);
 }
 
 future<> revoke_permissions(
         const service& ser,
         std::string_view role_name,
         permission_set perms,
-        const resource& r) {
-    return validate_role_exists(ser, role_name).then([&ser, role_name, perms, &r] {
-        return ser.underlying_authorizer().revoke(role_name, perms, r);
-    });
+        const resource& r,
+        ::service::group0_batch& mc) {
+    co_await validate_role_exists(ser, role_name);
+    co_await ser.underlying_authorizer().revoke(role_name, perms, r, mc);
+}
+
+future<> revoke_all(const service& ser, const resource& r, ::service::group0_batch& mc) {
+    return ser.underlying_authorizer().revoke_all(r, mc);
 }
 
 future<std::vector<permission_details>> list_filtered_permissions(
@@ -592,10 +606,14 @@ future<std::vector<permission_details>> list_filtered_permissions(
     });
 }
 
+future<> commit_mutations(service& ser, ::service::group0_batch&& mc) {
+    return ser.commit_mutations(std::move(mc));
+}
+
 future<> migrate_to_auth_v2(db::system_keyspace& sys_ks, ::service::raft_group0_client& g0, start_operation_func_t start_operation_func, abort_source& as) {
     // FIXME: if this function fails it may leave partial data in the new tables
     // that should be cleared
-    auto gen = [&sys_ks] (api::timestamp_type& ts) -> mutations_generator {
+    auto gen = [&sys_ks] (api::timestamp_type ts) -> ::service::mutations_generator {
         auto& qp = sys_ks.query_processor();
         for (const auto& cf_name : std::vector<sstring>{
                 "roles", "role_members", "role_attributes", "role_permissions"}) {

auth/service.hh

@@ -27,7 +27,6 @@
 #include "service/raft/raft_group0_client.hh"
 #include "utils/observable.hh"
 #include "utils/serialized_action.hh"
-#include "db/config.hh"
 #include "service/maintenance_mode.hh"
 
 namespace cql3 {
@@ -79,6 +78,8 @@ class service final : public seastar::peering_sharded_service<service> {
 
     cql3::query_processor& _qp;
 
+    ::service::raft_group0_client& _group0_client;
+
     ::service::migration_notifier& _mnotifier;
 
     authorizer::ptr_type _authorizer;
@@ -99,10 +100,13 @@ class service final : public seastar::peering_sharded_service<service> {
 
     maintenance_socket_enabled _used_by_maintenance_socket;
 
+    abort_source _as;
+
 public:
     service(
             utils::loading_cache_config,
             cql3::query_processor&,
+            ::service::raft_group0_client&,
             ::service::migration_notifier&,
             std::unique_ptr<authorizer>,
             std::unique_ptr<authenticator>,
@@ -150,6 +154,18 @@ public:
     ///
     future<bool> has_superuser(std::string_view role_name) const;
 
+    ///
+    /// Create a role with optional authentication information.
+    ///
+    /// \returns an exceptional future with \ref role_already_exists if the user or role exists.
+    ///
+    /// \returns an exceptional future with \ref unsupported_authentication_option if an unsupported option is included.
+    ///
+    future<> create_role(std::string_view name,
+            const role_config& config,
+            const authentication_options& options,
+            ::service::group0_batch& mc) const;
+
     ///
     /// Return the set of all roles granted to the given role, including itself and roles granted through other roles.
     ///
@@ -174,8 +190,13 @@ public:
         return _qp;
     }
 
+    future<> commit_mutations(::service::group0_batch&& mc) {
+        return std::move(mc).commit(_group0_client, _as, ::service::raft_timeout{});
+    }
+
 private:
     future<> create_legacy_keyspace_if_missing(::service::migration_manager& mm) const;
+    future<bool> has_superuser(std::string_view role_name, const role_set& roles) const;
 };
 
 future<bool> has_superuser(const service&, const authenticated_user&);
@@ -184,14 +205,6 @@ future<role_set> get_roles(const service&, const authenticated_user&);
 
 future<permission_set> get_permissions(const service&, const authenticated_user&, const resource&);
 
-///
-/// Access-control is "enforcing" when either the authenticator or the authorizer are not their "allow-all" variants.
-///
-/// Put differently, when access control is not enforcing, all operations on resources will be allowed and users do not
-/// need to authenticate themselves.
-///
-bool is_enforcing(const service&);
-
 /// A description of a CQL command from which auth::service can tell whether or not this command could endanger
 /// internal data on which auth::service depends.
 struct command_desc {
@@ -219,7 +232,8 @@ future<> create_role(
         const service&,
         std::string_view name,
         const role_config&,
-        const authentication_options&);
+        const authentication_options&,
+        ::service::group0_batch&);
 
 ///
 /// Alter an existing role and its authentication information.
@@ -232,14 +246,34 @@ future<> alter_role(
         const service&,
         std::string_view name,
         const role_config_update&,
-        const authentication_options&);
+        const authentication_options&,
+        ::service::group0_batch& mc);
 
 ///
 /// Drop a role from the system, including all permissions and authentication information.
 ///
 /// \returns an exceptional future with \ref nonexistant_role if the named role does not exist.
 ///
-future<> drop_role(const service&, std::string_view name);
+future<> drop_role(const service&, std::string_view name, ::service::group0_batch& mc);
+
+///
+/// Grant `role_name` to `grantee_name`.
+///
+/// \returns an exceptional future with \ref nonexistant_role if either the role or the grantee do not exist.
+///
+/// \returns an exceptional future with \ref role_already_included if granting the role would be redundant, or
+/// create a cycle.
+///
+future<> grant_role(const service&, std::string_view grantee_name, std::string_view role_name, ::service::group0_batch& mc);
+
+///
+/// Revoke `role_name` from `revokee_name`.
+///
+/// \returns an exceptional future with \ref nonexistant_role if either the role or the revokee do not exist.
+///
+/// \returns an exceptional future with \ref revoke_ungranted_role if the role was not granted.
+///
+future<> revoke_role(const service&, std::string_view revokee_name, std::string_view role_name, ::service::group0_batch& mc);
 
 ///
 /// Check if `grantee` has been granted the named role.
@@ -254,6 +288,18 @@ future<bool> has_role(const service&, std::string_view grantee, std::string_view
 ///
 future<bool> has_role(const service&, const authenticated_user&, std::string_view name);
 
+
+/// Sets `attribute_name` with `attribute_value` for `role_name`.
+/// \returns an exceptional future with nonexistant_role if the role does not exist.
+///
+future<> set_attribute(const service&, std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value, ::service::group0_batch& mc);
+
+/// Removes `attribute_name` for `role_name`.
+/// \returns an exceptional future with nonexistant_role if the role does not exist.
+/// \note: This is a no-op if the role does not have the named attribute set.
+///
+future<> remove_attribute(const service&, std::string_view role_name, std::string_view attribute_name, ::service::group0_batch& mc);
+
 ///
 /// \returns an exceptional future with \ref nonexistent_role if the named role does not exist.
 ///
@@ -264,7 +310,8 @@ future<> grant_permissions(
         const service&,
         std::string_view role_name,
         permission_set,
-        const resource&);
+        const resource&,
+        ::service::group0_batch&);
 
 ///
 /// Like \ref grant_permissions, but grants all applicable permissions on the resource.
@@ -274,8 +321,8 @@ future<> grant_permissions(
 /// \returns an exceptional future with \ref unsupported_authorization_operation if granting permissions is not
 /// supported.
 ///
-future<> grant_applicable_permissions(const service&, std::string_view role_name, const resource&);
-future<> grant_applicable_permissions(const service&, const authenticated_user&, const resource&);
+future<> grant_applicable_permissions(const service&, std::string_view role_name, const resource&, ::service::group0_batch&);
+future<> grant_applicable_permissions(const service&, const authenticated_user&, const resource&, ::service::group0_batch&);
 
 ///
 /// \returns an exceptional future with \ref nonexistent_role if the named role does not exist.
@@ -287,7 +334,15 @@ future<> revoke_permissions(
         const service&,
         std::string_view role_name,
         permission_set,
-        const resource&);
+        const resource&,
+        ::service::group0_batch&);
+
+///
+/// Revoke all permissions granted to any role for a particular resource.
+///
+/// \throws \ref unsupported_authorization_operation if revoking permissions is not supported.
+///
+future<> revoke_all(const service&, const resource&, ::service::group0_batch&);
 
 using recursive_permissions = bool_class<struct recursive_permissions_tag>;
 
@@ -313,6 +368,10 @@ future<std::vector<permission_details>> list_filtered_permissions(
         std::optional<std::string_view> role_name,
         const std::optional<std::pair<resource, recursive_permissions>>& resource_filter);
 
+
+// Finalizes write operations performed in auth by committing mutations via raft group0.
+future<> commit_mutations(service& ser, ::service::group0_batch&& mc);
+
 // Migrates data from old keyspace to new one which supports linearizable writes via raft.
 future<> migrate_to_auth_v2(db::system_keyspace& sys_ks, ::service::raft_group0_client& g0, start_operation_func_t start_operation_func, abort_source& as);
 

auth/standard_role_manager.cc

@@ -14,6 +14,7 @@
 
 #include <boost/algorithm/string/join.hpp>
 #include <seastar/core/future-util.hh>
+#include <seastar/core/on_internal_error.hh>
 #include <seastar/core/print.hh>
 #include <seastar/core/sleep.hh>
 #include <seastar/core/sstring.hh>
@@ -26,6 +27,7 @@
 #include "db/consistency_level_type.hh"
 #include "exceptions/exceptions.hh"
 #include "log.hh"
+#include "service/raft/raft_group0_client.hh"
 #include "utils/class_registrator.hh"
 #include "service/migration_manager.hh"
 #include "password_authenticator.hh"
@@ -222,8 +224,9 @@ future<> standard_role_manager::migrate_legacy_metadata() {
             return do_with(
                     row.get_as<sstring>("name"),
                     std::move(config),
-                    [this](const auto& name, const auto& config) {
-                return this->create_or_replace(name, config);
+                    ::service::group0_batch::unused(),
+                    [this](const auto& name, const auto& config, auto& mc) {
+                return create_or_replace(name, config, mc);
             });
         }).finally([results] {});
     }).then([] {
@@ -238,7 +241,7 @@ future<> standard_role_manager::start() {
     return once_among_shards([this] {
         return futurize_invoke([this] () {
             if (legacy_mode(_qp)) {
-                return this->create_legacy_metadata_tables_if_missing();
+                return create_legacy_metadata_tables_if_missing();
             }
             return make_ready_future<>();
         }).then([this] {
@@ -248,15 +251,15 @@ future<> standard_role_manager::start() {
                         _migration_manager.wait_for_schema_agreement(_qp.db().real_database(), db::timeout_clock::time_point::max(), &_as).get();
 
                         if (any_nondefault_role_row_satisfies(_qp, &has_can_login).get()) {
-                            if (this->legacy_metadata_exists()) {
+                            if (legacy_metadata_exists()) {
                                 log.warn("Ignoring legacy user metadata since nondefault roles already exist.");
                             }
 
                             return;
                         }
 
-                        if (this->legacy_metadata_exists()) {
-                            this->migrate_legacy_metadata().get();
+                        if (legacy_metadata_exists()) {
+                            migrate_legacy_metadata().get();
                             return;
                         }
                     }
@@ -272,7 +275,7 @@ future<> standard_role_manager::stop() {
     return _stopped.handle_exception_type([] (const sleep_aborted&) { }).handle_exception_type([](const abort_requested_exception&) {});;
 }
 
-future<> standard_role_manager::create_or_replace(std::string_view role_name, const role_config& c) {
+future<> standard_role_manager::create_or_replace(std::string_view role_name, const role_config& c, ::service::group0_batch& mc) {
     const sstring query = format("INSERT INTO {}.{} ({}, is_superuser, can_login) VALUES (?, ?, ?)",
             get_auth_ks_name(_qp),
             meta::roles_table::name,
@@ -285,23 +288,23 @@ future<> standard_role_manager::create_or_replace(std::string_view role_name, co
                 {sstring(role_name), c.is_superuser, c.can_login},
                 cql3::query_processor::cache_internal::yes).discard_result();
     } else {
-        co_await announce_mutations(_qp, _group0_client, query, {sstring(role_name), c.is_superuser, c.can_login}, _as, ::service::raft_timeout{});
+        co_await collect_mutations(_qp, mc,  query, {sstring(role_name), c.is_superuser, c.can_login});
     }
 }
 
 future<>
-standard_role_manager::create(std::string_view role_name, const role_config& c) {
-    return this->exists(role_name).then([this, role_name, &c](bool role_exists) {
+standard_role_manager::create(std::string_view role_name, const role_config& c, ::service::group0_batch& mc) {
+    return exists(role_name).then([this, role_name, &c, &mc](bool role_exists) {
         if (role_exists) {
             throw role_already_exists(role_name);
         }
 
-        return this->create_or_replace(role_name, c);
+        return create_or_replace(role_name, c, mc);
     });
 }
 
 future<>
-standard_role_manager::alter(std::string_view role_name, const role_config_update& u) {
+standard_role_manager::alter(std::string_view role_name, const role_config_update& u, ::service::group0_batch& mc) {
     static const auto build_column_assignments = [](const role_config_update& u) -> sstring {
         std::vector<sstring> assignments;
 
@@ -316,7 +319,7 @@ standard_role_manager::alter(std::string_view role_name, const role_config_updat
         return boost::algorithm::join(assignments, ", ");
     };
 
-    return require_record(_qp, role_name).then([this, role_name, &u](record) {
+    return require_record(_qp, role_name).then([this, role_name, &u, &mc](record) {
         if (!u.is_superuser && !u.can_login) {
             return make_ready_future<>();
         }
@@ -333,17 +336,17 @@ standard_role_manager::alter(std::string_view role_name, const role_config_updat
                     {sstring(role_name)},
                     cql3::query_processor::cache_internal::no).discard_result();
         } else {
-            return announce_mutations(_qp, _group0_client, std::move(query), {sstring(role_name)}, _as, ::service::raft_timeout{});
+            return collect_mutations(_qp, mc, std::move(query), {sstring(role_name)});
         }
     });
 }
 
-future<> standard_role_manager::drop(std::string_view role_name) {
-    if (!co_await this->exists(role_name)) {
+future<> standard_role_manager::drop(std::string_view role_name, ::service::group0_batch& mc) {
+    if (!co_await exists(role_name)) {
         throw nonexistant_role(role_name);
     }
     // First, revoke this role from all roles that are members of it.
-    const auto revoke_from_members = [this, role_name] () -> future<> {
+    const auto revoke_from_members = [this, role_name, &mc] () -> future<> {
         const sstring query = format("SELECT member FROM {}.{} WHERE role = ?",
                 get_auth_ks_name(_qp),
                 meta::role_members_table::name);
@@ -356,26 +359,26 @@ future<> standard_role_manager::drop(std::string_view role_name) {
         co_await parallel_for_each(
                 members->begin(),
                 members->end(),
-                [this, role_name] (const cql3::untyped_result_set_row& member_row) -> future<> {
+                [this, role_name, &mc] (const cql3::untyped_result_set_row& member_row) -> future<> {
                     const sstring member = member_row.template get_as<sstring>("member");
-                    co_await this->modify_membership(member, role_name, membership_change::remove);
+                    co_await modify_membership(member, role_name, membership_change::remove, mc);
                 }
         );
     };
     // In parallel, revoke all roles that this role is members of.
-    const auto revoke_members_of = [this, grantee = role_name] () -> future<> {
-        const role_set granted_roles = co_await this->query_granted(
+    const auto revoke_members_of = [this, grantee = role_name, &mc] () -> future<> {
+        const role_set granted_roles = co_await query_granted(
                 grantee,
                 recursive_role_query::no);
         co_await parallel_for_each(
                 granted_roles.begin(),
                 granted_roles.end(),
-                [this, grantee](const sstring& role_name) {
-            return this->modify_membership(grantee, role_name, membership_change::remove);
+                [this, grantee, &mc](const sstring& role_name) {
+            return modify_membership(grantee, role_name, membership_change::remove, mc);
         });
     };
     // Delete all attributes for that role
-    const auto remove_attributes_of = [this, role_name] () -> future<> {
+    const auto remove_attributes_of = [this, role_name, &mc] () -> future<> {
         const sstring query = format("DELETE FROM {}.{} WHERE role = ?",
                 get_auth_ks_name(_qp),
                 meta::role_attributes_table::name);
@@ -383,11 +386,11 @@ future<> standard_role_manager::drop(std::string_view role_name) {
             co_await _qp.execute_internal(query, {sstring(role_name)},
                 cql3::query_processor::cache_internal::yes).discard_result();
         } else {
-            co_await announce_mutations(_qp, _group0_client, query, {sstring(role_name)}, _as, ::service::raft_timeout{});
+            co_await collect_mutations(_qp, mc, query, {sstring(role_name)});
         }
     };
     // Finally, delete the role itself.
-    const auto delete_role = [this, role_name] () -> future<> {
+    const auto delete_role = [this, role_name, &mc] () -> future<> {
         const sstring query = format("DELETE FROM {}.{} WHERE {} = ?",
                 get_auth_ks_name(_qp),
                 meta::roles_table::name,
@@ -401,7 +404,7 @@ future<> standard_role_manager::drop(std::string_view role_name) {
                     {sstring(role_name)},
                     cql3::query_processor::cache_internal::no).discard_result();
         } else {
-            co_await announce_mutations(_qp, _group0_client, query, {sstring(role_name)}, _as, ::service::raft_timeout{});
+            co_await collect_mutations(_qp, mc, query, {sstring(role_name)});
         }
     };
 
@@ -410,14 +413,10 @@ future<> standard_role_manager::drop(std::string_view role_name) {
 }
 
 future<>
-standard_role_manager::modify_membership(
+standard_role_manager::legacy_modify_membership(
         std::string_view grantee_name,
         std::string_view role_name,
         membership_change ch) {
-    // FIXME: in auth-v2 mode callers of this function should use a single guard
-    // to achieve consistent data across read and write, but the structure of calls
-    // is too complex to make such a refactor a quick fix.
-
     const auto modify_roles = [this, role_name, grantee_name, ch] () -> future<> {
         const auto query = format(
                 "UPDATE {}.{} SET member_of = member_of {} ? WHERE {} = ?",
@@ -425,17 +424,12 @@ standard_role_manager::modify_membership(
                 meta::roles_table::name,
                 (ch == membership_change::add ? '+' : '-'),
                 meta::roles_table::role_col_name);
-        if (legacy_mode(_qp)) {
-            co_await _qp.execute_internal(
-                    query,
-                    consistency_for_role(grantee_name),
-                    internal_distributed_query_state(),
-                    {role_set{sstring(role_name)}, sstring(grantee_name)},
-                    cql3::query_processor::cache_internal::no).discard_result();
-        } else {
-            co_await announce_mutations(_qp, _group0_client, std::move(query),
-                    {role_set{sstring(role_name)}, sstring(grantee_name)}, _as, ::service::raft_timeout{});
-        }
+        co_await _qp.execute_internal(
+                query,
+                consistency_for_role(grantee_name),
+                internal_distributed_query_state(),
+                {role_set{sstring(role_name)}, sstring(grantee_name)},
+                cql3::query_processor::cache_internal::no).discard_result();
     };
 
     const auto modify_role_members = [this, role_name, grantee_name, ch] () -> future<> {
@@ -444,34 +438,24 @@ standard_role_manager::modify_membership(
                 const sstring insert_query = format("INSERT INTO {}.{} (role, member) VALUES (?, ?)",
                         get_auth_ks_name(_qp),
                         meta::role_members_table::name);
-                if (legacy_mode(_qp)) {
-                    co_return co_await _qp.execute_internal(
-                            insert_query,
-                            consistency_for_role(role_name),
-                            internal_distributed_query_state(),
-                            {sstring(role_name), sstring(grantee_name)},
-                            cql3::query_processor::cache_internal::no).discard_result();
-                } else {
-                    co_return co_await announce_mutations(_qp, _group0_client, insert_query,
-                            {sstring(role_name), sstring(grantee_name)}, _as, ::service::raft_timeout{});
-                }
+                co_return co_await _qp.execute_internal(
+                        insert_query,
+                        consistency_for_role(role_name),
+                        internal_distributed_query_state(),
+                        {sstring(role_name), sstring(grantee_name)},
+                        cql3::query_processor::cache_internal::no).discard_result();
             }
 
             case membership_change::remove: {
                 const sstring delete_query = format("DELETE FROM {}.{} WHERE role = ? AND member = ?",
                         get_auth_ks_name(_qp),
                         meta::role_members_table::name);
-                if (legacy_mode(_qp)) {
-                    co_return co_await _qp.execute_internal(
-                            delete_query,
-                            consistency_for_role(role_name),
-                            internal_distributed_query_state(),
-                            {sstring(role_name), sstring(grantee_name)},
-                            cql3::query_processor::cache_internal::no).discard_result();
-                } else {
-                    co_return co_await announce_mutations(_qp, _group0_client, delete_query,
-                            {sstring(role_name), sstring(grantee_name)}, _as, ::service::raft_timeout{});
-                }
+                co_return co_await _qp.execute_internal(
+                        delete_query,
+                        consistency_for_role(role_name),
+                        internal_distributed_query_state(),
+                        {sstring(role_name), sstring(grantee_name)},
+                        cql3::query_processor::cache_internal::no).discard_result();
             }
         }
     };
@@ -480,9 +464,47 @@ standard_role_manager::modify_membership(
 }
 
 future<>
-standard_role_manager::grant(std::string_view grantee_name, std::string_view role_name) {
+standard_role_manager::modify_membership(
+        std::string_view grantee_name,
+        std::string_view role_name,
+        membership_change ch,
+        ::service::group0_batch& mc) {
+    if (legacy_mode(_qp)) {
+        co_return co_await legacy_modify_membership(grantee_name, role_name, ch);
+    }
+
+    const auto modify_roles = format(
+            "UPDATE {}.{} SET member_of = member_of {} ? WHERE {} = ?",
+            get_auth_ks_name(_qp),
+            meta::roles_table::name,
+            (ch == membership_change::add ? '+' : '-'),
+            meta::roles_table::role_col_name);
+    co_await collect_mutations(_qp, mc, modify_roles,
+            {role_set{sstring(role_name)}, sstring(grantee_name)});
+
+    sstring modify_role_members;
+    switch (ch) {
+    case membership_change::add:
+        modify_role_members = format("INSERT INTO {}.{} (role, member) VALUES (?, ?)",
+                get_auth_ks_name(_qp),
+                meta::role_members_table::name);
+        break;
+    case membership_change::remove:
+        modify_role_members = format("DELETE FROM {}.{} WHERE role = ? AND member = ?",
+                get_auth_ks_name(_qp),
+                meta::role_members_table::name);
+        break;
+    default:
+        on_internal_error(log, format("unknown membership_change value: {}", int(ch)));
+    }
+    co_await collect_mutations(_qp, mc, modify_role_members,
+            {sstring(role_name), sstring(grantee_name)});
+}
+
+future<>
+standard_role_manager::grant(std::string_view grantee_name, std::string_view role_name, ::service::group0_batch& mc) {
     const auto check_redundant = [this, role_name, grantee_name] {
-        return this->query_granted(
+        return query_granted(
                 grantee_name,
                 recursive_role_query::yes).then([role_name, grantee_name](role_set roles) {
             if (roles.contains(sstring(role_name))) {
@@ -494,7 +516,7 @@ standard_role_manager::grant(std::string_view grantee_name, std::string_view rol
     };
 
     const auto check_cycle = [this, role_name, grantee_name] {
-        return this->query_granted(
+        return query_granted(
                 role_name,
                 recursive_role_query::yes).then([role_name, grantee_name](role_set roles) {
             if (roles.contains(sstring(grantee_name))) {
@@ -505,19 +527,19 @@ standard_role_manager::grant(std::string_view grantee_name, std::string_view rol
         });
     };
 
-   return when_all_succeed(check_redundant(), check_cycle()).then_unpack([this, role_name, grantee_name] {
-       return this->modify_membership(grantee_name, role_name, membership_change::add);
-   });
+    return when_all_succeed(check_redundant(), check_cycle()).then_unpack([this, role_name, grantee_name, &mc] {
+        return modify_membership(grantee_name, role_name, membership_change::add, mc);
+    });
 }
 
 future<>
-standard_role_manager::revoke(std::string_view revokee_name, std::string_view role_name) {
-    return this->exists(role_name).then([role_name](bool role_exists) {
+standard_role_manager::revoke(std::string_view revokee_name, std::string_view role_name, ::service::group0_batch& mc) {
+    return exists(role_name).then([role_name](bool role_exists) {
         if (!role_exists) {
             throw nonexistant_role(sstring(role_name));
         }
-    }).then([this, revokee_name, role_name] {
-        return this->query_granted(
+    }).then([this, revokee_name, role_name, &mc] {
+        return query_granted(
                 revokee_name,
                 recursive_role_query::no).then([revokee_name, role_name](role_set roles) {
             if (!roles.contains(sstring(role_name))) {
@@ -525,8 +547,8 @@ standard_role_manager::revoke(std::string_view revokee_name, std::string_view ro
             }
 
             return make_ready_future<>();
-        }).then([this, revokee_name, role_name] {
-            return this->modify_membership(revokee_name, role_name, membership_change::remove);
+        }).then([this, revokee_name, role_name, &mc] {
+            return modify_membership(revokee_name, role_name, membership_change::remove, mc);
         });
     });
 }
@@ -633,7 +655,7 @@ future<role_manager::attribute_vals> standard_role_manager::query_attribute_for_
     });
 }
 
-future<> standard_role_manager::set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value) {
+future<> standard_role_manager::set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value, ::service::group0_batch& mc) {
     if (!co_await exists(role_name)) {
         throw auth::nonexistant_role(role_name);
     }
@@ -643,12 +665,12 @@ future<> standard_role_manager::set_attribute(std::string_view role_name, std::s
     if (legacy_mode(_qp)) {
         co_await _qp.execute_internal(query, {sstring(role_name), sstring(attribute_name), sstring(attribute_value)}, cql3::query_processor::cache_internal::yes).discard_result();
     } else {
-        co_await announce_mutations(_qp, _group0_client, query,
-                {sstring(role_name), sstring(attribute_name), sstring(attribute_value)}, _as, ::service::raft_timeout{});
+        co_await collect_mutations(_qp, mc, query,
+                {sstring(role_name), sstring(attribute_name), sstring(attribute_value)});
     }
 }
 
-future<> standard_role_manager::remove_attribute(std::string_view role_name, std::string_view attribute_name) {
+future<> standard_role_manager::remove_attribute(std::string_view role_name, std::string_view attribute_name, ::service::group0_batch& mc) {
     if (!co_await exists(role_name)) {
         throw auth::nonexistant_role(role_name);
     }
@@ -658,8 +680,8 @@ future<> standard_role_manager::remove_attribute(std::string_view role_name, std
     if (legacy_mode(_qp)) {
         co_await _qp.execute_internal(query, {sstring(role_name), sstring(attribute_name)}, cql3::query_processor::cache_internal::yes).discard_result();
     } else {
-        co_await announce_mutations(_qp, _group0_client, query,
-                {sstring(role_name), sstring(attribute_name)}, _as, ::service::raft_timeout{});
+        co_await collect_mutations(_qp, mc, query,
+                {sstring(role_name), sstring(attribute_name)});
     }
 }
 }

auth/standard_role_manager.hh

@@ -8,6 +8,7 @@
 
 #pragma once
 
+#include "auth/common.hh"
 #include "auth/role_manager.hh"
 
 #include <string_view>
@@ -48,15 +49,15 @@ public:
 
     virtual future<> stop() override;
 
-    virtual future<> create(std::string_view role_name, const role_config&) override;
+    virtual future<> create(std::string_view role_name, const role_config&, ::service::group0_batch&) override;
 
-    virtual future<> drop(std::string_view role_name) override;
+    virtual future<> drop(std::string_view role_name, ::service::group0_batch& mc) override;
 
-    virtual future<> alter(std::string_view role_name, const role_config_update&) override;
+    virtual future<> alter(std::string_view role_name, const role_config_update&, ::service::group0_batch&) override;
 
-    virtual future<> grant(std::string_view grantee_name, std::string_view role_name) override;
+    virtual future<> grant(std::string_view grantee_name, std::string_view role_name, ::service::group0_batch& mc) override;
 
-    virtual future<> revoke(std::string_view revokee_name, std::string_view role_name) override;
+    virtual future<> revoke(std::string_view revokee_name, std::string_view role_name, ::service::group0_batch& mc) override;
 
     virtual future<role_set> query_granted(std::string_view grantee_name, recursive_role_query) override;
 
@@ -72,9 +73,9 @@ public:
 
     virtual future<role_manager::attribute_vals> query_attribute_for_all(std::string_view attribute_name) override;
 
-    virtual future<> set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value) override;
+    virtual future<> set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value, ::service::group0_batch& mc) override;
 
-    virtual future<> remove_attribute(std::string_view role_name, std::string_view attribute_name) override;
+    virtual future<> remove_attribute(std::string_view role_name, std::string_view attribute_name, ::service::group0_batch& mc) override;
 
 private:
     enum class membership_change { add, remove };
@@ -87,9 +88,11 @@ private:
 
     future<> create_default_role_if_missing();
 
-    future<> create_or_replace(std::string_view role_name, const role_config&);
+    future<> create_or_replace(std::string_view role_name, const role_config&, ::service::group0_batch&);
 
-    future<> modify_membership(std::string_view role_name, std::string_view grantee_name, membership_change);
+    future<> legacy_modify_membership(std::string_view role_name, std::string_view grantee_name, membership_change);
+
+    future<> modify_membership(std::string_view role_name, std::string_view grantee_name, membership_change, ::service::group0_batch& mc);
 };
 
 }

auth/transitional.cc

@@ -87,16 +87,16 @@ public:
         });
     }
 
-    virtual future<> create(std::string_view role_name, const authentication_options& options) override {
-        return _authenticator->create(role_name, options);
+    virtual future<> create(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) override {
+        return _authenticator->create(role_name, options, mc);
     }
 
-    virtual future<> alter(std::string_view role_name, const authentication_options& options) override {
-        return _authenticator->alter(role_name, options);
+    virtual future<> alter(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) override {
+        return _authenticator->alter(role_name, options, mc);
     }
 
-    virtual future<> drop(std::string_view role_name) override {
-        return _authenticator->drop(role_name);
+    virtual future<> drop(std::string_view role_name, ::service::group0_batch& mc) override {
+        return _authenticator->drop(role_name, mc);
     }
 
     virtual future<custom_options> query_custom_options(std::string_view role_name) const override {
@@ -187,24 +187,24 @@ public:
         return make_ready_future<permission_set>(transitional_permissions);
     }
 
-    virtual future<> grant(std::string_view s, permission_set ps, const resource& r)  override {
-        return _authorizer->grant(s, std::move(ps), r);
+    virtual future<> grant(std::string_view s, permission_set ps, const resource& r, ::service::group0_batch& mc)  override {
+        return _authorizer->grant(s, std::move(ps), r, mc);
     }
 
-    virtual future<> revoke(std::string_view s, permission_set ps, const resource& r) override {
-        return _authorizer->revoke(s, std::move(ps), r);
+    virtual future<> revoke(std::string_view s, permission_set ps, const resource& r, ::service::group0_batch& mc) override {
+        return _authorizer->revoke(s, std::move(ps), r, mc);
     }
 
     virtual future<std::vector<permission_details>> list_all() const override {
         return _authorizer->list_all();
     }
 
-    virtual future<> revoke_all(std::string_view s) override {
-        return _authorizer->revoke_all(s);
+    virtual future<> revoke_all(std::string_view s, ::service::group0_batch& mc) override {
+        return _authorizer->revoke_all(s, mc);
     }
 
-    virtual future<> revoke_all(const resource& r) override {
-        return _authorizer->revoke_all(r);
+    virtual future<> revoke_all(const resource& r, ::service::group0_batch& mc) override {
+        return _authorizer->revoke_all(r, mc);
     }
 
     virtual const resource_set& protected_resources() const override {

bytes.cc

@@ -61,19 +61,6 @@ sstring to_hex(const bytes_opt& b) {
     return !b ? "null" : to_hex(*b);
 }
 
-std::ostream& operator<<(std::ostream& os, const bytes& b) {
-    fmt::print(os, "{}", b);
-    return os;
-}
-
-std::ostream& operator<<(std::ostream& os, const bytes_opt& b) {
-    if (b) {
-        fmt::print(os, "{}", *b);
-        return os;
-    }
-    return os << "null";
-}
-
 namespace std {
 
 std::ostream& operator<<(std::ostream& os, const bytes_view& b) {
@@ -82,8 +69,3 @@ std::ostream& operator<<(std::ostream& os, const bytes_view& b) {
 }
 
 }
-
-std::ostream& operator<<(std::ostream& os, const fmt_hex& b) {
-    fmt::print(os, "{}", b);
-    return os;
-}

bytes.hh

@@ -42,16 +42,11 @@ struct fmt_hex {
     fmt_hex(const bytes_view& v) noexcept : v(v) {}
 };
 
-std::ostream& operator<<(std::ostream& os, const fmt_hex& hex);
-
 bytes from_hex(sstring_view s);
 sstring to_hex(bytes_view b);
 sstring to_hex(const bytes& b);
 sstring to_hex(const bytes_opt& b);
 
-std::ostream& operator<<(std::ostream& os, const bytes& b);
-std::ostream& operator<<(std::ostream& os, const bytes_opt& b);
-
 template <>
 struct fmt::formatter<fmt_hex> {
     size_t _group_size_in_bytes = 0;

cache_mutation_reader.hh

@@ -21,7 +21,7 @@ namespace cache {
 
 extern logging::logger clogger;
 
-class cache_flat_mutation_reader final : public flat_mutation_reader_v2::impl {
+class cache_mutation_reader final : public mutation_reader::impl {
     enum class state {
         before_static_row,
 
@@ -70,7 +70,7 @@ class cache_flat_mutation_reader final : public flat_mutation_reader_v2::impl {
     position_in_partition_view _upper_bound; // Query schema domain
     std::optional<position_in_partition> _underlying_upper_bound; // Query schema domain
 
-    // cache_flat_mutation_reader may be constructed either
+    // cache_mutation_reader may be constructed either
     // with a read_context&, where it knows that the read_context
     // is owned externally, by the caller.  In this case
     // _read_context_holder would be disengaged.
@@ -115,8 +115,8 @@ class cache_flat_mutation_reader final : public flat_mutation_reader_v2::impl {
 
     // Points to the underlying reader conforming to _schema,
     // either to *_underlying_holder or _read_context.underlying().underlying().
-    flat_mutation_reader_v2* _underlying = nullptr;
-    flat_mutation_reader_v2_opt _underlying_holder;
+    mutation_reader* _underlying = nullptr;
+    mutation_reader_opt _underlying_holder;
 
     gc_clock::time_point _read_time;
     gc_clock::time_point _gc_before;
@@ -206,13 +206,13 @@ class cache_flat_mutation_reader final : public flat_mutation_reader_v2::impl {
     }
 
 public:
-    cache_flat_mutation_reader(schema_ptr s,
+    cache_mutation_reader(schema_ptr s,
                                dht::decorated_key dk,
                                query::clustering_key_filter_ranges&& crr,
                                read_context& ctx,
                                partition_snapshot_ptr snp,
                                row_cache& cache)
-        : flat_mutation_reader_v2::impl(std::move(s), ctx.permit())
+        : mutation_reader::impl(std::move(s), ctx.permit())
         , _snp(std::move(snp))
         , _ck_ranges(std::move(crr))
         , _ck_ranges_curr(_ck_ranges.begin())
@@ -230,20 +230,20 @@ public:
                       fmt::ptr(&*_snp));
         push_mutation_fragment(*_schema, _permit, partition_start(std::move(dk), _snp->partition_tombstone()));
     }
-    cache_flat_mutation_reader(schema_ptr s,
+    cache_mutation_reader(schema_ptr s,
                                dht::decorated_key dk,
                                query::clustering_key_filter_ranges&& crr,
                                std::unique_ptr<read_context> unique_ctx,
                                partition_snapshot_ptr snp,
                                row_cache& cache)
-        : cache_flat_mutation_reader(s, std::move(dk), std::move(crr), *unique_ctx, std::move(snp), cache)
+        : cache_mutation_reader(s, std::move(dk), std::move(crr), *unique_ctx, std::move(snp), cache)
     {
         // Assume ownership of the read_context.
         // It is our responsibility to close it now.
         _read_context_holder = std::move(unique_ctx);
     }
-    cache_flat_mutation_reader(const cache_flat_mutation_reader&) = delete;
-    cache_flat_mutation_reader(cache_flat_mutation_reader&&) = delete;
+    cache_mutation_reader(const cache_mutation_reader&) = delete;
+    cache_mutation_reader(cache_mutation_reader&&) = delete;
     virtual future<> fill_buffer() override;
     virtual future<> next_partition() override {
         clear_buffer_to_next_partition();
@@ -268,7 +268,7 @@ public:
 };
 
 inline
-future<> cache_flat_mutation_reader::process_static_row() {
+future<> cache_mutation_reader::process_static_row() {
     if (_snp->static_row_continuous()) {
         _read_context.cache().on_row_hit();
         static_row sr = _lsa_manager.run_in_read_section([this] {
@@ -294,12 +294,12 @@ future<> cache_flat_mutation_reader::process_static_row() {
 }
 
 inline
-void cache_flat_mutation_reader::touch_partition() {
+void cache_mutation_reader::touch_partition() {
     _snp->touch();
 }
 
 inline
-future<> cache_flat_mutation_reader::fill_buffer() {
+future<> cache_mutation_reader::fill_buffer() {
     if (_state == state::before_static_row) {
         touch_partition();
         auto after_static_row = [this] {
@@ -326,12 +326,12 @@ future<> cache_flat_mutation_reader::fill_buffer() {
 }
 
 inline
-future<> cache_flat_mutation_reader::ensure_underlying() {
+future<> cache_mutation_reader::ensure_underlying() {
     if (_underlying) {
         return make_ready_future<>();
     }
     return _read_context.ensure_underlying().then([this] {
-        flat_mutation_reader_v2& ctx_underlying = _read_context.underlying().underlying();
+        mutation_reader& ctx_underlying = _read_context.underlying().underlying();
         if (ctx_underlying.schema() != _schema) {
             _underlying_holder = make_delegating_reader(ctx_underlying);
             _underlying_holder->upgrade_schema(_schema);
@@ -343,7 +343,7 @@ future<> cache_flat_mutation_reader::ensure_underlying() {
 }
 
 inline
-future<> cache_flat_mutation_reader::do_fill_buffer() {
+future<> cache_mutation_reader::do_fill_buffer() {
     if (_state == state::move_to_underlying) {
         if (!_underlying) {
             return ensure_underlying().then([this] {
@@ -411,7 +411,7 @@ future<> cache_flat_mutation_reader::do_fill_buffer() {
 }
 
 inline
-future<> cache_flat_mutation_reader::read_from_underlying() {
+future<> cache_mutation_reader::read_from_underlying() {
     return consume_mutation_fragments_until(*_underlying,
         [this] { return _state != state::reading_from_underlying || is_buffer_full(); },
         [this] (mutation_fragment_v2 mf) {
@@ -509,7 +509,7 @@ future<> cache_flat_mutation_reader::read_from_underlying() {
 }
 
 inline
-bool cache_flat_mutation_reader::ensure_population_lower_bound() {
+bool cache_mutation_reader::ensure_population_lower_bound() {
     if (_population_range_starts_before_all_rows) {
         return true;
     }
@@ -544,7 +544,7 @@ bool cache_flat_mutation_reader::ensure_population_lower_bound() {
 }
 
 inline
-void cache_flat_mutation_reader::maybe_update_continuity() {
+void cache_mutation_reader::maybe_update_continuity() {
     position_in_partition::equal_compare eq(*_schema);
     if (can_populate()
             && ensure_population_lower_bound()
@@ -617,7 +617,7 @@ void cache_flat_mutation_reader::maybe_update_continuity() {
 }
 
 inline
-void cache_flat_mutation_reader::maybe_add_to_cache(const clustering_row& cr) {
+void cache_mutation_reader::maybe_add_to_cache(const clustering_row& cr) {
     if (!can_populate()) {
         _last_row = nullptr;
         _population_range_starts_before_all_rows = false;
@@ -670,7 +670,7 @@ void cache_flat_mutation_reader::maybe_add_to_cache(const clustering_row& cr) {
 }
 
 inline
-bool cache_flat_mutation_reader::maybe_add_to_cache(const range_tombstone_change& rtc) {
+bool cache_mutation_reader::maybe_add_to_cache(const range_tombstone_change& rtc) {
     rows_entry::tri_compare q_cmp(*_schema);
 
     clogger.trace("csm {}: maybe_add_to_cache({})", fmt::ptr(this), rtc);
@@ -741,20 +741,20 @@ bool cache_flat_mutation_reader::maybe_add_to_cache(const range_tombstone_change
 }
 
 inline
-bool cache_flat_mutation_reader::after_current_range(position_in_partition_view p) {
+bool cache_mutation_reader::after_current_range(position_in_partition_view p) {
     position_in_partition::tri_compare cmp(*_schema);
     return cmp(p, _upper_bound) >= 0;
 }
 
 inline
-void cache_flat_mutation_reader::start_reading_from_underlying() {
+void cache_mutation_reader::start_reading_from_underlying() {
     clogger.trace("csm {}: start_reading_from_underlying(), range=[{}, {})", fmt::ptr(this), _lower_bound, _next_row_in_range ? _next_row.position() : _upper_bound);
     _state = state::move_to_underlying;
     _next_row.touch();
 }
 
 inline
-void cache_flat_mutation_reader::copy_from_cache_to_buffer() {
+void cache_mutation_reader::copy_from_cache_to_buffer() {
     clogger.trace("csm {}: copy_from_cache, next_row_in_range={}, next={}", fmt::ptr(this), _next_row_in_range, _next_row);
     _next_row.touch();
 
@@ -857,13 +857,13 @@ void cache_flat_mutation_reader::copy_from_cache_to_buffer() {
 }
 
 inline
-void cache_flat_mutation_reader::move_to_end() {
+void cache_mutation_reader::move_to_end() {
     finish_reader();
     clogger.trace("csm {}: eos", fmt::ptr(this));
 }
 
 inline
-void cache_flat_mutation_reader::move_to_next_range() {
+void cache_mutation_reader::move_to_next_range() {
     if (_current_tombstone) {
         clogger.trace("csm {}: move_to_next_range: emit rtc({}, null)", fmt::ptr(this), _upper_bound);
         push_mutation_fragment(mutation_fragment_v2(*_schema, _permit, range_tombstone_change(_upper_bound, {})));
@@ -879,7 +879,7 @@ void cache_flat_mutation_reader::move_to_next_range() {
 }
 
 inline
-void cache_flat_mutation_reader::move_to_range(query::clustering_row_ranges::const_iterator next_it) {
+void cache_mutation_reader::move_to_range(query::clustering_row_ranges::const_iterator next_it) {
     auto lb = position_in_partition::for_range_start(*next_it);
     auto ub = position_in_partition_view::for_range_end(*next_it);
     _last_row = nullptr;
@@ -928,7 +928,7 @@ void cache_flat_mutation_reader::move_to_range(query::clustering_row_ranges::con
 // _next_row must have a greater position than _last_row.
 // Invalidates references but keeps the _next_row valid.
 inline
-void cache_flat_mutation_reader::maybe_drop_last_entry(tombstone rt) noexcept {
+void cache_mutation_reader::maybe_drop_last_entry(tombstone rt) noexcept {
     // Drop dummy entry if it falls inside a continuous range.
     // This prevents unnecessary dummy entries from accumulating in cache and slowing down scans.
     //
@@ -962,7 +962,7 @@ void cache_flat_mutation_reader::maybe_drop_last_entry(tombstone rt) noexcept {
 
 // _next_row must be inside the range.
 inline
-void cache_flat_mutation_reader::move_to_next_entry() {
+void cache_mutation_reader::move_to_next_entry() {
     clogger.trace("csm {}: move_to_next_entry(), curr={}", fmt::ptr(this), _next_row.position());
     if (no_clustering_row_between(*_schema, _next_row.position(), _upper_bound)) {
         move_to_next_range();
@@ -984,7 +984,7 @@ void cache_flat_mutation_reader::move_to_next_entry() {
 }
 
 inline
-void cache_flat_mutation_reader::offer_from_underlying(mutation_fragment_v2&& mf) {
+void cache_mutation_reader::offer_from_underlying(mutation_fragment_v2&& mf) {
     clogger.trace("csm {}: offer_from_underlying({})", fmt::ptr(this), mutation_fragment_v2::printer(*_schema, mf));
     if (mf.is_clustering_row()) {
         maybe_add_to_cache(mf.as_clustering_row());
@@ -999,7 +999,7 @@ void cache_flat_mutation_reader::offer_from_underlying(mutation_fragment_v2&& mf
 }
 
 inline
-void cache_flat_mutation_reader::add_to_buffer(const partition_snapshot_row_cursor& row) {
+void cache_mutation_reader::add_to_buffer(const partition_snapshot_row_cursor& row) {
     position_in_partition::less_compare less(*_schema);
     if (!row.dummy()) {
         _read_context.cache().on_row_hit();
@@ -1019,7 +1019,7 @@ void cache_flat_mutation_reader::add_to_buffer(const partition_snapshot_row_curs
 //   (1) no fragment with position >= _lower_bound was pushed yet
 //   (2) If _lower_bound > mf.position(), mf was emitted
 inline
-void cache_flat_mutation_reader::add_clustering_row_to_buffer(mutation_fragment_v2&& mf) {
+void cache_mutation_reader::add_clustering_row_to_buffer(mutation_fragment_v2&& mf) {
     clogger.trace("csm {}: add_clustering_row_to_buffer({})", fmt::ptr(this), mutation_fragment_v2::printer(*_schema, mf));
     auto& row = mf.as_clustering_row();
     auto new_lower_bound = position_in_partition::after_key(*_schema, row.key());
@@ -1031,7 +1031,7 @@ void cache_flat_mutation_reader::add_clustering_row_to_buffer(mutation_fragment_
 }
 
 inline
-void cache_flat_mutation_reader::add_to_buffer(range_tombstone_change&& rtc) {
+void cache_mutation_reader::add_to_buffer(range_tombstone_change&& rtc) {
     clogger.trace("csm {}: add_to_buffer({})", fmt::ptr(this), rtc);
     _has_rt = true;
     position_in_partition::less_compare less(*_schema);
@@ -1041,7 +1041,7 @@ void cache_flat_mutation_reader::add_to_buffer(range_tombstone_change&& rtc) {
 }
 
 inline
-void cache_flat_mutation_reader::maybe_add_to_cache(const static_row& sr) {
+void cache_mutation_reader::maybe_add_to_cache(const static_row& sr) {
     if (can_populate()) {
         clogger.trace("csm {}: populate({})", fmt::ptr(this), static_row::printer(*_schema, sr));
         _read_context.cache().on_static_row_insert();
@@ -1058,7 +1058,7 @@ void cache_flat_mutation_reader::maybe_add_to_cache(const static_row& sr) {
 }
 
 inline
-void cache_flat_mutation_reader::maybe_set_static_row_continuous() {
+void cache_mutation_reader::maybe_set_static_row_continuous() {
     if (can_populate()) {
         clogger.trace("csm {}: set static row continuous", fmt::ptr(this));
         _snp->version()->partition().set_static_row_continuous(true);
@@ -1074,7 +1074,7 @@ void cache_flat_mutation_reader::maybe_set_static_row_continuous() {
 // Thus, when we make an entry continuous, we must ensure that it isn't an
 // unlinked last dummy.
 inline
-void cache_flat_mutation_reader::set_rows_entry_continuous(rows_entry& e) {
+void cache_mutation_reader::set_rows_entry_continuous(rows_entry& e) {
     e.set_continuous(true);
     if (!e.is_linked()) [[unlikely]] {
         _snp->tracker()->touch(e);
@@ -1082,7 +1082,7 @@ void cache_flat_mutation_reader::set_rows_entry_continuous(rows_entry& e) {
 }
 
 inline
-void cache_flat_mutation_reader::restore_continuity_after_insertion(const mutation_partition::rows_type::iterator& it) {
+void cache_mutation_reader::restore_continuity_after_insertion(const mutation_partition::rows_type::iterator& it) {
     if (auto x = std::next(it); x->continuous()) {
         it->set_continuous(true);
         it->set_range_tombstone(x->range_tombstone());
@@ -1090,33 +1090,33 @@ void cache_flat_mutation_reader::restore_continuity_after_insertion(const mutati
 }
 
 inline
-bool cache_flat_mutation_reader::can_populate() const {
+bool cache_mutation_reader::can_populate() const {
     return _snp->at_latest_version() && _read_context.cache().phase_of(_read_context.key()) == _read_context.phase();
 }
 
 } // namespace cache
 
-// pass a reference to ctx to cache_flat_mutation_reader
+// pass a reference to ctx to cache_mutation_reader
 // keeping its ownership at caller's.
-inline flat_mutation_reader_v2 make_cache_flat_mutation_reader(schema_ptr s,
+inline mutation_reader make_cache_mutation_reader(schema_ptr s,
                                                             dht::decorated_key dk,
                                                             query::clustering_key_filter_ranges crr,
                                                             row_cache& cache,
                                                             cache::read_context& ctx,
                                                             partition_snapshot_ptr snp)
 {
-    return make_flat_mutation_reader_v2<cache::cache_flat_mutation_reader>(
+    return make_mutation_reader<cache::cache_mutation_reader>(
         std::move(s), std::move(dk), std::move(crr), ctx, std::move(snp), cache);
 }
 
-// transfer ownership of ctx to cache_flat_mutation_reader
-inline flat_mutation_reader_v2 make_cache_flat_mutation_reader(schema_ptr s,
+// transfer ownership of ctx to cache_mutation_reader
+inline mutation_reader make_cache_mutation_reader(schema_ptr s,
                                                             dht::decorated_key dk,
                                                             query::clustering_key_filter_ranges crr,
                                                             row_cache& cache,
                                                             std::unique_ptr<cache::read_context> unique_ctx,
                                                             partition_snapshot_ptr snp)
 {
-    return make_flat_mutation_reader_v2<cache::cache_flat_mutation_reader>(
+    return make_mutation_reader<cache::cache_mutation_reader>(
         std::move(s), std::move(dk), std::move(crr), std::move(unique_ctx), std::move(snp), cache);
 }

cdc/cdc_extension.hh

@@ -12,7 +12,6 @@
 #include <seastar/core/sstring.hh>
 
 #include "bytes.hh"
-#include "serializer.hh"
 #include "cdc/cdc_options.hh"
 #include "schema/schema.hh"
 #include "serializer_impl.hh"

cdc/generation.cc

@@ -962,10 +962,10 @@ future<> generation_service::check_and_repair_cdc_streams() {
     // Update _gen_id first, so that legacy_do_handle_cdc_generation (which will get called due to the status update)
     // won't try to update the gossiper, which would result in a deadlock inside add_local_application_state
     _gen_id = new_gen_id;
-    co_await _gossiper.add_local_application_state({
-            { gms::application_state::CDC_GENERATION_ID, gms::versioned_value::cdc_generation_id(new_gen_id) },
-            { gms::application_state::STATUS, *status }
-    });
+    co_await _gossiper.add_local_application_state(
+            std::pair(gms::application_state::CDC_GENERATION_ID, gms::versioned_value::cdc_generation_id(new_gen_id)),
+            std::pair(gms::application_state::STATUS, *status)
+    );
     co_await _sys_ks.local().update_cdc_generation_id(new_gen_id);
 }
 
@@ -1110,7 +1110,9 @@ future<bool> generation_service::legacy_do_handle_cdc_generation(cdc::generation
     auto sys_dist_ks = get_sys_dist_ks();
     auto gen = co_await retrieve_generation_data(gen_id, _sys_ks.local(), *sys_dist_ks, { _token_metadata.get()->count_normal_token_owners() });
     if (!gen) {
-        throw std::runtime_error(format(
+        // This may happen during raft upgrade when a node gossips about a generation that
+        // was propagated through raft and we didn't apply it yet.
+        throw generation_handling_nonfatal_exception(format(
             "Could not find CDC generation {} in distributed system tables (current time: {}),"
             " even though some node gossiped about it.",
             gen_id, db_clock::now()));

cdc/log.cc

@@ -207,7 +207,7 @@ public:
             auto new_log_schema = create_log_schema(new_schema, log_schema ? std::make_optional(log_schema->id()) : std::nullopt, log_schema);
 
             auto log_mut = log_schema 
-                ? db::schema_tables::make_update_table_mutations(db, keyspace.metadata(), log_schema, new_log_schema, timestamp, false)
+                ? db::schema_tables::make_update_table_mutations(db, keyspace.metadata(), log_schema, new_log_schema, timestamp)
                 : db::schema_tables::make_create_table_mutations(new_log_schema, timestamp)
                 ;
 

cdc/metadata.cc

@@ -7,7 +7,6 @@
  */
 
 #include "dht/token-sharding.hh"
-#include "utils/exceptions.hh"
 #include "exceptions/exceptions.hh"
 
 #include "cdc/generation.hh"
@@ -187,7 +186,7 @@ bool cdc::metadata::prepare(db_clock::time_point tp) {
     }
 
     auto ts = to_ts(tp);
-    auto emplaced = _gens.emplace(to_ts(tp), std::nullopt).second;
+    auto [it, emplaced] = _gens.emplace(to_ts(tp), std::nullopt);
 
     if (_last_stream_timestamp != api::missing_timestamp) {
         auto last_correct_gen = gen_used_at(_last_stream_timestamp);
@@ -202,5 +201,5 @@ bool cdc::metadata::prepare(db_clock::time_point tp) {
         }
     }
 
-    return emplaced;
+    return !it->second;
 }

cdc/split.hh

@@ -8,10 +8,9 @@
 
 #pragma once
 
-#include <boost/dynamic_bitset.hpp>
+#include <boost/dynamic_bitset.hpp>  // IWYU pragma: keep
 #include "replica/database_fwd.hh"
 #include "timestamp.hh"
-#include <seastar/util/noncopyable_function.hh>
 
 class mutation;
 

cdc/stats.hh

@@ -10,11 +10,8 @@
 
 #include <array>
 #include <cstdint>
-#include <string>
 #include <seastar/core/metrics_registration.hh>
 #include "enum_set.hh"
-#include "utils/histogram.hh"
-#include "utils/estimated_histogram.hh"
 
 namespace cdc {
 

client_data.hh

@@ -33,7 +33,7 @@ sstring to_string(client_connection_stage ct);
 struct client_data {
     net::inet_address ip;
     int32_t port;
-    client_type ct;
+    client_type ct = client_type::cql;
     client_connection_stage connection_stage = client_connection_stage::established;
     int32_t shard_id;  /// ID of server-side shard which is processing the connection.
 

clustering_bounds_comparator.hh

@@ -25,8 +25,6 @@ enum class bound_kind : uint8_t {
     excl_start = 7,
 };
 
-std::ostream& operator<<(std::ostream& out, const bound_kind k);
-
 // Swaps start <-> end && incl <-> excl
 bound_kind invert_kind(bound_kind k);
 // Swaps start <-> end

clustering_ranges_walker.hh

@@ -103,7 +103,7 @@ public:
         // which positions you call advance_to(), provided that you change
         // the current tombstone at the same positions.
         // Redundant changes will not be generated.
-        // This is to support the guarantees of flat_mutation_reader_v2.
+        // This is to support the guarantees of mutation_reader.
         range_tombstones rts;
     };
 

cmake/FindThrift.cmake

@@ -1,47 +0,0 @@
-#
-# Copyright 2023-present ScyllaDB
-#
-
-#
-# SPDX-License-Identifier: AGPL-3.0-or-later
-#
-find_package(PkgConfig REQUIRED)
-
-pkg_check_modules(PC_thrift QUIET thrift)
-
-find_library(thrift_LIBRARY
-  NAMES thrift
-  HINTS
-    ${PC_thrift_LIBDIR}
-    ${PC_thrift_LIBRARY_DIRS})
-
-find_path(thrift_INCLUDE_DIR
-  NAMES thrift/Thrift.h
-  HINTS
-    ${PC_thrift_INCLUDEDIR}
-    ${PC_thrift_INCLUDE_DIRS})
-
-mark_as_advanced(
-  thrift_LIBRARY
-  thrift_INCLUDE_DIR)
-
-include(FindPackageHandleStandardArgs)
-
-find_package_handle_standard_args(Thrift
-  REQUIRED_VARS
-    thrift_LIBRARY
-    thrift_INCLUDE_DIR
-  VERSION_VAR PC_thrift_VERSION)
-
-if(Thrift_FOUND)
-  set(thrift_LIBRARIES ${thrift_LIBRARY})
-  set(thrift_INCLUDE_DIRS ${thrift_INCLUDE_DIR})
-  if(NOT(TARGET Thrift::thrift))
-    add_library(Thrift::thrift UNKNOWN IMPORTED)
-
-    set_target_properties(Thrift::thrift
-      PROPERTIES
-        IMPORTED_LOCATION ${thrift_LIBRARY}
-        INTERFACE_INCLUDE_DIRECTORIES ${thrift_INCLUDE_DIRS})
-  endif()
-endif()