dist: systemd: use default KillMode

before this change, we specify the KillMode of the scylla-service
service unit explicitly to "process". according to
according to
https://www.freedesktop.org/software/systemd/man/latest/systemd.kill.html,

> If set to process, only the main process itself is killed (not recommended!).

and the document suggests use "control-group" over "process".
but scylla server is not a multi-process server, it is a multi-threaded
server. so it should not make any difference even if we switch to
the recommended "control-group".

in the light that we've been seeing "defunct" scylla process after
stopping the scylla service using systemd. we are wondering if we should
try to change the `KillMode` to "control-group", which is the default
value of this setting.

in this change, we just drop the setting so that the systemd stops the
service by stopping all processes in the control group of this unit
are stopped.

Fixes scylladb/scylladb#21507

Signed-off-by: Kefu Chai <kefu.chai@scylladb.com>

Closes scylladb/scylladb#21508

(cherry picked from commit 961a53f716)

Closes scylladb/scylladb#23177

.gitattributes

@@ -1,3 +1,4 @@
 *.cc diff=cpp
 *.hh diff=cpp
 *.svg binary
+docs/_static/api/js/* binary

.github/clang-include-cleaner.json

@@ -0,0 +1,20 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "clang-include-cleaner",
+            "severity": "error",
+            "pattern": [
+                {
+                    "regexp": "^([^\\-\\+].*)$",
+                    "file": 1
+                },
+                {
+                    "regexp": "^(-\\s+[^\\s]+)\\s+@Line:(\\d+)$",
+                    "line": 2,
+                    "message": 1,
+                    "loop": true
+                }
+            ]
+        }
+    ]
+}

.github/clang-matcher.json

@@ -0,0 +1,18 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "clang",
+            "pattern": [
+                {
+                    "regexp": "^([^:]+):(\\d+):(\\d+):\\s+(warning|error):\\s+(.*?)\\s+\\[(.*?)\\]$",
+                    "file": 1,
+                    "line": 2,
+                    "column": 3,
+                    "severity": 4,
+                    "message": 5,
+                    "code": 6
+                }
+            ]
+        }
+    ]
+}

.github/mergify.yml

@@ -65,3 +65,28 @@ pull_request_rules:
           - branch-5.4
         assignees:
           - "{{ author }}"
+  - name: Automate backport pull request 6.0
+    conditions:
+      - or:
+        - closed
+        - merged
+      - or:
+          - base=master
+          - base=next
+      - label=backport/6.0 # The PR must have this label to trigger the backport
+      - label=promoted-to-master
+    actions:
+      copy:
+        title: "[Backport 6.0] {{ title }}"
+        body: |
+          {{ body }}
+
+          {% for c in commits %}
+          (cherry picked from commit {{ c.sha }})
+          {% endfor %}
+
+           Refs #{{number}}
+        branches:
+          - branch-6.0
+        assignees:
+          - "{{ author }}"

.github/pull_request_template.md

@@ -0,0 +1 @@
+**Please replace this line with justification for the backport/\* labels added to this PR**

.github/scripts/auto-backport.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import re
+import sys
+import tempfile
+import logging
+
+from github import Github, GithubException
+from git import Repo, GitCommandError
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+try:
+    github_token = os.environ["GITHUB_TOKEN"]
+except KeyError:
+    print("Please set the 'GITHUB_TOKEN' environment variable")
+    sys.exit(1)
+
+
+def is_pull_request():
+    return '--pull-request' in sys.argv[1:]
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--repo', type=str, required=True, help='Github repository name')
+    parser.add_argument('--base-branch', type=str, default='refs/heads/master', help='Base branch')
+    parser.add_argument('--commits', default=None, type=str, help='Range of promoted commits.')
+    parser.add_argument('--pull-request', type=int, help='Pull request number to be backported')
+    parser.add_argument('--head-commit', type=str, required=is_pull_request(), help='The HEAD of target branch after the pull request specified by --pull-request is merged')
+    return parser.parse_args()
+
+
+def create_pull_request(repo, new_branch_name, base_branch_name, pr, backport_pr_title, commits, is_draft=False):
+    pr_body = f'{pr.body}\n\n'
+    for commit in commits:
+        pr_body += f'- (cherry picked from commit {commit})\n\n'
+    pr_body += f'Parent PR: #{pr.number}'
+    try:
+        backport_pr = repo.create_pull(
+            title=backport_pr_title,
+            body=pr_body,
+            head=f'scylladbbot:{new_branch_name}',
+            base=base_branch_name,
+            draft=is_draft
+        )
+        logging.info(f"Pull request created: {backport_pr.html_url}")
+        backport_pr.add_to_assignees(pr.user)
+        if is_draft:
+            backport_pr.add_to_labels("conflicts")
+            pr_comment = f"@{pr.user} - This PR was marked as draft because it has conflicts\n"
+            pr_comment += "Please resolve them and mark this PR as ready for review"
+            backport_pr.create_issue_comment(pr_comment)
+        logging.info(f"Assigned PR to original author: {pr.user}")
+        return backport_pr
+    except GithubException as e:
+        if 'A pull request already exists' in str(e):
+            logging.warning(f'A pull request already exists for {pr.user}:{new_branch_name}')
+        else:
+            logging.error(f'Failed to create PR: {e}')
+
+
+def get_pr_commits(repo, pr, stable_branch, start_commit=None):
+    commits = []
+    if pr.merged:
+        merge_commit = repo.get_commit(pr.merge_commit_sha)
+        if len(merge_commit.parents) > 1:  # Check if this merge commit includes multiple commits
+            commits.append(pr.merge_commit_sha)
+        else:
+            if start_commit:
+                promoted_commits = repo.compare(start_commit, stable_branch).commits
+            else:
+                promoted_commits = repo.get_commits(sha=stable_branch)
+            for commit in pr.get_commits():
+                for promoted_commit in promoted_commits:
+                    commit_title = commit.commit.message.splitlines()[0]
+                    # In Scylla-pkg and scylla-dtest, for example,
+                    # we don't create a merge commit for a PR with multiple commits,
+                    # according to the GitHub API, the last commit will be the merge commit,
+                    # which is not what we need when backporting (we need all the commits).
+                    # So here, we are validating the correct SHA for each commit so we can cherry-pick
+                    if promoted_commit.commit.message.startswith(commit_title):
+                        commits.append(promoted_commit.sha)
+
+    elif pr.state == 'closed':
+        events = pr.get_issue_events()
+        for event in events:
+            if event.event == 'closed':
+                commits.append(event.commit_id)
+    return commits
+
+
+def create_pr_comment_and_remove_label(pr, comment_body):
+    labels = pr.get_labels()
+    pattern = re.compile(r"backport/\d+\.\d+$")
+    for label in labels:
+        if pattern.match(label.name):
+            print(f"Removing label: {label.name}")
+            comment_body += f'- {label.name}\n'
+            pr.remove_from_labels(label)
+    pr.create_issue_comment(comment_body)
+
+
+def backport(repo, pr, version, commits, backport_base_branch):
+    new_branch_name = f'backport/{pr.number}/to-{version}'
+    backport_pr_title = f'[Backport {version}] {pr.title}'
+    repo_url = f'https://scylladbbot:{github_token}@github.com/{repo.full_name}.git'
+    fork_repo = f'https://scylladbbot:{github_token}@github.com/scylladbbot/{repo.name}.git'
+    with (tempfile.TemporaryDirectory() as local_repo_path):
+        try:
+            repo_local = Repo.clone_from(repo_url, local_repo_path, branch=backport_base_branch)
+            repo_local.git.checkout(b=new_branch_name)
+            is_draft = False
+            for commit in commits:
+                try:
+                    repo_local.git.cherry_pick(commit, '-m1', '-x')
+                except GitCommandError as e:
+                    logging.warning(f'Cherry-pick conflict on commit {commit}: {e}')
+                    is_draft = True
+                    repo_local.git.add(A=True)
+                    repo_local.git.cherry_pick('--continue')
+            if not repo.private and not repo.has_in_collaborators(pr.user.login):
+                repo.add_to_collaborators(pr.user.login, permission="push")
+                comment = f':warning:  @{pr.user.login} you have been added as collaborator to scylladbbot fork '
+                comment += f'Please check your inbox and approve the invitation, once it is done, please add the backport labels again'
+                create_pr_comment_and_remove_label(pr, comment)
+                return
+            repo_local.git.push(fork_repo, new_branch_name, force=True)
+            create_pull_request(repo, new_branch_name, backport_base_branch, pr, backport_pr_title, commits,
+                                is_draft=is_draft)
+
+        except GitCommandError as e:
+            logging.warning(f"GitCommandError: {e}")
+
+
+def main():
+    args = parse_args()
+    base_branch = args.base_branch.split('/')[2]
+    promoted_label = 'promoted-to-master'
+    repo_name = args.repo
+    if 'scylla-enterprise' in args.repo:
+        promoted_label = 'promoted-to-enterprise'
+    stable_branch = base_branch
+    backport_branch = 'branch-'
+
+    backport_label_pattern = re.compile(r'backport/\d+\.\d+$')
+
+    g = Github(github_token)
+    repo = g.get_repo(repo_name)
+    closed_prs = []
+    start_commit = None
+
+    if args.commits:
+        start_commit, end_commit = args.commits.split('..')
+        commits = repo.compare(start_commit, end_commit).commits
+        for commit in commits:
+            match = re.search(rf"Closes .*#([0-9]+)", commit.commit.message, re.IGNORECASE)
+            if match:
+                pr_number = int(match.group(1))
+                pr = repo.get_pull(pr_number)
+                closed_prs.append(pr)
+    if args.pull_request:
+        start_commit = args.head_commit
+        pr = repo.get_pull(args.pull_request)
+        closed_prs = [pr]
+
+    for pr in closed_prs:
+        labels = [label.name for label in pr.labels]
+        backport_labels = [label for label in labels if backport_label_pattern.match(label)]
+        if promoted_label not in labels:
+            print(f'no {promoted_label} label: {pr.number}')
+            continue
+        if not backport_labels:
+            print(f'no backport label: {pr.number}')
+            continue
+        commits = get_pr_commits(repo, pr, stable_branch, start_commit)
+        logging.info(f"Found PR #{pr.number} with commit {commits} and the following labels: {backport_labels}")
+        for backport_label in backport_labels:
+            version = backport_label.replace('backport/', '')
+            backport_base_branch = backport_label.replace('backport/', backport_branch)
+            backport(repo, pr, version, commits, backport_base_branch)
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/label_promoted_commits.py

@@ -1,9 +1,9 @@
-import requests
-from github import Github
 import argparse
 import re
 import sys
 import os
+from github import Github
+from github.GithubException import UnknownObjectException
 
 try:
     github_token = os.environ["GITHUB_TOKEN"]
@@ -16,43 +16,71 @@ def parser():
     parser = argparse.ArgumentParser()
     parser.add_argument('--repository', type=str, required=True,
                         help='Github repository name (e.g., scylladb/scylladb)')
-    parser.add_argument('--commit_before_merge', type=str, required=True, help='Git commit ID to start labeling from ('
-                                                                               'newest commit).')
-    parser.add_argument('--commit_after_merge', type=str, required=True,
-                        help='Git commit ID to end labeling at (oldest '
-                             'commit, exclusive).')
-    parser.add_argument('--update_issue', type=bool, default=False, help='Set True to update issues when backport was '
-                                                                         'done')
-    parser.add_argument('--label', type=str, required=True, help='Label to use')
+    parser.add_argument('--commits', type=str, required=True, help='Range of promoted commits.')
+    parser.add_argument('--label', type=str, default='promoted-to-master', help='Label to use')
+    parser.add_argument('--ref', type=str, required=True, help='PR target branch')
     return parser.parse_args()
 
 
+def add_comment_and_close_pr(pr, comment):
+    if pr.state == 'open':
+        pr.create_issue_comment(comment)
+        pr.edit(state="closed")
+
+
+def mark_backport_done(repo, ref_pr_number, branch):
+    pr = repo.get_pull(int(ref_pr_number))
+    label_to_remove = f'backport/{branch}'
+    label_to_add = f'{label_to_remove}-done'
+    current_labels = [label.name for label in pr.get_labels()]
+    if label_to_remove in current_labels:
+        pr.remove_from_labels(label_to_remove)
+    if label_to_add not in current_labels:
+        pr.add_to_labels(label_to_add)
+
+
 def main():
+    # This script is triggered by a push event to either the master branch or a branch named branch-x.y (where x and y represent version numbers). Based on the pushed branch, the script performs the following actions:
+    # - When ref branch is `master`, it will add the `promoted-to-master` label, which we need later for the auto backport process
+    # - When ref branch is `branch-x.y` (which means we backported a patch), it will replace in the original PR the `backport/x.y` label with `backport/x.y-done` and will close the backport PR (Since GitHub close only the one referring to default branch)
     args = parser()
     pr_pattern = re.compile(r'Closes .*#([0-9]+)')
+    target_branch = re.search(r'branch-(\d+\.\d+)', args.ref)
     g = Github(github_token)
     repo = g.get_repo(args.repository, lazy=False)
-
-    commits = repo.compare(head=args.commit_after_merge, base=args.commit_before_merge)
+    start_commit, end_commit = args.commits.split('..')
+    commits = repo.compare(start_commit, end_commit).commits
+    processed_prs = set()
     # Print commit information
-    for commit in commits.commits:
-        print(commit.sha)
+    for commit in commits:
+        print(f'Commit sha is: {commit.sha}')
         match = pr_pattern.search(commit.commit.message)
         if match:
-            pr_number = match.group(1)
-            url = f'https://api.github.com/repos/{args.repository}/issues/{pr_number}/labels'
-            data = {
-                "labels": [f'{args.label}']
-            }
-            headers = {
-                "Authorization": f"token {github_token}",
-                "Accept": "application/vnd.github.v3+json"
-            }
-            response = requests.post(url, headers=headers, json=data)
-            if response.ok:
-                print(f"Label added successfully to {url}")
+            pr_number = int(match.group(1))
+            if pr_number in processed_prs:
+                continue
+            if target_branch:
+                pr = repo.get_pull(pr_number)
+                branch_name = target_branch[1]
+                refs_pr = re.findall(r'Parent PR: (?:#|https.*?)(\d+)', pr.body)
+                if refs_pr:
+                    print(f'branch-{target_branch.group(1)}, pr number is: {pr_number}')
+                    # 1. change the backport label of the parent PR to note that
+                    #    we've merged the corresponding backport PR
+                    # 2. close the backport PR and leave a comment on it to note
+                    #    that it has been merged with a certain git commit.
+                    ref_pr_number = refs_pr[0]
+                    mark_backport_done(repo, ref_pr_number, branch_name)
+                    comment = f'Closed via {commit.sha}'
+                    add_comment_and_close_pr(pr, comment)
             else:
-                print(f"No label was added to {url}")
+                try:
+                    pr = repo.get_pull(pr_number)
+                    pr.add_to_labels('promoted-to-master')
+                    print(f'master branch, pr number is: {pr_number}')
+                except UnknownObjectException:
+                    print(f'{pr_number} is not a PR but an issue, no need to add label')
+            processed_prs.add(pr_number)
 
 
 if __name__ == "__main__":

.github/workflows/add-label-when-promoted.yaml

@@ -4,6 +4,11 @@ on:
   push:
     branches:
       - master
+      - branch-*.*
+      - enterprise
+    pull_request_target:
+      types: [labeled]
+      branches: [master, next, enterprise]
 
 jobs:
   check-commit:
@@ -12,15 +17,55 @@ jobs:
       pull-requests: write
       issues: write
     steps:
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - name: Set Default Branch
+        id: set_branch
+        run: |
+          if [[ "${{ github.repository }}" == *enterprise* ]]; then
+            echo "DEFAULT_BRANCH=enterprise" >> $GITHUB_ENV
+          else
+            echo "DEFAULT_BRANCH=master" >> $GITHUB_ENV
+          fi
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
+          repository: ${{ github.repository }}
+          ref: ${{ env.DEFAULT_BRANCH }}
+          token: ${{ secrets.AUTO_BACKPORT_TOKEN }}
           fetch-depth: 0  # Fetch all history for all tags and branches
-
+      - name: Set up Git identity
+        run: |
+          git config --global user.name "GitHub Action"
+          git config --global user.email "action@github.com"
+          git config --global merge.conflictstyle diff3
       - name: Install dependencies
-        run: sudo apt-get install -y python3-github
-
+        run: sudo apt-get install -y python3-github python3-git
       - name: Run python script
+        if: github.event_name == 'push'
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: python .github/scripts/label_promoted_commits.py --commit_before_merge ${{ github.event.before }} --commit_after_merge ${{ github.event.after }} --repository ${{ github.repository }} --label promoted-to-master
+          GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }}
+        run: python .github/scripts/label_promoted_commits.py  --commits ${{ github.event.before }}..${{ github.sha }} --repository ${{ github.repository }} --ref ${{ github.ref }}
+      - name: Run auto-backport.py when promotion completed
+        if: ${{ github.event_name == 'push' && github.ref == format('refs/heads/{0}', env.DEFAULT_BRANCH) }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }}
+        run: python .github/scripts/auto-backport.py --repo ${{ github.repository }} --base-branch ${{ github.ref }} --commits ${{ github.event.before }}..${{ github.sha }}
+      - name: Check if label starts with 'backport/' and contains digits
+        id: check_label
+        run: |
+          label_name="${{ github.event.label.name }}"
+          if [[ "$label_name" =~ ^backport/[0-9]+\.[0-9]+$ ]]; then
+            echo "Label matches backport/X.X pattern."
+            echo "backport_label=true" >> $GITHUB_OUTPUT
+          else
+            echo "Label does not match the required pattern."
+            echo "backport_label=false" >> $GITHUB_OUTPUT
+          fi
+      - name: Run auto-backport.py when label was added
+        if: ${{ github.event_name == 'pull_request_target' && steps.check_label.outputs.backport_label == 'true' && github.event.pull_request.state == 'closed' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }}
+        run: python .github/scripts/auto-backport.py --repo ${{ github.repository }} --base-branch ${{ github.ref }} --pull-request ${{ github.event.pull_request.number }} --head-commit ${{ github.event.pull_request.base.sha }}

.github/workflows/build-scylla.yaml

@@ -0,0 +1,35 @@
+name: Build Scylla
+
+on:
+  workflow_call:
+    inputs:
+      build_mode:
+        description: 'the build mode'
+        type: string
+        required: true
+    outputs:
+      md5sum:
+        description: 'the md5sum for scylla executable'
+        value: ${{ jobs.build.outputs.md5sum }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    # be consistent with tools/toolchain/image
+    container: scylladb/scylla-toolchain:fedora-40-20240621
+    outputs:
+      md5sum: ${{ steps.checksum.outputs.md5sum }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Generate the building system
+        run: |
+          git config --global --add safe.directory $GITHUB_WORKSPACE
+          ./configure.py --mode ${{ inputs.build_mode }} --with scylla
+      - run: |
+          ninja build/${{ inputs.build_mode }}/scylla
+      - id: checksum
+        run: |
+          checksum=$(md5sum build/${{ inputs.build_mode }}/scylla | cut -c -32)
+          echo "md5sum=$checksum" >> $GITHUB_OUTPUT

.github/workflows/clang-nightly.yaml

@@ -0,0 +1,65 @@
+name: clang-nightly
+
+on:
+  schedule:
+    # only at 5AM Saturday
+    - cron: '0 5 * * SAT'
+
+env:
+  # use the development branch explicitly
+  CLANG_VERSION: 19
+  BUILD_DIR: build
+
+permissions: {}
+
+# cancel the in-progress run upon a repush
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  clang-dev:
+    name: Build with clang nightly
+    runs-on: ubuntu-latest
+    container: fedora:40
+    strategy:
+      matrix:
+        build_type:
+          - Debug
+          - RelWithDebInfo
+          - Dev
+    steps:
+      - run: |
+          sudo dnf -y install git
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Install build dependencies
+        run: |
+          # use the copr repo for llvm snapshot builds, see
+          # https://copr.fedorainfracloud.org/coprs/g/fedora-llvm-team/llvm-snapshots/
+          sudo dnf -y install 'dnf-command(copr)'
+          sudo dnf copr enable -y @fedora-llvm-team/llvm-snapshots
+          # do not install java dependencies, which is not only not used here
+          sed -i.orig \
+            -e '/tools\/.*\/install-dependencies.sh/d' \
+            -e 's/(minio_download_jobs)/(true)/' \
+            ./install-dependencies.sh
+          sudo ./install-dependencies.sh
+          sudo dnf -y install lld
+      - name: Generate the building system
+        run: |
+          cmake                                         \
+            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+            -DCMAKE_C_COMPILER=clang-$CLANG_VERSION     \
+            -DCMAKE_CXX_COMPILER=clang++-$CLANG_VERSION \
+            -G Ninja                                    \
+            -B $BUILD_DIR                               \
+            -S .
+      # see https://github.com/actions/toolkit/blob/main/docs/problem-matchers.md
+      - run: |
+          echo "::add-matcher::.github/clang-matcher.json"
+      - run: |
+          cmake --build $BUILD_DIR --target scylla
+      - run: |
+          echo "::remove-matcher owner=clang::"

.github/workflows/clang-tidy.yaml

@@ -0,0 +1,67 @@
+name: clang-tidy
+
+on:
+  pull_request:
+    branches:
+      - master
+    paths-ignore:
+      - '**/*.rst'
+      - '**/*.md'
+      - 'docs/**'
+      - '.github/**'
+  workflow_dispatch:
+  schedule:
+    # only at 5AM Saturday
+    - cron: '0 5 * * SAT'
+
+env:
+  BUILD_TYPE: RelWithDebInfo
+  BUILD_DIR: build
+  CLANG_TIDY_CHECKS: '-*,bugprone-use-after-move'
+
+permissions: {}
+
+# cancel the in-progress run upon a repush
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  read-toolchain:
+    uses: ./.github/workflows/read-toolchain.yaml
+  clang-tidy:
+    name: Run clang-tidy
+    needs:
+      - read-toolchain
+    runs-on: ubuntu-latest
+    container: ${{ needs.read-toolchain.outputs.image }}
+    steps:
+      - env:
+          IMAGE: ${{ needs.read-toolchain.image }}
+        run: |
+          echo ${{ needs.read-toolchain.image }}
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - run: |
+          sudo dnf -y install clang-tools-extra
+      - name: Generate the building system
+        run: |
+          cmake                                         \
+            -DCMAKE_BUILD_TYPE=$BUILD_TYPE              \
+            -DCMAKE_C_COMPILER=clang                    \
+            -DScylla_USE_LINKER=ld.lld                  \
+            -DCMAKE_CXX_COMPILER=clang++                \
+            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON          \
+            -DCMAKE_CXX_CLANG_TIDY="clang-tidy;--checks=$CLANG_TIDY_CHECKS" \
+            -G Ninja                                    \
+            -B $BUILD_DIR                               \
+            -S .
+      # see https://github.com/actions/toolkit/blob/main/docs/problem-matchers.md
+      - run: |
+          echo "::add-matcher::.github/clang-matcher.json"
+      - name: Build with clang-tidy enabled
+        run: |
+          cmake --build $BUILD_DIR --target scylla
+      - run: |
+          echo "::remove-matcher owner=clang::"

.github/workflows/codespell.yaml

@@ -14,4 +14,4 @@ jobs:
         with:
           only_warn: 1
           ignore_words_list: "ans,datas,fo,ser,ue,crate,nd,reenable,strat,stap,te,raison"
-          skip: "./.git,./build,./tools,*.js,*.thrift,*.lock,./test,./licenses,./redis/lolwut.cc,*.svg"
+          skip: "./.git,./build,./tools,*.js,*.lock,./test,./licenses,./redis/lolwut.cc,*.svg"

.github/workflows/iwyu.yaml

@@ -0,0 +1,80 @@
+name: iwyu
+
+on:
+  pull_request:
+    branches:
+      - master
+
+env:
+  BUILD_TYPE: RelWithDebInfo
+  BUILD_DIR: build
+  CLEANER_OUTPUT_PATH: build/clang-include-cleaner.log
+  CLEANER_DIRS: test/unit exceptions alternator api auth cdc compaction
+
+permissions: {}
+
+# cancel the in-progress run upon a repush
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  read-toolchain:
+    uses: ./.github/workflows/read-toolchain.yaml
+  clang-include-cleaner:
+    name: "Analyze #includes in source files"
+    needs:
+      - read-toolchain
+    runs-on: ubuntu-latest
+    container: ${{ needs.read-toolchain.outputs.image }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - run: |
+          sudo dnf -y install clang-tools-extra
+      - name: Generate compilation database
+        run: |
+          cmake                                         \
+            -DCMAKE_BUILD_TYPE=$BUILD_TYPE              \
+            -DCMAKE_C_COMPILER=clang                    \
+            -DCMAKE_CXX_COMPILER=clang++                \
+            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON          \
+            -G Ninja                                    \
+            -B $BUILD_DIR                               \
+            -S .
+      - name: Build headers
+        run: |
+          swagger_targets=''
+          for f in api/api-doc/*.json; do
+            if test "${f#*.}" = json; then
+              name=$(basename "$f" .json)
+              if test $name != swagger20_header; then
+                swagger_targets+=" scylla_swagger_gen_$name"
+              fi
+            fi
+          done
+          cmake                                         \
+            --build build                               \
+             --target seastar_http_request_parser       \
+             --target idl-sources                       \
+             --target $swagger_targets
+      - run: |
+          echo "::add-matcher::.github/clang-include-cleaner.json"
+      - name: clang-include-cleaner
+        run: |
+          for d in $CLEANER_DIRS; do
+            find $d -name '*.cc' -o -name '*.hh'          \
+              -exec echo {} \;                            \
+              -exec clang-include-cleaner                 \
+                --ignore-headers=seastarx.hh              \
+                --print=changes                           \
+                -p $BUILD_DIR                             \
+                {} \; | tee --append $CLEANER_OUTPUT_PATH
+          done
+      - run: |
+          echo "::remove-matcher owner=clang-include-cleaner::"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: Logs (clang-include-cleaner)
+          path: "./${{ env.CLEANER_OUTPUT_PATH }}"

.github/workflows/read-toolchain.yaml

@@ -0,0 +1,23 @@
+name: Read Toolchain
+
+on:
+  workflow_call:
+    outputs:
+      image:
+        description: "the toolchain docker image"
+        value: ${{ jobs.read-toolchain.outputs.image }}
+
+jobs:
+  read-toolchain:
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.read.outputs.image }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: tools/toolchain/image
+          sparse-checkout-cone-mode: false
+      - id: read
+        run: |
+          image=$(cat tools/toolchain/image)
+          echo "image=$image" >> $GITHUB_OUTPUT

.github/workflows/reproducible-build.yaml

@@ -0,0 +1,34 @@
+name: Check Reproducible Build
+
+on:
+  schedule:
+    # 5AM every friday
+    - cron: '0 5 * * FRI'
+
+permissions: {}
+
+env:
+  BUILD_MODE: release
+jobs:
+  build-a:
+    uses: ./.github/workflows/build-scylla.yaml
+    with:
+      build_mode: release
+  build-b:
+    uses: ./.github/workflows/build-scylla.yaml
+    with:
+      build_mode: release
+  compare-checksum:
+    runs-on: ubuntu-latest
+    needs:
+      - build-a
+      - build-b
+    steps:
+      - env:
+          CHECKSUM_A: ${{needs.build-a.outputs.md5sum}}
+          CHECKSUM_B: ${{needs.build-b.outputs.md5sum}}
+        run: |
+          if [ $CHECKSUM_A != $CHECKSUM_B ]; then                             \
+            echo "::error::mismatched checksums: $CHECKSUM_A != $CHECKSUM_B"; \
+            exit 1;                                                           \
+          fi

.github/workflows/seastar.yaml

@@ -0,0 +1,50 @@
+name: Build with the latest Seastar
+
+on:
+  schedule:
+    # 5AM everyday
+    - cron: '0 5 * * *'
+
+permissions: {}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  BUILD_DIR: build
+
+jobs:
+  build-with-the-latest-seastar:
+    runs-on: ubuntu-latest
+    # be consistent with tools/toolchain/image
+    container: scylladb/scylla-toolchain:fedora-40-20240621
+    strategy:
+      matrix:
+        build_type:
+          - Debug
+          - RelWithDebInfo
+          - Dev
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - run: |
+          rm -rf seastar
+      - uses: actions/checkout@v4
+        with:
+          repository: scylladb/seastar
+          submodules: true
+          path: seastar
+      - name: Generate the building system
+        run: |
+          git config --global --add safe.directory $GITHUB_WORKSPACE
+          cmake                                         \
+            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+            -DCMAKE_C_COMPILER=clang                    \
+            -DCMAKE_CXX_COMPILER=clang++                \
+            -G Ninja                                    \
+            -B $BUILD_DIR                               \
+            -S .
+      - run: |
+          cmake --build $BUILD_DIR --target scylla

.github/workflows/sync-labels.yaml

@@ -16,6 +16,10 @@ jobs:
       pull-requests: write
       issues: write
     steps:
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
@@ -33,7 +37,7 @@ jobs:
         run: python .github/scripts/sync_labels.py --repo ${{ github.repository }} --number ${{ github.event.number }} --action ${{ github.event.action }}
 
       - name: Pull request labeled or unlabeled event
-        if: github.event_name == 'pull_request' && startsWith(github.event.label.name, 'backport/')
+        if: github.event_name == 'pull_request_target' && startsWith(github.event.label.name, 'backport/')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: python .github/scripts/sync_labels.py --repo ${{ github.repository }} --number ${{ github.event.number }} --action ${{ github.event.action }} --label ${{ github.event.label.name }}

.gitignore

@@ -3,6 +3,7 @@
 .settings
 build
 build.ninja
+build.ninja.new
 cscope.*
 /debian/
 dist/ami/files/*.rpm
@@ -18,7 +19,7 @@ CMakeLists.txt.user
 *.egg-info
 __pycache__CMakeLists.txt.user
 .gdbinit
-resources
+/resources
 .pytest_cache
 /expressions.tokens
 tags
@@ -30,3 +31,4 @@ compile_commands.json
 .ccls-cache/
 .mypy_cache
 .envrc
+clang_build

.gitmodules

@@ -1,11 +1,14 @@
 [submodule "seastar"]
 	path = seastar
-	url = ../seastar
+	url = ../scylla-seastar
 	ignore = dirty
 [submodule "swagger-ui"]
 	path = swagger-ui
 	url = ../scylla-swagger-ui
 	ignore = dirty
+[submodule "abseil"]
+	path = abseil
+	url = ../abseil-cpp
 [submodule "scylla-jmx"]
 	path = tools/jmx
 	url = ../scylla-jmx

CMakeLists.txt

@@ -42,21 +42,48 @@ else()
         COMMENT "List configured modes")
 endif()
 
-add_compile_definitions(
-    FMT_DEPRECATED_OSTREAM)
 include(limit_jobs)
 # Configure Seastar compile options to align with Scylla
-set(CMAKE_CXX_STANDARD "20" CACHE INTERNAL "")
+set(CMAKE_CXX_STANDARD "23" CACHE INTERNAL "")
 set(CMAKE_CXX_EXTENSIONS ON CACHE INTERNAL "")
+set(CMAKE_CXX_SCAN_FOR_MODULES OFF CACHE INTERNAL "")
 set(CMAKE_CXX_VISIBILITY_PRESET hidden)
 
 set(Seastar_TESTING ON CACHE BOOL "" FORCE)
 set(Seastar_API_LEVEL 7 CACHE STRING "" FORCE)
+set(Seastar_DEPRECATED_OSTREAM_FORMATTERS OFF CACHE BOOL "" FORCE)
 set(Seastar_APPS ON CACHE BOOL "" FORCE)
 set(Seastar_EXCLUDE_APPS_FROM_ALL ON CACHE BOOL "" FORCE)
 set(Seastar_EXCLUDE_TESTS_FROM_ALL ON CACHE BOOL "" FORCE)
 set(Seastar_UNUSED_RESULT_ERROR ON CACHE BOOL "" FORCE)
 add_subdirectory(seastar)
+set(ABSL_PROPAGATE_CXX_STD ON CACHE BOOL "" FORCE)
+
+find_package(Sanitizers QUIET)
+set(sanitizer_cxx_flags
+    $<$<IN_LIST:$<CONFIG>,Debug;Sanitize>:$<TARGET_PROPERTY:Sanitizers::address,INTERFACE_COMPILE_OPTIONS>;$<TARGET_PROPERTY:Sanitizers::undefined_behavior,INTERFACE_COMPILE_OPTIONS>>)
+if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    set(ABSL_GCC_FLAGS ${sanitizer_cxx_flags})
+elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+    set(ABSL_LLVM_FLAGS ${sanitizer_cxx_flags})
+endif()
+set(ABSL_DEFAULT_LINKOPTS
+    $<$<IN_LIST:$<CONFIG>,Debug;Sanitize>:$<TARGET_PROPERTY:Sanitizers::address,INTERFACE_LINK_LIBRARIES>;$<TARGET_PROPERTY:Sanitizers::undefined_behavior,INTERFACE_LINK_LIBRARIES>>)
+add_subdirectory(abseil)
+add_library(absl-headers INTERFACE)
+target_include_directories(absl-headers SYSTEM INTERFACE
+    "${PROJECT_SOURCE_DIR}/abseil")
+add_library(absl::headers ALIAS absl-headers)
+
+# Exclude absl::strerror from the default "all" target since it's not
+# used in Scylla build and, moreover, makes use of deprecated glibc APIs,
+# such as sys_nerr, which are not exposed from "stdio.h" since glibc 2.32,
+# which happens to be the case for recent Fedora distribution versions.
+#
+# Need to use the internal "absl_strerror" target name instead of namespaced
+# variant because `set_target_properties` does not understand the latter form,
+# unfortunately.
+set_target_properties(absl_strerror PROPERTIES EXCLUDE_FROM_ALL TRUE)
 
 # System libraries dependencies
 find_package(Boost REQUIRED
@@ -68,13 +95,13 @@ target_link_libraries(Boost::regex
 find_package(Lua REQUIRED)
 find_package(ZLIB REQUIRED)
 find_package(ICU COMPONENTS uc i18n REQUIRED)
-find_package(absl COMPONENTS hash raw_hash_set REQUIRED)
+find_package(fmt 9.0.0 REQUIRED)
 find_package(libdeflate REQUIRED)
 find_package(libxcrypt REQUIRED)
 find_package(Snappy REQUIRED)
 find_package(RapidJSON REQUIRED)
-find_package(Thrift REQUIRED)
 find_package(xxHash REQUIRED)
+find_package(zstd REQUIRED)
 
 set(scylla_gen_build_dir "${CMAKE_BINARY_DIR}/gen")
 file(MAKE_DIRECTORY "${scylla_gen_build_dir}")
@@ -82,6 +109,14 @@ file(MAKE_DIRECTORY "${scylla_gen_build_dir}")
 include(add_version_library)
 generate_scylla_version()
 
+add_library(scylla-zstd STATIC
+    zstd.cc)
+target_link_libraries(scylla-zstd
+  PRIVATE
+    db
+    Seastar::seastar
+    zstd::libzstd)
+
 add_library(scylla-main STATIC)
 target_sources(scylla-main
   PRIVATE
@@ -120,11 +155,13 @@ target_sources(scylla-main
     timeout_config.cc
     unimplemented.cc
     validation.cc
-    vint-serialization.cc
-    zstd.cc)
+    vint-serialization.cc)
 target_link_libraries(scylla-main
   PRIVATE
+    "$<LINK_LIBRARY:WHOLE_ARCHIVE,scylla-zstd>"
     db
+    absl::headers
+    absl::btree
     absl::hash
     absl::raw_hash_set
     Seastar::seastar
@@ -169,7 +206,6 @@ add_subdirectory(dht)
 add_subdirectory(gms)
 add_subdirectory(idl)
 add_subdirectory(index)
-add_subdirectory(interface)
 add_subdirectory(lang)
 add_subdirectory(locator)
 add_subdirectory(message)
@@ -187,7 +223,6 @@ add_subdirectory(service)
 add_subdirectory(sstables)
 add_subdirectory(streaming)
 add_subdirectory(test)
-add_subdirectory(thrift)
 add_subdirectory(tools)
 add_subdirectory(tracing)
 add_subdirectory(transport)
@@ -228,7 +263,6 @@ target_link_libraries(scylla PRIVATE
     sstables
     streaming
     test-perf
-    thrift
     tools
     tracing
     transport
@@ -237,6 +271,7 @@ target_link_libraries(scylla PRIVATE
 
 target_link_libraries(scylla PRIVATE
     seastar
+    absl::headers
     Boost::program_options)
 
 target_include_directories(scylla PRIVATE

HACKING.md

@@ -199,7 +199,7 @@ The `scylla.yaml` file in the repository by default writes all database data to
 
 Scylla has a number of requirements for the file-system and operating system to operate ideally and at peak performance. However, during development, these requirements can be relaxed with the `--developer-mode` flag.
 
-Additionally, when running on under-powered platforms like portable laptops, the `--overprovisined` flag is useful.
+Additionally, when running on under-powered platforms like portable laptops, the `--overprovisioned` flag is useful.
 
 On a development machine, one might run Scylla as
 

README.md

@@ -65,11 +65,13 @@ $ ./tools/toolchain/dbuild ./build/release/scylla --help
 
 ## Testing
 
+[![Build with the latest Seastar](https://github.com/scylladb/scylladb/actions/workflows/seastar.yaml/badge.svg)](https://github.com/scylladb/scylladb/actions/workflows/seastar.yaml) [![Check Reproducible Build](https://github.com/scylladb/scylladb/actions/workflows/reproducible-build.yaml/badge.svg)](https://github.com/scylladb/scylladb/actions/workflows/reproducible-build.yaml) [![clang-nightly](https://github.com/scylladb/scylladb/actions/workflows/clang-nightly.yaml/badge.svg)](https://github.com/scylladb/scylladb/actions/workflows/clang-nightly.yaml)
+
 See [test.py manual](docs/dev/testing.md).
 
 ## Scylla APIs and compatibility
-By default, Scylla is compatible with Apache Cassandra and its APIs - CQL and
-Thrift. There is also support for the API of Amazon DynamoDB™,
+By default, Scylla is compatible with Apache Cassandra and its API - CQL.
+There is also support for the API of Amazon DynamoDB™,
 which needs to be enabled and configured in order to be used. For more
 information on how to enable the DynamoDB™ API in Scylla,
 and the current compatibility of this feature as well as Scylla-specific extensions, see

SCYLLA-VERSION-GEN

@@ -78,7 +78,7 @@ fi
 
 # Default scylla product/version tags
 PRODUCT=scylla
-VERSION=5.5.0-dev
+VERSION=6.1.6
 
 if test -f version
 then
@@ -88,10 +88,13 @@ else
 	SCYLLA_VERSION=$VERSION
 	if [ -z "$SCYLLA_RELEASE" ]; then
 		GIT_COMMIT=$(git -C "$SCRIPT_DIR" log --pretty=format:'%h' -n 1 --abbrev=12)
-		# For custom package builds, replace "0" with "counter.your_name",
+		# For custom package builds, replace "0" with "counter.yourname",
 		# where counter starts at 1 and increments for successive versions.
 		# This ensures that the package manager will select your custom
 		# package over the standard release.
+		# Do not use any special characters like - or _ in the name above!
+		# These characters either have special meaning or are illegal in
+		# version strings.
 		SCYLLA_BUILD=0
 		SCYLLA_RELEASE=$SCYLLA_BUILD.$DATE.$GIT_COMMIT
 	elif [ -f "$OUTPUT_DIR/SCYLLA-RELEASE-FILE" ]; then
@@ -101,7 +104,7 @@ else
 fi
 
 if [ -f "$OUTPUT_DIR/SCYLLA-RELEASE-FILE" ]; then
-	GIT_COMMIT_FILE=$(cat "$OUTPUT_DIR/SCYLLA-RELEASE-FILE" |cut -d . -f 3)
+	GIT_COMMIT_FILE=$(cat "$OUTPUT_DIR/SCYLLA-RELEASE-FILE" | rev | cut -d . -f 1 | rev)
 	if [ "$GIT_COMMIT" = "$GIT_COMMIT_FILE" ]; then
 		exit 0
 	fi

alternator/CMakeLists.txt

@@ -27,7 +27,8 @@ target_link_libraries(alternator
   cql3
   idl
   Seastar::seastar
-  xxHash::xxhash)
+  xxHash::xxhash
+  absl::headers)
 
 check_headers(check-headers alternator
   GLOB_RECURSE ${CMAKE_CURRENT_SOURCE_DIR}/*.hh)

alternator/auth.cc

@@ -19,6 +19,7 @@
 #include "alternator/executor.hh"
 #include "cql3/selection/selection.hh"
 #include "cql3/result_set.hh"
+#include "types/types.hh"
 #include <seastar/core/coroutine.hh>
 
 namespace alternator {
@@ -31,11 +32,12 @@ future<std::string> get_key_from_roles(service::storage_proxy& proxy, auth::serv
     dht::partition_range_vector partition_ranges{dht::partition_range(dht::decorate_key(*schema, pk))};
     std::vector<query::clustering_range> bounds{query::clustering_range::make_open_ended_both_sides()};
     const column_definition* salted_hash_col = schema->get_column_definition(bytes("salted_hash"));
-    if (!salted_hash_col) {
+    const column_definition* can_login_col = schema->get_column_definition(bytes("can_login"));
+    if (!salted_hash_col || !can_login_col) {
         co_await coroutine::return_exception(api_error::unrecognized_client(format("Credentials cannot be fetched for: {}", username)));
     }
-    auto selection = cql3::selection::selection::for_columns(schema, {salted_hash_col});
-    auto partition_slice = query::partition_slice(std::move(bounds), {}, query::column_id_vector{salted_hash_col->id}, selection->get_query_options());
+    auto selection = cql3::selection::selection::for_columns(schema, {salted_hash_col, can_login_col});
+    auto partition_slice = query::partition_slice(std::move(bounds), {}, query::column_id_vector{salted_hash_col->id, can_login_col->id}, selection->get_query_options());
     auto command = ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice,
             proxy.get_max_result_size(partition_slice), query::tombstone_limit(proxy.get_tombstone_limit()));
     auto cl = auth::password_authenticator::consistency_for_user(username);
@@ -51,7 +53,14 @@ future<std::string> get_key_from_roles(service::storage_proxy& proxy, auth::serv
     if (result_set->empty()) {
         co_await coroutine::return_exception(api_error::unrecognized_client(format("User not found: {}", username)));
     }
-    const managed_bytes_opt& salted_hash = result_set->rows().front().front(); // We only asked for 1 row and 1 column
+    const auto& result = result_set->rows().front();
+    bool can_login = result[1] && value_cast<bool>(boolean_type->deserialize(*result[1]));
+    if (!can_login) {
+        // This is a valid role name, but has "login=False" so should not be
+        // usable for authentication (see #19735).
+        co_await coroutine::return_exception(api_error::unrecognized_client(format("Role {} has login=false so cannot be used for login", username)));
+    }
+    const managed_bytes_opt& salted_hash = result.front();
     if (!salted_hash) {
         co_await coroutine::return_exception(api_error::unrecognized_client(format("No password found for user: {}", username)));
     }

alternator/controller.cc

@@ -32,8 +32,10 @@ controller::controller(
         sharded<service::memory_limiter>& memory_limiter,
         sharded<auth::service>& auth_service,
         sharded<qos::service_level_controller>& sl_controller,
-        const db::config& config)
-    : _gossiper(gossiper)
+        const db::config& config,
+        seastar::scheduling_group sg)
+    : protocol_server(sg)
+    , _gossiper(gossiper)
     , _proxy(proxy)
     , _mm(mm)
     , _sys_dist_ks(sys_dist_ks)
@@ -62,7 +64,9 @@ std::vector<socket_address> controller::listen_addresses() const {
 }
 
 future<> controller::start_server() {
-    return seastar::async([this] {
+    seastar::thread_attributes attr;
+    attr.sched_group = _sched_group;
+    return seastar::async(std::move(attr), [this] {
         _listen_addresses.clear();
 
         auto preferred = _config.listen_interface_prefer_ipv6() ? std::make_optional(net::inet_address::family::INET6) : std::nullopt;
@@ -156,7 +160,9 @@ future<> controller::stop_server() {
 }
 
 future<> controller::request_stop_server() {
-    return stop_server();
+    return with_scheduling_group(_sched_group, [this] {
+        return stop_server();
+    });
 }
 
 }

alternator/controller.hh

@@ -80,7 +80,8 @@ public:
         sharded<service::memory_limiter>& memory_limiter,
         sharded<auth::service>& auth_service,
         sharded<qos::service_level_controller>& sl_controller,
-        const db::config& config);
+        const db::config& config,
+        seastar::scheduling_group sg);
 
     virtual sstring name() const override;
     virtual sstring protocol() const override;

alternator/executor.cc

@@ -6,8 +6,10 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
+#include <fmt/ranges.h>
 #include <seastar/core/sleep.hh>
 #include "alternator/executor.hh"
+#include "cdc/log.hh"
 #include "db/config.hh"
 #include "log.hh"
 #include "schema/schema_builder.hh"
@@ -1250,7 +1252,7 @@ future<executor::request_return_type> executor::update_table(client_state& clien
 
         auto schema = builder.build();
 
-        auto m = co_await service::prepare_column_family_update_announcement(p.local(), schema, false,  std::vector<view_ptr>(), group0_guard.write_timestamp());
+        auto m = co_await service::prepare_column_family_update_announcement(p.local(), schema,  std::vector<view_ptr>(), group0_guard.write_timestamp());
 
         co_await mm.announce(std::move(m), std::move(group0_guard), format("alternator-executor: update {} table", tab->cf_name()));
 
@@ -4438,8 +4440,10 @@ future<executor::request_return_type> executor::list_tables(client_state& client
 
     auto tables = _proxy.data_dictionary().get_tables(); // hold on to temporary, table_names isn't a container, it's a view
     auto table_names = tables
-            | boost::adaptors::filtered([] (data_dictionary::table t) {
-                        return t.schema()->ks_name().find(KEYSPACE_NAME_PREFIX) == 0 && !t.schema()->is_view();
+            | boost::adaptors::filtered([this] (data_dictionary::table t) {
+                        return t.schema()->ks_name().find(KEYSPACE_NAME_PREFIX) == 0 &&
+                            !t.schema()->is_view() &&
+                            !cdc::is_log_for_some_table(_proxy.local_db(), t.schema()->ks_name(), t.schema()->cf_name());
                     })
             | boost::adaptors::transformed([] (data_dictionary::table t) {
                         return t.schema()->cf_name();
@@ -4575,7 +4579,7 @@ static lw_shared_ptr<keyspace_metadata> create_keyspace_metadata(std::string_vie
     // used by default on new Alternator tables. Change this initialization
     // to 0 enable tablets by default, with automatic number of tablets.
     std::optional<unsigned> initial_tablets;
-    if (sp.get_db().local().get_config().check_experimental(db::experimental_features_t::feature::TABLETS)) {
+    if (sp.get_db().local().get_config().enable_tablets()) {
         auto it = tags_map.find(INITIAL_TABLETS_TAG_KEY);
         if (it != tags_map.end()) {
             // Tag set. If it's a valid number, use it. If not - e.g., it's

alternator/executor.hh

@@ -9,7 +9,6 @@
 #pragma once
 
 #include <seastar/core/future.hh>
-#include <seastar/http/httpd.hh>
 #include "seastarx.hh"
 #include <seastar/json/json_elements.hh>
 #include <seastar/core/sharded.hh>

alternator/expressions.cc

@@ -28,7 +28,7 @@
 
 namespace alternator {
 
-template <typename Func, typename Result = std::result_of_t<Func(expressionsParser&)>>
+template <typename Func, typename Result = std::invoke_result_t<Func, expressionsParser&>>
 static Result do_with_parser(std::string_view input, Func&& f) {
     expressionsLexer::InputStreamType input_stream{
         reinterpret_cast<const ANTLR_UINT8*>(input.data()),
@@ -43,7 +43,7 @@ static Result do_with_parser(std::string_view input, Func&& f) {
     return result;
 }
 
-template <typename Func, typename Result = std::result_of_t<Func(expressionsParser&)>>
+template <typename Func, typename Result = std::invoke_result_t<Func, expressionsParser&>>
 static Result parse(const char* input_name, std::string_view input, Func&& f) {
     if (input.length() > 4096) {
         throw expressions_syntax_error(format("{} expression size {} exceeds allowed maximum 4096.",

alternator/expressions_types.hh

@@ -66,7 +66,6 @@ public:
     std::vector<std::variant<std::string, unsigned>>& operators() {
         return _operators;
     }
-    friend std::ostream& operator<<(std::ostream&, const path&);
 };
 
 // When an expression is first parsed, all constants are references, like
@@ -256,6 +255,6 @@ public:
 } // namespace parsed
 } // namespace alternator
 
-template <> struct fmt::formatter<alternator::parsed::path> : fmt::formatter<std::string_view> {
+template <> struct fmt::formatter<alternator::parsed::path> : fmt::formatter<string_view> {
     auto format(const alternator::parsed::path&, fmt::format_context& ctx) const -> decltype(ctx.out());
 };

alternator/server.cc

@@ -8,6 +8,7 @@
 
 #include "alternator/server.hh"
 #include "log.hh"
+#include <fmt/ranges.h>
 #include <seastar/http/function_handlers.hh>
 #include <seastar/http/short_streams.hh>
 #include <seastar/core/coroutine.hh>
@@ -20,6 +21,8 @@
 #include "utils/rjson.hh"
 #include "auth.hh"
 #include <cctype>
+#include <string_view>
+#include <utility>
 #include "service/storage_proxy.hh"
 #include "gms/gossiper.hh"
 #include "utils/overloaded_functor.hh"
@@ -33,8 +36,6 @@ using reply = http::reply;
 
 namespace alternator {
 
-static constexpr auto TARGET = "X-Amz-Target";
-
 inline std::vector<std::string_view> split(std::string_view text, char separator) {
     std::vector<std::string_view> tokens;
     if (text == "") {
@@ -210,8 +211,13 @@ protected:
         sstring local_dc = topology.get_datacenter();
         std::unordered_set<gms::inet_address> local_dc_nodes = topology.get_datacenter_endpoints().at(local_dc);
         for (auto& ip : local_dc_nodes) {
-            if (_gossiper.is_alive(ip)) {
-                rjson::push_back(results, rjson::from_string(ip.to_sstring()));
+            // Note that it's not enough for the node to be is_alive() - a
+            // node joining the cluster is also "alive" but not responsive to
+            // requests. We alive *and* normal. See #19694, #21538.
+            if (_gossiper.is_alive(ip) && _gossiper.is_normal(ip)) {
+                // Use the gossiped broadcast_rpc_address if available instead
+                // of the internal IP address "ip". See discussion in #18711.
+                rjson::push_back(results, rjson::from_string(_gossiper.get_rpc_address(ip)));
             }
         }
         rep->set_status(reply::status_type::ok);
@@ -384,10 +390,10 @@ static tracing::trace_state_ptr maybe_trace_query(service::client_state& client_
 
 future<executor::request_return_type> server::handle_api_request(std::unique_ptr<request> req) {
     _executor._stats.total_operations++;
-    sstring target = req->get_header(TARGET);
-    std::vector<std::string_view> split_target = split(target, '.');
-    //NOTICE(sarna): Target consists of Dynamo API version followed by a dot '.' and operation type (e.g. CreateTable)
-    std::string op = split_target.empty() ? std::string() : std::string(split_target.back());
+    sstring target = req->get_header("X-Amz-Target");
+    // target is DynamoDB API version followed by a dot '.' and operation type (e.g. CreateTable)
+    auto dot = target.find('.');
+    std::string_view op = (dot == sstring::npos) ? std::string_view() : std::string_view(target).substr(dot+1);
     // JSON parsing can allocate up to roughly 2x the size of the raw
     // document, + a couple of bytes for maintenance.
     // TODO: consider the case where req->content_length is missing. Maybe
@@ -633,7 +639,7 @@ future<> server::json_parser::stop() {
 
 const char* api_error::what() const noexcept {
     if (_what_string.empty()) {
-        _what_string = format("{} {}: {}", static_cast<int>(_http_code), _type, _msg);
+        _what_string = format("{} {}: {}", std::to_underlying(_http_code), _type, _msg);
     }
     return _what_string.c_str();
 }

alternator/stats.hh

@@ -11,7 +11,6 @@
 #include <cstdint>
 
 #include <seastar/core/metrics_registration.hh>
-#include "utils/estimated_histogram.hh"
 #include "utils/histogram.hh"
 #include "cql3/stats.hh"
 

alternator/streams.cc

@@ -233,11 +233,8 @@ struct shard_id {
 
     // dynamo specifies shardid as max 65 chars. 
     friend std::ostream& operator<<(std::ostream& os, const shard_id& id) {
-        boost::io::ios_flags_saver fs(os);
-        return os << marker << std::hex  
-            << id.time.time_since_epoch().count()
-            << ':' << id.id.to_bytes()
-            ;
+        fmt::print(os, "{} {:x}:{}", marker, id.time.time_since_epoch().count(), id.id.to_bytes());
+        return os;
     }
 };
 
@@ -779,7 +776,7 @@ struct event_id {
     cdc::stream_id stream;
     utils::UUID timestamp;
 
-    static const auto marker = 'E';
+    static constexpr auto marker = 'E';
 
     event_id(cdc::stream_id s, utils::UUID ts)
         : stream(s)
@@ -787,10 +784,8 @@ struct event_id {
     {}
     
     friend std::ostream& operator<<(std::ostream& os, const event_id& id) {
-        boost::io::ios_flags_saver fs(os);
-        return os << marker << std::hex << id.stream.to_bytes()
-            << ':' << id.timestamp
-            ;
+        fmt::print(os, "{}{}:{}", marker, id.stream.to_bytes(), id.timestamp);
+        return os;
     }
 };
 }
@@ -1057,9 +1052,6 @@ void executor::add_stream_options(const rjson::value& stream_specification, sche
     if (stream_enabled->GetBool()) {
         auto db = sp.data_dictionary();
 
-        if (!db.features().cdc) {
-            throw api_error::validation("StreamSpecification: streams (CDC) feature not enabled in cluster.");
-        }
         if (!db.features().alternator_streams) {
             throw api_error::validation("StreamSpecification: alternator streams feature not enabled in cluster.");
         }

alternator/ttl.cc

@@ -26,6 +26,7 @@
 #include "log.hh"
 #include "gc_clock.hh"
 #include "replica/database.hh"
+#include "service/client_state.hh"
 #include "service_permit.hh"
 #include "timestamp.hh"
 #include "service/storage_proxy.hh"
@@ -312,7 +313,7 @@ static size_t random_offset(size_t min, size_t max) {
 // this range's primary node is down. For this we need to return not just
 // a list of this node's secondary ranges - but also the primary owner of
 // each of those ranges.
-static std::vector<std::pair<dht::token_range, gms::inet_address>> get_secondary_ranges(
+static future<std::vector<std::pair<dht::token_range, gms::inet_address>>> get_secondary_ranges(
         const locator::effective_replication_map_ptr& erm,
         gms::inet_address ep) {
     const auto& tm = *erm->get_token_metadata_ptr();
@@ -323,6 +324,7 @@ static std::vector<std::pair<dht::token_range, gms::inet_address>> get_secondary
     }
     auto prev_tok = sorted_tokens.back();
     for (const auto& tok : sorted_tokens) {
+        co_await coroutine::maybe_yield();
         inet_address_vector_replica_set eps = erm->get_natural_endpoints(tok);
         if (eps.size() <= 1 || eps[1] != ep) {
             prev_tok = tok;
@@ -350,7 +352,7 @@ static std::vector<std::pair<dht::token_range, gms::inet_address>> get_secondary
         }
         prev_tok = tok;
     }
-    return ret;
+    co_return ret;
 }
 
 
@@ -383,63 +385,66 @@ static std::vector<std::pair<dht::token_range, gms::inet_address>> get_secondary
 // the chances of covering all ranges during a scan when restarts occur.
 // A more deterministic way would be to regularly persist the scanning state,
 // but that incurs overhead that we want to avoid if not needed.
-enum primary_or_secondary_t {primary, secondary};
-template<primary_or_secondary_t primary_or_secondary>
-class token_ranges_owned_by_this_shard {
-    // ranges_holder_primary holds just the primary ranges themselves
-    class ranges_holder_primary {
-        const dht::token_range_vector _token_ranges;
-     public:
-        ranges_holder_primary(const locator::vnode_effective_replication_map_ptr& erm, gms::gossiper& g, gms::inet_address ep)
-            : _token_ranges(erm->get_primary_ranges(ep)) {}
-        std::size_t size() const { return _token_ranges.size(); }
-        const dht::token_range& operator[](std::size_t i) const {
-            return _token_ranges[i];
-        }
-        bool should_skip(std::size_t i) const {
-            return false;
-        }
-    };
-    // ranges_holder<secondary> holds the secondary token ranges plus each
-    // range's primary owner, needed to implement should_skip().
-    class ranges_holder_secondary {
-        std::vector<std::pair<dht::token_range, gms::inet_address>> _token_ranges;
-        gms::gossiper& _gossiper;
-     public:
-        ranges_holder_secondary(const locator::effective_replication_map_ptr& erm, gms::gossiper& g, gms::inet_address ep)
-            : _token_ranges(get_secondary_ranges(erm, ep))
-            , _gossiper(g) {}
-        std::size_t size() const { return _token_ranges.size(); }
-        const dht::token_range& operator[](std::size_t i) const {
-            return _token_ranges[i].first;
-        }
-        // range i should be skipped if its primary owner is alive.
-        bool should_skip(std::size_t i) const {
-            return _gossiper.is_alive(_token_ranges[i].second);
-        }
-    };
+//
+// FIXME: Check if this algorithm is safe with tablet migration.
+// https://github.com/scylladb/scylladb/issues/16567
 
+// ranges_holder_primary holds just the primary ranges themselves
+class ranges_holder_primary {
+    dht::token_range_vector _token_ranges;
+public:
+    explicit ranges_holder_primary(dht::token_range_vector token_ranges) : _token_ranges(std::move(token_ranges)) {}
+    static future<ranges_holder_primary> make(const locator::vnode_effective_replication_map_ptr& erm, gms::inet_address ep) {
+        co_return ranges_holder_primary(co_await erm->get_primary_ranges(ep));
+    }
+    std::size_t size() const { return _token_ranges.size(); }
+    const dht::token_range& operator[](std::size_t i) const {
+        return _token_ranges[i];
+    }
+    bool should_skip(std::size_t i) const {
+        return false;
+    }
+};
+// ranges_holder<secondary> holds the secondary token ranges plus each
+// range's primary owner, needed to implement should_skip().
+class ranges_holder_secondary {
+    std::vector<std::pair<dht::token_range, gms::inet_address>> _token_ranges;
+    const gms::gossiper& _gossiper;
+public:
+    explicit ranges_holder_secondary(std::vector<std::pair<dht::token_range, gms::inet_address>> token_ranges, const gms::gossiper& g)
+        : _token_ranges(std::move(token_ranges))
+        , _gossiper(g) {}
+    static future<ranges_holder_secondary> make(const locator::effective_replication_map_ptr& erm, gms::inet_address ep, const gms::gossiper& g) {
+        co_return ranges_holder_secondary(co_await get_secondary_ranges(erm, ep), g);
+    }
+    std::size_t size() const { return _token_ranges.size(); }
+    const dht::token_range& operator[](std::size_t i) const {
+        return _token_ranges[i].first;
+    }
+    // range i should be skipped if its primary owner is alive.
+    bool should_skip(std::size_t i) const {
+        return _gossiper.is_alive(_token_ranges[i].second);
+    }
+};
+
+template<class primary_or_secondary_t>
+class token_ranges_owned_by_this_shard {
     schema_ptr _s;
     locator::effective_replication_map_ptr _erm;
     // _token_ranges will contain a list of token ranges owned by this node.
     // We'll further need to split each such range to the pieces owned by
     // the current shard, using _intersecter.
-    using ranges_holder = std::conditional_t<
-            primary_or_secondary == primary_or_secondary_t::primary,
-            ranges_holder_primary,
-            ranges_holder_secondary>;
-    const ranges_holder _token_ranges;
+    const primary_or_secondary_t _token_ranges;
     // NOTICE: _range_idx is used modulo _token_ranges size when accessing
     // the data to ensure that it doesn't go out of bounds
     size_t _range_idx;
     size_t _end_idx;
     std::optional<dht::selective_token_range_sharder> _intersecter;
 public:
-    token_ranges_owned_by_this_shard(replica::database& db, gms::gossiper& g, schema_ptr s)
+    token_ranges_owned_by_this_shard(schema_ptr s, primary_or_secondary_t token_ranges)
         :  _s(s)
         , _erm(s->table().get_effective_replication_map())
-        , _token_ranges(db.find_keyspace(s->ks_name()).get_vnode_effective_replication_map(),
-                g, _erm->get_topology().my_address())
+        , _token_ranges(std::move(token_ranges))
         , _range_idx(random_offset(0, _token_ranges.size() - 1))
         , _end_idx(_range_idx + _token_ranges.size())
     {
@@ -495,6 +500,7 @@ struct scan_ranges_context {
     bytes column_name;
     std::optional<std::string> member;
 
+    service::client_state internal_client_state;
     ::shared_ptr<cql3::selection::selection> selection;
     std::unique_ptr<service::query_state> query_state_ptr;
     std::unique_ptr<cql3::query_options> query_options;
@@ -504,6 +510,7 @@ struct scan_ranges_context {
         : s(s)
         , column_name(column_name)
         , member(member)
+        , internal_client_state(service::client_state::internal_tag())
     {
         // FIXME: don't read the entire items - read only parts of it.
         // We must read the key columns (to be able to delete) and also
@@ -522,10 +529,9 @@ struct scan_ranges_context {
         std::vector<query::clustering_range> ck_bounds{query::clustering_range::make_open_ended_both_sides()};
         auto partition_slice = query::partition_slice(std::move(ck_bounds), {}, std::move(regular_columns), opts);
         command = ::make_lw_shared<query::read_command>(s->id(), s->version(), partition_slice, proxy.get_max_result_size(partition_slice), query::tombstone_limit(proxy.get_tombstone_limit()));
-        executor::client_state client_state{executor::client_state::internal_tag()};
         tracing::trace_state_ptr trace_state;
         // NOTICE: empty_service_permit is used because the TTL service has fixed parallelism
-        query_state_ptr = std::make_unique<service::query_state>(client_state, trace_state, empty_service_permit());
+        query_state_ptr = std::make_unique<service::query_state>(internal_client_state, trace_state, empty_service_permit());
         // FIXME: What should we do on multi-DC? Will we run the expiration on the same ranges on all
         // DCs or only once for each range? If the latter, we need to change the CLs in the
         // scanner and deleter.
@@ -721,7 +727,9 @@ static future<bool> scan_table(
     expiration_stats.scan_table++;
     // FIXME: need to pace the scan, not do it all at once.
     scan_ranges_context scan_ctx{s, proxy, std::move(column_name), std::move(member)};
-    token_ranges_owned_by_this_shard<primary> my_ranges(db.real_database(), gossiper, s);
+    auto erm = db.real_database().find_keyspace(s->ks_name()).get_vnode_effective_replication_map();
+    auto my_address = erm->get_topology().my_address();
+    token_ranges_owned_by_this_shard my_ranges(s, co_await ranges_holder_primary::make(erm, my_address));
     while (std::optional<dht::partition_range> range = my_ranges.next_partition_range()) {
         // Note that because of issue #9167 we need to run a separate
         // query on each partition range, and can't pass several of
@@ -741,7 +749,7 @@ static future<bool> scan_table(
     // by tasking another node to take over scanning of the dead node's primary
     // ranges. What we do here is that this node will also check expiration
     // on its *secondary* ranges - but only those whose primary owner is down.
-    token_ranges_owned_by_this_shard<secondary> my_secondary_ranges(db.real_database(), gossiper, s);
+    token_ranges_owned_by_this_shard my_secondary_ranges(s, co_await ranges_holder_secondary::make(erm, my_address, gossiper));
     while (std::optional<dht::partition_range> range = my_secondary_ranges.next_partition_range()) {
         expiration_stats.secondary_ranges_scanned++;
         dht::partition_range_vector partition_ranges;

api/CMakeLists.txt

@@ -72,7 +72,8 @@ target_link_libraries(api
   idl
   wasmtime_bindings
   Seastar::seastar
-  xxHash::xxhash)
+  xxHash::xxhash
+  absl::headers)
 
 check_headers(check-headers api
   GLOB_RECURSE ${CMAKE_CURRENT_SOURCE_DIR}/*.hh)

api/api-doc/collectd.json

@@ -67,7 +67,7 @@
                "parameters":[
                   {
                      "name":"pluginid",
-                     "description":"The plugin ID, describe the component the metric belongs to. Examples are cache, thrift, etc'. Regex are supported.The plugin ID, describe the component the metric belong to. Examples are: cache, thrift etc'. regex are supported",
+                     "description":"The plugin ID, describe the component the metric belongs to. Examples are cache and alternator, etc'. Regex are supported.",
                      "required":true,
                      "allowMultiple":false,
                      "type":"string",
@@ -199,4 +199,4 @@
          }
       }
    }
-}
+}

api/api-doc/error_injection.json

@@ -63,6 +63,28 @@
                      "paramType":"path"
                   }
                ]
+            },
+            {
+               "method":"GET",
+               "summary":"Read the state of an injection from all shards",
+               "type":"array",
+               "items":{
+                  "type":"error_injection_info"
+               },
+               "nickname":"read_injection",
+               "produces":[
+                  "application/json"
+               ],
+               "parameters":[
+                  {
+                     "name":"injection",
+                     "description":"injection name",
+                     "required":true,
+                     "allowMultiple":false,
+                     "type":"string",
+                     "paramType":"path"
+                  }
+               ]
             }
          ]
       },
@@ -152,5 +174,39 @@
             }
          }
       }
+   },
+   "models":{
+      "mapper":{
+         "id":"mapper",
+         "description":"A key value mapping",
+         "properties":{
+            "key":{
+               "type":"string",
+               "description":"The key"
+            },
+            "value":{
+               "type":"string",
+               "description":"The value"
+            }
+         }
+      },
+       "error_injection_info":{
+         "id":"error_injection_info",
+         "description":"Information about an error injection",
+         "properties":{
+            "enabled":{
+               "type":"boolean",
+               "description":"Is the error injection enabled"
+            },
+            "parameters":{
+               "type":"array",
+               "items":{
+                  "type":"mapper"
+               },
+               "description":"The parameter values"
+            }
+         },
+         "required":["enabled"]
+      }
    }
 }

api/api-doc/raft.json

@@ -62,6 +62,38 @@
                ]
             }
          ]
+      },
+      {
+         "path": "/raft/read_barrier",
+         "operations": [
+            {
+               "method": "POST",
+               "summary": "Triggers read barrier for the given Raft group to wait for previously committed commands in this group to be applied locally. For example, can be used on group 0 to wait for the node to obtain latest schema changes.",
+               "type": "string",
+               "nickname": "read_barrier",
+               "produces": [
+                  "application/json"
+               ],
+               "parameters": [
+                  {
+                     "name": "group_id",
+                     "description": "The ID of the group. When absent, group0 is used.",
+                     "required": false,
+                     "allowMultiple": false,
+                     "type": "string",
+                     "paramType": "query"
+                  },
+                  {
+                     "name": "timeout",
+                     "description": "Timeout in seconds after which the endpoint returns a failure. If not provided, 60s is used.",
+                     "required": false,
+                     "allowMultiple": false,
+                     "type": "long",
+                     "paramType": "query"
+                  }
+               ]
+            }
+         ]
       }
    ]
 }

api/api-doc/storage_service.json

@@ -90,7 +90,7 @@
          "operations":[
             {
                "method":"GET",
-               "summary":"Returns a list of the tokens endpoint mapping",
+               "summary":"Returns a list of the tokens endpoint mapping, provide keyspace and cf param to get tablet mapping",
                "type":"array",
                "items":{
                   "type":"mapper"
@@ -100,6 +100,22 @@
                   "application/json"
                ],
                "parameters":[
+                  {
+                     "name":"keyspace",
+                     "description":"The keyspace to provide the tablet mapping for",
+                     "required":false,
+                     "allowMultiple":false,
+                     "type":"string",
+                     "paramType":"query"
+                  },
+                  {
+                     "name":"cf",
+                     "description":"The table to provide the tablet mapping for",
+                     "required":false,
+                     "allowMultiple":false,
+                     "type":"string",
+                     "paramType":"query"
+                  }
                ]
             }
          ]
@@ -1673,33 +1689,11 @@
       {
          "path":"/storage_service/rpc_server",
          "operations":[
-            {
-               "method":"DELETE",
-               "summary":"Allows a user to disable thrift",
-               "type":"void",
-               "nickname":"stop_rpc_server",
-               "produces":[
-                  "application/json"
-               ],
-               "parameters":[
-               ]
-            },
-            {
-               "method":"POST",
-               "summary":"allows a user to re-enable thrift",
-               "type":"void",
-               "nickname":"start_rpc_server",
-               "produces":[
-                  "application/json"
-               ],
-               "parameters":[
-               ]
-            },
             {
                "method":"GET",
                "summary":"Determine if thrift is running",
                "type":"boolean",
-               "nickname":"is_rpc_server_running",
+               "nickname":"is_thrift_server_running",
                "produces":[
                   "application/json"
                ],
@@ -1897,6 +1891,14 @@
                      "allowMultiple":false,
                      "type":"string",
                      "paramType":"query"
+                  },
+                  {
+                     "name":"force",
+                     "description":"Enforce the source_dc option, even if it unsafe to use for rebuild",
+                     "required":false,
+                     "allowMultiple":false,
+                     "type":"boolean",
+                     "paramType":"query"
                   }
                ]
             }
@@ -2054,7 +2056,7 @@
          "operations":[
             {
                "method":"POST",
-               "summary":"Enables/Disables tracing for the whole system. Only thrift requests can start tracing currently",
+               "summary":"Enables/Disables tracing for the whole system.",
                "type":"void",
                "nickname":"set_trace_probability",
                "produces":[
@@ -2726,6 +2728,22 @@
             }
          ]
       },
+      {
+         "path":"/storage_service/quiesce_topology",
+         "operations":[
+            {
+               "nickname":"quiesce_topology",
+               "method":"POST",
+               "summary":"Waits until there are no ongoing topology operations. Guarantees that topology operations which started before the call are finished after the call. This doesn't consider requested but not started operations. Such operations may start after the call succeeds.",
+               "type":"void",
+               "produces":[
+                  "application/json"
+               ],
+               "parameters":[
+               ]
+            }
+         ]
+      },
       {
          "path":"/storage_service/metrics/total_hints",
          "operations":[

api/api-doc/system.json

@@ -194,6 +194,21 @@
                "parameters":[]
             }
          ]
+      },
+      {
+         "path":"/system/highest_supported_sstable_version",
+         "operations":[
+            {
+               "method":"GET",
+               "summary":"Get highest supported sstable version",
+               "type":"string",
+               "nickname":"get_highest_supported_sstable_version",
+               "produces":[
+                  "application/json"
+               ],
+               "parameters":[]
+            }
+         ]
       }
    ]
 }

api/api-doc/utils.json

@@ -75,7 +75,7 @@
                "items":{
                   "type":"double"
                },
-               "description":"One, five and fifteen mintues rates"
+               "description":"One, five and fifteen minutes rates"
             },
             "mean_rate": {
                "type":"double",

api/api.cc

@@ -71,6 +71,8 @@ future<> set_server_init(http_context& ctx) {
         rb->register_function(r, "error_injection",
             "The error injection API");
         set_error_injection(ctx, r);
+        rb->register_function(r, "storage_proxy",
+                "The storage proxy API");
     });
 }
 
@@ -81,6 +83,10 @@ future<> set_server_config(http_context& ctx, const db::config& cfg) {
     });
 }
 
+future<> unset_server_config(http_context& ctx) {
+    return ctx.http_server.set_routes([&ctx] (routes& r) { unset_config(ctx, r); });
+}
+
 static future<> register_api(http_context& ctx, const sstring& api_name,
         const sstring api_desc,
         std::function<void(http_context& ctx, routes& r)> f) {
@@ -100,12 +106,12 @@ future<> unset_transport_controller(http_context& ctx) {
     return ctx.http_server.set_routes([&ctx] (routes& r) { unset_transport_controller(ctx, r); });
 }
 
-future<> set_rpc_controller(http_context& ctx, thrift_controller& ctl) {
-    return ctx.http_server.set_routes([&ctx, &ctl] (routes& r) { set_rpc_controller(ctx, r, ctl); });
+future<> set_thrift_controller(http_context& ctx) {
+    return ctx.http_server.set_routes([&ctx] (routes& r) { set_thrift_controller(ctx, r); });
 }
 
-future<> unset_rpc_controller(http_context& ctx) {
-    return ctx.http_server.set_routes([&ctx] (routes& r) { unset_rpc_controller(ctx, r); });
+future<> unset_thrift_controller(http_context& ctx) {
+    return ctx.http_server.set_routes([&ctx] (routes& r) { unset_thrift_controller(ctx, r); });
 }
 
 future<> set_server_storage_service(http_context& ctx, sharded<service::storage_service>& ss, service::raft_group0_client& group0_client) {
@@ -118,6 +124,14 @@ future<> unset_server_storage_service(http_context& ctx) {
     return ctx.http_server.set_routes([&ctx] (routes& r) { unset_storage_service(ctx, r); });
 }
 
+future<> set_load_meter(http_context& ctx, service::load_meter& lm) {
+    return ctx.http_server.set_routes([&ctx, &lm] (routes& r) { set_load_meter(ctx, r, lm); });
+}
+
+future<> unset_load_meter(http_context& ctx) {
+    return ctx.http_server.set_routes([&ctx] (routes& r) { unset_load_meter(ctx, r); });
+}
+
 future<> set_server_sstables_loader(http_context& ctx, sharded<sstables_loader>& sst_loader) {
     return ctx.http_server.set_routes([&ctx, &sst_loader] (routes& r) { set_sstables_loader(ctx, r, sst_loader); });
 }
@@ -180,10 +194,21 @@ future<> unset_server_snitch(http_context& ctx) {
 }
 
 future<> set_server_gossip(http_context& ctx, sharded<gms::gossiper>& g) {
-    return register_api(ctx, "gossiper",
+    co_await register_api(ctx, "gossiper",
                 "The gossiper API", [&g] (http_context& ctx, routes& r) {
                     set_gossiper(ctx, r, g.local());
                 });
+    co_await register_api(ctx, "failure_detector",
+                "The failure detector API", [&g] (http_context& ctx, routes& r) {
+                    set_failure_detector(ctx, r, g.local());
+                });
+}
+
+future<> unset_server_gossip(http_context& ctx) {
+    return ctx.http_server.set_routes([&ctx] (routes& r) {
+        unset_gossiper(ctx, r);
+        unset_failure_detector(ctx, r);
+    });
 }
 
 future<> set_server_column_family(http_context& ctx, sharded<db::system_keyspace>& sys_ks) {
@@ -208,10 +233,7 @@ future<> unset_server_messaging_service(http_context& ctx) {
 }
 
 future<> set_server_storage_proxy(http_context& ctx, sharded<service::storage_proxy>& proxy) {
-    return register_api(ctx, "storage_proxy",
-                "The storage proxy API", [&proxy] (http_context& ctx, routes& r) {
-                    set_storage_proxy(ctx, r, proxy);
-                });
+    return ctx.http_server.set_routes([&ctx, &proxy] (routes& r) { set_storage_proxy(ctx, r, proxy); });
 }
 
 future<> unset_server_storage_proxy(http_context& ctx) {
@@ -245,16 +267,6 @@ future<> unset_hinted_handoff(http_context& ctx) {
     return ctx.http_server.set_routes([&ctx] (routes& r) { unset_hinted_handoff(ctx, r); });
 }
 
-future<> set_server_gossip_settle(http_context& ctx, sharded<gms::gossiper>& g) {
-    auto rb = std::make_shared < api_registry_builder > (ctx.api_doc);
-
-    return ctx.http_server.set_routes([rb, &ctx, &g](routes& r) {
-        rb->register_function(r, "failure_detector",
-                "The failure detector API");
-        set_failure_detector(ctx, r, g.local());
-    });
-}
-
 future<> set_server_compaction_manager(http_context& ctx) {
     auto rb = std::make_shared < api_registry_builder > (ctx.api_doc);
 
@@ -346,7 +358,7 @@ void req_params::process(const request& req) {
             continue;
         }
         try {
-            ent.value = req.param[name];
+            ent.value = req.get_path_param(name);
         } catch (std::out_of_range&) {
             throw httpd::bad_param_exception(fmt::format("Mandatory parameter '{}' was not provided", name));
         }

api/api_init.hh

@@ -46,7 +46,6 @@ class snitch_ptr;
 } // namespace locator
 
 namespace cql_transport { class controller; }
-class thrift_controller;
 namespace db {
 class snapshot_ctl;
 class config;
@@ -77,17 +76,16 @@ struct http_context {
     sstring api_doc;
     httpd::http_server_control http_server;
     distributed<replica::database>& db;
-    service::load_meter& lmeter;
 
-    http_context(distributed<replica::database>& _db,
-            service::load_meter& _lm)
-            : db(_db), lmeter(_lm)
+    http_context(distributed<replica::database>& _db)
+            : db(_db)
     {
     }
 };
 
 future<> set_server_init(http_context& ctx);
 future<> set_server_config(http_context& ctx, const db::config& cfg);
+future<> unset_server_config(http_context& ctx);
 future<> set_server_snitch(http_context& ctx, sharded<locator::snitch_ptr>& snitch);
 future<> unset_server_snitch(http_context& ctx);
 future<> set_server_storage_service(http_context& ctx, sharded<service::storage_service>& ss, service::raft_group0_client&);
@@ -100,8 +98,8 @@ future<> set_server_repair(http_context& ctx, sharded<repair_service>& repair);
 future<> unset_server_repair(http_context& ctx);
 future<> set_transport_controller(http_context& ctx, cql_transport::controller& ctl);
 future<> unset_transport_controller(http_context& ctx);
-future<> set_rpc_controller(http_context& ctx, thrift_controller& ctl);
-future<> unset_rpc_controller(http_context& ctx);
+future<> set_thrift_controller(http_context& ctx);
+future<> unset_thrift_controller(http_context& ctx);
 future<> set_server_authorization_cache(http_context& ctx, sharded<auth::service> &auth_service);
 future<> unset_server_authorization_cache(http_context& ctx);
 future<> set_server_snapshot(http_context& ctx, sharded<db::snapshot_ctl>& snap_ctl);
@@ -109,6 +107,7 @@ future<> unset_server_snapshot(http_context& ctx);
 future<> set_server_token_metadata(http_context& ctx, sharded<locator::shared_token_metadata>& tm);
 future<> unset_server_token_metadata(http_context& ctx);
 future<> set_server_gossip(http_context& ctx, sharded<gms::gossiper>& g);
+future<> unset_server_gossip(http_context& ctx);
 future<> set_server_column_family(http_context& ctx, sharded<db::system_keyspace>& sys_ks);
 future<> unset_server_column_family(http_context& ctx);
 future<> set_server_messaging_service(http_context& ctx, sharded<netw::messaging_service>& ms);
@@ -119,7 +118,6 @@ future<> set_server_stream_manager(http_context& ctx, sharded<streaming::stream_
 future<> unset_server_stream_manager(http_context& ctx);
 future<> set_hinted_handoff(http_context& ctx, sharded<service::storage_proxy>& p);
 future<> unset_hinted_handoff(http_context& ctx);
-future<> set_server_gossip_settle(http_context& ctx, sharded<gms::gossiper>& g);
 future<> set_server_cache(http_context& ctx);
 future<> set_server_compaction_manager(http_context& ctx);
 future<> set_server_done(http_context& ctx);
@@ -131,5 +129,7 @@ future<> set_server_tasks_compaction_module(http_context& ctx, sharded<service::
 future<> unset_server_tasks_compaction_module(http_context& ctx);
 future<> set_server_raft(http_context&, sharded<service::raft_group_registry>&);
 future<> unset_server_raft(http_context&);
+future<> set_load_meter(http_context& ctx, service::load_meter& lm);
+future<> unset_load_meter(http_context& ctx);
 
 }

api/authorization_cache.hh

@@ -8,11 +8,20 @@
 
 #pragma once
 
-#include "api.hh"
+#include <seastar/core/sharded.hh>
+
+namespace seastar::httpd {
+class routes;
+}
+
+namespace auth {
+class service;
+}
 
 namespace api {
 
-void set_authorization_cache(http_context& ctx, httpd::routes& r, sharded<auth::service> &auth_service);
-void unset_authorization_cache(http_context& ctx, httpd::routes& r);
+struct http_context;
+void set_authorization_cache(http_context& ctx, seastar::httpd::routes& r, seastar::sharded<auth::service> &auth_service);
+void unset_authorization_cache(http_context& ctx, seastar::httpd::routes& r);
 
 }

api/cache_service.cc

@@ -7,6 +7,7 @@
  */
 
 #include "cache_service.hh"
+#include "api/api.hh"
 #include "api/api-doc/cache_service.json.hh"
 #include "column_family.hh"
 
@@ -195,9 +196,9 @@ void set_cache_service(http_context& ctx, routes& r) {
         return make_ready_future<json::json_return_type>(0);
     });
 
-    cs::get_row_capacity.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return ctx.db.map_reduce0([](replica::database& db) -> uint64_t {
-            return memory::stats().total_memory();
+    cs::get_row_capacity.set(r, [] (std::unique_ptr<http::request> req) {
+        return seastar::map_reduce(smp::all_cpus(), [] (int cpu) {
+            return make_ready_future<uint64_t>(memory::stats().total_memory());
         }, uint64_t(0), std::plus<uint64_t>()).then([](const int64_t& res) {
             return make_ready_future<json::json_return_type>(res);
         });

api/cache_service.hh

@@ -8,10 +8,13 @@
 
 #pragma once
 
-#include "api.hh"
+namespace seastar::httpd {
+class routes;
+}
 
 namespace api {
 
-void set_cache_service(http_context& ctx, httpd::routes& r);
+struct http_context;
+void set_cache_service(http_context& ctx, seastar::httpd::routes& r);
 
 }

api/collectd.cc

@@ -54,7 +54,7 @@ static const char* str_to_regex(const sstring& v) {
 void set_collectd(http_context& ctx, routes& r) {
     cd::get_collectd.set(r, [](std::unique_ptr<request> req) {
 
-        auto id = ::make_shared<scollectd::type_instance_id>(req->param["pluginid"],
+        auto id = ::make_shared<scollectd::type_instance_id>(req->get_path_param("pluginid"),
                 req->get_query_param("instance"), req->get_query_param("type"),
                 req->get_query_param("type_instance"));
 
@@ -91,7 +91,7 @@ void set_collectd(http_context& ctx, routes& r) {
     });
 
     cd::enable_collectd.set(r, [](std::unique_ptr<request> req) -> future<json::json_return_type> {
-        std::regex plugin(req->param["pluginid"].c_str());
+        std::regex plugin(req->get_path_param("pluginid").c_str());
         std::regex instance(str_to_regex(req->get_query_param("instance")));
         std::regex type(str_to_regex(req->get_query_param("type")));
         std::regex type_instance(str_to_regex(req->get_query_param("type_instance")));

api/column_family.cc

@@ -6,9 +6,11 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
+#include <fmt/ranges.h>
 #include "column_family.hh"
 #include "api/api.hh"
 #include "api/api-doc/column_family.json.hh"
+#include "api/api-doc/storage_service.json.hh"
 #include <vector>
 #include <seastar/http/exception.hh>
 #include "sstables/sstables.hh"
@@ -28,6 +30,7 @@ using namespace httpd;
 
 using namespace json;
 namespace cf = httpd::column_family_json;
+namespace ss = httpd::storage_service_json;
 
 std::tuple<sstring, sstring> parse_fully_qualified_cf_name(sstring name) {
     auto pos = name.find("%3A");
@@ -79,6 +82,65 @@ future<json::json_return_type>  get_cf_stats(http_context& ctx,
     }, std::plus<int64_t>());
 }
 
+static future<json::json_return_type> set_tables(http_context& ctx, const sstring& keyspace, std::vector<sstring> tables, std::function<future<>(replica::table&)> set) {
+    if (tables.empty()) {
+        tables = map_keys(ctx.db.local().find_keyspace(keyspace).metadata().get()->cf_meta_data());
+    }
+
+    return do_with(keyspace, std::move(tables), [&ctx, set] (const sstring& keyspace, const std::vector<sstring>& tables) {
+        return ctx.db.invoke_on_all([&keyspace, &tables, set] (replica::database& db) {
+            return parallel_for_each(tables, [&db, &keyspace, set] (const sstring& table) {
+                replica::table& t = db.find_column_family(keyspace, table);
+                return set(t);
+            });
+        });
+    }).then([] {
+        return make_ready_future<json::json_return_type>(json_void());
+    });
+}
+
+class autocompaction_toggle_guard {
+    replica::database& _db;
+public:
+    autocompaction_toggle_guard(replica::database& db) : _db(db) {
+        assert(this_shard_id() == 0);
+        if (!_db._enable_autocompaction_toggle) {
+            throw std::runtime_error("Autocompaction toggle is busy");
+        }
+        _db._enable_autocompaction_toggle = false;
+    }
+    autocompaction_toggle_guard(const autocompaction_toggle_guard&) = delete;
+    autocompaction_toggle_guard(autocompaction_toggle_guard&&) = default;
+    ~autocompaction_toggle_guard() {
+        assert(this_shard_id() == 0);
+        _db._enable_autocompaction_toggle = true;
+    }
+};
+
+static future<json::json_return_type> set_tables_autocompaction(http_context& ctx, const sstring &keyspace, std::vector<sstring> tables, bool enabled) {
+    apilog.info("set_tables_autocompaction: enabled={} keyspace={} tables={}", enabled, keyspace, tables);
+
+    return ctx.db.invoke_on(0, [&ctx, keyspace, tables = std::move(tables), enabled] (replica::database& db) {
+        auto g = autocompaction_toggle_guard(db);
+        return set_tables(ctx, keyspace, tables, [enabled] (replica::table& cf) {
+            if (enabled) {
+                cf.enable_auto_compaction();
+            } else {
+                return cf.disable_auto_compaction();
+            }
+            return make_ready_future<>();
+        }).finally([g = std::move(g)] {});
+    });
+}
+
+static future<json::json_return_type> set_tables_tombstone_gc(http_context& ctx, const sstring &keyspace, std::vector<sstring> tables, bool enabled) {
+    apilog.info("set_tables_tombstone_gc: enabled={} keyspace={} tables={}", enabled, keyspace, tables);
+    return set_tables(ctx, keyspace, std::move(tables), [enabled] (replica::table& t) {
+        t.set_tombstone_gc_enabled(enabled);
+        return make_ready_future<>();
+    });
+}
+
 static future<json::json_return_type>  get_cf_stats_count(http_context& ctx, const sstring& name,
         utils::timed_rate_moving_average_summary_and_histogram replica::column_family_stats::*f) {
     return map_reduce_cf(ctx, name, int64_t(0), [f](const replica::column_family& cf) {
@@ -304,6 +366,14 @@ ratio_holder filter_recent_false_positive_as_ratio_holder(const sstables::shared
     return ratio_holder(f + sst->filter_get_recent_true_positive(), f);
 }
 
+uint64_t accumulate_on_active_memtables(replica::table& t, noncopyable_function<uint64_t(replica::memtable& mt)> action) {
+    uint64_t ret = 0;
+    t.for_each_active_memtable([&] (replica::memtable& mt) {
+        ret += action(mt);
+    });
+    return ret;
+}
+
 void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace>& sys_ks) {
     cf::get_column_family_name.set(r, [&ctx] (const_req req){
         std::vector<sstring> res;
@@ -338,14 +408,14 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_memtable_columns_count.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], uint64_t{0}, [](replica::column_family& cf) {
-            return boost::accumulate(cf.active_memtables() | boost::adaptors::transformed(std::mem_fn(&replica::memtable::partition_count)), uint64_t(0));
+        return map_reduce_cf(ctx, req->get_path_param("name"), uint64_t{0}, [](replica::column_family& cf) {
+            return accumulate_on_active_memtables(cf, std::mem_fn(&replica::memtable::partition_count));
         }, std::plus<>());
     });
 
     cf::get_all_memtable_columns_count.set(r, [&ctx] (std::unique_ptr<http::request> req) {
         return map_reduce_cf(ctx, uint64_t{0}, [](replica::column_family& cf) {
-            return boost::accumulate(cf.active_memtables() | boost::adaptors::transformed(std::mem_fn(&replica::memtable::partition_count)), uint64_t(0));
+            return accumulate_on_active_memtables(cf, std::mem_fn(&replica::memtable::partition_count));
         }, std::plus<>());
     });
 
@@ -358,34 +428,34 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_memtable_off_heap_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](replica::column_family& cf) {
-            return boost::accumulate(cf.active_memtables() | boost::adaptors::transformed([] (replica::memtable* active_memtable) {
-                return active_memtable->region().occupancy().total_space();
-            }), uint64_t(0));
+        return map_reduce_cf(ctx, req->get_path_param("name"), int64_t(0), [](replica::column_family& cf) {
+            return accumulate_on_active_memtables(cf, [] (replica::memtable& active_memtable) {
+                return active_memtable.region().occupancy().total_space();
+            });
         }, std::plus<int64_t>());
     });
 
     cf::get_all_memtable_off_heap_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
         return map_reduce_cf(ctx, int64_t(0), [](replica::column_family& cf) {
-            return boost::accumulate(cf.active_memtables() | boost::adaptors::transformed([] (replica::memtable* active_memtable) {
-                return active_memtable->region().occupancy().total_space();
-            }), uint64_t(0));
+            return accumulate_on_active_memtables(cf, [] (replica::memtable& active_memtable) {
+                return active_memtable.region().occupancy().total_space();
+            });
         }, std::plus<int64_t>());
     });
 
     cf::get_memtable_live_data_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](replica::column_family& cf) {
-            return boost::accumulate(cf.active_memtables() | boost::adaptors::transformed([] (replica::memtable* active_memtable) {
-                return active_memtable->region().occupancy().used_space();
-            }), uint64_t(0));
+        return map_reduce_cf(ctx, req->get_path_param("name"), int64_t(0), [](replica::column_family& cf) {
+            return accumulate_on_active_memtables(cf, [] (replica::memtable& active_memtable) {
+                return active_memtable.region().occupancy().used_space();
+            });
         }, std::plus<int64_t>());
     });
 
     cf::get_all_memtable_live_data_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
         return map_reduce_cf(ctx, int64_t(0), [](replica::column_family& cf) {
-            return boost::accumulate(cf.active_memtables() | boost::adaptors::transformed([] (replica::memtable* active_memtable) {
-                return active_memtable->region().occupancy().used_space();
-            }), uint64_t(0));
+            return accumulate_on_active_memtables(cf, [] (replica::memtable& active_memtable) {
+                return active_memtable.region().occupancy().used_space();
+            });
         }, std::plus<int64_t>());
     });
 
@@ -399,7 +469,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
 
     cf::get_cf_all_memtables_off_heap_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
         warn(unimplemented::cause::INDEXES);
-        return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](replica::column_family& cf) {
+        return map_reduce_cf(ctx, req->get_path_param("name"), int64_t(0), [](replica::column_family& cf) {
             return cf.occupancy().total_space();
         }, std::plus<int64_t>());
     });
@@ -415,7 +485,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
 
     cf::get_cf_all_memtables_live_data_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
         warn(unimplemented::cause::INDEXES);
-        return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](replica::column_family& cf) {
+        return map_reduce_cf(ctx, req->get_path_param("name"), int64_t(0), [](replica::column_family& cf) {
             return cf.occupancy().used_space();
         }, std::plus<int64_t>());
     });
@@ -423,14 +493,14 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     cf::get_all_cf_all_memtables_live_data_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
         warn(unimplemented::cause::INDEXES);
         return map_reduce_cf(ctx, int64_t(0), [](replica::column_family& cf) {
-            return boost::accumulate(cf.active_memtables() | boost::adaptors::transformed([] (replica::memtable* active_memtable) {
-                return active_memtable->region().occupancy().used_space();
-            }), uint64_t(0));
+            return accumulate_on_active_memtables(cf, [] (replica::memtable& active_memtable) {
+                return active_memtable.region().occupancy().used_space();
+            });
         }, std::plus<int64_t>());
     });
 
     cf::get_memtable_switch_count.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return get_cf_stats(ctx,req->param["name"] ,&replica::column_family_stats::memtable_switch_count);
+        return get_cf_stats(ctx,req->get_path_param("name") ,&replica::column_family_stats::memtable_switch_count);
     });
 
     cf::get_all_memtable_switch_count.set(r, [&ctx] (std::unique_ptr<http::request> req) {
@@ -439,7 +509,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
 
     // FIXME: this refers to partitions, not rows.
     cf::get_estimated_row_size_histogram.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], utils::estimated_histogram(0), [](replica::column_family& cf) {
+        return map_reduce_cf(ctx, req->get_path_param("name"), utils::estimated_histogram(0), [](replica::column_family& cf) {
             utils::estimated_histogram res(0);
             for (auto sstables = cf.get_sstables(); auto& i : *sstables) {
                 res.merge(i->get_stats_metadata().estimated_partition_size);
@@ -451,7 +521,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
 
     // FIXME: this refers to partitions, not rows.
     cf::get_estimated_row_count.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](replica::column_family& cf) {
+        return map_reduce_cf(ctx, req->get_path_param("name"), int64_t(0), [](replica::column_family& cf) {
             uint64_t res = 0;
             for (auto sstables = cf.get_sstables(); auto& i : *sstables) {
                 res += i->get_stats_metadata().estimated_partition_size.count();
@@ -462,7 +532,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_estimated_column_count_histogram.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], utils::estimated_histogram(0), [](replica::column_family& cf) {
+        return map_reduce_cf(ctx, req->get_path_param("name"), utils::estimated_histogram(0), [](replica::column_family& cf) {
             utils::estimated_histogram res(0);
             for (auto sstables = cf.get_sstables(); auto& i : *sstables) {
                 res.merge(i->get_stats_metadata().estimated_cells_count);
@@ -479,7 +549,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_pending_flushes.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return get_cf_stats(ctx,req->param["name"] ,&replica::column_family_stats::pending_flushes);
+        return get_cf_stats(ctx,req->get_path_param("name") ,&replica::column_family_stats::pending_flushes);
     });
 
     cf::get_all_pending_flushes.set(r, [&ctx] (std::unique_ptr<http::request> req) {
@@ -487,7 +557,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_read.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return get_cf_stats_count(ctx,req->param["name"] ,&replica::column_family_stats::reads);
+        return get_cf_stats_count(ctx,req->get_path_param("name") ,&replica::column_family_stats::reads);
     });
 
     cf::get_all_read.set(r, [&ctx] (std::unique_ptr<http::request> req) {
@@ -495,7 +565,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_write.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return get_cf_stats_count(ctx, req->param["name"] ,&replica::column_family_stats::writes);
+        return get_cf_stats_count(ctx, req->get_path_param("name") ,&replica::column_family_stats::writes);
     });
 
     cf::get_all_write.set(r, [&ctx] (std::unique_ptr<http::request> req) {
@@ -503,19 +573,19 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_read_latency_histogram_depricated.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return get_cf_histogram(ctx, req->param["name"], &replica::column_family_stats::reads);
+        return get_cf_histogram(ctx, req->get_path_param("name"), &replica::column_family_stats::reads);
     });
 
     cf::get_read_latency_histogram.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return get_cf_rate_and_histogram(ctx, req->param["name"], &replica::column_family_stats::reads);
+        return get_cf_rate_and_histogram(ctx, req->get_path_param("name"), &replica::column_family_stats::reads);
     });
 
     cf::get_read_latency.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return get_cf_stats_sum(ctx,req->param["name"] ,&replica::column_family_stats::reads);
+        return get_cf_stats_sum(ctx,req->get_path_param("name") ,&replica::column_family_stats::reads);
     });
 
     cf::get_write_latency.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return get_cf_stats_sum(ctx, req->param["name"] ,&replica::column_family_stats::writes);
+        return get_cf_stats_sum(ctx, req->get_path_param("name") ,&replica::column_family_stats::writes);
     });
 
     cf::get_all_read_latency_histogram_depricated.set(r, [&ctx] (std::unique_ptr<http::request> req) {
@@ -527,11 +597,11 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_write_latency_histogram_depricated.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return get_cf_histogram(ctx, req->param["name"], &replica::column_family_stats::writes);
+        return get_cf_histogram(ctx, req->get_path_param("name"), &replica::column_family_stats::writes);
     });
 
     cf::get_write_latency_histogram.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return get_cf_rate_and_histogram(ctx, req->param["name"], &replica::column_family_stats::writes);
+        return get_cf_rate_and_histogram(ctx, req->get_path_param("name"), &replica::column_family_stats::writes);
     });
 
     cf::get_all_write_latency_histogram_depricated.set(r, [&ctx] (std::unique_ptr<http::request> req) {
@@ -543,7 +613,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_pending_compactions.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](replica::column_family& cf) {
+        return map_reduce_cf(ctx, req->get_path_param("name"), int64_t(0), [](replica::column_family& cf) {
             return cf.estimate_pending_compactions();
         }, std::plus<int64_t>());
     });
@@ -555,7 +625,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_live_ss_table_count.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return get_cf_stats(ctx, req->param["name"], &replica::column_family_stats::live_sstable_count);
+        return get_cf_stats(ctx, req->get_path_param("name"), &replica::column_family_stats::live_sstable_count);
     });
 
     cf::get_all_live_ss_table_count.set(r, [&ctx] (std::unique_ptr<http::request> req) {
@@ -563,11 +633,11 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_unleveled_sstables.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return get_cf_unleveled_sstables(ctx, req->param["name"]);
+        return get_cf_unleveled_sstables(ctx, req->get_path_param("name"));
     });
 
     cf::get_live_disk_space_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return sum_sstable(ctx, req->param["name"], false);
+        return sum_sstable(ctx, req->get_path_param("name"), false);
     });
 
     cf::get_all_live_disk_space_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
@@ -575,7 +645,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_total_disk_space_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return sum_sstable(ctx, req->param["name"], true);
+        return sum_sstable(ctx, req->get_path_param("name"), true);
     });
 
     cf::get_all_total_disk_space_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
@@ -584,7 +654,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
 
     // FIXME: this refers to partitions, not rows.
     cf::get_min_row_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], INT64_MAX, min_partition_size, min_int64);
+        return map_reduce_cf(ctx, req->get_path_param("name"), INT64_MAX, min_partition_size, min_int64);
     });
 
     // FIXME: this refers to partitions, not rows.
@@ -594,7 +664,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
 
     // FIXME: this refers to partitions, not rows.
     cf::get_max_row_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], int64_t(0), max_partition_size, max_int64);
+        return map_reduce_cf(ctx, req->get_path_param("name"), int64_t(0), max_partition_size, max_int64);
     });
 
     // FIXME: this refers to partitions, not rows.
@@ -605,7 +675,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     // FIXME: this refers to partitions, not rows.
     cf::get_mean_row_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
         // Cassandra 3.x mean values are truncated as integrals.
-        return map_reduce_cf(ctx, req->param["name"], integral_ratio_holder(), mean_partition_size, std::plus<integral_ratio_holder>());
+        return map_reduce_cf(ctx, req->get_path_param("name"), integral_ratio_holder(), mean_partition_size, std::plus<integral_ratio_holder>());
     });
 
     // FIXME: this refers to partitions, not rows.
@@ -615,7 +685,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_bloom_filter_false_positives.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], uint64_t(0), [] (replica::column_family& cf) {
+        return map_reduce_cf(ctx, req->get_path_param("name"), uint64_t(0), [] (replica::column_family& cf) {
             auto sstables = cf.get_sstables();
             return std::accumulate(sstables->begin(), sstables->end(), uint64_t(0), [](uint64_t s, auto& sst) {
                 return s + sst->filter_get_false_positive();
@@ -633,7 +703,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_recent_bloom_filter_false_positives.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], uint64_t(0), [] (replica::column_family& cf) {
+        return map_reduce_cf(ctx, req->get_path_param("name"), uint64_t(0), [] (replica::column_family& cf) {
             auto sstables = cf.get_sstables();
             return std::accumulate(sstables->begin(), sstables->end(), uint64_t(0), [](uint64_t s, auto& sst) {
                 return s + sst->filter_get_recent_false_positive();
@@ -651,7 +721,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_bloom_filter_false_ratio.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], ratio_holder(), [] (replica::column_family& cf) {
+        return map_reduce_cf(ctx, req->get_path_param("name"), ratio_holder(), [] (replica::column_family& cf) {
             return boost::accumulate(*cf.get_sstables() | boost::adaptors::transformed(filter_false_positive_as_ratio_holder), ratio_holder());
         }, std::plus<>());
     });
@@ -663,7 +733,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_recent_bloom_filter_false_ratio.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], ratio_holder(), [] (replica::column_family& cf) {
+        return map_reduce_cf(ctx, req->get_path_param("name"), ratio_holder(), [] (replica::column_family& cf) {
             return boost::accumulate(*cf.get_sstables() | boost::adaptors::transformed(filter_recent_false_positive_as_ratio_holder), ratio_holder());
         }, std::plus<>());
     });
@@ -675,7 +745,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_bloom_filter_disk_space_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], uint64_t(0), [] (replica::column_family& cf) {
+        return map_reduce_cf(ctx, req->get_path_param("name"), uint64_t(0), [] (replica::column_family& cf) {
             auto sstables = cf.get_sstables();
             return std::accumulate(sstables->begin(), sstables->end(), uint64_t(0), [](uint64_t s, auto& sst) {
                 return s + sst->filter_size();
@@ -693,7 +763,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_bloom_filter_off_heap_memory_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], uint64_t(0), [] (replica::column_family& cf) {
+        return map_reduce_cf(ctx, req->get_path_param("name"), uint64_t(0), [] (replica::column_family& cf) {
             auto sstables = cf.get_sstables();
             return std::accumulate(sstables->begin(), sstables->end(), uint64_t(0), [](uint64_t s, auto& sst) {
                 return s + sst->filter_memory_size();
@@ -711,7 +781,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_index_summary_off_heap_memory_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], uint64_t(0), [] (replica::column_family& cf) {
+        return map_reduce_cf(ctx, req->get_path_param("name"), uint64_t(0), [] (replica::column_family& cf) {
             auto sstables = cf.get_sstables();
             return std::accumulate(sstables->begin(), sstables->end(), uint64_t(0), [](uint64_t s, auto& sst) {
                 return s + sst->get_summary().memory_footprint();
@@ -734,7 +804,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
         // We are missing the off heap memory calculation
         // Return 0 is the wrong value. It's a work around
         // until the memory calculation will be available
-        //auto id = get_uuid(req->param["name"], ctx.db.local());
+        //auto id = get_uuid(req->get_path_param("name"), ctx.db.local());
         return make_ready_future<json::json_return_type>(0);
     });
 
@@ -747,7 +817,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     cf::get_speculative_retries.set(r, [] (std::unique_ptr<http::request> req) {
         //TBD
         unimplemented();
-        //auto id = get_uuid(req->param["name"], ctx.db.local());
+        //auto id = get_uuid(req->get_path_param("name"), ctx.db.local());
         return make_ready_future<json::json_return_type>(0);
     });
 
@@ -760,32 +830,14 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     cf::get_key_cache_hit_rate.set(r, [] (std::unique_ptr<http::request> req) {
         //TBD
         unimplemented();
-        //auto id = get_uuid(req->param["name"], ctx.db.local());
-        return make_ready_future<json::json_return_type>(0);
-    });
-
-    cf::get_true_snapshots_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        auto uuid = get_uuid(req->param["name"], ctx.db.local());
-        return ctx.db.local().find_column_family(uuid).get_snapshot_details().then([](
-                const std::unordered_map<sstring, replica::column_family::snapshot_details>& sd) {
-            int64_t res = 0;
-            for (auto i : sd) {
-                res += i.second.total;
-            }
-            return make_ready_future<json::json_return_type>(res);
-        });
-    });
-
-    cf::get_all_true_snapshots_size.set(r, [] (std::unique_ptr<http::request> req) {
-        //TBD
-        unimplemented();
+        //auto id = get_uuid(req->get_path_param("name"), ctx.db.local());
         return make_ready_future<json::json_return_type>(0);
     });
 
     cf::get_row_cache_hit_out_of_range.set(r, [] (std::unique_ptr<http::request> req) {
         //TBD
         unimplemented();
-        //auto id = get_uuid(req->param["name"], ctx.db.local());
+        //auto id = get_uuid(req->get_path_param("name"), ctx.db.local());
         return make_ready_future<json::json_return_type>(0);
     });
 
@@ -796,7 +848,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_row_cache_hit.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf_raw(ctx, req->param["name"], utils::rate_moving_average(), [](const replica::column_family& cf) {
+        return map_reduce_cf_raw(ctx, req->get_path_param("name"), utils::rate_moving_average(), [](const replica::column_family& cf) {
             return cf.get_row_cache().stats().hits.rate();
         }, std::plus<utils::rate_moving_average>()).then([](const utils::rate_moving_average& m) {
             return make_ready_future<json::json_return_type>(meter_to_json(m));
@@ -812,7 +864,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_row_cache_miss.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf_raw(ctx, req->param["name"], utils::rate_moving_average(), [](const replica::column_family& cf) {
+        return map_reduce_cf_raw(ctx, req->get_path_param("name"), utils::rate_moving_average(), [](const replica::column_family& cf) {
             return cf.get_row_cache().stats().misses.rate();
         }, std::plus<utils::rate_moving_average>()).then([](const utils::rate_moving_average& m) {
             return make_ready_future<json::json_return_type>(meter_to_json(m));
@@ -829,102 +881,120 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_cas_prepare.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf_time_histogram(ctx, req->param["name"], [](const replica::column_family& cf) {
+        return map_reduce_cf_time_histogram(ctx, req->get_path_param("name"), [](const replica::column_family& cf) {
             return cf.get_stats().cas_prepare.histogram();
         });
     });
 
     cf::get_cas_propose.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf_time_histogram(ctx, req->param["name"], [](const replica::column_family& cf) {
+        return map_reduce_cf_time_histogram(ctx, req->get_path_param("name"), [](const replica::column_family& cf) {
             return cf.get_stats().cas_accept.histogram();
         });
     });
 
     cf::get_cas_commit.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf_time_histogram(ctx, req->param["name"], [](const replica::column_family& cf) {
+        return map_reduce_cf_time_histogram(ctx, req->get_path_param("name"), [](const replica::column_family& cf) {
             return cf.get_stats().cas_learn.histogram();
         });
     });
 
     cf::get_sstables_per_read_histogram.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return map_reduce_cf(ctx, req->param["name"], utils::estimated_histogram(0), [](replica::column_family& cf) {
+        return map_reduce_cf(ctx, req->get_path_param("name"), utils::estimated_histogram(0), [](replica::column_family& cf) {
             return cf.get_stats().estimated_sstable_per_read;
         },
         utils::estimated_histogram_merge, utils_json::estimated_histogram());
     });
 
     cf::get_tombstone_scanned_histogram.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return get_cf_histogram(ctx, req->param["name"], &replica::column_family_stats::tombstone_scanned);
+        return get_cf_histogram(ctx, req->get_path_param("name"), &replica::column_family_stats::tombstone_scanned);
     });
 
     cf::get_live_scanned_histogram.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return get_cf_histogram(ctx, req->param["name"], &replica::column_family_stats::live_scanned);
+        return get_cf_histogram(ctx, req->get_path_param("name"), &replica::column_family_stats::live_scanned);
     });
 
     cf::get_col_update_time_delta_histogram.set(r, [] (std::unique_ptr<http::request> req) {
         //TBD
         unimplemented();
-        //auto id = get_uuid(req->param["name"], ctx.db.local());
+        //auto id = get_uuid(req->get_path_param("name"), ctx.db.local());
         std::vector<double> res;
         return make_ready_future<json::json_return_type>(res);
     });
 
     cf::get_auto_compaction.set(r, [&ctx] (const_req req) {
-        auto uuid = get_uuid(req.param["name"], ctx.db.local());
+        auto uuid = get_uuid(req.get_path_param("name"), ctx.db.local());
         replica::column_family& cf = ctx.db.local().find_column_family(uuid);
         return !cf.is_auto_compaction_disabled_by_user();
     });
 
     cf::enable_auto_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        apilog.info("column_family/enable_auto_compaction: name={}", req->param["name"]);
-        return ctx.db.invoke_on(0, [&ctx, req = std::move(req)] (replica::database& db) {
-            auto g = replica::database::autocompaction_toggle_guard(db);
-            return foreach_column_family(ctx, req->param["name"], [](replica::column_family &cf) {
-                cf.enable_auto_compaction();
-            }).then([g = std::move(g)] {
-                return make_ready_future<json::json_return_type>(json_void());
-            });
-        });
+        apilog.info("column_family/enable_auto_compaction: name={}", req->get_path_param("name"));
+        auto [ks, cf] = parse_fully_qualified_cf_name(req->get_path_param("name"));
+        validate_table(ctx, ks, cf);
+        return set_tables_autocompaction(ctx, ks, {std::move(cf)}, true);
     });
 
     cf::disable_auto_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        apilog.info("column_family/disable_auto_compaction: name={}", req->param["name"]);
-        return ctx.db.invoke_on(0, [&ctx, req = std::move(req)] (replica::database& db) {
-            auto g = replica::database::autocompaction_toggle_guard(db);
-            return foreach_column_family(ctx, req->param["name"], [](replica::column_family &cf) {
-                return cf.disable_auto_compaction();
-            }).then([g = std::move(g)] {
-                return make_ready_future<json::json_return_type>(json_void());
-            });
-        });
+        apilog.info("column_family/disable_auto_compaction: name={}", req->get_path_param("name"));
+        auto [ks, cf] = parse_fully_qualified_cf_name(req->get_path_param("name"));
+        validate_table(ctx, ks, cf);
+        return set_tables_autocompaction(ctx, ks, {std::move(cf)}, false);
+    });
+
+    ss::enable_auto_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) {
+        auto keyspace = validate_keyspace(ctx, req);
+        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
+
+        apilog.info("enable_auto_compaction: keyspace={} tables={}", keyspace, tables);
+        return set_tables_autocompaction(ctx, keyspace, tables, true);
+    });
+
+    ss::disable_auto_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) {
+        auto keyspace = validate_keyspace(ctx, req);
+        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
+
+        apilog.info("disable_auto_compaction: keyspace={} tables={}", keyspace, tables);
+        return set_tables_autocompaction(ctx, keyspace, tables, false);
     });
 
     cf::get_tombstone_gc.set(r, [&ctx] (const_req req) {
-        auto uuid = get_uuid(req.param["name"], ctx.db.local());
+        auto uuid = get_uuid(req.get_path_param("name"), ctx.db.local());
         replica::table& t = ctx.db.local().find_column_family(uuid);
         return t.tombstone_gc_enabled();
     });
 
     cf::enable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        apilog.info("column_family/enable_tombstone_gc: name={}", req->param["name"]);
-        return foreach_column_family(ctx, req->param["name"], [](replica::table& t) {
-            t.set_tombstone_gc_enabled(true);
-        }).then([] {
-            return make_ready_future<json::json_return_type>(json_void());
-        });
+        apilog.info("column_family/enable_tombstone_gc: name={}", req->get_path_param("name"));
+        auto [ks, cf] = parse_fully_qualified_cf_name(req->get_path_param("name"));
+        validate_table(ctx, ks, cf);
+        return set_tables_tombstone_gc(ctx, ks, {std::move(cf)}, true);
     });
 
     cf::disable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        apilog.info("column_family/disable_tombstone_gc: name={}", req->param["name"]);
-        return foreach_column_family(ctx, req->param["name"], [](replica::table& t) {
-            t.set_tombstone_gc_enabled(false);
-        }).then([] {
-            return make_ready_future<json::json_return_type>(json_void());
-        });
+        apilog.info("column_family/disable_tombstone_gc: name={}", req->get_path_param("name"));
+        auto [ks, cf] = parse_fully_qualified_cf_name(req->get_path_param("name"));
+        validate_table(ctx, ks, cf);
+        return set_tables_tombstone_gc(ctx, ks, {std::move(cf)}, false);
+    });
+
+    ss::enable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
+        auto keyspace = validate_keyspace(ctx, req);
+        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
+
+        apilog.info("enable_tombstone_gc: keyspace={} tables={}", keyspace, tables);
+        return set_tables_tombstone_gc(ctx, keyspace, tables, true);
+    });
+
+    ss::disable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
+        auto keyspace = validate_keyspace(ctx, req);
+        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
+
+        apilog.info("disable_tombstone_gc: keyspace={} tables={}", keyspace, tables);
+        return set_tables_tombstone_gc(ctx, keyspace, tables, false);
     });
 
     cf::get_built_indexes.set(r, [&ctx, &sys_ks](std::unique_ptr<http::request> req) {
-        auto ks_cf = parse_fully_qualified_cf_name(req->param["name"]);
+        auto ks_cf = parse_fully_qualified_cf_name(req->get_path_param("name"));
         auto&& ks = std::get<0>(ks_cf);
         auto&& cf_name = std::get<1>(ks_cf);
         return sys_ks.local().load_view_build_progress().then([ks, cf_name, &ctx](const std::vector<db::system_keyspace_view_build_progress>& vb) mutable {
@@ -962,7 +1032,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_compression_ratio.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        auto uuid = get_uuid(req->param["name"], ctx.db.local());
+        auto uuid = get_uuid(req->get_path_param("name"), ctx.db.local());
 
         return ctx.db.map_reduce(sum_ratio<double>(), [uuid](replica::database& db) {
             replica::column_family& cf = db.find_column_family(uuid);
@@ -973,21 +1043,21 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_read_latency_estimated_histogram.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        return map_reduce_cf_time_histogram(ctx, req->param["name"], [](const replica::column_family& cf) {
+        return map_reduce_cf_time_histogram(ctx, req->get_path_param("name"), [](const replica::column_family& cf) {
             return cf.get_stats().reads.histogram();
         });
     });
 
     cf::get_write_latency_estimated_histogram.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        return map_reduce_cf_time_histogram(ctx, req->param["name"], [](const replica::column_family& cf) {
+        return map_reduce_cf_time_histogram(ctx, req->get_path_param("name"), [](const replica::column_family& cf) {
             return cf.get_stats().writes.histogram();
         });
     });
 
     cf::set_compaction_strategy_class.set(r, [&ctx](std::unique_ptr<http::request> req) {
         sstring strategy = req->get_query_param("class_name");
-        apilog.info("column_family/set_compaction_strategy_class: name={} strategy={}", req->param["name"], strategy);
-        return foreach_column_family(ctx, req->param["name"], [strategy](replica::column_family& cf) {
+        apilog.info("column_family/set_compaction_strategy_class: name={} strategy={}", req->get_path_param("name"), strategy);
+        return foreach_column_family(ctx, req->get_path_param("name"), [strategy](replica::column_family& cf) {
             cf.set_compaction_strategy(sstables::compaction_strategy::type(strategy));
         }).then([] {
                 return make_ready_future<json::json_return_type>(json_void());
@@ -995,7 +1065,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_compaction_strategy_class.set(r, [&ctx](const_req req) {
-        return ctx.db.local().find_column_family(get_uuid(req.param["name"], ctx.db.local())).get_compaction_strategy().name();
+        return ctx.db.local().find_column_family(get_uuid(req.get_path_param("name"), ctx.db.local())).get_compaction_strategy().name();
     });
 
     cf::set_compression_parameters.set(r, [](std::unique_ptr<http::request> req) {
@@ -1011,7 +1081,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_sstable_count_per_level.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        return map_reduce_cf_raw(ctx, req->param["name"], std::vector<uint64_t>(), [](const replica::column_family& cf) {
+        return map_reduce_cf_raw(ctx, req->get_path_param("name"), std::vector<uint64_t>(), [](const replica::column_family& cf) {
             return cf.sstable_count_per_level();
         }, concat_sstable_count_per_level).then([](const std::vector<uint64_t>& res) {
             return make_ready_future<json::json_return_type>(res);
@@ -1020,7 +1090,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
 
     cf::get_sstables_for_key.set(r, [&ctx](std::unique_ptr<http::request> req) {
         auto key = req->get_query_param("key");
-        auto uuid = get_uuid(req->param["name"], ctx.db.local());
+        auto uuid = get_uuid(req->get_path_param("name"), ctx.db.local());
 
         return ctx.db.map_reduce0([key, uuid] (replica::database& db) -> future<std::unordered_set<sstring>> {
             auto sstables = co_await db.find_column_family(uuid).get_sstables_by_partition_key(key);
@@ -1036,7 +1106,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
 
 
     cf::toppartitions.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        auto name = req->param["name"];
+        auto name = req->get_path_param("name");
         auto [ks, cf] = parse_fully_qualified_cf_name(name);
 
         api::req_param<std::chrono::milliseconds, unsigned> duration{*req, "duration", 1000ms};
@@ -1063,7 +1133,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
         }
         auto [ks, cf] = parse_fully_qualified_cf_name(*params.get("name"));
         auto flush = params.get_as<bool>("flush_memtables").value_or(true);
-        apilog.info("column_family/force_major_compaction: name={} flush={}", req->param["name"], flush);
+        apilog.info("column_family/force_major_compaction: name={} flush={}", req->get_path_param("name"), flush);
 
         auto keyspace = validate_keyspace(ctx, ks);
         std::vector<table_info> table_infos = {table_info{
@@ -1156,8 +1226,6 @@ void unset_column_family(http_context& ctx, routes& r) {
     cf::get_speculative_retries.unset(r);
     cf::get_all_speculative_retries.unset(r);
     cf::get_key_cache_hit_rate.unset(r);
-    cf::get_true_snapshots_size.unset(r);
-    cf::get_all_true_snapshots_size.unset(r);
     cf::get_row_cache_hit_out_of_range.unset(r);
     cf::get_all_row_cache_hit_out_of_range.unset(r);
     cf::get_row_cache_hit.unset(r);
@@ -1174,6 +1242,13 @@ void unset_column_family(http_context& ctx, routes& r) {
     cf::get_auto_compaction.unset(r);
     cf::enable_auto_compaction.unset(r);
     cf::disable_auto_compaction.unset(r);
+    ss::enable_auto_compaction.unset(r);
+    ss::disable_auto_compaction.unset(r);
+    cf::get_tombstone_gc.unset(r);
+    cf::enable_tombstone_gc.unset(r);
+    cf::disable_tombstone_gc.unset(r);
+    ss::enable_tombstone_gc.unset(r);
+    ss::disable_tombstone_gc.unset(r);
     cf::get_built_indexes.unset(r);
     cf::get_compression_metadata_off_heap_memory_used.unset(r);
     cf::get_compression_parameters.unset(r);

api/column_family.hh

@@ -9,7 +9,6 @@
 #pragma once
 
 #include "replica/database.hh"
-#include <seastar/core/future-util.hh>
 #include <seastar/json/json_elements.hh>
 #include <any>
 #include "api/api_init.hh"

api/compaction_manager.cc

@@ -7,6 +7,7 @@
  */
 
 #include <seastar/core/coroutine.hh>
+#include <seastar/coroutine/exception.hh>
 
 #include "compaction_manager.hh"
 #include "compaction/compaction_manager.hh"
@@ -110,7 +111,7 @@ void set_compaction_manager(http_context& ctx, routes& r) {
     });
 
     cm::stop_keyspace_compaction.set(r, [&ctx] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
-        auto ks_name = validate_keyspace(ctx, req->param);
+        auto ks_name = validate_keyspace(ctx, req);
         auto table_names = parse_tables(ks_name, ctx, req->query_parameters, "tables");
         if (table_names.empty()) {
             table_names = map_keys(ctx.db.local().find_keyspace(ks_name).metadata().get()->cf_meta_data());
@@ -153,10 +154,13 @@ void set_compaction_manager(http_context& ctx, routes& r) {
     });
 
     cm::get_compaction_history.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        std::function<future<>(output_stream<char>&&)> f = [&ctx](output_stream<char>&& s) {
-            return do_with(output_stream<char>(std::move(s)), true, [&ctx] (output_stream<char>& s, bool& first){
-                return s.write("[").then([&ctx, &s, &first] {
-                    return ctx.db.local().get_compaction_manager().get_compaction_history([&s, &first](const db::compaction_history_entry& entry) mutable {
+        std::function<future<>(output_stream<char>&&)> f = [&ctx] (output_stream<char>&& out) -> future<> {
+            auto s = std::move(out);
+            bool first = true;
+            std::exception_ptr ex;
+            try {
+                co_await s.write("[");
+                co_await ctx.db.local().get_compaction_manager().get_compaction_history([&s, &first](const db::compaction_history_entry& entry) mutable -> future<> {
                         cm::history h;
                         h.id = fmt::to_string(entry.id);
                         h.ks = std::move(entry.ks);
@@ -170,18 +174,21 @@ void set_compaction_manager(http_context& ctx, routes& r) {
                             e.value = it.second;
                             h.rows_merged.push(std::move(e));
                         }
-                        auto fut = first ? make_ready_future<>() : s.write(", ");
+                        if (!first) {
+                            co_await s.write(", ");
+                        }
                         first = false;
-                        return fut.then([&s, h = std::move(h)] {
-                            return formatter::write(s, h);
-                        });
-                    }).then([&s] {
-                        return s.write("]").then([&s] {
-                            return s.close();
-                        });
+                        co_await formatter::write(s, h);
                     });
-                });
-            });
+                co_await s.write("]");
+                co_await s.flush();
+            } catch (...) {
+                ex = std::current_exception();
+            }
+            co_await s.close();
+            if (ex) {
+                co_await coroutine::return_exception_ptr(std::move(ex));
+            }
         };
         return make_ready_future<json::json_return_type>(std::move(f));
     });

api/config.cc

@@ -6,8 +6,11 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
+#include "api/api.hh"
 #include "api/config.hh"
 #include "api/api-doc/config.json.hh"
+#include "api/api-doc/storage_proxy.json.hh"
+#include "replica/database.hh"
 #include "db/config.hh"
 #include <sstream>
 #include <boost/algorithm/string/replace.hpp>
@@ -15,6 +18,7 @@
 
 namespace api {
 using namespace seastar::httpd;
+namespace sp = httpd::storage_proxy_json;
 
 template<class T>
 json::json_return_type get_json_return_type(const T& val) {
@@ -92,7 +96,7 @@ void set_config(std::shared_ptr < api_registry_builder20 > rb, http_context& ctx
     });
 
     cs::find_config_id.set(r, [&cfg] (const_req r) {
-        auto id = r.param["id"];
+        auto id = r.get_path_param("id");
         for (auto&& cfg_ref : cfg.values()) {
             auto&& cfg = cfg_ref.get();
             if (id == cfg.name()) {
@@ -101,6 +105,102 @@ void set_config(std::shared_ptr < api_registry_builder20 > rb, http_context& ctx
         }
         throw bad_param_exception(sstring("No such config entry: ") + id);
     });
+
+    sp::get_rpc_timeout.set(r, [&cfg](const_req req)  {
+        return cfg.request_timeout_in_ms()/1000.0;
+    });
+
+    sp::set_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
+        //TBD
+        unimplemented();
+        auto enable = req->get_query_param("timeout");
+        return make_ready_future<json::json_return_type>(seastar::json::json_void());
+    });
+
+    sp::get_read_rpc_timeout.set(r, [&cfg](const_req req)  {
+        return cfg.read_request_timeout_in_ms()/1000.0;
+    });
+
+    sp::set_read_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
+        //TBD
+        unimplemented();
+        auto enable = req->get_query_param("timeout");
+        return make_ready_future<json::json_return_type>(seastar::json::json_void());
+    });
+
+    sp::get_write_rpc_timeout.set(r, [&cfg](const_req req)  {
+        return cfg.write_request_timeout_in_ms()/1000.0;
+    });
+
+    sp::set_write_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
+        //TBD
+        unimplemented();
+        auto enable = req->get_query_param("timeout");
+        return make_ready_future<json::json_return_type>(seastar::json::json_void());
+    });
+
+    sp::get_counter_write_rpc_timeout.set(r, [&cfg](const_req req)  {
+        return cfg.counter_write_request_timeout_in_ms()/1000.0;
+    });
+
+    sp::set_counter_write_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
+        //TBD
+        unimplemented();
+        auto enable = req->get_query_param("timeout");
+        return make_ready_future<json::json_return_type>(seastar::json::json_void());
+    });
+
+    sp::get_cas_contention_timeout.set(r, [&cfg](const_req req)  {
+        return cfg.cas_contention_timeout_in_ms()/1000.0;
+    });
+
+    sp::set_cas_contention_timeout.set(r, [](std::unique_ptr<http::request> req)  {
+        //TBD
+        unimplemented();
+        auto enable = req->get_query_param("timeout");
+        return make_ready_future<json::json_return_type>(seastar::json::json_void());
+    });
+
+    sp::get_range_rpc_timeout.set(r, [&cfg](const_req req)  {
+        return cfg.range_request_timeout_in_ms()/1000.0;
+    });
+
+    sp::set_range_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
+        //TBD
+        unimplemented();
+        auto enable = req->get_query_param("timeout");
+        return make_ready_future<json::json_return_type>(seastar::json::json_void());
+    });
+
+    sp::get_truncate_rpc_timeout.set(r, [&cfg](const_req req)  {
+        return cfg.truncate_request_timeout_in_ms()/1000.0;
+    });
+
+    sp::set_truncate_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
+        //TBD
+        unimplemented();
+        auto enable = req->get_query_param("timeout");
+        return make_ready_future<json::json_return_type>(seastar::json::json_void());
+    });
+
+}
+
+void unset_config(http_context& ctx, routes& r) {
+    cs::find_config_id.unset(r);
+    sp::get_rpc_timeout.unset(r);
+    sp::set_rpc_timeout.unset(r);
+    sp::get_read_rpc_timeout.unset(r);
+    sp::set_read_rpc_timeout.unset(r);
+    sp::get_write_rpc_timeout.unset(r);
+    sp::set_write_rpc_timeout.unset(r);
+    sp::get_counter_write_rpc_timeout.unset(r);
+    sp::set_counter_write_rpc_timeout.unset(r);
+    sp::get_cas_contention_timeout.unset(r);
+    sp::set_cas_contention_timeout.unset(r);
+    sp::get_range_rpc_timeout.unset(r);
+    sp::set_range_rpc_timeout.unset(r);
+    sp::get_truncate_rpc_timeout.unset(r);
+    sp::set_truncate_rpc_timeout.unset(r);
 }
 
 }

api/config.hh

@@ -14,4 +14,5 @@
 namespace api {
 
 void set_config(std::shared_ptr<httpd::api_registry_builder20> rb, http_context& ctx, httpd::routes& r, const db::config& cfg, bool first = false);
+void unset_config(http_context& ctx, httpd::routes& r);
 }

api/error_injection.cc

@@ -7,10 +7,8 @@
  */
 
 #include "api/api-doc/error_injection.json.hh"
-#include "api/api.hh"
-
+#include "api/api_init.hh"
 #include <seastar/http/exception.hh>
-#include "log.hh"
 #include "utils/error_injection.hh"
 #include "utils/rjson.hh"
 #include <seastar/core/future-util.hh>
@@ -24,7 +22,7 @@ namespace hf = httpd::error_injection_json;
 void set_error_injection(http_context& ctx, routes& r) {
 
     hf::enable_injection.set(r, [](std::unique_ptr<request> req) {
-        sstring injection = req->param["injection"];
+        sstring injection = req->get_path_param("injection");
         bool one_shot = req->get_query_param("one_shot") == "True";
         auto params = req->content;
 
@@ -56,7 +54,7 @@ void set_error_injection(http_context& ctx, routes& r) {
     });
 
     hf::disable_injection.set(r, [](std::unique_ptr<request> req) {
-        sstring injection = req->param["injection"];
+        sstring injection = req->get_path_param("injection");
 
         auto& errinj = utils::get_local_injector();
         return errinj.disable_on_all(injection).then([] {
@@ -64,6 +62,32 @@ void set_error_injection(http_context& ctx, routes& r) {
         });
     });
 
+    hf::read_injection.set(r, [](std::unique_ptr<request> req) -> future<json::json_return_type> {
+        const sstring injection = req->get_path_param("injection");
+
+        std::vector<error_injection_json::error_injection_info> error_injection_infos(smp::count, error_injection_json::error_injection_info{});
+
+        co_await smp::invoke_on_all([&] {
+            auto& info = error_injection_infos[this_shard_id()];
+            auto& errinj = utils::get_local_injector();
+            const auto enabled = errinj.is_enabled(injection);
+            info.enabled = enabled;
+            if (!enabled) {
+                return;
+            }
+            std::vector<error_injection_json::mapper> parameters;
+            for (const auto& p : errinj.get_injection_parameters(injection)) {
+                error_injection_json::mapper param;
+                param.key = p.first;
+                param.value = p.second;
+                parameters.push_back(std::move(param));
+            }
+            info.parameters = std::move(parameters);
+        });
+
+        co_return json::json_return_type(error_injection_infos);
+    });
+
     hf::disable_on_all.set(r, [](std::unique_ptr<request> req) {
         auto& errinj = utils::get_local_injector();
         return errinj.disable_on_all().then([] {
@@ -72,7 +96,7 @@ void set_error_injection(http_context& ctx, routes& r) {
     });
 
     hf::message_injection.set(r, [](std::unique_ptr<request> req) {
-        sstring injection = req->param["injection"];
+        sstring injection = req->get_path_param("injection");
         auto& errinj = utils::get_local_injector();
         return errinj.receive_message_on_all(injection).then([] {
             return make_ready_future<json::json_return_type>(json::json_void());

api/failure_detector.cc

@@ -66,7 +66,7 @@ void set_failure_detector(http_context& ctx, routes& r, gms::gossiper& g) {
         return g.container().invoke_on(0, [] (gms::gossiper& g) {
             std::map<sstring, sstring> nodes_status;
             g.for_each_endpoint_state([&] (const gms::inet_address& node, const gms::endpoint_state&) {
-                nodes_status.emplace(node.to_sstring(), g.is_alive(node) ? "UP" : "DOWN");
+                nodes_status.emplace(fmt::to_string(node), g.is_alive(node) ? "UP" : "DOWN");
             });
             return make_ready_future<json::json_return_type>(map_to_key_value<fd::mapper>(nodes_status));
         });
@@ -81,9 +81,9 @@ void set_failure_detector(http_context& ctx, routes& r, gms::gossiper& g) {
 
     fd::get_endpoint_state.set(r, [&g] (std::unique_ptr<request> req) {
         return g.container().invoke_on(0, [req = std::move(req)] (gms::gossiper& g) {
-            auto state = g.get_endpoint_state_ptr(gms::inet_address(req->param["addr"]));
+            auto state = g.get_endpoint_state_ptr(gms::inet_address(req->get_path_param("addr")));
             if (!state) {
-                return make_ready_future<json::json_return_type>(format("unknown endpoint {}", req->param["addr"]));
+                return make_ready_future<json::json_return_type>(format("unknown endpoint {}", req->get_path_param("addr")));
             }
             std::stringstream ss;
             g.append_endpoint_state(ss, *state);
@@ -99,5 +99,16 @@ void set_failure_detector(http_context& ctx, routes& r, gms::gossiper& g) {
     });
 }
 
+void unset_failure_detector(http_context& ctx, routes& r) {
+    fd::get_all_endpoint_states.unset(r);
+    fd::get_up_endpoint_count.unset(r);
+    fd::get_down_endpoint_count.unset(r);
+    fd::get_phi_convict_threshold.unset(r);
+    fd::get_simple_states.unset(r);
+    fd::set_phi_convict_threshold.unset(r);
+    fd::get_endpoint_state.unset(r);
+    fd::get_endpoint_phi_values.unset(r);
+}
+
 }
 

api/failure_detector.hh

@@ -19,5 +19,6 @@ class gossiper;
 namespace api {
 
 void set_failure_detector(http_context& ctx, httpd::routes& r, gms::gossiper& g);
+void unset_failure_detector(http_context& ctx, httpd::routes& r);
 
 }

api/gossiper.cc

@@ -32,21 +32,21 @@ void set_gossiper(http_context& ctx, routes& r, gms::gossiper& g) {
     });
 
     httpd::gossiper_json::get_endpoint_downtime.set(r, [&g] (std::unique_ptr<request> req) -> future<json::json_return_type> {
-        gms::inet_address ep(req->param["addr"]);
+        gms::inet_address ep(req->get_path_param("addr"));
         // synchronize unreachable_members on all shards
         co_await g.get_unreachable_members_synchronized();
         co_return g.get_endpoint_downtime(ep);
     });
 
     httpd::gossiper_json::get_current_generation_number.set(r, [&g] (std::unique_ptr<http::request> req) {
-        gms::inet_address ep(req->param["addr"]);
+        gms::inet_address ep(req->get_path_param("addr"));
         return g.get_current_generation_number(ep).then([] (gms::generation_type res) {
             return make_ready_future<json::json_return_type>(res.value());
         });
     });
 
     httpd::gossiper_json::get_current_heart_beat_version.set(r, [&g] (std::unique_ptr<http::request> req) {
-        gms::inet_address ep(req->param["addr"]);
+        gms::inet_address ep(req->get_path_param("addr"));
         return g.get_current_heart_beat_version(ep).then([] (gms::version_type res) {
             return make_ready_future<json::json_return_type>(res.value());
         });
@@ -54,21 +54,31 @@ void set_gossiper(http_context& ctx, routes& r, gms::gossiper& g) {
 
     httpd::gossiper_json::assassinate_endpoint.set(r, [&g](std::unique_ptr<http::request> req) {
         if (req->get_query_param("unsafe") != "True") {
-            return g.assassinate_endpoint(req->param["addr"]).then([] {
+            return g.assassinate_endpoint(req->get_path_param("addr")).then([] {
                 return make_ready_future<json::json_return_type>(json_void());
             });
         }
-        return g.unsafe_assassinate_endpoint(req->param["addr"]).then([] {
+        return g.unsafe_assassinate_endpoint(req->get_path_param("addr")).then([] {
             return make_ready_future<json::json_return_type>(json_void());
         });
     });
 
     httpd::gossiper_json::force_remove_endpoint.set(r, [&g](std::unique_ptr<http::request> req) {
-        gms::inet_address ep(req->param["addr"]);
+        gms::inet_address ep(req->get_path_param("addr"));
         return g.force_remove_endpoint(ep, gms::null_permit_id).then([] {
             return make_ready_future<json::json_return_type>(json_void());
         });
     });
 }
 
+void unset_gossiper(http_context& ctx, routes& r) {
+    httpd::gossiper_json::get_down_endpoint.unset(r);
+    httpd::gossiper_json::get_live_endpoint.unset(r);
+    httpd::gossiper_json::get_endpoint_downtime.unset(r);
+    httpd::gossiper_json::get_current_generation_number.unset(r);
+    httpd::gossiper_json::get_current_heart_beat_version.unset(r);
+    httpd::gossiper_json::assassinate_endpoint.unset(r);
+    httpd::gossiper_json::force_remove_endpoint.unset(r);
+}
+
 }

api/gossiper.hh

@@ -19,5 +19,6 @@ class gossiper;
 namespace api {
 
 void set_gossiper(http_context& ctx, httpd::routes& r, gms::gossiper& g);
+void unset_gossiper(http_context& ctx, httpd::routes& r);
 
 }

api/messaging_service.cc

@@ -146,7 +146,7 @@ void set_messaging_service(http_context& ctx, routes& r, sharded<netw::messaging
     });
 
     hf::inject_disconnect.set(r, [&ms] (std::unique_ptr<request> req) -> future<json::json_return_type> {
-        auto ip = msg_addr(req->param["ip"]);
+        auto ip = msg_addr(req->get_path_param("ip"));
         co_await ms.invoke_on_all([ip] (netw::messaging_service& ms) {
             ms.remove_rpc_client(ip);
         });

api/raft.cc

@@ -8,10 +8,10 @@
 
 #include <seastar/core/coroutine.hh>
 
-#include "api/api.hh"
 #include "api/api-doc/raft.json.hh"
 
 #include "service/raft/raft_group_registry.hh"
+#include "log.hh"
 
 using namespace seastar::httpd;
 
@@ -19,34 +19,44 @@ extern logging::logger apilog;
 
 namespace api {
 
+struct http_context;
 namespace r = httpd::raft_json;
 using namespace json;
 
+
+namespace {
+
+::service::raft_timeout get_request_timeout(const http::request& req) {
+    return std::invoke([timeout_str = req.get_query_param("timeout")] {
+        if (timeout_str.empty()) {
+            return ::service::raft_timeout{};
+        }
+        auto dur = std::stoll(timeout_str);
+        if (dur <= 0) {
+            throw bad_param_exception{"Timeout must be a positive number."};
+        }
+        return ::service::raft_timeout{.value = lowres_clock::now() + std::chrono::seconds{dur}};
+    });
+}
+
+}  // namespace
+
+
 void set_raft(http_context&, httpd::routes& r, sharded<service::raft_group_registry>& raft_gr) {
     r::trigger_snapshot.set(r, [&raft_gr] (std::unique_ptr<http::request> req) -> future<json_return_type> {
-        raft::group_id gid{utils::UUID{req->param["group_id"]}};
-        auto timeout_dur = std::invoke([timeout_str = req->get_query_param("timeout")] {
-            if (timeout_str.empty()) {
-                return std::chrono::seconds{60};
-            }
-            auto dur = std::stoll(timeout_str);
-            if (dur <= 0) {
-                throw std::runtime_error{"Timeout must be a positive number."};
-            }
-            return std::chrono::seconds{dur};
-        });
+        raft::group_id gid{utils::UUID{req->get_path_param("group_id")}};
+        auto timeout = get_request_timeout(*req);
 
         std::atomic<bool> found_srv{false};
-        co_await raft_gr.invoke_on_all([gid, timeout_dur, &found_srv] (service::raft_group_registry& raft_gr) -> future<> {
-            auto* srv = raft_gr.find_server(gid);
-            if (!srv) {
+        co_await raft_gr.invoke_on_all([gid, timeout, &found_srv] (service::raft_group_registry& raft_gr) -> future<> {
+            if (!raft_gr.find_server(gid)) {
                 co_return;
             }
 
             found_srv = true;
-            abort_on_expiry aoe(lowres_clock::now() + timeout_dur);
             apilog.info("Triggering Raft group {} snapshot", gid);
-            auto result = co_await srv->trigger_snapshot(&aoe.abort_source());
+            auto srv = raft_gr.get_server_with_timeouts(gid);
+            auto result = co_await srv.trigger_snapshot(nullptr, timeout);
             if (result) {
                 apilog.info("New snapshot for Raft group {} created", gid);
             } else {
@@ -55,30 +65,72 @@ void set_raft(http_context&, httpd::routes& r, sharded<service::raft_group_regis
         });
 
         if (!found_srv) {
-            throw std::runtime_error{fmt::format("Server for group ID {} not found", gid)};
+            throw bad_param_exception{fmt::format("Server for group ID {} not found", gid)};
         }
 
         co_return json_void{};
     });
     r::get_leader_host.set(r, [&raft_gr] (std::unique_ptr<http::request> req) -> future<json_return_type> {
-        return smp::submit_to(0, [&] {
-            auto& srv = std::invoke([&] () -> raft::server& {
-                if (req->query_parameters.contains("group_id")) {
-                    raft::group_id id{utils::UUID{req->get_query_param("group_id")}};
-                    return raft_gr.local().get_server(id);
-                } else {
-                    return raft_gr.local().group0();
-                }
+        if (!req->query_parameters.contains("group_id")) {
+            const auto leader_id = co_await raft_gr.invoke_on(0, [] (service::raft_group_registry& raft_gr) {
+                auto& srv = raft_gr.group0();
+                return srv.current_leader();
             });
-            return json_return_type(srv.current_leader().to_sstring());
+            co_return json_return_type{leader_id.to_sstring()};
+        }
+
+        const raft::group_id gid{utils::UUID{req->get_query_param("group_id")}};
+
+        std::atomic<bool> found_srv{false};
+        std::atomic<raft::server_id> leader_id = raft::server_id::create_null_id();
+        co_await raft_gr.invoke_on_all([gid, &found_srv, &leader_id] (service::raft_group_registry& raft_gr) {
+            if (raft_gr.find_server(gid)) {
+                found_srv = true;
+                leader_id = raft_gr.get_server(gid).current_leader();
+            }
+            return make_ready_future<>();
         });
+
+        if (!found_srv) {
+            throw bad_param_exception{fmt::format("Server for group ID {} not found", gid)};
+        }
+
+        co_return json_return_type(leader_id.load().to_sstring());
+    });
+    r::read_barrier.set(r, [&raft_gr] (std::unique_ptr<http::request> req) -> future<json_return_type> {
+        auto timeout = get_request_timeout(*req);
+
+        if (!req->query_parameters.contains("group_id")) {
+            // Read barrier on group 0 by default
+            co_await raft_gr.invoke_on(0, [timeout] (service::raft_group_registry& raft_gr) {
+                return raft_gr.group0_with_timeouts().read_barrier(nullptr, timeout);
+            });
+            co_return json_void{};
+        }
+
+        raft::group_id gid{utils::UUID{req->get_query_param("group_id")}};
+
+        std::atomic<bool> found_srv{false};
+        co_await raft_gr.invoke_on_all([gid, timeout, &found_srv] (service::raft_group_registry& raft_gr) {
+            if (!raft_gr.find_server(gid)) {
+                return make_ready_future<>();
+            }
+            found_srv = true;
+            return raft_gr.get_server_with_timeouts(gid).read_barrier(nullptr, timeout);
+        });
+
+        if (!found_srv) {
+            throw bad_param_exception{fmt::format("Server for group ID {} not found", gid)};
+        }
+
+        co_return json_void{};
     });
 }
 
 void unset_raft(http_context&, httpd::routes& r) {
     r::trigger_snapshot.unset(r);
     r::get_leader_host.unset(r);
+    r::read_barrier.unset(r);
 }
 
 }
-

api/scrub_status.hh

@@ -6,6 +6,8 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
+#pragma once
+
 namespace api {
 
 enum class scrub_status {

api/storage_proxy.cc

@@ -13,7 +13,6 @@
 #include "api/api-doc/utils.json.hh"
 #include "db/config.hh"
 #include "utils/histogram.hh"
-#include "replica/database.hh"
 #include <seastar/core/scheduling_specific.hh>
 
 namespace api {
@@ -259,83 +258,6 @@ void set_storage_proxy(http_context& ctx, routes& r, sharded<service::storage_pr
         return make_ready_future<json::json_return_type>(0);
     });
 
-    sp::get_rpc_timeout.set(r, [&ctx](const_req req)  {
-        return ctx.db.local().get_config().request_timeout_in_ms()/1000.0;
-    });
-
-    sp::set_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
-        //TBD
-        unimplemented();
-        auto enable = req->get_query_param("timeout");
-        return make_ready_future<json::json_return_type>(json_void());
-    });
-
-    sp::get_read_rpc_timeout.set(r, [&ctx](const_req req)  {
-        return ctx.db.local().get_config().read_request_timeout_in_ms()/1000.0;
-    });
-
-    sp::set_read_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
-        //TBD
-        unimplemented();
-        auto enable = req->get_query_param("timeout");
-        return make_ready_future<json::json_return_type>(json_void());
-    });
-
-    sp::get_write_rpc_timeout.set(r, [&ctx](const_req req)  {
-        return ctx.db.local().get_config().write_request_timeout_in_ms()/1000.0;
-    });
-
-    sp::set_write_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
-        //TBD
-        unimplemented();
-        auto enable = req->get_query_param("timeout");
-        return make_ready_future<json::json_return_type>(json_void());
-    });
-
-    sp::get_counter_write_rpc_timeout.set(r, [&ctx](const_req req)  {
-        return ctx.db.local().get_config().counter_write_request_timeout_in_ms()/1000.0;
-    });
-
-    sp::set_counter_write_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
-        //TBD
-        unimplemented();
-        auto enable = req->get_query_param("timeout");
-        return make_ready_future<json::json_return_type>(json_void());
-    });
-
-    sp::get_cas_contention_timeout.set(r, [&ctx](const_req req)  {
-        return ctx.db.local().get_config().cas_contention_timeout_in_ms()/1000.0;
-    });
-
-    sp::set_cas_contention_timeout.set(r, [](std::unique_ptr<http::request> req)  {
-        //TBD
-        unimplemented();
-        auto enable = req->get_query_param("timeout");
-        return make_ready_future<json::json_return_type>(json_void());
-    });
-
-    sp::get_range_rpc_timeout.set(r, [&ctx](const_req req)  {
-        return ctx.db.local().get_config().range_request_timeout_in_ms()/1000.0;
-    });
-
-    sp::set_range_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
-        //TBD
-        unimplemented();
-        auto enable = req->get_query_param("timeout");
-        return make_ready_future<json::json_return_type>(json_void());
-    });
-
-    sp::get_truncate_rpc_timeout.set(r, [&ctx](const_req req)  {
-        return ctx.db.local().get_config().truncate_request_timeout_in_ms()/1000.0;
-    });
-
-    sp::set_truncate_rpc_timeout.set(r, [](std::unique_ptr<http::request> req)  {
-        //TBD
-        unimplemented();
-        auto enable = req->get_query_param("timeout");
-        return make_ready_future<json::json_return_type>(json_void());
-    });
-
     sp::reload_trigger_classes.set(r, [](std::unique_ptr<http::request> req)  {
         //TBD
         unimplemented();
@@ -516,20 +438,6 @@ void unset_storage_proxy(http_context& ctx, routes& r) {
     sp::get_max_hints_in_progress.unset(r);
     sp::set_max_hints_in_progress.unset(r);
     sp::get_hints_in_progress.unset(r);
-    sp::get_rpc_timeout.unset(r);
-    sp::set_rpc_timeout.unset(r);
-    sp::get_read_rpc_timeout.unset(r);
-    sp::set_read_rpc_timeout.unset(r);
-    sp::get_write_rpc_timeout.unset(r);
-    sp::set_write_rpc_timeout.unset(r);
-    sp::get_counter_write_rpc_timeout.unset(r);
-    sp::set_counter_write_rpc_timeout.unset(r);
-    sp::get_cas_contention_timeout.unset(r);
-    sp::set_cas_contention_timeout.unset(r);
-    sp::get_range_rpc_timeout.unset(r);
-    sp::set_range_rpc_timeout.unset(r);
-    sp::get_truncate_rpc_timeout.unset(r);
-    sp::set_truncate_rpc_timeout.unset(r);
     sp::reload_trigger_classes.unset(r);
     sp::get_read_repair_attempted.unset(r);
     sp::get_read_repair_repaired_blocking.unset(r);

api/storage_service.cc

@@ -26,6 +26,7 @@
 #include <boost/algorithm/string/trim_all.hpp>
 #include <boost/algorithm/string/case_conv.hpp>
 #include <boost/functional/hash.hpp>
+#include <fmt/ranges.h>
 #include "service/raft/raft_group0_client.hh"
 #include "service/storage_service.hh"
 #include "service/load_meter.hh"
@@ -35,6 +36,7 @@
 #include <seastar/http/exception.hh>
 #include <seastar/core/coroutine.hh>
 #include <seastar/coroutine/parallel_for_each.hh>
+#include <seastar/coroutine/exception.hh>
 #include "repair/row_level.hh"
 #include "locator/snitch_base.hh"
 #include "column_family.hh"
@@ -47,12 +49,12 @@
 #include "db/extensions.hh"
 #include "db/snapshot-ctl.hh"
 #include "transport/controller.hh"
-#include "thrift/controller.hh"
 #include "locator/token_metadata.hh"
 #include "cdc/generation_service.hh"
 #include "locator/abstract_replication_strategy.hh"
 #include "sstables_loader.hh"
 #include "db/view/view_builder.hh"
+#include "utils/user_provided_param.hh"
 
 using namespace seastar::httpd;
 using namespace std::chrono_literals;
@@ -63,6 +65,7 @@ namespace api {
 
 namespace ss = httpd::storage_service_json;
 namespace sp = httpd::storage_proxy_json;
+namespace cf = httpd::column_family_json;
 using namespace json;
 
 sstring validate_keyspace(const http_context& ctx, sstring ks_name) {
@@ -72,11 +75,15 @@ sstring validate_keyspace(const http_context& ctx, sstring ks_name) {
     throw bad_param_exception(replica::no_such_keyspace(ks_name).what());
 }
 
-sstring validate_keyspace(const http_context& ctx, const parameters& param) {
-    return validate_keyspace(ctx, param["keyspace"]);
+sstring validate_keyspace(const http_context& ctx, const std::unique_ptr<http::request>& req) {
+    return validate_keyspace(ctx, req->get_path_param("keyspace"));
 }
 
-static void validate_table(const http_context& ctx, sstring ks_name, sstring table_name) {
+sstring validate_keyspace(const http_context& ctx, const http::request& req) {
+    return validate_keyspace(ctx, req.get_path_param("keyspace"));
+}
+
+void validate_table(const http_context& ctx, sstring ks_name, sstring table_name) {
     auto& db = ctx.db.local();
     try {
         db.find_column_family(ks_name, table_name);
@@ -199,14 +206,13 @@ using ks_cf_func = std::function<future<json::json_return_type>(http_context&, s
 
 static auto wrap_ks_cf(http_context &ctx, ks_cf_func f) {
     return [&ctx, f = std::move(f)](std::unique_ptr<http::request> req) {
-        auto keyspace = validate_keyspace(ctx, req->param);
+        auto keyspace = validate_keyspace(ctx, req);
         auto table_infos = parse_table_infos(keyspace, ctx, req->query_parameters, "cf");
         return f(ctx, std::move(req), std::move(keyspace), std::move(table_infos));
     };
 }
 
 seastar::future<json::json_return_type> run_toppartitions_query(db::toppartitions_query& q, http_context &ctx, bool legacy_request) {
-    namespace cf = httpd::column_family_json;
     return q.scatter().then([&q, legacy_request] {
         return sleep(q.duration()).then([&q, legacy_request] {
             return q.gather(q.capacity()).then([&q, legacy_request] (auto topk_results) {
@@ -236,47 +242,6 @@ seastar::future<json::json_return_type> run_toppartitions_query(db::toppartition
     });
 }
 
-static future<json::json_return_type> set_tables(http_context& ctx, const sstring& keyspace, std::vector<sstring> tables, std::function<future<>(replica::table&)> set) {
-    if (tables.empty()) {
-        tables = map_keys(ctx.db.local().find_keyspace(keyspace).metadata().get()->cf_meta_data());
-    }
-
-    return do_with(keyspace, std::move(tables), [&ctx, set] (const sstring& keyspace, const std::vector<sstring>& tables) {
-        return ctx.db.invoke_on_all([&keyspace, &tables, set] (replica::database& db) {
-            return parallel_for_each(tables, [&db, &keyspace, set] (const sstring& table) {
-                replica::table& t = db.find_column_family(keyspace, table);
-                return set(t);
-            });
-        });
-    }).then([] {
-        return make_ready_future<json::json_return_type>(json_void());
-    });
-}
-
-future<json::json_return_type> set_tables_autocompaction(http_context& ctx, const sstring &keyspace, std::vector<sstring> tables, bool enabled) {
-    apilog.info("set_tables_autocompaction: enabled={} keyspace={} tables={}", enabled, keyspace, tables);
-
-    return ctx.db.invoke_on(0, [&ctx, keyspace, tables = std::move(tables), enabled] (replica::database& db) {
-        auto g = replica::database::autocompaction_toggle_guard(db);
-        return set_tables(ctx, keyspace, tables, [enabled] (replica::table& cf) {
-            if (enabled) {
-                cf.enable_auto_compaction();
-            } else {
-                return cf.disable_auto_compaction();
-            }
-            return make_ready_future<>();
-        }).finally([g = std::move(g)] {});
-    });
-}
-
-future<json::json_return_type> set_tables_tombstone_gc(http_context& ctx, const sstring &keyspace, std::vector<sstring> tables, bool enabled) {
-    apilog.info("set_tables_tombstone_gc: enabled={} keyspace={} tables={}", enabled, keyspace, tables);
-    return set_tables(ctx, keyspace, std::move(tables), [enabled] (replica::table& t) {
-        t.set_tombstone_gc_enabled(enabled);
-        return make_ready_future<>();
-    });
-}
-
 future<scrub_info> parse_scrub_options(const http_context& ctx, sharded<db::snapshot_ctl>& snap_ctl, std::unique_ptr<http::request> req) {
     scrub_info info;
     auto rp = req_params({
@@ -342,21 +307,17 @@ future<scrub_info> parse_scrub_options(const http_context& ctx, sharded<db::snap
 }
 
 void set_transport_controller(http_context& ctx, routes& r, cql_transport::controller& ctl) {
-    ss::start_native_transport.set(r, [&ctx, &ctl](std::unique_ptr<http::request> req) {
+    ss::start_native_transport.set(r, [&ctl](std::unique_ptr<http::request> req) {
         return smp::submit_to(0, [&] {
-            return with_scheduling_group(ctx.db.local().get_statement_scheduling_group(), [&ctl] {
-                return ctl.start_server();
-            });
+            return ctl.start_server();
         }).then([] {
             return make_ready_future<json::json_return_type>(json_void());
         });
     });
 
-    ss::stop_native_transport.set(r, [&ctx, &ctl](std::unique_ptr<http::request> req) {
+    ss::stop_native_transport.set(r, [&ctl](std::unique_ptr<http::request> req) {
         return smp::submit_to(0, [&] {
-            return with_scheduling_group(ctx.db.local().get_statement_scheduling_group(), [&ctl] {
-                return ctl.request_stop_server();
-            });
+            return ctl.request_stop_server();
         }).then([] {
             return make_ready_future<json::json_return_type>(json_void());
         });
@@ -377,44 +338,21 @@ void unset_transport_controller(http_context& ctx, routes& r) {
     ss::is_native_transport_running.unset(r);
 }
 
-void set_rpc_controller(http_context& ctx, routes& r, thrift_controller& ctl) {
-    ss::stop_rpc_server.set(r, [&ctx, &ctl](std::unique_ptr<http::request> req) {
-        return smp::submit_to(0, [&] {
-            return with_scheduling_group(ctx.db.local().get_statement_scheduling_group(), [&ctl] {
-                return ctl.request_stop_server();
-            });
-        }).then([] {
-            return make_ready_future<json::json_return_type>(json_void());
-        });
-    });
-
-    ss::start_rpc_server.set(r, [&ctx, &ctl](std::unique_ptr<http::request> req) {
-        return smp::submit_to(0, [&] {
-            return with_scheduling_group(ctx.db.local().get_statement_scheduling_group(), [&ctl] {
-                return ctl.start_server();
-            });
-        }).then([] {
-            return make_ready_future<json::json_return_type>(json_void());
-        });
-    });
-
-    ss::is_rpc_server_running.set(r, [&ctl] (std::unique_ptr<http::request> req) {
-        return smp::submit_to(0, [&] {
-            return !ctl.listen_addresses().empty();
-        }).then([] (bool running) {
-            return make_ready_future<json::json_return_type>(running);
+// NOTE: preserved only for backward compatibility
+void set_thrift_controller(http_context& ctx, routes& r) {
+    ss::is_thrift_server_running.set(r, [] (std::unique_ptr<http::request> req) {
+        return smp::submit_to(0, [] {
+            return make_ready_future<json::json_return_type>(false);
         });
     });
 }
 
-void unset_rpc_controller(http_context& ctx, routes& r) {
-    ss::stop_rpc_server.unset(r);
-    ss::start_rpc_server.unset(r);
-    ss::is_rpc_server_running.unset(r);
+void unset_thrift_controller(http_context& ctx, routes& r) {
+    ss::is_thrift_server_running.unset(r);
 }
 
 void set_repair(http_context& ctx, routes& r, sharded<repair_service>& repair) {
-    ss::repair_async.set(r, [&ctx, &repair](std::unique_ptr<http::request> req) {
+    ss::repair_async.set(r, [&ctx, &repair](std::unique_ptr<http::request> req) -> future<json::json_return_type> {
         static std::unordered_set<sstring> options = {"primaryRange", "parallelism", "incremental",
                 "jobThreads", "ranges", "columnFamilies", "dataCenters", "hosts", "ignore_nodes", "trace",
                 "startToken", "endToken", "ranges_parallelism", "small_table_optimization"};
@@ -427,8 +365,7 @@ void set_repair(http_context& ctx, routes& r, sharded<repair_service>& repair) {
                 continue;
             }
             if (!options.contains(x.first)) {
-                return make_exception_future<json::json_return_type>(
-                        httpd::bad_param_exception(format("option {} is not supported", x.first)));
+                throw httpd::bad_param_exception(format("option {} is not supported", x.first));
             }
         }
         std::unordered_map<sstring, sstring> options_map;
@@ -443,10 +380,14 @@ void set_repair(http_context& ctx, routes& r, sharded<repair_service>& repair) {
         // returns immediately, not waiting for the repair to finish. The user
         // then has other mechanisms to track the ongoing repair's progress,
         // or stop it.
-        return repair_start(repair, validate_keyspace(ctx, req->param),
-                options_map).then([] (int i) {
-                    return make_ready_future<json::json_return_type>(i);
-                });
+        try {
+            int res = co_await repair_start(repair, validate_keyspace(ctx, req), options_map);
+            co_return json::json_return_type(res);
+        } catch (const std::invalid_argument& e) {
+            // if the option is not sane, repair_start() throws immediately, so
+            // convert the exception to an HTTP error
+            throw httpd::bad_param_exception(e.what());
+        }
     });
 
     ss::get_active_repair_async.set(r, [&repair] (std::unique_ptr<http::request> req) {
@@ -526,7 +467,7 @@ void unset_repair(http_context& ctx, routes& r) {
 
 void set_sstables_loader(http_context& ctx, routes& r, sharded<sstables_loader>& sst_loader) {
     ss::load_new_ss_tables.set(r, [&ctx, &sst_loader](std::unique_ptr<http::request> req) {
-        auto ks = validate_keyspace(ctx, req->param);
+        auto ks = validate_keyspace(ctx, req);
         auto cf = req->get_query_param("cf");
         auto stream = req->get_query_param("load_and_stream");
         auto primary_replica = req->get_query_param("primary_replica_only");
@@ -557,8 +498,8 @@ void unset_sstables_loader(http_context& ctx, routes& r) {
 
 void set_view_builder(http_context& ctx, routes& r, sharded<db::view::view_builder>& vb) {
     ss::view_build_statuses.set(r, [&ctx, &vb] (std::unique_ptr<http::request> req) {
-        auto keyspace = validate_keyspace(ctx, req->param);
-        auto view = req->param["view"];
+        auto keyspace = validate_keyspace(ctx, req);
+        auto view = req->get_path_param("view");
         return vb.local().view_build_statuses(std::move(keyspace), std::move(view)).then([] (std::unordered_map<sstring, sstring> status) {
             std::vector<storage_service_json::mapper> res;
             return make_ready_future<json::json_return_type>(map_to_key_value(std::move(status), res));
@@ -584,8 +525,24 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
         return ctx.db.local().commitlog()->active_config().commit_log_location;
     });
 
-    ss::get_token_endpoint.set(r, [&ss] (std::unique_ptr<http::request> req) {
-        return make_ready_future<json::json_return_type>(stream_range_as_array(ss.local().get_token_to_endpoint_map(), [](const auto& i) {
+    ss::get_token_endpoint.set(r, [&ctx, &ss] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
+        const auto keyspace_name = req->get_query_param("keyspace");
+        const auto table_name = req->get_query_param("cf");
+
+        std::map<dht::token, gms::inet_address> token_endpoints;
+        if (keyspace_name.empty() && table_name.empty()) {
+            token_endpoints = ss.local().get_token_to_endpoint_map();
+        } else if (!keyspace_name.empty() && !table_name.empty()) {
+            auto& db = ctx.db.local();
+            if (!db.has_schema(keyspace_name, table_name)) {
+                throw bad_param_exception(fmt::format("Failed to find table {}.{}", keyspace_name, table_name));
+            }
+            token_endpoints = co_await ss.local().get_tablet_to_endpoint_map(db.find_schema(keyspace_name, table_name)->id());
+        } else {
+            throw bad_param_exception("Either provide both keyspace and table (for tablet table) or neither (for vnodes)");
+        }
+
+        co_return json::json_return_type(stream_range_as_array(token_endpoints, [](const auto& i) {
             storage_service_json::mapper val;
             val.key = fmt::to_string(i.first);
             val.value = fmt::to_string(i.second);
@@ -663,7 +620,7 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
     });
 
     ss::get_range_to_endpoint_map.set(r, [&ctx, &ss](std::unique_ptr<http::request> req) -> future<json::json_return_type> {
-        auto keyspace = validate_keyspace(ctx, req->param);
+        auto keyspace = validate_keyspace(ctx, req);
         auto table = req->get_query_param("cf");
 
         auto erm = std::invoke([&]() -> locator::effective_replication_map_ptr {
@@ -694,7 +651,7 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
                 m.key.push("");
             }
             for (const gms::inet_address& address : entry.second) {
-                m.value.push(address.to_sstring());
+                m.value.push(fmt::to_string(address));
             }
             return m;
         });
@@ -703,7 +660,7 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
     ss::get_pending_range_to_endpoint_map.set(r, [&ctx](std::unique_ptr<http::request> req) {
         //TBD
         unimplemented();
-        auto keyspace = validate_keyspace(ctx, req->param);
+        auto keyspace = validate_keyspace(ctx, req);
         std::vector<ss::maplist_mapper> res;
         return make_ready_future<json::json_return_type>(res);
     });
@@ -712,32 +669,19 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
         if (!req->param.exists("keyspace")) {
             throw bad_param_exception("The keyspace param is not provided");
         }
-        auto keyspace = req->param["keyspace"];
+        auto keyspace = req->get_path_param("keyspace");
         auto table = req->get_query_param("table");
         if (!table.empty()) {
             validate_table(ctx, keyspace, table);
             return describe_ring_as_json_for_table(ss, keyspace, table);
         }
-        return describe_ring_as_json(ss, validate_keyspace(ctx, req->param));
+        return describe_ring_as_json(ss, validate_keyspace(ctx, req));
     });
 
     ss::get_load.set(r, [&ctx](std::unique_ptr<http::request> req) {
         return get_cf_stats(ctx, &replica::column_family_stats::live_disk_space_used);
     });
 
-    ss::get_load_map.set(r, [&ctx] (std::unique_ptr<http::request> req) {
-        return ctx.lmeter.get_load_map().then([] (auto&& load_map) {
-            std::vector<ss::map_string_double> res;
-            for (auto i : load_map) {
-                ss::map_string_double val;
-                val.key = i.first;
-                val.value = i.second;
-                res.push_back(val);
-            }
-            return make_ready_future<json::json_return_type>(res);
-        });
-    });
-
     ss::get_current_generation_number.set(r, [&ss](std::unique_ptr<http::request> req) {
         auto ep = ss.local().get_token_metadata().get_topology().my_address();
         return ss.local().gossiper().get_current_generation_number(ep).then([](gms::generation_type res) {
@@ -746,7 +690,7 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
     });
 
     ss::get_natural_endpoints.set(r, [&ctx, &ss](const_req req) {
-        auto keyspace = validate_keyspace(ctx, req.param);
+        auto keyspace = validate_keyspace(ctx, req);
         return container_to_vec(ss.local().get_natural_endpoints(keyspace, req.get_query_param("cf"),
                 req.get_query_param("key")));
     });
@@ -815,7 +759,7 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
 
     ss::force_keyspace_cleanup.set(r, [&ctx, &ss](std::unique_ptr<http::request> req) -> future<json::json_return_type> {
         auto& db = ctx.db;
-        auto keyspace = validate_keyspace(ctx, req->param);
+        auto keyspace = validate_keyspace(ctx, req);
         auto table_infos = parse_table_infos(keyspace, ctx, req->query_parameters, "cf");
         const auto& rs = db.local().find_keyspace(keyspace).get_replication_strategy();
         if (rs.get_type() == locator::replication_strategy_type::local || !rs.is_vnode_based()) {
@@ -910,7 +854,7 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
     });
 
     ss::force_keyspace_flush.set(r, [&ctx](std::unique_ptr<http::request> req) -> future<json::json_return_type> {
-        auto keyspace = validate_keyspace(ctx, req->param);
+        auto keyspace = validate_keyspace(ctx, req);
         auto column_families = parse_tables(keyspace, ctx, req->query_parameters, "cf");
         apilog.info("perform_keyspace_flush: keyspace={} tables={}", keyspace, column_families);
         auto& db = ctx.db;
@@ -1019,7 +963,7 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
     ss::truncate.set(r, [&ctx](std::unique_ptr<http::request> req) {
         //TBD
         unimplemented();
-        auto keyspace = validate_keyspace(ctx, req->param);
+        auto keyspace = validate_keyspace(ctx, req);
         auto column_family = req->get_query_param("cf");
         return make_ready_future<json::json_return_type>(json_void());
     });
@@ -1153,7 +1097,16 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
     });
 
     ss::rebuild.set(r, [&ss](std::unique_ptr<http::request> req) {
-        auto source_dc = req->get_query_param("source_dc");
+        utils::optional_param source_dc;
+        if (auto source_dc_str = req->get_query_param("source_dc"); !source_dc_str.empty()) {
+            source_dc.emplace(std::move(source_dc_str)).set_user_provided();
+        }
+        if (auto force_str = req->get_query_param("force"); !force_str.empty() && service::loosen_constraints(validate_bool(force_str))) {
+            if (!source_dc) {
+                throw bad_param_exception("The `source_dc` option must be provided for using the `force` option");
+            }
+            source_dc.set_force();
+        }
         apilog.info("rebuild: source_dc={}", source_dc);
         return ss.local().rebuild(std::move(source_dc)).then([] {
             return make_ready_future<json::json_return_type>(json_void());
@@ -1163,14 +1116,14 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
     ss::bulk_load.set(r, [](std::unique_ptr<http::request> req) {
         //TBD
         unimplemented();
-        auto path = req->param["path"];
+        auto path = req->get_path_param("path");
         return make_ready_future<json::json_return_type>(json_void());
     });
 
     ss::bulk_load_async.set(r, [](std::unique_ptr<http::request> req) {
         //TBD
         unimplemented();
-        auto path = req->param["path"];
+        auto path = req->get_path_param("path");
         return make_ready_future<json::json_return_type>(json_void());
     });
 
@@ -1257,38 +1210,6 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
         }
     });
 
-    ss::enable_auto_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        auto keyspace = validate_keyspace(ctx, req->param);
-        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
-
-        apilog.info("enable_auto_compaction: keyspace={} tables={}", keyspace, tables);
-        return set_tables_autocompaction(ctx, keyspace, tables, true);
-    });
-
-    ss::disable_auto_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        auto keyspace = validate_keyspace(ctx, req->param);
-        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
-
-        apilog.info("disable_auto_compaction: keyspace={} tables={}", keyspace, tables);
-        return set_tables_autocompaction(ctx, keyspace, tables, false);
-    });
-
-    ss::enable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        auto keyspace = validate_keyspace(ctx, req->param);
-        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
-
-        apilog.info("enable_tombstone_gc: keyspace={} tables={}", keyspace, tables);
-        return set_tables_tombstone_gc(ctx, keyspace, tables, true);
-    });
-
-    ss::disable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        auto keyspace = validate_keyspace(ctx, req->param);
-        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
-
-        apilog.info("disable_tombstone_gc: keyspace={} tables={}", keyspace, tables);
-        return set_tables_tombstone_gc(ctx, keyspace, tables, false);
-    });
-
     ss::deliver_hints.set(r, [](std::unique_ptr<http::request> req) {
         //TBD
         unimplemented();
@@ -1382,7 +1303,7 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
     });
 
     ss::get_effective_ownership.set(r, [&ctx, &ss] (std::unique_ptr<http::request> req) {
-        auto keyspace_name = req->param["keyspace"] == "null" ? "" : validate_keyspace(ctx, req->param);
+        auto keyspace_name = req->get_path_param("keyspace") == "null" ? "" : validate_keyspace(ctx, req);
         auto table_name = req->get_query_param("cf");
 
         if (!keyspace_name.empty()) {
@@ -1622,6 +1543,11 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
         co_return json_void();
     });
 
+    ss::quiesce_topology.set(r, [&ss] (std::unique_ptr<http::request> req) -> future<json_return_type> {
+        co_await ss.local().await_topology_quiesced();
+        co_return json_void();
+    });
+
     sp::get_schema_versions.set(r, [&ss](std::unique_ptr<http::request> req)  {
         return ss.local().describe_schema_versions().then([] (auto result) {
             std::vector<sp::mapper_list> res;
@@ -1649,7 +1575,6 @@ void unset_storage_service(http_context& ctx, routes& r) {
     ss::get_pending_range_to_endpoint_map.unset(r);
     ss::describe_ring.unset(r);
     ss::get_load.unset(r);
-    ss::get_load_map.unset(r);
     ss::get_current_generation_number.unset(r);
     ss::get_natural_endpoints.unset(r);
     ss::cdc_streams_check_and_repair.unset(r);
@@ -1697,10 +1622,6 @@ void unset_storage_service(http_context& ctx, routes& r) {
     ss::get_trace_probability.unset(r);
     ss::get_slow_query_info.unset(r);
     ss::set_slow_query.unset(r);
-    ss::enable_auto_compaction.unset(r);
-    ss::disable_auto_compaction.unset(r);
-    ss::enable_tombstone_gc.unset(r);
-    ss::disable_tombstone_gc.unset(r);
     ss::deliver_hints.unset(r);
     ss::get_cluster_name.unset(r);
     ss::get_partitioner_name.unset(r);
@@ -1725,39 +1646,67 @@ void unset_storage_service(http_context& ctx, routes& r) {
     ss::add_tablet_replica.unset(r);
     ss::del_tablet_replica.unset(r);
     ss::tablet_balancing_enable.unset(r);
+    ss::quiesce_topology.unset(r);
     sp::get_schema_versions.unset(r);
 }
 
+void set_load_meter(http_context& ctx, routes& r, service::load_meter& lm) {
+    ss::get_load_map.set(r, [&lm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
+        auto load_map = co_await lm.get_load_map();
+        std::vector<ss::map_string_double> res;
+        for (auto i : load_map) {
+            ss::map_string_double val;
+            val.key = i.first;
+            val.value = i.second;
+            res.push_back(val);
+        }
+        co_return res;
+    });
+}
+
+void unset_load_meter(http_context& ctx, routes& r) {
+    ss::get_load_map.unset(r);
+}
+
 void set_snapshot(http_context& ctx, routes& r, sharded<db::snapshot_ctl>& snap_ctl) {
     ss::get_snapshot_details.set(r, [&snap_ctl](std::unique_ptr<http::request> req) -> future<json::json_return_type> {
         auto result = co_await snap_ctl.local().get_snapshot_details();
         co_return std::function([res = std::move(result)] (output_stream<char>&& o) -> future<> {
-            auto result = std::move(res);
+            std::exception_ptr ex;
             output_stream<char> out = std::move(o);
-            bool first = true;
+            try {
+                auto result = std::move(res);
+                bool first = true;
 
-            co_await out.write("[");
-            for (auto&& map : result) {
-                if (!first) {
-                    co_await out.write(", ");
+                co_await out.write("[");
+                for (auto& [name, details] : result) {
+                    if (!first) {
+                        co_await out.write(", ");
+                    }
+                    std::vector<ss::snapshot> snapshot;
+                    for (auto& cf : details) {
+                        ss::snapshot snp;
+                        snp.ks = cf.ks;
+                        snp.cf = cf.cf;
+                        snp.live = cf.details.live;
+                        snp.total = cf.details.total;
+                        snapshot.push_back(std::move(snp));
+                    }
+                    ss::snapshots all_snapshots;
+                    all_snapshots.key = name;
+                    all_snapshots.value = std::move(snapshot);
+                    co_await all_snapshots.write(out);
+                    first = false;
                 }
-                std::vector<ss::snapshot> snapshot;
-                for (auto& cf : std::get<1>(map)) {
-                    ss::snapshot snp;
-                    snp.ks = cf.ks;
-                    snp.cf = cf.cf;
-                    snp.live = cf.live;
-                    snp.total = cf.total;
-                    snapshot.push_back(std::move(snp));
-                }
-                ss::snapshots all_snapshots;
-                all_snapshots.key = std::get<0>(map);
-                all_snapshots.value = std::move(snapshot);
-                co_await all_snapshots.write(out);
-                first = false;
+                co_await out.write("]");
+                co_await out.flush();
+            } catch (...) {
+              ex = std::current_exception();
             }
-            co_await out.write("]");
             co_await out.close();
+            if (ex) {
+                co_await coroutine::return_exception_ptr(std::move(ex));
+            }
         });
     });
 
@@ -1836,6 +1785,20 @@ void set_snapshot(http_context& ctx, routes& r, sharded<db::snapshot_ctl>& snap_
 
         co_return json::json_return_type(static_cast<int>(scrub_status::successful));
     });
+
+    cf::get_true_snapshots_size.set(r, [&snap_ctl] (std::unique_ptr<http::request> req) {
+        auto [ks, cf] = parse_fully_qualified_cf_name(req->get_path_param("name"));
+        return snap_ctl.local().true_snapshots_size(std::move(ks), std::move(cf)).then([] (int64_t res) {
+            return make_ready_future<json::json_return_type>(res);
+        });
+    });
+
+    cf::get_all_true_snapshots_size.set(r, [] (std::unique_ptr<http::request> req) {
+        //TBD
+        unimplemented();
+        return make_ready_future<json::json_return_type>(0);
+    });
+
 }
 
 void unset_snapshot(http_context& ctx, routes& r) {
@@ -1844,6 +1807,8 @@ void unset_snapshot(http_context& ctx, routes& r) {
     ss::del_snapshot.unset(r);
     ss::true_snapshots_size.unset(r);
     ss::scrub.unset(r);
+    cf::get_true_snapshots_size.unset(r);
+    cf::get_all_true_snapshots_size.unset(r);
 }
 
 }

api/storage_service.hh

@@ -14,7 +14,6 @@
 #include "db/data_listeners.hh"
 
 namespace cql_transport { class controller; }
-class thrift_controller;
 namespace db {
 class snapshot_ctl;
 namespace view {
@@ -40,7 +39,11 @@ sstring validate_keyspace(const http_context& ctx, sstring ks_name);
 
 // verify that the keyspace parameter is found, otherwise a bad_param_exception exception is thrown
 // containing the description of the respective keyspace error.
-sstring validate_keyspace(const http_context& ctx, const httpd::parameters& param);
+sstring validate_keyspace(const http_context& ctx, const std::unique_ptr<http::request>& req);
+
+// verify that the table parameter is found, otherwise a bad_param_exception exception is thrown
+// containing the description of the respective table error.
+void validate_table(const http_context& ctx, sstring ks_name, sstring table_name);
 
 // splits a request parameter assumed to hold a comma-separated list of table names
 // verify that the tables are found, otherwise a bad_param_exception exception is thrown
@@ -76,10 +79,12 @@ void set_repair(http_context& ctx, httpd::routes& r, sharded<repair_service>& re
 void unset_repair(http_context& ctx, httpd::routes& r);
 void set_transport_controller(http_context& ctx, httpd::routes& r, cql_transport::controller& ctl);
 void unset_transport_controller(http_context& ctx, httpd::routes& r);
-void set_rpc_controller(http_context& ctx, httpd::routes& r, thrift_controller& ctl);
-void unset_rpc_controller(http_context& ctx, httpd::routes& r);
+void set_thrift_controller(http_context& ctx, httpd::routes& r);
+void unset_thrift_controller(http_context& ctx, httpd::routes& r);
 void set_snapshot(http_context& ctx, httpd::routes& r, sharded<db::snapshot_ctl>& snap_ctl);
 void unset_snapshot(http_context& ctx, httpd::routes& r);
+void set_load_meter(http_context& ctx, httpd::routes& r, service::load_meter& lm);
+void unset_load_meter(http_context& ctx, httpd::routes& r);
 seastar::future<json::json_return_type> run_toppartitions_query(db::toppartitions_query& q, http_context &ctx, bool legacy_request = false);
 
 } // namespace api

api/stream_manager.cc

@@ -108,7 +108,7 @@ void set_stream_manager(http_context& ctx, routes& r, sharded<streaming::stream_
     });
 
     hs::get_total_incoming_bytes.set(r, [&sm](std::unique_ptr<request> req) {
-        gms::inet_address peer(req->param["peer"]);
+        gms::inet_address peer(req->get_path_param("peer"));
         return sm.map_reduce0([peer](streaming::stream_manager& sm) {
             return sm.get_progress_on_all_shards(peer).then([] (auto sbytes) {
                 return sbytes.bytes_received;
@@ -129,7 +129,7 @@ void set_stream_manager(http_context& ctx, routes& r, sharded<streaming::stream_
     });
 
     hs::get_total_outgoing_bytes.set(r, [&sm](std::unique_ptr<request> req) {
-        gms::inet_address peer(req->param["peer"]);
+        gms::inet_address peer(req->get_path_param("peer"));
         return sm.map_reduce0([peer] (streaming::stream_manager& sm) {
             return sm.get_progress_on_all_shards(peer).then([] (auto sbytes) {
                 return sbytes.bytes_sent;

api/system.cc

@@ -10,6 +10,7 @@
 #include "api/api-doc/system.json.hh"
 #include "api/api-doc/metrics.json.hh"
 #include "replica/database.hh"
+#include "sstables/sstables_manager.hh"
 
 #include <rapidjson/document.h>
 #include <seastar/core/reactor.hh>
@@ -122,9 +123,9 @@ void set_system(http_context& ctx, routes& r) {
 
     hs::get_logger_level.set(r, [](const_req req) {
         try {
-            return logging::level_name(logging::logger_registry().get_logger_level(req.param["name"]));
+            return logging::level_name(logging::logger_registry().get_logger_level(req.get_path_param("name")));
         } catch (std::out_of_range& e) {
-            throw bad_param_exception("Unknown logger name " + req.param["name"]);
+            throw bad_param_exception("Unknown logger name " + req.get_path_param("name"));
         }
         // just to keep the compiler happy
         return sstring();
@@ -133,9 +134,9 @@ void set_system(http_context& ctx, routes& r) {
     hs::set_logger_level.set(r, [](const_req req) {
         try {
             logging::log_level level = boost::lexical_cast<logging::log_level>(std::string(req.get_query_param("level")));
-            logging::logger_registry().set_logger_level(req.param["name"], level);
+            logging::logger_registry().set_logger_level(req.get_path_param("name"), level);
         } catch (std::out_of_range& e) {
-            throw bad_param_exception("Unknown logger name " + req.param["name"]);
+            throw bad_param_exception("Unknown logger name " + req.get_path_param("name"));
         } catch (boost::bad_lexical_cast& e) {
             throw bad_param_exception("Unknown logging level " + req.get_query_param("level"));
         }
@@ -182,6 +183,11 @@ void set_system(http_context& ctx, routes& r) {
         apilog.info("Profile dumped to {}", profile_dest);
         return make_ready_future<json::json_return_type>(json::json_return_type(json::json_void()));
     }) ;
+
+    hs::get_highest_supported_sstable_version.set(r, [&ctx] (const_req req) {
+        auto& table = ctx.db.local().find_column_family("system", "local");
+        return seastar::to_sstring(table.get_sstables_manager().get_highest_supported_format());
+    });
 }
 
 }

api/system.hh

@@ -8,10 +8,13 @@
 
 #pragma once
 
-#include "api.hh"
+namespace seastar::httpd {
+class routes;
+}
 
 namespace api {
 
-void set_system(http_context& ctx, httpd::routes& r);
+struct http_context;
+void set_system(http_context& ctx, seastar::httpd::routes& r);
 
 }

api/task_manager.cc

@@ -7,6 +7,7 @@
  */
 
 #include <seastar/core/coroutine.hh>
+#include <seastar/coroutine/exception.hh>
 #include <seastar/http/exception.hh>
 
 #include "task_manager.hh"
@@ -23,6 +24,8 @@ namespace tm = httpd::task_manager_json;
 using namespace json;
 using namespace seastar::httpd;
 
+using task_variant = std::variant<tasks::task_manager::foreign_task_ptr, tasks::task_manager::task::task_essentials>;
+
 inline bool filter_tasks(tasks::task_manager::task_ptr task, std::unordered_map<sstring, sstring>& query_params) {
     return (!query_params.contains("keyspace") || query_params["keyspace"] == task->get_status().keyspace) &&
         (!query_params.contains("table") || query_params["table"] == task->get_status().table);
@@ -32,7 +35,6 @@ struct full_task_status {
     tasks::task_manager::task::status task_status;
     std::string type;
     tasks::task_manager::task::progress progress;
-    std::string module;
     tasks::task_id parent_id;
     tasks::is_abortable abortable;
     std::vector<std::string> children_ids;
@@ -99,16 +101,16 @@ future<full_task_status> retrieve_status(const tasks::task_manager::foreign_task
     s.type = task->type();
     s.parent_id = task->get_parent_id();
     s.abortable = task->is_abortable();
-    s.module = task->get_module_name();
     s.progress.completed = progress.completed;
     s.progress.total = progress.total;
-    std::vector<std::string> ct{task->get_children().size()};
-    boost::transform(task->get_children(), ct.begin(), [] (const auto& child) {
+    std::vector<std::string> ct = co_await task->get_children().map_each_task<std::string>([] (const tasks::task_manager::foreign_task_ptr& child) {
         return child->id().to_sstring();
+    }, [] (const tasks::task_manager::task::task_essentials& child) {
+        return child.task_status.id.to_sstring();
     });
     s.children_ids = std::move(ct);
     co_return s;
-}
+};
 
 void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>& tm, db::config& cfg) {
     tm::get_modules.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
@@ -123,7 +125,7 @@ void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>
             chunked_stats local_res;
             tasks::task_manager::module_ptr module;
             try {
-                module = tm.find_module(req->param["module"]);
+                module = tm.find_module(req->get_path_param("module"));
             } catch (...) {
                 throw bad_param_exception(fmt::format("{}", std::current_exception()));
             }
@@ -138,25 +140,34 @@ void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>
 
         std::function<future<>(output_stream<char>&&)> f = [r = std::move(res)] (output_stream<char>&& os) -> future<> {
             auto s = std::move(os);
-            auto res = std::move(r);
-            co_await s.write("[");
-            std::string delim = "";
-            for (auto& v: res) {
-                for (auto& stats: v) {
-                    co_await s.write(std::exchange(delim, ", "));
-                    tm::task_stats ts;
-                    ts = stats;
-                    co_await formatter::write(s, ts);
+            std::exception_ptr ex;
+            try {
+                auto res = std::move(r);
+                co_await s.write("[");
+                std::string delim = "";
+                for (auto& v: res) {
+                    for (auto& stats: v) {
+                        co_await s.write(std::exchange(delim, ", "));
+                        tm::task_stats ts;
+                        ts = stats;
+                        co_await formatter::write(s, ts);
+                    }
                 }
+                co_await s.write("]");
+                co_await s.flush();
+            } catch (...) {
+                ex = std::current_exception();
             }
-            co_await s.write("]");
             co_await s.close();
+            if (ex) {
+                co_await coroutine::return_exception_ptr(std::move(ex));
+            }
         };
         co_return std::move(f);
     });
 
     tm::get_task_status.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
-        auto id = tasks::task_id{utils::UUID{req->param["task_id"]}};
+        auto id = tasks::task_id{utils::UUID{req->get_path_param("task_id")}};
         tasks::task_manager::foreign_task_ptr task;
         try {
             task = co_await tasks::task_manager::invoke_on_task(tm, id, std::function([] (tasks::task_manager::task_ptr task) -> future<tasks::task_manager::foreign_task_ptr> {
@@ -173,13 +184,13 @@ void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>
     });
 
     tm::abort_task.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
-        auto id = tasks::task_id{utils::UUID{req->param["task_id"]}};
+        auto id = tasks::task_id{utils::UUID{req->get_path_param("task_id")}};
         try {
             co_await tasks::task_manager::invoke_on_task(tm, id, [] (tasks::task_manager::task_ptr task) -> future<> {
                 if (!task->is_abortable()) {
                     co_await coroutine::return_exception(std::runtime_error("Requested task cannot be aborted"));
                 }
-                co_await task->abort();
+                task->abort();
             });
         } catch (tasks::task_manager::task_not_found& e) {
             throw bad_param_exception(e.what());
@@ -188,12 +199,11 @@ void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>
     });
 
     tm::wait_task.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
-        auto id = tasks::task_id{utils::UUID{req->param["task_id"]}};
+        auto id = tasks::task_id{utils::UUID{req->get_path_param("task_id")}};
         tasks::task_manager::foreign_task_ptr task;
         try {
             task = co_await tasks::task_manager::invoke_on_task(tm, id, std::function([] (tasks::task_manager::task_ptr task) {
                 return task->done().then_wrapped([task] (auto f) {
-                    task->unregister_task();
                     // done() is called only because we want the task to be complete before getting its status.
                     // The future should be ignored here as the result does not matter.
                     f.ignore_ready_future();
@@ -209,8 +219,8 @@ void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>
 
     tm::get_task_status_recursively.set(r, [&_tm = tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
         auto& tm = _tm;
-        auto id = tasks::task_id{utils::UUID{req->param["task_id"]}};
-        std::queue<tasks::task_manager::foreign_task_ptr> q;
+        auto id = tasks::task_id{utils::UUID{req->get_path_param("task_id")}};
+        std::queue<task_variant> q;
         utils::chunked_vector<full_task_status> res;
 
         tasks::task_manager::foreign_task_ptr task;
@@ -230,10 +240,33 @@ void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>
         q.push(co_await task.copy());   // Task cannot be moved since we need it to be alive during whole loop execution.
         while (!q.empty()) {
             auto& current = q.front();
-            res.push_back(co_await retrieve_status(current));
-            for (auto& child: current->get_children()) {
-                q.push(co_await child.copy());
-            }
+            co_await std::visit(overloaded_functor {
+                [&] (const tasks::task_manager::foreign_task_ptr& task) -> future<> {
+                    res.push_back(co_await retrieve_status(task));
+                    co_await task->get_children().for_each_task([&q] (const tasks::task_manager::foreign_task_ptr& child) -> future<> {
+                        q.push(co_await child.copy());
+                    }, [&] (const tasks::task_manager::task::task_essentials& child) {
+                        q.push(child);
+                        return make_ready_future();
+                    });
+                },
+                [&] (const tasks::task_manager::task::task_essentials& task) -> future<> {
+                    res.push_back(full_task_status{
+                        .task_status = task.task_status,
+                        .type = task.type,
+                        .progress = task.task_progress,
+                        .parent_id = task.parent_id,
+                        .abortable = task.abortable,
+                        .children_ids = boost::copy_range<std::vector<std::string>>(task.failed_children | boost::adaptors::transformed([] (auto& child) {
+                            return child.task_status.id.to_sstring();
+                        }))
+                    });
+                    for (auto& child: task.failed_children) {
+                        q.push(child);
+                    }
+                    return make_ready_future();
+                }
+            }, current);
             q.pop();
         }
 

api/task_manager_test.cc

@@ -83,20 +83,19 @@ void set_task_manager_test(http_context& ctx, routes& r, sharded<tasks::task_man
     });
 
     tmt::finish_test_task.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
-        auto id = tasks::task_id{utils::UUID{req->param["task_id"]}};
+        auto id = tasks::task_id{utils::UUID{req->get_path_param("task_id")}};
         auto it = req->query_parameters.find("error");
         bool fail = it != req->query_parameters.end();
         std::string error = fail ? it->second : "";
 
         try {
-            co_await tasks::task_manager::invoke_on_task(tm, id, [fail, error = std::move(error)] (tasks::task_manager::task_ptr task) {
+            co_await tasks::task_manager::invoke_on_task(tm, id, [fail, error = std::move(error)] (tasks::task_manager::task_ptr task) -> future<> {
                 tasks::test_task test_task{task};
                 if (fail) {
-                    test_task.finish_failed(std::make_exception_ptr(std::runtime_error(error)));
+                    co_await test_task.finish_failed(std::make_exception_ptr(std::runtime_error(error)));
                 } else {
-                    test_task.finish();
+                    co_await test_task.finish();
                 }
-                return make_ready_future<>();
             });
         } catch (tasks::task_manager::task_not_found& e) {
             throw bad_param_exception(e.what());

api/task_manager_test.hh

@@ -11,16 +11,19 @@
 #pragma once
 
 #include <seastar/core/sharded.hh>
-#include "api.hh"
 
 namespace tasks {
 class task_manager;
 }
 
-namespace api {
+namespace seastar::httpd {
+class routes;
+}
 
-void set_task_manager_test(http_context& ctx, httpd::routes& r, sharded<tasks::task_manager>& tm);
-void unset_task_manager_test(http_context& ctx, httpd::routes& r);
+namespace api {
+struct http_context;
+void set_task_manager_test(http_context& ctx, seastar::httpd::routes& r, seastar::sharded<tasks::task_manager>& tm);
+void unset_task_manager_test(http_context& ctx, seastar::httpd::routes& r);
 
 }
 

api/tasks.cc

@@ -7,6 +7,7 @@
  */
 
 #include <seastar/core/coroutine.hh>
+#include <fmt/ranges.h>
 
 #include "api/api.hh"
 #include "api/storage_service.hh"
@@ -29,7 +30,7 @@ using ks_cf_func = std::function<future<json::json_return_type>(http_context&, s
 
 static auto wrap_ks_cf(http_context &ctx, ks_cf_func f) {
     return [&ctx, f = std::move(f)](std::unique_ptr<http::request> req) {
-        auto keyspace = validate_keyspace(ctx, req->param);
+        auto keyspace = validate_keyspace(ctx, req);
         auto table_infos = parse_table_infos(keyspace, ctx, req->query_parameters, "cf");
         return f(ctx, std::move(req), std::move(keyspace), std::move(table_infos));
     };
@@ -61,7 +62,7 @@ void set_tasks_compaction_module(http_context& ctx, routes& r, sharded<service::
 
     t::force_keyspace_cleanup_async.set(r, [&ctx, &ss](std::unique_ptr<http::request> req) -> future<json::json_return_type> {
         auto& db = ctx.db;
-        auto keyspace = validate_keyspace(ctx, req->param);
+        auto keyspace = validate_keyspace(ctx, req);
         auto table_infos = parse_table_infos(keyspace, ctx, req->query_parameters, "cf");
         apilog.info("force_keyspace_cleanup_async: keyspace={} tables={}", keyspace, table_infos);
         if (!co_await ss.local().is_cleanup_allowed(keyspace)) {

api/tasks.hh

@@ -8,11 +8,20 @@
 
 #pragma once
 
-#include "api.hh"
-#include "db/config.hh"
+#include <seastar/core/sharded.hh>
+#include "db/snapshot-ctl.hh"
+
+namespace seastar::httpd {
+class routes;
+}
+
+namespace service {
+class storage_service;
+}
 
 namespace api {
 
+struct http_context;
 void set_tasks_compaction_module(http_context& ctx, httpd::routes& r, sharded<service::storage_service>& ss, sharded<db::snapshot_ctl>& snap_ctl);
 void unset_tasks_compaction_module(http_context& ctx, httpd::routes& r);
 

api/token_metadata.cc

@@ -31,7 +31,7 @@ void set_token_metadata(http_context& ctx, routes& r, sharded<locator::shared_to
     });
 
     ss::get_node_tokens.set(r, [&tm] (std::unique_ptr<http::request> req) {
-        gms::inet_address addr(req->param["endpoint"]);
+        gms::inet_address addr(req->get_path_param("endpoint"));
         auto& local_tm = *tm.local().get();
         const auto host_id = local_tm.get_host_id_if_known(addr);
         return make_ready_future<json::json_return_type>(stream_range_as_array(host_id ? local_tm.get_tokens(*host_id): std::vector<dht::token>{}, [](const dht::token& i) {

api/token_metadata.hh

@@ -9,13 +9,16 @@
 #pragma once
 
 #include <seastar/core/sharded.hh>
-#include "api/api_init.hh"
+
+namespace seastar::httpd {
+class routes;
+}
 
 namespace locator { class shared_token_metadata; }
 
 namespace api {
-
-void set_token_metadata(http_context& ctx, httpd::routes& r, sharded<locator::shared_token_metadata>& tm);
-void unset_token_metadata(http_context& ctx, httpd::routes& r);
+struct http_context;
+void set_token_metadata(http_context& ctx, seastar::httpd::routes& r, seastar::sharded<locator::shared_token_metadata>& tm);
+void unset_token_metadata(http_context& ctx, seastar::httpd::routes& r);
 
 }

auth/CMakeLists.txt

@@ -30,6 +30,7 @@ target_link_libraries(scylla_auth
     Seastar::seastar
     xxHash::xxhash
   PRIVATE
+    absl::headers
     cql3
     idl
     wasmtime_bindings

auth/allow_all_authenticator.hh

@@ -59,15 +59,15 @@ public:
         return make_ready_future<authenticated_user>(anonymous_user());
     }
 
-    virtual future<> create(std::string_view, const authentication_options& options) override {
+    virtual future<> create(std::string_view, const authentication_options& options, ::service::group0_batch&) override {
         return make_ready_future();
     }
 
-    virtual future<> alter(std::string_view, const authentication_options& options) override {
+    virtual future<> alter(std::string_view, const authentication_options& options, ::service::group0_batch&) override {
         return make_ready_future();
     }
 
-    virtual future<> drop(std::string_view) override {
+    virtual future<> drop(std::string_view, ::service::group0_batch&) override {
         return make_ready_future();
     }
 

auth/allow_all_authorizer.hh

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "auth/authorizer.hh"
+#include <seastar/core/future.hh>
 
 namespace cql3 {
 class query_processor;
@@ -44,12 +45,12 @@ public:
         return make_ready_future<permission_set>(permissions::ALL);
     }
 
-    virtual future<> grant(std::string_view, permission_set, const resource&) override {
+    virtual future<> grant(std::string_view, permission_set, const resource&, ::service::group0_batch&) override {
         return make_exception_future<>(
                 unsupported_authorization_operation("GRANT operation is not supported by AllowAllAuthorizer"));
     }
 
-    virtual future<> revoke(std::string_view, permission_set, const resource&) override {
+    virtual future<> revoke(std::string_view, permission_set, const resource&, ::service::group0_batch&) override {
         return make_exception_future<>(
                 unsupported_authorization_operation("REVOKE operation is not supported by AllowAllAuthorizer"));
     }
@@ -60,14 +61,12 @@ public:
                         "LIST PERMISSIONS operation is not supported by AllowAllAuthorizer"));
     }
 
-    virtual future<> revoke_all(std::string_view) override {
-        return make_exception_future(
-                unsupported_authorization_operation("REVOKE operation is not supported by AllowAllAuthorizer"));
+    virtual future<> revoke_all(std::string_view, ::service::group0_batch&) override {
+        return make_ready_future();
     }
 
-    virtual future<> revoke_all(const resource&) override {
-        return make_exception_future(
-                unsupported_authorization_operation("REVOKE operation is not supported by AllowAllAuthorizer"));
+    virtual future<> revoke_all(const resource&, ::service::group0_batch&) override {
+        return make_ready_future();
     }
 
     virtual const resource_set& protected_resources() const override {

auth/authenticated_user.hh

@@ -50,7 +50,7 @@ inline bool is_anonymous(const authenticated_user& u) noexcept {
 /// The user name, or "anonymous".
 ///
 template <>
-struct fmt::formatter<auth::authenticated_user> : fmt::formatter<std::string_view> {
+struct fmt::formatter<auth::authenticated_user> : fmt::formatter<string_view> {
     template <typename FormatContext>
     auto format(const auth::authenticated_user& u, FormatContext& ctx) const {
         if (u.name) {

auth/authentication_options.hh

@@ -48,15 +48,15 @@ public:
 }
 
 template <>
-struct fmt::formatter<auth::authentication_option> : fmt::formatter<std::string_view> {
+struct fmt::formatter<auth::authentication_option> : fmt::formatter<string_view> {
     template <typename FormatContext>
     auto format(const auth::authentication_option a, FormatContext& ctx) const {
         using enum auth::authentication_option;
         switch (a) {
         case password:
-            return formatter<std::string_view>::format("PASSWORD", ctx);
+            return formatter<string_view>::format("PASSWORD", ctx);
         case options:
-            return formatter<std::string_view>::format("OPTIONS", ctx);
+            return formatter<string_view>::format("OPTIONS", ctx);
         }
         std::abort();
     }

auth/authenticator.hh

@@ -16,15 +16,15 @@
 #include <optional>
 #include <functional>
 
-#include <seastar/core/enum.hh>
 #include <seastar/core/future.hh>
 #include <seastar/core/sstring.hh>
-#include <seastar/core/shared_ptr.hh>
 
 #include "auth/authentication_options.hh"
 #include "auth/resource.hh"
 #include "auth/sasl_challenge.hh"
 
+#include "service/raft/raft_group0_client.hh"
+
 namespace db {
     class config;
 }
@@ -106,7 +106,7 @@ public:
     ///
     /// The options provided must be a subset of `supported_options()`.
     ///
-    virtual future<> create(std::string_view role_name, const authentication_options& options) = 0;
+    virtual future<> create(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) = 0;
 
     ///
     /// Alter the authentication record of an existing user.
@@ -115,12 +115,12 @@ public:
     ///
     /// Callers must ensure that the specification of `alterable_options()` is adhered to.
     ///
-    virtual future<> alter(std::string_view role_name, const authentication_options& options) = 0;
+    virtual future<> alter(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) = 0;
 
     ///
     /// Delete the authentication record for a user. This will disallow the user from logging in.
     ///
-    virtual future<> drop(std::string_view role_name) = 0;
+    virtual future<> drop(std::string_view role_name, ::service::group0_batch&) = 0;
 
     ///
     /// Query for custom options (those corresponding to \ref authentication_options::options).

auth/authorizer.hh

@@ -16,10 +16,10 @@
 #include <vector>
 
 #include <seastar/core/future.hh>
-#include <seastar/core/shared_ptr.hh>
 
 #include "auth/permission.hh"
 #include "auth/resource.hh"
+#include "service/raft/raft_group0_client.hh"
 #include "seastarx.hh"
 
 namespace auth {
@@ -81,14 +81,14 @@ public:
     ///
     /// \throws \ref unsupported_authorization_operation if granting permissions is not supported.
     ///
-    virtual future<> grant(std::string_view role_name, permission_set, const resource&) = 0;
+    virtual future<> grant(std::string_view role_name, permission_set, const resource&, ::service::group0_batch&) = 0;
 
     ///
     /// Revoke a set of permissions from a role for a particular \ref resource.
     ///
     /// \throws \ref unsupported_authorization_operation if revoking permissions is not supported.
     ///
-    virtual future<> revoke(std::string_view role_name, permission_set, const resource&) = 0;
+    virtual future<> revoke(std::string_view role_name, permission_set, const resource&, ::service::group0_batch&) = 0;
 
     ///
     /// Query for all directly granted permissions.
@@ -102,14 +102,14 @@ public:
     ///
     /// \throws \ref unsupported_authorization_operation if revoking permissions is not supported.
     ///
-    virtual future<> revoke_all(std::string_view role_name) = 0;
+    virtual future<> revoke_all(std::string_view role_name, ::service::group0_batch&) = 0;
 
     ///
     /// Revoke all permissions granted to any role for a particular resource.
     ///
     /// \throws \ref unsupported_authorization_operation if revoking permissions is not supported.
     ///
-    virtual future<> revoke_all(const resource&) = 0;
+    virtual future<> revoke_all(const resource&, ::service::group0_batch&) = 0;
 
     ///
     /// System resources used internally as part of the implementation. These are made inaccessible to users.

auth/certificate_authenticator.cc

@@ -10,8 +10,10 @@
 #include "auth/certificate_authenticator.hh"
 
 #include <regex>
+#include <fmt/ranges.h>
 
 #include "utils/class_registrator.hh"
+#include "utils/to_string.hh"
 #include "data_dictionary/data_dictionary.hh"
 #include "cql3/query_processor.hh"
 #include "db/config.hh"
@@ -74,7 +76,7 @@ auth::certificate_authenticator::certificate_authenticator(cql3::query_processor
                     continue;
                 } catch (std::out_of_range&) {
                     // just fallthrough
-                } catch (std::regex_error&) {
+                } catch (boost::regex_error&) {
                     std::throw_with_nested(std::invalid_argument(fmt::format("Invalid query expression: {}", map.at(cfg_query_attr))));
                 }
             }
@@ -155,16 +157,16 @@ future<auth::authenticated_user> auth::certificate_authenticator::authenticate(c
     throw exceptions::authentication_exception("Cannot authenticate using attribute map");
 }
 
-future<> auth::certificate_authenticator::create(std::string_view role_name, const authentication_options& options) {
+future<> auth::certificate_authenticator::create(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) {
     // TODO: should we keep track of roles/enforce existence? Role manager should deal with this...
     co_return;
 }
 
-future<> auth::certificate_authenticator::alter(std::string_view role_name, const authentication_options& options) {
+future<> auth::certificate_authenticator::alter(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) {
     co_return;
 }
 
-future<> auth::certificate_authenticator::drop(std::string_view role_name) {
+future<> auth::certificate_authenticator::drop(std::string_view role_name, ::service::group0_batch&) {
     co_return;
 }
 

auth/certificate_authenticator.hh

@@ -9,7 +9,6 @@
 
 #pragma once
 
-#include <boost/regex.hpp>
 #include "auth/authenticator.hh"
 
 namespace cql3 {
@@ -47,9 +46,9 @@ public:
     future<authenticated_user> authenticate(const credentials_map& credentials) const override;
     future<std::optional<authenticated_user>> authenticate(session_dn_func) const override;
 
-    future<> create(std::string_view role_name, const authentication_options& options) override;
-    future<> alter(std::string_view role_name, const authentication_options& options) override;
-    future<> drop(std::string_view role_name) override;
+    future<> create(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) override;
+    future<> alter(std::string_view role_name, const authentication_options& options, ::service::group0_batch&) override;
+    future<> drop(std::string_view role_name, ::service::group0_batch&) override;
 
     future<custom_options> query_custom_options(std::string_view role_name) const override;
 

auth/common.cc

@@ -23,8 +23,6 @@
 #include "service/migration_manager.hh"
 #include "service/raft/group0_state_machine.hh"
 #include "timeout_config.hh"
-#include "db/config.hh"
-#include "db/system_auth_keyspace.hh"
 #include "utils/error_injection.hh"
 
 namespace auth {
@@ -41,14 +39,14 @@ constinit const std::string_view AUTH_PACKAGE_NAME("org.apache.cassandra.auth.")
 static logging::logger auth_log("auth");
 
 bool legacy_mode(cql3::query_processor& qp) {
-    return qp.auth_version < db::system_auth_keyspace::version_t::v2;
+    return qp.auth_version < db::system_keyspace::auth_version_t::v2;
 }
 
 std::string_view get_auth_ks_name(cql3::query_processor& qp) {
     if (legacy_mode(qp)) {
         return meta::legacy::AUTH_KS;
     }
-    return db::system_auth_keyspace::NAME;
+    return db::system_keyspace::NAME;
 }
 
 // Func must support being invoked more than once.
@@ -65,7 +63,7 @@ future<> do_after_system_ready(seastar::abort_source& as, seastar::noncopyable_f
     }).discard_result();
 }
 
-static future<> create_metadata_table_if_missing_impl(
+static future<> create_legacy_metadata_table_if_missing_impl(
         std::string_view table_name,
         cql3::query_processor& qp,
         std::string_view cql,
@@ -73,7 +71,7 @@ static future<> create_metadata_table_if_missing_impl(
     assert(this_shard_id() == 0); // once_among_shards makes sure a function is executed on shard 0 only
 
     auto db = qp.db();
-    auto parsed_statement = cql3::query_processor::parse_statement(cql);
+    auto parsed_statement = cql3::query_processor::parse_statement(cql, cql3::dialect{});
     auto& parsed_cf_statement = static_cast<cql3::statements::raw::cf_statement&>(*parsed_statement);
 
     parsed_cf_statement.prepare_keyspace(meta::legacy::AUTH_KS);
@@ -98,12 +96,12 @@ static future<> create_metadata_table_if_missing_impl(
     }
 }
 
-future<> create_metadata_table_if_missing(
+future<> create_legacy_metadata_table_if_missing(
         std::string_view table_name,
         cql3::query_processor& qp,
         std::string_view cql,
         ::service::migration_manager& mm) noexcept {
-    return futurize_invoke(create_metadata_table_if_missing_impl, table_name, qp, cql, mm);
+    return futurize_invoke(create_legacy_metadata_table_if_missing_impl, table_name, qp, cql, mm);
 }
 
 ::service::query_state& internal_distributed_query_state() noexcept {
@@ -123,7 +121,7 @@ static future<> announce_mutations_with_guard(
         ::service::raft_group0_client& group0_client,
         std::vector<canonical_mutation> muts,
         ::service::group0_guard group0_guard,
-        seastar::abort_source* as,
+        seastar::abort_source& as,
         std::optional<::service::raft_timeout> timeout) {
     auto group0_cmd = group0_client.prepare_command(
         ::service::write_mutations{
@@ -138,8 +136,8 @@ static future<> announce_mutations_with_guard(
 future<> announce_mutations_with_batching(
         ::service::raft_group0_client& group0_client,
         start_operation_func_t start_operation_func,
-        std::function<mutations_generator(api::timestamp_type& t)> gen,
-        seastar::abort_source* as,
+        std::function<::service::mutations_generator(api::timestamp_type t)> gen,
+        seastar::abort_source& as,
         std::optional<::service::raft_timeout> timeout) {
     // account for command's overhead, it's better to use smaller threshold than constantly bounce off the limit
     size_t memory_threshold = group0_client.max_command_size() * 0.75;
@@ -190,7 +188,7 @@ future<> announce_mutations(
         ::service::raft_group0_client& group0_client,
         const sstring query_string,
         std::vector<data_value_or_unset> values,
-        seastar::abort_source* as,
+        seastar::abort_source& as,
         std::optional<::service::raft_timeout> timeout) {
     auto group0_guard = co_await group0_client.start_operation(as, timeout);
     auto timestamp = group0_guard.write_timestamp();
@@ -203,4 +201,17 @@ future<> announce_mutations(
     co_await announce_mutations_with_guard(group0_client, std::move(cmuts), std::move(group0_guard), as, timeout);
 }
 
+future<> collect_mutations(
+        cql3::query_processor& qp,
+        ::service::group0_batch& collector,
+        const sstring query_string,
+        std::vector<data_value_or_unset> values) {
+    auto muts = co_await qp.get_mutations_internal(
+            query_string,
+            internal_distributed_query_state(),
+            collector.write_timestamp(),
+            std::move(values));
+    collector.add_mutations(std::move(muts), format("auth internal statement: {}", query_string));
+}
+
 }

auth/common.hh

@@ -13,12 +13,8 @@
 #include <seastar/core/future.hh>
 #include <seastar/core/abort_source.hh>
 #include <seastar/util/noncopyable_function.hh>
-#include <seastar/core/seastar.hh>
-#include <seastar/core/resource.hh>
 #include <seastar/core/sstring.hh>
-#include <seastar/core/smp.hh>
 
-#include "schema/schema_registry.hh"
 #include "types/types.hh"
 #include "service/raft/raft_group0_client.hh"
 
@@ -70,7 +66,7 @@ future<> once_among_shards(Task&& f) {
 // Func must support being invoked more than once.
 future<> do_after_system_ready(seastar::abort_source& as, seastar::noncopyable_function<future<>()> func);
 
-future<> create_metadata_table_if_missing(
+future<> create_legacy_metadata_table_if_missing(
         std::string_view table_name,
         cql3::query_processor&,
         std::string_view cql,
@@ -84,16 +80,15 @@ future<> create_metadata_table_if_missing(
 // Execute update query via group0 mechanism, mutations will be applied on all nodes.
 // Use this function when need to perform read before write on a single guard or if
 // you have more than one mutation and potentially exceed single command size limit.
-using start_operation_func_t = std::function<future<::service::group0_guard>(abort_source*)>;
-using mutations_generator = coroutine::experimental::generator<mutation>;
+using start_operation_func_t = std::function<future<::service::group0_guard>(abort_source&)>;
 future<> announce_mutations_with_batching(
         ::service::raft_group0_client& group0_client,
         // since we can operate also in topology coordinator context where we need stronger
         // guarantees than start_operation from group0_client gives we allow to inject custom
         // function here
         start_operation_func_t start_operation_func,
-        std::function<mutations_generator(api::timestamp_type& t)> gen,
-        seastar::abort_source* as,
+        std::function<::service::mutations_generator(api::timestamp_type t)> gen,
+        seastar::abort_source& as,
         std::optional<::service::raft_timeout> timeout);
 
 // Execute update query via group0 mechanism, mutations will be applied on all nodes.
@@ -102,7 +97,13 @@ future<> announce_mutations(
         ::service::raft_group0_client& group0_client,
         const sstring query_string,
         std::vector<data_value_or_unset> values,
-        seastar::abort_source* as,
+        seastar::abort_source& as,
         std::optional<::service::raft_timeout> timeout);
 
+// Appends mutations to a collector, they will be applied later on all nodes via group0 mechanism.
+future<> collect_mutations(
+        cql3::query_processor& qp,
+        ::service::group0_batch& collector,
+        const sstring query_string,
+        std::vector<data_value_or_unset> values);
 }

auth/default_authorizer.cc

@@ -9,7 +9,7 @@
  */
 
 #include "auth/default_authorizer.hh"
-#include "db/system_auth_keyspace.hh"
+#include "db/system_keyspace.hh"
 
 extern "C" {
 #include <crypt.h>
@@ -90,9 +90,10 @@ future<> default_authorizer::migrate_legacy_metadata() {
             return do_with(
                     row.get_as<sstring>("username"),
                     parse_resource(row.get_as<sstring>(RESOURCE_NAME)),
-                    [this, &row](const auto& username, const auto& r) {
+                    ::service::group0_batch::unused(),
+                    [this, &row](const auto& username, const auto& r, auto& mc) {
                 const permission_set perms = permissions::from_strings(row.get_set<sstring>(PERMISSIONS_NAME));
-                return grant(username, perms, r);
+                return grant(username, perms, r, mc);
             });
         }).finally([results] {});
     }).then([] {
@@ -103,7 +104,7 @@ future<> default_authorizer::migrate_legacy_metadata() {
     });
 }
 
-future<> default_authorizer::start() {
+future<> default_authorizer::start_legacy() {
     static const sstring create_table = fmt::format(
             "CREATE TABLE {}.{} ("
             "{} text,"
@@ -121,7 +122,7 @@ future<> default_authorizer::start() {
             90 * 24 * 60 * 60); // 3 months.
 
     return once_among_shards([this] {
-        return create_metadata_table_if_missing(
+        return create_legacy_metadata_table_if_missing(
                 PERMISSIONS_CF,
                 _qp,
                 create_table,
@@ -144,6 +145,13 @@ future<> default_authorizer::start() {
     });
 }
 
+future<> default_authorizer::start() {
+    if (legacy_mode(_qp)) {
+        return start_legacy();
+    }
+    return make_ready_future<>();
+}
+
 future<> default_authorizer::stop() {
     _as.request_abort();
     return _finished.handle_exception_type([](const sleep_aborted&) {}).handle_exception_type([](const abort_requested_exception&) {});
@@ -178,7 +186,8 @@ default_authorizer::modify(
         std::string_view role_name,
         permission_set set,
         const resource& resource,
-        std::string_view op) {
+        std::string_view op,
+        ::service::group0_batch& mc) {
     const sstring query = format("UPDATE {}.{} SET {} = {} {} ? WHERE {} = ? AND {} = ?",
             get_auth_ks_name(_qp),
             PERMISSIONS_CF,
@@ -195,17 +204,17 @@ default_authorizer::modify(
                 {permissions::to_strings(set), sstring(role_name), resource.name()},
                 cql3::query_processor::cache_internal::no).discard_result();
     }
-    co_return co_await announce_mutations(_qp, _group0_client, query,
-        {permissions::to_strings(set), sstring(role_name), resource.name()}, &_as, ::service::raft_timeout{});
+    co_await collect_mutations(_qp, mc, query,
+            {permissions::to_strings(set), sstring(role_name), resource.name()});
 }
 
 
-future<> default_authorizer::grant(std::string_view role_name, permission_set set, const resource& resource) {
-    return modify(role_name, std::move(set), resource, "+");
+future<> default_authorizer::grant(std::string_view role_name, permission_set set, const resource& resource, ::service::group0_batch& mc) {
+    return modify(role_name, std::move(set), resource, "+", mc);
 }
 
-future<> default_authorizer::revoke(std::string_view role_name, permission_set set, const resource& resource) {
-    return modify(role_name, std::move(set), resource, "-");
+future<> default_authorizer::revoke(std::string_view role_name, permission_set set, const resource& resource, ::service::group0_batch& mc) {
+    return modify(role_name, std::move(set), resource, "-", mc);
 }
 
 future<std::vector<permission_details>> default_authorizer::list_all() const {
@@ -235,7 +244,7 @@ future<std::vector<permission_details>> default_authorizer::list_all() const {
     co_return all_details;
 }
 
-future<> default_authorizer::revoke_all(std::string_view role_name) {
+future<> default_authorizer::revoke_all(std::string_view role_name, ::service::group0_batch& mc) {
     try {
         const sstring query = format("DELETE FROM {}.{} WHERE {} = ?",
                 get_auth_ks_name(_qp),
@@ -249,7 +258,7 @@ future<> default_authorizer::revoke_all(std::string_view role_name) {
                     {sstring(role_name)},
                     cql3::query_processor::cache_internal::no).discard_result();
         } else {
-            co_await announce_mutations(_qp, _group0_client, query, {sstring(role_name)}, &_as, ::service::raft_timeout{});
+            co_await collect_mutations(_qp, mc, query, {sstring(role_name)});
         }
     } catch (exceptions::request_execution_exception& e) {
         alogger.warn("CassandraAuthorizer failed to revoke all permissions of {}: {}", role_name, e);
@@ -301,51 +310,88 @@ future<> default_authorizer::revoke_all_legacy(const resource& resource) {
     });
 }
 
-future<> default_authorizer::revoke_all(const resource& resource) {
+future<> default_authorizer::revoke_all(const resource& resource, ::service::group0_batch& mc) {
     if (legacy_mode(_qp)) {
         co_return co_await revoke_all_legacy(resource);
     }
+
+    if (resource.kind() == resource_kind::data &&
+            data_resource_view(resource).is_keyspace()) {
+        revoke_all_keyspace_resources(resource, mc);
+        co_return;
+    }
+
     auto name = resource.name();
-    try {
-        auto gen = [this, name] (api::timestamp_type& t) -> mutations_generator {
-            const sstring query = format("SELECT {} FROM {}.{} WHERE {} = ? ALLOW FILTERING",
-                    ROLE_NAME,
+    auto gen = [this, name] (api::timestamp_type t) -> ::service::mutations_generator {
+        const sstring query = format("SELECT {} FROM {}.{} WHERE {} = ? ALLOW FILTERING",
+                ROLE_NAME,
+                get_auth_ks_name(_qp),
+                PERMISSIONS_CF,
+                RESOURCE_NAME);
+        auto res = co_await _qp.execute_internal(
+                query,
+                db::consistency_level::LOCAL_ONE,
+                {name},
+                cql3::query_processor::cache_internal::no);
+        for (const auto& r : *res) {
+            const sstring query = format("DELETE FROM {}.{} WHERE {} = ? AND {} = ?",
                     get_auth_ks_name(_qp),
                     PERMISSIONS_CF,
+                    ROLE_NAME,
                     RESOURCE_NAME);
-            auto res = co_await _qp.execute_internal(
+            auto muts = co_await _qp.get_mutations_internal(
                     query,
-                    db::consistency_level::LOCAL_ONE,
-                    {name},
-                    cql3::query_processor::cache_internal::no);
-            for (const auto& r : *res) {
-                const sstring query = format("DELETE FROM {}.{} WHERE {} = ? AND {} = ?",
-                        get_auth_ks_name(_qp),
-                        PERMISSIONS_CF,
-                        ROLE_NAME,
-                        RESOURCE_NAME);
-                auto muts = co_await _qp.get_mutations_internal(
-                        query,
-                        internal_distributed_query_state(),
-                        t,
-                        {r.get_as<sstring>(ROLE_NAME), name});
-                if (muts.size() != 1) {
-                    on_internal_error(alogger,
-                        format("expecting single delete mutation, got {}", muts.size()));
-                }
-                co_yield std::move(muts[0]);
+                    internal_distributed_query_state(),
+                    t,
+                    {r.get_as<sstring>(ROLE_NAME), name});
+            if (muts.size() != 1) {
+                on_internal_error(alogger,
+                    format("expecting single delete mutation, got {}", muts.size()));
             }
-        };
-        const auto timeout = ::service::raft_timeout{};
-        co_await announce_mutations_with_batching(
-                _group0_client,
-                [this, timeout](abort_source* as) { return _group0_client.start_operation(as, timeout); },
-                std::move(gen),
-                &_as,
-            timeout);
-    } catch (exceptions::request_execution_exception& e) {
-        alogger.warn("CassandraAuthorizer failed to revoke all permissions on {}: {}", name, e);
-    }
+            co_yield std::move(muts[0]);
+        }
+    };
+    mc.add_generator(std::move(gen), "default_authorizer::revoke_all");
+}
+
+void default_authorizer::revoke_all_keyspace_resources(const resource& ks_resource, ::service::group0_batch& mc) {
+    auto ks_name = ks_resource.name();
+    auto gen = [this, ks_name] (api::timestamp_type t) -> ::service::mutations_generator {
+        const sstring query = format("SELECT {}, {} FROM {}.{}",
+                ROLE_NAME,
+                RESOURCE_NAME,
+                get_auth_ks_name(_qp),
+                PERMISSIONS_CF);
+        auto res = co_await _qp.execute_internal(
+                query,
+                db::consistency_level::LOCAL_ONE,
+                {},
+                cql3::query_processor::cache_internal::no);
+        auto ks_prefix = ks_name + "/";
+        for (const auto& r : *res) {
+            auto name = r.get_as<sstring>(RESOURCE_NAME);
+            if (name != ks_name && !name.starts_with(ks_prefix)) {
+                // r doesn't represent resource related to ks_resource
+                continue;
+            }
+            const sstring query = format("DELETE FROM {}.{} WHERE {} = ? AND {} = ?",
+                    get_auth_ks_name(_qp),
+                    PERMISSIONS_CF,
+                    ROLE_NAME,
+                    RESOURCE_NAME);
+            auto muts = co_await _qp.get_mutations_internal(
+                    query,
+                    internal_distributed_query_state(),
+                    t,
+                    {r.get_as<sstring>(ROLE_NAME), name});
+            if (muts.size() != 1) {
+                on_internal_error(alogger,
+                    format("expecting single delete mutation, got {}", muts.size()));
+            }
+            co_yield std::move(muts[0]);
+        }
+    };
+    mc.add_generator(std::move(gen), "default_authorizer::revoke_all_keyspace_resources");
 }
 
 const resource_set& default_authorizer::protected_resources() const {

auth/default_authorizer.hh

@@ -47,19 +47,21 @@ public:
 
     virtual future<permission_set> authorize(const role_or_anonymous&, const resource&) const override;
 
-    virtual future<> grant(std::string_view, permission_set, const resource&) override;
+    virtual future<> grant(std::string_view, permission_set, const resource&, ::service::group0_batch&) override;
 
-    virtual future<> revoke( std::string_view, permission_set, const resource&) override;
+    virtual future<> revoke( std::string_view, permission_set, const resource&, ::service::group0_batch&) override;
 
     virtual future<std::vector<permission_details>> list_all() const override;
 
-    virtual future<> revoke_all(std::string_view) override;
+    virtual future<> revoke_all(std::string_view, ::service::group0_batch&) override;
 
-    virtual future<> revoke_all(const resource&) override;
+    virtual future<> revoke_all(const resource&, ::service::group0_batch&) override;
 
     virtual const resource_set& protected_resources() const override;
 
 private:
+    future<> start_legacy();
+
     bool legacy_metadata_exists() const;
 
     future<> revoke_all_legacy(const resource&);
@@ -68,7 +70,9 @@ private:
 
     future<> migrate_legacy_metadata();
 
-    future<> modify(std::string_view, permission_set, const resource&, std::string_view);
+    future<> modify(std::string_view, permission_set, const resource&, std::string_view, ::service::group0_batch&);
+
+    void revoke_all_keyspace_resources(const resource& ks_resource, ::service::group0_batch& mc);
 };
 
 } /* namespace auth */

auth/maintenance_socket_role_manager.cc

@@ -49,23 +49,23 @@ future<T> operation_not_supported_exception(std::string_view operation) {
         std::runtime_error(format("role manager: {} operation not supported through maintenance socket", operation)));
 }
 
-future<> maintenance_socket_role_manager::create(std::string_view role_name, const role_config&) {
+future<> maintenance_socket_role_manager::create(std::string_view role_name, const role_config&, ::service::group0_batch&) {
     return operation_not_supported_exception("CREATE");
 }
 
-future<> maintenance_socket_role_manager::drop(std::string_view role_name) {
+future<> maintenance_socket_role_manager::drop(std::string_view role_name, ::service::group0_batch& mc) {
     return operation_not_supported_exception("DROP");
 }
 
-future<> maintenance_socket_role_manager::alter(std::string_view role_name, const role_config_update&) {
+future<> maintenance_socket_role_manager::alter(std::string_view role_name, const role_config_update&, ::service::group0_batch&) {
     return operation_not_supported_exception("ALTER");
 }
 
-future<> maintenance_socket_role_manager::grant(std::string_view grantee_name, std::string_view role_name) {
+future<> maintenance_socket_role_manager::grant(std::string_view grantee_name, std::string_view role_name, ::service::group0_batch& mc) {
     return operation_not_supported_exception("GRANT");
 }
 
-future<> maintenance_socket_role_manager::revoke(std::string_view revokee_name, std::string_view role_name) {
+future<> maintenance_socket_role_manager::revoke(std::string_view revokee_name, std::string_view role_name, ::service::group0_batch& mc) {
     return operation_not_supported_exception("REVOKE");
 }
 
@@ -97,11 +97,11 @@ future<role_manager::attribute_vals> maintenance_socket_role_manager::query_attr
     return operation_not_supported_exception<role_manager::attribute_vals>("QUERY ATTRIBUTE");
 }
 
-future<> maintenance_socket_role_manager::set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value) {
+future<> maintenance_socket_role_manager::set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value, ::service::group0_batch& mc) {
     return operation_not_supported_exception("SET ATTRIBUTE");
 }
 
-future<> maintenance_socket_role_manager::remove_attribute(std::string_view role_name, std::string_view attribute_name) {
+future<> maintenance_socket_role_manager::remove_attribute(std::string_view role_name, std::string_view attribute_name, ::service::group0_batch& mc) {
     return operation_not_supported_exception("REMOVE ATTRIBUTE");
 }
 

auth/maintenance_socket_role_manager.hh

@@ -10,7 +10,7 @@
 
 #include "auth/resource.hh"
 #include "auth/role_manager.hh"
-#include "seastar/core/future.hh"
+#include <seastar/core/future.hh>
 
 namespace cql3 {
 class query_processor;
@@ -39,15 +39,15 @@ public:
 
     virtual future<> stop() override;
 
-    virtual future<> create(std::string_view role_name, const role_config&) override;
+    virtual future<> create(std::string_view role_name, const role_config&, ::service::group0_batch&) override;
 
-    virtual future<> drop(std::string_view role_name) override;
+    virtual future<> drop(std::string_view role_name, ::service::group0_batch& mc) override;
 
-    virtual future<> alter(std::string_view role_name, const role_config_update&) override;
+    virtual future<> alter(std::string_view role_name, const role_config_update&, ::service::group0_batch&) override;
 
-    virtual future<> grant(std::string_view grantee_name, std::string_view role_name) override;
+    virtual future<> grant(std::string_view grantee_name, std::string_view role_name, ::service::group0_batch& mc) override;
 
-    virtual future<> revoke(std::string_view revokee_name, std::string_view role_name) override;
+    virtual future<> revoke(std::string_view revokee_name, std::string_view role_name, ::service::group0_batch& mc) override;
 
     virtual future<role_set> query_granted(std::string_view grantee_name, recursive_role_query) override;
 
@@ -63,9 +63,9 @@ public:
 
     virtual future<role_manager::attribute_vals> query_attribute_for_all(std::string_view attribute_name) override;
 
-    virtual future<> set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value) override;
+    virtual future<> set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value, ::service::group0_batch& mc) override;
 
-    virtual future<> remove_attribute(std::string_view role_name, std::string_view attribute_name) override;
+    virtual future<> remove_attribute(std::string_view role_name, std::string_view attribute_name, ::service::group0_batch& mc) override;
 };
 
 }

auth/password_authenticator.cc

@@ -132,48 +132,48 @@ future<> password_authenticator::create_default_if_missing() {
             db::consistency_level::QUORUM,
             internal_distributed_query_state(),
             {salted_pwd, _superuser},
-            cql3::query_processor::cache_internal::no).then([](auto&&) {
-            plogger.info("Created default superuser authentication record.");
-        });
+            cql3::query_processor::cache_internal::no);
+        plogger.info("Created default superuser authentication record.");
     } else {
         co_await announce_mutations(_qp, _group0_client, query,
-            {salted_pwd, _superuser}, &_as, ::service::raft_timeout{}).then([]() {
-            plogger.info("Created default superuser authentication record.");
-        });
+            {salted_pwd, _superuser}, _as, ::service::raft_timeout{});
+        plogger.info("Created default superuser authentication record.");
     }
 }
 
 future<> password_authenticator::start() {
-     return once_among_shards([this] {
-         auto f = create_metadata_table_if_missing(
-                 meta::roles_table::name,
-                 _qp,
-                 meta::roles_table::creation_query(),
-                 _migration_manager);
+    return once_among_shards([this] {
+        _stopped = do_after_system_ready(_as, [this] {
+            return async([this] {
+                if (legacy_mode(_qp)) {
+                    _migration_manager.wait_for_schema_agreement(_qp.db().real_database(), db::timeout_clock::time_point::max(), &_as).get();
 
-         _stopped = do_after_system_ready(_as, [this] {
-             return async([this] {
-                 _migration_manager.wait_for_schema_agreement(_qp.db().real_database(), db::timeout_clock::time_point::max(), &_as).get();
+                    if (any_nondefault_role_row_satisfies(_qp, &has_salted_hash, _superuser).get()) {
+                        if (legacy_metadata_exists()) {
+                            plogger.warn("Ignoring legacy authentication metadata since nondefault data already exist.");
+                        }
 
-                 if (any_nondefault_role_row_satisfies(_qp, &has_salted_hash, _superuser).get()) {
-                     if (legacy_metadata_exists()) {
-                         plogger.warn("Ignoring legacy authentication metadata since nondefault data already exist.");
-                     }
+                        return;
+                    }
 
-                     return;
-                 }
+                    if (legacy_metadata_exists()) {
+                        migrate_legacy_metadata().get();
+                        return;
+                    }
+                }
+                create_default_if_missing().get();
+            });
+        });
 
-                 if (legacy_metadata_exists()) {
-                     migrate_legacy_metadata().get();
-                     return;
-                 }
-
-                 create_default_if_missing().get();
-             });
-         });
-
-         return f;
-     });
+        if (legacy_mode(_qp)) {
+            return create_legacy_metadata_table_if_missing(
+                    meta::roles_table::name,
+                    _qp,
+                    meta::roles_table::creation_query(),
+                    _migration_manager);
+        }
+        return make_ready_future<>();
+    });
  }
 
 future<> password_authenticator::stop() {
@@ -257,7 +257,7 @@ future<authenticated_user> password_authenticator::authenticate(
     }
 }
 
-future<> password_authenticator::create(std::string_view role_name, const authentication_options& options) {
+future<> password_authenticator::create(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) {
     if (!options.password) {
         co_return;
     }
@@ -270,12 +270,12 @@ future<> password_authenticator::create(std::string_view role_name, const authen
                 {passwords::hash(*options.password, rng_for_salt), sstring(role_name)},
                 cql3::query_processor::cache_internal::no).discard_result();
     } else {
-        co_await announce_mutations(_qp, _group0_client, query,
-                {passwords::hash(*options.password, rng_for_salt), sstring(role_name)}, &_as, ::service::raft_timeout{});
+        co_await collect_mutations(_qp, mc, query,
+                {passwords::hash(*options.password, rng_for_salt), sstring(role_name)});
     }
 }
 
-future<> password_authenticator::alter(std::string_view role_name, const authentication_options& options) {
+future<> password_authenticator::alter(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) {
     if (!options.password) {
         co_return;
     }
@@ -293,12 +293,12 @@ future<> password_authenticator::alter(std::string_view role_name, const authent
                 {passwords::hash(*options.password, rng_for_salt), sstring(role_name)},
                 cql3::query_processor::cache_internal::no).discard_result();
     } else {
-        co_await announce_mutations(_qp, _group0_client, query,
-            {passwords::hash(*options.password, rng_for_salt), sstring(role_name)}, &_as, ::service::raft_timeout{});
+        co_await collect_mutations(_qp, mc, query,
+                {passwords::hash(*options.password, rng_for_salt), sstring(role_name)});
     }
 }
 
-future<> password_authenticator::drop(std::string_view name) {
+future<> password_authenticator::drop(std::string_view name, ::service::group0_batch& mc) {
     const sstring query = format("DELETE {} FROM {}.{} WHERE {} = ?",
             SALTED_HASH,
             get_auth_ks_name(_qp),
@@ -311,7 +311,7 @@ future<> password_authenticator::drop(std::string_view name) {
                 {sstring(name)},
                 cql3::query_processor::cache_internal::no).discard_result();
     } else {
-        co_await announce_mutations(_qp, _group0_client, query, {sstring(name)}, &_as, ::service::raft_timeout{});
+        co_await collect_mutations(_qp, mc, query, {sstring(name)});
     }
 }
 
@@ -329,7 +329,7 @@ const resource_set& password_authenticator::protected_resources() const {
         credentials_map credentials{};
         credentials[USERNAME_KEY] = sstring(username);
         credentials[PASSWORD_KEY] = sstring(password);
-        return this->authenticate(credentials);
+        return authenticate(credentials);
     });
 }
 

auth/password_authenticator.hh

@@ -64,11 +64,11 @@ public:
 
     virtual future<authenticated_user> authenticate(const credentials_map& credentials) const override;
 
-    virtual future<> create(std::string_view role_name, const authentication_options& options) override;
+    virtual future<> create(std::string_view role_name, const authentication_options& options, ::service::group0_batch& mc) override;
 
-    virtual future<> alter(std::string_view role_name, const authentication_options& options) override;
+    virtual future<> alter(std::string_view role_name, const authentication_options& options, ::service::group0_batch&) override;
 
-    virtual future<> drop(std::string_view role_name) override;
+    virtual future<> drop(std::string_view role_name, ::service::group0_batch&) override;
 
     virtual future<custom_options> query_custom_options(std::string_view role_name) const override;
 

auth/permissions_cache.cc

@@ -8,6 +8,7 @@
 
 #include "auth/permissions_cache.hh"
 
+#include <fmt/ranges.h>
 #include "auth/authorizer.hh"
 #include "auth/service.hh"
 

auth/permissions_cache.hh

@@ -13,8 +13,6 @@
 
 #include <fmt/core.h>
 #include <seastar/core/future.hh>
-#include <seastar/core/shared_ptr.hh>
-#include <seastar/core/sstring.hh>
 
 #include "auth/permission.hh"
 #include "auth/resource.hh"

auth/resource.cc

@@ -19,11 +19,16 @@
 #include <boost/algorithm/string/split.hpp>
 #include <boost/algorithm/string/classification.hpp>
 
+#include "cql3/functions/aggregate_function.hh"
+#include "cql3/functions/user_function.hh"
 #include "cql3/util.hh"
 #include "db/marshal/type_parser.hh"
+#include "log.hh"
 
 namespace auth {
 
+static logging::logger logger("auth_resource");
+
 static const std::unordered_map<resource_kind, std::string_view> roots{
         {resource_kind::data, "data"},
         {resource_kind::role, "roles"},
@@ -223,6 +228,15 @@ static sstring decoded_signature_string(std::string_view encoded_signature) {
             }), ", "));
 }
 
+resource make_functions_resource(const cql3::functions::function& f) {
+    if (!dynamic_cast<const cql3::functions::user_function*>(&f) &&
+            !dynamic_cast<const cql3::functions::aggregate_function*>(&f)) {
+        on_internal_error(logger, "unsuppported function type");
+    }
+    auto&& sig = auth::encode_signature(f.name().name, f.arg_types());
+    return make_functions_resource(f.name().keyspace, sig);
+}
+
 functions_resource_view::functions_resource_view(const resource& r) : _resource(r) {
     if (r._kind != resource_kind::functions) {
         throw resource_kind_mismatch(resource_kind::functions, r._kind);
@@ -282,6 +296,10 @@ std::optional<std::string_view> data_resource_view::keyspace() const {
     return _resource._parts[1];
 }
 
+bool data_resource_view::is_keyspace() const {
+    return _resource._parts.size() == 2;
+}
+
 std::optional<std::string_view> data_resource_view::table() const {
     if (_resource._parts.size() <= 2) {
         return {};

auth/resource.hh

@@ -18,11 +18,11 @@
 #include <unordered_set>
 
 #include <fmt/core.h>
-#include <boost/range/adaptor/transformed.hpp>
 #include <seastar/core/print.hh>
 #include <seastar/core/sstring.hh>
 
 #include "auth/permission.hh"
+#include "cql3/functions/function.hh"
 #include "seastarx.hh"
 #include "utils/hash.hh"
 #include "utils/small_vector.hh"
@@ -41,6 +41,28 @@ enum class resource_kind {
     data, role, service_level, functions
 };
 
+}
+
+template <>
+struct fmt::formatter<auth::resource_kind> : fmt::formatter<string_view> {
+    template <typename FormatContext>
+    auto format(const auth::resource_kind kind, FormatContext& ctx) const {
+        using enum auth::resource_kind;
+        switch (kind) {
+        case data:
+            return formatter<string_view>::format("data", ctx);
+        case role:
+            return formatter<string_view>::format("role", ctx);
+        case service_level:
+            return formatter<string_view>::format("service_level", ctx);
+        case functions:
+            return formatter<string_view>::format("functions", ctx);
+        }
+        std::abort();
+    }
+};
+
+namespace auth {
 ///
 /// Type tag for constructing data resources.
 ///
@@ -144,6 +166,7 @@ public:
     explicit data_resource_view(const resource& r);
 
     std::optional<std::string_view> keyspace() const;
+    bool is_keyspace() const;
 
     std::optional<std::string_view> table() const;
 };
@@ -240,31 +263,14 @@ inline resource make_functions_resource(std::string_view keyspace, std::string_v
     return resource(functions_resource_t{}, keyspace, function_name, function_signature);
 }
 
+resource make_functions_resource(const cql3::functions::function& f);
+
 sstring encode_signature(std::string_view name, std::vector<data_type> args);
 
 std::pair<sstring, std::vector<data_type>> decode_signature(std::string_view encoded_signature);
 
 }
 
-template <>
-struct fmt::formatter<auth::resource_kind> : fmt::formatter<std::string_view> {
-    template <typename FormatContext>
-    auto format(const auth::resource_kind kind, FormatContext& ctx) const {
-        using enum auth::resource_kind;
-        switch (kind) {
-        case data:
-            return formatter<std::string_view>::format("data", ctx);
-        case role:
-            return formatter<std::string_view>::format("role", ctx);
-        case service_level:
-            return formatter<std::string_view>::format("service_level", ctx);
-        case functions:
-            return formatter<std::string_view>::format("functions", ctx);
-        }
-        std::abort();
-    }
-};
-
 namespace std {
 
 template <>

auth/role_manager.hh

@@ -20,6 +20,7 @@
 #include "auth/resource.hh"
 #include "seastarx.hh"
 #include "exceptions/exceptions.hh"
+#include "service/raft/raft_group0_client.hh"
 
 namespace auth {
 
@@ -107,17 +108,17 @@ public:
     ///
     /// \returns an exceptional future with \ref role_already_exists for a role that has previously been created.
     ///
-    virtual future<> create(std::string_view role_name, const role_config&) = 0;
+    virtual future<> create(std::string_view role_name, const role_config&, ::service::group0_batch&) = 0;
 
     ///
     /// \returns an exceptional future with \ref nonexistant_role if the role does not exist.
     ///
-    virtual future<> drop(std::string_view role_name) = 0;
+    virtual future<> drop(std::string_view role_name, ::service::group0_batch&) = 0;
 
     ///
     /// \returns an exceptional future with \ref nonexistant_role if the role does not exist.
     ///
-    virtual future<> alter(std::string_view role_name, const role_config_update&) = 0;
+    virtual future<> alter(std::string_view role_name, const role_config_update&, ::service::group0_batch&) = 0;
 
     ///
     /// Grant `role_name` to `grantee_name`.
@@ -127,7 +128,7 @@ public:
     /// \returns an exceptional future with \ref role_already_included if granting the role would be redundant, or
     /// create a cycle.
     ///
-    virtual future<> grant(std::string_view grantee_name, std::string_view role_name) = 0;
+    virtual future<> grant(std::string_view grantee_name, std::string_view role_name, ::service::group0_batch& mc) = 0;
 
     ///
     /// Revoke `role_name` from `revokee_name`.
@@ -136,7 +137,7 @@ public:
     ///
     /// \returns an exceptional future with \ref revoke_ungranted_role if the role was not granted.
     ///
-    virtual future<> revoke(std::string_view revokee_name, std::string_view role_name) = 0;
+    virtual future<> revoke(std::string_view revokee_name, std::string_view role_name, ::service::group0_batch& mc) = 0;
 
     ///
     /// \returns an exceptional future with \ref nonexistant_role if the role does not exist.
@@ -170,12 +171,12 @@ public:
     /// Sets `attribute_name` with `attribute_value` for `role_name`.
     /// \returns an exceptional future with nonexistant_role if the role does not exist.
     ///
-    virtual future<> set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value) = 0;
+    virtual future<> set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value, ::service::group0_batch& mc) = 0;
 
     /// Removes `attribute_name` for `role_name`.
     /// \returns an exceptional future with nonexistant_role if the role does not exist.
     /// \note: This is a no-op if the role does not have the named attribute set.
     ///
-    virtual future<> remove_attribute(std::string_view role_name, std::string_view attribute_name) = 0;
+    virtual future<> remove_attribute(std::string_view role_name, std::string_view attribute_name, ::service::group0_batch& mc) = 0;
 };
 }

auth/role_or_anonymous.cc

@@ -10,11 +10,6 @@
 
 namespace auth {
 
-std::ostream& operator<<(std::ostream& os, const role_or_anonymous& mr) {
-    os << mr.name.value_or("<anonymous>");
-    return os;
-}
-
 bool is_anonymous(const role_or_anonymous& mr) noexcept {
     return !mr.name.has_value();
 }

auth/roles-metadata.hh

@@ -8,6 +8,7 @@
 
 #pragma once
 
+#include <optional>
 #include <string_view>
 #include <functional>
 

auth/service.cc

@@ -6,6 +6,7 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
+#include <exception>
 #include <seastar/core/coroutine.hh>
 #include "auth/resource.hh"
 #include "auth/service.hh"
@@ -28,11 +29,12 @@
 #include "db/config.hh"
 #include "db/consistency_level_type.hh"
 #include "db/functions/function_name.hh"
-#include "db/system_auth_keyspace.hh"
 #include "log.hh"
 #include "schema/schema_fwd.hh"
-#include "seastar/core/future.hh"
+#include <seastar/core/future.hh>
+#include <seastar/coroutine/parallel_for_each.hh>
 #include "service/migration_manager.hh"
+#include "service/raft/raft_group0_client.hh"
 #include "timestamp.hh"
 #include "utils/class_registrator.hh"
 #include "locator/abstract_replication_strategy.hh"
@@ -55,9 +57,10 @@ static logging::logger log("auth_service");
 
 class auth_migration_listener final : public ::service::migration_listener {
     authorizer& _authorizer;
+    cql3::query_processor& _qp;
 
 public:
-    explicit auth_migration_listener(authorizer& a) : _authorizer(a) {
+    explicit auth_migration_listener(authorizer& a, cql3::query_processor& qp) : _authorizer(a),  _qp(qp) {
     }
 
 private:
@@ -77,27 +80,33 @@ private:
     void on_update_tablet_metadata() override {}
 
     void on_drop_keyspace(const sstring& ks_name) override {
+        if (!legacy_mode(_qp)) {
+            // in non legacy path revoke is part of schema change statement execution
+            return;
+        }
         // Do it in the background.
-        (void)_authorizer.revoke_all(
-                auth::make_data_resource(ks_name)).handle_exception_type([](const unsupported_authorization_operation&) {
-            // Nothing.
+        (void)do_with(::service::group0_batch::unused(), [this, &ks_name] (auto& mc) mutable {
+            return _authorizer.revoke_all(auth::make_data_resource(ks_name), mc);
         }).handle_exception([] (std::exception_ptr e) {
             log.error("Unexpected exception while revoking all permissions on dropped keyspace: {}", e);
         });
-        (void)_authorizer.revoke_all(
-            auth::make_functions_resource(ks_name)).handle_exception_type([](const unsupported_authorization_operation&) {
-            // Nothing.
+
+        (void)do_with(::service::group0_batch::unused(), [this, &ks_name] (auto& mc) mutable {
+            return _authorizer.revoke_all(auth::make_functions_resource(ks_name), mc);
         }).handle_exception([] (std::exception_ptr e) {
             log.error("Unexpected exception while revoking all permissions on functions in dropped keyspace: {}", e);
         });
     }
 
     void on_drop_column_family(const sstring& ks_name, const sstring& cf_name) override {
+        if (!legacy_mode(_qp)) {
+            // in non legacy path revoke is part of schema change statement execution
+            return;
+        }
         // Do it in the background.
-        (void)_authorizer.revoke_all(
-                auth::make_data_resource(
-                        ks_name, cf_name)).handle_exception_type([](const unsupported_authorization_operation&) {
-            // Nothing.
+        (void)do_with(::service::group0_batch::unused(), [this, &ks_name, &cf_name] (auto& mc) mutable {
+            return _authorizer.revoke_all(
+                    auth::make_data_resource(ks_name, cf_name), mc);
         }).handle_exception([] (std::exception_ptr e) {
             log.error("Unexpected exception while revoking all permissions on dropped table: {}", e);
         });
@@ -105,17 +114,26 @@ private:
 
     void on_drop_user_type(const sstring& ks_name, const sstring& type_name) override {}
     void on_drop_function(const sstring& ks_name, const sstring& function_name) override {
-        (void)_authorizer.revoke_all(
-            auth::make_functions_resource(ks_name, function_name)).handle_exception_type([](const unsupported_authorization_operation&) {
-            // Nothing.
+        if (!legacy_mode(_qp)) {
+            // in non legacy path revoke is part of schema change statement execution
+            return;
+        }
+        // Do it in the background.
+        (void)do_with(::service::group0_batch::unused(), [this, &ks_name, &function_name] (auto& mc) mutable {
+            return _authorizer.revoke_all(
+                    auth::make_functions_resource(ks_name, function_name), mc);
         }).handle_exception([] (std::exception_ptr e) {
             log.error("Unexpected exception while revoking all permissions on dropped function: {}", e);
         });
     }
     void on_drop_aggregate(const sstring& ks_name, const sstring& aggregate_name) override {
-        (void)_authorizer.revoke_all(
-            auth::make_functions_resource(ks_name, aggregate_name)).handle_exception_type([](const unsupported_authorization_operation&) {
-            // Nothing.
+        if (!legacy_mode(_qp)) {
+            // in non legacy path revoke is part of schema change statement execution
+            return;
+        }
+        (void)do_with(::service::group0_batch::unused(), [this, &ks_name, &aggregate_name] (auto& mc) mutable {
+            return _authorizer.revoke_all(
+                    auth::make_functions_resource(ks_name, aggregate_name), mc);
         }).handle_exception([] (std::exception_ptr e) {
             log.error("Unexpected exception while revoking all permissions on dropped aggregate: {}", e);
         });
@@ -134,6 +152,7 @@ static future<> validate_role_exists(const service& ser, std::string_view role_n
 service::service(
         utils::loading_cache_config c,
         cql3::query_processor& qp,
+        ::service::raft_group0_client& g0,
         ::service::migration_notifier& mn,
         std::unique_ptr<authorizer> z,
         std::unique_ptr<authenticator> a,
@@ -142,11 +161,12 @@ service::service(
             : _loading_cache_config(std::move(c))
             , _permissions_cache(nullptr)
             , _qp(qp)
+            , _group0_client(g0)
             , _mnotifier(mn)
             , _authorizer(std::move(z))
             , _authenticator(std::move(a))
             , _role_manager(std::move(r))
-            , _migration_listener(std::make_unique<auth_migration_listener>(*_authorizer))
+            , _migration_listener(std::make_unique<auth_migration_listener>(*_authorizer, qp))
             , _permissions_cache_cfg_cb([this] (uint32_t) { (void) _permissions_cache_config_action.trigger_later(); })
             , _permissions_cache_config_action([this] { update_cache_config(); return make_ready_future<>(); })
             , _permissions_cache_max_entries_observer(_qp.db().get_config().permissions_cache_max_entries.observe(_permissions_cache_cfg_cb))
@@ -165,6 +185,7 @@ service::service(
             : service(
                       std::move(c),
                       qp,
+                      g0,
                       mn,
                       create_object<authorizer>(sc.authorizer_java_name, qp, g0, mm),
                       create_object<authenticator>(sc.authenticator_java_name, qp, g0, mm),
@@ -172,7 +193,7 @@ service::service(
                       used_by_maintenance_socket) {
 }
 
-future<> service::create_keyspace_if_missing(::service::migration_manager& mm) const {
+future<> service::create_legacy_keyspace_if_missing(::service::migration_manager& mm) const {
     assert(this_shard_id() == 0); // once_among_shards makes sure a function is executed on shard 0 only
     auto db = _qp.db();
 
@@ -204,8 +225,12 @@ future<> service::start(::service::migration_manager& mm, db::system_keyspace& s
     // version is set in query processor to be easily available in various places we call auth::legacy_mode check.
     _qp.auth_version = auth_version;
     if (!_used_by_maintenance_socket) {
+        // this legacy keyspace is only used by cqlsh
+        // it's needed when executing `list roles` or `list users`
+        // it doesn't affect anything except that cqlsh fails if keyspace
+        // is not found
         co_await once_among_shards([this, &mm] {
-            return create_keyspace_if_missing(mm);
+            return create_legacy_keyspace_if_missing(mm);
         });
     }
     co_await _role_manager->start();
@@ -218,6 +243,7 @@ future<> service::start(::service::migration_manager& mm, db::system_keyspace& s
 }
 
 future<> service::stop() {
+    _as.request_abort();
     // Only one of the shards has the listener registered, but let's try to
     // unregister on each one just to make sure.
     return _mnotifier.unregister_listener(_migration_listener.get()).then([this] {
@@ -248,104 +274,86 @@ void service::reset_authorization_cache() {
     _qp.reset_cache();
 }
 
-future<bool> service::has_existing_legacy_users() const {
-    if (!_qp.db().has_schema(meta::legacy::AUTH_KS, meta::legacy::USERS_CF)) {
-        return make_ready_future<bool>(false);
-    }
-
-    static const sstring default_user_query = format("SELECT * FROM {}.{} WHERE {} = ?",
-            meta::legacy::AUTH_KS,
-            meta::legacy::USERS_CF,
-            meta::user_name_col_name);
-
-    static const sstring all_users_query = format("SELECT * FROM {}.{} LIMIT 1",
-            meta::legacy::AUTH_KS,
-            meta::legacy::USERS_CF);
-
-    // This logic is borrowed directly from Apache Cassandra. By first checking for the presence of the default user, we
-    // can potentially avoid doing a range query with a high consistency level.
-
-    return _qp.execute_internal(
-            default_user_query,
-            db::consistency_level::ONE,
-            {meta::DEFAULT_SUPERUSER_NAME},
-            cql3::query_processor::cache_internal::yes).then([this](auto results) {
-        if (!results->empty()) {
-            return make_ready_future<bool>(true);
-        }
-
-        return _qp.execute_internal(
-                default_user_query,
-                db::consistency_level::QUORUM,
-                {meta::DEFAULT_SUPERUSER_NAME},
-                cql3::query_processor::cache_internal::yes).then([this](auto results) {
-            if (!results->empty()) {
-                return make_ready_future<bool>(true);
-            }
-
-            return _qp.execute_internal(
-                    all_users_query,
-                    db::consistency_level::QUORUM,
-                    cql3::query_processor::cache_internal::no).then([](auto results) {
-                return make_ready_future<bool>(!results->empty());
-            });
-        });
-    });
-}
-
 future<permission_set>
 service::get_uncached_permissions(const role_or_anonymous& maybe_role, const resource& r) const {
     if (is_anonymous(maybe_role)) {
-        return _authorizer->authorize(maybe_role, r);
+        co_return co_await _authorizer->authorize(maybe_role, r);
     }
-
     const std::string_view role_name = *maybe_role.name;
-
-    return has_superuser(role_name).then([this, role_name, &r](bool superuser) {
-        if (superuser) {
-            return make_ready_future<permission_set>(r.applicable_permissions());
-        }
-
-        //
-        // Aggregate the permissions from all granted roles.
-        //
-
-        return do_with(permission_set(), [this, role_name, &r](auto& all_perms) {
-            return get_roles(role_name).then([this, &r, &all_perms](role_set all_roles) {
-                return do_with(std::move(all_roles), [this, &r, &all_perms](const auto& all_roles) {
-                    return parallel_for_each(all_roles, [this, &r, &all_perms](std::string_view role_name) {
-                        return _authorizer->authorize(role_name, r).then([&all_perms](permission_set perms) {
-                            all_perms = permission_set::from_mask(all_perms.mask() | perms.mask());
-                        });
-                    });
-                });
-            }).then([&all_perms] {
-                return all_perms;
-            });
-        });
+    auto all_roles = co_await get_roles(role_name);
+    auto superuser = co_await has_superuser(role_name, all_roles);
+    if (superuser) {
+        co_return r.applicable_permissions();
+    }
+    // Aggregate the permissions from all granted roles.
+    permission_set all_perms;
+    co_await coroutine::parallel_for_each(all_roles, [this, &r, &all_perms](std::string_view role_name) -> future<> {
+        auto perms = co_await _authorizer->authorize(role_name, r);
+        all_perms = permission_set::from_mask(all_perms.mask() | perms.mask());
     });
+    co_return std::move(all_perms);
 }
 
 future<permission_set> service::get_permissions(const role_or_anonymous& maybe_role, const resource& r) const {
     return _permissions_cache->get(maybe_role, r);
 }
 
+future<bool> service::has_superuser(std::string_view role_name, const role_set& roles) const {
+    for (const auto& role : roles) {
+        if (co_await _role_manager->is_superuser(role)) {
+            co_return true;
+        }
+    }
+    co_return false;
+}
+
 future<bool> service::has_superuser(std::string_view role_name) const {
-    return this->get_roles(std::move(role_name)).then([this](role_set roles) {
-        return do_with(std::move(roles), [this](const role_set& roles) {
-            return do_with(false, roles.begin(), [this, &roles](bool& any_super, auto& iter) {
-                return do_until(
-                        [&roles, &any_super, &iter] { return any_super || (iter == roles.end()); },
-                        [this, &any_super, &iter] {
-                    return _role_manager->is_superuser(*iter++).then([&any_super](bool super) {
-                        any_super = super;
-                    });
-                }).then([&any_super] {
-                    return any_super;
-                });
-            });
-        });
-    });
+    auto roles = co_await get_roles(role_name);
+    co_return co_await has_superuser(role_name, roles);
+}
+
+static void validate_authentication_options_are_supported(
+        const authentication_options& options,
+        const authentication_option_set& supported) {
+    const auto check = [&supported](authentication_option k) {
+        if (!supported.contains(k)) {
+            throw unsupported_authentication_option(k);
+        }
+    };
+
+    if (options.password) {
+        check(authentication_option::password);
+    }
+
+    if (options.options) {
+        check(authentication_option::options);
+    }
+}
+
+future<> service::create_role(std::string_view name,
+        const role_config& config,
+        const authentication_options& options,
+        ::service::group0_batch& mc) const {
+    co_await underlying_role_manager().create(name, config, mc);
+    if (!auth::any_authentication_options(options)) {
+        co_return;
+    }
+    std::exception_ptr ep;
+    try {
+        validate_authentication_options_are_supported(options,
+                underlying_authenticator().supported_options());
+        co_await underlying_authenticator().create(name, options, mc);
+    } catch (...) {
+        ep = std::current_exception();
+    }
+    if (ep) {
+        // Rollback only in legacy mode as normally mutations won't be
+        // applied in case exception is raised
+        if (legacy_mode(_qp)) {
+            co_await underlying_role_manager().drop(name, mc);
+        }
+        std::rethrow_exception(std::move(ep));
+    }
 }
 
 future<role_set> service::get_roles(std::string_view role_name) const {
@@ -402,7 +410,7 @@ future<bool> service::exists(const resource& r) const {
                 return make_ready_future<bool>(db.has_keyspace(sstring(*keyspace)));
             }
             auto [name, function_args] = auth::decode_signature(*function_signature);
-            return make_ready_future<bool>(cql3::functions::functions::find(db::functions::function_name{sstring(*keyspace), name}, function_args));
+            return make_ready_future<bool>(cql3::functions::instance().find(db::functions::function_name{sstring(*keyspace), name}, function_args));
         }
     }
 
@@ -436,15 +444,6 @@ future<permission_set> get_permissions(const service& ser, const authenticated_u
     });
 }
 
-bool is_enforcing(const service& ser)  {
-    const bool enforcing_authorizer = ser.underlying_authorizer().qualified_java_name() != allow_all_authorizer_name;
-
-    const bool enforcing_authenticator = ser.underlying_authenticator().qualified_java_name()
-            != allow_all_authenticator_name;
-
-    return enforcing_authorizer || enforcing_authenticator;
-}
-
 bool is_protected(const service& ser, command_desc cmd) noexcept {
     if (cmd.type_ == command_desc::type::ALTER_WITH_OPTS) {
         return false; // Table attributes are OK to modify; see #7057.
@@ -454,84 +453,45 @@ bool is_protected(const service& ser, command_desc cmd) noexcept {
             || ser.underlying_authorizer().protected_resources().contains(cmd.resource);
 }
 
-static void validate_authentication_options_are_supported(
-        const authentication_options& options,
-        const authentication_option_set& supported) {
-    const auto check = [&supported](authentication_option k) {
-        if (!supported.contains(k)) {
-            throw unsupported_authentication_option(k);
-        }
-    };
-
-    if (options.password) {
-        check(authentication_option::password);
-    }
-
-    if (options.options) {
-        check(authentication_option::options);
-    }
-}
-
-
 future<> create_role(
         const service& ser,
         std::string_view name,
         const role_config& config,
-        const authentication_options& options) {
-    return ser.underlying_role_manager().create(name, config).then([&ser, name, &options] {
-        if (!auth::any_authentication_options(options)) {
-            return make_ready_future<>();
-        }
-
-        return futurize_invoke(
-                &validate_authentication_options_are_supported,
-                options,
-                ser.underlying_authenticator().supported_options()).then([&ser, name, &options] {
-            return ser.underlying_authenticator().create(name, options);
-        }).handle_exception([&ser, name](std::exception_ptr ep) {
-            // Roll-back.
-            return ser.underlying_role_manager().drop(name).then([ep = std::move(ep)] {
-                std::rethrow_exception(ep);
-            });
-        });
-    });
+        const authentication_options& options,
+        ::service::group0_batch& mc) {
+    return ser.create_role(name, config, options, mc);
 }
 
 future<> alter_role(
         const service& ser,
         std::string_view name,
         const role_config_update& config_update,
-        const authentication_options& options) {
-    return ser.underlying_role_manager().alter(name, config_update).then([&ser, name, &options] {
-        if (!any_authentication_options(options)) {
-            return make_ready_future<>();
-        }
-
-        return futurize_invoke(
-                &validate_authentication_options_are_supported,
-                options,
-                ser.underlying_authenticator().supported_options()).then([&ser, name, &options] {
-            return ser.underlying_authenticator().alter(name, options);
-        });
-    });
+        const authentication_options& options,
+        ::service::group0_batch& mc) {
+    co_await ser.underlying_role_manager().alter(name, config_update, mc);
+    if (!any_authentication_options(options)) {
+        co_return;
+    }
+    validate_authentication_options_are_supported(options,
+            ser.underlying_authenticator().supported_options());
+    co_await ser.underlying_authenticator().alter(name, options, mc);
 }
 
-future<> drop_role(const service& ser, std::string_view name) {
-    return do_with(make_role_resource(name), [&ser, name](const resource& r) {
-        auto& a = ser.underlying_authorizer();
+future<> drop_role(const service& ser, std::string_view name, ::service::group0_batch& mc) {
+    auto& a = ser.underlying_authorizer();
+    auto r = make_role_resource(name);
+    co_await a.revoke_all(name, mc);
+    co_await a.revoke_all(r, mc);
+    co_await ser.underlying_authenticator().drop(name, mc);
+    co_await ser.underlying_role_manager().drop(name, mc);
+}
 
-        return when_all_succeed(
-                a.revoke_all(name),
-                a.revoke_all(r))
-                    .discard_result()
-                    .handle_exception_type([](const unsupported_authorization_operation&) {
-            // Nothing.
-        });
-    }).then([&ser, name] {
-        return ser.underlying_authenticator().drop(name);
-    }).then([&ser, name] {
-        return ser.underlying_role_manager().drop(name);
-    });
+future<> grant_role(const service& ser, std::string_view grantee_name, std::string_view role_name, ::service::group0_batch& mc) {
+    return ser.underlying_role_manager().grant(grantee_name, role_name, mc);
+}
+
+future<> revoke_role(const service& ser, std::string_view revokee_name, std::string_view role_name, ::service::group0_batch& mc) {
+    return ser.underlying_role_manager().revoke(revokee_name, role_name, mc);
 }
 
 future<bool> has_role(const service& ser, std::string_view grantee, std::string_view name) {
@@ -549,35 +509,47 @@ future<bool> has_role(const service& ser, const authenticated_user& u, std::stri
     return has_role(ser, *u.name, name);
 }
 
+future<> set_attribute(const service& ser, std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value, ::service::group0_batch& mc) {
+    return ser.underlying_role_manager().set_attribute(role_name, attribute_name, attribute_value, mc);
+}
+
+future<> remove_attribute(const service& ser, std::string_view role_name, std::string_view attribute_name, ::service::group0_batch& mc) {
+    return ser.underlying_role_manager().remove_attribute(role_name, attribute_name, mc);
+}
+
 future<> grant_permissions(
         const service& ser,
         std::string_view role_name,
         permission_set perms,
-        const resource& r) {
-    return validate_role_exists(ser, role_name).then([&ser, role_name, perms, &r] {
-        return ser.underlying_authorizer().grant(role_name, perms, r);
-    });
+        const resource& r,
+        ::service::group0_batch& mc) {
+    co_await validate_role_exists(ser, role_name);
+    co_await ser.underlying_authorizer().grant(role_name, perms, r, mc);
 }
 
-future<> grant_applicable_permissions(const service& ser, std::string_view role_name, const resource& r) {
-    return grant_permissions(ser, role_name, r.applicable_permissions(), r);
+future<> grant_applicable_permissions(const service& ser, std::string_view role_name, const resource& r, ::service::group0_batch& mc) {
+    return grant_permissions(ser, role_name, r.applicable_permissions(), r, mc);
 }
-future<> grant_applicable_permissions(const service& ser, const authenticated_user& u, const resource& r) {
+
+future<> grant_applicable_permissions(const service& ser, const authenticated_user& u, const resource& r, ::service::group0_batch& mc) {
     if (is_anonymous(u)) {
         return make_ready_future<>();
     }
-
-    return grant_applicable_permissions(ser, *u.name, r);
+    return grant_applicable_permissions(ser, *u.name, r, mc);
 }
 
 future<> revoke_permissions(
         const service& ser,
         std::string_view role_name,
         permission_set perms,
-        const resource& r) {
-    return validate_role_exists(ser, role_name).then([&ser, role_name, perms, &r] {
-        return ser.underlying_authorizer().revoke(role_name, perms, r);
-    });
+        const resource& r,
+        ::service::group0_batch& mc) {
+    co_await validate_role_exists(ser, role_name);
+    co_await ser.underlying_authorizer().revoke(role_name, perms, r, mc);
+}
+
+future<> revoke_all(const service& ser, const resource& r, ::service::group0_batch& mc) {
+    return ser.underlying_authorizer().revoke_all(r, mc);
 }
 
 future<std::vector<permission_details>> list_filtered_permissions(
@@ -634,10 +606,14 @@ future<std::vector<permission_details>> list_filtered_permissions(
     });
 }
 
+future<> commit_mutations(service& ser, ::service::group0_batch&& mc) {
+    return ser.commit_mutations(std::move(mc));
+}
+
 future<> migrate_to_auth_v2(db::system_keyspace& sys_ks, ::service::raft_group0_client& g0, start_operation_func_t start_operation_func, abort_source& as) {
     // FIXME: if this function fails it may leave partial data in the new tables
     // that should be cleared
-    auto gen = [&sys_ks] (api::timestamp_type& ts) -> mutations_generator {
+    auto gen = [&sys_ks] (api::timestamp_type ts) -> ::service::mutations_generator {
         auto& qp = sys_ks.query_processor();
         for (const auto& cf_name : std::vector<sstring>{
                 "roles", "role_members", "role_attributes", "role_permissions"}) {
@@ -685,7 +661,7 @@ future<> migrate_to_auth_v2(db::system_keyspace& sys_ks, ::service::raft_group0_
                 }
                 auto muts = co_await qp.get_mutations_internal(
                         format("INSERT INTO {}.{} ({}) VALUES ({})",
-                                db::system_auth_keyspace::NAME,
+                                db::system_keyspace::NAME,
                                 cf_name,
                                 col_names_str,
                                 val_binders_str),
@@ -700,12 +676,12 @@ future<> migrate_to_auth_v2(db::system_keyspace& sys_ks, ::service::raft_group0_
             }
         }
         co_yield co_await sys_ks.make_auth_version_mutation(ts,
-                db::system_auth_keyspace::version_t::v2);
+                db::system_keyspace::auth_version_t::v2);
     };
     co_await announce_mutations_with_batching(g0,
             start_operation_func,
             std::move(gen),
-            &as,
+            as,
             std::nullopt);
 }