./github/scripts/auto-backport.py: don't remove backport label when backport process has an error

Today, when the `Fixes` prefix is missing or the developer is not a collaborator with `scylladbbot` we remove the backport labels to prevent the process from starting and notifying the developers.

Developers are worried that removing these backport labels will cause us to forget we need to do these backports. @nyh suggested to add a `scylladbbot/backport_error` label instead

Applied those changes, so when a `Fixes` prefix is missing we will add a `scylladbbot/backport_error` label and stop the process

When a user doesn't accept the invite we will still open the PR but he will not be assigned and will not be able to edit the branch when we have conflicts

Fixes: https://github.com/scylladb/scylla-pkg/issues/4898
Fixes: https://github.com/scylladb/scylla-pkg/issues/4897

.github/scripts/auto-backport.py

@@ -29,10 +29,11 @@ def parse_args():
     parser.add_argument('--commits', default=None, type=str, help='Range of promoted commits.')
     parser.add_argument('--pull-request', type=int, help='Pull request number to be backported')
     parser.add_argument('--head-commit', type=str, required=is_pull_request(), help='The HEAD of target branch after the pull request specified by --pull-request is merged')
+    parser.add_argument('--github-event', type=str, help='Get GitHub event type')
     return parser.parse_args()
 
 
-def create_pull_request(repo, new_branch_name, base_branch_name, pr, backport_pr_title, commits, is_draft=False):
+def create_pull_request(repo, new_branch_name, base_branch_name, pr, backport_pr_title, commits, is_draft, is_collaborator):
     pr_body = f'{pr.body}\n\n'
     for commit in commits:
         pr_body += f'- (cherry picked from commit {commit})\n\n'
@@ -46,11 +47,12 @@ def create_pull_request(repo, new_branch_name, base_branch_name, pr, backport_pr
             draft=is_draft
         )
         logging.info(f"Pull request created: {backport_pr.html_url}")
-        backport_pr.add_to_assignees(pr.user)
+        if is_collaborator:
+            backport_pr.add_to_assignees(pr.user)
         if is_draft:
             backport_pr.add_to_labels("conflicts")
             pr_comment = f"@{pr.user.login} - This PR was marked as draft because it has conflicts\n"
-            pr_comment += "Please resolve them and remove the 'conflicts' label. The PR will be made ready for review automatically."
+            pr_comment += "Please resolve them and mark this PR as ready for review"
             backport_pr.create_issue_comment(pr_comment)
         logging.info(f"Assigned PR to original author: {pr.user}")
         return backport_pr
@@ -92,18 +94,7 @@ def get_pr_commits(repo, pr, stable_branch, start_commit=None):
     return commits
 
 
-def create_pr_comment_and_remove_label(pr, comment_body):
-    labels = pr.get_labels()
-    pattern = re.compile(r"backport/\d+\.\d+$")
-    for label in labels:
-        if pattern.match(label.name):
-            print(f"Removing label: {label.name}")
-            comment_body += f'- {label.name}\n'
-            pr.remove_from_labels(label)
-    pr.create_issue_comment(comment_body)
-
-
-def backport(repo, pr, version, commits, backport_base_branch):
+def backport(repo, pr, version, commits, backport_base_branch, is_collaborator):
     new_branch_name = f'backport/{pr.number}/to-{version}'
     backport_pr_title = f'[Backport {version}] {pr.title}'
     repo_url = f'https://scylladbbot:{github_token}@github.com/{repo.full_name}.git'
@@ -121,46 +112,29 @@ def backport(repo, pr, version, commits, backport_base_branch):
                     is_draft = True
                     repo_local.git.add(A=True)
                     repo_local.git.cherry_pick('--continue')
+            repo_local.git.push(fork_repo, new_branch_name, force=True)
+            create_pull_request(repo, new_branch_name, backport_base_branch, pr, backport_pr_title, commits,
+                                is_draft, is_collaborator)
 
-            # Check if the branch already exists in the remote fork
-            remote_refs = repo_local.git.ls_remote('--heads', fork_repo, new_branch_name)
-            if not remote_refs:
-                # Branch does not exist, create it with a regular push
-                repo_local.git.push(fork_repo, new_branch_name)
-                create_pull_request(repo, new_branch_name, backport_base_branch, pr, backport_pr_title, commits,
-                                    is_draft)
-            else:
-                logging.info(f"Remote branch {new_branch_name} already exists in fork. Skipping push.")
         except GitCommandError as e:
             logging.warning(f"GitCommandError: {e}")
 
 
 def with_github_keyword_prefix(repo, pr):
-    # GitHub issue pattern: #123, scylladb/scylladb#123, or full GitHub URLs
-    github_pattern = rf"(?:fix(?:|es|ed))\s*:?\s*(?:(?:(?:{repo.full_name})?#)|https://github\.com/{repo.full_name}/issues/)(\d+)"
-    
-    # JIRA issue pattern: PKG-92 or https://scylladb.atlassian.net/browse/PKG-92
-    jira_pattern = r"(?:fix(?:|es|ed))\s*:?\s*(?:(?:https://scylladb\.atlassian\.net/browse/)?([A-Z]+-\d+))"
-    
-    # Check PR body for GitHub issues
-    github_match = re.findall(github_pattern, pr.body, re.IGNORECASE)
-    # Check PR body for JIRA issues
-    jira_match = re.findall(jira_pattern, pr.body, re.IGNORECASE)
-    
-    match = github_match or jira_match
-
-    if match:
+    pattern = rf"(?:fix(?:|es|ed))\s*:?\s*(?:(?:(?:{repo.full_name})?#)|https://github\.com/{repo.full_name}/issues/)(\d+)"
+    match = re.findall(pattern, pr.body, re.IGNORECASE)
+    if not match:
+        for commit in pr.get_commits():
+            match = re.findall(pattern, commit.commit.message, re.IGNORECASE)
+            if match:
+                print(f'{pr.number} has a valid close reference in commit message {commit.sha}')
+                break
+    if not match:
+        print(f'No valid close reference for {pr.number}')
+        return False
+    else:
         return True
 
-    for commit in pr.get_commits():
-        github_match = re.findall(github_pattern, commit.commit.message, re.IGNORECASE)
-        jira_match = re.findall(jira_pattern, commit.commit.message, re.IGNORECASE)
-        if github_match or jira_match:
-            print(f'{pr.number} has a valid close reference in commit message {commit.sha}')
-            return True
-
-    print(f'No valid close reference for {pr.number}')
-    return False
 
 def main():
     args = parse_args()
@@ -181,6 +155,7 @@ def main():
     scylladbbot_repo = g.get_repo(fork_repo_name)
     closed_prs = []
     start_commit = None
+    is_collaborator = True
 
     if args.commits:
         start_commit, end_commit = args.commits.split('..')
@@ -205,21 +180,33 @@ def main():
         if not backport_labels:
             print(f'no backport label: {pr.number}')
             continue
-        if args.commits and not with_github_keyword_prefix(repo, pr):
+        if not with_github_keyword_prefix(repo, pr) and args.github_event != 'unlabeled':
+            comment = f''':warning:  @{pr.user.login} PR body or PR commits do not contain a Fixes reference to an issue and can not be backported
+            please update PR body with a valid ref to an issue. Then remove `scylladbbot/backport_error` label to re-trigger the backport process
+            '''
+            pr.create_issue_comment(comment)
+            pr.add_to_labels("scylladbbot/backport_error")
             continue
         if not repo.private and not scylladbbot_repo.has_in_collaborators(pr.user.login):
             logging.info(f"Sending an invite to {pr.user.login} to become a collaborator to {scylladbbot_repo.full_name} ")
             scylladbbot_repo.add_to_collaborators(pr.user.login)
-            comment = f':warning:  @{pr.user.login} you have been added as collaborator to scylladbbot fork '
-            comment += f'Please check your inbox and approve the invitation, once it is done, please add the backport labels again\n'
-            create_pr_comment_and_remove_label(pr, comment)
-            continue
+            comment = f''':warning:  @{pr.user.login} you have been added as collaborator to scylladbbot fork
+            Please check your inbox and approve the invitation, otherwise you will not be able to edit PR branch when needed
+            '''
+            # When a pull request is pending for backport but its author is not yet a collaborator of "scylladbbot",
+            # we attach a "scylladbbot/backport_error" label to the PR.
+            # This prevents the workflow from proceeding with the backport process
+            # until the author has been granted proper permissions
+            # the author should remove the label manually to re-trigger the backport workflow.
+            pr.add_to_labels("scylladbbot/backport_error")
+            pr.create_issue_comment(comment)
+            is_collaborator = False
         commits = get_pr_commits(repo, pr, stable_branch, start_commit)
         logging.info(f"Found PR #{pr.number} with commit {commits} and the following labels: {backport_labels}")
         for backport_label in backport_labels:
             version = backport_label.replace('backport/', '')
             backport_base_branch = backport_label.replace('backport/', backport_branch)
-            backport(repo, pr, version, commits, backport_base_branch)
+            backport(repo, pr, version, commits, backport_base_branch, is_collaborator)
 
 
 if __name__ == "__main__":

.github/scripts/check-license.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024-present ScyllaDB
+#
+#
+# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+#
+
+import argparse
+import sys
+from pathlib import Path
+from typing import Set
+
+
+def parse_args() -> argparse.Namespace:
+    """Parses command-line arguments."""
+    parser = argparse.ArgumentParser(description='Check license headers in files')
+    parser.add_argument('--files', required=True, nargs="+", type=Path,
+                        help='List of files to check')
+    parser.add_argument('--license', required=True,
+                        help='License to check for')
+    parser.add_argument('--check-lines', type=int, default=10,
+                        help='Number of lines to check (default: %(default)s)')
+    parser.add_argument('--extensions', required=True, nargs="+",
+                        help='List of file extensions to check')
+    parser.add_argument('--verbose', action='store_true',
+                        help='Print verbose output (default: %(default)s)')
+    return parser.parse_args()
+
+
+def should_check_file(file_path: Path, allowed_extensions: Set[str]) -> bool:
+    return file_path.suffix in allowed_extensions
+
+
+def check_license_header(file_path: Path, license_header: str, check_lines: int) -> bool:
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            for _ in range(check_lines):
+                line = f.readline()
+                if license_header in line:
+                    return True
+        return False
+    except (UnicodeDecodeError, StopIteration):
+        # Handle files that can't be read as text or have fewer lines
+        return False
+
+
+def main() -> int:
+    args = parse_args()
+
+    if not args.files:
+        print("No files to check")
+        return 0
+
+    num_errors = 0
+
+    for file_path in args.files:
+        # Skip non-existent files
+        if not file_path.exists():
+            continue
+
+        # Skip files with non-matching extensions
+        if not should_check_file(file_path, args.extensions):
+            print(f"ℹ️ Skipping file with unchecked extension: {file_path}")
+            continue
+
+        # Check license header
+        if check_license_header(file_path, args.license, args.check_lines):
+            if args.verbose:
+                print(f"✅ License header found in: {file_path}")
+        else:
+            print(f"❌ Missing license header in: {file_path}")
+            num_errors += 1
+
+    if num_errors > 0:
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

.github/workflows/add-label-when-promoted.yaml

@@ -7,7 +7,7 @@ on:
       - branch-*.*
       - enterprise
   pull_request_target:
-    types: [labeled]
+    types: [labeled, unlabeled]
     branches: [master, next, enterprise]
 
 jobs:
@@ -53,19 +53,28 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }}
         run: python .github/scripts/auto-backport.py --repo ${{ github.repository }} --base-branch ${{ github.ref }} --commits ${{ github.event.before }}..${{ github.sha }}
-      - name: Check if label starts with 'backport/' and contains digits
+      - name: Check if a valid backport label exists and no backport_error
         id: check_label
         run: |
-          label_name="${{ github.event.label.name }}"
-          if [[ "$label_name" =~ ^backport/[0-9]+\.[0-9]+$ ]]; then
-            echo "Label matches backport/X.X pattern."
-            echo "backport_label=true" >> $GITHUB_OUTPUT
+          labels_json='${{ toJson(github.event.pull_request.labels) }}'
+          echo "Checking labels: $(echo "$labels_json" | jq -r '.[].name')"
+
+          # Check if a valid backport label exists
+          if echo "$labels_json" | jq -e 'any(.[] | .name; test("backport/[0-9]+\\.[0-9]+$"))' > /dev/null; then
+            # Ensure scylladbbot/backport_error is NOT present
+            if ! echo "$labels_json" | jq -e '.[] | select(.name == "scylladbbot/backport_error")' > /dev/null; then
+              echo "A matching backport label was found and no backport_error label exists."
+              echo "ready_for_backport=true" >> "$GITHUB_OUTPUT"
+              exit 0
+            else
+              echo "The label 'scylladbbot/backport_error' is present, invalidating backport."
+            fi
           else
-            echo "Label does not match the required pattern."
-            echo "backport_label=false" >> $GITHUB_OUTPUT
+            echo "No matching backport label found."
           fi
-      - name: Run auto-backport.py when label was added
-        if: ${{ github.event_name == 'pull_request_target' && steps.check_label.outputs.backport_label == 'true' && github.event.pull_request.state == 'closed' }}
+          echo "ready_for_backport=false" >> "$GITHUB_OUTPUT"
+      - name: Run auto-backport.py when PR is closed
+        if: ${{ github.event_name == 'pull_request_target' && steps.check_label.outputs.ready_for_backport == 'true' && github.event.pull_request.state == 'closed' }}
         env:
           GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }}
-        run: python .github/scripts/auto-backport.py --repo ${{ github.repository }} --base-branch ${{ github.ref }} --pull-request ${{ github.event.pull_request.number }} --head-commit ${{ github.event.pull_request.base.sha }}
+        run: python .github/scripts/auto-backport.py --repo ${{ github.repository }} --base-branch ${{ github.ref }} --pull-request ${{ github.event.pull_request.number }} --head-commit ${{ github.event.pull_request.base.sha }} --github-event ${{ github.event.action }}

.github/workflows/backport-pr-fixes-validation.yaml

@@ -18,7 +18,7 @@ jobs:
             
             // Regular expression pattern to check for "Fixes" prefix
             // Adjusted to dynamically insert the repository full name
-            const pattern = `Fixes:? ((?:#|${repo.replace('/', '\\/')}#|https://github\\.com/${repo.replace('/', '\\/')}/issues/)(\\d+)|(?:https://scylladb\\.atlassian\\.net/browse/)?([A-Z]+-\\d+))`;
+            const pattern = `Fixes:? (?:#|${repo.replace('/', '\\/')}#|https://github\\.com/${repo.replace('/', '\\/')}/issues/)(\\d+)`;
             const regex = new RegExp(pattern);
             
             if (!regex.test(body)) {

.github/workflows/call_backport_with_jira.yaml

@@ -1,53 +0,0 @@
-name: Backport with Jira Integration
-
-on:
-  push:
-    branches:
-      - master
-      - next-*.*
-      - branch-*.*
-  pull_request_target:
-    types: [labeled, closed]
-    branches: 
-      - master
-      - next
-      - next-*.*
-      - branch-*.*
-
-jobs:
-  backport-on-push:
-    if: github.event_name == 'push'
-    uses: scylladb/github-automation/.github/workflows/backport-with-jira.yaml@main
-    with:
-      event_type: 'push'
-      base_branch: ${{ github.ref }}
-      commits: ${{ github.event.before }}..${{ github.sha }}
-    secrets:
-      gh_token: ${{ secrets.AUTO_BACKPORT_TOKEN }}
-      jira_auth: ${{ secrets.USER_AND_KEY_FOR_JIRA_AUTOMATION }}
-
-  backport-on-label:
-    if: github.event_name == 'pull_request_target' && github.event.action == 'labeled'
-    uses: scylladb/github-automation/.github/workflows/backport-with-jira.yaml@main
-    with:
-      event_type: 'labeled'
-      base_branch: refs/heads/${{ github.event.pull_request.base.ref }}
-      pull_request_number: ${{ github.event.pull_request.number }}
-      head_commit: ${{ github.event.pull_request.base.sha }}
-      label_name: ${{ github.event.label.name }}
-      pr_state: ${{ github.event.pull_request.state }}
-    secrets:
-      gh_token: ${{ secrets.AUTO_BACKPORT_TOKEN }}
-      jira_auth: ${{ secrets.USER_AND_KEY_FOR_JIRA_AUTOMATION }}
-
-  backport-chain:
-    if: github.event_name == 'pull_request_target' && github.event.action == 'closed' && github.event.pull_request.merged == true
-    uses: scylladb/github-automation/.github/workflows/backport-with-jira.yaml@main
-    with:
-      event_type: 'chain'
-      base_branch: refs/heads/${{ github.event.pull_request.base.ref }}
-      pull_request_number: ${{ github.event.pull_request.number }}
-      pr_body: ${{ github.event.pull_request.body }}
-    secrets:
-      gh_token: ${{ secrets.AUTO_BACKPORT_TOKEN }}
-      jira_auth: ${{ secrets.USER_AND_KEY_FOR_JIRA_AUTOMATION }}

.github/workflows/check-license-header.yaml

@@ -0,0 +1,52 @@
+name: License Header Check
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    branches: [master]
+
+env:
+  HEADER_CHECK_LINES: 10
+  LICENSE: "LicenseRef-ScyllaDB-Source-Available-1.0"
+  CHECKED_EXTENSIONS: ".cc .hh .py"
+
+jobs:
+  check-license-headers:
+    name: Check License Headers
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Get changed files
+        id: changed-files
+        run: |
+          # Get list of added files comparing with base branch
+          echo "files=$(git diff --name-only --diff-filter=A ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | tr '\n' ' ')" >> $GITHUB_OUTPUT
+
+      - name: Check license headers
+        if: steps.changed-files.outputs.files != ''
+        run: |
+          .github/scripts/check-license.py \
+            --files ${{ steps.changed-files.outputs.files }} \
+            --license "${{ env.LICENSE }}" \
+            --check-lines "${{ env.HEADER_CHECK_LINES }}" \
+            --extensions ${{ env.CHECKED_EXTENSIONS }}
+
+      - name: Comment on PR if check fails
+        if: failure()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const license = '${{ env.LICENSE }}';
+            await github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: `❌ License header check failed. Please ensure all new files include the header within the first ${{ env.HEADER_CHECK_LINES }} lines:\n\`\`\`\n${license}\n\`\`\`\nSee action logs for details.`
+            });

.github/workflows/clang-nightly.yaml

@@ -7,7 +7,7 @@ on:
 
 env:
   # use the development branch explicitly
-  CLANG_VERSION: 20
+  CLANG_VERSION: 21
   BUILD_DIR: build
 
 permissions: {}

.github/workflows/clang-tidy.yaml

@@ -34,6 +34,7 @@ jobs:
     name: Run clang-tidy
     needs:
       - read-toolchain
+    if: "${{ needs.read-toolchain.result == 'success' }}"
     runs-on: ubuntu-latest
     container: ${{ needs.read-toolchain.outputs.image }}
     steps:

.github/workflows/make-pr-ready-for-review.yaml

@@ -12,15 +12,10 @@ jobs:
   mark-ready:
     if: github.event.label.name == 'conflicts'
     runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ github.repository }}
-          ref: ${{ env.DEFAULT_BRANCH }}
-          token: ${{ secrets.AUTO_BACKPORT_TOKEN }}
-          fetch-depth: 1
       - name: Mark pull request as ready for review
         run:  gh pr ready "${{ github.event.pull_request.number }}"
         env:

.github/workflows/pr-require-backport-label.yaml

@@ -17,6 +17,6 @@ jobs:
         with:
           mode: minimum
           count: 1
-          labels: "backport/none\nbackport/\\d.\\d"
+          labels: "backport/none\nbackport/\\d{4}\\.\\d+\nbackport/\\d+\\.\\d+"
           use_regex: true
           add_comment: false

.github/workflows/trigger_jenkins.yaml

@@ -0,0 +1,50 @@
+name: Trigger next gating
+
+on:
+  push:
+    branches:
+      - next**
+
+jobs:
+  trigger-jenkins:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Determine Jenkins Job Name
+        run: |
+          if [[ "${{ github.ref_name }}" == "next" ]]; then
+            FOLDER_NAME="scylla-master"
+          elif [[ "${{ github.ref_name }}" == "next-enterprise" ]]; then
+            FOLDER_NAME="scylla-enterprise"
+          else
+            VERSION=$(echo "${{ github.ref_name }}" | awk -F'-' '{print $2}')
+            if [[ "$VERSION" =~ ^202[0-4]\.[0-9]+$ ]]; then
+              FOLDER_NAME="enterprise-$VERSION"
+            elif [[ "$VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then
+              FOLDER_NAME="scylla-$VERSION"
+            fi
+          fi
+          echo "JOB_NAME=${FOLDER_NAME}/job/next" >> $GITHUB_ENV
+
+      - name: Trigger Jenkins Job
+        env:
+          JENKINS_USER: ${{ secrets.JENKINS_USERNAME }}
+          JENKINS_API_TOKEN: ${{ secrets.JENKINS_TOKEN }}
+          JENKINS_URL: "https://jenkins.scylladb.com"
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+        run: |
+          echo "Triggering Jenkins Job: $JOB_NAME"
+          if ! curl -X POST "$JENKINS_URL/job/$JOB_NAME/buildWithParameters" --fail --user "$JENKINS_USER:$JENKINS_API_TOKEN" -i -v; then
+            echo "Error: Jenkins job trigger failed"
+
+            # Send Slack message
+            curl -X POST -H 'Content-type: application/json' \
+              -H "Authorization: Bearer $SLACK_BOT_TOKEN" \
+              --data '{
+                "channel": "#releng-team",
+                "text": "🚨 @here '$JOB_NAME' failed to be triggered, please check https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} for more details",
+                "icon_emoji": ":warning:"
+              }' \
+              https://slack.com/api/chat.postMessage
+
+            exit 1
+          fi

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "seastar"]
 	path = seastar
-	url = ../scylla-seastar
+	url = ../seastar
 	ignore = dirty
 [submodule "swagger-ui"]
 	path = swagger-ui

CMakeLists.txt

@@ -66,24 +66,25 @@ if(is_multi_config)
     #   establishing proper dependencies between them
     include(ExternalProject)
 
+    # should be consistent with configure_seastar() in configure.py
+    set(seastar_build_dir "${CMAKE_BINARY_DIR}/$<CONFIG>/seastar")
     ExternalProject_Add(Seastar
         SOURCE_DIR "${PROJECT_SOURCE_DIR}/seastar"
-        BINARY_DIR "${CMAKE_BINARY_DIR}/$<CONFIG>/seastar"
         CONFIGURE_COMMAND ""
-        BUILD_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR>
+        BUILD_COMMAND ${CMAKE_COMMAND} --build "${seastar_build_dir}"
           --target seastar
           --target seastar_testing
           --target seastar_perf_testing
           --target app_iotune
         BUILD_ALWAYS ON
         BUILD_BYPRODUCTS
-          <BINARY_DIR>/libseastar.$<IF:$<CONFIG:Debug,Dev>,so,a>
-          <BINARY_DIR>/libseastar_testing.$<IF:$<CONFIG:Debug,Dev>,so,a>
-          <BINARY_DIR>/libseastar_perf_testing.$<IF:$<CONFIG:Debug,Dev>,so,a>
-          <BINARY_DIR>/apps/iotune/iotune
-          <BINARY_DIR>/gen/include/seastar/http/chunk_parsers.hh
-          <BINARY_DIR>/gen/include/seastar/http/request_parser.hh
-          <BINARY_DIR>/gen/include/seastar/http/response_parser.hh
+          ${seastar_build_dir}/libseastar.$<IF:$<CONFIG:Debug,Dev>,so,a>
+          ${seastar_build_dir}/libseastar_testing.$<IF:$<CONFIG:Debug,Dev>,so,a>
+          ${seastar_build_dir}/libseastar_perf_testing.$<IF:$<CONFIG:Debug,Dev>,so,a>
+          ${seastar_build_dir}/apps/iotune/iotune
+          ${seastar_build_dir}/gen/include/seastar/http/chunk_parsers.hh
+          ${seastar_build_dir}/gen/include/seastar/http/request_parser.hh
+          ${seastar_build_dir}/gen/include/seastar/http/response_parser.hh
         INSTALL_COMMAND "")
     add_dependencies(Seastar::seastar Seastar)
     add_dependencies(Seastar::seastar_testing Seastar)
@@ -201,7 +202,6 @@ target_sources(scylla-main
     tombstone_gc.cc
     reader_concurrency_semaphore.cc
     reader_concurrency_semaphore_group.cc
-    row_cache.cc
     schema_mutations.cc
     serializer.cc
     sstables_loader.cc

CONTRIBUTING.md

@@ -12,7 +12,7 @@ Please use the [issue tracker](https://github.com/scylladb/scylla/issues/) to re
 
 ## Contributing code to Scylla
 
-Before you can contribute code to Scylla for the first time, you should sign the [Contributor License Agreement](https://www.scylladb.com/open-source/contributor-agreement/) and send the signed form cla@scylladb.com. You can then submit your changes as patches to the to the [scylladb-dev mailing list](https://groups.google.com/forum/#!forum/scylladb-dev) or as a pull request to the [Scylla project on github](https://github.com/scylladb/scylla).
+Before you can contribute code to Scylla for the first time, you should sign the [Contributor License Agreement](https://www.scylladb.com/open-source/contributor-agreement/) and send the signed form cla@scylladb.com. You can then submit your changes as patches to the [scylladb-dev mailing list](https://groups.google.com/forum/#!forum/scylladb-dev) or as a pull request to the [Scylla project on github](https://github.com/scylladb/scylla).
 If you need help formatting or sending patches, [check out these instructions](https://github.com/scylladb/scylla/wiki/Formatting-and-sending-patches).
 
 The Scylla C++ source code uses the [Seastar coding style](https://github.com/scylladb/seastar/blob/master/coding-style.md) so please adhere to that in your patches. Note that Scylla code is written with `using namespace seastar`, so should not explicitly add the `seastar::` prefix to Seastar symbols. You will usually not need to add `using namespace seastar` to new source files, because most Scylla header files have `#include "seastarx.hh"`, which does this.

HACKING.md

@@ -280,21 +280,45 @@ Once the patch set is ready to be reviewed, push the branch to the public remote
 
 ### Development environment and source code navigation
 
-Scylla includes a [CMake](https://cmake.org/) file, `CMakeLists.txt`, for use only with development environments (not for building) so that they can properly analyze the source code.
+Scylla includes a [CMake](https://cmake.org/) file, `CMakeLists.txt` that can be used with development environments so
+that they can properly analyze the source code. However, building with CMake is not yet officially supported.
 
-[CLion](https://www.jetbrains.com/clion/) is a commercial IDE offers reasonably good source code navigation and advice for code hygiene, though its C++ parser sometimes makes errors and flags false issues.
+Good IDEs that have support for CMake build toolchain are [CLion](https://www.jetbrains.com/clion/),
+[KDevelop](https://www.kdevelop.org/) and [QtCreator](https://wiki.qt.io/Qt_Creator).
 
-Other good options that directly parse CMake files are [KDevelop](https://www.kdevelop.org/) and [QtCreator](https://wiki.qt.io/Qt_Creator).
+[Eclipse](https://eclipse.org/cdt/) is another open-source option. It doesn't natively work with CMake projects and its
+C++ parser has many issues.
 
-To use the `CMakeLists.txt` file with these programs, define the `FOR_IDE` CMake variable or shell environmental variable.
+#### CLion
 
-[Eclipse](https://eclipse.org/cdt/) is another open-source option. It doesn't natively work with CMake projects, and its C++ parser has many similar issues as CLion.
+[CLion](https://www.jetbrains.com/clion/) is a commercial IDE offers reasonably good source code navigation and advice
+for code hygiene, though its C++ parser sometimes makes errors and flags false issues. In order to enable proper code
+analysis in CLion, the following steps are needed:
+
+1. Get the ScyllaDB source code by following the [Getting the source code](#getting-the-source-code).
+2. Follow the steps in [Dependencies](#dependencies) in order to install the required tools natively into your system.
+   **Don't** follow the *frozen toolchain* part described there, since CMake checks for the build dependencies installed
+   in the system, not in the container image provided by the toolchain.
+3. In CLion, select `File`→`Open` and select the main ScyllaDB directory in order to open the CMake project there. The
+   project should open and fail to process the `CMakeLists.txt`. That's expected.
+4. In CLion, open `File`→`Settings`.
+5. Find and click on `Toolchains` (type *toolchains* into search box).
+6. Select the toolchain you will use, for instance the `Default` one.
+7. Type in the following system-installed tools to be used:
+    - `CMake`: *cmake*
+    - `Build Tool`: *ninja*
+    - `C Compiler`: *clang*
+    - `C++ Compiler`: *clang*
+8. On the `CMake` panel/tab, click on `Reload CMake Project`
+
+After that, CLion should successfully initialize the CMake project (marked by `[Finished]` in the console) and the
+source code editor should provide code analysis support normally from now on.
 
 ### Distributed compilation: `distcc` and `ccache`
 
 Scylla's compilations times can be long. Two tools help somewhat:
 
-- [ccache](https://ccache.samba.org/) caches compiled object files on disk and re-uses them when possible
+- [ccache](https://ccache.samba.org/) caches compiled object files on disk and reuses them when possible
 - [distcc](https://github.com/distcc/distcc) distributes compilation jobs to remote machines
 
 A reasonably-powered laptop acts as the coordinator for compilation. A second, more powerful, machine acts as a passive compilation server.

LICENSE-ScyllaDB-Source-Available.md

@@ -49,7 +49,7 @@ The terms "**You**" or "**Licensee**" refer to any individual accessing or using
 
 * **Ownership:** Licensor retains sole and exclusive ownership of all rights, interests and title in the Software and any scripts, processes, techniques, methodologies, inventions, know-how, concepts, formatting, arrangements, visual attributes, ideas, database rights, copyrights, patents, trade secrets, and other intellectual property related thereto, and all derivatives, enhancements, modifications and improvements thereof. Except for the limited license rights granted herein, Licensee has no rights in or to the Software and/ or Licensor’s trademarks, logo, or branding and You acknowledge that such Software, trademarks, logo, or branding is the sole property of Licensor.
 * **Feedback:** Licensee is not required to provide any suggestions, enhancement requests, recommendations or other feedback regarding the Software ("Feedback").  If, notwithstanding this policy, Licensee submits Feedback, Licensee understands and acknowledges that such Feedback is not submitted in confidence and Licensor assumes no obligation, expressed or implied, by considering it.  All right in any trademark or logo of Licensor or its affiliates and You shall make no claim of right to the Software or any part thereof to be supplied by Licensor hereunder and acknowledges that as between Licensor and You, such Software is the sole proprietary, title and interest in and to Licensor.such Feedback shall be assigned to, and shall become the sole and exclusive property of, Licensor upon its creation.
-* Except for the rights expressly granted to You under this Agreement, You are not granted any other licenses or rights in the Software or otherwise. This Agreement constitutes the entire agreement between the You and the Licensor with respect to the subject matter hereof and supersedes all prior or contemporaneous communications, representations, or agreements, whether oral or written.
+* Except for the rights expressly granted to You under this Agreement, You are not granted any other licenses or rights in the Software or otherwise. This Agreement constitutes the entire agreement between You and the Licensor with respect to the subject matter hereof and supersedes all prior or contemporaneous communications, representations, or agreements, whether oral or written.
 * **Third-Party Software:** Customer acknowledges that the Software may contain open and closed source components (“OSS Components”) that are governed separately by certain licenses, in each case as further provided by Company upon request. Any applicable OSS Component license is solely between Licensee and the applicable licensor of the OSS Component and Licensee shall comply with the applicable OSS Component license.
 * If any provision of this Agreement is held to be invalid or unenforceable, such provision shall be struck and the remaining provisions shall remain in full force and effect.
 

README.md

@@ -102,7 +102,7 @@ If you are a developer working on Scylla, please read the [developer guidelines]
 
 ## Contact
 
-* The [community forum] and [Slack channel] are for users to discuss configuration, management, and operations of the ScyllaDB open source.
+* The [community forum] and [Slack channel] are for users to discuss configuration, management, and operations of ScyllaDB.
 * The [developers mailing list] is for developers and people interested in following the development of ScyllaDB to discuss technical topics.
 
 [Community forum]: https://forum.scylladb.com/

SCYLLA-VERSION-GEN

@@ -78,7 +78,7 @@ fi
 
 # Default scylla product/version tags
 PRODUCT=scylla
-VERSION=2025.1.12
+VERSION=2025.2.0-dev
 
 if test -f version
 then

alternator/consumed_capacity.cc

@@ -24,7 +24,7 @@ static constexpr uint64_t KB = 1024ULL;
 static constexpr uint64_t RCU_BLOCK_SIZE_LENGTH = 4*KB;
 static constexpr uint64_t WCU_BLOCK_SIZE_LENGTH = 1*KB;
 
-bool consumed_capacity_counter::should_add_capacity(const rjson::value& request) {
+static bool should_add_capacity(const rjson::value& request) {
     const rjson::value* return_consumed = rjson::find(request, "ReturnConsumedCapacity");
     if (!return_consumed) {
         return false;
@@ -62,22 +62,15 @@ static uint64_t calculate_half_units(uint64_t unit_block_size, uint64_t total_by
 rcu_consumed_capacity_counter::rcu_consumed_capacity_counter(const rjson::value& request, bool is_quorum) :
         consumed_capacity_counter(should_add_capacity(request)),_is_quorum(is_quorum) {
 }
-uint64_t rcu_consumed_capacity_counter::get_half_units(uint64_t total_bytes, bool is_quorum) noexcept {
-    return calculate_half_units(RCU_BLOCK_SIZE_LENGTH, total_bytes, is_quorum);
-}
 
 uint64_t rcu_consumed_capacity_counter::get_half_units() const noexcept {
-    return get_half_units(_total_bytes, _is_quorum);
+    return calculate_half_units(RCU_BLOCK_SIZE_LENGTH, _total_bytes, _is_quorum);
 }
 
 uint64_t wcu_consumed_capacity_counter::get_half_units() const noexcept {
     return calculate_half_units(WCU_BLOCK_SIZE_LENGTH, _total_bytes, true);
 }
 
-uint64_t wcu_consumed_capacity_counter::get_units(uint64_t total_bytes) noexcept {
-    return calculate_half_units(WCU_BLOCK_SIZE_LENGTH, total_bytes, true) * HALF_UNIT_MULTIPLIER;
-}
-
 wcu_consumed_capacity_counter::wcu_consumed_capacity_counter(const rjson::value& request) :
         consumed_capacity_counter(should_add_capacity(request)) {
 }

alternator/consumed_capacity.hh

@@ -42,25 +42,21 @@ public:
      */
     virtual uint64_t get_half_units() const noexcept = 0;
     uint64_t _total_bytes = 0;
-    static bool should_add_capacity(const rjson::value& request);
 protected:
     bool _should_add_to_reponse = false;
 };
 
 class rcu_consumed_capacity_counter : public consumed_capacity_counter {
+    virtual uint64_t get_half_units() const noexcept;
     bool _is_quorum = false;
 public:
     rcu_consumed_capacity_counter(const rjson::value& request, bool is_quorum);
-    rcu_consumed_capacity_counter(): consumed_capacity_counter(false), _is_quorum(false){}
-    virtual uint64_t get_half_units() const noexcept;
-    static uint64_t get_half_units(uint64_t total_bytes, bool is_quorum) noexcept;
 };
 
 class wcu_consumed_capacity_counter : public consumed_capacity_counter {
     virtual uint64_t get_half_units() const noexcept;
 public:
     wcu_consumed_capacity_counter(const rjson::value& request);
-    static uint64_t get_units(uint64_t total_bytes) noexcept;
 };
 
 }

alternator/executor.hh

@@ -241,8 +241,7 @@ public:
         const query::partition_slice&& slice,
         shared_ptr<cql3::selection::selection> selection,
         foreign_ptr<lw_shared_ptr<query::result>> query_result,
-        shared_ptr<const std::optional<attrs_to_get>> attrs_to_get,
-        uint64_t& rcu_half_units);
+        shared_ptr<const std::optional<attrs_to_get>> attrs_to_get);
 
     static void describe_single_item(const cql3::selection::selection&,
         const std::vector<managed_bytes_opt>&,

alternator/expressions.g

@@ -91,18 +91,6 @@ options {
         throw expressions_syntax_error(format("{} at char {}", err,
             ex->get_charPositionInLine()));
     }
-
-    // ANTLR3 tries to recover missing tokens - it tries to finish parsing
-    // and create valid objects, as if the missing token was there.
-    // But it has a bug and leaks these tokens.
-    // We override offending method and handle abandoned pointers.
-    std::vector<std::unique_ptr<TokenType>> _missing_tokens;
-    TokenType* getMissingSymbol(IntStreamType* istream, ExceptionBaseType* e,
-                                ANTLR_UINT32 expectedTokenType, BitsetListType* follow) {
-        auto token = BaseType::getMissingSymbol(istream, e, expectedTokenType, follow);
-        _missing_tokens.emplace_back(token);
-        return token;
-    }
 }
 @lexer::context {
     void displayRecognitionError(ANTLR_UINT8** token_names, ExceptionBaseType* ex) {

alternator/server.cc

@@ -606,14 +606,24 @@ future<> server::init(net::inet_address addr, std::optional<uint16_t> port, std:
             set_routes(_https_server._routes);
             _https_server.set_content_length_limit(server::content_length_limit);
             _https_server.set_content_streaming(true);
-            auto server_creds = creds->build_reloadable_server_credentials([](const std::unordered_set<sstring>& files, std::exception_ptr ep) {
-                if (ep) {
-                    slogger.warn("Exception loading {}: {}", files, ep);
-                } else {
-                    slogger.info("Reloaded {}", files);
-                }
-            }).get();
-            _https_server.listen(socket_address{addr, *https_port}, std::move(server_creds)).get();
+
+            if (this_shard_id() == 0) {
+                _credentials = creds->build_reloadable_server_credentials([this](const tls::credentials_builder& b, const std::unordered_set<sstring>& files, std::exception_ptr ep) -> future<> {
+                    if (ep) {
+                        slogger.warn("Exception loading {}: {}", files, ep);
+                    } else {
+                        co_await container().invoke_on_others([&b](server& s) {
+                            if (s._credentials) {
+                                b.rebuild(*s._credentials);
+                            }
+                        });
+                        slogger.info("Reloaded {}", files);
+                    }
+                }).get();
+            } else {
+                _credentials = creds->build_server_credentials();
+            }
+            _https_server.listen(socket_address{addr, *https_port}, _credentials).get();
             _enabled_servers.push_back(std::ref(_https_server));
         }
     });

alternator/server.hh

@@ -24,7 +24,7 @@ namespace alternator {
 
 using chunked_content = rjson::chunked_content;
 
-class server {
+class server : public peering_sharded_service<server> {
     static constexpr size_t content_length_limit = 16*MB;
     using alternator_callback = std::function<future<executor::request_return_type>(executor&, executor::client_state&,
             tracing::trace_state_ptr, service_permit, rjson::value, std::unique_ptr<http::request>)>;
@@ -52,6 +52,8 @@ class server {
     semaphore* _memory_limiter;
     utils::updateable_value<uint32_t> _max_concurrent_requests;
 
+    ::shared_ptr<seastar::tls::server_credentials> _credentials;
+
     class json_parser {
         static constexpr size_t yieldable_parsing_threshold = 16*KB;
         chunked_content _raw_document;

alternator/stats.cc

@@ -9,6 +9,7 @@
 #include "stats.hh"
 #include "utils/histogram_metrics_helper.hh"
 #include <seastar/core/metrics.hh>
+#include "utils/labels.hh"
 
 namespace alternator {
 
@@ -21,12 +22,12 @@ stats::stats() : api_operations{} {
     _metrics.add_group("alternator", {
 #define OPERATION(name, CamelCaseName) \
                 seastar::metrics::make_total_operations("operation", api_operations.name, \
-                        seastar::metrics::description("number of operations via Alternator API"), {op(CamelCaseName)}).set_skip_when_empty(),
+                        seastar::metrics::description("number of operations via Alternator API"), {op(CamelCaseName), alternator_label, basic_level}).set_skip_when_empty(),
 #define OPERATION_LATENCY(name, CamelCaseName) \
                 seastar::metrics::make_histogram("op_latency", \
-                        seastar::metrics::description("Latency histogram of an operation via Alternator API"), {op(CamelCaseName)}, [this]{return to_metrics_histogram(api_operations.name.histogram());}).aggregate({seastar::metrics::shard_label}).set_skip_when_empty(), \
+                        seastar::metrics::description("Latency histogram of an operation via Alternator API"), {op(CamelCaseName), alternator_label, basic_level}, [this]{return to_metrics_histogram(api_operations.name.histogram());}).aggregate({seastar::metrics::shard_label}).set_skip_when_empty(), \
 				seastar::metrics::make_summary("op_latency_summary", \
-						                        seastar::metrics::description("Latency summary of an operation via Alternator API"), [this]{return to_metrics_summary(api_operations.name.summary());})(op(CamelCaseName)).set_skip_when_empty(),
+						                        seastar::metrics::description("Latency summary of an operation via Alternator API"), [this]{return to_metrics_summary(api_operations.name.summary());})(op(CamelCaseName))(basic_level)(alternator_label).set_skip_when_empty(),
             OPERATION(batch_get_item, "BatchGetItem")
             OPERATION(batch_write_item, "BatchWriteItem")
             OPERATION(create_backup, "CreateBackup")
@@ -77,39 +78,39 @@ stats::stats() : api_operations{} {
     });
     _metrics.add_group("alternator", {
             seastar::metrics::make_total_operations("unsupported_operations", unsupported_operations,
-                    seastar::metrics::description("number of unsupported operations via Alternator API")),
+                    seastar::metrics::description("number of unsupported operations via Alternator API"))(alternator_label).set_skip_when_empty(),
             seastar::metrics::make_total_operations("total_operations", total_operations,
-                    seastar::metrics::description("number of total operations via Alternator API")),
+                    seastar::metrics::description("number of total operations via Alternator API"))(basic_level)(alternator_label).set_skip_when_empty(),
             seastar::metrics::make_total_operations("reads_before_write", reads_before_write,
-                    seastar::metrics::description("number of performed read-before-write operations")),
+                    seastar::metrics::description("number of performed read-before-write operations"))(alternator_label).set_skip_when_empty(),
             seastar::metrics::make_total_operations("write_using_lwt", write_using_lwt,
-                    seastar::metrics::description("number of writes that used LWT")),
+                    seastar::metrics::description("number of writes that used LWT"))(alternator_label).set_skip_when_empty(),
             seastar::metrics::make_total_operations("shard_bounce_for_lwt", shard_bounce_for_lwt,
-                    seastar::metrics::description("number writes that had to be bounced from this shard because of LWT requirements")),
+                    seastar::metrics::description("number writes that had to be bounced from this shard because of LWT requirements"))(alternator_label).set_skip_when_empty(),
             seastar::metrics::make_total_operations("requests_blocked_memory", requests_blocked_memory,
-                    seastar::metrics::description("Counts a number of requests blocked due to memory pressure.")),
+                    seastar::metrics::description("Counts a number of requests blocked due to memory pressure."))(alternator_label).set_skip_when_empty(),
             seastar::metrics::make_total_operations("requests_shed", requests_shed,
-                    seastar::metrics::description("Counts a number of requests shed due to overload.")),
+                    seastar::metrics::description("Counts a number of requests shed due to overload."))(alternator_label).set_skip_when_empty(),
             seastar::metrics::make_total_operations("filtered_rows_read_total", cql_stats.filtered_rows_read_total,
-                    seastar::metrics::description("number of rows read during filtering operations")),
+                    seastar::metrics::description("number of rows read during filtering operations"))(alternator_label).set_skip_when_empty(),
             seastar::metrics::make_total_operations("filtered_rows_matched_total", cql_stats.filtered_rows_matched_total,
                     seastar::metrics::description("number of rows read and matched during filtering operations")),
-            seastar::metrics::make_counter("rcu_total", [this]{return 0.5 * rcu_half_units_total;},
-                    seastar::metrics::description("total number of consumed read units")).set_skip_when_empty(),
+            seastar::metrics::make_counter("rcu_total", rcu_total,
+                    seastar::metrics::description("total number of consumed read units, counted as half units"))(alternator_label).set_skip_when_empty(),
             seastar::metrics::make_counter("wcu_total", wcu_total[wcu_types::PUT_ITEM],
-                    seastar::metrics::description("total number of consumed write units"),{op("PutItem")}).set_skip_when_empty(),
+                    seastar::metrics::description("total number of consumed write units, counted as half units"),{op("PutItem")})(alternator_label).set_skip_when_empty(),
             seastar::metrics::make_counter("wcu_total", wcu_total[wcu_types::DELETE_ITEM],
-                    seastar::metrics::description("total number of consumed write units"),{op("DeleteItem")}).set_skip_when_empty(),
+                    seastar::metrics::description("total number of consumed write units, counted as half units"),{op("DeleteItem")})(alternator_label).set_skip_when_empty(),
             seastar::metrics::make_counter("wcu_total", wcu_total[wcu_types::UPDATE_ITEM],
-                    seastar::metrics::description("total number of consumed write units"),{op("UpdateItem")}).set_skip_when_empty(),
+                    seastar::metrics::description("total number of consumed write units, counted as half units"),{op("UpdateItem")})(alternator_label).set_skip_when_empty(),
             seastar::metrics::make_counter("wcu_total", wcu_total[wcu_types::INDEX],
-                    seastar::metrics::description("total number of consumed write units"),{op("Index")}).set_skip_when_empty(),
+                    seastar::metrics::description("total number of consumed write units, counted as half units"),{op("Index")})(alternator_label).set_skip_when_empty(),
             seastar::metrics::make_total_operations("filtered_rows_dropped_total", [this] { return cql_stats.filtered_rows_read_total - cql_stats.filtered_rows_matched_total; },
-                    seastar::metrics::description("number of rows read and dropped during filtering operations")),
+                    seastar::metrics::description("number of rows read and dropped during filtering operations"))(alternator_label).set_skip_when_empty(),
             seastar::metrics::make_counter("batch_item_count", seastar::metrics::description("The total number of items processed across all batches"),{op("BatchWriteItem")},
-                    api_operations.batch_write_item_batch_total).set_skip_when_empty(),
+                    api_operations.batch_write_item_batch_total)(alternator_label).set_skip_when_empty(),
             seastar::metrics::make_counter("batch_item_count", seastar::metrics::description("The total number of items processed across all batches"),{op("BatchGetItem")},
-                    api_operations.batch_get_item_batch_total).set_skip_when_empty(),
+                    api_operations.batch_get_item_batch_total)(alternator_label).set_skip_when_empty(),
     });
 }
 

alternator/stats.hh

@@ -84,7 +84,7 @@ public:
     uint64_t shard_bounce_for_lwt = 0;
     uint64_t requests_blocked_memory = 0;
     uint64_t requests_shed = 0;
-    uint64_t rcu_half_units_total = 0;
+    uint64_t rcu_total = 0;
     // wcu can results from put, update, delete and index
     // Index related will be done on top of the operation it comes with
     enum wcu_types {

alternator/streams.cc

@@ -808,9 +808,6 @@ future<executor::request_return_type> executor::get_records(client_state& client
     if (limit < 1) {
         throw api_error::validation("Limit must be 1 or more");
     }
-    if (limit > 1000) {
-        throw api_error::validation("Limit must be less than or equal to 1000");
-    }
 
     auto db = _proxy.data_dictionary();
     schema_ptr schema, base;

alternator/ttl.cc

@@ -16,7 +16,6 @@
 #include <seastar/core/future.hh>
 #include <seastar/core/lowres_clock.hh>
 #include <seastar/coroutine/maybe_yield.hh>
-#include <boost/multiprecision/cpp_int.hpp>
 
 #include "exceptions/exceptions.hh"
 #include "gms/gossiper.hh"
@@ -49,6 +48,7 @@
 #include "dht/sharder.hh"
 #include "db/config.hh"
 #include "db/tags/utils.hh"
+#include "utils/labels.hh"
 
 #include "ttl.hh"
 
@@ -851,13 +851,13 @@ future<> expiration_service::stop() {
 expiration_service::stats::stats() {
     _metrics.add_group("expiration", {
         seastar::metrics::make_total_operations("scan_passes", scan_passes,
-            seastar::metrics::description("number of passes over the database")),
+            seastar::metrics::description("number of passes over the database"))(alternator_label).set_skip_when_empty(),
         seastar::metrics::make_total_operations("scan_table", scan_table,
-            seastar::metrics::description("number of table scans (counting each scan of each table that enabled expiration)")),
+            seastar::metrics::description("number of table scans (counting each scan of each table that enabled expiration)"))(alternator_label).set_skip_when_empty(),
         seastar::metrics::make_total_operations("items_deleted", items_deleted,
-            seastar::metrics::description("number of items deleted after expiration")),
+            seastar::metrics::description("number of items deleted after expiration"))(basic_level)(alternator_label).set_skip_when_empty(),
         seastar::metrics::make_total_operations("secondary_ranges_scanned", secondary_ranges_scanned,
-            seastar::metrics::description("number of token ranges scanned by this node while their primary owner was down")),
+            seastar::metrics::description("number of token ranges scanned by this node while their primary owner was down"))(alternator_label).set_skip_when_empty(),
     });
 }
 

api/api-doc/storage_service.json

@@ -813,6 +813,14 @@
                           "allowMultiple":false,
                           "type":"string",
                           "paramType":"query"
+                      },
+                      {
+                          "name":"move_files",
+                          "description":"Move component files instead of copying them",
+                          "required":false,
+                          "allowMultiple":false,
+                          "type":"boolean",
+                          "paramType":"query"
                       }
                   ]
               }
@@ -976,7 +984,7 @@
          ]
       },
       {
-         "path":"/storage_service/cleanup_all/",
+         "path":"/storage_service/cleanup_all",
          "operations":[
             {
                "method":"POST",
@@ -986,30 +994,6 @@
                "produces":[
                   "application/json"
                ],
-               "parameters":[
-                    {
-                     "name":"global",
-                     "description":"true if cleanup of entire cluster is requested",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"boolean",
-                     "paramType":"query"
-                  }
-               ]
-            }
-         ]
-      },
-      {
-         "path":"/storage_service/mark_node_as_clean",
-         "operations":[
-            {
-               "method":"POST",
-               "summary":"Mark the node as clean. After that the node will not be considered as needing cleanup during automatic cleanup which is triggered by some topology operations",
-               "type":"void",
-               "nickname":"reset_cleanup_needed",
-               "produces":[
-                  "application/json"
-               ],
                "parameters":[]
             }
          ]
@@ -3085,22 +3069,6 @@
                ]
             }
          ]
-      },
-      {
-         "path":"/storage_service/raft_topology/cmd_rpc_status",
-         "operations":[
-            {
-               "method":"GET",
-               "summary":"Get information about currently running topology cmd rpc",
-               "type":"string",
-               "nickname":"raft_topology_get_cmd_status",
-               "produces":[
-                  "application/json"
-               ],
-               "parameters":[
-               ]
-            }
-         ]
       }
    ],
    "models":{
@@ -3237,11 +3205,11 @@
          "properties":{
             "start_token":{
                "type":"string",
-               "description":"The range start token (exclusive)"
+               "description":"The range start token"
             },
             "end_token":{
                "type":"string",
-               "description":"The range end token (inclusive)"
+               "description":"The range start token"
             },
             "endpoints":{
                "type":"array",

api/api-doc/task_manager.json

@@ -344,6 +344,18 @@
             "sequence_number":{
                "type":"long",
                "description":"The running sequence number of the task"
+            },
+            "shard":{
+               "type":"long",
+               "description":"The shard the task is running on"
+            },
+            "start_time":{
+               "type":"datetime",
+               "description":"The start time of the task; unspecified (equal to epoch) when state == created"
+            },
+            "end_time":{
+               "type":"datetime",
+               "description":"The end time of the task; unspecified (equal to epoch) when the task is not completed"
             }
          }
       },

api/api-doc/tasks.json

@@ -42,14 +42,6 @@
                      "allowMultiple":false,
                      "type":"boolean",
                      "paramType":"query"
-                  },
-                  {
-                     "name":"consider_only_existing_data",
-                     "description":"Set to \"true\" to flush all memtables and force tombstone garbage collection to check only the sstables being compacted (false by default). The memtable, commitlog and other uncompacted sstables will not be checked during tombstone garbage collection.",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"boolean",
-                     "paramType":"query"
                   }
                ]
             }

api/api.cc

@@ -317,13 +317,13 @@ future<> unset_server_commitlog(http_context& ctx) {
     return ctx.http_server.set_routes([&ctx] (routes& r) { unset_commitlog(ctx, r); });
 }
 
-future<> set_server_task_manager(http_context& ctx, sharded<tasks::task_manager>& tm, lw_shared_ptr<db::config> cfg) {
+future<> set_server_task_manager(http_context& ctx, sharded<tasks::task_manager>& tm, lw_shared_ptr<db::config> cfg, sharded<gms::gossiper>& gossiper) {
     auto rb = std::make_shared < api_registry_builder > (ctx.api_doc);
 
-    return ctx.http_server.set_routes([rb, &ctx, &tm, &cfg = *cfg](routes& r) {
+    return ctx.http_server.set_routes([rb, &ctx, &tm, &cfg = *cfg, &gossiper](routes& r) {
         rb->register_function(r, "task_manager",
                 "The task manager API");
-        set_task_manager(ctx, r, tm, cfg);
+        set_task_manager(ctx, r, tm, cfg, gossiper);
     });
 }
 

api/api_init.hh

@@ -131,7 +131,7 @@ future<> unset_server_cache(http_context& ctx);
 future<> set_server_compaction_manager(http_context& ctx, sharded<compaction_manager>& cm);
 future<> unset_server_compaction_manager(http_context& ctx);
 future<> set_server_done(http_context& ctx);
-future<> set_server_task_manager(http_context& ctx, sharded<tasks::task_manager>& tm, lw_shared_ptr<db::config> cfg);
+future<> set_server_task_manager(http_context& ctx, sharded<tasks::task_manager>& tm, lw_shared_ptr<db::config> cfg, sharded<gms::gossiper>& gossiper);
 future<> unset_server_task_manager(http_context& ctx);
 future<> set_server_task_manager_test(http_context& ctx, sharded<tasks::task_manager>& tm);
 future<> unset_server_task_manager_test(http_context& ctx);

api/collectd.cc

@@ -10,7 +10,6 @@
 #include "api/api-doc/collectd.json.hh"
 #include <seastar/core/scollectd.hh>
 #include <seastar/core/scollectd_api.hh>
-#include <boost/range/irange.hpp>
 #include <ranges>
 #include <regex>
 #include "api/api_init.hh"

api/column_family.cc

@@ -24,9 +24,6 @@
 #include "compaction/compaction_manager.hh"
 #include "unimplemented.hh"
 
-#include <boost/range/algorithm/copy.hpp>
-#include <boost/range/numeric.hpp>
-
 extern logging::logger apilog;
 
 namespace api {
@@ -51,17 +48,9 @@ std::tuple<sstring, sstring> parse_fully_qualified_cf_name(sstring name) {
     return std::make_tuple(name.substr(0, pos), name.substr(end));
 }
 
-table_id get_uuid(const sstring& ks, const sstring& cf, const replica::database& db) {
-    try {
-        return db.find_uuid(ks, cf);
-    } catch (replica::no_such_column_family& e) {
-        throw bad_param_exception(e.what());
-    }
-}
-
-table_id get_uuid(const sstring& name, const replica::database& db) {
+table_info parse_table_info(const sstring& name, const replica::database& db) {
     auto [ks, cf] = parse_fully_qualified_cf_name(name);
-    return get_uuid(ks, cf, db);
+    return table_info{ .name = cf, .id = validate_table(db, ks, cf) };
 }
 
 future<json::json_return_type>  get_cf_stats(http_context& ctx, const sstring& name,
@@ -78,15 +67,11 @@ future<json::json_return_type>  get_cf_stats(http_context& ctx,
     }, std::plus<int64_t>());
 }
 
-static future<json::json_return_type> for_tables_on_all_shards(http_context& ctx, const sstring& keyspace, std::vector<sstring> tables, std::function<future<>(replica::table&)> set) {
-    if (tables.empty()) {
-        tables = map_keys(ctx.db.local().find_keyspace(keyspace).metadata().get()->cf_meta_data());
-    }
-
-    return do_with(keyspace, std::move(tables), [&ctx, set] (const sstring& keyspace, const std::vector<sstring>& tables) {
-        return ctx.db.invoke_on_all([&keyspace, &tables, set] (replica::database& db) {
-            return parallel_for_each(tables, [&db, &keyspace, set] (const sstring& table) {
-                replica::table& t = db.find_column_family(keyspace, table);
+static future<json::json_return_type> for_tables_on_all_shards(http_context& ctx, std::vector<table_info> tables, std::function<future<>(replica::table&)> set) {
+    return do_with(std::move(tables), [&ctx, set] (const std::vector<table_info>& tables) {
+        return ctx.db.invoke_on_all([&tables, set] (replica::database& db) {
+            return parallel_for_each(tables, [&db, set] (const table_info& table) {
+                replica::table& t = db.find_column_family(table.id);
                 return set(t);
             });
         });
@@ -113,12 +98,12 @@ public:
     }
 };
 
-static future<json::json_return_type> set_tables_autocompaction(http_context& ctx, const sstring &keyspace, std::vector<sstring> tables, bool enabled) {
-    apilog.info("set_tables_autocompaction: enabled={} keyspace={} tables={}", enabled, keyspace, tables);
+static future<json::json_return_type> set_tables_autocompaction(http_context& ctx, std::vector<table_info> tables, bool enabled) {
+    apilog.info("set_tables_autocompaction: enabled={} tables={}", enabled, tables);
 
-    return ctx.db.invoke_on(0, [&ctx, keyspace, tables = std::move(tables), enabled] (replica::database& db) {
+    return ctx.db.invoke_on(0, [&ctx, tables = std::move(tables), enabled] (replica::database& db) {
         auto g = autocompaction_toggle_guard(db);
-        return for_tables_on_all_shards(ctx, keyspace, tables, [enabled] (replica::table& cf) {
+        return for_tables_on_all_shards(ctx, tables, [enabled] (replica::table& cf) {
             if (enabled) {
                 cf.enable_auto_compaction();
             } else {
@@ -129,9 +114,9 @@ static future<json::json_return_type> set_tables_autocompaction(http_context& ct
     });
 }
 
-static future<json::json_return_type> set_tables_tombstone_gc(http_context& ctx, const sstring &keyspace, std::vector<sstring> tables, bool enabled) {
-    apilog.info("set_tables_tombstone_gc: enabled={} keyspace={} tables={}", enabled, keyspace, tables);
-    return for_tables_on_all_shards(ctx, keyspace, std::move(tables), [enabled] (replica::table& t) {
+static future<json::json_return_type> set_tables_tombstone_gc(http_context& ctx, std::vector<table_info> tables, bool enabled) {
+    apilog.info("set_tables_tombstone_gc: enabled={} tables={}", enabled, tables);
+    return for_tables_on_all_shards(ctx, std::move(tables), [enabled] (replica::table& t) {
         t.set_tombstone_gc_enabled(enabled);
         return make_ready_future<>();
     });
@@ -146,7 +131,7 @@ static future<json::json_return_type>  get_cf_stats_count(http_context& ctx, con
 
 static future<json::json_return_type>  get_cf_stats_sum(http_context& ctx, const sstring& name,
         utils::timed_rate_moving_average_summary_and_histogram replica::column_family_stats::*f) {
-    auto uuid = get_uuid(name, ctx.db.local());
+    auto uuid = parse_table_info(name, ctx.db.local()).id;
     return ctx.db.map_reduce0([uuid, f](replica::database& db) {
         // Histograms information is sample of the actual load
         // so to get an estimation of sum, we multiply the mean
@@ -169,7 +154,7 @@ static future<json::json_return_type>  get_cf_stats_count(http_context& ctx,
 
 static future<json::json_return_type>  get_cf_histogram(http_context& ctx, const sstring& name,
         utils::timed_rate_moving_average_and_histogram replica::column_family_stats::*f) {
-    auto uuid = get_uuid(name, ctx.db.local());
+    auto uuid = parse_table_info(name, ctx.db.local()).id;
     return ctx.db.map_reduce0([f, uuid](const replica::database& p) {
         return (p.find_column_family(uuid).get_stats().*f).hist;},
             utils::ihistogram(),
@@ -181,7 +166,7 @@ static future<json::json_return_type>  get_cf_histogram(http_context& ctx, const
 
 static future<json::json_return_type>  get_cf_histogram(http_context& ctx, const sstring& name,
         utils::timed_rate_moving_average_summary_and_histogram replica::column_family_stats::*f) {
-    auto uuid = get_uuid(name, ctx.db.local());
+    auto uuid = parse_table_info(name, ctx.db.local()).id;
     return ctx.db.map_reduce0([f, uuid](const replica::database& p) {
         return (p.find_column_family(uuid).get_stats().*f).hist;},
             utils::ihistogram(),
@@ -208,7 +193,7 @@ static future<json::json_return_type> get_cf_histogram(http_context& ctx, utils:
 
 static future<json::json_return_type>  get_cf_rate_and_histogram(http_context& ctx, const sstring& name,
         utils::timed_rate_moving_average_summary_and_histogram replica::column_family_stats::*f) {
-    auto uuid = get_uuid(name, ctx.db.local());
+    auto uuid = parse_table_info(name, ctx.db.local()).id;
     return ctx.db.map_reduce0([f, uuid](const replica::database& p) {
         return (p.find_column_family(uuid).get_stats().*f).rate();},
             utils::rate_moving_average_and_histogram(),
@@ -265,48 +250,29 @@ static integral_ratio_holder mean_partition_size(replica::column_family& cf) {
     return res;
 }
 
-static std::unordered_map<sstring, uint64_t> merge_maps(std::unordered_map<sstring, uint64_t> a,
-        const std::unordered_map<sstring, uint64_t>& b) {
-    a.insert(b.begin(), b.end());
-    return a;
-}
-
-static json::json_return_type sum_map(const std::unordered_map<sstring, uint64_t>& val) {
-    uint64_t res = 0;
-    for (auto i : val) {
-        res += i.second;
+static auto count_bytes_on_disk(const replica::column_family& cf, bool total) {
+    uint64_t bytes_on_disk = 0;
+    auto sstables = (total) ? cf.get_sstables_including_compacted_undeleted() : cf.get_sstables();
+    for (auto t : *sstables) {
+        bytes_on_disk += t->bytes_on_disk();
     }
-    return res;
+    return bytes_on_disk;
 }
 
 static future<json::json_return_type>  sum_sstable(http_context& ctx, const sstring name, bool total) {
-    auto uuid = get_uuid(name, ctx.db.local());
-    return ctx.db.map_reduce0([uuid, total](replica::database& db) {
-        std::unordered_map<sstring, uint64_t> m;
-        auto sstables = (total) ? db.find_column_family(uuid).get_sstables_including_compacted_undeleted() :
-                db.find_column_family(uuid).get_sstables();
-        for (auto t : *sstables) {
-            m[t->get_filename()] = t->bytes_on_disk();
-        }
-        return m;
-    }, std::unordered_map<sstring, uint64_t>(), merge_maps).
-            then([](const std::unordered_map<sstring, uint64_t>& val) {
-        return sum_map(val);
+    return map_reduce_cf_raw(ctx, name, uint64_t(0), [total](replica::column_family& cf) {
+        return count_bytes_on_disk(cf, total);
+    }, std::plus<>()).then([] (uint64_t val) {
+        return make_ready_future<json::json_return_type>(val);
     });
 }
 
 
 static future<json::json_return_type> sum_sstable(http_context& ctx, bool total) {
-    return map_reduce_cf_raw(ctx, std::unordered_map<sstring, uint64_t>(), [total](replica::column_family& cf) {
-        std::unordered_map<sstring, uint64_t> m;
-        auto sstables = (total) ? cf.get_sstables_including_compacted_undeleted() :
-                cf.get_sstables();
-        for (auto t : *sstables) {
-            m[t->get_filename()] = t->bytes_on_disk();
-        }
-        return m;
-    },merge_maps).then([](const std::unordered_map<sstring, uint64_t>& val) {
-        return sum_map(val);
+    return map_reduce_cf_raw(ctx, uint64_t(0), [total](replica::column_family& cf) {
+        return count_bytes_on_disk(cf, total);
+    }, std::plus<>()).then([] (uint64_t val) {
+        return make_ready_future<json::json_return_type>(val);
     });
 }
 
@@ -918,75 +884,71 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_auto_compaction.set(r, [&ctx] (const_req req) {
-        auto uuid = get_uuid(req.get_path_param("name"), ctx.db.local());
+        auto uuid = parse_table_info(req.get_path_param("name"), ctx.db.local()).id;
         replica::column_family& cf = ctx.db.local().find_column_family(uuid);
         return !cf.is_auto_compaction_disabled_by_user();
     });
 
     cf::enable_auto_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) {
         apilog.info("column_family/enable_auto_compaction: name={}", req->get_path_param("name"));
-        auto [ks, cf] = parse_fully_qualified_cf_name(req->get_path_param("name"));
-        validate_table(ctx, ks, cf);
-        return set_tables_autocompaction(ctx, ks, {std::move(cf)}, true);
+        auto ti = parse_table_info(req->get_path_param("name"), ctx.db.local());
+        return set_tables_autocompaction(ctx, {std::move(ti)}, true);
     });
 
     cf::disable_auto_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) {
         apilog.info("column_family/disable_auto_compaction: name={}", req->get_path_param("name"));
-        auto [ks, cf] = parse_fully_qualified_cf_name(req->get_path_param("name"));
-        validate_table(ctx, ks, cf);
-        return set_tables_autocompaction(ctx, ks, {std::move(cf)}, false);
+        auto ti = parse_table_info(req->get_path_param("name"), ctx.db.local());
+        return set_tables_autocompaction(ctx, {std::move(ti)}, false);
     });
 
     ss::enable_auto_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) {
         auto keyspace = validate_keyspace(ctx, req);
-        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
+        auto tables = parse_table_infos(keyspace, ctx, req->query_parameters, "cf");
 
         apilog.info("enable_auto_compaction: keyspace={} tables={}", keyspace, tables);
-        return set_tables_autocompaction(ctx, keyspace, tables, true);
+        return set_tables_autocompaction(ctx, std::move(tables), true);
     });
 
     ss::disable_auto_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) {
         auto keyspace = validate_keyspace(ctx, req);
-        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
+        auto tables = parse_table_infos(keyspace, ctx, req->query_parameters, "cf");
 
         apilog.info("disable_auto_compaction: keyspace={} tables={}", keyspace, tables);
-        return set_tables_autocompaction(ctx, keyspace, tables, false);
+        return set_tables_autocompaction(ctx, std::move(tables), false);
     });
 
     cf::get_tombstone_gc.set(r, [&ctx] (const_req req) {
-        auto uuid = get_uuid(req.get_path_param("name"), ctx.db.local());
+        auto uuid = parse_table_info(req.get_path_param("name"), ctx.db.local()).id;
         replica::table& t = ctx.db.local().find_column_family(uuid);
         return t.tombstone_gc_enabled();
     });
 
     cf::enable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
         apilog.info("column_family/enable_tombstone_gc: name={}", req->get_path_param("name"));
-        auto [ks, cf] = parse_fully_qualified_cf_name(req->get_path_param("name"));
-        validate_table(ctx, ks, cf);
-        return set_tables_tombstone_gc(ctx, ks, {std::move(cf)}, true);
+        auto ti = parse_table_info(req->get_path_param("name"), ctx.db.local());
+        return set_tables_tombstone_gc(ctx, {std::move(ti)}, true);
     });
 
     cf::disable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
         apilog.info("column_family/disable_tombstone_gc: name={}", req->get_path_param("name"));
-        auto [ks, cf] = parse_fully_qualified_cf_name(req->get_path_param("name"));
-        validate_table(ctx, ks, cf);
-        return set_tables_tombstone_gc(ctx, ks, {std::move(cf)}, false);
+        auto ti = parse_table_info(req->get_path_param("name"), ctx.db.local());
+        return set_tables_tombstone_gc(ctx, {std::move(ti)}, false);
     });
 
     ss::enable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
         auto keyspace = validate_keyspace(ctx, req);
-        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
+        auto tables = parse_table_infos(keyspace, ctx, req->query_parameters, "cf");
 
         apilog.info("enable_tombstone_gc: keyspace={} tables={}", keyspace, tables);
-        return set_tables_tombstone_gc(ctx, keyspace, tables, true);
+        return set_tables_tombstone_gc(ctx, std::move(tables), true);
     });
 
     ss::disable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
         auto keyspace = validate_keyspace(ctx, req);
-        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
+        auto tables = parse_table_infos(keyspace, ctx, req->query_parameters, "cf");
 
         apilog.info("disable_tombstone_gc: keyspace={} tables={}", keyspace, tables);
-        return set_tables_tombstone_gc(ctx, keyspace, tables, false);
+        return set_tables_tombstone_gc(ctx, std::move(tables), false);
     });
 
     cf::get_built_indexes.set(r, [&ctx, &sys_ks](std::unique_ptr<http::request> req) {
@@ -1003,7 +965,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
                 }
             }
             std::vector<sstring> res;
-            auto uuid = get_uuid(ks, cf_name, ctx.db.local());
+            auto uuid = validate_table(ctx.db.local(), ks, cf_name);
             replica::column_family& cf = ctx.db.local().find_column_family(uuid);
             res.reserve(cf.get_index_manager().list_indexes().size());
             for (auto&& i : cf.get_index_manager().list_indexes()) {
@@ -1030,7 +992,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_compression_ratio.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        auto uuid = get_uuid(req->get_path_param("name"), ctx.db.local());
+        auto uuid = parse_table_info(req->get_path_param("name"), ctx.db.local()).id;
 
         return ctx.db.map_reduce(sum_ratio<double>(), [uuid](replica::database& db) {
             replica::column_family& cf = db.find_column_family(uuid);
@@ -1053,17 +1015,17 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::set_compaction_strategy_class.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        auto [ks, cf] = parse_fully_qualified_cf_name(req->get_path_param("name"));
+        auto ti = parse_table_info(req->get_path_param("name"), ctx.db.local());
         sstring strategy = req->get_query_param("class_name");
         apilog.info("column_family/set_compaction_strategy_class: name={} strategy={}", req->get_path_param("name"), strategy);
-        return for_tables_on_all_shards(ctx, ks, {std::move(cf)}, [strategy] (replica::table& cf) {
+        return for_tables_on_all_shards(ctx, {std::move(ti)}, [strategy] (replica::table& cf) {
             cf.set_compaction_strategy(sstables::compaction_strategy::type(strategy));
             return make_ready_future<>();
         });
     });
 
     cf::get_compaction_strategy_class.set(r, [&ctx](const_req req) {
-        return ctx.db.local().find_column_family(get_uuid(req.get_path_param("name"), ctx.db.local())).get_compaction_strategy().name();
+        return ctx.db.local().find_column_family(parse_table_info(req.get_path_param("name"), ctx.db.local()).id).get_compaction_strategy().name();
     });
 
     cf::set_compression_parameters.set(r, [](std::unique_ptr<http::request> req) {
@@ -1088,7 +1050,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
 
     cf::get_sstables_for_key.set(r, [&ctx](std::unique_ptr<http::request> req) {
         auto key = req->get_query_param("key");
-        auto uuid = get_uuid(req->get_path_param("name"), ctx.db.local());
+        auto uuid = parse_table_info(req->get_path_param("name"), ctx.db.local()).id;
 
         return ctx.db.map_reduce0([key, uuid] (replica::database& db) -> future<std::unordered_set<sstring>> {
             auto sstables = co_await db.find_column_family(uuid).get_sstables_by_partition_key(key);

api/column_family.hh

@@ -22,12 +22,12 @@ namespace api {
 void set_column_family(http_context& ctx, httpd::routes& r, sharded<db::system_keyspace>& sys_ks);
 void unset_column_family(http_context& ctx, httpd::routes& r);
 
-table_id get_uuid(const sstring& name, const replica::database& db);
+table_info parse_table_info(const sstring& name, const replica::database& db);
 
 template<class Mapper, class I, class Reducer>
 future<I> map_reduce_cf_raw(http_context& ctx, const sstring& name, I init,
         Mapper mapper, Reducer reducer) {
-    auto uuid = get_uuid(name, ctx.db.local());
+    auto uuid = parse_table_info(name, ctx.db.local()).id;
     using mapper_type = std::function<std::unique_ptr<std::any>(replica::database&)>;
     using reducer_type = std::function<std::unique_ptr<std::any>(std::unique_ptr<std::any>, std::unique_ptr<std::any>)>;
     return ctx.db.map_reduce0(mapper_type([mapper, uuid](replica::database& db) {

api/compaction_manager.cc

@@ -112,12 +112,12 @@ void set_compaction_manager(http_context& ctx, routes& r, sharded<compaction_man
 
     cm::stop_keyspace_compaction.set(r, [&ctx] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
         auto ks_name = validate_keyspace(ctx, req);
-        auto table_names = parse_tables(ks_name, ctx, req->query_parameters, "tables");
+        auto tables = parse_table_infos(ks_name, ctx, req->query_parameters, "tables");
         auto type = req->get_query_param("type");
         co_await ctx.db.invoke_on_all([&] (replica::database& db) {
             auto& cm = db.get_compaction_manager();
-            return parallel_for_each(table_names, [&] (sstring& table_name) {
-                auto& t = db.find_column_family(ks_name, table_name);
+            return parallel_for_each(tables, [&] (const table_info& ti) {
+                auto& t = db.find_column_family(ti.id);
                 return t.parallel_foreach_table_state([&] (compaction::table_state& ts) {
                     return cm.stop_compaction(type, &ts);
                 });

api/storage_service.hh

@@ -43,16 +43,9 @@ sstring validate_keyspace(const http_context& ctx, sstring ks_name);
 // containing the description of the respective keyspace error.
 sstring validate_keyspace(const http_context& ctx, const std::unique_ptr<http::request>& req);
 
-// verify that the table parameter is found, otherwise a bad_param_exception exception is thrown
-// containing the description of the respective table error.
-void validate_table(const http_context& ctx, sstring ks_name, sstring table_name);
-
-// splits a request parameter assumed to hold a comma-separated list of table names
-// verify that the tables are found, otherwise a bad_param_exception exception is thrown
-// containing the description of the respective no_such_column_family error.
-// Returns an empty vector if no parameter was found.
-// If the parameter is found and empty, returns a list of all table names in the keyspace.
-std::vector<sstring> parse_tables(const sstring& ks_name, const http_context& ctx, const std::unordered_map<sstring, sstring>& query_params, sstring param_name);
+// verify that the keyspace:table is found, otherwise a bad_param_exception exception is thrown
+// returns the table_id of the table if found
+table_id validate_table(const replica::database& db, sstring ks_name, sstring table_name);
 
 // splits a request parameter assumed to hold a comma-separated list of table names
 // verify that the tables are found, otherwise a bad_param_exception exception is thrown

api/task_manager.cc

@@ -14,6 +14,7 @@
 #include "api/api.hh"
 #include "api/api-doc/task_manager.json.hh"
 #include "db/system_keyspace.hh"
+#include "gms/gossiper.hh"
 #include "tasks/task_handler.hh"
 #include "utils/overloaded_functor.hh"
 
@@ -25,18 +26,23 @@ namespace tm = httpd::task_manager_json;
 using namespace json;
 using namespace seastar::httpd;
 
-tm::task_status make_status(tasks::task_status status) {
-    auto start_time = db_clock::to_time_t(status.start_time);
-    auto end_time = db_clock::to_time_t(status.end_time);
-    ::tm st, et;
-    ::gmtime_r(&end_time, &et);
-    ::gmtime_r(&start_time, &st);
+static ::tm get_time(db_clock::time_point tp) {
+    auto time = db_clock::to_time_t(tp);
+    ::tm t;
+    ::gmtime_r(&time, &t);
+    return t;
+}
 
+tm::task_status make_status(tasks::task_status status, sharded<gms::gossiper>& gossiper) {
     std::vector<tm::task_identity> tis{status.children.size()};
-    std::ranges::transform(status.children, tis.begin(), [] (const auto& child) {
+    std::ranges::transform(status.children, tis.begin(), [&gossiper] (const auto& child) {
         tm::task_identity ident;
+        gms::inet_address addr{};
+        if (gossiper.local_is_initialized()) {
+            addr = gossiper.local().get_address_map().find(child.host_id).value_or(gms::inet_address{});
+        }
         ident.task_id = child.task_id.to_sstring();
-        ident.node = fmt::format("{}", child.node);
+        ident.node = fmt::format("{}", addr);
         return ident;
     });
 
@@ -47,8 +53,8 @@ tm::task_status make_status(tasks::task_status status) {
     res.scope = status.scope;
     res.state = status.state;
     res.is_abortable = bool(status.is_abortable);
-    res.start_time = st;
-    res.end_time = et;
+    res.start_time = get_time(status.start_time);
+    res.end_time = get_time(status.end_time);
     res.error = status.error;
     res.parent_id = status.parent_id ? status.parent_id.to_sstring() : "none";
     res.sequence_number = status.sequence_number;
@@ -74,10 +80,13 @@ tm::task_stats make_stats(tasks::task_stats stats) {
     res.keyspace = stats.keyspace;
     res.table = stats.table;
     res.entity = stats.entity;
+    res.shard = stats.shard;
+    res.start_time = get_time(stats.start_time);
+    res.end_time = get_time(stats.end_time);;
     return res;
 }
 
-void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>& tm, db::config& cfg) {
+void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>& tm, db::config& cfg, sharded<gms::gossiper>& gossiper) {
     tm::get_modules.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
         std::vector<std::string> v = tm.local().get_modules() | std::views::keys | std::ranges::to<std::vector>();
         co_return v;
@@ -135,7 +144,7 @@ void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>
         co_return std::move(f);
     });
 
-    tm::get_task_status.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
+    tm::get_task_status.set(r, [&tm, &gossiper] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
         auto id = tasks::task_id{utils::UUID{req->get_path_param("task_id")}};
         tasks::task_status status;
         try {
@@ -144,7 +153,7 @@ void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>
         } catch (tasks::task_manager::task_not_found& e) {
             throw bad_param_exception(e.what());
         }
-        co_return make_status(status);
+        co_return make_status(status, gossiper);
     });
 
     tm::abort_task.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
@@ -160,7 +169,7 @@ void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>
         co_return json_void();
     });
 
-    tm::wait_task.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
+    tm::wait_task.set(r, [&tm, &gossiper] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
         auto id = tasks::task_id{utils::UUID{req->get_path_param("task_id")}};
         tasks::task_status status;
         std::optional<std::chrono::seconds> timeout = std::nullopt;
@@ -175,24 +184,24 @@ void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>
         } catch (timed_out_error& e) {
             throw httpd::base_exception{e.what(), http::reply::status_type::request_timeout};
         }
-        co_return make_status(status);
+        co_return make_status(status, gossiper);
     });
 
-    tm::get_task_status_recursively.set(r, [&_tm = tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
+    tm::get_task_status_recursively.set(r, [&_tm = tm, &gossiper] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
         auto& tm = _tm;
         auto id = tasks::task_id{utils::UUID{req->get_path_param("task_id")}};
         try {
             auto task = tasks::task_handler{tm.local(), id};
             auto res = co_await task.get_status_recursively(true);
 
-            std::function<future<>(output_stream<char>&&)> f = [r = std::move(res)] (output_stream<char>&& os) -> future<> {
+            std::function<future<>(output_stream<char>&&)> f = [r = std::move(res), &gossiper] (output_stream<char>&& os) -> future<> {
                 auto s = std::move(os);
                 auto res = std::move(r);
                 co_await s.write("[");
                 std::string delim = "";
                 for (auto& status: res) {
                     co_await s.write(std::exchange(delim, ", "));
-                    co_await formatter::write(s, make_status(status));
+                    co_await formatter::write(s, make_status(status, gossiper));
                 }
                 co_await s.write("]");
                 co_await s.close();

api/task_manager.hh

@@ -18,7 +18,7 @@ namespace tasks {
 
 namespace api {
 
-void set_task_manager(http_context& ctx, httpd::routes& r, sharded<tasks::task_manager>& tm, db::config& cfg);
+void set_task_manager(http_context& ctx, httpd::routes& r, sharded<tasks::task_manager>& tm, db::config& cfg, sharded<gms::gossiper>& gossiper);
 void unset_task_manager(http_context& ctx, httpd::routes& r);
 
 }

api/tasks.cc

@@ -37,74 +37,45 @@ static auto wrap_ks_cf(http_context &ctx, ks_cf_func f) {
     };
 }
 
-future<tasks::task_manager::task_ptr> force_keyspace_compaction(http_context& ctx, std::unique_ptr<http::request> req) {
-    auto& db = ctx.db;
-    auto params = req_params({
-        std::pair("keyspace", mandatory::yes),
-        std::pair("cf", mandatory::no),
-        std::pair("flush_memtables", mandatory::no),
-        std::pair("consider_only_existing_data", mandatory::no),
-    });
-    params.process(*req);
-    auto keyspace = validate_keyspace(ctx, *params.get("keyspace"));
-    auto table_infos = parse_table_infos(keyspace, ctx, params.get("cf").value_or(""));
-    auto flush = params.get_as<bool>("flush_memtables").value_or(true);
-    auto consider_only_existing_data = params.get_as<bool>("consider_only_existing_data").value_or(false);
-    apilog.info("force_keyspace_compaction: keyspace={} tables={}, flush={} consider_only_existing_data={}", keyspace, table_infos, flush, consider_only_existing_data);
-
-    auto& compaction_module = db.local().get_compaction_manager().get_task_manager_module();
-    std::optional<compaction::flush_mode> fmopt;
-    if (!flush && !consider_only_existing_data) {
-        fmopt = compaction::flush_mode::skip;
-    }
-    return compaction_module.make_and_start_task<compaction::major_keyspace_compaction_task_impl>({}, std::move(keyspace), tasks::task_id::create_null_id(), db, table_infos, fmopt, consider_only_existing_data);
-}
-
-future<tasks::task_manager::task_ptr> upgrade_sstables(http_context& ctx, std::unique_ptr<http::request> req, sstring keyspace, std::vector<table_info> table_infos) {
-    auto& db = ctx.db;
-    bool exclude_current_version = req_param<bool>(*req, "exclude_current_version", false);
-
-    apilog.info("upgrade_sstables: keyspace={} tables={} exclude_current_version={}", keyspace, table_infos, exclude_current_version);
-
-    auto& compaction_module = db.local().get_compaction_manager().get_task_manager_module();
-    return compaction_module.make_and_start_task<compaction::upgrade_sstables_compaction_task_impl>({}, std::move(keyspace), db, table_infos, exclude_current_version);
-}
-
-future<tasks::task_manager::task_ptr> force_keyspace_cleanup(http_context& ctx, sharded<service::storage_service>& ss, std::unique_ptr<http::request> req) {
-    auto& db = ctx.db;
-    auto keyspace = validate_keyspace(ctx, req);
-    auto table_infos = parse_table_infos(keyspace, ctx, req->query_parameters, "cf");
-    const auto& rs = db.local().find_keyspace(keyspace).get_replication_strategy();
-    if (rs.get_type() == locator::replication_strategy_type::local || !rs.is_vnode_based()) {
-        auto reason = rs.get_type() == locator::replication_strategy_type::local ? "require" : "support";
-        apilog.info("Keyspace {} does not {} cleanup", keyspace, reason);
-        co_return nullptr;
-    }
-    apilog.info("force_keyspace_cleanup: keyspace={} tables={}", keyspace, table_infos);
-    if (!co_await ss.local().is_cleanup_allowed(keyspace)) {
-        auto msg = "Can not perform cleanup operation when topology changes";
-        apilog.warn("force_keyspace_cleanup: keyspace={} tables={}: {}", keyspace, table_infos, msg);
-        co_await coroutine::return_exception(std::runtime_error(msg));
-    }
-
-    auto& compaction_module = db.local().get_compaction_manager().get_task_manager_module();
-    co_return co_await compaction_module.make_and_start_task<compaction::cleanup_keyspace_compaction_task_impl>(
-        {}, std::move(keyspace), db, table_infos, compaction::flush_mode::all_tables, tasks::is_user_task::yes);
-}
-
 void set_tasks_compaction_module(http_context& ctx, routes& r, sharded<service::storage_service>& ss, sharded<db::snapshot_ctl>& snap_ctl) {
     t::force_keyspace_compaction_async.set(r, [&ctx](std::unique_ptr<http::request> req) -> future<json::json_return_type> {
-        auto task = co_await force_keyspace_compaction(ctx, std::move(req));
+        auto& db = ctx.db;
+        auto params = req_params({
+            std::pair("keyspace", mandatory::yes),
+            std::pair("cf", mandatory::no),
+            std::pair("flush_memtables", mandatory::no),
+        });
+        params.process(*req);
+        auto keyspace = validate_keyspace(ctx, *params.get("keyspace"));
+        auto table_infos = parse_table_infos(keyspace, ctx, params.get("cf").value_or(""));
+        auto flush = params.get_as<bool>("flush_memtables").value_or(true);
+        apilog.debug("force_keyspace_compaction_async: keyspace={} tables={}, flush={}", keyspace, table_infos, flush);
+
+        auto& compaction_module = db.local().get_compaction_manager().get_task_manager_module();
+        std::optional<flush_mode> fmopt;
+        if (!flush) {
+            fmopt = flush_mode::skip;
+        }
+        auto task = co_await compaction_module.make_and_start_task<major_keyspace_compaction_task_impl>({}, std::move(keyspace), tasks::task_id::create_null_id(), db, table_infos, fmopt);
+
         co_return json::json_return_type(task->get_status().id.to_sstring());
     });
 
     t::force_keyspace_cleanup_async.set(r, [&ctx, &ss](std::unique_ptr<http::request> req) -> future<json::json_return_type> {
-        tasks::task_id id = tasks::task_id::create_null_id();
-        auto task = co_await force_keyspace_cleanup(ctx, ss, std::move(req));
-        if (task) {
-            id = task->get_status().id;
+        auto& db = ctx.db;
+        auto keyspace = validate_keyspace(ctx, req);
+        auto table_infos = parse_table_infos(keyspace, ctx, req->query_parameters, "cf");
+        apilog.info("force_keyspace_cleanup_async: keyspace={} tables={}", keyspace, table_infos);
+        if (!co_await ss.local().is_cleanup_allowed(keyspace)) {
+            auto msg = "Can not perform cleanup operation when topology changes";
+            apilog.warn("force_keyspace_cleanup_async: keyspace={} tables={}: {}", keyspace, table_infos, msg);
+            co_await coroutine::return_exception(std::runtime_error(msg));
         }
-        co_return json::json_return_type(id.to_sstring());
+
+        auto& compaction_module = db.local().get_compaction_manager().get_task_manager_module();
+        auto task = co_await compaction_module.make_and_start_task<cleanup_keyspace_compaction_task_impl>({}, std::move(keyspace), db, table_infos, flush_mode::all_tables, tasks::is_user_task::yes);
+
+        co_return json::json_return_type(task->get_status().id.to_sstring());
     });
 
     t::perform_keyspace_offstrategy_compaction_async.set(r, wrap_ks_cf(ctx, [] (http_context& ctx, std::unique_ptr<http::request> req, sstring keyspace, std::vector<table_info> table_infos) -> future<json::json_return_type> {
@@ -116,7 +87,14 @@ void set_tasks_compaction_module(http_context& ctx, routes& r, sharded<service::
     }));
 
     t::upgrade_sstables_async.set(r, wrap_ks_cf(ctx, [] (http_context& ctx, std::unique_ptr<http::request> req, sstring keyspace, std::vector<table_info> table_infos) -> future<json::json_return_type> {
-        auto task = co_await upgrade_sstables(ctx, std::move(req), std::move(keyspace), std::move(table_infos));
+        auto& db = ctx.db;
+        bool exclude_current_version = req_param<bool>(*req, "exclude_current_version", false);
+
+        apilog.info("upgrade_sstables: keyspace={} tables={} exclude_current_version={}", keyspace, table_infos, exclude_current_version);
+
+        auto& compaction_module = db.local().get_compaction_manager().get_task_manager_module();
+        auto task = co_await compaction_module.make_and_start_task<upgrade_sstables_compaction_task_impl>({}, std::move(keyspace), db, table_infos, exclude_current_version);
+
         co_return json::json_return_type(task->get_status().id.to_sstring());
     }));
 

api/tasks.hh

@@ -15,10 +15,6 @@ namespace seastar::httpd {
 class routes;
 }
 
-namespace seastar::http {
-struct request;
-}
-
 namespace service {
 class storage_service;
 }
@@ -29,8 +25,4 @@ struct http_context;
 void set_tasks_compaction_module(http_context& ctx, httpd::routes& r, sharded<service::storage_service>& ss, sharded<db::snapshot_ctl>& snap_ctl);
 void unset_tasks_compaction_module(http_context& ctx, httpd::routes& r);
 
-future<tasks::task_manager::task_ptr> force_keyspace_compaction(http_context& ctx, std::unique_ptr<http::request> req);
-future<tasks::task_manager::task_ptr> force_keyspace_cleanup(http_context& ctx, sharded<service::storage_service>& ss, std::unique_ptr<http::request> req);
-future<tasks::task_manager::task_ptr> upgrade_sstables(http_context& ctx, std::unique_ptr<http::request> req, sstring keyspace, std::vector<table_info> table_infos);
-
 }

api/token_metadata.cc

@@ -74,9 +74,6 @@ void set_token_metadata(http_context& ctx, routes& r, sharded<locator::shared_to
     });
 
     ss::get_host_id_map.set(r, [&tm, &g](const_req req) {
-        if (!g.local().is_enabled()) {
-            throw std::runtime_error("The gossiper is not ready yet");
-        }
         std::vector<ss::mapper> res;
         auto map = tm.local().get()->get_host_ids() |
             std::views::transform([&g] (locator::host_id id) { return std::make_pair(g.local().get_address_map().get(id), id); }) |

audit/audit.cc

@@ -26,8 +26,9 @@ namespace audit {
 
 logging::logger logger("audit");
 
-sstring audit_info::category_string() const {
-    switch (_category) {
+static sstring category_to_string(statement_category category)
+{
+    switch (category) {
         case statement_category::QUERY: return "QUERY";
         case statement_category::DML: return "DML";
         case statement_category::DDL: return "DDL";
@@ -38,19 +39,9 @@ sstring audit_info::category_string() const {
     return "";
 }
 
-audit::audit(locator::shared_token_metadata& token_metadata,
-             sstring&& storage_helper_name,
-             std::set<sstring>&& audited_keyspaces,
-             std::map<sstring, std::set<sstring>>&& audited_tables,
-             category_set&& audited_categories)
-    : _token_metadata(token_metadata)
-    , _audited_keyspaces(std::move(audited_keyspaces))
-    , _audited_tables(std::move(audited_tables))
-    , _audited_categories(std::move(audited_categories))
-    , _storage_helper_class_name(std::move(storage_helper_name))
-{ }
-
-audit::~audit() = default;
+sstring audit_info::category_string() const {
+    return category_to_string(_category);
+}
 
 static category_set parse_audit_categories(const sstring& data) {
     category_set result;
@@ -111,6 +102,25 @@ static std::set<sstring> parse_audit_keyspaces(const sstring& data) {
     return result;
 }
 
+audit::audit(locator::shared_token_metadata& token_metadata,
+             sstring&& storage_helper_name,
+             std::set<sstring>&& audited_keyspaces,
+             std::map<sstring, std::set<sstring>>&& audited_tables,
+             category_set&& audited_categories,
+             const db::config& cfg)
+    : _token_metadata(token_metadata)
+    , _audited_keyspaces(std::move(audited_keyspaces))
+    , _audited_tables(std::move(audited_tables))
+    , _audited_categories(std::move(audited_categories))
+    , _storage_helper_class_name(std::move(storage_helper_name))
+    , _cfg(cfg)
+    , _cfg_keyspaces_observer(cfg.audit_keyspaces.observe([this] (sstring const& new_value){ update_config<std::set<sstring>>(new_value, parse_audit_keyspaces, _audited_keyspaces); }))
+    , _cfg_tables_observer(cfg.audit_tables.observe([this] (sstring const& new_value){ update_config<std::map<sstring, std::set<sstring>>>(new_value, parse_audit_tables, _audited_tables); }))
+    , _cfg_categories_observer(cfg.audit_categories.observe([this] (sstring const& new_value){ update_config<category_set>(new_value, parse_audit_categories, _audited_categories); }))
+{ }
+
+audit::~audit() = default;
+
 future<> audit::create_audit(const db::config& cfg, sharded<locator::shared_token_metadata>& stm) {
     sstring storage_helper_name;
     if (cfg.audit() == "table") {
@@ -126,18 +136,9 @@ future<> audit::create_audit(const db::config& cfg, sharded<locator::shared_toke
         throw audit_exception(fmt::format("Bad configuration: invalid 'audit': {}", cfg.audit()));
     }
     category_set audited_categories = parse_audit_categories(cfg.audit_categories());
-    if (!audited_categories) {
-        return make_ready_future<>();
-    }
     std::map<sstring, std::set<sstring>> audited_tables = parse_audit_tables(cfg.audit_tables());
     std::set<sstring> audited_keyspaces = parse_audit_keyspaces(cfg.audit_keyspaces());
-    if (audited_tables.empty()
-        && audited_keyspaces.empty()
-        && !audited_categories.contains(statement_category::AUTH)
-        && !audited_categories.contains(statement_category::ADMIN)
-        && !audited_categories.contains(statement_category::DCL)) {
-        return make_ready_future<>();
-    }
+
     logger.info("Audit is enabled. Auditing to: \"{}\", with the following categories: \"{}\", keyspaces: \"{}\", and tables: \"{}\"",
                 cfg.audit(), cfg.audit_categories(), cfg.audit_keyspaces(), cfg.audit_tables());
 
@@ -145,7 +146,8 @@ future<> audit::create_audit(const db::config& cfg, sharded<locator::shared_toke
                                   std::move(storage_helper_name),
                                   std::move(audited_keyspaces),
                                   std::move(audited_tables),
-                                  std::move(audited_categories));
+                                  std::move(audited_categories),
+                                  std::cref(cfg));
 }
 
 future<> audit::start_audit(const db::config& cfg, sharded<cql3::query_processor>& qp, sharded<service::migration_manager>& mm) {
@@ -260,4 +262,33 @@ bool audit::should_log(const audit_info* audit_info) const {
                          || audit_info->category() == statement_category::DCL);
 }
 
+template<class T>
+void audit::update_config(const sstring & new_value, std::function<T(const sstring&)> parse_func, T& cfg_parameter)
+{
+    try {
+        cfg_parameter = parse_func(new_value);
+    } catch (...) {
+        logger.error("Audit configuration update failed because cannot parse value=\"{}\".", new_value);
+        return;
+    }
+
+    // If update_config is called with an invalid new_value, this line is not reached.
+    // But logging the invalid value must be avoided later, when a different configuration parameter is changed to a correct value.
+    // That's why values from _audited_{categories, keyspaces, tables} are logged instead of _cfg.audit_{categories, keyspaces, tables}
+
+    // Each table as "keyspace.table_name" like in the configuration file
+    auto table_entries = _audited_tables | std::views::transform([](const auto& pair) {
+        return pair.second | std::views::transform([&](const std::string& table_name) {
+            return fmt::format("{}.{}", pair.first, table_name);
+        });
+    }) | std::views::join;
+
+    logger.info(
+        "Audit configuration is updated. Auditing to: \"{}\", with the following categories: \"{}\", keyspaces: \"{}\", and tables: \"{}\".",
+        _cfg.audit(),
+        fmt::join(std::views::transform(_audited_categories, category_to_string), ","),
+        fmt::join(_audited_keyspaces, ","),
+        fmt::join(table_entries, ","));
+}
+
 }

audit/audit.hh

@@ -9,6 +9,7 @@
 
 #include "seastarx.hh"
 #include "utils/log.hh"
+#include "utils/observable.hh"
 #include "db/consistency_level.hh"
 #include "locator/token_metadata_fwd.hh"
 #include <seastar/core/sharded.hh>
@@ -96,13 +97,22 @@ class storage_helper;
 
 class audit final : public seastar::async_sharded_service<audit> {
     locator::shared_token_metadata& _token_metadata;
-    const std::set<sstring> _audited_keyspaces;
+    std::set<sstring> _audited_keyspaces;
     // Maps keyspace name to set of table names in that keyspace
-    const std::map<sstring, std::set<sstring>> _audited_tables;
-    const category_set _audited_categories;
+    std::map<sstring, std::set<sstring>> _audited_tables;
+    category_set _audited_categories;
+
     sstring _storage_helper_class_name;
     std::unique_ptr<storage_helper> _storage_helper_ptr;
 
+    const db::config& _cfg;
+    utils::observer<sstring> _cfg_keyspaces_observer;
+    utils::observer<sstring> _cfg_tables_observer;
+    utils::observer<sstring> _cfg_categories_observer;
+
+    template<class T>
+    void update_config(const sstring & new_value, std::function<T(const sstring&)> parse_func, T& cfg_parameter);
+
     bool should_log_table(const sstring& keyspace, const sstring& name) const;
 public:
     static seastar::sharded<audit>& audit_instance() {
@@ -123,7 +133,8 @@ public:
     audit(locator::shared_token_metadata& stm, sstring&& storage_helper_name,
           std::set<sstring>&& audited_keyspaces,
           std::map<sstring, std::set<sstring>>&& audited_tables,
-          category_set&& audited_categories);
+          category_set&& audited_categories,
+          const db::config& cfg);
     ~audit();
     future<> start(const db::config& cfg, cql3::query_processor& qp, service::migration_manager& mm);
     future<> stop();

audit/audit_syslog_storage_helper.cc

@@ -33,6 +33,20 @@ namespace audit {
 
 namespace {
 
+future<> syslog_send_helper(net::datagram_channel& sender,
+                            const socket_address& address,
+                            const sstring& msg) {
+    return sender.send(address, net::packet{msg.data(), msg.size()}).handle_exception([address](auto&& exception_ptr) {
+        auto error_msg = seastar::format(
+            "Syslog audit backend failed (sending a message to {} resulted in {}).",
+            address,
+            exception_ptr
+        );
+        logger.error("{}", error_msg);
+        throw audit_exception(std::move(error_msg));
+    });
+}
+
 static auto syslog_address_helper(const db::config& cfg)
 {
     return cfg.audit_unix_socket_path.is_set()
@@ -42,26 +56,9 @@ static auto syslog_address_helper(const db::config& cfg)
 
 }
 
-future<> audit_syslog_storage_helper::syslog_send_helper(const sstring& msg) {
-    try {
-        auto lock = co_await get_units(_semaphore, 1, std::chrono::hours(1));
-        co_await _sender.send(_syslog_address, net::packet{msg.data(), msg.size()});
-    }
-    catch (const std::exception& e) {
-        auto error_msg = seastar::format(
-            "Syslog audit backend failed (sending a message to {} resulted in {}).",
-            _syslog_address,
-            e
-        );
-        logger.error("{}", error_msg);
-        throw audit_exception(std::move(error_msg));
-    }
-}
-
 audit_syslog_storage_helper::audit_syslog_storage_helper(cql3::query_processor& qp, service::migration_manager&) :
     _syslog_address(syslog_address_helper(qp.db().get_config())),
-    _sender(make_unbound_datagram_channel(AF_UNIX)),
-    _semaphore(1) {
+    _sender(make_unbound_datagram_channel(AF_UNIX)) {
 }
 
 audit_syslog_storage_helper::~audit_syslog_storage_helper() {
@@ -76,10 +73,10 @@ audit_syslog_storage_helper::~audit_syslog_storage_helper() {
  */
 future<> audit_syslog_storage_helper::start(const db::config& cfg) {
     if (this_shard_id() != 0) {
-        co_return;
+        return make_ready_future();
     }
 
-    co_await syslog_send_helper("Initializing syslog audit backend.");
+    return syslog_send_helper(_sender, _syslog_address, "Initializing syslog audit backend.");
 }
 
 future<> audit_syslog_storage_helper::stop() {
@@ -109,7 +106,7 @@ future<> audit_syslog_storage_helper::write(const audit_info* audit_info,
                                     audit_info->table(),
                                     username);
 
-    co_await syslog_send_helper(msg);
+    return syslog_send_helper(_sender, _syslog_address, msg);
 }
 
 future<> audit_syslog_storage_helper::write_login(const sstring& username,
@@ -128,7 +125,7 @@ future<> audit_syslog_storage_helper::write_login(const sstring& username,
                                     username,
                                     (error ? "true" : "false"));
 
-    co_await syslog_send_helper(msg.c_str());
+    co_await syslog_send_helper(_sender, _syslog_address, msg.c_str());
 }
 
 using registry = class_registrator<storage_helper, audit_syslog_storage_helper, cql3::query_processor&, service::migration_manager&>;

audit/audit_syslog_storage_helper.hh

@@ -24,9 +24,6 @@ namespace audit {
 class audit_syslog_storage_helper : public storage_helper {
     socket_address _syslog_address;
     net::datagram_channel _sender;
-    seastar::semaphore _semaphore;
-
-    future<> syslog_send_helper(const sstring& msg);
 public:
     explicit audit_syslog_storage_helper(cql3::query_processor&, service::migration_manager&);
     virtual ~audit_syslog_storage_helper();

auth/allow_all_authenticator.hh

@@ -83,6 +83,10 @@ public:
     virtual ::shared_ptr<sasl_challenge> new_sasl_challenge() const override {
         throw std::runtime_error("Should not reach");
     }
+
+    virtual future<> ensure_superuser_is_created() const override {
+        return make_ready_future<>();
+    }
 };
 
 }

auth/authenticator.hh

@@ -159,6 +159,8 @@ public:
     virtual const resource_set& protected_resources() const = 0;
 
     virtual ::shared_ptr<sasl_challenge> new_sasl_challenge() const = 0;
+
+    virtual future<> ensure_superuser_is_created() const = 0;
 };
 
 }

auth/certificate_authenticator.hh

@@ -56,6 +56,10 @@ public:
     const resource_set& protected_resources() const override;
 
     ::shared_ptr<sasl_challenge> new_sasl_challenge() const override;
+
+    virtual future<> ensure_superuser_is_created() const override {
+        return make_ready_future<>();
+    }
 private:
 };
 

auth/common.cc

@@ -119,11 +119,6 @@ future<> create_legacy_metadata_table_if_missing(
     return qs;
 }
 
-::service::raft_timeout get_raft_timeout() noexcept {
-    auto dur = internal_distributed_query_state().get_client_state().get_timeout_config().other_timeout;
-    return ::service::raft_timeout{.value = lowres_clock::now() + dur};
-}
-
 static future<> announce_mutations_with_guard(
         ::service::raft_group0_client& group0_client,
         std::vector<canonical_mutation> muts,

auth/common.hh

@@ -17,7 +17,6 @@
 
 #include "types/types.hh"
 #include "service/raft/raft_group0_client.hh"
-#include "timeout_config.hh"
 
 using namespace std::chrono_literals;
 
@@ -78,8 +77,6 @@ future<> create_legacy_metadata_table_if_missing(
 ///
 ::service::query_state& internal_distributed_query_state() noexcept;
 
-::service::raft_timeout get_raft_timeout() noexcept;
-
 // Execute update query via group0 mechanism, mutations will be applied on all nodes.
 // Use this function when need to perform read before write on a single guard or if
 // you have more than one mutation and potentially exceed single command size limit.

auth/default_authorizer.cc

@@ -16,7 +16,6 @@ extern "C" {
 #include <unistd.h>
 }
 
-#include <boost/range.hpp>
 #include <seastar/core/seastar.hh>
 #include <seastar/core/sleep.hh>
 

auth/ldap_role_manager.cc

@@ -233,9 +233,9 @@ future<role_set> ldap_role_manager::query_granted(std::string_view grantee_name,
 }
 
 future<role_to_directly_granted_map>
-ldap_role_manager::query_all_directly_granted(::service::query_state& qs) {
+ldap_role_manager::query_all_directly_granted() {
     role_to_directly_granted_map result;
-    auto roles = co_await query_all(qs);
+    auto roles = co_await query_all();
     for (auto& role: roles) {
         auto granted_set = co_await query_granted(role, recursive_role_query::no);
         for (auto& granted: granted_set) {
@@ -247,8 +247,8 @@ ldap_role_manager::query_all_directly_granted(::service::query_state& qs) {
     co_return result;
 }
 
-future<role_set> ldap_role_manager::query_all(::service::query_state& qs) {
-    return _std_mgr.query_all(qs);
+future<role_set> ldap_role_manager::query_all() {
+    return _std_mgr.query_all();
 }
 
 future<> ldap_role_manager::create_role(std::string_view role_name) {
@@ -311,12 +311,12 @@ future<bool> ldap_role_manager::can_login(std::string_view role_name) {
 }
 
 future<std::optional<sstring>> ldap_role_manager::get_attribute(
-        std::string_view role_name, std::string_view attribute_name, ::service::query_state& qs) {
-    return _std_mgr.get_attribute(role_name, attribute_name, qs);
+        std::string_view role_name, std::string_view attribute_name) {
+    return _std_mgr.get_attribute(role_name, attribute_name);
 }
 
-future<role_manager::attribute_vals> ldap_role_manager::query_attribute_for_all(std::string_view attribute_name, ::service::query_state& qs) {
-    return _std_mgr.query_attribute_for_all(attribute_name, qs);
+future<role_manager::attribute_vals> ldap_role_manager::query_attribute_for_all(std::string_view attribute_name) {
+    return _std_mgr.query_attribute_for_all(attribute_name);
 }
 
 future<> ldap_role_manager::set_attribute(
@@ -338,7 +338,8 @@ future<std::vector<cql3::description>> ldap_role_manager::describe_role_grants()
 }
 
 future<> ldap_role_manager::ensure_superuser_is_created() {
-    return _std_mgr.ensure_superuser_is_created();
+    // ldap is responsible for users
+    co_return;
 }
 
 } // namespace auth

auth/ldap_role_manager.hh

@@ -9,11 +9,9 @@
 
 #pragma once
 
-#include <memory>
 #include <seastar/core/abort_source.hh>
 #include <stdexcept>
 
-#include "bytes.hh"
 #include "ent/ldap/ldap_connection.hh"
 #include "standard_role_manager.hh"
 
@@ -77,9 +75,9 @@ class ldap_role_manager : public role_manager {
 
     future<role_set> query_granted(std::string_view, recursive_role_query) override;
 
-    future<role_to_directly_granted_map> query_all_directly_granted(::service::query_state&) override;
+    future<role_to_directly_granted_map> query_all_directly_granted() override;
 
-    future<role_set> query_all(::service::query_state&) override;
+    future<role_set> query_all() override;
 
     future<bool> exists(std::string_view) override;
 
@@ -87,9 +85,9 @@ class ldap_role_manager : public role_manager {
 
     future<bool> can_login(std::string_view) override;
 
-    future<std::optional<sstring>> get_attribute(std::string_view, std::string_view, ::service::query_state&) override;
+    future<std::optional<sstring>> get_attribute(std::string_view, std::string_view) override;
 
-    future<role_manager::attribute_vals> query_attribute_for_all(std::string_view, ::service::query_state&) override;
+    future<role_manager::attribute_vals> query_attribute_for_all(std::string_view) override;
 
     future<> set_attribute(std::string_view, std::string_view, std::string_view, ::service::group0_batch& mc) override;
 

auth/maintenance_socket_role_manager.cc

@@ -78,11 +78,11 @@ future<role_set> maintenance_socket_role_manager::query_granted(std::string_view
     return operation_not_supported_exception<role_set>("QUERY GRANTED");
 }
 
-future<role_to_directly_granted_map> maintenance_socket_role_manager::query_all_directly_granted(::service::query_state&) {
+future<role_to_directly_granted_map> maintenance_socket_role_manager::query_all_directly_granted() {
     return operation_not_supported_exception<role_to_directly_granted_map>("QUERY ALL DIRECTLY GRANTED");
 }
 
-future<role_set> maintenance_socket_role_manager::query_all(::service::query_state&) {
+future<role_set> maintenance_socket_role_manager::query_all() {
     return operation_not_supported_exception<role_set>("QUERY ALL");
 }
 
@@ -98,11 +98,11 @@ future<bool> maintenance_socket_role_manager::can_login(std::string_view role_na
     return make_ready_future<bool>(true);
 }
 
-future<std::optional<sstring>> maintenance_socket_role_manager::get_attribute(std::string_view role_name, std::string_view attribute_name, ::service::query_state&) {
+future<std::optional<sstring>> maintenance_socket_role_manager::get_attribute(std::string_view role_name, std::string_view attribute_name) {
     return operation_not_supported_exception<std::optional<sstring>>("GET ATTRIBUTE");
 }
 
-future<role_manager::attribute_vals> maintenance_socket_role_manager::query_attribute_for_all(std::string_view attribute_name, ::service::query_state&) {
+future<role_manager::attribute_vals> maintenance_socket_role_manager::query_attribute_for_all(std::string_view attribute_name) {
     return operation_not_supported_exception<role_manager::attribute_vals>("QUERY ATTRIBUTE");
 }
 

auth/maintenance_socket_role_manager.hh

@@ -53,9 +53,9 @@ public:
 
     virtual future<role_set> query_granted(std::string_view grantee_name, recursive_role_query) override;
 
-    virtual future<role_to_directly_granted_map> query_all_directly_granted(::service::query_state&) override;
+    virtual future<role_to_directly_granted_map> query_all_directly_granted() override;
 
-    virtual future<role_set> query_all(::service::query_state&) override;
+    virtual future<role_set> query_all() override;
 
     virtual future<bool> exists(std::string_view role_name) override;
 
@@ -63,9 +63,9 @@ public:
 
     virtual future<bool> can_login(std::string_view role_name) override;
 
-    virtual future<std::optional<sstring>> get_attribute(std::string_view role_name, std::string_view attribute_name, ::service::query_state&) override;
+    virtual future<std::optional<sstring>> get_attribute(std::string_view role_name, std::string_view attribute_name) override;
 
-    virtual future<role_manager::attribute_vals> query_attribute_for_all(std::string_view attribute_name, ::service::query_state&) override;
+    virtual future<role_manager::attribute_vals> query_attribute_for_all(std::string_view attribute_name) override;
 
     virtual future<> set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value, ::service::group0_batch& mc) override;
 

auth/password_authenticator.cc

@@ -117,8 +117,7 @@ future<> password_authenticator::migrate_legacy_metadata() const {
     });
 }
 
-future<> password_authenticator::legacy_create_default_if_missing() {
-    SCYLLA_ASSERT(legacy_mode(_qp));
+future<> password_authenticator::create_default_if_missing() {
     const auto exists = co_await default_role_row_satisfies(_qp, &has_salted_hash, _superuser);
     if (exists) {
         co_return;
@@ -128,75 +127,18 @@ future<> password_authenticator::legacy_create_default_if_missing() {
         salted_pwd = passwords::hash(DEFAULT_USER_PASSWORD, rng_for_salt);
     }
     const auto query = update_row_query();
-    co_await _qp.execute_internal(
+    if (legacy_mode(_qp)) {
+        co_await _qp.execute_internal(
             query,
             db::consistency_level::QUORUM,
             internal_distributed_query_state(),
             {salted_pwd, _superuser},
             cql3::query_processor::cache_internal::no);
-    plogger.info("Created default superuser authentication record.");
-}
-
-future<> password_authenticator::maybe_create_default_password() {
-    auto needs_password = [this] () -> future<bool> {
-        const sstring query = seastar::format("SELECT * FROM {}.{} WHERE is_superuser = true ALLOW FILTERING", get_auth_ks_name(_qp), meta::roles_table::name);
-        auto results = co_await _qp.execute_internal(query,
-                db::consistency_level::LOCAL_ONE,
-                internal_distributed_query_state(), cql3::query_processor::cache_internal::yes);
-        // Don't add default password if
-        // - there is no default superuser
-        // - there is a superuser with a password.
-        bool has_default = false;
-        bool has_superuser_with_password = false;
-        for (auto& result : *results) {
-            if (result.get_as<sstring>(meta::roles_table::role_col_name) == _superuser) {
-                has_default = true;
-            }
-            if (has_salted_hash(result)) {
-                has_superuser_with_password = true;
-            }
-        }
-        co_return has_default && !has_superuser_with_password;
-    };
-    if (!co_await needs_password()) {
-        co_return;
-    }
-    // We don't want to start operation earlier to avoid quorum requirement in
-    // a common case.
-    ::service::group0_batch batch(
-            co_await _group0_client.start_operation(_as, get_raft_timeout()));
-    // Check again as the state may have changed before we took the guard (batch).
-    if (!co_await needs_password()) {
-        co_return;
-    }
-    // Set default superuser's password.
-    std::string salted_pwd(get_config_value(_qp.db().get_config().auth_superuser_salted_password(), ""));
-    if (salted_pwd.empty()) {
-        salted_pwd = passwords::hash(DEFAULT_USER_PASSWORD, rng_for_salt);
-    }
-    const auto update_query = update_row_query();
-    co_await collect_mutations(_qp, batch, update_query, {salted_pwd, _superuser});
-    co_await std::move(batch).commit(_group0_client, _as, get_raft_timeout());
-    plogger.info("Created default superuser authentication record.");
-}
-
-future<> password_authenticator::maybe_create_default_password_with_retries() {
-    size_t retries = _migration_manager.get_concurrent_ddl_retries();
-    while (true)  {
-        try {
-            co_return co_await maybe_create_default_password();
-        } catch (const ::service::group0_concurrent_modification& ex) {
-            plogger.warn("Failed to execute maybe_create_default_password due to guard conflict.{}.", retries ? " Retrying" : " Number of retries exceeded, giving up");
-            if (retries--) {
-                continue;
-            }
-            // Log error but don't crash the whole node startup sequence.
-            plogger.error("Failed to create default superuser password due to guard conflict.");
-            co_return;
-        } catch (const ::service::raft_operation_timeout_error& ex) {
-            plogger.error("Failed to create default superuser password due to exception: {}", ex.what());
-            co_return;
-        }
+        plogger.info("Created default superuser authentication record.");
+    } else {
+        co_await announce_mutations(_qp, _group0_client, query,
+            {salted_pwd, _superuser}, _as, ::service::raft_timeout{});
+        plogger.info("Created default superuser authentication record.");
     }
 }
 
@@ -205,6 +147,9 @@ future<> password_authenticator::start() {
         _stopped = do_after_system_ready(_as, [this] {
             return async([this] {
                 if (legacy_mode(_qp)) {
+                    if (!_superuser_created_promise.available()) {
+                        _superuser_created_promise.set_value();
+                    }
                     _migration_manager.wait_for_schema_agreement(_qp.db().real_database(), db::timeout_clock::time_point::max(), &_as).get();
 
                     if (any_nondefault_role_row_satisfies(_qp, &has_salted_hash, _superuser).get()) {
@@ -219,9 +164,12 @@ future<> password_authenticator::start() {
                         migrate_legacy_metadata().get();
                         return;
                     }
-                    legacy_create_default_if_missing().get();
                 }
-                    maybe_create_default_password_with_retries().get();
+                utils::get_local_injector().inject("password_authenticator_start_pause", utils::wait_for_message(5min)).get();
+                create_default_if_missing().get();
+                if (!legacy_mode(_qp)) {
+                    _superuser_created_promise.set_value();
+                }
             });
         });
 
@@ -420,4 +368,8 @@ const resource_set& password_authenticator::protected_resources() const {
     });
 }
 
+future<> password_authenticator::ensure_superuser_is_created() const {
+    return _superuser_created_promise.get_shared_future();
+}
+
 }

auth/password_authenticator.hh

@@ -11,6 +11,7 @@
 #pragma once
 
 #include <seastar/core/abort_source.hh>
+#include <seastar/core/shared_future.hh>
 
 #include "db/consistency_level_type.hh"
 #include "auth/authenticator.hh"
@@ -40,7 +41,8 @@ class password_authenticator : public authenticator {
     ::service::migration_manager& _migration_manager;
     future<> _stopped;
     abort_source _as;
-    std::string _superuser; // default superuser name from the config (may or may not be present in roles table)
+    std::string _superuser;
+    shared_promise<> _superuser_created_promise;
 
 public:
     static db::consistency_level consistency_for_user(std::string_view role_name);
@@ -80,15 +82,14 @@ public:
 
     virtual ::shared_ptr<sasl_challenge> new_sasl_challenge() const override;
 
+    virtual future<> ensure_superuser_is_created() const override;
+
 private:
     bool legacy_metadata_exists() const;
 
     future<> migrate_legacy_metadata() const;
 
-    future<> legacy_create_default_if_missing();
-
-    future<> maybe_create_default_password();
-    future<> maybe_create_default_password_with_retries();
+    future<> create_default_if_missing();
 
     sstring update_row_query() const;
 };

auth/role_manager.hh

@@ -17,17 +17,12 @@
 #include <seastar/core/format.hh>
 #include <seastar/core/sstring.hh>
 
-#include "auth/common.hh"
 #include "auth/resource.hh"
 #include "cql3/description.hh"
 #include "seastarx.hh"
 #include "exceptions/exceptions.hh"
 #include "service/raft/raft_group0_client.hh"
 
-namespace service {
-class query_state;
-};
-
 namespace auth {
 
 struct role_config final {
@@ -172,9 +167,9 @@ public:
     ///   (role2, role3)
     /// }
     ///  
-    virtual future<role_to_directly_granted_map> query_all_directly_granted(::service::query_state& = internal_distributed_query_state()) = 0;
+    virtual future<role_to_directly_granted_map> query_all_directly_granted() = 0;
 
-    virtual future<role_set> query_all(::service::query_state& = internal_distributed_query_state()) = 0;
+    virtual future<role_set> query_all() = 0;
 
     virtual future<bool> exists(std::string_view role_name) = 0;
 
@@ -191,12 +186,12 @@ public:
     ///
     /// \returns the value of the named attribute, if one is set.
     ///
-    virtual future<std::optional<sstring>> get_attribute(std::string_view role_name, std::string_view attribute_name, ::service::query_state& = internal_distributed_query_state()) = 0;
+    virtual future<std::optional<sstring>> get_attribute(std::string_view role_name, std::string_view attribute_name) = 0;
 
     ///
     /// \returns a mapping of each role's value for the named attribute, if one is set for the role.
     ///
-    virtual future<attribute_vals> query_attribute_for_all(std::string_view attribute_name, ::service::query_state& = internal_distributed_query_state()) = 0;
+    virtual future<attribute_vals> query_attribute_for_all(std::string_view attribute_name) = 0;
 
     /// Sets `attribute_name` with `attribute_value` for `role_name`.
     /// \returns an exceptional future with nonexistant_role if the role does not exist.

auth/saslauthd_authenticator.hh

@@ -55,6 +55,10 @@ public:
     const resource_set& protected_resources() const override;
 
     ::shared_ptr<sasl_challenge> new_sasl_challenge() const override;
+
+    virtual future<> ensure_superuser_is_created() const override {
+        return make_ready_future<>();
+    }
 };
 
 /// A set of four credential strings that saslauthd expects.

auth/service.cc

@@ -240,13 +240,6 @@ future<> service::start(::service::migration_manager& mm, db::system_keyspace& s
         });
     }
     co_await _role_manager->start();
-    if (this_shard_id() == 0) {
-        // Role manager and password authenticator have this odd startup
-        // mechanism where they asynchronously create the superuser role
-        // in the background. Correct password creation depends on role
-        // creation therefore we need to wait here.
-        co_await _role_manager->ensure_superuser_is_created();
-    }
     co_await when_all_succeed(_authorizer->start(), _authenticator->start()).discard_result();
     _permissions_cache = std::make_unique<permissions_cache>(_loading_cache_config, *this, log);
     co_await once_among_shards([this] {
@@ -270,7 +263,8 @@ future<> service::stop() {
 }
 
 future<> service::ensure_superuser_is_created() {
-    return _role_manager->ensure_superuser_is_created();
+    co_await _role_manager->ensure_superuser_is_created();
+    co_await _authenticator->ensure_superuser_is_created();
 }
 
 void service::update_cache_config() {
@@ -682,7 +676,8 @@ future<permission_set> get_permissions(const service& ser, const authenticated_u
 }
 
 bool is_protected(const service& ser, command_desc cmd) noexcept {
-    if (cmd.type_ == command_desc::type::ALTER_WITH_OPTS) {
+    if (cmd.type_ == command_desc::type::ALTER_WITH_OPTS ||
+        cmd.type_ == command_desc::type::ALTER_SYSTEM_WITH_ALLOWED_OPTS) {
         return false; // Table attributes are OK to modify; see #7057.
     }
     return ser.underlying_role_manager().protected_resources().contains(cmd.resource)

auth/service.hh

@@ -224,6 +224,7 @@ struct command_desc {
     const ::auth::resource& resource; ///< Resource impacted by this command.
     enum class type {
         ALTER_WITH_OPTS, ///< Command is ALTER ... WITH ...
+        ALTER_SYSTEM_WITH_ALLOWED_OPTS,
         OTHER
     } type_ = type::OTHER;
 };

auth/standard_role_manager.cc

@@ -9,7 +9,6 @@
 #include "auth/standard_role_manager.hh"
 
 #include <optional>
-#include <stdexcept>
 #include <unordered_set>
 #include <vector>
 
@@ -29,7 +28,6 @@
 #include "cql3/util.hh"
 #include "db/consistency_level_type.hh"
 #include "exceptions/exceptions.hh"
-#include "utils/error_injection.hh"
 #include "utils/log.hh"
 #include <seastar/core/loop.hh>
 #include <seastar/coroutine/maybe_yield.hh>
@@ -180,8 +178,7 @@ future<> standard_role_manager::create_legacy_metadata_tables_if_missing() const
                     _migration_manager)).discard_result();
 }
 
-future<> standard_role_manager::legacy_create_default_role_if_missing() {
-    SCYLLA_ASSERT(legacy_mode(_qp));
+future<> standard_role_manager::create_default_role_if_missing() {
     try {
         const auto exists = co_await default_role_row_satisfies(_qp, &has_can_login, _superuser);
         if (exists) {
@@ -191,12 +188,16 @@ future<> standard_role_manager::legacy_create_default_role_if_missing() {
                 get_auth_ks_name(_qp),
                 meta::roles_table::name,
                 meta::roles_table::role_col_name);
-        co_await _qp.execute_internal(
-                query,
-                db::consistency_level::QUORUM,
-                internal_distributed_query_state(),
-                {_superuser},
-                cql3::query_processor::cache_internal::no).discard_result();
+        if (legacy_mode(_qp)) {
+            co_await _qp.execute_internal(
+                    query,
+                    db::consistency_level::QUORUM,
+                    internal_distributed_query_state(),
+                    {_superuser},
+                    cql3::query_processor::cache_internal::no).discard_result();
+        } else {
+            co_await announce_mutations(_qp, _group0_client, query, {_superuser}, _as, ::service::raft_timeout{});
+        }
         log.info("Created default superuser role '{}'.", _superuser);
     } catch(const exceptions::unavailable_exception& e) {
         log.warn("Skipped default role setup: some nodes were not ready; will retry");
@@ -204,60 +205,6 @@ future<> standard_role_manager::legacy_create_default_role_if_missing() {
     }
 }
 
-future<> standard_role_manager::maybe_create_default_role() {
-    auto has_superuser = [this] () -> future<bool> {
-        const sstring query = seastar::format("SELECT * FROM {}.{} WHERE is_superuser = true ALLOW FILTERING", get_auth_ks_name(_qp), meta::roles_table::name);
-        auto results = co_await _qp.execute_internal(query, db::consistency_level::LOCAL_ONE,
-                internal_distributed_query_state(), cql3::query_processor::cache_internal::yes);
-        for (const auto& result : *results) {
-            if (has_can_login(result)) {
-                co_return true;
-            }
-        }
-        co_return false;
-    };
-    if (co_await has_superuser()) {
-        co_return;
-    }
-    // We don't want to start operation earlier to avoid quorum requirement in
-    // a common case.
-    ::service::group0_batch batch(
-            co_await _group0_client.start_operation(_as, get_raft_timeout()));
-    // Check again as the state may have changed before we took the guard (batch).
-    if (co_await has_superuser()) {
-        co_return;
-    }
-    // There is no superuser which has can_login field - create default role.
-    // Note that we don't check if can_login is set to true.
-    const sstring insert_query = seastar::format("INSERT INTO {}.{} ({}, is_superuser, can_login) VALUES (?, true, true)",
-            get_auth_ks_name(_qp),
-            meta::roles_table::name,
-            meta::roles_table::role_col_name);
-    co_await collect_mutations(_qp, batch, insert_query, {_superuser});
-    co_await std::move(batch).commit(_group0_client, _as, get_raft_timeout());
-    log.info("Created default superuser role '{}'.", _superuser);
-}
-
-future<> standard_role_manager::maybe_create_default_role_with_retries() {
-    size_t retries = _migration_manager.get_concurrent_ddl_retries();
-    while (true)  {
-        try {
-            co_return co_await maybe_create_default_role();
-        } catch (const ::service::group0_concurrent_modification& ex) {
-            log.warn("Failed to execute maybe_create_default_role due to guard conflict.{}.", retries ? " Retrying" : " Number of retries exceeded, giving up");
-            if (retries--) {
-                continue;
-            }
-            // Log error but don't crash the whole node startup sequence.
-            log.error("Failed to create default superuser role due to guard conflict.");
-            co_return;
-        } catch (const ::service::raft_operation_timeout_error& ex) {
-            log.error("Failed to create default superuser role due to exception: {}", ex.what());
-            co_return;
-        }
-    }
-}
-
 static const sstring legacy_table_name{"users"};
 
 bool standard_role_manager::legacy_metadata_exists() {
@@ -319,13 +266,10 @@ future<> standard_role_manager::start() {
                     co_await migrate_legacy_metadata();
                     co_return;
                 }
-                co_await legacy_create_default_role_if_missing();
             }
+            co_await create_default_role_if_missing();
             if (!legacy) {
-                co_await maybe_create_default_role_with_retries();
-                if (!_superuser_created_promise.available()) {
-                    _superuser_created_promise.set_value();
-                }
+                _superuser_created_promise.set_value();
             }
         };
 
@@ -652,30 +596,21 @@ future<role_set> standard_role_manager::query_granted(std::string_view grantee_n
     });
 }
 
-future<role_to_directly_granted_map> standard_role_manager::query_all_directly_granted(::service::query_state& qs) {
+future<role_to_directly_granted_map> standard_role_manager::query_all_directly_granted() {
     const sstring query = seastar::format("SELECT * FROM {}.{}",
             get_auth_ks_name(_qp),
             meta::role_members_table::name);
 
-    const auto results = co_await _qp.execute_internal(
-            query,
-            db::consistency_level::ONE,
-            qs,
-            cql3::query_processor::cache_internal::yes);
-
     role_to_directly_granted_map roles_map;
-    std::transform(
-            results->begin(),
-            results->end(),
-            std::inserter(roles_map, roles_map.begin()),
-            [] (const cql3::untyped_result_set_row& row) {
-                return std::make_pair(row.get_as<sstring>("member"), row.get_as<sstring>("role")); }
-    );
+    co_await _qp.query_internal(query, [&roles_map] (const cql3::untyped_result_set_row& row) -> future<stop_iteration> {
+        roles_map.insert({row.get_as<sstring>("member"), row.get_as<sstring>("role")});
+        co_return stop_iteration::no;
+    });
 
     co_return roles_map;
 }
 
-future<role_set> standard_role_manager::query_all(::service::query_state& qs) {
+future<role_set> standard_role_manager::query_all() {
     const sstring query = seastar::format("SELECT {} FROM {}.{}",
             meta::roles_table::role_col_name,
             get_auth_ks_name(_qp),
@@ -684,16 +619,10 @@ future<role_set> standard_role_manager::query_all(::service::query_state& qs) {
     // To avoid many copies of a view.
     static const auto role_col_name_string = sstring(meta::roles_table::role_col_name);
 
-    if (utils::get_local_injector().enter("standard_role_manager_fail_legacy_query")) {
-        if (legacy_mode(_qp)) {
-            throw std::runtime_error("standard_role_manager::query_all: failed due to error injection");
-        }
-    }
-
     const auto results = co_await _qp.execute_internal(
             query,
             db::consistency_level::QUORUM,
-            qs,
+            internal_distributed_query_state(),
             cql3::query_processor::cache_internal::yes);
 
     role_set roles;
@@ -725,11 +654,11 @@ future<bool> standard_role_manager::can_login(std::string_view role_name) {
     });
 }
 
-future<std::optional<sstring>> standard_role_manager::get_attribute(std::string_view role_name, std::string_view attribute_name, ::service::query_state& qs) {
+future<std::optional<sstring>> standard_role_manager::get_attribute(std::string_view role_name, std::string_view attribute_name) {
     const sstring query = seastar::format("SELECT name, value FROM {}.{} WHERE role = ? AND name = ?",
             get_auth_ks_name(_qp),
             meta::role_attributes_table::name);
-    const auto result_set = co_await _qp.execute_internal(query, db::consistency_level::ONE, qs, {sstring(role_name), sstring(attribute_name)}, cql3::query_processor::cache_internal::yes);
+    const auto result_set = co_await _qp.execute_internal(query, {sstring(role_name), sstring(attribute_name)}, cql3::query_processor::cache_internal::yes);
     if (!result_set->empty()) {
         const cql3::untyped_result_set_row &row = result_set->one();
         co_return std::optional<sstring>(row.get_as<sstring>("value"));
@@ -737,11 +666,11 @@ future<std::optional<sstring>> standard_role_manager::get_attribute(std::string_
     co_return std::optional<sstring>{};
 }
 
-future<role_manager::attribute_vals> standard_role_manager::query_attribute_for_all (std::string_view attribute_name, ::service::query_state& qs) {
-    return query_all(qs).then([this, attribute_name, &qs] (role_set roles) {
-        return do_with(attribute_vals{}, [this, attribute_name, roles = std::move(roles), &qs] (attribute_vals &role_to_att_val) {
-            return parallel_for_each(roles.begin(), roles.end(), [this, &role_to_att_val, attribute_name, &qs] (sstring role) {
-                return get_attribute(role, attribute_name, qs).then([&role_to_att_val, role] (std::optional<sstring> att_val) {
+future<role_manager::attribute_vals> standard_role_manager::query_attribute_for_all (std::string_view attribute_name) {
+    return query_all().then([this, attribute_name] (role_set roles) {
+        return do_with(attribute_vals{}, [this, attribute_name, roles = std::move(roles)] (attribute_vals &role_to_att_val) {
+            return parallel_for_each(roles.begin(), roles.end(), [this, &role_to_att_val, attribute_name] (sstring role) {
+                return get_attribute(role, attribute_name).then([&role_to_att_val, role] (std::optional<sstring> att_val) {
                     if (att_val) {
                         role_to_att_val.emplace(std::move(role), std::move(*att_val));
                     }
@@ -786,7 +715,7 @@ future<> standard_role_manager::remove_attribute(std::string_view role_name, std
 future<std::vector<cql3::description>> standard_role_manager::describe_role_grants() {
     std::vector<cql3::description> result{};
 
-    const auto grants = co_await query_all_directly_granted(internal_distributed_query_state());
+    const auto grants = co_await query_all_directly_granted();
     result.reserve(grants.size());
 
     for (const auto& [grantee_role, granted_role] : grants) {

auth/standard_role_manager.hh

@@ -66,9 +66,9 @@ public:
 
     virtual future<role_set> query_granted(std::string_view grantee_name, recursive_role_query) override;
 
-    virtual future<role_to_directly_granted_map> query_all_directly_granted(::service::query_state&) override;
+    virtual future<role_to_directly_granted_map> query_all_directly_granted() override;
 
-    virtual future<role_set> query_all(::service::query_state&) override;
+    virtual future<role_set> query_all() override;
 
     virtual future<bool> exists(std::string_view role_name) override;
 
@@ -76,9 +76,9 @@ public:
 
     virtual future<bool> can_login(std::string_view role_name) override;
 
-    virtual future<std::optional<sstring>> get_attribute(std::string_view role_name, std::string_view attribute_name, ::service::query_state&) override;
+    virtual future<std::optional<sstring>> get_attribute(std::string_view role_name, std::string_view attribute_name) override;
 
-    virtual future<role_manager::attribute_vals> query_attribute_for_all(std::string_view attribute_name, ::service::query_state&) override;
+    virtual future<role_manager::attribute_vals> query_attribute_for_all(std::string_view attribute_name) override;
 
     virtual future<> set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value, ::service::group0_batch& mc) override;
 
@@ -95,10 +95,7 @@ private:
 
     future<> migrate_legacy_metadata();
 
-    future<> legacy_create_default_role_if_missing();
-
-    future<> maybe_create_default_role();
-    future<> maybe_create_default_role_with_retries();
+    future<> create_default_role_if_missing();
 
     future<> create_or_replace(std::string_view role_name, const role_config&, ::service::group0_batch&);
 

auth/transitional.cc

@@ -159,6 +159,10 @@ public:
         };
         return ::make_shared<sasl_wrapper>(_authenticator->new_sasl_challenge());
     }
+
+    virtual future<> ensure_superuser_is_created() const override {
+        return _authenticator->ensure_superuser_is_created();
+    }
 };
 
 class transitional_authorizer : public authorizer {

bin/cqlsh

@@ -1,8 +1,10 @@
-#!/bin/bash
+#!/bin/bash -e
 
 # Copyright (C) 2023-present ScyllaDB
 # SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
 
+trap 'echo "error $? in $0 line $LINENO"' ERR
+
 here=$(dirname "$0")
 exec "$here/../tools/cqlsh/bin/cqlsh.py" "$@"
 

bin/nodetool

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
 #
 # Copyright (C) 2024-present ScyllaDB
 #
@@ -6,6 +6,8 @@
 # SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
 #
 
+trap 'echo "error $? in $0 line $LINENO"' ERR
+
 SCRIPT_PATH=$(dirname $(realpath "$0"))
 
 INSTALLED_SCYLLA_PATH="${SCRIPT_PATH}/scylla"

cdc/generation.cc

@@ -365,9 +365,6 @@ cdc::topology_description make_new_generation_description(
         const noncopyable_function<std::pair<size_t, uint8_t>(dht::token)>& get_sharding_info,
         const locator::token_metadata_ptr tmptr) {
     const auto tokens = get_tokens(bootstrap_tokens, tmptr);
-    if (tokens.empty()) {
-        on_internal_error(cdc_log, "Attempted to create a CDC generation from an empty list of tokens");
-    }
 
     utils::chunked_vector<token_range_description> vnode_descriptions;
     vnode_descriptions.reserve(tokens.size());
@@ -1028,7 +1025,7 @@ future<> generation_service::legacy_handle_cdc_generation(std::optional<cdc::gen
     if (using_this_gen) {
         cdc_log.info("Starting to use generation {}", *gen_id);
         co_await update_streams_description(*gen_id, _sys_ks.local(), get_sys_dist_ks(),
-                [tmptr = _token_metadata.get()] { return tmptr->count_normal_token_owners(); },
+                [&tm = _token_metadata] { return tm.get()->count_normal_token_owners(); },
                 _abort_src);
     }
 }
@@ -1045,7 +1042,7 @@ void generation_service::legacy_async_handle_cdc_generation(cdc::generation_id g
                 if (using_this_gen) {
                     cdc_log.info("Starting to use generation {}", gen_id);
                     co_await update_streams_description(gen_id, svc->_sys_ks.local(), svc->get_sys_dist_ks(),
-                            [tmptr = svc->_token_metadata.get()] { return tmptr->count_normal_token_owners(); },
+                            [&tm = svc->_token_metadata] { return tm.get()->count_normal_token_owners(); },
                             svc->_abort_src);
                 }
                 co_return;

cdc/log.cc

@@ -10,7 +10,6 @@
 #include <algorithm>
 
 #include <boost/range/irange.hpp>
-#include <boost/algorithm/string/predicate.hpp>
 #include <seastar/core/thread.hh>
 #include <seastar/core/metrics.hh>
 
@@ -41,6 +40,7 @@
 #include "types/listlike_partial_deserializing_iterator.hh"
 #include "tracing/trace_state.hh"
 #include "stats.hh"
+#include "utils/labels.hh"
 
 namespace std {
 
@@ -56,17 +56,8 @@ using namespace std::chrono_literals;
 
 logging::logger cdc_log("cdc");
 
-namespace {
-
-// When dropping a column from a CDC log table, we set the drop timestamp
-// `column_drop_leeway` seconds into the future to ensure that for writes concurrent
-// with column drop, the write timestamp is before the column drop timestamp.
-constexpr auto column_drop_leeway = std::chrono::seconds(5);
-
-} // anonymous namespace
-
 namespace cdc {
-static schema_ptr create_log_schema(const schema&, api::timestamp_type, std::optional<table_id> = {}, schema_ptr = nullptr);
+static schema_ptr create_log_schema(const schema&, std::optional<table_id> = {}, schema_ptr = nullptr);
 }
 
 static constexpr auto cdc_group_name = "cdc";
@@ -77,7 +68,7 @@ void cdc::stats::parts_touched_stats::register_metrics(seastar::metrics::metric_
         metrics.add_group(cdc_group_name, {
                 sm::make_total_operations(seastar::format("operations_on_{}_performed_{}", part_name, suffix), count[(size_t)part],
                         sm::description(seastar::format("number of {} CDC operations that processed a {}", suffix, part_name)),
-                        {})
+                        {cdc_label}).set_skip_when_empty()
             });
     };
 
@@ -100,23 +91,23 @@ cdc::stats::stats() {
         _metrics.add_group(cdc_group_name, {
                 sm::make_total_operations("operations_" + kind, counters.unsplit_count,
                         sm::description(format("number of {} CDC operations", kind)),
-                        {split_label(false)}),
+                        {split_label(false), basic_level, cdc_label}).set_skip_when_empty(),
 
                 sm::make_total_operations("operations_" + kind, counters.split_count,
                         sm::description(format("number of {} CDC operations", kind)),
-                        {split_label(true)}),
+                        {split_label(true), basic_level, cdc_label}).set_skip_when_empty(),
 
                 sm::make_total_operations("preimage_selects_" + kind, counters.preimage_selects,
                         sm::description(format("number of {} preimage queries performed", kind)),
-                        {}),
+                        {cdc_label}).set_skip_when_empty(),
 
                 sm::make_total_operations("operations_with_preimage_" + kind, counters.with_preimage_count,
                         sm::description(format("number of {} operations that included preimage", kind)),
-                        {}),
+                        {cdc_label}).set_skip_when_empty(),
 
                 sm::make_total_operations("operations_with_postimage_" + kind, counters.with_postimage_count,
                         sm::description(format("number of {} operations that included postimage", kind)),
-                        {})
+                        {cdc_label}).set_skip_when_empty()
             });
 
         counters.touches.register_metrics(_metrics, kind);
@@ -176,7 +167,7 @@ public:
             ensure_that_table_uses_vnodes(ksm, schema);
 
             // in seastar thread
-            auto log_schema = create_log_schema(schema, timestamp);
+            auto log_schema = create_log_schema(schema);
 
             auto log_mut = db::schema_tables::make_create_table_mutations(log_schema, timestamp);
 
@@ -214,7 +205,7 @@ public:
             ensure_that_table_has_no_counter_columns(new_schema);
             ensure_that_table_uses_vnodes(*keyspace.metadata(), new_schema);
 
-            auto new_log_schema = create_log_schema(new_schema, timestamp, log_schema ? std::make_optional(log_schema->id()) : std::nullopt, log_schema);
+            auto new_log_schema = create_log_schema(new_schema, log_schema ? std::make_optional(log_schema->id()) : std::nullopt, log_schema);
 
             auto log_mut = log_schema 
                 ? db::schema_tables::make_update_table_mutations(db, keyspace.metadata(), log_schema, new_log_schema, timestamp)
@@ -429,7 +420,7 @@ static const sstring cdc_deleted_column_prefix = cdc_meta_column_prefix + "delet
 static const sstring cdc_deleted_elements_column_prefix = cdc_meta_column_prefix + "deleted_elements_";
 
 bool is_log_name(const std::string_view& table_name) {
-    return boost::ends_with(table_name, cdc_log_suffix);
+    return table_name.ends_with(cdc_log_suffix);
 }
 
 bool is_cdc_metacolumn_name(const sstring& name) {
@@ -505,7 +496,7 @@ bytes log_data_column_deleted_elements_name_bytes(const bytes& column_name) {
     return to_bytes(cdc_deleted_elements_column_prefix) + column_name;
 }
 
-static schema_ptr create_log_schema(const schema& s, api::timestamp_type timestamp, std::optional<table_id> uuid, schema_ptr old) {
+static schema_ptr create_log_schema(const schema& s, std::optional<table_id> uuid, schema_ptr old) {
     schema_builder b(s.ks_name(), log_name(s.cf_name()));
     b.with_partitioner(cdc::cdc_partitioner::classname);
     b.set_compaction_strategy(sstables::compaction_strategy_type::time_window);
@@ -540,28 +531,6 @@ static schema_ptr create_log_schema(const schema& s, api::timestamp_type timesta
     b.with_column(log_meta_column_name_bytes("ttl"), long_type);
     b.with_column(log_meta_column_name_bytes("end_of_batch"), boolean_type);
     b.set_caching_options(caching_options::get_disabled_caching_options());
-
-    auto validate_new_column = [&] (const sstring& name) {
-        // When dropping a column from a CDC log table, we set the drop timestamp to be
-        // `column_drop_leeway` seconds into the future (see `create_log_schema`).
-        // Therefore, when recreating a column with the same name, we need to validate
-        // that it's not recreated too soon and that the drop timestamp has passed.
-        if (old && old->dropped_columns().contains(name)) {
-            const auto& drop_info = old->dropped_columns().at(name);
-            auto create_time = api::timestamp_clock::time_point(api::timestamp_clock::duration(timestamp));
-            auto drop_time = api::timestamp_clock::time_point(api::timestamp_clock::duration(drop_info.timestamp));
-            if (drop_time > create_time) {
-                throw exceptions::invalid_request_exception(format("Cannot add column {} because a column with the same name was dropped too recently. Please retry after {} seconds",
-                        name, std::chrono::duration_cast<std::chrono::seconds>(drop_time - create_time).count() + 1));
-            }
-        }
-    };
-
-    auto add_column = [&] (sstring name, data_type type) {
-        validate_new_column(name);
-        b.with_column(to_bytes(name), type);
-    };
-
     auto add_columns = [&] (const schema::const_iterator_range_type& columns, bool is_data_col = false) {
         for (const auto& column : columns) {
             auto type = column.type;
@@ -583,9 +552,9 @@ static schema_ptr create_log_schema(const schema& s, api::timestamp_type timesta
                     }
                 ));
             }
-            add_column(log_data_column_name(column.name_as_text()), type);
+            b.with_column(log_data_column_name_bytes(column.name()), type);
             if (is_data_col) {
-                add_column(log_data_column_deleted_name(column.name_as_text()), boolean_type);
+                b.with_column(log_data_column_deleted_name_bytes(column.name()), boolean_type);
             }
             if (column.type->is_multi_cell()) {
                 auto dtype = visit(*type, make_visitor(
@@ -601,7 +570,7 @@ static schema_ptr create_log_schema(const schema& s, api::timestamp_type timesta
                         throw std::invalid_argument("Should not reach");
                     }
                 ));
-                add_column(log_data_column_deleted_elements_name(column.name_as_text()), dtype);
+                b.with_column(log_data_column_deleted_elements_name_bytes(column.name()), dtype);
             }
         }
     };
@@ -623,8 +592,7 @@ static schema_ptr create_log_schema(const schema& s, api::timestamp_type timesta
         // not super efficient, but we don't do this often.
         for (auto& col : old->all_columns()) {
             if (!b.has_column({col.name(), col.name_as_text() })) {
-                auto drop_ts = api::timestamp_clock::now() + column_drop_leeway;
-                b.without_column(col.name_as_text(), col.type, drop_ts.time_since_epoch().count());
+                b.without_column(col.name_as_text(), col.type, api::new_timestamp());
             }
         }
     }
@@ -992,12 +960,8 @@ public:
     // Given a reference to such a column from the base schema, this function sets the corresponding column
     // in the log to the given value for the given row.
     void set_value(const clustering_key& log_ck, const column_definition& base_cdef, const managed_bytes_view& value) {
-        auto log_cdef_ptr = _log_schema.get_column_definition(log_data_column_name_bytes(base_cdef.name()));
-        if (!log_cdef_ptr) {
-            throw exceptions::invalid_request_exception(format("CDC log schema for {}.{} does not have base column {}",
-                _log_schema.ks_name(), _log_schema.cf_name(), base_cdef.name_as_text()));
-        }
-        _log_mut.set_cell(log_ck, *log_cdef_ptr, atomic_cell::make_live(*base_cdef.type, _ts, value, _ttl));
+        auto& log_cdef = *_log_schema.get_column_definition(log_data_column_name_bytes(base_cdef.name()));
+        _log_mut.set_cell(log_ck, log_cdef, atomic_cell::make_live(*base_cdef.type, _ts, value, _ttl));
     }
 
     // Each regular and static column in the base schema has a corresponding column in the log schema
@@ -1005,13 +969,7 @@ public:
     // Given a reference to such a column from the base schema, this function sets the corresponding column
     // in the log to `true` for the given row. If not called, the column will be `null`.
     void set_deleted(const clustering_key& log_ck, const column_definition& base_cdef) {
-        auto log_cdef_ptr = _log_schema.get_column_definition(log_data_column_deleted_name_bytes(base_cdef.name()));
-        if (!log_cdef_ptr) {
-            throw exceptions::invalid_request_exception(format("CDC log schema for {}.{} does not have base column {}",
-                _log_schema.ks_name(), _log_schema.cf_name(), base_cdef.name_as_text()));
-        }
-        auto& log_cdef = *log_cdef_ptr;
-        _log_mut.set_cell(log_ck, *log_cdef_ptr, atomic_cell::make_live(*log_cdef.type, _ts, log_cdef.type->decompose(true), _ttl));
+        _log_mut.set_cell(log_ck, log_data_column_deleted_name_bytes(base_cdef.name()), data_value(true), _ts, _ttl);
     }
 
     // Each regular and static non-atomic column in the base schema has a corresponding column in the log schema
@@ -1020,12 +978,7 @@ public:
     // Given a reference to such a column from the base schema, this function sets the corresponding column
     // in the log to the given set of keys for the given row.
     void set_deleted_elements(const clustering_key& log_ck, const column_definition& base_cdef, const managed_bytes& deleted_elements) {
-        auto log_cdef_ptr = _log_schema.get_column_definition(log_data_column_deleted_elements_name_bytes(base_cdef.name()));
-        if (!log_cdef_ptr) {
-            throw exceptions::invalid_request_exception(format("CDC log schema for {}.{} does not have base column {}",
-                _log_schema.ks_name(), _log_schema.cf_name(), base_cdef.name_as_text()));
-        }
-        auto& log_cdef = *log_cdef_ptr;
+        auto& log_cdef = *_log_schema.get_column_definition(log_data_column_deleted_elements_name_bytes(base_cdef.name()));
         _log_mut.set_cell(log_ck, log_cdef, atomic_cell::make_live(*log_cdef.type, _ts, deleted_elements, _ttl));
     }
 
@@ -1912,10 +1865,5 @@ bool cdc::cdc_service::needs_cdc_augmentation(const std::vector<mutation>& mutat
 
 future<std::tuple<std::vector<mutation>, lw_shared_ptr<cdc::operation_result_tracker>>>
 cdc::cdc_service::augment_mutation_call(lowres_clock::time_point timeout, std::vector<mutation>&& mutations, tracing::trace_state_ptr tr_state, db::consistency_level write_cl) {
-    if (utils::get_local_injector().enter("sleep_before_cdc_augmentation")) {
-        return seastar::sleep(std::chrono::milliseconds(100)).then([this, timeout, mutations = std::move(mutations), tr_state = std::move(tr_state), write_cl] () mutable {
-            return _impl->augment_mutation_call(timeout, std::move(mutations), std::move(tr_state), write_cl);
-        });
-    }
     return _impl->augment_mutation_call(timeout, std::move(mutations), std::move(tr_state), write_cl);
 }

clustering_ranges_walker.hh

@@ -16,13 +16,13 @@
 #include "mutation/mutation_fragment.hh"
 #include "mutation/mutation_fragment_v2.hh"
 
-#include <boost/range/iterator_range.hpp>
+#include <ranges>
 
 // Utility for in-order checking of overlap with position ranges.
 class clustering_ranges_walker {
     const schema& _schema;
     const query::clustering_row_ranges& _ranges;
-    boost::iterator_range<query::clustering_row_ranges::const_iterator> _current_range;
+    std::ranges::subrange<query::clustering_row_ranges::const_iterator> _current_range;
     bool _in_current; // next position is known to be >= _current_start
     bool _past_current; // next position is known to be >= _current_end
     bool _using_clustering_range; // Whether current range comes from _current_range
@@ -40,7 +40,7 @@ private:
             if (!_current_range) {
                 return false;
             }
-            _current_range.advance_begin(1);
+            _current_range.advance(1);
         }
         ++_change_counter;
         _using_clustering_range = true;

cmake/limit_jobs.cmake

@@ -1,16 +1,36 @@
-set(LINK_MEM_PER_JOB 4096 CACHE INTERNAL "Maximum memory used by each link job in (in MiB)")
+if(NOT DEFINED Scylla_PARALLEL_LINK_JOBS)
+  if(NOT DEFINED Scylla_RAM_PER_LINK_JOB)
+    # preserve user-provided value
+    set(_default_ram_value 4096)
+    if(Scylla_ENABLE_LTO)
+      # When ThinLTO optimization is enabled, the linker uses all available CPU threads.
+      # To prevent excessive memory usage, we limit parallel link jobs based on available RAM,
+      # as each link job requires significant memory during optimization.
+      set(_default_ram_value 16384)
+    endif()
+    set(Scylla_RAM_PER_LINK_JOB ${_default_ram_value} CACHE STRING
+      "Maximum amount of memory used by each link job (in MiB)")
+  endif()
+  cmake_host_system_information(
+    RESULT _total_mem_mb
+    QUERY AVAILABLE_PHYSICAL_MEMORY)
+  math(EXPR _link_pool_depth "${_total_mem_mb} / ${Scylla_RAM_PER_LINK_JOB}")
+  # Use 2 parallel link jobs to optimize build throughput. The main executable requires
+  # LTO (slower link phase) while tests are linked without LTO (faster link phase).
+  # This allows simultaneous linking of LTO and non-LTO targets, enabling better CPU
+  # utilization by overlapping the slower LTO link with faster test links.
+  if(_link_pool_depth LESS 2)
+    set(_link_pool_depth 2)
+  endif()
 
-cmake_host_system_information(
-  RESULT _total_mem
-  QUERY AVAILABLE_PHYSICAL_MEMORY)
-math(EXPR _link_pool_depth "${_total_mem} / ${LINK_MEM_PER_JOB}")
-if(_link_pool_depth EQUAL 0)
-  set(_link_pool_depth 1)
+  set(Scylla_PARALLEL_LINK_JOBS "${_link_pool_depth}" CACHE STRING
+    "Maximum number of concurrent link jobs")
 endif()
+
 set_property(
   GLOBAL
   APPEND
   PROPERTY JOB_POOLS
-    link_pool=${_link_pool_depth}
+    link_pool=${Scylla_PARALLEL_LINK_JOBS}
     submodule_pool=1)
 set(CMAKE_JOB_POOL_LINK link_pool)

cmake/mode.common.cmake

@@ -122,7 +122,9 @@ if(target_arch)
   add_compile_options("-march=${target_arch}")
 endif()
 
-add_compile_options("SHELL:-Xclang -fexperimental-assignment-tracking=disabled")
+if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+  add_compile_options("SHELL:-Xclang -fexperimental-assignment-tracking=disabled")
+endif()
 
 function(maybe_limit_stack_usage_in_KB stack_usage_threshold_in_KB config)
   math(EXPR _stack_usage_threshold_in_bytes "${stack_usage_threshold_in_KB} * 1024")

collection_mutation.cc

@@ -16,8 +16,6 @@
 
 #include "collection_mutation.hh"
 
-#include <boost/range/numeric.hpp>
-
 bytes_view collection_mutation_input_stream::read_linearized(size_t n) {
     managed_bytes_view mbv = ::read_simple_bytes(_src, n);
     if (mbv.is_linearized()) {

compaction/compaction.cc

@@ -769,7 +769,7 @@ private:
     }
 
     virtual sstables::sstable_set make_sstable_set_for_input() const {
-        return _table_s.get_compaction_strategy().make_sstable_set(_table_s);
+        return _table_s.get_compaction_strategy().make_sstable_set(_schema);
     }
 
     const tombstone_gc_state& get_tombstone_gc_state() const {
@@ -1301,7 +1301,7 @@ public:
     }
 
     virtual sstables::sstable_set make_sstable_set_for_input() const override {
-        return sstables::make_partitioned_sstable_set(_schema, _table_s.token_range());
+        return sstables::make_partitioned_sstable_set(_schema, false);
     }
 
     // Unconditionally enable incremental compaction if the strategy specifies a max output size, e.g. LCS.
@@ -1910,11 +1910,7 @@ static future<compaction_result> scrub_sstables_validate_mode(sstables::compacti
     using scrub = sstables::compaction_type_options::scrub;
     if (validation_errors != 0 && descriptor.options.as<scrub>().quarantine_sstables == scrub::quarantine_invalid_sstables::yes) {
         for (auto& sst : descriptor.sstables) {
-            try {
-                co_await sst->change_state(sstables::sstable_state::quarantine);
-            } catch (...) {
-                clogger.error("Moving {} to quarantine failed due to {}, continuing.", sst->get_filename(), std::current_exception());
-            }
+            co_await sst->change_state(sstables::sstable_state::quarantine);
         }
     }
 

compaction/compaction_manager.cc

@@ -29,6 +29,7 @@
 #include "db/system_keyspace.hh"
 #include "tombstone_gc-internals.hh"
 #include <cmath>
+#include "utils/labels.hh"
 
 static logging::logger cmlog("compaction_manager");
 using namespace std::chrono_literals;
@@ -394,7 +395,7 @@ future<sstables::sstable_set> compaction_task_executor::sstable_set_for_tombston
     auto compound_set = t.sstable_set_for_tombstone_gc();
     // Compound set will be linearized into a single set, since compaction might add or remove sstables
     // to it for incremental compaction to work.
-    auto new_set = sstables::make_partitioned_sstable_set(t.schema(), t.token_range());
+    auto new_set = sstables::make_partitioned_sstable_set(t.schema(), false);
     co_await compound_set->for_each_sstable_gently([&] (const sstables::shared_sstable& sst) {
         auto inserted = new_set.insert(sst);
         if (!inserted) {
@@ -997,7 +998,7 @@ void compaction_manager::register_metrics() {
 
     _metrics.add_group("compaction_manager", {
         sm::make_gauge("compactions", [this] { return _stats.active_tasks; },
-                       sm::description("Holds the number of currently active compactions.")),
+                       sm::description("Holds the number of currently active compactions."))(basic_level),
         sm::make_gauge("pending_compactions", [this] { return _stats.pending_tasks; },
                        sm::description("Holds the number of compaction tasks waiting for an opportunity to run.")),
         sm::make_counter("completed_compactions", [this] { return _stats.completed_tasks; },
@@ -1127,16 +1128,16 @@ future<> compaction_manager::drain() {
         // Disable the state so that it can be enabled later if requested.
         _state = state::disabled;
     }
-    _compaction_submission_timer.cancel();
     // Stop ongoing compactions, if the request has not been sent already and wait for them to stop.
     co_await stop_ongoing_compactions("drain");
-    // Trigger a signal to properly exit from postponed_compactions_reevaluation() fiber
-    reevaluate_postponed_compactions();
     cmlog.info("Drained");
 }
 
 future<> compaction_manager::stop() {
     do_stop();
+    if (auto cm = std::exchange(_task_manager_module, nullptr)) {
+        co_await cm->stop();
+    }
     if (_stop_future) {
         co_await std::exchange(*_stop_future, make_ready_future());
     }
@@ -1147,15 +1148,14 @@ future<> compaction_manager::really_do_stop() noexcept {
     // Reset the metrics registry
     _metrics.clear();
     co_await stop_ongoing_compactions("shutdown");
-    co_await _task_manager_module->stop();
+    if (!_tasks.empty()) {
+        on_fatal_internal_error(cmlog, format("{} tasks still exist after being stopped", _tasks.size()));
+    }
     co_await coroutine::parallel_for_each(_compaction_state | std::views::values, [] (compaction_state& cs) -> future<> {
         if (!cs.gate.is_closed()) {
             co_await cs.gate.close();
         }
     });
-    if (!_tasks.empty()) {
-        on_fatal_internal_error(cmlog, format("{} tasks still exist after being stopped", _tasks.size()));
-    }
     reevaluate_postponed_compactions();
     co_await std::move(_waiting_reevalution);
     _weight_tracker.clear();
@@ -1818,21 +1818,8 @@ future<compaction_manager::compaction_stats_opt> compaction_manager::perform_sst
     if (!gh) {
         co_return compaction_stats_opt{};
     }
-
-    // Collect and register all sstables as compacting while compaction is disabled, to avoid a race condition where
-    // regular compaction runs in between and picks the same files.
-    std::vector<sstables::shared_sstable> all_sstables;
-    compacting_sstable_registration compacting(*this, get_compaction_state(&t));
-    co_await run_with_compaction_disabled(t, [&all_sstables, &compacting, &t] () -> future<> {
-        // All sstables must be included.
-        all_sstables = get_all_sstables(t);
-        compacting.register_compacting(all_sstables);
-        return make_ready_future<>();
-    });
-    if (all_sstables.empty()) {
-        co_return compaction_stats_opt{};
-    }
-
+    // All sstables must be included, even the ones being compacted, such that everything in table is validated.
+    auto all_sstables = get_all_sstables(t);
     co_return co_await perform_compaction<validate_sstables_compaction_task_executor>(throw_if_stopping::no, info, &t, info.id, std::move(all_sstables), quarantine_sstables);
 }
 

compaction/compaction_manager.hh

@@ -300,11 +300,6 @@ public:
     // unless it is moved back to enabled state.
     future<> drain();
 
-    // Check if compaction manager is running, i.e. it was enabled or drained
-    bool is_running() const noexcept {
-        return _state == state::enabled || _state == state::disabled;
-    }
-
     using compaction_history_consumer = noncopyable_function<future<>(const db::compaction_history_entry&)>;
     future<> get_compaction_history(compaction_history_consumer&& f);
 

compaction/compaction_strategy.cc

@@ -789,8 +789,8 @@ future<reshape_config> make_reshape_config(const sstables::storage& storage, res
     };
 }
 
-std::unique_ptr<sstable_set_impl> incremental_compaction_strategy::make_sstable_set(const table_state& ts) const {
-    return std::make_unique<partitioned_sstable_set>(ts.schema(), ts.token_range());
+std::unique_ptr<sstable_set_impl> incremental_compaction_strategy::make_sstable_set(schema_ptr schema) const {
+    return std::make_unique<partitioned_sstable_set>(std::move(schema), false);
 }
 
 }

compaction/compaction_strategy.hh

@@ -105,7 +105,7 @@ public:
         return name(type());
     }
 
-    sstable_set make_sstable_set(const table_state& ts) const;
+    sstable_set make_sstable_set(schema_ptr schema) const;
 
     compaction_backlog_tracker make_backlog_tracker() const;
 

compaction/compaction_strategy_impl.hh

@@ -56,7 +56,7 @@ public:
         return true;
     }
     virtual int64_t estimated_pending_compactions(table_state& table_s) const = 0;
-    virtual std::unique_ptr<sstable_set_impl> make_sstable_set(const table_state& ts) const;
+    virtual std::unique_ptr<sstable_set_impl> make_sstable_set(schema_ptr schema) const;
 
     bool use_clustering_key_filter() const {
         return _use_clustering_key_filter;

compaction/incremental_backlog_tracker.cc

@@ -6,7 +6,6 @@
 
 #include "incremental_backlog_tracker.hh"
 #include "sstables/sstables.hh"
-#include <boost/range/adaptor/map.hpp>
 
 using namespace sstables;
 

compaction/incremental_compaction_strategy.cc

@@ -11,10 +11,6 @@
 #include "compaction_manager.hh"
 #include "incremental_compaction_strategy.hh"
 #include "incremental_backlog_tracker.hh"
-#include <boost/algorithm/cxx11/any_of.hpp>
-#include <boost/range/numeric.hpp>
-#include <boost/range/algorithm.hpp>
-#include <boost/range/adaptors.hpp>
 #include <ranges>
 
 namespace sstables {
@@ -152,7 +148,7 @@ bool incremental_compaction_strategy::is_bucket_interesting(const std::vector<ss
 }
 
 bool incremental_compaction_strategy::is_any_bucket_interesting(const std::vector<std::vector<sstables::frozen_sstable_run>>& buckets, size_t min_threshold) const {
-    return boost::algorithm::any_of(buckets, [&] (const std::vector<sstables::frozen_sstable_run>& bucket) {
+    return std::ranges::any_of(buckets, [&] (const std::vector<sstables::frozen_sstable_run>& bucket) {
         return this->is_bucket_interesting(bucket, min_threshold);
     });
 }
@@ -282,7 +278,7 @@ incremental_compaction_strategy::find_garbage_collection_job(const compaction::t
     };
     auto compaction_time = gc_clock::now();
     auto can_garbage_collect = [&] (const size_bucket_t& bucket) {
-        return boost::algorithm::any_of(bucket, [&] (const frozen_sstable_run& r) {
+        return std::ranges::any_of(bucket, [&] (const frozen_sstable_run& r) {
             return worth_dropping_tombstones(*r, compaction_time);
         });
     };
@@ -418,11 +414,10 @@ int64_t incremental_compaction_strategy::estimated_pending_compactions(table_sta
 
 std::vector<shared_sstable>
 incremental_compaction_strategy::runs_to_sstables(std::vector<frozen_sstable_run> runs) {
-    return boost::accumulate(runs, std::vector<shared_sstable>(), [&] (std::vector<shared_sstable>&& v, const frozen_sstable_run& run_ptr) {
-        auto& run = *run_ptr;
-        v.insert(v.end(), run.all().begin(), run.all().end());
-        return std::move(v);
-    });
+    return runs
+        | std::views::transform([] (auto& run) -> auto& { return run->all(); })
+        | std::views::join
+        | std::ranges::to<std::vector>();
 }
 
 std::vector<frozen_sstable_run>
@@ -441,8 +436,8 @@ void incremental_compaction_strategy::sort_run_bucket_by_first_key(size_bucket_t
         auto sst_first_key_less = [&schema] (const shared_sstable& sst_a, const shared_sstable& sst_b) {
             return sst_a->get_first_decorated_key().tri_compare(*schema, sst_b->get_first_decorated_key()) <= 0;
         };
-        auto& a_first = *boost::min_element(a->all(), sst_first_key_less);
-        auto& b_first = *boost::min_element(b->all(), sst_first_key_less);
+        auto& a_first = *std::ranges::min_element(a->all(), sst_first_key_less);
+        auto& b_first = *std::ranges::min_element(b->all(), sst_first_key_less);
         return a_first->get_first_decorated_key().tri_compare(*schema, b_first->get_first_decorated_key()) <= 0;
     });
 }

compaction/incremental_compaction_strategy.hh

@@ -7,7 +7,6 @@
 #pragma once
 
 #include "compaction_strategy_impl.hh"
-#include <boost/range/algorithm.hpp>
 
 class incremental_backlog_tracker;
 
@@ -99,7 +98,7 @@ public:
 
     virtual compaction_descriptor get_reshaping_job(std::vector<shared_sstable> input, schema_ptr schema, reshape_config cfg) const override;
 
-    virtual std::unique_ptr<sstable_set_impl> make_sstable_set(const table_state& ts) const override;
+    virtual std::unique_ptr<sstable_set_impl> make_sstable_set(schema_ptr schema) const override;
 
     friend class ::incremental_backlog_tracker;
 };

compaction/leveled_compaction_strategy.cc

@@ -11,8 +11,6 @@
 #include "compaction_strategy_state.hh"
 #include <algorithm>
 
-#include <boost/range/algorithm/remove_if.hpp>
-
 namespace sstables {
 
 leveled_compaction_strategy_state& leveled_compaction_strategy::get_state(table_state& table_s) const {
@@ -50,10 +48,9 @@ compaction_descriptor leveled_compaction_strategy::get_sstables_for_compaction(t
     for (auto level = int(manifest.get_level_count()); level >= 0; level--) {
         auto& sstables = manifest.get_level(level);
         // filter out sstables which droppable tombstone ratio isn't greater than the defined threshold.
-        auto e = boost::range::remove_if(sstables, [this, compaction_time, &table_s] (const sstables::shared_sstable& sst) -> bool {
+        std::erase_if(sstables, [this, compaction_time, &table_s] (const sstables::shared_sstable& sst) -> bool {
             return !worth_dropping_tombstones(sst, compaction_time, table_s);
         });
-        sstables.erase(e, sstables.end());
         if (sstables.empty()) {
             continue;
         }

compaction/leveled_compaction_strategy.hh

@@ -70,7 +70,7 @@ public:
     virtual compaction_strategy_type type() const override {
         return compaction_strategy_type::leveled;
     }
-    virtual std::unique_ptr<sstable_set_impl> make_sstable_set(const table_state& ts) const override;
+    virtual std::unique_ptr<sstable_set_impl> make_sstable_set(schema_ptr schema) const override;
 
     virtual std::unique_ptr<compaction_backlog_tracker::impl> make_backlog_tracker() const override;
 

compaction/size_tiered_compaction_strategy.cc

@@ -11,8 +11,6 @@
 #include "size_tiered_compaction_strategy.hh"
 #include "cql3/statements/property_definitions.hh"
 
-#include <boost/range/algorithm.hpp>
-
 namespace sstables {
 
 static long validate_sstable_size(const std::map<sstring, sstring>& options) {
@@ -242,10 +240,9 @@ size_tiered_compaction_strategy::get_sstables_for_compaction(table_state& table_
     // tombstone purge, i.e. less likely to shadow even older data.
     for (auto&& sstables : buckets | std::views::reverse) {
         // filter out sstables which droppable tombstone ratio isn't greater than the defined threshold.
-        auto e = boost::range::remove_if(sstables, [this, compaction_time, &table_s] (const sstables::shared_sstable& sst) -> bool {
+        std::erase_if(sstables, [this, compaction_time, &table_s] (const sstables::shared_sstable& sst) -> bool {
             return !worth_dropping_tombstones(sst, compaction_time, table_s);
         });
-        sstables.erase(e, sstables.end());
         if (sstables.empty()) {
             continue;
         }

compaction/table_state.hh

@@ -33,7 +33,6 @@ namespace compaction {
 class table_state {
 public:
     virtual ~table_state() {}
-    virtual dht::token_range token_range() const noexcept = 0;
     virtual const schema_ptr& schema() const noexcept = 0;
     // min threshold as defined by table.
     virtual unsigned min_compaction_threshold() const noexcept = 0;

compaction/task_manager_module.cc

@@ -6,8 +6,6 @@
  * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
  */
 
-#include <boost/range/algorithm/min_element.hpp>
-#include <boost/range/numeric.hpp>
 #include <seastar/coroutine/maybe_yield.hh>
 #include <seastar/coroutine/parallel_for_each.hh>
 
@@ -109,7 +107,7 @@ distribute_reshard_jobs(sstables::sstable_directory::sstable_open_info_vector so
         // Choose the stable shard owner with the smallest amount of accumulated work.
         // Note that for sstables that need cleanup via resharding, owners may contain
         // a single shard.
-        auto shard_it = boost::min_element(info.owners, [&] (const shard_id& lhs, const shard_id& rhs) {
+        auto shard_it = std::ranges::min_element(info.owners, [&] (const shard_id& lhs, const shard_id& rhs) {
             return destinations[lhs].total_size_smaller(destinations[rhs]);
         });
         auto& dest = destinations[*shard_it];

compaction/time_window_compaction_strategy.cc

@@ -13,10 +13,6 @@
 #include "sstables/sstables.hh"
 #include "compaction_strategy_state.hh"
 
-#include <boost/range/algorithm/find.hpp>
-#include <boost/range/algorithm/remove_if.hpp>
-#include <boost/range/algorithm/min_element.hpp>
-
 #include <ranges>
 
 namespace sstables {
@@ -167,7 +163,7 @@ public:
     explicit classify_by_timestamp(time_window_compaction_strategy_options options) : _options(std::move(options)) { }
     int64_t operator()(api::timestamp_type ts) {
         const auto window = time_window_compaction_strategy::get_window_for(_options, ts);
-        if (const auto it = boost::range::find(_known_windows, window); it != _known_windows.end()) {
+        if (const auto it = std::ranges::find(_known_windows, window); it != _known_windows.end()) {
             std::swap(*it, _known_windows.front());
             return window;
         }
@@ -400,14 +396,13 @@ time_window_compaction_strategy::get_next_non_expired_sstables(table_state& tabl
 
     // if there is no sstable to compact in standard way, try compacting single sstable whose droppable tombstone
     // ratio is greater than threshold.
-    auto e = boost::range::remove_if(non_expiring_sstables, [this, compaction_time, &table_s] (const shared_sstable& sst) -> bool {
+    std::erase_if(non_expiring_sstables, [this, compaction_time, &table_s] (const shared_sstable& sst) -> bool {
         return !worth_dropping_tombstones(sst, compaction_time, table_s);
     });
-    non_expiring_sstables.erase(e, non_expiring_sstables.end());
     if (non_expiring_sstables.empty()) {
         return {};
     }
-    auto it = boost::min_element(non_expiring_sstables, [] (auto& i, auto& j) {
+    auto it = std::ranges::min_element(non_expiring_sstables, [] (auto& i, auto& j) {
         return i->get_stats_metadata().min_timestamp < j->get_stats_metadata().min_timestamp;
     });
     return { *it };

compaction/time_window_compaction_strategy.hh

@@ -150,7 +150,7 @@ public:
         return compaction_strategy_type::time_window;
     }
 
-    virtual std::unique_ptr<sstable_set_impl> make_sstable_set(const table_state& ts) const override;
+    virtual std::unique_ptr<sstable_set_impl> make_sstable_set(schema_ptr schema) const override;
 
     virtual std::unique_ptr<compaction_backlog_tracker::impl> make_backlog_tracker() const override;
 

compound.hh

@@ -255,9 +255,6 @@ public:
     // Returns true iff given prefix has no missing components
     bool is_full(managed_bytes_view v) const {
         SCYLLA_ASSERT(AllowPrefixes == allow_prefixes::yes);
-        if (_types.size() == 0) {
-            return v.empty();
-        }
         return std::distance(begin(v), end(v)) == (ssize_t)_types.size();
     }
     bool is_empty(managed_bytes_view v) const {

compress.cc

@@ -14,7 +14,9 @@
 #include "exceptions/exceptions.hh"
 #include "utils/class_registrator.hh"
 
-const sstring compressor::namespace_prefix = "org.apache.cassandra.io.compress.";
+sstring compressor::make_name(std::string_view short_name) {
+    return seastar::format("org.apache.cassandra.io.compress.{}", short_name);
+}
 
 class lz4_processor: public compressor {
 public:
@@ -66,7 +68,7 @@ compressor::ptr_type compressor::create(const sstring& name, const opt_getter& o
         return {};
     }
 
-    qualified_name qn(namespace_prefix, name);
+    qualified_name qn(make_name(""), name);
 
     for (auto& c : { lz4, snappy, deflate }) {
         if (c->name() == static_cast<const sstring&>(qn)) {
@@ -91,9 +93,9 @@ shared_ptr<compressor> compressor::create(const std::map<sstring, sstring>& opti
     return {};
 }
 
-thread_local const shared_ptr<compressor> compressor::lz4 = ::make_shared<lz4_processor>(namespace_prefix + "LZ4Compressor");
-thread_local const shared_ptr<compressor> compressor::snappy = ::make_shared<snappy_processor>(namespace_prefix + "SnappyCompressor");
-thread_local const shared_ptr<compressor> compressor::deflate = ::make_shared<deflate_processor>(namespace_prefix + "DeflateCompressor");
+thread_local const shared_ptr<compressor> compressor::lz4 = ::make_shared<lz4_processor>(make_name("LZ4Compressor"));
+thread_local const shared_ptr<compressor> compressor::snappy = ::make_shared<snappy_processor>(make_name("SnappyCompressor"));
+thread_local const shared_ptr<compressor> compressor::deflate = ::make_shared<deflate_processor>(make_name("DeflateCompressor"));
 
 const sstring compression_parameters::SSTABLE_COMPRESSION = "sstable_compression";
 const sstring compression_parameters::CHUNK_LENGTH_KB = "chunk_length_in_kb";

compress.hh

@@ -69,7 +69,7 @@ public:
     static thread_local const ptr_type snappy;
     static thread_local const ptr_type deflate;
 
-    static const sstring namespace_prefix;
+    static sstring make_name(std::string_view short_name);
 };
 
 template<typename BaseType, typename... Args>

concrete_types.hh

@@ -15,6 +15,7 @@
 #include "types/map.hh"
 #include "types/set.hh"
 #include "types/tuple.hh"
+#include "types/vector.hh"
 #include "types/user.hh"
 #include "utils/big_decimal.hh"
 
@@ -170,6 +171,7 @@ template <typename Func> concept CanHandleAllTypes = requires(Func f) {
     { f(*static_cast<const utf8_type_impl*>(nullptr)) }        -> std::same_as<visit_ret_type<Func>>;
     { f(*static_cast<const uuid_type_impl*>(nullptr)) }        -> std::same_as<visit_ret_type<Func>>;
     { f(*static_cast<const varint_type_impl*>(nullptr)) }      -> std::same_as<visit_ret_type<Func>>;
+    { f(*static_cast<const vector_type_impl*>(nullptr)) }      -> std::same_as<visit_ret_type<Func>>;
 };
 
 template<typename Func>
@@ -232,6 +234,8 @@ inline visit_ret_type<Func> visit(const abstract_type& t, Func&& f) {
         return f(*static_cast<const uuid_type_impl*>(&t));
     case abstract_type::kind::varint:
         return f(*static_cast<const varint_type_impl*>(&t));
+    case abstract_type::kind::vector:
+        return f(*static_cast<const vector_type_impl*>(&t));
     }
     __builtin_unreachable();
 }

conf/scylla.yaml

@@ -473,6 +473,7 @@ commitlog_total_space_in_mb: -1
 #    certficate_revocation_list: <not set>
 #    require_client_auth: False
 #    priority_string: <not set, use default>
+#    enable_session_tickets: <default false>
 
 # internode_compression controls whether traffic between nodes is
 # compressed.
@@ -662,6 +663,153 @@ maintenance_socket: ignore
 # maximum_replication_factor_warn_threshold: -1
 # maximum_replication_factor_fail_threshold: -1
 
+#
+# System information encryption settings
+#
+# If enabled, system tables that may contain sensitive information (system.batchlog,
+# system.paxos), hints files and commit logs are encrypted with the
+# encryption settings below.
+#
+# When enabling system table encryption on a node with existing data, run
+# `nodetool upgradesstables -a` on the listed tables to encrypt existing data.
+#
+# When tracing is enabled, sensitive info will be written into the tables in the
+# system_traces keyspace. Those tables should be configured to encrypt their data
+# on disk.
+#
+# It is recommended to use remote encryption keys from a KMIP server/KMS when using
+# Transparent Data Encryption (TDE) features.
+# Local key support is provided when a KMIP server/KMS is not available.
+#
+# See the scylla documentation for more info on available key providers and 
+# their properties.
+#
+# system_info_encryption:
+#   enabled: true
+#   cipher_algorithm: AES
+#   secret_key_strength: 128  
+#   key_provider: LocalFileSystemKeyProviderFactory
+#   secret_key_file: <key file>
+#
+# system_info_encryption:
+#   enabled: true
+#   cipher_algorithm: AES
+#   secret_key_strength: 128  
+#   key_provider: KmipKeyProviderFactory
+#   kmip_host: <kmip host group>
+#   template_name: <kmip key template name> (optional)
+#   key_namespace: <kmip key namespace> (optional)
+#
+
+#
+# The directory where system keys are kept
+# This directory should have 700 permissions and belong to the scylla user
+#
+# system_key_directory: /etc/scylla/conf/resources/system_keys
+#
+
+#
+# KMIP host(s).
+#
+# The unique name of kmip host/cluster that can be referenced in table schema.
+#
+# host.yourdomain.com={ hosts=[<host1>, <host2>...], keyfile=/path/to/keyfile, truststore=/path/to/truststore.pem, key_cache_millis=<cache ms>, timeout=<timeout ms> }:...
+#
+# The KMIP connection management only supports failover, so all requests will go through a
+# single KMIP server. There is no load balancing, as no KMIP servers (at the time of this writing)
+# support read replication, or other strategies for availability.
+#
+# Hosts are tried in the order they appear here. Add them in the same sequence they'll fail over in.
+# 
+# KMIP requests will fail over/retry 'max_command_retries' times (default 3)
+#
+# kmip_hosts:
+#   <name>:
+#       hosts: <address1[:port]> [, <address2[:port]>...]
+#       certificate: <identifying certificate> (optional)
+#       keyfile: <identifying key> (optional)
+#       truststore: <truststore for SSL connection> (optional)
+#       priority_string: <kmip tls priority string> (optional)
+#       username: <login> (optional>
+#       password: <password> (optional)
+#       max_command_retries: <int> (optional; default 3)
+#       key_cache_expiry: <key cache expiry period>
+#       key_cache_refresh: <key cache refresh/prune period>
+#   <name>:
+#       ...
+#
+
+#
+# KMS host(s).
+#
+# The unique name of kms host/account config that can be referenced in table schema.
+#
+# host.yourdomain.com={ endpoint=<http(s)://host[:port]>, aws_access_key_id=<AWS access id>, aws_secret_access_key=<AWS secret key>, aws_region=<AWS region>, master_key=<alias or id>, keyfile=/path/to/keyfile, truststore=/path/to/truststore.pem, key_cache_millis=<cache ms>, timeout=<timeout ms> }:...
+#
+# Actual connection can be either an explicit endpoint (<host>:<port>), or selected automatic via aws_region.
+# 
+# Authentication can be explicit with aws_access_key_id and aws_secret_access_key. Either secret or both can be omitted
+# in which case the provider will try to read them from AWS credentials in ~/.aws/credentials. If aws_profile is set, the
+# credentials in this section is used.
+#
+# master_key is an AWS KMS key id or alias from which all keys used for actual encryption of scylla data will be derived.
+# This key must be pre-created with access policy allowing the above AWS id Encrypt, Decrypt and GenerateDataKey operations.
+#
+# kms_hosts:
+#   <name>:
+#       endpoint: http(s)://<host>(:port) (optional)
+#       aws_region: <aws region> (optional)
+#       aws_access_key_id: <aws access key id> (optional)
+#       aws_secret_access_key: <aws secret access key> (optional)
+#       aws_profile: <aws credentials profile> (optional)
+#       aws_use_ec2_credentials: <bool> (default false) If true, KMS queries will use the credentials provided by ec2 instance role metadata as initial access key. 
+#       aws_use_ec2_region: <bool> (default false) If true, KMS queries will use the AWS region indicated by ec2 instance metadata
+#       aws_assume_role_arn: <aws role arn> (optional) If set, any KMS query will first attempt to assume this role. 
+#       master_key: <named KMS key for encrypting data keys> (required)
+#       certificate: <identifying certificate> (optional)
+#       keyfile: <identifying key> (optional)
+#       truststore: <truststore for SSL connection> (optional)
+#       priority_string: <kmip tls priority string> (optional)
+#       key_cache_expiry: <key cache expiry period>
+#       key_cache_refresh: <key cache refresh/prune period>
+#   <name>:
+#       ...
+#
+
+#
+# Server-global user information encryption settings
+#
+# If enabled, all user tables are encrypted with the
+# encryption settings below, unless the table has local scylla_encryption_options
+# specified.
+#
+# When enabling user table encryption on a node with existing data, run
+# `nodetool upgradesstables -a` on all user tables to encrypt existing data.
+#
+# It is recommended to use remote encryption keys from a KMIP server or KMS when using
+# Transparent Data Encryption (TDE) features.
+# Local key support is provided when a KMIP server/KMS is not available.
+#
+# See the scylla documentation for more info on available key providers and 
+# their properties.
+#
+# user_info_encryption:
+#   enabled: true
+#   cipher_algorithm: AES
+#   secret_key_strength: 128  
+#   key_provider: LocalFileSystemKeyProviderFactory
+#   secret_key_file: <key file>
+#
+# user_info_encryption:
+#   enabled: true
+#   cipher_algorithm: AES
+#   secret_key_strength: 128  
+#   key_provider: KmipKeyProviderFactory
+#   kmip_host: <kmip host group>
+#   template_name: <kmip key template name> (optional)
+#   key_namespace: <kmip key namespace> (optional)
+#
+
 # Guardrails to warn about or disallow creating a keyspace with specific replication strategy.
 # Each of these 2 settings is a list storing replication strategies considered harmful.
 # The replication strategies to choose from are:
@@ -677,9 +825,7 @@ maintenance_socket: ignore
 # Guardrail to enable the deprecated feature of CREATE TABLE WITH COMPACT STORAGE.
 # enable_create_table_with_compact_storage: false
 
-# Control tablets for new keyspaces.
-# Can be set to: disabled|enabled
-#
+# Enable tablets for new keyspaces.
 # When enabled, newly created keyspaces will have tablets enabled by default.
 # That can be explicitly disabled in the CREATE KEYSPACE query
 # by using the `tablets = {'enabled': false}` replication option.
@@ -688,15 +834,6 @@ maintenance_socket: ignore
 # unless tablets are explicitly enabled in the CREATE KEYSPACE query
 # by using the `tablets = {'enabled': true}` replication option.
 #
-# When set to `enforced`, newly created keyspaces will always have tablets enabled by default.
-# This prevents explicitly disabling tablets in the CREATE KEYSPACE query
-# using the `tablets = {'enabled': false}` replication option.
-# It also mandates a replication strategy supporting tablets, like
-# NetworkTopologyStrategy
-#
 # Note that creating keyspaces with tablets enabled or disabled is irreversible.
 # The `tablets` option cannot be changed using `ALTER KEYSPACE`.
-tablets_mode_for_new_keyspaces: enabled
-
-# Enforce RF-rack-valid keyspaces.
-rf_rack_valid_keyspaces: false
+enable_tablets: true

configure.py

@@ -795,7 +795,6 @@ scylla_core = (['message/messaging_service.cc',
                 'frozen_schema.cc',
                 'bytes.cc',
                 'timeout_config.cc',
-                'row_cache.cc',
                 'schema_mutations.cc',
                 'generic_server.cc',
                 'utils/alien_worker.cc',
@@ -813,10 +812,10 @@ scylla_core = (['message/messaging_service.cc',
                 'utils/rjson.cc',
                 'utils/human_readable.cc',
                 'utils/histogram_metrics_helper.cc',
-                'utils/io-wrappers.cc',
                 'utils/on_internal_error.cc',
                 'utils/pretty_printers.cc',
                 'utils/stream_compressor.cc',
+                'utils/labels.cc',
                 'converting_mutation_partition_applier.cc',
                 'readers/combined.cc',
                 'readers/multishard.cc',
@@ -976,44 +975,45 @@ scylla_core = (['message/messaging_service.cc',
                 'cql3/restrictions/statement_restrictions.cc',
                 'cql3/result_set.cc',
                 'cql3/prepare_context.cc',
-                'db/consistency_level.cc',
-                'db/system_keyspace.cc',
-                'db/virtual_table.cc',
-                'db/virtual_tables.cc',
-                'db/system_distributed_keyspace.cc',
-                'db/size_estimates_virtual_reader.cc',
-                'db/schema_applier.cc',
-                'db/schema_tables.cc',
-                'db/cql_type_parser.cc',
-                'db/legacy_schema_migrator.cc',
+                'db/batchlog_manager.cc',
                 'db/commitlog/commitlog.cc',
-                'db/commitlog/commitlog_replayer.cc',
                 'db/commitlog/commitlog_entry.cc',
+                'db/commitlog/commitlog_replayer.cc',
+                'db/config.cc',
+                'db/consistency_level.cc',
+                'db/cql_type_parser.cc',
                 'db/data_listeners.cc',
+                'db/extensions.cc',
                 'db/functions/function.cc',
+                'db/heat_load_balance.cc',
+                'db/hints/host_filter.cc',
                 'db/hints/internal/hint_endpoint_manager.cc',
                 'db/hints/internal/hint_sender.cc',
                 'db/hints/internal/hint_storage.cc',
                 'db/hints/manager.cc',
                 'db/hints/resource_manager.cc',
-                'db/hints/host_filter.cc',
                 'db/hints/sync_point.cc',
-                'db/config.cc',
-                'db/extensions.cc',
-                'db/heat_load_balance.cc',
                 'db/large_data_handler.cc',
-                'db/corrupt_data_handler.cc',
+                'db/legacy_schema_migrator.cc',
                 'db/marshal/type_parser.cc',
-                'db/batchlog_manager.cc',
+                'db/per_partition_rate_limit_options.cc',
+                'db/rate_limiter.cc',
+                'db/row_cache.cc',
+                'db/schema_applier.cc',
+                'db/schema_tables.cc',
+                'db/size_estimates_virtual_reader.cc',
+                'db/snapshot-ctl.cc',
+                'db/snapshot/backup_task.cc',
+                'db/sstables-format-selector.cc',
+                'db/system_distributed_keyspace.cc',
+                'db/system_keyspace.cc',
                 'db/tags/utils.cc',
+                'db/view/row_locking.cc',
                 'db/view/view.cc',
                 'db/view/view_update_generator.cc',
-                'db/view/row_locking.cc',
-                'db/sstables-format-selector.cc',
-                'db/snapshot-ctl.cc',
-                'db/rate_limiter.cc',
-                'db/per_partition_rate_limit_options.cc',
-                'db/snapshot/backup_task.cc',
+                'db/virtual_table.cc',
+                'db/virtual_tables.cc',
+                'db/tablet_options.cc',
                 'index/secondary_index_manager.cc',
                 'index/secondary_index.cc',
                 'utils/UUID_gen.cc',
@@ -1029,10 +1029,14 @@ scylla_core = (['message/messaging_service.cc',
                 'utils/multiprecision_int.cc',
                 'utils/gz/crc_combine.cc',
                 'utils/gz/crc_combine_table.cc',
-                'utils/http.cc',
                 'utils/s3/aws_error.cc',
                 'utils/s3/client.cc',
                 'utils/s3/retry_strategy.cc',
+                'utils/s3/credentials_providers/aws_credentials_provider.cc',
+                'utils/s3/credentials_providers/environment_aws_credentials_provider.cc',
+                'utils/s3/credentials_providers/instance_profile_credentials_provider.cc',
+                'utils/s3/credentials_providers/sts_assume_role_credentials_provider.cc',
+                'utils/s3/credentials_providers/aws_credentials_provider_chain.cc',
                 'utils/advanced_rpc_compressor.cc',
                 'gms/version_generator.cc',
                 'gms/versioned_value.cc',
@@ -1102,7 +1106,7 @@ scylla_core = (['message/messaging_service.cc',
                 'utils/lister.cc',
                 'repair/repair.cc',
                 'repair/row_level.cc',
-                'streaming/table_check.cc',
+                'repair/table_check.cc',
                 'exceptions/exceptions.cc',
                 'auth/allow_all_authenticator.cc',
                 'auth/allow_all_authorizer.cc',
@@ -1324,7 +1328,6 @@ idls = ['idl/gossip_digest.idl.hh',
         'idl/replica_exception.idl.hh',
         'idl/per_partition_rate_limit_info.idl.hh',
         'idl/position_in_partition.idl.hh',
-        'idl/full_position.idl.hh',
         'idl/experimental/broadcast_tables_lang.idl.hh',
         'idl/storage_service.idl.hh',
         'idl/join_node.idl.hh',
@@ -1332,7 +1335,7 @@ idls = ['idl/gossip_digest.idl.hh',
         'idl/gossip.idl.hh',
         'idl/migration_manager.idl.hh',
         "idl/node_ops.idl.hh",
-
+        "idl/tasks.idl.hh"
         ]
 
 scylla_tests_generic_dependencies = [
@@ -1349,6 +1352,7 @@ scylla_tests_dependencies = scylla_core + alternator + idls + scylla_tests_gener
     'test/lib/cql_assertions.cc',
     'test/lib/result_set_assertions.cc',
     'test/lib/mutation_source_test.cc',
+    'test/lib/mutation_assertions.cc',
     'test/lib/sstable_utils.cc',
     'test/lib/data_model.cc',
     'test/lib/exception_utils.cc',
@@ -1546,13 +1550,14 @@ deps['test/boost/bytes_ostream_test'] = [
     "bytes.cc",
     "utils/managed_bytes.cc",
     "utils/logalloc.cc",
+    "utils/labels.cc",
     "utils/dynamic_bitset.cc",
     "test/lib/log.cc",
 ]
 deps['test/boost/input_stream_test'] = ['test/boost/input_stream_test.cc']
 deps['test/boost/UUID_test'] = ['clocks-impl.cc', 'utils/UUID_gen.cc', 'test/boost/UUID_test.cc', 'utils/uuid.cc', 'utils/dynamic_bitset.cc', 'utils/hashers.cc', 'utils/on_internal_error.cc']
 deps['test/boost/murmur_hash_test'] = ['bytes.cc', 'utils/murmur_hash.cc', 'test/boost/murmur_hash_test.cc']
-deps['test/boost/allocation_strategy_test'] = ['test/boost/allocation_strategy_test.cc', 'utils/logalloc.cc', 'utils/dynamic_bitset.cc']
+deps['test/boost/allocation_strategy_test'] = ['test/boost/allocation_strategy_test.cc', 'utils/logalloc.cc', 'utils/dynamic_bitset.cc', 'utils/labels.cc']
 deps['test/boost/log_heap_test'] = ['test/boost/log_heap_test.cc']
 deps['test/boost/estimated_histogram_test'] = ['test/boost/estimated_histogram_test.cc']
 deps['test/boost/summary_test'] = ['test/boost/summary_test.cc']
@@ -1661,7 +1666,14 @@ defines = ' '.join(['-D' + d for d in defines])
 globals().update(vars(args))
 
 total_memory = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
+# assuming each link job takes around 7GiB of memory without LTO
 link_pool_depth = max(int(total_memory / 7e9), 1)
+if args.lto:
+    # ThinLTO provides its own parallel linking, use 16GiB for RAM size used
+    # by each link job
+    depth_with_lto = max(int(total_memory / 16e9), 2)
+    if depth_with_lto < link_pool_depth:
+        link_pool_depth = depth_with_lto
 
 selected_modes = args.selected_modes or modes.keys()
 default_modes = args.selected_modes or [mode for mode, mode_cfg in modes.items() if mode_cfg["default"]]
@@ -2141,13 +2153,14 @@ def get_extra_cxxflags(mode, mode_config, cxx, debuginfo):
     if debuginfo and mode_config['can_have_debug_info']:
         cxxflags += ['-g', '-gz']
 
-    # Since AssignmentTracking was enabled by default in clang
-    # (llvm/llvm-project@de6da6ad55d3ca945195d1cb109cb8efdf40a52a)
-    # coroutine frame debugging info (`coro_frame_ty`) is broken.
-    # 
-    # It seems that we aren't losing much by disabling AssigmentTracking,
-    # so for now we choose to disable it to get `coro_frame_ty` back.
-    cxxflags.append('-Xclang -fexperimental-assignment-tracking=disabled')
+    if 'clang' in cxx:
+        # Since AssignmentTracking was enabled by default in clang
+        # (llvm/llvm-project@de6da6ad55d3ca945195d1cb109cb8efdf40a52a)
+        # coroutine frame debugging info (`coro_frame_ty`) is broken.
+        #
+        # It seems that we aren't losing much by disabling AssigmentTracking,
+        # so for now we choose to disable it to get `coro_frame_ty` back.
+        cxxflags.append('-Xclang -fexperimental-assignment-tracking=disabled')
 
     return cxxflags
 

cql3/Cql.g

@@ -709,23 +709,17 @@ batchStatement returns [std::unique_ptr<cql3::statements::raw::batch_statement>
     : K_BEGIN
       ( K_UNLOGGED { type = btype::UNLOGGED; } | K_COUNTER { type = btype::COUNTER; } )?
       K_BATCH ( usingClause[attrs] )?
-          ( s=batchStatementObjective ';'?
-              {
-                  auto&& stmt = *$s.statement;
-                  stmt->add_raw(sstring{$s.text});
-                  statements.push_back(std::move(stmt));
-              } )*
+          ( s=batchStatementObjective ';'? { statements.push_back(std::move(s)); } )*
       K_APPLY K_BATCH
       {
           $expr = std::make_unique<cql3::statements::raw::batch_statement>(type, std::move(attrs), std::move(statements));
       }
     ;
 
-batchStatementObjective returns [::lw_shared_ptr<std::unique_ptr<cql3::statements::raw::modification_statement>> statement]
-    @init { using original_ret_type = std::unique_ptr<cql3::statements::raw::modification_statement>; }
-    : i=insertStatement  { $statement = make_lw_shared<original_ret_type>(std::move(i)); }
-    | u=updateStatement  { $statement = make_lw_shared<original_ret_type>(std::move(u)); }
-    | d=deleteStatement  { $statement = make_lw_shared<original_ret_type>(std::move(d)); }
+batchStatementObjective returns [std::unique_ptr<cql3::statements::raw::modification_statement> statement]
+    : i=insertStatement  { $statement = std::move(i); }
+    | u=updateStatement  { $statement = std::move(u); }
+    | d=deleteStatement  { $statement = std::move(d); }
     ;
 
 dropAggregateStatement returns [std::unique_ptr<cql3::statements::drop_aggregate_statement> expr]
@@ -1627,7 +1621,7 @@ collectionLiteral returns [uexpression value]
 	@init{ std::vector<expression> l; }
     : '['
           ( t1=term { l.push_back(std::move(t1)); } ( ',' tn=term { l.push_back(std::move(tn)); } )* )?
-      ']' { $value = collection_constructor{collection_constructor::style_type::list, std::move(l)}; }
+      ']' { $value = collection_constructor{collection_constructor::style_type::list_or_vector, std::move(l)}; }
     | '{' t=term v=setOrMapLiteral[t] { $value = std::move(v); } '}'
     // Note that we have an ambiguity between maps and set for "{}". So we force it to a set literal,
     // and deal with it later based on the type of the column (SetLiteral.java).
@@ -1771,7 +1765,7 @@ subscriptExpr returns [uexpression e]
     ;
 
 singleColumnInValuesOrMarkerExpr returns [uexpression e]
-    : values=singleColumnInValues { e = collection_constructor{collection_constructor::style_type::list, std::move(values)}; }
+    : values=singleColumnInValues { e = collection_constructor{collection_constructor::style_type::list_or_vector, std::move(values)}; }
     | m=marker { e = std::move(m); }
     ;
 
@@ -1849,7 +1843,7 @@ relation returns [uexpression e]
     | name=cident K_IN in_values=singleColumnInValues
         { $e = binary_operator(unresolved_identifier{std::move(name)}, oper_t::IN,
         collection_constructor {
-            .style = collection_constructor::style_type::list,
+            .style = collection_constructor::style_type::list_or_vector,
             .elements = std::move(in_values)
         }); }
     | name=cident K_NOT K_IN marker1=marker
@@ -1857,7 +1851,7 @@ relation returns [uexpression e]
     | name=cident K_NOT K_IN in_values=singleColumnInValues
         { $e = binary_operator(unresolved_identifier{std::move(name)}, oper_t::NOT_IN,
         collection_constructor {
-            .style = collection_constructor::style_type::list,
+            .style = collection_constructor::style_type::list_or_vector,
             .elements = std::move(in_values)
         }); }
     | name=cident K_CONTAINS { rt = oper_t::CONTAINS; } (K_KEY { rt = oper_t::CONTAINS_KEY; })?
@@ -1871,7 +1865,7 @@ relation returns [uexpression e]
                     ids,
                     oper_t::IN,
                     collection_constructor {
-                      .style = collection_constructor::style_type::list,
+                      .style = collection_constructor::style_type::list_or_vector,
                       .elements = std::vector<expression>()
                     }
                   );
@@ -1890,7 +1884,7 @@ relation returns [uexpression e]
                     ids,
                     oper_t::IN,
                     collection_constructor {
-                      .style = collection_constructor::style_type::list,
+                      .style = collection_constructor::style_type::list_or_vector,
                       .elements = std::move(literals)
                     }
                   );
@@ -1901,7 +1895,7 @@ relation returns [uexpression e]
                     ids,
                     oper_t::IN,
                     collection_constructor {
-                      .style = collection_constructor::style_type::list,
+                      .style = collection_constructor::style_type::list_or_vector,
                       .elements = std::move(markers)
                     }
                   );
@@ -1957,6 +1951,7 @@ comparator_type [bool internal] returns [shared_ptr<cql3_type::raw> t]
     : n=native_or_internal_type[internal]     { $t = cql3_type::raw::from(n); }
     | c=collection_type[internal]   { $t = c; }
     | tt=tuple_type[internal]       { $t = tt; }
+    | vt=vector_type                { $t = vt; }
     | id=userTypeName   { $t = cql3::cql3_type::raw::user_type(std::move(id)); }
     | K_FROZEN '<' f=comparator_type[internal] '>'
       {
@@ -2041,6 +2036,15 @@ tuple_type [bool internal] returns [shared_ptr<cql3::cql3_type::raw> t]
       '>' { $t = cql3::cql3_type::raw::tuple(std::move(types)); }
     ;
 
+vector_type returns [shared_ptr<cql3::cql3_type::raw> pt]
+    : K_VECTOR '<' t=comparator_type[false] ',' d=INTEGER '>'
+        {
+            if ($d.text[0] == '-')
+                throw exceptions::invalid_request_exception("Vectors must have a dimension greater than 0");
+            $pt = cql3::cql3_type::raw::vector(t, std::stoul($d.text));
+        }
+    ;
+
 username returns [sstring str]
     : t=IDENT { $str = $t.text; }
     | t=STRING_LITERAL { $str = $t.text; }
@@ -2106,6 +2110,7 @@ basic_unreserved_keyword returns [sstring str]
         | K_STATIC
         | K_FROZEN
         | K_TUPLE
+        | K_VECTOR
         | K_FUNCTION
         | K_FUNCTIONS
         | K_AGGREGATE
@@ -2298,6 +2303,7 @@ K_LIST:        L I S T;
 K_NAN:         N A N;
 K_INFINITY:    I N F I N I T Y;
 K_TUPLE:       T U P L E;
+K_VECTOR:      V E C T O R;
 
 K_TRIGGER:     T R I G G E R;
 K_STATIC:      S T A T I C;

cql3/cql3_type.cc

@@ -20,6 +20,7 @@
 #include "types/map.hh"
 #include "types/set.hh"
 #include "types/list.hh"
+#include "types/vector.hh"
 #include "concrete_types.hh"
 
 namespace cql3 {
@@ -49,6 +50,7 @@ static cql3_type::kind get_cql3_kind(const abstract_type& t) {
         cql3_type::kind operator()(const varint_type_impl&) { return cql3_type::kind::VARINT; }
         cql3_type::kind operator()(const reversed_type_impl& r) { return get_cql3_kind(*r.underlying_type()); }
         cql3_type::kind operator()(const tuple_type_impl&) { SCYLLA_ASSERT(0 && "no kind for this type"); }
+        cql3_type::kind operator()(const vector_type_impl&) { SCYLLA_ASSERT(0 && "no kind for this type"); }
         cql3_type::kind operator()(const collection_type_impl&) { SCYLLA_ASSERT(0 && "no kind for this type"); }
     };
     return visit(t, visitor{});
@@ -303,6 +305,56 @@ public:
     }
 };
 
+class cql3_type::raw_vector : public raw {
+    shared_ptr<raw> _type;
+    size_t _dimension;
+
+    // This limitation is acquired from the maximum number of dimensions in OpenSearch. 
+    static constexpr size_t MAX_VECTOR_DIMENSION = 16000;
+
+    virtual sstring to_string() const override {
+        return seastar::format("vector<{}, {}>", _type, _dimension);
+    }
+
+public:
+    raw_vector(shared_ptr<raw> type, size_t dimension)
+            : _type(std::move(type)), _dimension(dimension) {
+    }
+
+    virtual bool supports_freezing() const override {
+        return true;
+    }
+
+    virtual bool is_collection() const override {
+        return false;
+    }
+
+    virtual void freeze() override {
+        _frozen = true;
+    }
+
+    virtual cql3_type prepare_internal(const sstring& keyspace, const data_dictionary::user_types_metadata& user_types) override {
+        if (!is_frozen()) {
+            freeze();
+        }
+
+        if (_dimension == 0) {
+            throw exceptions::invalid_request_exception("Vectors must have a dimension greater than 0");
+        }
+
+        if (_dimension > MAX_VECTOR_DIMENSION) {
+            throw exceptions::invalid_request_exception(
+                    format("Vectors must have a dimension less than or equal to {}", MAX_VECTOR_DIMENSION));
+        }
+
+        return vector_type_impl::get_instance(_type->prepare_internal(keyspace, user_types).get_type(), _dimension)->as_cql3_type();
+    }
+
+    bool references_user_type(const sstring& name) const override {
+        return _type->references_user_type(name);
+    }
+};
+
 bool
 cql3_type::raw::is_collection() const {
     return false;
@@ -364,6 +416,11 @@ cql3_type::raw::tuple(std::vector<shared_ptr<raw>> ts) {
     return ::make_shared<raw_tuple>(std::move(ts));
 }
 
+shared_ptr<cql3_type::raw>
+cql3_type::raw::vector(shared_ptr<raw> t, size_t dimension) {
+    return ::make_shared<raw_vector>(std::move(t), dimension);
+}
+
 shared_ptr<cql3_type::raw>
 cql3_type::raw::frozen(shared_ptr<raw> t) {
     t->freeze();