Create urgent_issue_reminder.yml

.gitattributes

@@ -2,4 +2,3 @@
 *.hh diff=cpp
 *.svg binary
 docs/_static/api/js/* binary
-pgo/profiles/** filter=lfs diff=lfs merge=lfs -text

.github/CODEOWNERS

@@ -1,5 +1,5 @@
 # AUTH
-auth/* @nuivall @ptrsmrn
+auth/* @nuivall @ptrsmrn @KrzaQ
 
 # CACHE
 row_cache* @tgrabiec
@@ -25,15 +25,15 @@ compaction/* @raphaelsc
 transport/*
 
 # CQL QUERY LANGUAGE
-cql3/* @tgrabiec @nuivall @ptrsmrn
+cql3/* @tgrabiec @nuivall @ptrsmrn @KrzaQ
 
 # COUNTERS
-counters* @nuivall @ptrsmrn
-tests/counter_test* @nuivall @ptrsmrn
+counters* @nuivall @ptrsmrn @KrzaQ
+tests/counter_test* @nuivall @ptrsmrn @KrzaQ
 
 # DOCS
 docs/* @annastuchlik @tzach
-docs/alternator @annastuchlik @tzach @nyh
+docs/alternator @annastuchlik @tzach @nyh @nuivall @ptrsmrn @KrzaQ
 
 # GOSSIP
 gms/* @tgrabiec @asias @kbr-scylla
@@ -74,8 +74,8 @@ streaming/* @tgrabiec @asias
 service/storage_service.* @tgrabiec @asias
 
 # ALTERNATOR
-alternator/* @nyh
-test/alternator/* @nyh
+alternator/* @nyh @nuivall @ptrsmrn @KrzaQ
+test/alternator/* @nyh @nuivall @ptrsmrn @KrzaQ
 
 # HINTED HANDOFF
 db/hints/* @piodul @vladzcloudius @eliransin

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,86 +1,15 @@
-name: "Report a bug"
-description: "File a bug report."
-title: "[Bug]: "
-type: "bug"
-labels: bug
-body:
-  - type: checkboxes
-    id: terms
-    attributes:
-      label: Code of Conduct
-      description: "This is Scylla's bug tracker, to be used for reporting bugs only.
+This is Scylla's bug tracker, to be used for reporting bugs only.
 If you have a question about Scylla, and not a bug, please ask it in
-our forum at https://forum.scylladb.com/ or in our slack channel https://slack.scylladb.com/ "
-      options:
-        - label: I have read the disclaimer above and am reporting a suspected malfunction in Scylla.
-          required: true
+our mailing-list at scylladb-dev@googlegroups.com or in our slack channel.
 
-  - type: input
-    id: product-version
-    attributes:
-      label: product version
-      description: Scylla version (or git commit hash)
-      placeholder: ex. scylla-6.1.1
-    validations:
-      required: true
-      
-  - type: input
-    id: cluster-size
-    attributes:
-      label: Cluster Size
-    validations:
-      required: true  
-      
-  - type: input
-    id: os
-    attributes:
-      label: OS
-      placeholder: RHEL/CentOS/Ubuntu/AWS AMI
-    validations:
-      required: true
-      
-  - type: textarea
-    id: additional-data
-    attributes:
-      label: Additional Environmental Data
-      #description: 
-      placeholder: Add additional data
-      value: "Platform (physical/VM/cloud instance type/docker):\n
-Hardware: sockets=   cores=   hyperthreading=   memory=\n
-Disks: (SSD/HDD, count)"
-    validations:
-      required: false
-      
-  - type: textarea
-    id: reproducer-steps
-    attributes:
-      label: Reproduction Steps
-      placeholder: Describe how to reproduce the problem
-      value: "The steps to reproduce the problem are:"
-    validations:
-      required: true
-      
-  - type: textarea
-    id: the-problem
-    attributes:
-      label: What is the problem?
-      placeholder: Describe the problem you found
-      value: "The problem is that"
-    validations:
-      required: true
-      
-  - type: textarea
-    id: what-happened
-    attributes:
-      label: Expected behavior?
-      placeholder: Describe what should have happened
-      value: "I expected that "
-    validations:
-      required: true
-      
-  - type: textarea
-    id: logs
-    attributes:
-      label: Relevant log output
-      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
-      render: shell
+- [] I have read the disclaimer above, and I am reporting a suspected malfunction in Scylla.
+
+*Installation details*
+Scylla version (or git commit hash):
+Cluster size:
+OS (RHEL/CentOS/Ubuntu/AWS AMI):
+
+*Hardware details (for performance issues)*          Delete if unneeded
+Platform (physical/VM/cloud instance type/docker):
+Hardware: sockets= cores= hyperthreading= memory=
+Disks: (SSD/HDD, count)

.github/scripts/auto-backport.py

@@ -29,11 +29,10 @@ def parse_args():
     parser.add_argument('--commits', default=None, type=str, help='Range of promoted commits.')
     parser.add_argument('--pull-request', type=int, help='Pull request number to be backported')
     parser.add_argument('--head-commit', type=str, required=is_pull_request(), help='The HEAD of target branch after the pull request specified by --pull-request is merged')
-    parser.add_argument('--github-event', type=str, help='Get GitHub event type')
     return parser.parse_args()
 
 
-def create_pull_request(repo, new_branch_name, base_branch_name, pr, backport_pr_title, commits, is_draft, is_collaborator):
+def create_pull_request(repo, new_branch_name, base_branch_name, pr, backport_pr_title, commits, is_draft=False):
     pr_body = f'{pr.body}\n\n'
     for commit in commits:
         pr_body += f'- (cherry picked from commit {commit})\n\n'
@@ -47,12 +46,11 @@ def create_pull_request(repo, new_branch_name, base_branch_name, pr, backport_pr
             draft=is_draft
         )
         logging.info(f"Pull request created: {backport_pr.html_url}")
-        if is_collaborator:
-            backport_pr.add_to_assignees(pr.user)
+        backport_pr.add_to_assignees(pr.user)
         if is_draft:
             backport_pr.add_to_labels("conflicts")
-            pr_comment = f"@{pr.user.login} - This PR was marked as draft because it has conflicts\n"
-            pr_comment += "Please resolve them and remove the 'conflicts' label. The PR will be made ready for review automatically."
+            pr_comment = f"@{pr.user} - This PR was marked as draft because it has conflicts\n"
+            pr_comment += "Please resolve them and mark this PR as ready for review"
             backport_pr.create_issue_comment(pr_comment)
         logging.info(f"Assigned PR to original author: {pr.user}")
         return backport_pr
@@ -68,8 +66,7 @@ def get_pr_commits(repo, pr, stable_branch, start_commit=None):
     if pr.merged:
         merge_commit = repo.get_commit(pr.merge_commit_sha)
         if len(merge_commit.parents) > 1:  # Check if this merge commit includes multiple commits
-            for commit in pr.get_commits():
-                commits.append(commit.sha)
+            commits.append(pr.merge_commit_sha)
         else:
             if start_commit:
                 promoted_commits = repo.compare(start_commit, stable_branch).commits
@@ -94,7 +91,18 @@ def get_pr_commits(repo, pr, stable_branch, start_commit=None):
     return commits
 
 
-def backport(repo, pr, version, commits, backport_base_branch, is_collaborator):
+def create_pr_comment_and_remove_label(pr, comment_body):
+    labels = pr.get_labels()
+    pattern = re.compile(r"backport/\d+\.\d+$")
+    for label in labels:
+        if pattern.match(label.name):
+            print(f"Removing label: {label.name}")
+            comment_body += f'- {label.name}\n'
+            pr.remove_from_labels(label)
+    pr.create_issue_comment(comment_body)
+
+
+def backport(repo, pr, version, commits, backport_base_branch):
     new_branch_name = f'backport/{pr.number}/to-{version}'
     backport_pr_title = f'[Backport {version}] {pr.title}'
     repo_url = f'https://scylladbbot:{github_token}@github.com/{repo.full_name}.git'
@@ -106,51 +114,33 @@ def backport(repo, pr, version, commits, backport_base_branch, is_collaborator):
             is_draft = False
             for commit in commits:
                 try:
-                    repo_local.git.cherry_pick(commit, '-x')
+                    repo_local.git.cherry_pick(commit, '-m1', '-x')
                 except GitCommandError as e:
                     logging.warning(f'Cherry-pick conflict on commit {commit}: {e}')
                     is_draft = True
                     repo_local.git.add(A=True)
                     repo_local.git.cherry_pick('--continue')
-            # Check if the branch already exists in the remote fork
-            remote_refs = repo_local.git.ls_remote('--heads', fork_repo, new_branch_name)
-            if not remote_refs:
-                # Branch does not exist, create it with a regular push
-                repo_local.git.push(fork_repo, new_branch_name)
-                create_pull_request(repo, new_branch_name, backport_base_branch, pr, backport_pr_title, commits,
-                                    is_draft, is_collaborator)
-            else:
-                logging.info(f"Remote branch {new_branch_name} already exists in fork. Skipping push.")
+            repo_local.git.push(fork_repo, new_branch_name, force=True)
+            create_pull_request(repo, new_branch_name, backport_base_branch, pr, backport_pr_title, commits,
+                                is_draft=is_draft)
+
         except GitCommandError as e:
             logging.warning(f"GitCommandError: {e}")
 
 
 def with_github_keyword_prefix(repo, pr):
-    # GitHub issue pattern: #123, scylladb/scylladb#123, or full GitHub URLs
-    github_pattern = rf"(?:fix(?:|es|ed))\s*:?\s*(?:(?:(?:{repo.full_name})?#)|https://github\.com/{repo.full_name}/issues/)(\d+)"
-    
-    # JIRA issue pattern: PKG-92 or https://scylladb.atlassian.net/browse/PKG-92
-    jira_pattern = r"(?:fix(?:|es|ed))\s*:?\s*(?:(?:https://scylladb\.atlassian\.net/browse/)?([A-Z]+-\d+))"
-    
-    # Check PR body for GitHub issues
-    github_match = re.findall(github_pattern, pr.body, re.IGNORECASE)
-    # Check PR body for JIRA issues
-    jira_match = re.findall(jira_pattern, pr.body, re.IGNORECASE)
-    
-    match = github_match or jira_match
-
-    if match:
+    pattern = rf"(?:fix(?:|es|ed))\s*:?\s*(?:(?:(?:{repo.full_name})?#)|https://github\.com/{repo.full_name}/issues/)(\d+)"
+    match = re.findall(pattern, pr.body, re.IGNORECASE)
+    if not match:
+        print(f'No valid close reference for {pr.number}')
+        comment = f':warning:  @{pr.user.login} PR body does not contain a Fixes reference to an issue '
+        comment += ' and can not be backported\n\n'
+        comment += 'The following labels were removed:\n'
+        create_pr_comment_and_remove_label(pr, comment)
+        return False
+    else:
         return True
 
-    for commit in pr.get_commits():
-        github_match = re.findall(github_pattern, commit.commit.message, re.IGNORECASE)
-        jira_match = re.findall(jira_pattern, commit.commit.message, re.IGNORECASE)
-        if github_match or jira_match:
-            print(f'{pr.number} has a valid close reference in commit message {commit.sha}')
-            return True
-
-    print(f'No valid close reference for {pr.number}')
-    return False
 
 def main():
     args = parse_args()
@@ -171,7 +161,6 @@ def main():
     scylladbbot_repo = g.get_repo(fork_repo_name)
     closed_prs = []
     start_commit = None
-    is_collaborator = True
 
     if args.commits:
         start_commit, end_commit = args.commits.split('..')
@@ -196,33 +185,21 @@ def main():
         if not backport_labels:
             print(f'no backport label: {pr.number}')
             continue
-        if not with_github_keyword_prefix(repo, pr) and args.github_event != 'unlabeled':
-            comment = f''':warning:  @{pr.user.login} PR body or PR commits do not contain a Fixes reference to an issue and can not be backported
-            please update PR body with a valid ref to an issue. Then remove `scylladbbot/backport_error` label to re-trigger the backport process
-            '''
-            pr.create_issue_comment(comment)
-            pr.add_to_labels("scylladbbot/backport_error")
+        if args.commits and not with_github_keyword_prefix(repo, pr):
             continue
         if not repo.private and not scylladbbot_repo.has_in_collaborators(pr.user.login):
             logging.info(f"Sending an invite to {pr.user.login} to become a collaborator to {scylladbbot_repo.full_name} ")
             scylladbbot_repo.add_to_collaborators(pr.user.login)
-            comment = f''':warning:  @{pr.user.login} you have been added as collaborator to scylladbbot fork
-            Please check your inbox and approve the invitation, otherwise you will not be able to edit PR branch when needed
-            '''
-            # When a pull request is pending for backport but its author is not yet a collaborator of "scylladbbot",
-            # we attach a "scylladbbot/backport_error" label to the PR.
-            # This prevents the workflow from proceeding with the backport process
-            # until the author has been granted proper permissions
-            # the author should remove the label manually to re-trigger the backport workflow.
-            pr.add_to_labels("scylladbbot/backport_error")
-            pr.create_issue_comment(comment)
-            is_collaborator = False
+            comment = f':warning:  @{pr.user.login} you have been added as collaborator to scylladbbot fork '
+            comment += f'Please check your inbox and approve the invitation, once it is done, please add the backport labels again\n'
+            create_pr_comment_and_remove_label(pr, comment)
+            continue
         commits = get_pr_commits(repo, pr, stable_branch, start_commit)
         logging.info(f"Found PR #{pr.number} with commit {commits} and the following labels: {backport_labels}")
         for backport_label in backport_labels:
             version = backport_label.replace('backport/', '')
             backport_base_branch = backport_label.replace('backport/', backport_branch)
-            backport(repo, pr, version, commits, backport_base_branch, is_collaborator)
+            backport(repo, pr, version, commits, backport_base_branch)
 
 
 if __name__ == "__main__":

.github/scripts/check-license.py

@@ -1,81 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2024-present ScyllaDB
-#
-#
-# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
-#
-
-import argparse
-import sys
-from pathlib import Path
-from typing import Set
-
-
-def parse_args() -> argparse.Namespace:
-    """Parses command-line arguments."""
-    parser = argparse.ArgumentParser(description='Check license headers in files')
-    parser.add_argument('--files', required=True, nargs="+", type=Path,
-                        help='List of files to check')
-    parser.add_argument('--license', required=True,
-                        help='License to check for')
-    parser.add_argument('--check-lines', type=int, default=10,
-                        help='Number of lines to check (default: %(default)s)')
-    parser.add_argument('--extensions', required=True, nargs="+",
-                        help='List of file extensions to check')
-    parser.add_argument('--verbose', action='store_true',
-                        help='Print verbose output (default: %(default)s)')
-    return parser.parse_args()
-
-
-def should_check_file(file_path: Path, allowed_extensions: Set[str]) -> bool:
-    return file_path.suffix in allowed_extensions
-
-
-def check_license_header(file_path: Path, license_header: str, check_lines: int) -> bool:
-    try:
-        with open(file_path, 'r', encoding='utf-8') as f:
-            for _ in range(check_lines):
-                line = f.readline()
-                if license_header in line:
-                    return True
-        return False
-    except (UnicodeDecodeError, StopIteration):
-        # Handle files that can't be read as text or have fewer lines
-        return False
-
-
-def main() -> int:
-    args = parse_args()
-
-    if not args.files:
-        print("No files to check")
-        return 0
-
-    num_errors = 0
-
-    for file_path in args.files:
-        # Skip non-existent files
-        if not file_path.exists():
-            continue
-
-        # Skip files with non-matching extensions
-        if not should_check_file(file_path, args.extensions):
-            print(f"ℹ️ Skipping file with unchecked extension: {file_path}")
-            continue
-
-        # Check license header
-        if check_license_header(file_path, args.license, args.check_lines):
-            if args.verbose:
-                print(f"✅ License header found in: {file_path}")
-        else:
-            print(f"❌ Missing license header in: {file_path}")
-            num_errors += 1
-
-    if num_errors > 0:
-        sys.exit(1)
-
-
-if __name__ == '__main__':
-    main()

.github/seastar-bad-include.json

@@ -1,16 +0,0 @@
-{
-    "problemMatcher": [
-        {
-            "owner": "seastar-bad-include",
-            "severity": "error",
-            "pattern": [
-                {
-                    "regexp": "^(.+):(\\d+):(.+)$",
-                    "file": 1,
-                    "line": 2,
-                    "message": 3
-                }
-            ]
-        }
-    ]
-}

.github/workflows/add-label-when-promoted.yaml

@@ -7,7 +7,7 @@ on:
       - branch-*.*
       - enterprise
   pull_request_target:
-    types: [labeled, unlabeled]
+    types: [labeled]
     branches: [master, next, enterprise]
 
 jobs:
@@ -53,28 +53,19 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }}
         run: python .github/scripts/auto-backport.py --repo ${{ github.repository }} --base-branch ${{ github.ref }} --commits ${{ github.event.before }}..${{ github.sha }}
-      - name: Check if a valid backport label exists and no backport_error
+      - name: Check if label starts with 'backport/' and contains digits
         id: check_label
         run: |
-          labels_json='${{ toJson(github.event.pull_request.labels) }}'
-          echo "Checking labels: $(echo "$labels_json" | jq -r '.[].name')"
-
-          # Check if a valid backport label exists
-          if echo "$labels_json" | jq -e 'any(.[] | .name; test("backport/[0-9]+\\.[0-9]+$"))' > /dev/null; then
-            # Ensure scylladbbot/backport_error is NOT present
-            if ! echo "$labels_json" | jq -e '.[] | select(.name == "scylladbbot/backport_error")' > /dev/null; then
-              echo "A matching backport label was found and no backport_error label exists."
-              echo "ready_for_backport=true" >> "$GITHUB_OUTPUT"
-              exit 0
-            else
-              echo "The label 'scylladbbot/backport_error' is present, invalidating backport."
-            fi
+          label_name="${{ github.event.label.name }}"
+          if [[ "$label_name" =~ ^backport/[0-9]+\.[0-9]+$ ]]; then
+            echo "Label matches backport/X.X pattern."
+            echo "backport_label=true" >> $GITHUB_OUTPUT
           else
-            echo "No matching backport label found."
+            echo "Label does not match the required pattern."
+            echo "backport_label=false" >> $GITHUB_OUTPUT
           fi
-          echo "ready_for_backport=false" >> "$GITHUB_OUTPUT"
-      - name: Run auto-backport.py when PR is closed
-        if: ${{ github.event_name == 'pull_request_target' && steps.check_label.outputs.ready_for_backport == 'true' && github.event.pull_request.state == 'closed' }}
+      - name: Run auto-backport.py when label was added
+        if: ${{ github.event_name == 'pull_request_target' && steps.check_label.outputs.backport_label == 'true' && github.event.pull_request.state == 'closed' }}
         env:
           GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }}
-        run: python .github/scripts/auto-backport.py --repo ${{ github.repository }} --base-branch ${{ github.ref }} --pull-request ${{ github.event.pull_request.number }} --head-commit ${{ github.event.pull_request.base.sha }} --github-event ${{ github.event.action }}
+        run: python .github/scripts/auto-backport.py --repo ${{ github.repository }} --base-branch ${{ github.ref }} --pull-request ${{ github.event.pull_request.number }} --head-commit ${{ github.event.pull_request.base.sha }}

.github/workflows/backport-pr-fixes-validation.yaml

@@ -18,7 +18,7 @@ jobs:
             
             // Regular expression pattern to check for "Fixes" prefix
             // Adjusted to dynamically insert the repository full name
-            const pattern = `Fixes:? ((?:#|${repo.replace('/', '\\/')}#|https://github\\.com/${repo.replace('/', '\\/')}/issues/)(\\d+)|([A-Z]+-\\d+))`;
+            const pattern = `Fixes:? (?:#|${repo.replace('/', '\\/')}#|https://github\\.com/${repo.replace('/', '\\/')}/issues/)(\\d+)`;
             const regex = new RegExp(pattern);
             
             if (!regex.test(body)) {

.github/workflows/check-license-header.yaml

@@ -1,52 +0,0 @@
-name: License Header Check
-
-on:
-  pull_request:
-    types: [opened, synchronize, reopened]
-    branches: [master]
-
-env:
-  HEADER_CHECK_LINES: 10
-  LICENSE: "LicenseRef-ScyllaDB-Source-Available-1.0"
-  CHECKED_EXTENSIONS: ".cc .hh .py"
-
-jobs:
-  check-license-headers:
-    name: Check License Headers
-    runs-on: ubuntu-latest
-    permissions:
-      pull-requests: write
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Get changed files
-        id: changed-files
-        run: |
-          # Get list of added files comparing with base branch
-          echo "files=$(git diff --name-only --diff-filter=A ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | tr '\n' ' ')" >> $GITHUB_OUTPUT
-
-      - name: Check license headers
-        if: steps.changed-files.outputs.files != ''
-        run: |
-          .github/scripts/check-license.py \
-            --files ${{ steps.changed-files.outputs.files }} \
-            --license "${{ env.LICENSE }}" \
-            --check-lines "${{ env.HEADER_CHECK_LINES }}" \
-            --extensions ${{ env.CHECKED_EXTENSIONS }}
-
-      - name: Comment on PR if check fails
-        if: failure()
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const license = '${{ env.LICENSE }}';
-            await github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: `❌ License header check failed. Please ensure all new files include the header within the first ${{ env.HEADER_CHECK_LINES }} lines:\n\`\`\`\n${license}\n\`\`\`\nSee action logs for details.`
-            });

.github/workflows/clang-nightly.yaml

@@ -7,7 +7,7 @@ on:
 
 env:
   # use the development branch explicitly
-  CLANG_VERSION: 21
+  CLANG_VERSION: 20
   BUILD_DIR: build
 
 permissions: {}

.github/workflows/clang-tidy.yaml

@@ -34,7 +34,6 @@ jobs:
     name: Run clang-tidy
     needs:
       - read-toolchain
-    if: "${{ needs.read-toolchain.result == 'success' }}"
     runs-on: ubuntu-latest
     container: ${{ needs.read-toolchain.outputs.image }}
     steps:

.github/workflows/iwyu.yaml

@@ -11,8 +11,7 @@ env:
   CLEANER_OUTPUT_PATH: build/clang-include-cleaner.log
   # the "idl" subdirectory does not contain C++ source code. the .hh files in it are
   # supposed to be processed by idl-compiler.py, so we don't check them using the cleaner
-  CLEANER_DIRS: test/unit exceptions alternator api auth cdc compaction db dht gms index lang message mutation mutation_writer node_ops raft redis replica service
-  SEASTAR_BAD_INCLUDE_OUTPUT_PATH: build/seastar-bad-include.log
+  CLEANER_DIRS: test/unit exceptions alternator api auth cdc compaction db dht gms index lang message mutation
 
 permissions: {}
 
@@ -81,24 +80,7 @@ jobs:
           done
       - run: |
           echo "::remove-matcher owner=clang-include-cleaner::"
-      - run: |
-          echo "::add-matcher::.github/seastar-bad-include.json"
-      - name: check for seastar includes
-        run: |
-          git -c safe.directory="$PWD"    \
-            grep -nE '#include +"seastar/' \
-            | tee "$SEASTAR_BAD_INCLUDE_OUTPUT_PATH"
-      - run: |
-          echo "::remove-matcher owner=seastar-bad-include::"
       - uses: actions/upload-artifact@v4
         with:
-          name: Logs
-          path: |
-            ${{ env.CLEANER_OUTPUT_PATH }}
-            ${{ env.SEASTAR_BAD_INCLUDE_OUTPUT_PATH }}
-      - name: fail if seastar headers are included as an internal library
-        run: |
-          if [ -s "$SEASTAR_BAD_INCLUDE_OUTPUT_PATH" ]; then
-            echo "::error::Found #include \"seastar/ in the source code. Use angle brackets instead."
-            exit 1
-          fi
+          name: Logs (clang-include-cleaner)
+          path: "./${{ env.CLEANER_OUTPUT_PATH }}"

.github/workflows/make-pr-ready-for-review.yaml

@@ -1,29 +0,0 @@
-name: Mark PR as Ready When Conflicts Label is Removed
-
-on:
-  pull_request_target:
-    types:
-      - unlabeled
-
-env:
-  DEFAULT_BRANCH: 'master'
-
-jobs:
-  mark-ready:
-    if: github.event.label.name == 'conflicts'
-    runs-on: ubuntu-latest
-    permissions:
-      pull-requests: write
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ github.repository }}
-          ref: ${{ env.DEFAULT_BRANCH }}
-          token: ${{ secrets.AUTO_BACKPORT_TOKEN }}
-          fetch-depth: 1
-      - name: Mark pull request as ready for review
-        run:  gh pr ready "${{ github.event.pull_request.number }}"
-        env:
-          GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }}

.github/workflows/pr-require-backport-label.yaml

@@ -13,12 +13,10 @@ jobs:
       issues: write
       pull-requests: write
     steps:
-      - name: Wait for label to be added
-        run: sleep 1m
       - uses: mheap/github-action-required-labels@v5
         with:
           mode: minimum
           count: 1
-          labels: "backport/none\nbackport/\\d{4}\\.\\d+\nbackport/\\d+\\.\\d+"
+          labels: "backport/none\nbackport/\\d.\\d"
           use_regex: true
           add_comment: false

.github/workflows/seastar.yaml

@@ -15,13 +15,10 @@ env:
   BUILD_DIR: build
 
 jobs:
-  read-toolchain:
-    uses: ./.github/workflows/read-toolchain.yaml
   build-with-the-latest-seastar:
-    needs:
-      - read-toolchain
     runs-on: ubuntu-latest
-    container: ${{ needs.read-toolchain.outputs.image }}
+    # be consistent with tools/toolchain/image
+    container: scylladb/scylla-toolchain:fedora-40-20240621
     strategy:
       matrix:
         build_type:

.github/workflows/trigger_jenkins.yaml

@@ -1,50 +0,0 @@
-name: Trigger next gating
-
-on:
-  push:
-    branches:
-      - next**
-
-jobs:
-  trigger-jenkins:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Determine Jenkins Job Name
-        run: |
-          if [[ "${{ github.ref_name }}" == "next" ]]; then
-            FOLDER_NAME="scylla-master"
-          elif [[ "${{ github.ref_name }}" == "next-enterprise" ]]; then
-            FOLDER_NAME="scylla-enterprise"
-          else
-            VERSION=$(echo "${{ github.ref_name }}" | awk -F'-' '{print $2}')
-            if [[ "$VERSION" =~ ^202[0-4]\.[0-9]+$ ]]; then
-              FOLDER_NAME="enterprise-$VERSION"
-            elif [[ "$VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then
-              FOLDER_NAME="scylla-$VERSION"
-            fi
-          fi
-          echo "JOB_NAME=${FOLDER_NAME}/job/next" >> $GITHUB_ENV
-
-      - name: Trigger Jenkins Job
-        env:
-          JENKINS_USER: ${{ secrets.JENKINS_USERNAME }}
-          JENKINS_API_TOKEN: ${{ secrets.JENKINS_TOKEN }}
-          JENKINS_URL: "https://jenkins.scylladb.com"
-          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
-        run: |
-          echo "Triggering Jenkins Job: $JOB_NAME"
-          if ! curl -X POST "$JENKINS_URL/job/$JOB_NAME/buildWithParameters" --fail --user "$JENKINS_USER:$JENKINS_API_TOKEN" -i -v; then
-            echo "Error: Jenkins job trigger failed"
-
-            # Send Slack message
-            curl -X POST -H 'Content-type: application/json' \
-              -H "Authorization: Bearer $SLACK_BOT_TOKEN" \
-              --data '{
-                "channel": "#releng-team",
-                "text": "🚨 @here '$JOB_NAME' failed to be triggered, please check https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} for more details",
-                "icon_emoji": ":warning:"
-              }' \
-              https://slack.com/api/chat.postMessage
-
-            exit 1
-          fi

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "seastar"]
 	path = seastar
-	url = ../scylla-seastar
+	url = ../seastar
 	ignore = dirty
 [submodule "swagger-ui"]
 	path = swagger-ui
@@ -9,6 +9,9 @@
 [submodule "abseil"]
 	path = abseil
 	url = ../abseil-cpp
+[submodule "scylla-tools"]
+	path = tools/java
+	url = ../scylla-tools-java
 [submodule "scylla-python3"]
 	path = tools/python3
 	url = ../scylla-python3

CMakeLists.txt

@@ -22,8 +22,6 @@ if(DEFINED CMAKE_BUILD_TYPE)
     endif()
 endif(DEFINED CMAKE_BUILD_TYPE)
 
-option(Scylla_ENABLE_LTO "Turn on link-time optimization for the 'release' mode." ON)
-
 include(mode.common)
 get_property(is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
 if(is_multi_config)
@@ -44,7 +42,6 @@ else()
 endif()
 
 include(limit_jobs)
-
 # Configure Seastar compile options to align with Scylla
 set(CMAKE_CXX_STANDARD "23" CACHE INTERNAL "")
 set(CMAKE_CXX_EXTENSIONS ON CACHE INTERNAL "")
@@ -66,25 +63,24 @@ if(is_multi_config)
     #   establishing proper dependencies between them
     include(ExternalProject)
 
-    # should be consistent with configure_seastar() in configure.py
-    set(seastar_build_dir "${CMAKE_BINARY_DIR}/$<CONFIG>/seastar")
     ExternalProject_Add(Seastar
         SOURCE_DIR "${PROJECT_SOURCE_DIR}/seastar"
+        BINARY_DIR "${CMAKE_BINARY_DIR}/$<CONFIG>/seastar"
         CONFIGURE_COMMAND ""
-        BUILD_COMMAND ${CMAKE_COMMAND} --build "${seastar_build_dir}"
+        BUILD_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR>
           --target seastar
           --target seastar_testing
           --target seastar_perf_testing
           --target app_iotune
         BUILD_ALWAYS ON
         BUILD_BYPRODUCTS
-          ${seastar_build_dir}/libseastar.$<IF:$<CONFIG:Debug,Dev>,so,a>
-          ${seastar_build_dir}/libseastar_testing.$<IF:$<CONFIG:Debug,Dev>,so,a>
-          ${seastar_build_dir}/libseastar_perf_testing.$<IF:$<CONFIG:Debug,Dev>,so,a>
-          ${seastar_build_dir}/apps/iotune/iotune
-          ${seastar_build_dir}/gen/include/seastar/http/chunk_parsers.hh
-          ${seastar_build_dir}/gen/include/seastar/http/request_parser.hh
-          ${seastar_build_dir}/gen/include/seastar/http/response_parser.hh
+          <BINARY_DIR>/libseastar.$<IF:$<CONFIG:Debug,Dev>,so,a>
+          <BINARY_DIR>/libseastar_testing.$<IF:$<CONFIG:Debug,Dev>,so,a>
+          <BINARY_DIR>/libseastar_perf_testing.$<IF:$<CONFIG:Debug,Dev>,so,a>
+          <BINARY_DIR>/apps/iotune/iotune
+          <BINARY_DIR>/gen/include/seastar/http/chunk_parsers.hh
+          <BINARY_DIR>/gen/include/seastar/http/request_parser.hh
+          <BINARY_DIR>/gen/include/seastar/http/response_parser.hh
         INSTALL_COMMAND "")
     add_dependencies(Seastar::seastar Seastar)
     add_dependencies(Seastar::seastar_testing Seastar)
@@ -96,7 +92,7 @@ else()
     set(Seastar_EXCLUDE_APPS_FROM_ALL ON CACHE BOOL "" FORCE)
     set(Seastar_EXCLUDE_TESTS_FROM_ALL ON CACHE BOOL "" FORCE)
     set(Seastar_IO_URING ON CACHE BOOL "" FORCE)
-    set(Seastar_SCHEDULING_GROUPS_COUNT 19 CACHE STRING "" FORCE)
+    set(Seastar_SCHEDULING_GROUPS_COUNT 16 CACHE STRING "" FORCE)
     set(Seastar_UNUSED_RESULT_ERROR ON CACHE BOOL "" FORCE)
     add_subdirectory(seastar)
     target_compile_definitions (seastar
@@ -106,17 +102,13 @@ endif()
 
 set(ABSL_PROPAGATE_CXX_STD ON CACHE BOOL "" FORCE)
 
-if(Scylla_ENABLE_LTO)
-    list(APPEND absl_cxx_flags $<$<CONFIG:RelWithDebInfo>:${CMAKE_CXX_COMPILE_OPTIONS_IPO};-ffat-lto-objects>)
-endif()
-
 find_package(Sanitizers QUIET)
-list(APPEND absl_cxx_flags
+set(sanitizer_cxx_flags
     $<$<CONFIG:Debug,Sanitize>:$<TARGET_PROPERTY:Sanitizers::address,INTERFACE_COMPILE_OPTIONS>;$<TARGET_PROPERTY:Sanitizers::undefined_behavior,INTERFACE_COMPILE_OPTIONS>>)
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-    list(APPEND ABSL_GCC_FLAGS ${absl_cxx_flags})
+    set(ABSL_GCC_FLAGS ${sanitizer_cxx_flags})
 elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-    list(APPEND ABSL_LLVM_FLAGS ${absl_cxx_flags})
+    set(ABSL_LLVM_FLAGS ${sanitizer_cxx_flags})
 endif()
 set(ABSL_DEFAULT_LINKOPTS
     $<$<CONFIG:Debug,Sanitize>:$<TARGET_PROPERTY:Sanitizers::address,INTERFACE_LINK_LIBRARIES>;$<TARGET_PROPERTY:Sanitizers::undefined_behavior,INTERFACE_LINK_LIBRARIES>>)
@@ -149,13 +141,11 @@ find_package(ICU COMPONENTS uc i18n REQUIRED)
 find_package(fmt 10.0.0 REQUIRED)
 find_package(libdeflate REQUIRED)
 find_package(libxcrypt REQUIRED)
-find_package(p11-kit REQUIRED)
 find_package(Snappy REQUIRED)
 find_package(RapidJSON REQUIRED)
 find_package(xxHash REQUIRED)
 find_package(yaml-cpp REQUIRED)
 find_package(zstd REQUIRED)
-find_package(lz4 REQUIRED)
 
 set(scylla_gen_build_dir "${CMAKE_BINARY_DIR}/gen")
 file(MAKE_DIRECTORY "${scylla_gen_build_dir}")
@@ -163,6 +153,14 @@ file(MAKE_DIRECTORY "${scylla_gen_build_dir}")
 include(add_version_library)
 generate_scylla_version()
 
+add_library(scylla-zstd STATIC
+    zstd.cc)
+target_link_libraries(scylla-zstd
+  PRIVATE
+    db
+    Seastar::seastar
+    zstd::libzstd)
+
 add_library(scylla-main STATIC)
 target_sources(scylla-main
   PRIVATE
@@ -174,7 +172,7 @@ target_sources(scylla-main
     compress.cc
     converting_mutation_partition_applier.cc
     counters.cc
-    sstable_dict_autotrainer.cc
+    direct_failure_detector/failure_detector.cc
     duration.cc
     exceptions/exceptions.cc
     frozen_schema.cc
@@ -193,10 +191,9 @@ target_sources(scylla-main
     tombstone_gc_options.cc
     tombstone_gc.cc
     reader_concurrency_semaphore.cc
-    reader_concurrency_semaphore_group.cc
+    row_cache.cc
     schema_mutations.cc
     serializer.cc
-    service/direct_failure_detector/failure_detector.cc
     sstables_loader.cc
     table_helper.cc
     tasks/task_handler.cc
@@ -207,6 +204,7 @@ target_sources(scylla-main
     vint-serialization.cc)
 target_link_libraries(scylla-main
   PRIVATE
+    "$<LINK_LIBRARY:WHOLE_ARCHIVE,scylla-zstd>"
     db
     absl::headers
     absl::btree
@@ -215,10 +213,7 @@ target_link_libraries(scylla-main
     Seastar::seastar
     Snappy::snappy
     systemd
-    ZLIB::ZLIB
-    lz4::lz4_static
-    zstd::zstd_static
-)
+    ZLIB::ZLIB)
 
 option(Scylla_CHECK_HEADERS
   "Add check-headers target for checking the self-containness of headers")
@@ -253,7 +248,6 @@ add_custom_target(compiler-training)
 
 add_subdirectory(api)
 add_subdirectory(alternator)
-add_subdirectory(audit)
 add_subdirectory(db)
 add_subdirectory(auth)
 add_subdirectory(cdc)
@@ -261,7 +255,6 @@ add_subdirectory(compaction)
 add_subdirectory(cql3)
 add_subdirectory(data_dictionary)
 add_subdirectory(dht)
-add_subdirectory(ent)
 add_subdirectory(gms)
 add_subdirectory(idl)
 add_subdirectory(index)
@@ -292,8 +285,7 @@ add_version_library(scylla_version
 
 add_executable(scylla
   main.cc)
-set(scylla_libs
-    audit
+target_link_libraries(scylla PRIVATE
     scylla-main
     api
     auth
@@ -304,12 +296,10 @@ set(scylla_libs
     cql3
     data_dictionary
     dht
-    encryption
     gms
     idl
     index
     lang
-    ldap
     locator
     message
     mutation
@@ -330,18 +320,8 @@ set(scylla_libs
     transport
     types
     utils)
-target_link_libraries(scylla PRIVATE
-    ${scylla_libs})
-
-if(Scylla_ENABLE_LTO)
-  include(enable_lto)
-  foreach(target scylla ${scylla_libs})
-    enable_lto(${target})
-  endforeach()
-endif()
 
 target_link_libraries(scylla PRIVATE
-    p11-kit::p11-kit
     Seastar::seastar
     absl::headers
     yaml-cpp::yaml-cpp
@@ -359,7 +339,3 @@ add_dependencies(compiler-training
 if(Scylla_DIST)
   add_subdirectory(dist)
 endif()
-
-if(Scylla_BUILD_INSTRUMENTED)
-  add_subdirectory(pgo)
-endif()

CONTRIBUTING.md

@@ -12,7 +12,7 @@ Please use the [issue tracker](https://github.com/scylladb/scylla/issues/) to re
 
 ## Contributing code to Scylla
 
-Before you can contribute code to Scylla for the first time, you should sign the [Contributor License Agreement](https://www.scylladb.com/open-source/contributor-agreement/) and send the signed form cla@scylladb.com. You can then submit your changes as patches to the [scylladb-dev mailing list](https://groups.google.com/forum/#!forum/scylladb-dev) or as a pull request to the [Scylla project on github](https://github.com/scylladb/scylla).
+Before you can contribute code to Scylla for the first time, you should sign the [Contributor License Agreement](https://www.scylladb.com/open-source/contributor-agreement/) and send the signed form cla@scylladb.com. You can then submit your changes as patches to the to the [scylladb-dev mailing list](https://groups.google.com/forum/#!forum/scylladb-dev) or as a pull request to the [Scylla project on github](https://github.com/scylladb/scylla).
 If you need help formatting or sending patches, [check out these instructions](https://github.com/scylladb/scylla/wiki/Formatting-and-sending-patches).
 
 The Scylla C++ source code uses the [Seastar coding style](https://github.com/scylladb/seastar/blob/master/coding-style.md) so please adhere to that in your patches. Note that Scylla code is written with `using namespace seastar`, so should not explicitly add the `seastar::` prefix to Seastar symbols. You will usually not need to add `using namespace seastar` to new source files, because most Scylla header files have `#include "seastarx.hh"`, which does this.

HACKING.md

@@ -220,9 +220,28 @@ On a development machine, one might run Scylla as
 $ SCYLLA_HOME=$HOME/scylla build/release/scylla --overprovisioned --developer-mode=yes
 ```
 
-To interact with scylla it is recommended to build our version of
-cqlsh. It is available at
-https://github.com/scylladb/scylla-cqlsh and is available as a submodule.
+To interact with scylla it is recommended to build our versions of
+cqlsh and nodetool. They are available at
+https://github.com/scylladb/scylla-tools-java and can be built with
+
+```bash
+$ sudo ./install-dependencies.sh
+$ ant jar
+```
+
+cqlsh should work out of the box, but nodetool depends on a running
+scylla-jmx (https://github.com/scylladb/scylla-jmx). It can be build
+with
+
+```bash
+$ mvn package
+```
+
+and must be started with
+
+```bash
+$ ./scripts/scylla-jmx
+```
 
 ### Branches and tags
 
@@ -261,45 +280,21 @@ Once the patch set is ready to be reviewed, push the branch to the public remote
 
 ### Development environment and source code navigation
 
-Scylla includes a [CMake](https://cmake.org/) file, `CMakeLists.txt` that can be used with development environments so
-that they can properly analyze the source code. However, building with CMake is not yet officially supported.
+Scylla includes a [CMake](https://cmake.org/) file, `CMakeLists.txt`, for use only with development environments (not for building) so that they can properly analyze the source code.
 
-Good IDEs that have support for CMake build toolchain are [CLion](https://www.jetbrains.com/clion/),
-[KDevelop](https://www.kdevelop.org/) and [QtCreator](https://wiki.qt.io/Qt_Creator).
+[CLion](https://www.jetbrains.com/clion/) is a commercial IDE offers reasonably good source code navigation and advice for code hygiene, though its C++ parser sometimes makes errors and flags false issues.
 
-[Eclipse](https://eclipse.org/cdt/) is another open-source option. It doesn't natively work with CMake projects and its
-C++ parser has many issues.
+Other good options that directly parse CMake files are [KDevelop](https://www.kdevelop.org/) and [QtCreator](https://wiki.qt.io/Qt_Creator).
 
-#### CLion
+To use the `CMakeLists.txt` file with these programs, define the `FOR_IDE` CMake variable or shell environmental variable.
 
-[CLion](https://www.jetbrains.com/clion/) is a commercial IDE offers reasonably good source code navigation and advice
-for code hygiene, though its C++ parser sometimes makes errors and flags false issues. In order to enable proper code
-analysis in CLion, the following steps are needed:
-
-1. Get the ScyllaDB source code by following the [Getting the source code](#getting-the-source-code).
-2. Follow the steps in [Dependencies](#dependencies) in order to install the required tools natively into your system.
-   **Don't** follow the *frozen toolchain* part described there, since CMake checks for the build dependencies installed
-   in the system, not in the container image provided by the toolchain.
-3. In CLion, select `File`→`Open` and select the main ScyllaDB directory in order to open the CMake project there. The
-   project should open and fail to process the `CMakeLists.txt`. That's expected.
-4. In CLion, open `File`→`Settings`.
-5. Find and click on `Toolchains` (type *toolchains* into search box).
-6. Select the toolchain you will use, for instance the `Default` one.
-7. Type in the following system-installed tools to be used:
-    - `CMake`: *cmake*
-    - `Build Tool`: *ninja*
-    - `C Compiler`: *clang*
-    - `C++ Compiler`: *clang*
-8. On the `CMake` panel/tab, click on `Reload CMake Project`
-
-After that, CLion should successfully initialize the CMake project (marked by `[Finished]` in the console) and the
-source code editor should provide code analysis support normally from now on.
+[Eclipse](https://eclipse.org/cdt/) is another open-source option. It doesn't natively work with CMake projects, and its C++ parser has many similar issues as CLion.
 
 ### Distributed compilation: `distcc` and `ccache`
 
 Scylla's compilations times can be long. Two tools help somewhat:
 
-- [ccache](https://ccache.samba.org/) caches compiled object files on disk and reuses them when possible
+- [ccache](https://ccache.samba.org/) caches compiled object files on disk and re-uses them when possible
 - [distcc](https://github.com/distcc/distcc) distributes compilation jobs to remote machines
 
 A reasonably-powered laptop acts as the coordinator for compilation. A second, more powerful, machine acts as a passive compilation server.

LICENSE-ScyllaDB-Source-Available.md

@@ -49,7 +49,7 @@ The terms "**You**" or "**Licensee**" refer to any individual accessing or using
 
 * **Ownership:** Licensor retains sole and exclusive ownership of all rights, interests and title in the Software and any scripts, processes, techniques, methodologies, inventions, know-how, concepts, formatting, arrangements, visual attributes, ideas, database rights, copyrights, patents, trade secrets, and other intellectual property related thereto, and all derivatives, enhancements, modifications and improvements thereof. Except for the limited license rights granted herein, Licensee has no rights in or to the Software and/ or Licensor’s trademarks, logo, or branding and You acknowledge that such Software, trademarks, logo, or branding is the sole property of Licensor.
 * **Feedback:** Licensee is not required to provide any suggestions, enhancement requests, recommendations or other feedback regarding the Software ("Feedback").  If, notwithstanding this policy, Licensee submits Feedback, Licensee understands and acknowledges that such Feedback is not submitted in confidence and Licensor assumes no obligation, expressed or implied, by considering it.  All right in any trademark or logo of Licensor or its affiliates and You shall make no claim of right to the Software or any part thereof to be supplied by Licensor hereunder and acknowledges that as between Licensor and You, such Software is the sole proprietary, title and interest in and to Licensor.such Feedback shall be assigned to, and shall become the sole and exclusive property of, Licensor upon its creation.
-* Except for the rights expressly granted to You under this Agreement, You are not granted any other licenses or rights in the Software or otherwise. This Agreement constitutes the entire agreement between You and the Licensor with respect to the subject matter hereof and supersedes all prior or contemporaneous communications, representations, or agreements, whether oral or written.
+* Except for the rights expressly granted to You under this Agreement, You are not granted any other licenses or rights in the Software or otherwise. This Agreement constitutes the entire agreement between the You and the Licensor with respect to the subject matter hereof and supersedes all prior or contemporaneous communications, representations, or agreements, whether oral or written.
 * **Third-Party Software:** Customer acknowledges that the Software may contain open and closed source components (“OSS Components”) that are governed separately by certain licenses, in each case as further provided by Company upon request. Any applicable OSS Component license is solely between Licensee and the applicable licensor of the OSS Component and Licensee shall comply with the applicable OSS Component license.
 * If any provision of this Agreement is held to be invalid or unenforceable, such provision shall be struck and the remaining provisions shall remain in full force and effect.
 

README.md

@@ -102,7 +102,7 @@ If you are a developer working on Scylla, please read the [developer guidelines]
 
 ## Contact
 
-* The [community forum] and [Slack channel] are for users to discuss configuration, management, and operations of ScyllaDB.
+* The [community forum] and [Slack channel] are for users to discuss configuration, management, and operations of the ScyllaDB open source.
 * The [developers mailing list] is for developers and people interested in following the development of ScyllaDB to discuss technical topics.
 
 [Community forum]: https://forum.scylladb.com/

SCYLLA-VERSION-GEN

@@ -78,7 +78,7 @@ fi
 
 # Default scylla product/version tags
 PRODUCT=scylla
-VERSION=2025.2.6
+VERSION=6.3.0-dev
 
 if test -f version
 then

alternator/consumed_capacity.cc

@@ -24,7 +24,7 @@ static constexpr uint64_t KB = 1024ULL;
 static constexpr uint64_t RCU_BLOCK_SIZE_LENGTH = 4*KB;
 static constexpr uint64_t WCU_BLOCK_SIZE_LENGTH = 1*KB;
 
-bool consumed_capacity_counter::should_add_capacity(const rjson::value& request) {
+static bool should_add_capacity(const rjson::value& request) {
     const rjson::value* return_consumed = rjson::find(request, "ReturnConsumedCapacity");
     if (!return_consumed) {
         return false;
@@ -62,22 +62,15 @@ static uint64_t calculate_half_units(uint64_t unit_block_size, uint64_t total_by
 rcu_consumed_capacity_counter::rcu_consumed_capacity_counter(const rjson::value& request, bool is_quorum) :
         consumed_capacity_counter(should_add_capacity(request)),_is_quorum(is_quorum) {
 }
-uint64_t rcu_consumed_capacity_counter::get_half_units(uint64_t total_bytes, bool is_quorum) noexcept {
-    return calculate_half_units(RCU_BLOCK_SIZE_LENGTH, total_bytes, is_quorum);
-}
 
 uint64_t rcu_consumed_capacity_counter::get_half_units() const noexcept {
-    return get_half_units(_total_bytes, _is_quorum);
+    return calculate_half_units(RCU_BLOCK_SIZE_LENGTH, _total_bytes, _is_quorum);
 }
 
 uint64_t wcu_consumed_capacity_counter::get_half_units() const noexcept {
     return calculate_half_units(WCU_BLOCK_SIZE_LENGTH, _total_bytes, true);
 }
 
-uint64_t wcu_consumed_capacity_counter::get_units(uint64_t total_bytes) noexcept {
-    return calculate_half_units(WCU_BLOCK_SIZE_LENGTH, total_bytes, true) * HALF_UNIT_MULTIPLIER;
-}
-
 wcu_consumed_capacity_counter::wcu_consumed_capacity_counter(const rjson::value& request) :
         consumed_capacity_counter(should_add_capacity(request)) {
 }

alternator/consumed_capacity.hh

@@ -42,25 +42,21 @@ public:
      */
     virtual uint64_t get_half_units() const noexcept = 0;
     uint64_t _total_bytes = 0;
-    static bool should_add_capacity(const rjson::value& request);
 protected:
     bool _should_add_to_reponse = false;
 };
 
 class rcu_consumed_capacity_counter : public consumed_capacity_counter {
+    virtual uint64_t get_half_units() const noexcept;
     bool _is_quorum = false;
 public:
     rcu_consumed_capacity_counter(const rjson::value& request, bool is_quorum);
-    rcu_consumed_capacity_counter(): consumed_capacity_counter(false), _is_quorum(false){}
-    virtual uint64_t get_half_units() const noexcept;
-    static uint64_t get_half_units(uint64_t total_bytes, bool is_quorum) noexcept;
 };
 
 class wcu_consumed_capacity_counter : public consumed_capacity_counter {
     virtual uint64_t get_half_units() const noexcept;
 public:
     wcu_consumed_capacity_counter(const rjson::value& request);
-    static uint64_t get_units(uint64_t total_bytes) noexcept;
 };
 
 }

alternator/controller.cc

@@ -6,9 +6,7 @@
  * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
  */
 
-#include <seastar/core/with_scheduling_group.hh>
 #include <seastar/net/dns.hh>
-
 #include "controller.hh"
 #include "server.hh"
 #include "executor.hh"

alternator/error.hh

@@ -88,9 +88,6 @@ public:
     static api_error table_not_found(std::string msg) {
         return api_error("TableNotFoundException", std::move(msg));
     }
-    static api_error limit_exceeded(std::string msg) {
-        return api_error("LimitExceededException", std::move(msg));
-    }
     static api_error internal(std::string msg) {
         return api_error("InternalServerError", std::move(msg), http::reply::status_type::internal_server_error);
     }

alternator/executor.hh

@@ -241,8 +241,7 @@ public:
         const query::partition_slice&& slice,
         shared_ptr<cql3::selection::selection> selection,
         foreign_ptr<lw_shared_ptr<query::result>> query_result,
-        shared_ptr<const std::optional<attrs_to_get>> attrs_to_get,
-        uint64_t& rcu_half_units);
+        shared_ptr<const std::optional<attrs_to_get>> attrs_to_get);
 
     static void describe_single_item(const cql3::selection::selection&,
         const std::vector<managed_bytes_opt>&,

alternator/expressions.cc

@@ -165,9 +165,7 @@ static std::optional<std::string> resolve_path_component(const std::string& colu
                     fmt::format("ExpressionAttributeNames missing entry '{}' required by expression", column_name));
         }
         used_attribute_names.emplace(column_name);
-        auto result = std::string(rjson::to_string_view(*value));
-        validate_attr_name_length("", result.size(), false, "ExpressionAttributeNames contains invalid value: ");
-        return result;
+        return std::string(rjson::to_string_view(*value));
     }
     return std::nullopt;
 }
@@ -739,26 +737,6 @@ rjson::value calculate_value(const parsed::set_rhs& rhs,
     return rjson::null_value();
 }
 
-void validate_attr_name_length(std::string_view supplementary_context, size_t attr_name_length, bool is_key, std::string_view error_msg_prefix) {
-    constexpr const size_t DYNAMODB_KEY_ATTR_NAME_SIZE_MAX = 255;
-    constexpr const size_t DYNAMODB_NONKEY_ATTR_NAME_SIZE_MAX = 65535;
-
-    const size_t max_length = is_key ? DYNAMODB_KEY_ATTR_NAME_SIZE_MAX : DYNAMODB_NONKEY_ATTR_NAME_SIZE_MAX;
-    if (attr_name_length > max_length) {
-        std::string error_msg;
-        if (!error_msg_prefix.empty()) {
-            error_msg += error_msg_prefix;
-        }
-        if (!supplementary_context.empty()) {
-            error_msg += "in ";
-            error_msg += supplementary_context;
-            error_msg += " - ";
-        }
-        error_msg += fmt::format("Attribute name is too large, must be less than {} bytes", std::to_string(max_length + 1));
-        throw api_error::validation(error_msg);
-    }
-}
-
 } // namespace alternator
 
 auto fmt::formatter<alternator::parsed::path>::format(const alternator::parsed::path& p, fmt::format_context& ctx) const

alternator/expressions.g

@@ -91,18 +91,6 @@ options {
         throw expressions_syntax_error(format("{} at char {}", err,
             ex->get_charPositionInLine()));
     }
-
-    // ANTLR3 tries to recover missing tokens - it tries to finish parsing
-    // and create valid objects, as if the missing token was there.
-    // But it has a bug and leaks these tokens.
-    // We override offending method and handle abandoned pointers.
-    std::vector<std::unique_ptr<TokenType>> _missing_tokens;
-    TokenType* getMissingSymbol(IntStreamType* istream, ExceptionBaseType* e,
-                                ANTLR_UINT32 expectedTokenType, BitsetListType* follow) {
-        auto token = BaseType::getMissingSymbol(istream, e, expectedTokenType, follow);
-        _missing_tokens.emplace_back(token);
-        return token;
-    }
 }
 @lexer::context {
     void displayRecognitionError(ANTLR_UINT8** token_names, ExceptionBaseType* ex) {

alternator/expressions.hh

@@ -91,7 +91,5 @@ rjson::value calculate_value(const parsed::value& v,
 rjson::value calculate_value(const parsed::set_rhs& rhs,
         const rjson::value* previous_item);
 
-void validate_attr_name_length(std::string_view supplementary_context, size_t attr_name_length, bool is_key, std::string_view error_msg_prefix = {});
-
 
 } /* namespace alternator */

alternator/extract_from_attrs.hh

@@ -1,73 +0,0 @@
-/*
- * Copyright 2024-present ScyllaDB
- */
-
-/*
- * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
- */
-
-#pragma once
-
-#include <string>
-#include <string_view>
-
-#include "utils/rjson.hh"
-#include "serialization.hh"
-#include "column_computation.hh"
-#include "db/view/regular_column_transformation.hh"
-
-namespace alternator {
-
-// An implementation of a "column_computation" which extracts a specific
-// non-key attribute from the big map (":attrs") of all non-key attributes,
-// and deserializes it if it has the desired type. GSI will use this computed
-// column as a materialized-view key when the view key attribute isn't a
-// full-fledged CQL column but rather stored in ":attrs".
-class extract_from_attrs_column_computation : public regular_column_transformation {
-    // The name of the CQL column name holding the attribute map. It is a
-    // constant defined in executor.cc (as ":attrs"), so doesn't need
-    // to be specified when constructing the column computation.
-    static const bytes MAP_NAME;
-    // The top-level attribute name to extract from the ":attrs" map.
-    bytes _attr_name;
-    // The type we expect for the value stored in the attribute. If the type
-    // matches the expected type, it is decoded from the serialized format
-    // we store in the map's values) into the raw CQL type value that we use
-    // for keys, and returned by compute_value(). Only the types "S" (string),
-    // "B" (bytes) and "N" (number) are allowed as keys in DynamoDB, and
-    // therefore in desired_type.
-    alternator_type _desired_type;
-public:
-    virtual column_computation_ptr clone() const override;
-    // TYPE_NAME is a unique string that distinguishes this class from other
-    // column_computation subclasses. column_computation::deserialize() will
-    // construct an object of this subclass if it sees a "type" TYPE_NAME.
-    static inline const std::string TYPE_NAME = "alternator_extract_from_attrs";
-    // Serialize the *definition* of this column computation into a JSON
-    // string with a unique "type" string - TYPE_NAME - which then causes
-    // column_computation::deserialize() to create an object from this class.
-    virtual bytes serialize() const override;
-    // Construct this object based on the previous output of serialize().
-    // Calls on_internal_error() if the string doesn't match the output format
-    // of serialize(). "type" is not checked column_computation::deserialize()
-    // won't call this constructor if "type" doesn't match.
-    extract_from_attrs_column_computation(const rjson::value &v);
-    extract_from_attrs_column_computation(bytes_view attr_name, alternator_type desired_type)
-        : _attr_name(attr_name), _desired_type(desired_type)
-        {}
-    // Implement regular_column_transformation's compute_value() that
-    // accepts the full row:
-    result compute_value(const schema& schema, const partition_key& key,
-        const db::view::clustering_or_static_row& row) const override;
-    // But do not implement column_computation's compute_value() that
-    // accepts only a partition key - that's not enough so our implementation
-    // of this function does on_internal_error().
-    bytes compute_value(const schema& schema, const partition_key& key) const override;
-    // This computed column does depend on a non-primary key column, so
-    // its result may change in the update and we need to compute it
-    // before and after the update.
-    virtual bool depends_on_non_primary_key_column() const override {
-        return true;
-    }
-};
-} // namespace alternator

alternator/serialization.cc

@@ -245,27 +245,6 @@ rjson::value deserialize_item(bytes_view bv) {
     return deserialized;
 }
 
-// This function takes a bytes_view created earlier by serialize_item(), and
-// if has the type "expected_type", the function returns the value as a
-// raw Scylla type. If the type doesn't match, returns an unset optional.
-// This function only supports the key types S (string), B (bytes) and N
-// (number) - serialize_item() serializes those types as a single-byte type
-// followed by the serialized raw Scylla type, so all this function needs to
-// do is to remove the first byte. This makes this function much more
-// efficient than deserialize_item() above because it avoids transformation
-// to/from JSON.
-std::optional<bytes> serialized_value_if_type(bytes_view bv, alternator_type expected_type) {
-    if (bv.empty() || alternator_type(bv[0]) != expected_type) {
-        return std::nullopt;
-    }
-    // Currently, serialize_item() for types in alternator_type (notably S, B
-    // and N) are nothing more than Scylla's raw format for these types
-    // preceded by a type byte. So we just need to skip that byte and we are
-    // left by exactly what we need to return.
-    bv.remove_prefix(1);
-    return bytes(bv);
-}
-
 std::string type_to_string(data_type type) {
     static thread_local std::unordered_map<data_type, std::string> types = {
         {utf8_type, "S"},

alternator/serialization.hh

@@ -43,7 +43,6 @@ type_representation represent_type(alternator_type atype);
 
 bytes serialize_item(const rjson::value& item);
 rjson::value deserialize_item(bytes_view bv);
-std::optional<bytes> serialized_value_if_type(bytes_view bv, alternator_type expected_type);
 
 std::string type_to_string(data_type type);
 

alternator/server.cc

@@ -217,7 +217,7 @@ protected:
         // If the DC does not exist, we return an empty list - not an error.
         sstring query_dc = req->get_query_param("dc");
         sstring local_dc = query_dc.empty() ? topology.get_datacenter() : query_dc;
-        std::unordered_set<locator::host_id> local_dc_nodes;
+        std::unordered_set<gms::inet_address> local_dc_nodes;
         const auto& endpoints = topology.get_datacenter_endpoints();
         auto dc_it = endpoints.find(local_dc);
         if (dc_it != endpoints.end()) {
@@ -227,9 +227,9 @@ protected:
         // DC, unless a single rack is selected by the "rack" query option.
         // If the rack does not exist, we return an empty list - not an error.
         sstring query_rack = req->get_query_param("rack");
-        for (auto& id : local_dc_nodes) {
+        for (auto& ip : local_dc_nodes) {
             if (!query_rack.empty()) {
-                auto rack = _gossiper.get_application_state_value(id, gms::application_state::RACK);
+                auto rack = _gossiper.get_application_state_value(ip, gms::application_state::RACK);
                 if (rack != query_rack) {
                     continue;
                 }
@@ -237,10 +237,10 @@ protected:
             // Note that it's not enough for the node to be is_alive() - a
             // node joining the cluster is also "alive" but not responsive to
             // requests. We alive *and* normal. See #19694, #21538.
-            if (_gossiper.is_alive(id) && _gossiper.is_normal(id)) {
+            if (_gossiper.is_alive(ip) && _gossiper.is_normal(ip)) {
                 // Use the gossiped broadcast_rpc_address if available instead
                 // of the internal IP address "ip". See discussion in #18711.
-                rjson::push_back(results, rjson::from_string(_gossiper.get_rpc_address(id)));
+                rjson::push_back(results, rjson::from_string(_gossiper.get_rpc_address(ip)));
             }
         }
         rep->set_status(reply::status_type::ok);
@@ -456,16 +456,9 @@ future<executor::request_return_type> server::handle_api_request(std::unique_ptr
 
     tracing::trace_state_ptr trace_state = maybe_trace_query(client_state, username, op, content);
     tracing::trace(trace_state, "{}", op);
-
-    auto user = client_state.user();
-    auto f = [this, content = std::move(content), &callback = callback_it->second,
-            client_state = std::move(client_state), trace_state = std::move(trace_state),
-            units = std::move(units), req = std::move(req)] () mutable -> future<executor::request_return_type> {
-                rjson::value json_request = co_await _json_parser.parse(std::move(content));
-                co_return co_await callback(_executor, client_state, trace_state,
-                    make_service_permit(std::move(units)), std::move(json_request), std::move(req));
-    };
-    co_return co_await _sl_controller.with_user_service_level(user, std::ref(f));
+    rjson::value json_request = co_await _json_parser.parse(std::move(content));
+    co_return co_await callback_it->second(_executor, client_state, trace_state,
+            make_service_permit(std::move(units)), std::move(json_request), std::move(req));
 }
 
 void server::set_routes(routes& r) {
@@ -504,7 +497,7 @@ server::server(executor& exec, service::storage_proxy& proxy, gms::gossiper& gos
         , _key_cache(1024, 1min, slogger)
         , _enforce_authorization(false)
         , _enabled_servers{}
-        , _pending_requests("alternator::server::pending_requests")
+        , _pending_requests{}
         , _timeout_config(_proxy.data_dictionary().get_config())
       , _callbacks{
         {"CreateTable", [] (executor& e, executor::client_state& client_state, tracing::trace_state_ptr trace_state, service_permit permit, rjson::value json_request, std::unique_ptr<request> req) {
@@ -605,24 +598,14 @@ future<> server::init(net::inet_address addr, std::optional<uint16_t> port, std:
             set_routes(_https_server._routes);
             _https_server.set_content_length_limit(server::content_length_limit);
             _https_server.set_content_streaming(true);
-
-            if (this_shard_id() == 0) {
-                _credentials = creds->build_reloadable_server_credentials([this](const tls::credentials_builder& b, const std::unordered_set<sstring>& files, std::exception_ptr ep) -> future<> {
-                    if (ep) {
-                        slogger.warn("Exception loading {}: {}", files, ep);
-                    } else {
-                        co_await container().invoke_on_others([&b](server& s) {
-                            if (s._credentials) {
-                                b.rebuild(*s._credentials);
-                            }
-                        });
-                        slogger.info("Reloaded {}", files);
-                    }
-                }).get();
-            } else {
-                _credentials = creds->build_server_credentials();
-            }
-            _https_server.listen(socket_address{addr, *https_port}, _credentials).get();
+            auto server_creds = creds->build_reloadable_server_credentials([](const std::unordered_set<sstring>& files, std::exception_ptr ep) {
+                if (ep) {
+                    slogger.warn("Exception loading {}: {}", files, ep);
+                } else {
+                    slogger.info("Reloaded {}", files);
+                }
+            }).get();
+            _https_server.listen(socket_address{addr, *https_port}, std::move(server_creds)).get();
             _enabled_servers.push_back(std::ref(_https_server));
         }
     });

alternator/server.hh

@@ -24,7 +24,7 @@ namespace alternator {
 
 using chunked_content = rjson::chunked_content;
 
-class server : public peering_sharded_service<server> {
+class server {
     static constexpr size_t content_length_limit = 16*MB;
     using alternator_callback = std::function<future<executor::request_return_type>(executor&, executor::client_state&,
             tracing::trace_state_ptr, service_permit, rjson::value, std::unique_ptr<http::request>)>;
@@ -41,7 +41,7 @@ class server : public peering_sharded_service<server> {
     key_cache _key_cache;
     utils::updateable_value<bool> _enforce_authorization;
     utils::small_vector<std::reference_wrapper<seastar::httpd::http_server>, 2> _enabled_servers;
-    named_gate _pending_requests;
+    gate _pending_requests;
     // In some places we will need a CQL updateable_timeout_config object even
     // though it isn't really relevant for Alternator which defines its own
     // timeouts separately. We can create this object only once.
@@ -52,8 +52,6 @@ class server : public peering_sharded_service<server> {
     semaphore* _memory_limiter;
     utils::updateable_value<uint32_t> _max_concurrent_requests;
 
-    ::shared_ptr<seastar::tls::server_credentials> _credentials;
-
     class json_parser {
         static constexpr size_t yieldable_parsing_threshold = 16*KB;
         chunked_content _raw_document;

alternator/stats.cc

@@ -9,25 +9,11 @@
 #include "stats.hh"
 #include "utils/histogram_metrics_helper.hh"
 #include <seastar/core/metrics.hh>
-#include "utils/labels.hh"
 
 namespace alternator {
 
 const char* ALTERNATOR_METRICS = "alternator";
-static seastar::metrics::histogram estimated_histogram_to_metrics(const utils::estimated_histogram& histogram) {
-    seastar::metrics::histogram res;
-    res.buckets.resize(histogram.bucket_offsets.size());
-    uint64_t cumulative_count = 0;
-    res.sample_count = histogram._count;
-    res.sample_sum = histogram._sample_sum;
-    for (size_t i = 0; i < res.buckets.size(); i++) {
-        auto& v = res.buckets[i];
-        v.upper_bound = histogram.bucket_offsets[i];
-        cumulative_count += histogram.buckets[i];
-        v.count = cumulative_count;
-    }
-    return res;
-}
+
 stats::stats() : api_operations{} {
     // Register the
     seastar::metrics::label op("op");
@@ -35,12 +21,12 @@ stats::stats() : api_operations{} {
     _metrics.add_group("alternator", {
 #define OPERATION(name, CamelCaseName) \
                 seastar::metrics::make_total_operations("operation", api_operations.name, \
-                        seastar::metrics::description("number of operations via Alternator API"), {op(CamelCaseName), alternator_label, basic_level}).set_skip_when_empty(),
+                        seastar::metrics::description("number of operations via Alternator API"), {op(CamelCaseName)}).set_skip_when_empty(),
 #define OPERATION_LATENCY(name, CamelCaseName) \
                 seastar::metrics::make_histogram("op_latency", \
-                        seastar::metrics::description("Latency histogram of an operation via Alternator API"), {op(CamelCaseName), alternator_label, basic_level}, [this]{return to_metrics_histogram(api_operations.name.histogram());}).aggregate({seastar::metrics::shard_label}).set_skip_when_empty(), \
+                        seastar::metrics::description("Latency histogram of an operation via Alternator API"), {op(CamelCaseName)}, [this]{return to_metrics_histogram(api_operations.name.histogram());}).aggregate({seastar::metrics::shard_label}).set_skip_when_empty(), \
 				seastar::metrics::make_summary("op_latency_summary", \
-						                        seastar::metrics::description("Latency summary of an operation via Alternator API"), [this]{return to_metrics_summary(api_operations.name.summary());})(op(CamelCaseName))(basic_level)(alternator_label).set_skip_when_empty(),
+						                        seastar::metrics::description("Latency summary of an operation via Alternator API"), [this]{return to_metrics_summary(api_operations.name.summary());})(op(CamelCaseName)).set_skip_when_empty(),
             OPERATION(batch_get_item, "BatchGetItem")
             OPERATION(batch_write_item, "BatchWriteItem")
             OPERATION(create_backup, "CreateBackup")
@@ -91,43 +77,39 @@ stats::stats() : api_operations{} {
     });
     _metrics.add_group("alternator", {
             seastar::metrics::make_total_operations("unsupported_operations", unsupported_operations,
-                    seastar::metrics::description("number of unsupported operations via Alternator API"))(alternator_label).set_skip_when_empty(),
+                    seastar::metrics::description("number of unsupported operations via Alternator API")),
             seastar::metrics::make_total_operations("total_operations", total_operations,
-                    seastar::metrics::description("number of total operations via Alternator API"))(basic_level)(alternator_label).set_skip_when_empty(),
+                    seastar::metrics::description("number of total operations via Alternator API")),
             seastar::metrics::make_total_operations("reads_before_write", reads_before_write,
-                    seastar::metrics::description("number of performed read-before-write operations"))(alternator_label).set_skip_when_empty(),
+                    seastar::metrics::description("number of performed read-before-write operations")),
             seastar::metrics::make_total_operations("write_using_lwt", write_using_lwt,
-                    seastar::metrics::description("number of writes that used LWT"))(alternator_label).set_skip_when_empty(),
+                    seastar::metrics::description("number of writes that used LWT")),
             seastar::metrics::make_total_operations("shard_bounce_for_lwt", shard_bounce_for_lwt,
-                    seastar::metrics::description("number writes that had to be bounced from this shard because of LWT requirements"))(alternator_label).set_skip_when_empty(),
+                    seastar::metrics::description("number writes that had to be bounced from this shard because of LWT requirements")),
             seastar::metrics::make_total_operations("requests_blocked_memory", requests_blocked_memory,
-                    seastar::metrics::description("Counts a number of requests blocked due to memory pressure."))(alternator_label).set_skip_when_empty(),
+                    seastar::metrics::description("Counts a number of requests blocked due to memory pressure.")),
             seastar::metrics::make_total_operations("requests_shed", requests_shed,
-                    seastar::metrics::description("Counts a number of requests shed due to overload."))(alternator_label).set_skip_when_empty(),
+                    seastar::metrics::description("Counts a number of requests shed due to overload.")),
             seastar::metrics::make_total_operations("filtered_rows_read_total", cql_stats.filtered_rows_read_total,
-                    seastar::metrics::description("number of rows read during filtering operations"))(alternator_label).set_skip_when_empty(),
+                    seastar::metrics::description("number of rows read during filtering operations")),
             seastar::metrics::make_total_operations("filtered_rows_matched_total", cql_stats.filtered_rows_matched_total,
                     seastar::metrics::description("number of rows read and matched during filtering operations")),
-            seastar::metrics::make_counter("rcu_total", [this]{return 0.5 * rcu_half_units_total;},
-                    seastar::metrics::description("total number of consumed read units"))(alternator_label).set_skip_when_empty(),
+            seastar::metrics::make_counter("rcu_total", rcu_total,
+                    seastar::metrics::description("total number of consumed read units, counted as half units")).set_skip_when_empty(),
             seastar::metrics::make_counter("wcu_total", wcu_total[wcu_types::PUT_ITEM],
-                    seastar::metrics::description("total number of consumed write units"),{op("PutItem")})(alternator_label).set_skip_when_empty(),
+                    seastar::metrics::description("total number of consumed write units, counted as half units"),{op("PutItem")}).set_skip_when_empty(),
             seastar::metrics::make_counter("wcu_total", wcu_total[wcu_types::DELETE_ITEM],
-                    seastar::metrics::description("total number of consumed write units"),{op("DeleteItem")})(alternator_label).set_skip_when_empty(),
+                    seastar::metrics::description("total number of consumed write units, counted as half units"),{op("DeleteItem")}).set_skip_when_empty(),
             seastar::metrics::make_counter("wcu_total", wcu_total[wcu_types::UPDATE_ITEM],
-                    seastar::metrics::description("total number of consumed write units"),{op("UpdateItem")})(alternator_label).set_skip_when_empty(),
+                    seastar::metrics::description("total number of consumed write units, counted as half units"),{op("UpdateItem")}).set_skip_when_empty(),
             seastar::metrics::make_counter("wcu_total", wcu_total[wcu_types::INDEX],
-                    seastar::metrics::description("total number of consumed write units"),{op("Index")})(alternator_label).set_skip_when_empty(),
+                    seastar::metrics::description("total number of consumed write units, counted as half units"),{op("Index")}).set_skip_when_empty(),
             seastar::metrics::make_total_operations("filtered_rows_dropped_total", [this] { return cql_stats.filtered_rows_read_total - cql_stats.filtered_rows_matched_total; },
-                    seastar::metrics::description("number of rows read and dropped during filtering operations"))(alternator_label).set_skip_when_empty(),
+                    seastar::metrics::description("number of rows read and dropped during filtering operations")),
             seastar::metrics::make_counter("batch_item_count", seastar::metrics::description("The total number of items processed across all batches"),{op("BatchWriteItem")},
-                    api_operations.batch_write_item_batch_total)(alternator_label).set_skip_when_empty(),
+                    api_operations.batch_write_item_batch_total).set_skip_when_empty(),
             seastar::metrics::make_counter("batch_item_count", seastar::metrics::description("The total number of items processed across all batches"),{op("BatchGetItem")},
-                    api_operations.batch_get_item_batch_total)(alternator_label).set_skip_when_empty(),
-            seastar::metrics::make_histogram("batch_item_count_histogram", seastar::metrics::description("Histogram of the number of items in a batch request"),{op("BatchGetItem")},
-                    [this]{ return estimated_histogram_to_metrics(api_operations.batch_get_item_histogram);})(alternator_label).aggregate({seastar::metrics::shard_label}).set_skip_when_empty(),
-            seastar::metrics::make_histogram("batch_item_count_histogram", seastar::metrics::description("Histogram of the number of items in a batch request"),{op("BatchWriteItem")},
-                    [this]{ return estimated_histogram_to_metrics(api_operations.batch_write_item_histogram);})(alternator_label).aggregate({seastar::metrics::shard_label}).set_skip_when_empty(),
+                    api_operations.batch_get_item_batch_total).set_skip_when_empty(),
     });
 }
 

alternator/stats.hh

@@ -12,7 +12,6 @@
 
 #include <seastar/core/metrics_registration.hh>
 #include "utils/histogram.hh"
-#include "utils/estimated_histogram.hh"
 #include "cql3/stats.hh"
 
 namespace alternator {
@@ -76,9 +75,6 @@ public:
         utils::timed_rate_moving_average_summary_and_histogram batch_write_item_latency;
         utils::timed_rate_moving_average_summary_and_histogram batch_get_item_latency;
         utils::timed_rate_moving_average_summary_and_histogram get_records_latency;
-
-        utils::estimated_histogram batch_get_item_histogram{22}; // a histogram that covers the range 1 - 100
-        utils::estimated_histogram batch_write_item_histogram{22}; // a histogram that covers the range 1 - 100
     } api_operations;
     // Miscellaneous event counters
     uint64_t total_operations = 0;
@@ -88,7 +84,7 @@ public:
     uint64_t shard_bounce_for_lwt = 0;
     uint64_t requests_blocked_memory = 0;
     uint64_t requests_shed = 0;
-    uint64_t rcu_half_units_total = 0;
+    uint64_t rcu_total = 0;
     // wcu can results from put, update, delete and index
     // Index related will be done on top of the operation it comes with
     enum wcu_types {

alternator/streams.cc

@@ -808,9 +808,6 @@ future<executor::request_return_type> executor::get_records(client_state& client
     if (limit < 1) {
         throw api_error::validation("Limit must be 1 or more");
     }
-    if (limit > 1000) {
-        throw api_error::validation("Limit must be less than or equal to 1000");
-    }
 
     auto db = _proxy.data_dictionary();
     schema_ptr schema, base;

alternator/ttl.cc

@@ -16,6 +16,7 @@
 #include <seastar/core/future.hh>
 #include <seastar/core/lowres_clock.hh>
 #include <seastar/coroutine/maybe_yield.hh>
+#include <boost/multiprecision/cpp_int.hpp>
 
 #include "exceptions/exceptions.hh"
 #include "gms/gossiper.hh"
@@ -48,7 +49,6 @@
 #include "dht/sharder.hh"
 #include "db/config.hh"
 #include "db/tags/utils.hh"
-#include "utils/labels.hh"
 
 #include "ttl.hh"
 
@@ -851,13 +851,13 @@ future<> expiration_service::stop() {
 expiration_service::stats::stats() {
     _metrics.add_group("expiration", {
         seastar::metrics::make_total_operations("scan_passes", scan_passes,
-            seastar::metrics::description("number of passes over the database"))(alternator_label).set_skip_when_empty(),
+            seastar::metrics::description("number of passes over the database")),
         seastar::metrics::make_total_operations("scan_table", scan_table,
-            seastar::metrics::description("number of table scans (counting each scan of each table that enabled expiration)"))(alternator_label).set_skip_when_empty(),
+            seastar::metrics::description("number of table scans (counting each scan of each table that enabled expiration)")),
         seastar::metrics::make_total_operations("items_deleted", items_deleted,
-            seastar::metrics::description("number of items deleted after expiration"))(basic_level)(alternator_label).set_skip_when_empty(),
+            seastar::metrics::description("number of items deleted after expiration")),
         seastar::metrics::make_total_operations("secondary_ranges_scanned", secondary_ranges_scanned,
-            seastar::metrics::description("number of token ranges scanned by this node while their primary owner was down"))(alternator_label).set_skip_when_empty(),
+            seastar::metrics::description("number of token ranges scanned by this node while their primary owner was down")),
     });
 }
 

api/CMakeLists.txt

@@ -42,7 +42,6 @@ set(swagger_files
   api-doc/messaging_service.json
   api-doc/metrics.json
   api-doc/raft.json
-  api-doc/service_levels.json
   api-doc/storage_proxy.json
   api-doc/storage_service.json
   api-doc/stream_manager.json
@@ -83,7 +82,6 @@ target_sources(api
     lsa.cc
     messaging_service.cc
     raft.cc
-    service_levels.cc
     storage_proxy.cc
     storage_service.cc
     stream_manager.cc

api/api-doc/gossiper.json

@@ -136,6 +136,14 @@
                      "allowMultiple":false,
                      "type":"string",
                      "paramType":"path"
+                  },
+                  {
+                     "name":"unsafe",
+                     "description":"Set to True to perform an unsafe assassination",
+                     "required":false,
+                     "allowMultiple":false,
+                     "type":"boolean",
+                     "paramType":"query"
                   }
                ]
             }

api/api-doc/service_levels.json

@@ -1,56 +0,0 @@
-{
-    "apiVersion":"0.0.1",
-    "swaggerVersion":"1.2",
-    "basePath":"{{Protocol}}://{{Host}}",
-    "resourcePath":"/service_levels",
-    "produces":[
-        "application/json"
-    ],
-    "apis":[
-        {
-            "path":"/service_levels/switch_tenants",
-            "operations":[
-                {
-                    "method":"POST",
-                    "summary":"Switch tenants on all opened connections if needed",
-                    "type":"void",
-                    "nickname":"do_switch_tenants",
-                    "produces":[
-                        "application/json"
-                    ],
-                    "parameters":[]
-                }
-            ]
-        },
-        {
-            "path":"/service_levels/count_connections",
-            "operations":[
-                {
-                    "method":"GET",
-                    "summary":"Count opened CQL connections per scheduling group per user",
-                    "type":"connections_count_map",
-                    "nickname":"count_connections",
-                    "produces":[
-                        "application/json"
-                    ],
-                    "parameters":[]
-                }
-            ]
-        }
-    ],
-    "models":{},
-    "components": {
-        "schemas": {
-          "connections_count_map": {
-            "type": "object",
-            "additionalProperties": {
-              "type": "object",
-              "additionalProperties": {
-                "type": "integer"
-              }
-            }
-          }
-        }
-      }
-
-}

api/api-doc/storage_service.json

@@ -813,14 +813,6 @@
                           "allowMultiple":false,
                           "type":"string",
                           "paramType":"query"
-                      },
-                      {
-                          "name":"move_files",
-                          "description":"Move component files instead of copying them",
-                          "required":false,
-                          "allowMultiple":false,
-                          "type":"boolean",
-                          "paramType":"query"
                       }
                   ]
               }
@@ -889,15 +881,6 @@
                           "allowMultiple":false,
                           "type":"string",
                           "paramType":"query"
-                      },
-                      {
-                          "name":"scope",
-                          "description":"Defines the set of nodes to which mutations can be streamed",
-                          "required":false,
-                          "allowMultiple":false,
-                          "type":"string",
-                          "paramType":"query",
-                          "enum": ["all", "dc", "rack", "node"]
                       }
                   ]
               }
@@ -984,7 +967,7 @@
          ]
       },
       {
-         "path":"/storage_service/cleanup_all/",
+         "path":"/storage_service/cleanup_all",
          "operations":[
             {
                "method":"POST",
@@ -994,30 +977,6 @@
                "produces":[
                   "application/json"
                ],
-               "parameters":[
-                    {
-                     "name":"global",
-                     "description":"true if cleanup of entire cluster is requested",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"boolean",
-                     "paramType":"query"
-                  }
-               ]
-            }
-         ]
-      },
-      {
-         "path":"/storage_service/mark_node_as_clean",
-         "operations":[
-            {
-               "method":"POST",
-               "summary":"Mark the node as clean. After that the node will not be considered as needing cleanup during automatic cleanup which is triggered by some topology operations",
-               "type":"void",
-               "nickname":"reset_cleanup_needed",
-               "produces":[
-                  "application/json"
-               ],
                "parameters":[]
             }
          ]
@@ -1680,6 +1639,38 @@
             }
          ]
       },
+      {
+         "path":"/storage_service/truncate/{keyspace}",
+         "operations":[
+            {
+               "method":"POST",
+               "summary":"Truncates (deletes) the given columnFamily from the provided keyspace. Calling truncate results in actual deletion of all data in the cluster under the given columnFamily and it will fail unless all hosts are up. All data in the given column family will be deleted, but its definition will not be affected.",
+               "type":"void",
+               "nickname":"truncate",
+               "produces":[
+                  "application/json"
+               ],
+               "parameters":[
+                  {
+                     "name":"keyspace",
+                     "description":"The keyspace",
+                     "required":true,
+                     "allowMultiple":false,
+                     "type":"string",
+                     "paramType":"path"
+                  },
+                  {
+                     "name":"cf",
+                     "description":"Column family name",
+                     "required":false,
+                     "allowMultiple":false,
+                     "type":"string",
+                     "paramType":"query"
+                  }
+               ]
+            }
+         ]
+      },
       {
          "path":"/storage_service/keyspaces",
          "operations":[
@@ -2168,31 +2159,6 @@
                      "allowMultiple":false,
                      "type":"string",
                      "paramType":"query"
-                  },
-                  {
-                     "name":"skip_cleanup",
-                     "description":"Don't cleanup keys from loaded sstables. Invalid if load_and_stream is true",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
-                  },
-                  {
-                     "name":"skip_reshape",
-                     "description":"Don't reshape the loaded sstables. Invalid if load_and_stream is true",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
-                  },
-                  {
-                     "name":"scope",
-                     "description":"Defines the set of nodes to which mutations can be streamed",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query",
-                     "enum": ["all", "dc", "rack", "node"]
                   }
                ]
             }
@@ -2893,7 +2859,7 @@
                "nickname":"repair_tablet",
                "method":"POST",
                "summary":"Repair a tablet",
-               "type":"tablet_repair_result",
+               "type":"void",
                "produces":[
                   "application/json"
                ],
@@ -2921,30 +2887,6 @@
                      "allowMultiple":false,
                      "type":"string",
                      "paramType":"query"
-                  },
-                  {
-                     "name":"hosts_filter",
-                     "description":"Repair replicas listed in the comma-separated host_id list.",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
-                  },
-                  {
-                     "name":"dcs_filter",
-                     "description":"Repair replicas listed in the comma-separated DC list",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
-                  },
-                  {
-                     "name":"await_completion",
-                     "description":"Set true to wait for the repair to complete. Set false to skip waiting for the repair to complete. When the option is not provided, it defaults to false.",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
                   }
                ]
             }
@@ -3076,73 +3018,6 @@
             }
          ]
       },
-      {
-         "path":"/storage_service/retrain_dict",
-         "operations":[
-            {
-               "method":"POST",
-               "summary":"Retrain the SSTable compression dictionary for the target table.",
-               "type":"void",
-               "nickname":"retrain_dict",
-               "produces":[
-                  "application/json"
-               ],
-               "parameters":[
-                  {
-                     "name":"keyspace",
-                     "description":"Name of the keyspace containing the target table.",
-                     "required":true,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
-                  },
-                  {
-                     "name":"cf",
-                     "description":"Name of the target table.",
-                     "required":true,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
-                  }
-               ]
-            }
-         ]
-      },
-      {
-         "path":"/storage_service/estimate_compression_ratios",
-         "operations":[
-            {
-               "method":"GET",
-               "summary":"Compute an estimated compression ratio for SSTables of the given table, for various compression configurations.",
-               "type":"array",
-               "items":{
-                  "type":"compression_config_result"
-               },
-               "nickname":"estimate_compression_ratios",
-               "produces":[
-                  "application/json"
-               ],
-               "parameters":[
-                  {
-                     "name":"keyspace",
-                     "description":"Name of the keyspace containing the target table.",
-                     "required":true,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
-                  },
-                  {
-                     "name":"cf",
-                     "description":"Name of the target table.",
-                     "required":true,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
-                  }
-               ]
-            }
-         ]
-      },
       {
          "path":"/storage_service/raft_topology/reload",
          "operations":[
@@ -3185,22 +3060,6 @@
                ]
             }
          ]
-      },
-      {
-         "path":"/storage_service/raft_topology/cmd_rpc_status",
-         "operations":[
-            {
-               "method":"GET",
-               "summary":"Get information about currently running topology cmd rpc",
-               "type":"string",
-               "nickname":"raft_topology_get_cmd_status",
-               "produces":[
-                  "application/json"
-               ],
-               "parameters":[
-               ]
-            }
-         ]
       }
    ],
    "models":{
@@ -3337,11 +3196,11 @@
          "properties":{
             "start_token":{
                "type":"string",
-               "description":"The range start token (exclusive)"
+               "description":"The range start token"
             },
             "end_token":{
                "type":"string",
-               "description":"The range end token (inclusive)"
+               "description":"The range start token"
             },
             "endpoints":{
                "type":"array",
@@ -3451,41 +3310,6 @@
                 }
             }
         }
-      },
-      "tablet_repair_result":{
-        "id":"tablet_repair_result",
-        "description":"Tablet repair result",
-        "properties":{
-            "tablet_task_id":{
-                "type":"string"
-            }
-        }
-      },
-      "compression_config_result":{
-         "id":"compression_config_result",
-         "description":"Compression ratio estimation result for one config",
-         "properties":{
-            "level":{
-               "type":"long",
-               "description":"The used value of `compression_level`"
-            },
-            "chunk_length_in_kb":{
-               "type":"long",
-               "description":"The used value of `chunk_length_in_kb`"
-            },
-            "dict":{
-               "type":"string",
-               "description":"The used dictionary: `none`, `past` (== current), or `future`"
-            },
-            "sstable_compression":{
-               "type":"string",
-               "description":"The used compressor name (aka `sstable_compression`)"
-            },
-            "ratio":{
-               "type":"float",
-               "description":"The resulting compression ratio (estimated on a random sample of files)"
-            }
-         }
       }
    }
 }

api/api-doc/task_manager.json

@@ -253,30 +253,6 @@
                ]
             }
          ]
-      },
-      {
-         "path":"/task_manager/drain/{module}",
-         "operations":[
-            {
-               "method":"POST",
-               "summary":"Drain finished local tasks",
-               "type":"void",
-               "nickname":"drain_tasks",
-               "produces":[
-                  "application/json"
-               ],
-               "parameters":[
-                  {
-                     "name":"module",
-                     "description":"The module to drain",
-                     "required":true,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"path"
-                  }
-               ]
-            }
-         ]
       }
    ],
    "models":{
@@ -308,8 +284,7 @@
                   "created",
                   "running",
                   "done",
-                  "failed",
-                  "suspended"
+                  "failed"
                ],
                "description":"The state of a task"
             },
@@ -344,18 +319,6 @@
             "sequence_number":{
                "type":"long",
                "description":"The running sequence number of the task"
-            },
-            "shard":{
-               "type":"long",
-               "description":"The shard the task is running on"
-            },
-            "start_time":{
-               "type":"datetime",
-               "description":"The start time of the task; unspecified (equal to epoch) when state == created"
-            },
-            "end_time":{
-               "type":"datetime",
-               "description":"The end time of the task; unspecified (equal to epoch) when the task is not completed"
             }
          }
       },
@@ -389,8 +352,7 @@
                   "created",
                   "running",
                   "done",
-                  "failed",
-                  "suspended"
+                  "failed"
                ],
                "description":"The state of the task"
             },

api/api.cc

@@ -36,7 +36,6 @@
 #include "tasks.hh"
 #include "raft.hh"
 #include "gms/gossip_address_map.hh"
-#include "service_levels.hh"
 
 logging::logger apilog("api");
 
@@ -81,7 +80,7 @@ future<> set_server_init(http_context& ctx) {
     });
 }
 
-future<> set_server_config(http_context& ctx, db::config& cfg) {
+future<> set_server_config(http_context& ctx, const db::config& cfg) {
     auto rb02 = std::make_shared < api_registry_builder20 > (ctx.api_doc, "/v2");
     return ctx.http_server.set_routes([&ctx, &cfg, rb02](routes& r) {
         set_config(rb02, ctx, r, cfg, false);
@@ -153,8 +152,8 @@ future<> unset_server_sstables_loader(http_context& ctx) {
     return ctx.http_server.set_routes([&ctx] (routes& r) { unset_sstables_loader(ctx, r); });
 }
 
-future<> set_server_view_builder(http_context& ctx, sharded<db::view::view_builder>& vb, sharded<gms::gossiper>& g) {
-    return ctx.http_server.set_routes([&ctx, &vb, &g] (routes& r) { set_view_builder(ctx, r, vb, g); });
+future<> set_server_view_builder(http_context& ctx, sharded<db::view::view_builder>& vb) {
+    return ctx.http_server.set_routes([&ctx, &vb] (routes& r) { set_view_builder(ctx, r, vb); });
 }
 
 future<> unset_server_view_builder(http_context& ctx) {
@@ -188,8 +187,8 @@ future<> unset_server_snapshot(http_context& ctx) {
     return ctx.http_server.set_routes([&ctx] (routes& r) { unset_snapshot(ctx, r); });
 }
 
-future<> set_server_token_metadata(http_context& ctx, sharded<locator::shared_token_metadata>& tm, sharded<gms::gossiper>& g) {
-    return ctx.http_server.set_routes([&ctx, &tm, &g] (routes& r) { set_token_metadata(ctx, r, tm, g); });
+future<> set_server_token_metadata(http_context& ctx, sharded<locator::shared_token_metadata>& tm) {
+    return ctx.http_server.set_routes([&ctx, &tm] (routes& r) { set_token_metadata(ctx, r, tm); });
 }
 
 future<> unset_server_token_metadata(http_context& ctx) {
@@ -273,10 +272,10 @@ future<> unset_server_cache(http_context& ctx) {
     return ctx.http_server.set_routes([&ctx] (routes& r) { unset_cache_service(ctx, r); });
 }
 
-future<> set_hinted_handoff(http_context& ctx, sharded<service::storage_proxy>& proxy, sharded<gms::gossiper>& g) {
+future<> set_hinted_handoff(http_context& ctx, sharded<service::storage_proxy>& proxy) {
     return register_api(ctx, "hinted_handoff",
-                "The hinted handoff API", [&proxy, &g] (http_context& ctx, routes& r) {
-                    set_hinted_handoff(ctx, r, proxy, g);
+                "The hinted handoff API", [&proxy] (http_context& ctx, routes& r) {
+                    set_hinted_handoff(ctx, r, proxy);
                 });
 }
 
@@ -317,13 +316,13 @@ future<> unset_server_commitlog(http_context& ctx) {
     return ctx.http_server.set_routes([&ctx] (routes& r) { unset_commitlog(ctx, r); });
 }
 
-future<> set_server_task_manager(http_context& ctx, sharded<tasks::task_manager>& tm, lw_shared_ptr<db::config> cfg, sharded<gms::gossiper>& gossiper) {
+future<> set_server_task_manager(http_context& ctx, sharded<tasks::task_manager>& tm, lw_shared_ptr<db::config> cfg) {
     auto rb = std::make_shared < api_registry_builder > (ctx.api_doc);
 
-    return ctx.http_server.set_routes([rb, &ctx, &tm, &cfg = *cfg, &gossiper](routes& r) {
+    return ctx.http_server.set_routes([rb, &ctx, &tm, &cfg = *cfg](routes& r) {
         rb->register_function(r, "task_manager",
                 "The task manager API");
-        set_task_manager(ctx, r, tm, cfg, gossiper);
+        set_task_manager(ctx, r, tm, cfg);
     });
 }
 
@@ -359,12 +358,6 @@ future<> unset_server_cql_server_test(http_context& ctx) {
 
 #endif
 
-future<> set_server_service_levels(http_context &ctx, cql_transport::controller& ctl, sharded<cql3::query_processor>& qp) {
-    return register_api(ctx, "service_levels", "The service levels API", [&ctl, &qp] (http_context& ctx, routes& r) {
-        set_service_levels(ctx, r, ctl, qp);
-    });
-}
-
 future<> set_server_tasks_compaction_module(http_context& ctx, sharded<service::storage_service>& ss, sharded<db::snapshot_ctl>& snap_ctl) {
     auto rb = std::make_shared < api_registry_builder > (ctx.api_doc);
 

api/api_init.hh

@@ -73,10 +73,6 @@ namespace tasks {
 class task_manager;
 }
 
-namespace cql3 {
-class query_processor;
-}
-
 namespace api {
 
 struct http_context {
@@ -92,7 +88,7 @@ struct http_context {
 };
 
 future<> set_server_init(http_context& ctx);
-future<> set_server_config(http_context& ctx, db::config& cfg);
+future<> set_server_config(http_context& ctx, const db::config& cfg);
 future<> unset_server_config(http_context& ctx);
 future<> set_server_snitch(http_context& ctx, sharded<locator::snitch_ptr>& snitch);
 future<> unset_server_snitch(http_context& ctx);
@@ -100,7 +96,7 @@ future<> set_server_storage_service(http_context& ctx, sharded<service::storage_
 future<> unset_server_storage_service(http_context& ctx);
 future<> set_server_sstables_loader(http_context& ctx, sharded<sstables_loader>& sst_loader);
 future<> unset_server_sstables_loader(http_context& ctx);
-future<> set_server_view_builder(http_context& ctx, sharded<db::view::view_builder>& vb, sharded<gms::gossiper>& g);
+future<> set_server_view_builder(http_context& ctx, sharded<db::view::view_builder>& vb);
 future<> unset_server_view_builder(http_context& ctx);
 future<> set_server_repair(http_context& ctx, sharded<repair_service>& repair, sharded<gms::gossip_address_map>& am);
 future<> unset_server_repair(http_context& ctx);
@@ -112,7 +108,7 @@ future<> set_server_authorization_cache(http_context& ctx, sharded<auth::service
 future<> unset_server_authorization_cache(http_context& ctx);
 future<> set_server_snapshot(http_context& ctx, sharded<db::snapshot_ctl>& snap_ctl);
 future<> unset_server_snapshot(http_context& ctx);
-future<> set_server_token_metadata(http_context& ctx, sharded<locator::shared_token_metadata>& tm, sharded<gms::gossiper>& g);
+future<> set_server_token_metadata(http_context& ctx, sharded<locator::shared_token_metadata>& tm);
 future<> unset_server_token_metadata(http_context& ctx);
 future<> set_server_gossip(http_context& ctx, sharded<gms::gossiper>& g);
 future<> unset_server_gossip(http_context& ctx);
@@ -124,14 +120,14 @@ future<> set_server_storage_proxy(http_context& ctx, sharded<service::storage_pr
 future<> unset_server_storage_proxy(http_context& ctx);
 future<> set_server_stream_manager(http_context& ctx, sharded<streaming::stream_manager>& sm);
 future<> unset_server_stream_manager(http_context& ctx);
-future<> set_hinted_handoff(http_context& ctx, sharded<service::storage_proxy>& p, sharded<gms::gossiper>& g);
+future<> set_hinted_handoff(http_context& ctx, sharded<service::storage_proxy>& p);
 future<> unset_hinted_handoff(http_context& ctx);
 future<> set_server_cache(http_context& ctx);
 future<> unset_server_cache(http_context& ctx);
 future<> set_server_compaction_manager(http_context& ctx, sharded<compaction_manager>& cm);
 future<> unset_server_compaction_manager(http_context& ctx);
 future<> set_server_done(http_context& ctx);
-future<> set_server_task_manager(http_context& ctx, sharded<tasks::task_manager>& tm, lw_shared_ptr<db::config> cfg, sharded<gms::gossiper>& gossiper);
+future<> set_server_task_manager(http_context& ctx, sharded<tasks::task_manager>& tm, lw_shared_ptr<db::config> cfg);
 future<> unset_server_task_manager(http_context& ctx);
 future<> set_server_task_manager_test(http_context& ctx, sharded<tasks::task_manager>& tm);
 future<> unset_server_task_manager_test(http_context& ctx);
@@ -145,7 +141,6 @@ future<> set_format_selector(http_context& ctx, db::sstables_format_selector& se
 future<> unset_format_selector(http_context& ctx);
 future<> set_server_cql_server_test(http_context& ctx, cql_transport::controller& ctl);
 future<> unset_server_cql_server_test(http_context& ctx);
-future<> set_server_service_levels(http_context& ctx, cql_transport::controller& ctl, sharded<cql3::query_processor>& qp);
 future<> set_server_commitlog(http_context& ctx, sharded<replica::database>&);
 future<> unset_server_commitlog(http_context& ctx);
 

api/collectd.cc

@@ -10,6 +10,7 @@
 #include "api/api-doc/collectd.json.hh"
 #include <seastar/core/scollectd.hh>
 #include <seastar/core/scollectd_api.hh>
+#include <boost/range/irange.hpp>
 #include <ranges>
 #include <regex>
 #include "api/api_init.hh"

api/column_family.cc

@@ -24,6 +24,9 @@
 #include "compaction/compaction_manager.hh"
 #include "unimplemented.hh"
 
+#include <boost/range/algorithm/copy.hpp>
+#include <boost/range/numeric.hpp>
+
 extern logging::logger apilog;
 
 namespace api {
@@ -48,9 +51,17 @@ std::tuple<sstring, sstring> parse_fully_qualified_cf_name(sstring name) {
     return std::make_tuple(name.substr(0, pos), name.substr(end));
 }
 
-table_info parse_table_info(const sstring& name, const replica::database& db) {
+table_id get_uuid(const sstring& ks, const sstring& cf, const replica::database& db) {
+    try {
+        return db.find_uuid(ks, cf);
+    } catch (replica::no_such_column_family& e) {
+        throw bad_param_exception(e.what());
+    }
+}
+
+table_id get_uuid(const sstring& name, const replica::database& db) {
     auto [ks, cf] = parse_fully_qualified_cf_name(name);
-    return table_info{ .name = cf, .id = validate_table(db, ks, cf) };
+    return get_uuid(ks, cf, db);
 }
 
 future<json::json_return_type>  get_cf_stats(http_context& ctx, const sstring& name,
@@ -67,11 +78,15 @@ future<json::json_return_type>  get_cf_stats(http_context& ctx,
     }, std::plus<int64_t>());
 }
 
-static future<json::json_return_type> for_tables_on_all_shards(http_context& ctx, std::vector<table_info> tables, std::function<future<>(replica::table&)> set) {
-    return do_with(std::move(tables), [&ctx, set] (const std::vector<table_info>& tables) {
-        return ctx.db.invoke_on_all([&tables, set] (replica::database& db) {
-            return parallel_for_each(tables, [&db, set] (const table_info& table) {
-                replica::table& t = db.find_column_family(table.id);
+static future<json::json_return_type> for_tables_on_all_shards(http_context& ctx, const sstring& keyspace, std::vector<sstring> tables, std::function<future<>(replica::table&)> set) {
+    if (tables.empty()) {
+        tables = map_keys(ctx.db.local().find_keyspace(keyspace).metadata().get()->cf_meta_data());
+    }
+
+    return do_with(keyspace, std::move(tables), [&ctx, set] (const sstring& keyspace, const std::vector<sstring>& tables) {
+        return ctx.db.invoke_on_all([&keyspace, &tables, set] (replica::database& db) {
+            return parallel_for_each(tables, [&db, &keyspace, set] (const sstring& table) {
+                replica::table& t = db.find_column_family(keyspace, table);
                 return set(t);
             });
         });
@@ -98,12 +113,12 @@ public:
     }
 };
 
-static future<json::json_return_type> set_tables_autocompaction(http_context& ctx, std::vector<table_info> tables, bool enabled) {
-    apilog.info("set_tables_autocompaction: enabled={} tables={}", enabled, tables);
+static future<json::json_return_type> set_tables_autocompaction(http_context& ctx, const sstring &keyspace, std::vector<sstring> tables, bool enabled) {
+    apilog.info("set_tables_autocompaction: enabled={} keyspace={} tables={}", enabled, keyspace, tables);
 
-    return ctx.db.invoke_on(0, [&ctx, tables = std::move(tables), enabled] (replica::database& db) {
+    return ctx.db.invoke_on(0, [&ctx, keyspace, tables = std::move(tables), enabled] (replica::database& db) {
         auto g = autocompaction_toggle_guard(db);
-        return for_tables_on_all_shards(ctx, tables, [enabled] (replica::table& cf) {
+        return for_tables_on_all_shards(ctx, keyspace, tables, [enabled] (replica::table& cf) {
             if (enabled) {
                 cf.enable_auto_compaction();
             } else {
@@ -114,9 +129,9 @@ static future<json::json_return_type> set_tables_autocompaction(http_context& ct
     });
 }
 
-static future<json::json_return_type> set_tables_tombstone_gc(http_context& ctx, std::vector<table_info> tables, bool enabled) {
-    apilog.info("set_tables_tombstone_gc: enabled={} tables={}", enabled, tables);
-    return for_tables_on_all_shards(ctx, std::move(tables), [enabled] (replica::table& t) {
+static future<json::json_return_type> set_tables_tombstone_gc(http_context& ctx, const sstring &keyspace, std::vector<sstring> tables, bool enabled) {
+    apilog.info("set_tables_tombstone_gc: enabled={} keyspace={} tables={}", enabled, keyspace, tables);
+    return for_tables_on_all_shards(ctx, keyspace, std::move(tables), [enabled] (replica::table& t) {
         t.set_tombstone_gc_enabled(enabled);
         return make_ready_future<>();
     });
@@ -131,7 +146,7 @@ static future<json::json_return_type>  get_cf_stats_count(http_context& ctx, con
 
 static future<json::json_return_type>  get_cf_stats_sum(http_context& ctx, const sstring& name,
         utils::timed_rate_moving_average_summary_and_histogram replica::column_family_stats::*f) {
-    auto uuid = parse_table_info(name, ctx.db.local()).id;
+    auto uuid = get_uuid(name, ctx.db.local());
     return ctx.db.map_reduce0([uuid, f](replica::database& db) {
         // Histograms information is sample of the actual load
         // so to get an estimation of sum, we multiply the mean
@@ -154,7 +169,7 @@ static future<json::json_return_type>  get_cf_stats_count(http_context& ctx,
 
 static future<json::json_return_type>  get_cf_histogram(http_context& ctx, const sstring& name,
         utils::timed_rate_moving_average_and_histogram replica::column_family_stats::*f) {
-    auto uuid = parse_table_info(name, ctx.db.local()).id;
+    auto uuid = get_uuid(name, ctx.db.local());
     return ctx.db.map_reduce0([f, uuid](const replica::database& p) {
         return (p.find_column_family(uuid).get_stats().*f).hist;},
             utils::ihistogram(),
@@ -166,7 +181,7 @@ static future<json::json_return_type>  get_cf_histogram(http_context& ctx, const
 
 static future<json::json_return_type>  get_cf_histogram(http_context& ctx, const sstring& name,
         utils::timed_rate_moving_average_summary_and_histogram replica::column_family_stats::*f) {
-    auto uuid = parse_table_info(name, ctx.db.local()).id;
+    auto uuid = get_uuid(name, ctx.db.local());
     return ctx.db.map_reduce0([f, uuid](const replica::database& p) {
         return (p.find_column_family(uuid).get_stats().*f).hist;},
             utils::ihistogram(),
@@ -187,13 +202,13 @@ static future<json::json_return_type> get_cf_histogram(http_context& ctx, utils:
     return ctx.db.map(fun).then([](const std::vector<utils::ihistogram> &res) {
         std::vector<httpd::utils_json::histogram> r;
         std::ranges::copy(res | std::views::transform(to_json), std::back_inserter(r));
-        return make_ready_future<json::json_return_type>(std::move(r));
+        return make_ready_future<json::json_return_type>(r);
     });
 }
 
 static future<json::json_return_type>  get_cf_rate_and_histogram(http_context& ctx, const sstring& name,
         utils::timed_rate_moving_average_summary_and_histogram replica::column_family_stats::*f) {
-    auto uuid = parse_table_info(name, ctx.db.local()).id;
+    auto uuid = get_uuid(name, ctx.db.local());
     return ctx.db.map_reduce0([f, uuid](const replica::database& p) {
         return (p.find_column_family(uuid).get_stats().*f).rate();},
             utils::rate_moving_average_and_histogram(),
@@ -250,29 +265,48 @@ static integral_ratio_holder mean_partition_size(replica::column_family& cf) {
     return res;
 }
 
-static auto count_bytes_on_disk(const replica::column_family& cf, bool total) {
-    uint64_t bytes_on_disk = 0;
-    auto sstables = (total) ? cf.get_sstables_including_compacted_undeleted() : cf.get_sstables();
-    for (auto t : *sstables) {
-        bytes_on_disk += t->bytes_on_disk();
+static std::unordered_map<sstring, uint64_t> merge_maps(std::unordered_map<sstring, uint64_t> a,
+        const std::unordered_map<sstring, uint64_t>& b) {
+    a.insert(b.begin(), b.end());
+    return a;
+}
+
+static json::json_return_type sum_map(const std::unordered_map<sstring, uint64_t>& val) {
+    uint64_t res = 0;
+    for (auto i : val) {
+        res += i.second;
     }
-    return bytes_on_disk;
+    return res;
 }
 
 static future<json::json_return_type>  sum_sstable(http_context& ctx, const sstring name, bool total) {
-    return map_reduce_cf_raw(ctx, name, uint64_t(0), [total](replica::column_family& cf) {
-        return count_bytes_on_disk(cf, total);
-    }, std::plus<>()).then([] (uint64_t val) {
-        return make_ready_future<json::json_return_type>(val);
+    auto uuid = get_uuid(name, ctx.db.local());
+    return ctx.db.map_reduce0([uuid, total](replica::database& db) {
+        std::unordered_map<sstring, uint64_t> m;
+        auto sstables = (total) ? db.find_column_family(uuid).get_sstables_including_compacted_undeleted() :
+                db.find_column_family(uuid).get_sstables();
+        for (auto t : *sstables) {
+            m[t->get_filename()] = t->bytes_on_disk();
+        }
+        return m;
+    }, std::unordered_map<sstring, uint64_t>(), merge_maps).
+            then([](const std::unordered_map<sstring, uint64_t>& val) {
+        return sum_map(val);
     });
 }
 
 
 static future<json::json_return_type> sum_sstable(http_context& ctx, bool total) {
-    return map_reduce_cf_raw(ctx, uint64_t(0), [total](replica::column_family& cf) {
-        return count_bytes_on_disk(cf, total);
-    }, std::plus<>()).then([] (uint64_t val) {
-        return make_ready_future<json::json_return_type>(val);
+    return map_reduce_cf_raw(ctx, std::unordered_map<sstring, uint64_t>(), [total](replica::column_family& cf) {
+        std::unordered_map<sstring, uint64_t> m;
+        auto sstables = (total) ? cf.get_sstables_including_compacted_undeleted() :
+                cf.get_sstables();
+        for (auto t : *sstables) {
+            m[t->get_filename()] = t->bytes_on_disk();
+        }
+        return m;
+    },merge_maps).then([](const std::unordered_map<sstring, uint64_t>& val) {
+        return sum_map(val);
     });
 }
 
@@ -884,84 +918,94 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_auto_compaction.set(r, [&ctx] (const_req req) {
-        auto uuid = parse_table_info(req.get_path_param("name"), ctx.db.local()).id;
+        auto uuid = get_uuid(req.get_path_param("name"), ctx.db.local());
         replica::column_family& cf = ctx.db.local().find_column_family(uuid);
         return !cf.is_auto_compaction_disabled_by_user();
     });
 
     cf::enable_auto_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) {
         apilog.info("column_family/enable_auto_compaction: name={}", req->get_path_param("name"));
-        auto ti = parse_table_info(req->get_path_param("name"), ctx.db.local());
-        return set_tables_autocompaction(ctx, {std::move(ti)}, true);
+        auto [ks, cf] = parse_fully_qualified_cf_name(req->get_path_param("name"));
+        validate_table(ctx, ks, cf);
+        return set_tables_autocompaction(ctx, ks, {std::move(cf)}, true);
     });
 
     cf::disable_auto_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) {
         apilog.info("column_family/disable_auto_compaction: name={}", req->get_path_param("name"));
-        auto ti = parse_table_info(req->get_path_param("name"), ctx.db.local());
-        return set_tables_autocompaction(ctx, {std::move(ti)}, false);
+        auto [ks, cf] = parse_fully_qualified_cf_name(req->get_path_param("name"));
+        validate_table(ctx, ks, cf);
+        return set_tables_autocompaction(ctx, ks, {std::move(cf)}, false);
     });
 
     ss::enable_auto_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        auto [keyspace, tables] = parse_table_infos(ctx, *req);
+        auto keyspace = validate_keyspace(ctx, req);
+        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
+
         apilog.info("enable_auto_compaction: keyspace={} tables={}", keyspace, tables);
-        return set_tables_autocompaction(ctx, std::move(tables), true);
+        return set_tables_autocompaction(ctx, keyspace, tables, true);
     });
 
     ss::disable_auto_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        auto [keyspace, tables] = parse_table_infos(ctx, *req);
+        auto keyspace = validate_keyspace(ctx, req);
+        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
+
         apilog.info("disable_auto_compaction: keyspace={} tables={}", keyspace, tables);
-        return set_tables_autocompaction(ctx, std::move(tables), false);
+        return set_tables_autocompaction(ctx, keyspace, tables, false);
     });
 
     cf::get_tombstone_gc.set(r, [&ctx] (const_req req) {
-        auto uuid = parse_table_info(req.get_path_param("name"), ctx.db.local()).id;
+        auto uuid = get_uuid(req.get_path_param("name"), ctx.db.local());
         replica::table& t = ctx.db.local().find_column_family(uuid);
         return t.tombstone_gc_enabled();
     });
 
     cf::enable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
         apilog.info("column_family/enable_tombstone_gc: name={}", req->get_path_param("name"));
-        auto ti = parse_table_info(req->get_path_param("name"), ctx.db.local());
-        return set_tables_tombstone_gc(ctx, {std::move(ti)}, true);
+        auto [ks, cf] = parse_fully_qualified_cf_name(req->get_path_param("name"));
+        validate_table(ctx, ks, cf);
+        return set_tables_tombstone_gc(ctx, ks, {std::move(cf)}, true);
     });
 
     cf::disable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
         apilog.info("column_family/disable_tombstone_gc: name={}", req->get_path_param("name"));
-        auto ti = parse_table_info(req->get_path_param("name"), ctx.db.local());
-        return set_tables_tombstone_gc(ctx, {std::move(ti)}, false);
+        auto [ks, cf] = parse_fully_qualified_cf_name(req->get_path_param("name"));
+        validate_table(ctx, ks, cf);
+        return set_tables_tombstone_gc(ctx, ks, {std::move(cf)}, false);
     });
 
     ss::enable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        auto [keyspace, tables] = parse_table_infos(ctx, *req);
+        auto keyspace = validate_keyspace(ctx, req);
+        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
+
         apilog.info("enable_tombstone_gc: keyspace={} tables={}", keyspace, tables);
-        return set_tables_tombstone_gc(ctx, std::move(tables), true);
+        return set_tables_tombstone_gc(ctx, keyspace, tables, true);
     });
 
     ss::disable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        auto [keyspace, tables] = parse_table_infos(ctx, *req);
+        auto keyspace = validate_keyspace(ctx, req);
+        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
+
         apilog.info("disable_tombstone_gc: keyspace={} tables={}", keyspace, tables);
-        return set_tables_tombstone_gc(ctx, std::move(tables), false);
+        return set_tables_tombstone_gc(ctx, keyspace, tables, false);
     });
 
     cf::get_built_indexes.set(r, [&ctx, &sys_ks](std::unique_ptr<http::request> req) {
         auto ks_cf = parse_fully_qualified_cf_name(req->get_path_param("name"));
         auto&& ks = std::get<0>(ks_cf);
         auto&& cf_name = std::get<1>(ks_cf);
-        // Use of load_built_views() as filtering table should be in sync with
-        // built_indexes_virtual_reader filtering with BUILT_VIEWS table
-        return sys_ks.local().load_built_views().then([ks, cf_name, &ctx](const std::vector<db::system_keyspace::view_name>& vb) mutable {
+        return sys_ks.local().load_view_build_progress().then([ks, cf_name, &ctx](const std::vector<db::system_keyspace_view_build_progress>& vb) mutable {
             std::set<sstring> vp;
             for (auto b : vb) {
-                if (b.first == ks) {
-                    vp.insert(b.second);
+                if (b.view.first == ks) {
+                    vp.insert(b.view.second);
                 }
             }
             std::vector<sstring> res;
-            auto uuid = validate_table(ctx.db.local(), ks, cf_name);
+            auto uuid = get_uuid(ks, cf_name, ctx.db.local());
             replica::column_family& cf = ctx.db.local().find_column_family(uuid);
             res.reserve(cf.get_index_manager().list_indexes().size());
             for (auto&& i : cf.get_index_manager().list_indexes()) {
-                if (vp.contains(secondary_index::index_table_name(i.metadata().name()))) {
+                if (!vp.contains(secondary_index::index_table_name(i.metadata().name()))) {
                     res.emplace_back(i.metadata().name());
                 }
             }
@@ -984,7 +1028,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::get_compression_ratio.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        auto uuid = parse_table_info(req->get_path_param("name"), ctx.db.local()).id;
+        auto uuid = get_uuid(req->get_path_param("name"), ctx.db.local());
 
         return ctx.db.map_reduce(sum_ratio<double>(), [uuid](replica::database& db) {
             replica::column_family& cf = db.find_column_family(uuid);
@@ -1007,17 +1051,17 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
     });
 
     cf::set_compaction_strategy_class.set(r, [&ctx](std::unique_ptr<http::request> req) {
-        auto ti = parse_table_info(req->get_path_param("name"), ctx.db.local());
+        auto [ks, cf] = parse_fully_qualified_cf_name(req->get_path_param("name"));
         sstring strategy = req->get_query_param("class_name");
         apilog.info("column_family/set_compaction_strategy_class: name={} strategy={}", req->get_path_param("name"), strategy);
-        return for_tables_on_all_shards(ctx, {std::move(ti)}, [strategy] (replica::table& cf) {
+        return for_tables_on_all_shards(ctx, ks, {std::move(cf)}, [strategy] (replica::table& cf) {
             cf.set_compaction_strategy(sstables::compaction_strategy::type(strategy));
             return make_ready_future<>();
         });
     });
 
     cf::get_compaction_strategy_class.set(r, [&ctx](const_req req) {
-        return ctx.db.local().find_column_family(parse_table_info(req.get_path_param("name"), ctx.db.local()).id).get_compaction_strategy().name();
+        return ctx.db.local().find_column_family(get_uuid(req.get_path_param("name"), ctx.db.local())).get_compaction_strategy().name();
     });
 
     cf::set_compression_parameters.set(r, [](std::unique_ptr<http::request> req) {
@@ -1042,11 +1086,11 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
 
     cf::get_sstables_for_key.set(r, [&ctx](std::unique_ptr<http::request> req) {
         auto key = req->get_query_param("key");
-        auto uuid = parse_table_info(req->get_path_param("name"), ctx.db.local()).id;
+        auto uuid = get_uuid(req->get_path_param("name"), ctx.db.local());
 
         return ctx.db.map_reduce0([key, uuid] (replica::database& db) -> future<std::unordered_set<sstring>> {
             auto sstables = co_await db.find_column_family(uuid).get_sstables_by_partition_key(key);
-            co_return sstables | std::views::transform([] (auto s) -> sstring { return fmt::to_string(s->get_filename()); }) | std::ranges::to<std::unordered_set>();
+            co_return sstables | std::views::transform([] (auto s) { return s->get_filename(); }) | std::ranges::to<std::unordered_set>();
         }, std::unordered_set<sstring>(),
         [](std::unordered_set<sstring> a, std::unordered_set<sstring>&& b) mutable {
             a.merge(b);

api/column_family.hh

@@ -22,12 +22,12 @@ namespace api {
 void set_column_family(http_context& ctx, httpd::routes& r, sharded<db::system_keyspace>& sys_ks);
 void unset_column_family(http_context& ctx, httpd::routes& r);
 
-table_info parse_table_info(const sstring& name, const replica::database& db);
+table_id get_uuid(const sstring& name, const replica::database& db);
 
 template<class Mapper, class I, class Reducer>
 future<I> map_reduce_cf_raw(http_context& ctx, const sstring& name, I init,
         Mapper mapper, Reducer reducer) {
-    auto uuid = parse_table_info(name, ctx.db.local()).id;
+    auto uuid = get_uuid(name, ctx.db.local());
     using mapper_type = std::function<std::unique_ptr<std::any>(replica::database&)>;
     using reducer_type = std::function<std::unique_ptr<std::any>(std::unique_ptr<std::any>, std::unique_ptr<std::any>)>;
     return ctx.db.map_reduce0(mapper_type([mapper, uuid](replica::database& db) {

api/compaction_manager.cc

@@ -111,12 +111,13 @@ void set_compaction_manager(http_context& ctx, routes& r, sharded<compaction_man
     });
 
     cm::stop_keyspace_compaction.set(r, [&ctx] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
-        auto [ks_name, tables] = parse_table_infos(ctx, *req, "tables");
+        auto ks_name = validate_keyspace(ctx, req);
+        auto table_names = parse_tables(ks_name, ctx, req->query_parameters, "tables");
         auto type = req->get_query_param("type");
         co_await ctx.db.invoke_on_all([&] (replica::database& db) {
             auto& cm = db.get_compaction_manager();
-            return parallel_for_each(tables, [&] (const table_info& ti) {
-                auto& t = db.find_column_family(ti.id);
+            return parallel_for_each(table_names, [&] (sstring& table_name) {
+                auto& t = db.find_column_family(ks_name, table_name);
                 return t.parallel_foreach_table_state([&] (compaction::table_state& ts) {
                     return cm.stop_compaction(type, &ts);
                 });
@@ -203,6 +204,14 @@ void set_compaction_manager(http_context& ctx, routes& r, sharded<compaction_man
         int value = cm.local().throughput_mbs();
         return make_ready_future<json::json_return_type>(value);
     });
+
+    ss::set_compaction_throughput_mb_per_sec.set(r, [](std::unique_ptr<http::request> req) {
+        //TBD
+        unimplemented();
+        auto value = req->get_query_param("value");
+        return make_ready_future<json::json_return_type>(json_void());
+    });
+
 }
 
 void unset_compaction_manager(http_context& ctx, routes& r) {
@@ -218,6 +227,7 @@ void unset_compaction_manager(http_context& ctx, routes& r) {
     cm::get_compaction_history.unset(r);
     cm::get_compaction_info.unset(r);
     ss::get_compaction_throughput_mb_per_sec.unset(r);
+    ss::set_compaction_throughput_mb_per_sec.unset(r);
 }
 
 }

api/config.cc

@@ -14,7 +14,6 @@
 #include "replica/database.hh"
 #include "db/config.hh"
 #include <sstream>
-#include <fmt/ranges.h>
 #include <boost/algorithm/string/replace.hpp>
 #include <seastar/http/exception.hh>
 
@@ -84,7 +83,7 @@ future<> get_config_swagger_entry(std::string_view name, const std::string& desc
 
 namespace cs = httpd::config_json;
 
-void set_config(std::shared_ptr < api_registry_builder20 > rb, http_context& ctx, routes& r, db::config& cfg, bool first) {
+void set_config(std::shared_ptr < api_registry_builder20 > rb, http_context& ctx, routes& r, const db::config& cfg, bool first) {
     rb->register_function(r, [&cfg, first] (output_stream<char>& os) {
         return do_with(first, [&os, &cfg] (bool& first) {
             auto f = make_ready_future();
@@ -194,17 +193,6 @@ void set_config(std::shared_ptr < api_registry_builder20 > rb, http_context& ctx
         return cfg.saved_caches_directory();
     });
 
-    ss::set_compaction_throughput_mb_per_sec.set(r, [&cfg](std::unique_ptr<http::request> req) mutable {
-        api::req_param<uint32_t> value(*req, "value", 0);
-        cfg.compaction_throughput_mb_per_sec(value.value, utils::config_file::config_source::API);
-        return make_ready_future<json::json_return_type>(json::json_void());
-    });
-
-    ss::set_stream_throughput_mb_per_sec.set(r, [&cfg](std::unique_ptr<http::request> req) mutable {
-        api::req_param<uint32_t> value(*req, "value", 0);
-        cfg.stream_io_throughput_mb_per_sec(value.value, utils::config_file::config_source::API);
-        return make_ready_future<json::json_return_type>(json::json_void());
-    });
 }
 
 void unset_config(http_context& ctx, routes& r) {
@@ -225,8 +213,6 @@ void unset_config(http_context& ctx, routes& r) {
     sp::set_truncate_rpc_timeout.unset(r);
     ss::get_all_data_file_locations.unset(r);
     ss::get_saved_caches_location.unset(r);
-    ss::set_compaction_throughput_mb_per_sec.unset(r);
-    ss::set_stream_throughput_mb_per_sec.unset(r);
 }
 
 }

api/config.hh

@@ -13,6 +13,6 @@
 
 namespace api {
 
-void set_config(std::shared_ptr<httpd::api_registry_builder20> rb, http_context& ctx, httpd::routes& r, db::config& cfg, bool first = false);
+void set_config(std::shared_ptr<httpd::api_registry_builder20> rb, http_context& ctx, httpd::routes& r, const db::config& cfg, bool first = false);
 void unset_config(http_context& ctx, httpd::routes& r);
 }

api/cql_server_test.cc

@@ -6,8 +6,6 @@
  * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
  */
 
-#include "build_mode.hh"
-
 #ifndef SCYLLA_BUILD_MODE_RELEASE
 
 #include <seastar/core/coroutine.hh>
@@ -28,24 +26,21 @@ struct connection_sl_params : public json::json_base {
     json::json_element<sstring> _role_name;
     json::json_element<sstring> _workload_type;
     json::json_element<sstring> _timeout;
-    json::json_element<sstring> _scheduling_group;
 
-    connection_sl_params(const sstring& role_name, const sstring& workload_type, const sstring& timeout, const sstring& scheduling_group) {
+    connection_sl_params(const sstring& role_name, const sstring& workload_type, const sstring& timeout) {
         _role_name = role_name;
         _workload_type = workload_type;
         _timeout = timeout;
-        _scheduling_group = scheduling_group;
         register_params();
     }
 
     connection_sl_params(const connection_sl_params& params)
-        : connection_sl_params(params._role_name(), params._workload_type(), params._timeout(), params._scheduling_group()) {}
+        : connection_sl_params(params._role_name(), params._workload_type(), params._timeout()) {}
 
     void register_params() {
         add(&_role_name, "role_name");
         add(&_workload_type, "workload_type");
         add(&_timeout, "timeout");
-        add(&_scheduling_group, "scheduling_group");
     }    
 };
 
@@ -59,8 +54,7 @@ void set_cql_server_test(http_context& ctx, seastar::httpd::routes& r, cql_trans
             return connection_sl_params(
                     std::move(params.role_name), 
                     sstring(qos::service_level_options::to_string(params.workload_type)), 
-                    to_string(cql_duration(months_counter{0}, days_counter{0}, nanoseconds_counter{nanos})),
-                    std::move(params.scheduling_group_name));
+                    to_string(cql_duration(months_counter{0}, days_counter{0}, nanoseconds_counter{nanos})));
         });
         co_return result;
     });

api/failure_detector.cc

@@ -22,10 +22,10 @@ void set_failure_detector(http_context& ctx, routes& r, gms::gossiper& g) {
         return g.container().invoke_on(0, [] (gms::gossiper& g) {
             std::vector<fd::endpoint_state> res;
             res.reserve(g.num_endpoints());
-            g.for_each_endpoint_state([&] (const gms::endpoint_state& eps) {
+            g.for_each_endpoint_state([&] (const gms::inet_address& addr, const gms::endpoint_state& eps) {
                 fd::endpoint_state val;
-                val.addrs = fmt::to_string(eps.get_ip());
-                val.is_alive = g.is_alive(eps.get_host_id());
+                val.addrs = fmt::to_string(addr);
+                val.is_alive = g.is_alive(addr);
                 val.generation = eps.get_heart_beat_state().get_generation().value();
                 val.version = eps.get_heart_beat_state().get_heart_beat_version().value();
                 val.update_time = eps.get_update_timestamp().time_since_epoch().count();
@@ -65,8 +65,8 @@ void set_failure_detector(http_context& ctx, routes& r, gms::gossiper& g) {
     fd::get_simple_states.set(r, [&g] (std::unique_ptr<request> req) {
         return g.container().invoke_on(0, [] (gms::gossiper& g) {
             std::map<sstring, sstring> nodes_status;
-            g.for_each_endpoint_state([&] (const gms::endpoint_state& es) {
-                nodes_status.emplace(fmt::to_string(es.get_ip()), g.is_alive(es.get_host_id()) ? "UP" : "DOWN");
+            g.for_each_endpoint_state([&] (const gms::inet_address& node, const gms::endpoint_state&) {
+                nodes_status.emplace(fmt::to_string(node), g.is_alive(node) ? "UP" : "DOWN");
             });
             return make_ready_future<json::json_return_type>(map_to_key_value<fd::mapper>(nodes_status));
         });
@@ -81,7 +81,7 @@ void set_failure_detector(http_context& ctx, routes& r, gms::gossiper& g) {
 
     fd::get_endpoint_state.set(r, [&g] (std::unique_ptr<request> req) {
         return g.container().invoke_on(0, [req = std::move(req)] (gms::gossiper& g) {
-            auto state = g.get_endpoint_state_ptr(g.get_host_id(gms::inet_address(req->get_path_param("addr"))));
+            auto state = g.get_endpoint_state_ptr(gms::inet_address(req->get_path_param("addr")));
             if (!state) {
                 return make_ready_future<json::json_return_type>(format("unknown endpoint {}", req->get_path_param("addr")));
             }

api/gossiper.cc

@@ -35,32 +35,37 @@ void set_gossiper(http_context& ctx, routes& r, gms::gossiper& g) {
         gms::inet_address ep(req->get_path_param("addr"));
         // synchronize unreachable_members on all shards
         co_await g.get_unreachable_members_synchronized();
-        co_return g.get_endpoint_downtime(g.get_host_id(ep));
+        co_return g.get_endpoint_downtime(ep);
     });
 
     httpd::gossiper_json::get_current_generation_number.set(r, [&g] (std::unique_ptr<http::request> req) {
         gms::inet_address ep(req->get_path_param("addr"));
-        return g.get_current_generation_number(g.get_host_id(ep)).then([] (gms::generation_type res) {
+        return g.get_current_generation_number(ep).then([] (gms::generation_type res) {
             return make_ready_future<json::json_return_type>(res.value());
         });
     });
 
     httpd::gossiper_json::get_current_heart_beat_version.set(r, [&g] (std::unique_ptr<http::request> req) {
         gms::inet_address ep(req->get_path_param("addr"));
-        return g.get_current_heart_beat_version(g.get_host_id(ep)).then([] (gms::version_type res) {
+        return g.get_current_heart_beat_version(ep).then([] (gms::version_type res) {
             return make_ready_future<json::json_return_type>(res.value());
         });
     });
 
     httpd::gossiper_json::assassinate_endpoint.set(r, [&g](std::unique_ptr<http::request> req) {
-        return g.assassinate_endpoint(req->get_path_param("addr")).then([] {
+        if (req->get_query_param("unsafe") != "True") {
+            return g.assassinate_endpoint(req->get_path_param("addr")).then([] {
+                return make_ready_future<json::json_return_type>(json_void());
+            });
+        }
+        return g.unsafe_assassinate_endpoint(req->get_path_param("addr")).then([] {
             return make_ready_future<json::json_return_type>(json_void());
         });
     });
 
     httpd::gossiper_json::force_remove_endpoint.set(r, [&g](std::unique_ptr<http::request> req) {
         gms::inet_address ep(req->get_path_param("addr"));
-        return g.force_remove_endpoint(g.get_host_id(ep), gms::null_permit_id).then([] () {
+        return g.force_remove_endpoint(ep, gms::null_permit_id).then([] {
             return make_ready_future<json::json_return_type>(json_void());
         });
     });

api/hinted_handoff.cc

@@ -14,7 +14,6 @@
 
 #include "gms/inet_address.hh"
 #include "service/storage_proxy.hh"
-#include "gms/gossiper.hh"
 
 namespace api {
 
@@ -22,18 +21,18 @@ using namespace json;
 using namespace seastar::httpd;
 namespace hh = httpd::hinted_handoff_json;
 
-void set_hinted_handoff(http_context& ctx, routes& r, sharded<service::storage_proxy>& proxy, sharded<gms::gossiper>& g) {
-    hh::create_hints_sync_point.set(r, [&proxy, &g] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
-        auto parse_hosts_list = [&g] (sstring arg) {
+void set_hinted_handoff(http_context& ctx, routes& r, sharded<service::storage_proxy>& proxy) {
+    hh::create_hints_sync_point.set(r, [&proxy] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
+        auto parse_hosts_list = [] (sstring arg) {
             std::vector<sstring> hosts_str = split(arg, ",");
-            std::vector<locator::host_id> hosts;
+            std::vector<gms::inet_address> hosts;
             hosts.reserve(hosts_str.size());
 
             for (const auto& host_str : hosts_str) {
                 try {
                     gms::inet_address host;
                     host = gms::inet_address(host_str);
-                    hosts.push_back(g.local().get_host_id(host));
+                    hosts.push_back(host);
                 } catch (std::exception& e) {
                     throw httpd::bad_param_exception(format("Failed to parse host address {}: {}", host_str, e.what()));
                 }
@@ -42,7 +41,7 @@ void set_hinted_handoff(http_context& ctx, routes& r, sharded<service::storage_p
             return hosts;
         };
 
-        std::vector<locator::host_id> target_hosts = parse_hosts_list(req->get_query_param("target_hosts"));
+        std::vector<gms::inet_address> target_hosts = parse_hosts_list(req->get_query_param("target_hosts"));
         return proxy.local().create_hint_sync_point(std::move(target_hosts)).then([] (db::hints::sync_point sync_point) {
             return json::json_return_type(sync_point.encode());
         });

api/hinted_handoff.hh

@@ -10,13 +10,12 @@
 
 #include <seastar/core/sharded.hh>
 #include "api/api_init.hh"
-#include "gms/gossiper.hh"
 
 namespace service { class storage_proxy; }
 
 namespace api {
 
-void set_hinted_handoff(http_context& ctx, httpd::routes& r, sharded<service::storage_proxy>& p, sharded<gms::gossiper>& g);
+void set_hinted_handoff(http_context& ctx, httpd::routes& r, sharded<service::storage_proxy>& p);
 void unset_hinted_handoff(http_context& ctx, httpd::routes& r);
 
 }

api/messaging_service.cc

@@ -114,7 +114,7 @@ void set_messaging_service(http_context& ctx, routes& r, sharded<netw::messaging
     }));
 
     get_version.set(r, [&ms](const_req req) {
-        return ms.local().current_version;
+        return ms.local().get_raw_version(gms::inet_address(req.get_query_param("addr")));
     });
 
     get_dropped_messages_by_ver.set(r, [&ms](std::unique_ptr<request> req) {
@@ -148,7 +148,7 @@ void set_messaging_service(http_context& ctx, routes& r, sharded<netw::messaging
     hf::inject_disconnect.set(r, [&ms] (std::unique_ptr<request> req) -> future<json::json_return_type> {
         auto ip = msg_addr(req->get_path_param("ip"));
         co_await ms.invoke_on_all([ip] (netw::messaging_service& ms) {
-            ms.remove_rpc_client(ip, std::nullopt);
+            ms.remove_rpc_client(ip);
         });
         co_return json::json_void();
     });

api/service_levels.cc

@@ -1,63 +0,0 @@
-/*
- * Copyright (C) 2023-present ScyllaDB
- */
-
-/*
- * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
- */
-
-#include "service_levels.hh"
-#include "api/api-doc/service_levels.json.hh"
-#include "cql3/query_processor.hh"
-#include "cql3/untyped_result_set.hh"
-#include "db/consistency_level_type.hh"
-#include <seastar/json/json_elements.hh>
-#include "transport/controller.hh"
-#include <unordered_map>
-
-
-namespace api {
-
-namespace sl = httpd::service_levels_json;
-using namespace json;
-using namespace seastar::httpd;
-
-
-void set_service_levels(http_context& ctx, routes& r, cql_transport::controller& ctl, sharded<cql3::query_processor>& qp) {
-    sl::do_switch_tenants.set(r, [&ctl] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
-        co_await ctl.update_connections_scheduling_group();
-        co_return json_void();
-    });
-
-    sl::count_connections.set(r, [&qp] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
-        auto connections = co_await qp.local().execute_internal(
-            "SELECT username, scheduling_group FROM system.clients WHERE client_type='cql' ALLOW FILTERING",
-            db::consistency_level::LOCAL_ONE,
-            cql3::query_processor::cache_internal::no
-        );
-
-        using connections_per_user = std::unordered_map<sstring, uint64_t>;
-        using connections_per_scheduling_group = std::unordered_map<sstring, connections_per_user>;
-        connections_per_scheduling_group result;
-
-        for (auto it = connections->begin(); it != connections->end(); it++) {
-            auto user = it->get_as<sstring>("username");
-            auto shg = it->get_as<sstring>("scheduling_group");
-
-            if (result.contains(shg)) {
-                result[shg][user]++;
-            }
-            else {
-                result[shg] = {{user, 1}};
-            }
-        }
-
-        co_return result;
-    });
-
-}
-
-
-
-
-}

api/service_levels.hh

@@ -1,17 +0,0 @@
-/*
- * Copyright (C) 2023-present ScyllaDB
- */
-
-/*
- * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
- */
-
-#pragma once
-
-#include "api/api_init.hh"
-
-namespace api {
-
-void set_service_levels(http_context& ctx, httpd::routes& r, cql_transport::controller& ctl, sharded<cql3::query_processor>& qp);
-
-}

api/storage_service.hh

@@ -43,20 +43,26 @@ sstring validate_keyspace(const http_context& ctx, sstring ks_name);
 // containing the description of the respective keyspace error.
 sstring validate_keyspace(const http_context& ctx, const std::unique_ptr<http::request>& req);
 
-// verify that the keyspace:table is found, otherwise a bad_param_exception exception is thrown
-// returns the table_id of the table if found
-table_id validate_table(const replica::database& db, sstring ks_name, sstring table_name);
+// verify that the table parameter is found, otherwise a bad_param_exception exception is thrown
+// containing the description of the respective table error.
+void validate_table(const http_context& ctx, sstring ks_name, sstring table_name);
+
+// splits a request parameter assumed to hold a comma-separated list of table names
+// verify that the tables are found, otherwise a bad_param_exception exception is thrown
+// containing the description of the respective no_such_column_family error.
+// Returns an empty vector if no parameter was found.
+// If the parameter is found and empty, returns a list of all table names in the keyspace.
+std::vector<sstring> parse_tables(const sstring& ks_name, const http_context& ctx, const std::unordered_map<sstring, sstring>& query_params, sstring param_name);
 
 // splits a request parameter assumed to hold a comma-separated list of table names
 // verify that the tables are found, otherwise a bad_param_exception exception is thrown
 // containing the description of the respective no_such_column_family error.
 // Returns a vector of all table infos given by the parameter, or
 // if the parameter is not found or is empty, returns a list of all table infos in the keyspace.
+std::vector<table_info> parse_table_infos(const sstring& ks_name, const http_context& ctx, const std::unordered_map<sstring, sstring>& query_params, sstring param_name);
 
 std::vector<table_info> parse_table_infos(const sstring& ks_name, const http_context& ctx, sstring value);
 
-std::pair<sstring, std::vector<table_info>> parse_table_infos(const http_context& ctx, const http::request& req, sstring cf_param_name = "cf");
-
 struct scrub_info {
     sstables::compaction_type_options::scrub opts;
     sstring keyspace;
@@ -69,7 +75,7 @@ void set_storage_service(http_context& ctx, httpd::routes& r, sharded<service::s
 void unset_storage_service(http_context& ctx, httpd::routes& r);
 void set_sstables_loader(http_context& ctx, httpd::routes& r, sharded<sstables_loader>& sst_loader);
 void unset_sstables_loader(http_context& ctx, httpd::routes& r);
-void set_view_builder(http_context& ctx, httpd::routes& r, sharded<db::view::view_builder>& vb, sharded<gms::gossiper>& g);
+void set_view_builder(http_context& ctx, httpd::routes& r, sharded<db::view::view_builder>& vb);
 void unset_view_builder(http_context& ctx, httpd::routes& r);
 void set_repair(http_context& ctx, httpd::routes& r, sharded<repair_service>& repair, sharded<gms::gossip_address_map>& am);
 void unset_repair(http_context& ctx, httpd::routes& r);

api/stream_manager.cc

@@ -11,7 +11,6 @@
 #include "streaming/stream_result_future.hh"
 #include "api/api.hh"
 #include "api/api-doc/stream_manager.json.hh"
-#include "api/api-doc/storage_service.json.hh"
 #include <vector>
 #include <rapidjson/document.h>
 #include "gms/gossiper.hh"
@@ -19,7 +18,6 @@
 namespace api {
 using namespace seastar::httpd;
 
-namespace ss = httpd::storage_service_json;
 namespace hs = httpd::stream_manager_json;
 
 static void set_summaries(const std::vector<streaming::stream_summary>& from,
@@ -150,11 +148,6 @@ void set_stream_manager(http_context& ctx, routes& r, sharded<streaming::stream_
             return make_ready_future<json::json_return_type>(res);
         });
     });
-
-    ss::get_stream_throughput_mb_per_sec.set(r, [&sm](std::unique_ptr<http::request> req) {
-        auto value = sm.local().throughput_mbs();
-        return make_ready_future<json::json_return_type>(value);
-    });
 }
 
 void unset_stream_manager(http_context& ctx, routes& r) {
@@ -164,7 +157,6 @@ void unset_stream_manager(http_context& ctx, routes& r) {
     hs::get_all_total_incoming_bytes.unset(r);
     hs::get_total_outgoing_bytes.unset(r);
     hs::get_all_total_outgoing_bytes.unset(r);
-    ss::get_stream_throughput_mb_per_sec.unset(r);
 }
 
 }

api/system.cc

@@ -13,7 +13,6 @@
 #include "db/sstables-format-selector.hh"
 
 #include <rapidjson/document.h>
-#include <boost/lexical_cast.hpp>
 #include <seastar/core/reactor.hh>
 #include <seastar/core/metrics_api.hh>
 #include <seastar/core/relabel_config.hh>

api/task_manager.cc

@@ -14,7 +14,6 @@
 #include "api/api.hh"
 #include "api/api-doc/task_manager.json.hh"
 #include "db/system_keyspace.hh"
-#include "gms/gossiper.hh"
 #include "tasks/task_handler.hh"
 #include "utils/overloaded_functor.hh"
 
@@ -26,23 +25,18 @@ namespace tm = httpd::task_manager_json;
 using namespace json;
 using namespace seastar::httpd;
 
-static ::tm get_time(db_clock::time_point tp) {
-    auto time = db_clock::to_time_t(tp);
-    ::tm t;
-    ::gmtime_r(&time, &t);
-    return t;
-}
+tm::task_status make_status(tasks::task_status status) {
+    auto start_time = db_clock::to_time_t(status.start_time);
+    auto end_time = db_clock::to_time_t(status.end_time);
+    ::tm st, et;
+    ::gmtime_r(&end_time, &et);
+    ::gmtime_r(&start_time, &st);
 
-tm::task_status make_status(tasks::task_status status, sharded<gms::gossiper>& gossiper) {
     std::vector<tm::task_identity> tis{status.children.size()};
-    std::ranges::transform(status.children, tis.begin(), [&gossiper] (const auto& child) {
+    std::ranges::transform(status.children, tis.begin(), [] (const auto& child) {
         tm::task_identity ident;
-        gms::inet_address addr{};
-        if (gossiper.local_is_initialized()) {
-            addr = gossiper.local().get_address_map().find(child.host_id).value_or(gms::inet_address{});
-        }
         ident.task_id = child.task_id.to_sstring();
-        ident.node = fmt::format("{}", addr);
+        ident.node = fmt::format("{}", child.node);
         return ident;
     });
 
@@ -53,8 +47,8 @@ tm::task_status make_status(tasks::task_status status, sharded<gms::gossiper>& g
     res.scope = status.scope;
     res.state = status.state;
     res.is_abortable = bool(status.is_abortable);
-    res.start_time = get_time(status.start_time);
-    res.end_time = get_time(status.end_time);
+    res.start_time = st;
+    res.end_time = et;
     res.error = status.error;
     res.parent_id = status.parent_id ? status.parent_id.to_sstring() : "none";
     res.sequence_number = status.sequence_number;
@@ -80,13 +74,10 @@ tm::task_stats make_stats(tasks::task_stats stats) {
     res.keyspace = stats.keyspace;
     res.table = stats.table;
     res.entity = stats.entity;
-    res.shard = stats.shard;
-    res.start_time = get_time(stats.start_time);
-    res.end_time = get_time(stats.end_time);;
     return res;
 }
 
-void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>& tm, db::config& cfg, sharded<gms::gossiper>& gossiper) {
+void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>& tm, db::config& cfg) {
     tm::get_modules.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
         std::vector<std::string> v = tm.local().get_modules() | std::views::keys | std::ranges::to<std::vector>();
         co_return v;
@@ -144,7 +135,7 @@ void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>
         co_return std::move(f);
     });
 
-    tm::get_task_status.set(r, [&tm, &gossiper] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
+    tm::get_task_status.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
         auto id = tasks::task_id{utils::UUID{req->get_path_param("task_id")}};
         tasks::task_status status;
         try {
@@ -153,7 +144,7 @@ void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>
         } catch (tasks::task_manager::task_not_found& e) {
             throw bad_param_exception(e.what());
         }
-        co_return make_status(status, gossiper);
+        co_return make_status(status);
     });
 
     tm::abort_task.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
@@ -169,7 +160,7 @@ void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>
         co_return json_void();
     });
 
-    tm::wait_task.set(r, [&tm, &gossiper] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
+    tm::wait_task.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
         auto id = tasks::task_id{utils::UUID{req->get_path_param("task_id")}};
         tasks::task_status status;
         std::optional<std::chrono::seconds> timeout = std::nullopt;
@@ -184,24 +175,24 @@ void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>
         } catch (timed_out_error& e) {
             throw httpd::base_exception{e.what(), http::reply::status_type::request_timeout};
         }
-        co_return make_status(status, gossiper);
+        co_return make_status(status);
     });
 
-    tm::get_task_status_recursively.set(r, [&_tm = tm, &gossiper] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
+    tm::get_task_status_recursively.set(r, [&_tm = tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
         auto& tm = _tm;
         auto id = tasks::task_id{utils::UUID{req->get_path_param("task_id")}};
         try {
             auto task = tasks::task_handler{tm.local(), id};
             auto res = co_await task.get_status_recursively(true);
 
-            std::function<future<>(output_stream<char>&&)> f = [r = std::move(res), &gossiper] (output_stream<char>&& os) -> future<> {
+            std::function<future<>(output_stream<char>&&)> f = [r = std::move(res)] (output_stream<char>&& os) -> future<> {
                 auto s = std::move(os);
                 auto res = std::move(r);
                 co_await s.write("[");
                 std::string delim = "";
                 for (auto& status: res) {
                     co_await s.write(std::exchange(delim, ", "));
-                    co_await formatter::write(s, make_status(status, gossiper));
+                    co_await formatter::write(s, make_status(status));
                 }
                 co_await s.write("]");
                 co_await s.close();
@@ -241,32 +232,6 @@ void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>
         uint32_t user_ttl = cfg.user_task_ttl_seconds();
         co_return json::json_return_type(user_ttl);
     });
-
-    tm::drain_tasks.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
-        co_await tm.invoke_on_all([&req] (tasks::task_manager& tm) -> future<> {
-            tasks::task_manager::module_ptr module;
-            try {
-                module = tm.find_module(req->get_path_param("module"));
-            } catch (...) {
-                throw bad_param_exception(fmt::format("{}", std::current_exception()));
-            }
-
-            const auto& local_tasks = module->get_local_tasks();
-            std::vector<tasks::task_id> ids;
-            ids.reserve(local_tasks.size());
-            std::transform(begin(local_tasks), end(local_tasks), std::back_inserter(ids), [] (const auto& task) {
-                return task.second->is_complete() ? task.first : tasks::task_id::create_null_id();
-            });
-
-            for (auto&& id : ids) {
-                if (id) {
-                    module->unregister_task(id);
-                }
-                co_await maybe_yield();
-            }
-        });
-        co_return json_void();
-    });
 }
 
 void unset_task_manager(http_context& ctx, routes& r) {
@@ -278,7 +243,6 @@ void unset_task_manager(http_context& ctx, routes& r) {
     tm::get_task_status_recursively.unset(r);
     tm::get_and_update_ttl.unset(r);
     tm::get_ttl.unset(r);
-    tm::drain_tasks.unset(r);
 }
 
 }

api/task_manager.hh

@@ -18,7 +18,7 @@ namespace tasks {
 
 namespace api {
 
-void set_task_manager(http_context& ctx, httpd::routes& r, sharded<tasks::task_manager>& tm, db::config& cfg, sharded<gms::gossiper>& gossiper);
+void set_task_manager(http_context& ctx, httpd::routes& r, sharded<tasks::task_manager>& tm, db::config& cfg);
 void unset_task_manager(http_context& ctx, httpd::routes& r);
 
 }

api/task_manager_test.cc

@@ -6,9 +6,6 @@
  * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
  */
 
-
-#include "build_mode.hh"
-
 #ifndef SCYLLA_BUILD_MODE_RELEASE
 
 #include <seastar/core/coroutine.hh>

api/tasks.cc

@@ -31,7 +31,8 @@ using ks_cf_func = std::function<future<json::json_return_type>(http_context&, s
 
 static auto wrap_ks_cf(http_context &ctx, ks_cf_func f) {
     return [&ctx, f = std::move(f)](std::unique_ptr<http::request> req) {
-        auto [keyspace, table_infos] = parse_table_infos(ctx, *req);
+        auto keyspace = validate_keyspace(ctx, req);
+        auto table_infos = parse_table_infos(keyspace, ctx, req->query_parameters, "cf");
         return f(ctx, std::move(req), std::move(keyspace), std::move(table_infos));
     };
 }
@@ -62,7 +63,8 @@ void set_tasks_compaction_module(http_context& ctx, routes& r, sharded<service::
 
     t::force_keyspace_cleanup_async.set(r, [&ctx, &ss](std::unique_ptr<http::request> req) -> future<json::json_return_type> {
         auto& db = ctx.db;
-        auto [keyspace, table_infos] = parse_table_infos(ctx, *req);
+        auto keyspace = validate_keyspace(ctx, req);
+        auto table_infos = parse_table_infos(keyspace, ctx, req->query_parameters, "cf");
         apilog.info("force_keyspace_cleanup_async: keyspace={} tables={}", keyspace, table_infos);
         if (!co_await ss.local().is_cleanup_allowed(keyspace)) {
             auto msg = "Can not perform cleanup operation when topology changes";

api/token_metadata.cc

@@ -10,7 +10,6 @@
 #include "api/api-doc/storage_service.json.hh"
 #include "api/api-doc/endpoint_snitch_info.json.hh"
 #include "locator/token_metadata.hh"
-#include "gms/gossiper.hh"
 
 using namespace seastar::httpd;
 
@@ -19,7 +18,7 @@ namespace api {
 namespace ss = httpd::storage_service_json;
 using namespace json;
 
-void set_token_metadata(http_context& ctx, routes& r, sharded<locator::shared_token_metadata>& tm, sharded<gms::gossiper>& g) {
+void set_token_metadata(http_context& ctx, routes& r, sharded<locator::shared_token_metadata>& tm) {
     ss::local_hostid.set(r, [&tm](std::unique_ptr<http::request> req) {
         auto id = tm.local().get()->get_my_id();
         if (!bool(id)) {
@@ -34,25 +33,22 @@ void set_token_metadata(http_context& ctx, routes& r, sharded<locator::shared_to
         }));
     });
 
-    ss::get_node_tokens.set(r, [&tm, &g] (std::unique_ptr<http::request> req) {
+    ss::get_node_tokens.set(r, [&tm] (std::unique_ptr<http::request> req) {
         gms::inet_address addr(req->get_path_param("endpoint"));
         auto& local_tm = *tm.local().get();
-        std::optional<locator::host_id> host_id;
-        try {
-            host_id = g.local().get_host_id(addr);
-        } catch (...) {}
+        const auto host_id = local_tm.get_host_id_if_known(addr);
         return make_ready_future<json::json_return_type>(stream_range_as_array(host_id ? local_tm.get_tokens(*host_id): std::vector<dht::token>{}, [](const dht::token& i) {
             return fmt::to_string(i);
         }));
     });
 
-    ss::get_leaving_nodes.set(r, [&tm, &g](const_req req) {
+    ss::get_leaving_nodes.set(r, [&tm](const_req req) {
         const auto& local_tm = *tm.local().get();
         const auto& leaving_host_ids = local_tm.get_leaving_endpoints();
         std::unordered_set<gms::inet_address> eps;
         eps.reserve(leaving_host_ids.size());
         for (const auto host_id: leaving_host_ids) {
-            eps.insert(g.local().get_address_map().get(host_id));
+            eps.insert(local_tm.get_endpoint_for_host_id(host_id));
         }
         return container_to_vec(eps);
     });
@@ -62,26 +58,20 @@ void set_token_metadata(http_context& ctx, routes& r, sharded<locator::shared_to
         return container_to_vec(addr);
     });
 
-    ss::get_joining_nodes.set(r, [&tm, &g](const_req req) {
+    ss::get_joining_nodes.set(r, [&tm](const_req req) {
         const auto& local_tm = *tm.local().get();
         const auto& points = local_tm.get_bootstrap_tokens();
         std::unordered_set<gms::inet_address> eps;
         eps.reserve(points.size());
         for (const auto& [token, host_id]: points) {
-            eps.insert(g.local().get_address_map().get(host_id));
+            eps.insert(local_tm.get_endpoint_for_host_id(host_id));
         }
         return container_to_vec(eps);
     });
 
-    ss::get_host_id_map.set(r, [&tm, &g](const_req req) {
-        if (!g.local().is_enabled()) {
-            throw std::runtime_error("The gossiper is not ready yet");
-        }
+    ss::get_host_id_map.set(r, [&tm](const_req req) {
         std::vector<ss::mapper> res;
-        auto map = tm.local().get()->get_host_ids() |
-            std::views::transform([&g] (locator::host_id id) { return std::make_pair(g.local().get_address_map().get(id), id); }) |
-            std::ranges::to<std::unordered_map>();
-        return map_to_key_value(std::move(map), res);
+        return map_to_key_value(tm.local().get()->get_endpoint_to_host_id_map(), res);
     });
 
     static auto host_or_broadcast = [&tm](const_req req) {
@@ -89,34 +79,26 @@ void set_token_metadata(http_context& ctx, routes& r, sharded<locator::shared_to
         return host.empty() ? tm.local().get()->get_topology().my_address() : gms::inet_address(host);
     };
 
-    httpd::endpoint_snitch_info_json::get_datacenter.set(r, [&tm, &g](const_req req) {
+    httpd::endpoint_snitch_info_json::get_datacenter.set(r, [&tm](const_req req) {
         auto& topology = tm.local().get()->get_topology();
         auto ep = host_or_broadcast(req);
-        std::optional<locator::host_id> host_id;
-        try {
-            host_id = g.local().get_host_id(ep);
-        } catch (...) {}
-        if (!host_id || !topology.has_node(*host_id)) {
+        if (!topology.has_endpoint(ep)) {
             // Cannot return error here, nodetool status can race, request
             // info about just-left node and not handle it nicely
             return locator::endpoint_dc_rack::default_location.dc;
         }
-        return topology.get_datacenter(*host_id);
+        return topology.get_datacenter(ep);
     });
 
-    httpd::endpoint_snitch_info_json::get_rack.set(r, [&tm, &g](const_req req) {
+    httpd::endpoint_snitch_info_json::get_rack.set(r, [&tm](const_req req) {
         auto& topology = tm.local().get()->get_topology();
         auto ep = host_or_broadcast(req);
-        std::optional<locator::host_id> host_id;
-        try {
-            host_id = g.local().get_host_id(ep);
-        } catch (...) {}
-        if (!host_id || !topology.has_node(*host_id)) {
+        if (!topology.has_endpoint(ep)) {
             // Cannot return error here, nodetool status can race, request
             // info about just-left node and not handle it nicely
             return locator::endpoint_dc_rack::default_location.rack;
         }
-        return topology.get_rack(*host_id);
+        return topology.get_rack(ep);
     });
 }
 

api/token_metadata.hh

@@ -15,11 +15,10 @@ class routes;
 }
 
 namespace locator { class shared_token_metadata; }
-namespace gms { class gossiper; }
 
 namespace api {
 struct http_context;
-void set_token_metadata(http_context& ctx, seastar::httpd::routes& r, seastar::sharded<locator::shared_token_metadata>& tm, seastar::sharded<gms::gossiper>& g);
+void set_token_metadata(http_context& ctx, seastar::httpd::routes& r, seastar::sharded<locator::shared_token_metadata>& tm);
 void unset_token_metadata(http_context& ctx, seastar::httpd::routes& r);
 
 }

audit/CMakeLists.txt

@@ -1,19 +0,0 @@
-include(add_whole_archive)
-
-add_library(scylla_audit STATIC)
-target_sources(scylla_audit
-  PRIVATE
-    audit.cc
-    audit_cf_storage_helper.cc
-    audit_syslog_storage_helper.cc)
-target_include_directories(scylla_audit
-  PUBLIC
-    ${CMAKE_SOURCE_DIR})
-target_link_libraries(scylla_audit
-  PUBLIC
-    Seastar::seastar
-    xxHash::xxhash
-  PRIVATE
-    cql3)
-
-add_whole_archive(audit scylla_audit)

audit/audit.cc

@@ -1,294 +0,0 @@
-/*
- * Copyright (C) 2017 ScyllaDB
- */
-
-/*
- * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
- */
-
-#include <seastar/core/future-util.hh>
-#include "audit/audit.hh"
-#include "db/config.hh"
-#include "cql3/cql_statement.hh"
-#include "cql3/statements/batch_statement.hh"
-#include "cql3/statements/modification_statement.hh"
-#include "storage_helper.hh"
-#include "audit.hh"
-#include "../db/config.hh"
-#include "utils/class_registrator.hh"
-
-#include <boost/algorithm/string/split.hpp>
-#include <boost/algorithm/string/trim.hpp>
-#include <boost/algorithm/string/classification.hpp>
-
-
-namespace audit {
-
-logging::logger logger("audit");
-
-static sstring category_to_string(statement_category category)
-{
-    switch (category) {
-        case statement_category::QUERY: return "QUERY";
-        case statement_category::DML: return "DML";
-        case statement_category::DDL: return "DDL";
-        case statement_category::DCL: return "DCL";
-        case statement_category::AUTH: return "AUTH";
-        case statement_category::ADMIN: return "ADMIN";
-    }
-    return "";
-}
-
-sstring audit_info::category_string() const {
-    return category_to_string(_category);
-}
-
-static category_set parse_audit_categories(const sstring& data) {
-    category_set result;
-    if (!data.empty()) {
-        std::vector<sstring> tokens;
-        boost::split(tokens, data, boost::is_any_of(","));
-        for (sstring& category : tokens) {
-            boost::trim(category);
-            if (category == "QUERY") {
-                result.set(statement_category::QUERY);
-            } else if (category == "DML") {
-                result.set(statement_category::DML);
-            } else if (category == "DDL") {
-                result.set(statement_category::DDL);
-            } else if (category == "DCL") {
-                result.set(statement_category::DCL);
-            } else if (category == "AUTH") {
-                result.set(statement_category::AUTH);
-            } else if (category == "ADMIN") {
-                result.set(statement_category::ADMIN);
-            } else {
-                throw audit_exception(fmt::format("Bad configuration: invalid 'audit_categories': {}", data));
-            }
-        }
-    }
-    return result;
-}
-
-static std::map<sstring, std::set<sstring>> parse_audit_tables(const sstring& data) {
-    std::map<sstring, std::set<sstring>> result;
-    if (!data.empty()) {
-        std::vector<sstring> tokens;
-        boost::split(tokens, data, boost::is_any_of(","));
-        for (sstring& token : tokens) {
-            std::vector<sstring> parts;
-            boost::split(parts, token, boost::is_any_of("."));
-            if (parts.size() != 2) {
-                throw audit_exception(fmt::format("Bad configuration: invalid 'audit_tables': {}", data));
-            }
-            boost::trim(parts[0]);
-            boost::trim(parts[1]);
-            result[parts[0]].insert(std::move(parts[1]));
-        }
-    }
-    return result;
-}
-
-static std::set<sstring> parse_audit_keyspaces(const sstring& data) {
-    std::set<sstring> result;
-    if (!data.empty()) {
-        std::vector<sstring> tokens;
-        boost::split(tokens, data, boost::is_any_of(","));
-        for (sstring& token : tokens) {
-            boost::trim(token);
-            result.insert(std::move(token));
-        }
-    }
-    return result;
-}
-
-audit::audit(locator::shared_token_metadata& token_metadata,
-             sstring&& storage_helper_name,
-             std::set<sstring>&& audited_keyspaces,
-             std::map<sstring, std::set<sstring>>&& audited_tables,
-             category_set&& audited_categories,
-             const db::config& cfg)
-    : _token_metadata(token_metadata)
-    , _audited_keyspaces(std::move(audited_keyspaces))
-    , _audited_tables(std::move(audited_tables))
-    , _audited_categories(std::move(audited_categories))
-    , _storage_helper_class_name(std::move(storage_helper_name))
-    , _cfg(cfg)
-    , _cfg_keyspaces_observer(cfg.audit_keyspaces.observe([this] (sstring const& new_value){ update_config<std::set<sstring>>(new_value, parse_audit_keyspaces, _audited_keyspaces); }))
-    , _cfg_tables_observer(cfg.audit_tables.observe([this] (sstring const& new_value){ update_config<std::map<sstring, std::set<sstring>>>(new_value, parse_audit_tables, _audited_tables); }))
-    , _cfg_categories_observer(cfg.audit_categories.observe([this] (sstring const& new_value){ update_config<category_set>(new_value, parse_audit_categories, _audited_categories); }))
-{ }
-
-audit::~audit() = default;
-
-future<> audit::create_audit(const db::config& cfg, sharded<locator::shared_token_metadata>& stm) {
-    sstring storage_helper_name;
-    if (cfg.audit() == "table") {
-        storage_helper_name = "audit_cf_storage_helper";
-    } else if (cfg.audit() == "syslog") {
-        storage_helper_name = "audit_syslog_storage_helper";
-    } else if (cfg.audit() == "none") {
-        // Audit is off
-        logger.info("Audit is disabled");
-
-        return make_ready_future<>();
-    } else {
-        throw audit_exception(fmt::format("Bad configuration: invalid 'audit': {}", cfg.audit()));
-    }
-    category_set audited_categories = parse_audit_categories(cfg.audit_categories());
-    std::map<sstring, std::set<sstring>> audited_tables = parse_audit_tables(cfg.audit_tables());
-    std::set<sstring> audited_keyspaces = parse_audit_keyspaces(cfg.audit_keyspaces());
-
-    logger.info("Audit is enabled. Auditing to: \"{}\", with the following categories: \"{}\", keyspaces: \"{}\", and tables: \"{}\"",
-                cfg.audit(), cfg.audit_categories(), cfg.audit_keyspaces(), cfg.audit_tables());
-
-    return audit_instance().start(std::ref(stm),
-                                  std::move(storage_helper_name),
-                                  std::move(audited_keyspaces),
-                                  std::move(audited_tables),
-                                  std::move(audited_categories),
-                                  std::cref(cfg));
-}
-
-future<> audit::start_audit(const db::config& cfg, sharded<cql3::query_processor>& qp, sharded<service::migration_manager>& mm) {
-    if (!audit_instance().local_is_initialized()) {
-        return make_ready_future<>();
-    }
-    return audit_instance().invoke_on_all([&cfg, &qp, &mm] (audit& local_audit) {
-        return local_audit.start(cfg, qp.local(), mm.local());
-    });
-}
-
-future<> audit::stop_audit() {
-    if (!audit_instance().local_is_initialized()) {
-        return make_ready_future<>();
-    }
-    return audit::audit::audit_instance().invoke_on_all([] (auto& local_audit) {
-        return local_audit.shutdown();
-    }).then([] {
-        return audit::audit::audit_instance().stop();
-    });
-}
-
-audit_info_ptr audit::create_audit_info(statement_category cat, const sstring& keyspace, const sstring& table) {
-    if (!audit_instance().local_is_initialized()) {
-        return nullptr;
-    }
-    return std::make_unique<audit_info>(cat, keyspace, table);
-}
-
-audit_info_ptr audit::create_no_audit_info() {
-    return audit_info_ptr();
-}
-
-future<> audit::start(const db::config& cfg, cql3::query_processor& qp, service::migration_manager& mm) {
-    try {
-        _storage_helper_ptr = create_object<storage_helper>(_storage_helper_class_name, qp, mm);
-    } catch (no_such_class& e) {
-        logger.error("Can't create audit storage helper {}: not supported", _storage_helper_class_name);
-        throw;
-    } catch (...) {
-        throw;
-    }
-    return _storage_helper_ptr->start(cfg);
-}
-
-future<> audit::stop() {
-    return _storage_helper_ptr->stop();
-}
-
-future<> audit::shutdown() {
-    return make_ready_future<>();
-}
-
-future<> audit::log(const audit_info* audit_info, service::query_state& query_state, const cql3::query_options& options, bool error) {
-    const service::client_state& client_state = query_state.get_client_state();
-    socket_address node_ip = _token_metadata.get()->get_topology().my_address().addr();
-    db::consistency_level cl = options.get_consistency();
-    thread_local static sstring no_username("undefined");
-    static const sstring anonymous_username("anonymous");
-    const sstring& username = client_state.user() ? client_state.user()->name.value_or(anonymous_username) : no_username;
-    socket_address client_ip = client_state.get_client_address().addr();
-    return futurize_invoke(std::mem_fn(&storage_helper::write), _storage_helper_ptr, audit_info, node_ip, client_ip, cl, username, error)
-        .handle_exception([audit_info, node_ip, client_ip, cl, username, error] (auto ep) {
-            logger.error("Unexpected exception when writing log with: node_ip {} category {} cl {} error {} keyspace {} query '{}' client_ip {} table {} username {} exception {}",
-                node_ip, audit_info->category_string(), cl, error, audit_info->keyspace(),
-                audit_info->query(), client_ip, audit_info->table(),username, ep);
-    });
-}
-
-future<> audit::log_login(const sstring& username, socket_address client_ip, bool error) noexcept {
-    socket_address node_ip = _token_metadata.get()->get_topology().my_address().addr();
-    return futurize_invoke(std::mem_fn(&storage_helper::write_login), _storage_helper_ptr, username, node_ip, client_ip, error)
-        .handle_exception([username, node_ip, client_ip, error] (auto ep) {
-            logger.error("Unexpected exception when writing login log with: node_ip {} client_ip {} username {} error {} exception {}",
-                node_ip, client_ip, username, error, ep);
-    });
-}
-
-future<> inspect(shared_ptr<cql3::cql_statement> statement, service::query_state& query_state, const cql3::query_options& options, bool error) {
-    cql3::statements::batch_statement* batch = dynamic_cast<cql3::statements::batch_statement*>(statement.get());
-    if (batch != nullptr) {
-        return do_for_each(batch->statements().begin(), batch->statements().end(), [&query_state, &options, error] (auto&& m) {
-            return inspect(m.statement, query_state, options, error);
-        });
-    } else {
-        auto audit_info = statement->get_audit_info();
-        if (bool(audit_info) && audit::local_audit_instance().should_log(audit_info)) {
-            return audit::local_audit_instance().log(audit_info, query_state, options, error);
-        }
-    }
-    return make_ready_future<>();
-}
-
-future<> inspect_login(const sstring& username, socket_address client_ip, bool error) {
-    if (!audit::audit_instance().local_is_initialized() || !audit::local_audit_instance().should_log_login()) {
-        return make_ready_future<>();
-    }
-    return audit::local_audit_instance().log_login(username, client_ip, error);
-}
-
-bool audit::should_log_table(const sstring& keyspace, const sstring& name) const {
-    auto keyspace_it = _audited_tables.find(keyspace);
-    return keyspace_it != _audited_tables.cend() && keyspace_it->second.find(name) != keyspace_it->second.cend();
-}
-
-bool audit::should_log(const audit_info* audit_info) const {
-    return _audited_categories.contains(audit_info->category())
-           && (_audited_keyspaces.find(audit_info->keyspace()) != _audited_keyspaces.cend()
-                         || should_log_table(audit_info->keyspace(), audit_info->table())
-                         || audit_info->category() == statement_category::AUTH
-                         || audit_info->category() == statement_category::ADMIN
-                         || audit_info->category() == statement_category::DCL);
-}
-
-template<class T>
-void audit::update_config(const sstring & new_value, std::function<T(const sstring&)> parse_func, T& cfg_parameter)
-{
-    try {
-        cfg_parameter = parse_func(new_value);
-    } catch (...) {
-        logger.error("Audit configuration update failed because cannot parse value=\"{}\".", new_value);
-        return;
-    }
-
-    // If update_config is called with an invalid new_value, this line is not reached.
-    // But logging the invalid value must be avoided later, when a different configuration parameter is changed to a correct value.
-    // That's why values from _audited_{categories, keyspaces, tables} are logged instead of _cfg.audit_{categories, keyspaces, tables}
-
-    // Each table as "keyspace.table_name" like in the configuration file
-    auto table_entries = _audited_tables | std::views::transform([](const auto& pair) {
-        return pair.second | std::views::transform([&](const std::string& table_name) {
-            return fmt::format("{}.{}", pair.first, table_name);
-        });
-    }) | std::views::join;
-
-    logger.info(
-        "Audit configuration is updated. Auditing to: \"{}\", with the following categories: \"{}\", keyspaces: \"{}\", and tables: \"{}\".",
-        _cfg.audit(),
-        fmt::join(std::views::transform(_audited_categories, category_to_string), ","),
-        fmt::join(_audited_keyspaces, ","),
-        fmt::join(table_entries, ","));
-}
-
-}

audit/audit.hh

@@ -1,152 +0,0 @@
-/*
- * Copyright (C) 2017 ScyllaDB
- */
-
-/*
- * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
- */
-#pragma once
-
-#include "seastarx.hh"
-#include "utils/log.hh"
-#include "utils/observable.hh"
-#include "db/consistency_level.hh"
-#include "locator/token_metadata_fwd.hh"
-#include <seastar/core/sharded.hh>
-#include <seastar/util/log.hh>
-
-#include "enum_set.hh"
-
-#include <memory>
-
-namespace db {
-
-class config;
-
-}
-
-namespace cql3 {
-
-class cql_statement;
-class query_processor;
-class query_options;
-
-}
-
-namespace service {
-
-class migration_manager;
-class query_state;
-
-}
-
-namespace locator {
-
-class shared_token_metadata;
-
-}
-
-namespace audit {
-
-extern logging::logger logger;
-
-class audit_exception : public std::exception {
-    sstring _what;
-public:
-    explicit audit_exception(sstring&& what) : _what(std::move(what)) { }
-    const char* what() const noexcept override {
-        return _what.c_str();
-    }
-};
-
-enum class statement_category {
-    QUERY, DML, DDL, DCL, AUTH, ADMIN
-};
-
-using category_set = enum_set<super_enum<statement_category, statement_category::QUERY,
-                                                             statement_category::DML,
-                                                             statement_category::DDL,
-                                                             statement_category::DCL,
-                                                             statement_category::AUTH,
-                                                             statement_category::ADMIN>>;
-
-class audit_info final {
-    statement_category _category;
-    sstring _keyspace;
-    sstring _table;
-    sstring _query;
-public:
-    audit_info(statement_category cat, sstring keyspace, sstring table)
-        : _category(cat)
-        , _keyspace(std::move(keyspace))
-        , _table(std::move(table))
-    { }
-    void set_query_string(const std::string_view& query_string) {
-        _query = sstring(query_string);
-    }
-    const sstring& keyspace() const { return _keyspace; }
-    const sstring& table() const { return _table; }
-    const sstring& query() const { return _query; }
-    sstring category_string() const;
-    statement_category category() const { return _category; }
-};
-
-using audit_info_ptr = std::unique_ptr<audit_info>;
-
-class storage_helper;
-
-class audit final : public seastar::async_sharded_service<audit> {
-    locator::shared_token_metadata& _token_metadata;
-    std::set<sstring> _audited_keyspaces;
-    // Maps keyspace name to set of table names in that keyspace
-    std::map<sstring, std::set<sstring>> _audited_tables;
-    category_set _audited_categories;
-
-    sstring _storage_helper_class_name;
-    std::unique_ptr<storage_helper> _storage_helper_ptr;
-
-    const db::config& _cfg;
-    utils::observer<sstring> _cfg_keyspaces_observer;
-    utils::observer<sstring> _cfg_tables_observer;
-    utils::observer<sstring> _cfg_categories_observer;
-
-    template<class T>
-    void update_config(const sstring & new_value, std::function<T(const sstring&)> parse_func, T& cfg_parameter);
-
-    bool should_log_table(const sstring& keyspace, const sstring& name) const;
-public:
-    static seastar::sharded<audit>& audit_instance() {
-        // FIXME: leaked intentionally to avoid shutdown problems, see #293
-        static seastar::sharded<audit>* audit_inst = new seastar::sharded<audit>();
-
-        return *audit_inst;
-    }
-
-    static audit& local_audit_instance() {
-        return audit_instance().local();
-    }
-    static future<> create_audit(const db::config& cfg, sharded<locator::shared_token_metadata>& stm);
-    static future<> start_audit(const db::config& cfg, sharded<cql3::query_processor>& qp, sharded<service::migration_manager>& mm);
-    static future<> stop_audit();
-    static audit_info_ptr create_audit_info(statement_category cat, const sstring& keyspace, const sstring& table);
-    static audit_info_ptr create_no_audit_info();
-    audit(locator::shared_token_metadata& stm, sstring&& storage_helper_name,
-          std::set<sstring>&& audited_keyspaces,
-          std::map<sstring, std::set<sstring>>&& audited_tables,
-          category_set&& audited_categories,
-          const db::config& cfg);
-    ~audit();
-    future<> start(const db::config& cfg, cql3::query_processor& qp, service::migration_manager& mm);
-    future<> stop();
-    future<> shutdown();
-    bool should_log(const audit_info* audit_info) const;
-    bool should_log_login() const { return _audited_categories.contains(statement_category::AUTH); }
-    future<> log(const audit_info* audit_info, service::query_state& query_state, const cql3::query_options& options, bool error);
-    future<> log_login(const sstring& username, socket_address client_ip, bool error) noexcept;
-};
-
-future<> inspect(shared_ptr<cql3::cql_statement> statement, service::query_state& query_state, const cql3::query_options& options, bool error);
-
-future<> inspect_login(const sstring& username, socket_address client_ip, bool error);
-
-}

audit/audit_cf_storage_helper.cc

@@ -1,202 +0,0 @@
-/*
- * Copyright (C) 2017 ScyllaDB
- */
-
-/*
- * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
- */
-
-#include "audit/audit_cf_storage_helper.hh"
-
-#include "cql3/query_processor.hh"
-#include "data_dictionary/keyspace_metadata.hh"
-#include "utils/UUID_gen.hh"
-#include "utils/class_registrator.hh"
-#include "cql3/query_options.hh"
-#include "cql3/statements/ks_prop_defs.hh"
-#include "service/migration_manager.hh"
-#include "service/storage_proxy.hh"
-
-namespace audit {
-
-const sstring audit_cf_storage_helper::KEYSPACE_NAME("audit");
-const sstring audit_cf_storage_helper::TABLE_NAME("audit_log");
-
-audit_cf_storage_helper::audit_cf_storage_helper(cql3::query_processor& qp, service::migration_manager& mm)
-    : _qp(qp)
-    , _mm(mm)
-    , _table(KEYSPACE_NAME, TABLE_NAME,
-             fmt::format("CREATE TABLE IF NOT EXISTS {}.{} ("
-                       "date timestamp, "
-                       "node inet, "
-                       "event_time timeuuid, "
-                       "category text, "
-                       "consistency text, "
-                       "table_name text, "
-                       "keyspace_name text, "
-                       "operation text, "
-                       "source inet, "
-                       "username text, "
-                       "error boolean, "
-                       "PRIMARY KEY ((date, node), event_time))",
-                       KEYSPACE_NAME, TABLE_NAME),
-             fmt::format("INSERT INTO {}.{} ("
-                       "date,"
-                       "node,"
-                       "event_time,"
-                       "category,"
-                       "consistency,"
-                       "table_name,"
-                       "keyspace_name,"
-                       "operation,"
-                       "source,"
-                       "username,"
-                       "error) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
-                       KEYSPACE_NAME, TABLE_NAME))
-    , _dummy_query_state(service::client_state::for_internal_calls(), empty_service_permit())
-{
-}
-
-future<> audit_cf_storage_helper::migrate_audit_table(service::group0_guard group0_guard) {
-    while (true) {
-        auto const ks = _qp.db().try_find_keyspace(KEYSPACE_NAME);
-        if (ks && ks->metadata()->strategy_name() == "org.apache.cassandra.locator.SimpleStrategy") {
-            data_dictionary::database db = _qp.db();
-            cql3::statements::ks_prop_defs old_ks_prop_defs;
-            auto old_ks_metadata = old_ks_prop_defs.as_ks_metadata_update(
-                    ks->metadata(), *_qp.proxy().get_token_metadata_ptr(), db.features());
-            std::map<sstring, sstring> strategy_opts;
-            for (const auto &dc: _qp.proxy().get_token_metadata_ptr()->get_topology().get_datacenters())
-                strategy_opts[dc] = "3";
-
-            auto new_ks_metadata = keyspace_metadata::new_keyspace(KEYSPACE_NAME,
-                                                                   "org.apache.cassandra.locator.NetworkTopologyStrategy",
-                                                                   strategy_opts,
-                                                                   std::nullopt, // initial_tablets
-                                                                   old_ks_metadata->durable_writes(),
-                                                                   old_ks_metadata->get_storage_options(),
-                                                                   old_ks_metadata->tables());
-            auto ts = group0_guard.write_timestamp();
-            try {
-                co_await _mm.announce(
-                        service::prepare_keyspace_update_announcement(db.real_database(), new_ks_metadata, ts),
-                        std::move(group0_guard), format("audit: Alter {} keyspace", KEYSPACE_NAME));
-                break;
-            } catch (::service::group0_concurrent_modification &) {
-                logger.info("Concurrent operation is detected while altering {} keyspace, retrying.", KEYSPACE_NAME);
-            }
-            group0_guard = co_await _mm.start_group0_operation();
-        } else {
-            co_return;
-        }
-    }
-}
-
-future<> audit_cf_storage_helper::start(const db::config &cfg) {
-    if (this_shard_id() != 0) {
-        co_return;
-    }
-
-    if (auto ks = _qp.db().try_find_keyspace(KEYSPACE_NAME);
-            !ks ||
-            ks->metadata()->strategy_name() == "org.apache.cassandra.locator.SimpleStrategy") {
-
-        auto group0_guard = co_await _mm.start_group0_operation();
-        if (ks = _qp.db().try_find_keyspace(KEYSPACE_NAME); !ks) {
-            // releasing, because table_helper::setup_keyspace creates a raft guard of its own
-            service::release_guard(std::move(group0_guard));
-            co_return co_await table_helper::setup_keyspace(_qp, _mm, KEYSPACE_NAME,
-                                                            "org.apache.cassandra.locator.NetworkTopologyStrategy",
-                                                            "3", _dummy_query_state, {&_table});
-        } else if (ks->metadata()->strategy_name() == "org.apache.cassandra.locator.SimpleStrategy") {
-            // We want to migrate the old (pre-Scylla 6.0) SimpleStrategy to a newer one.
-            // The migrate_audit_table() function will do nothing if it races with another strategy change:
-            // - either by another node doing the same thing in parallel,
-            // - or a user manually changing the strategy of the same table.
-            // Note we only check the strategy, not the replication factor.
-            co_return co_await migrate_audit_table(std::move(group0_guard));
-        } else {
-            co_return;
-        }
-    }
-}
-
-future<> audit_cf_storage_helper::stop() {
-    return make_ready_future<>();
-}
-
-future<> audit_cf_storage_helper::write(const audit_info* audit_info,
-                                    socket_address node_ip,
-                                    socket_address client_ip,
-                                    db::consistency_level cl,
-                                    const sstring& username,
-                                    bool error) {
-    return _table.insert(_qp, _mm, _dummy_query_state, make_data, audit_info, node_ip, client_ip, cl, username, error);
-}
-
-future<> audit_cf_storage_helper::write_login(const sstring& username,
-                                              socket_address node_ip,
-                                              socket_address client_ip,
-                                              bool error) {
-    return _table.insert(_qp, _mm, _dummy_query_state, make_login_data, node_ip, client_ip, username, error);
-}
-
-cql3::query_options audit_cf_storage_helper::make_data(const audit_info* audit_info,
-                                                       socket_address node_ip,
-                                                       socket_address client_ip,
-                                                       db::consistency_level cl,
-                                                       const sstring& username,
-                                                       bool error) {
-    auto time = std::chrono::system_clock::now();
-    auto millis_since_epoch = std::chrono::duration_cast<std::chrono::milliseconds>(time.time_since_epoch()).count();
-    auto ticks_per_day = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::hours(24)).count();
-    auto date = millis_since_epoch / ticks_per_day * ticks_per_day;
-    thread_local static int64_t last_nanos = 0;
-    auto time_id = utils::UUID_gen::get_time_UUID(table_helper::make_monotonic_UUID_tp(last_nanos, time));
-    auto consistency_level = fmt::format("{}", cl);
-    std::vector<cql3::raw_value> values {
-        cql3::raw_value::make_value(timestamp_type->decompose(date)),
-        cql3::raw_value::make_value(inet_addr_type->decompose(node_ip.addr())),
-        cql3::raw_value::make_value(uuid_type->decompose(time_id)),
-        cql3::raw_value::make_value(utf8_type->decompose(audit_info->category_string())),
-        cql3::raw_value::make_value(utf8_type->decompose(sstring(consistency_level))),
-        cql3::raw_value::make_value(utf8_type->decompose(audit_info->table())),
-        cql3::raw_value::make_value(utf8_type->decompose(audit_info->keyspace())),
-        cql3::raw_value::make_value(utf8_type->decompose(audit_info->query())),
-        cql3::raw_value::make_value(inet_addr_type->decompose(client_ip.addr())),
-        cql3::raw_value::make_value(utf8_type->decompose(username)),
-        cql3::raw_value::make_value(boolean_type->decompose(error)),
-    };
-    return cql3::query_options(cql3::default_cql_config, db::consistency_level::ONE, std::nullopt, std::move(values), false, cql3::query_options::specific_options::DEFAULT);
-}
-
-cql3::query_options audit_cf_storage_helper::make_login_data(socket_address node_ip,
-                                                             socket_address client_ip,
-                                                             const sstring& username,
-                                                             bool error) {
-    auto time = std::chrono::system_clock::now();
-    auto millis_since_epoch = std::chrono::duration_cast<std::chrono::milliseconds>(time.time_since_epoch()).count();
-    auto ticks_per_day = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::hours(24)).count();
-    auto date = millis_since_epoch / ticks_per_day * ticks_per_day;
-    thread_local static int64_t last_nanos = 0;
-    auto time_id = utils::UUID_gen::get_time_UUID(table_helper::make_monotonic_UUID_tp(last_nanos, time));
-    std::vector<cql3::raw_value> values {
-            cql3::raw_value::make_value(timestamp_type->decompose(date)),
-            cql3::raw_value::make_value(inet_addr_type->decompose(node_ip.addr())),
-            cql3::raw_value::make_value(uuid_type->decompose(time_id)),
-            cql3::raw_value::make_value(utf8_type->decompose(sstring("AUTH"))),
-            cql3::raw_value::make_value(utf8_type->decompose(sstring(""))),
-            cql3::raw_value::make_value(utf8_type->decompose(sstring(""))),
-            cql3::raw_value::make_value(utf8_type->decompose(sstring(""))),
-            cql3::raw_value::make_value(utf8_type->decompose(sstring("LOGIN"))),
-            cql3::raw_value::make_value(inet_addr_type->decompose(client_ip.addr())),
-            cql3::raw_value::make_value(utf8_type->decompose(username)),
-            cql3::raw_value::make_value(boolean_type->decompose(error)),
-    };
-    return cql3::query_options(cql3::default_cql_config, db::consistency_level::ONE, std::nullopt, std::move(values), false, cql3::query_options::specific_options::DEFAULT);
-}
-
-using registry = class_registrator<storage_helper, audit_cf_storage_helper, cql3::query_processor&, service::migration_manager&>;
-static registry registrator1("audit_cf_storage_helper");
-
-}

audit/audit_cf_storage_helper.hh

@@ -1,67 +0,0 @@
-/*
- * Copyright (C) 2017 ScyllaDB
- */
-
-/*
- * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
- */
-#pragma once
-
-#include "audit/audit.hh"
-#include "table_helper.hh"
-#include "storage_helper.hh"
-#include "db/config.hh"
-#include "service/raft/raft_group0_client.hh"
-
-namespace cql3 {
-
-class query_processor;
-
-}
-
-namespace service {
-
-class migration_manager;
-
-}
-
-namespace audit {
-
-class audit_cf_storage_helper : public storage_helper {
-    static const sstring KEYSPACE_NAME;
-    static const sstring TABLE_NAME;
-    cql3::query_processor& _qp;
-    service::migration_manager& _mm;
-    table_helper _table;
-    service::query_state _dummy_query_state;
-    static cql3::query_options make_data(const audit_info* audit_info,
-                                         socket_address node_ip,
-                                         socket_address client_ip,
-                                         db::consistency_level cl,
-                                         const sstring& username,
-                                         bool error);
-    static cql3::query_options make_login_data(socket_address node_ip,
-                                               socket_address client_ip,
-                                               const sstring& username,
-                                               bool error);
-
-    future<> migrate_audit_table(service::group0_guard guard);
-
-public:
-    explicit audit_cf_storage_helper(cql3::query_processor& qp, service::migration_manager& mm);
-    virtual ~audit_cf_storage_helper() {}
-    virtual future<> start(const db::config& cfg) override;
-    virtual future<> stop() override;
-    virtual future<> write(const audit_info* audit_info,
-                           socket_address node_ip,
-                           socket_address client_ip,
-                           db::consistency_level cl,
-                           const sstring& username,
-                           bool error) override;
-    virtual future<> write_login(const sstring& username,
-                                 socket_address node_ip,
-                                 socket_address client_ip,
-                                 bool error) override;
-};
-
-}

audit/audit_syslog_storage_helper.cc

@@ -1,149 +0,0 @@
-/*
- * Copyright (C) 2017 ScyllaDB
- */
-
-/*
- * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
- */
-
-#include "audit/audit_syslog_storage_helper.hh"
-
-#include <sys/socket.h>
-#include <string.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <syslog.h>
-
-#include <seastar/core/coroutine.hh>
-#include <seastar/core/seastar.hh>
-#include <seastar/net/api.hh>
-
-#include <fmt/chrono.h>
-
-#include "cql3/query_processor.hh"
-#include "utils/class_registrator.hh"
-
-namespace cql3 {
-
-class query_processor;
-
-}
-
-namespace audit {
-
-namespace {
-
-static auto syslog_address_helper(const db::config& cfg)
-{
-    return cfg.audit_unix_socket_path.is_set()
-        ? unix_domain_addr(cfg.audit_unix_socket_path())
-        : unix_domain_addr(_PATH_LOG);
-}
-
-static std::string json_escape(std::string_view str) {
-    std::string result;
-    result.reserve(str.size() * 1.2);
-    for (auto c : str) {
-        if (c == '"' || c == '\\') {
-            result.push_back('\\');
-        }
-        result.push_back(c);
-    }
-    return result;
-}
-
-}
-
-future<> audit_syslog_storage_helper::syslog_send_helper(const sstring& msg) {
-    try {
-        auto lock = co_await get_units(_semaphore, 1, std::chrono::hours(1));
-        co_await _sender.send(_syslog_address, net::packet{msg.data(), msg.size()});
-    }
-    catch (const std::exception& e) {
-        auto error_msg = seastar::format(
-            "Syslog audit backend failed (sending a message to {} resulted in {}).",
-            _syslog_address,
-            e
-        );
-        logger.error("{}", error_msg);
-        throw audit_exception(std::move(error_msg));
-    }
-}
-
-audit_syslog_storage_helper::audit_syslog_storage_helper(cql3::query_processor& qp, service::migration_manager&) :
-    _syslog_address(syslog_address_helper(qp.db().get_config())),
-    _sender(make_unbound_datagram_channel(AF_UNIX)),
-    _semaphore(1) {
-}
-
-audit_syslog_storage_helper::~audit_syslog_storage_helper() {
-}
-
-/*
- * We don't use openlog and syslog directly because it's already used by logger.
- * Audit needs to use different ident so than logger but syslog.h uses a global ident
- * and it's not possible to use more than one in a program.
- *
- * To work around it we directly communicate with the socket.
- */
-future<> audit_syslog_storage_helper::start(const db::config& cfg) {
-    if (this_shard_id() != 0) {
-        co_return;
-    }
-
-    co_await syslog_send_helper("Initializing syslog audit backend.");
-}
-
-future<> audit_syslog_storage_helper::stop() {
-    _sender.shutdown_output();
-    co_return;
-}
-
-future<> audit_syslog_storage_helper::write(const audit_info* audit_info,
-                                            socket_address node_ip,
-                                            socket_address client_ip,
-                                            db::consistency_level cl,
-                                            const sstring& username,
-                                            bool error) {
-    auto now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
-    tm time;
-    localtime_r(&now, &time);
-    sstring msg = seastar::format(R"(<{}>{:%h %e %T} scylla-audit: node="{}" category="{}" cl="{}" error="{}" keyspace="{}" query="{}" client_ip="{}" table="{}" username="{}")",
-                                    LOG_NOTICE | LOG_USER,
-                                    time,
-                                    node_ip,
-                                    audit_info->category_string(),
-                                    cl,
-                                    (error ? "true" : "false"),
-                                    audit_info->keyspace(),
-                                    json_escape(audit_info->query()),
-                                    client_ip,
-                                    audit_info->table(),
-                                    username);
-
-    co_await syslog_send_helper(msg);
-}
-
-future<> audit_syslog_storage_helper::write_login(const sstring& username,
-                                                  socket_address node_ip,
-                                                  socket_address client_ip,
-                                                  bool error) {
-
-    auto now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
-    tm time;
-    localtime_r(&now, &time);
-    sstring msg = seastar::format(R"(<{}>{:%h %e %T} scylla-audit: node="{}", category="AUTH", cl="", error="{}", keyspace="", query="", client_ip="{}", table="", username="{}")",
-                                    LOG_NOTICE | LOG_USER,
-                                    time,
-                                    node_ip,
-                                    (error ? "true" : "false"),
-                                    client_ip,
-                                    username);
-
-    co_await syslog_send_helper(msg.c_str());
-}
-
-using registry = class_registrator<storage_helper, audit_syslog_storage_helper, cql3::query_processor&, service::migration_manager&>;
-static registry registrator1("audit_syslog_storage_helper");
-
-}

audit/audit_syslog_storage_helper.hh

@@ -1,47 +0,0 @@
-/*
- * Copyright (C) 2017 ScyllaDB
- */
-
-/*
- * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
- */
-#pragma once
-
-#include <seastar/net/api.hh>
-
-#include "audit/audit.hh"
-#include "storage_helper.hh"
-#include "db/config.hh"
-
-namespace service {
-
-class migration_manager;
-
-};
-
-namespace audit {
-
-class audit_syslog_storage_helper : public storage_helper {
-    socket_address _syslog_address;
-    net::datagram_channel _sender;
-    seastar::semaphore _semaphore;
-
-    future<> syslog_send_helper(const sstring& msg);
-public:
-    explicit audit_syslog_storage_helper(cql3::query_processor&, service::migration_manager&);
-    virtual ~audit_syslog_storage_helper();
-    virtual future<> start(const db::config& cfg) override;
-    virtual future<> stop() override;
-    virtual future<> write(const audit_info* audit_info,
-                           socket_address node_ip,
-                           socket_address client_ip,
-                           db::consistency_level cl,
-                           const sstring& username,
-                           bool error) override;
-    virtual future<> write_login(const sstring& username,
-                                 socket_address node_ip,
-                                 socket_address client_ip,
-                                 bool error) override;
-};
-
-}

audit/storage_helper.hh

@@ -1,34 +0,0 @@
-/*
- * Copyright (C) 2017 ScyllaDB
- */
-
-/*
- * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
- */
-#pragma once
-
-#include "audit/audit.hh"
-#include <seastar/core/future.hh>
-
-namespace audit {
-
-class storage_helper {
-public:
-    using ptr_type = std::unique_ptr<storage_helper>;
-    storage_helper() {}
-    virtual ~storage_helper() {}
-    virtual future<> start(const db::config& cfg) = 0;
-    virtual future<> stop() = 0;
-    virtual future<> write(const audit_info* audit_info,
-                           socket_address node_ip,
-                           socket_address client_ip,
-                           db::consistency_level cl,
-                           const sstring& username,
-                           bool error) = 0;
-    virtual future<> write_login(const sstring& username,
-                                 socket_address node_ip,
-                                 socket_address client_ip,
-                                 bool error) = 0;
-};
-
-}

auth/CMakeLists.txt

@@ -1,6 +1,4 @@
 include(add_whole_archive)
-find_package(OpenLDAP REQUIRED
-  ldap)
 
 add_library(scylla_auth STATIC)
 target_sources(scylla_auth
@@ -12,7 +10,6 @@ target_sources(scylla_auth
     certificate_authenticator.cc
     common.cc
     default_authorizer.cc
-    ldap_role_manager.cc
     password_authenticator.cc
     passwords.cc
     permission.cc
@@ -21,7 +18,6 @@ target_sources(scylla_auth
     role_or_anonymous.cc
     roles-metadata.cc
     sasl_challenge.cc
-    saslauthd_authenticator.cc
     service.cc
     standard_role_manager.cc
     transitional.cc
@@ -35,14 +31,12 @@ target_link_libraries(scylla_auth
     xxHash::xxhash
   PRIVATE
     absl::headers
-    OpenLDAP::ldap
     cql3
     idl
-    ldap
     wasmtime_bindings
     libxcrypt::libxcrypt)
 
 add_whole_archive(auth scylla_auth)
 
 check_headers(check-headers scylla_auth
-  GLOB_RECURSE ${CMAKE_CURRENT_SOURCE_DIR}/*.hh)
+  GLOB_RECURSE ${CMAKE_CURRENT_SOURCE_DIR}/*.hh)

auth/allow_all_authenticator.cc

@@ -9,7 +9,6 @@
 #include "auth/allow_all_authenticator.hh"
 
 #include "service/migration_manager.hh"
-#include "utils/alien_worker.hh"
 #include "utils/class_registrator.hh"
 
 namespace auth {
@@ -22,7 +21,6 @@ static const class_registrator<
         allow_all_authenticator,
         cql3::query_processor&,
         ::service::raft_group0_client&,
-        ::service::migration_manager&,
-        utils::alien_worker&> registration("org.apache.cassandra.auth.AllowAllAuthenticator");
+        ::service::migration_manager&> registration("org.apache.cassandra.auth.AllowAllAuthenticator");
 
 }

auth/allow_all_authenticator.hh

@@ -13,7 +13,6 @@
 #include "auth/authenticated_user.hh"
 #include "auth/authenticator.hh"
 #include "auth/common.hh"
-#include "utils/alien_worker.hh"
 
 namespace cql3 {
 class query_processor;
@@ -29,7 +28,7 @@ extern const std::string_view allow_all_authenticator_name;
 
 class allow_all_authenticator final : public authenticator {
 public:
-    allow_all_authenticator(cql3::query_processor&, ::service::raft_group0_client&, ::service::migration_manager&, utils::alien_worker&) {
+    allow_all_authenticator(cql3::query_processor&, ::service::raft_group0_client&, ::service::migration_manager&) {
     }
 
     virtual future<> start() override {
@@ -84,10 +83,6 @@ public:
     virtual ::shared_ptr<sasl_challenge> new_sasl_challenge() const override {
         throw std::runtime_error("Should not reach");
     }
-
-    virtual future<> ensure_superuser_is_created() const override {
-        return make_ready_future<>();
-    }
 };
 
 }

auth/authenticator.cc

@@ -14,8 +14,6 @@
 
 const sstring auth::authenticator::USERNAME_KEY("username");
 const sstring auth::authenticator::PASSWORD_KEY("password");
-const sstring auth::authenticator::SERVICE_KEY("service");
-const sstring auth::authenticator::REALM_KEY("realm");
 
 future<std::optional<auth::authenticated_user>> auth::authenticator::authenticate(session_dn_func) const {
     return make_ready_future<std::optional<auth::authenticated_user>>(std::nullopt);

auth/authenticator.hh

@@ -67,12 +67,6 @@ public:
     ///
     static const sstring PASSWORD_KEY;
 
-    /// Service for SASL authentication.
-    static const sstring SERVICE_KEY;
-
-    /// Realm for SASL authentication.
-    static const sstring REALM_KEY;
-
     using credentials_map = std::unordered_map<sstring, sstring>;
 
     virtual ~authenticator() = default;
@@ -159,8 +153,6 @@ public:
     virtual const resource_set& protected_resources() const = 0;
 
     virtual ::shared_ptr<sasl_challenge> new_sasl_challenge() const = 0;
-
-    virtual future<> ensure_superuser_is_created() const = 0;
 };
 
 }

auth/certificate_authenticator.cc

@@ -33,14 +33,13 @@ static const class_registrator<auth::authenticator
     , auth::certificate_authenticator
     , cql3::query_processor&
     , ::service::raft_group0_client&
-    , ::service::migration_manager&
-    , utils::alien_worker&> cert_auth_reg(CERT_AUTH_NAME);
+    , ::service::migration_manager&> cert_auth_reg(CERT_AUTH_NAME);
 
 enum class auth::certificate_authenticator::query_source {
     subject, altname
 };
 
-auth::certificate_authenticator::certificate_authenticator(cql3::query_processor& qp, ::service::raft_group0_client&, ::service::migration_manager&, utils::alien_worker&)
+auth::certificate_authenticator::certificate_authenticator(cql3::query_processor& qp, ::service::raft_group0_client&, ::service::migration_manager&)
     : _queries([&] {
         auto& conf = qp.db().get_config();
         auto queries = conf.auth_certificate_role_queries();

auth/certificate_authenticator.hh

@@ -10,7 +10,6 @@
 #pragma once
 
 #include "auth/authenticator.hh"
-#include "utils/alien_worker.hh"
 #include <boost/regex_fwd.hpp>  // IWYU pragma: keep
 
 namespace cql3 {
@@ -32,7 +31,7 @@ class certificate_authenticator : public authenticator {
     enum class query_source;
     std::vector<std::pair<query_source, boost::regex>> _queries;
 public:
-    certificate_authenticator(cql3::query_processor&, ::service::raft_group0_client&, ::service::migration_manager&, utils::alien_worker&);
+    certificate_authenticator(cql3::query_processor&, ::service::raft_group0_client&, ::service::migration_manager&);
     ~certificate_authenticator();
 
     future<> start() override;
@@ -57,10 +56,6 @@ public:
     const resource_set& protected_resources() const override;
 
     ::shared_ptr<sasl_challenge> new_sasl_challenge() const override;
-
-    virtual future<> ensure_superuser_is_created() const override {
-        return make_ready_future<>();
-    }
 private:
 };
 

auth/common.cc

@@ -119,11 +119,6 @@ future<> create_legacy_metadata_table_if_missing(
     return qs;
 }
 
-::service::raft_timeout get_raft_timeout() noexcept {
-    auto dur = internal_distributed_query_state().get_client_state().get_timeout_config().other_timeout;
-    return ::service::raft_timeout{.value = lowres_clock::now() + dur};
-}
-
 static future<> announce_mutations_with_guard(
         ::service::raft_group0_client& group0_client,
         std::vector<canonical_mutation> muts,

auth/common.hh

@@ -17,7 +17,6 @@
 
 #include "types/types.hh"
 #include "service/raft/raft_group0_client.hh"
-#include "timeout_config.hh"
 
 using namespace std::chrono_literals;
 
@@ -78,8 +77,6 @@ future<> create_legacy_metadata_table_if_missing(
 ///
 ::service::query_state& internal_distributed_query_state() noexcept;
 
-::service::raft_timeout get_raft_timeout() noexcept;
-
 // Execute update query via group0 mechanism, mutations will be applied on all nodes.
 // Use this function when need to perform read before write on a single guard or if
 // you have more than one mutation and potentially exceed single command size limit.

auth/default_authorizer.cc

@@ -16,6 +16,8 @@ extern "C" {
 #include <unistd.h>
 }
 
+#include <boost/algorithm/string/join.hpp>
+#include <boost/range.hpp>
 #include <seastar/core/seastar.hh>
 #include <seastar/core/sleep.hh>
 

auth/ldap_role_manager.cc

@@ -1,344 +0,0 @@
-/*
- * Copyright (C) 2019 ScyllaDB
- */
-
-/*
- * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
- */
-
-#include "ldap_role_manager.hh"
-
-#include <boost/algorithm/string/replace.hpp>
-#include <fmt/format.h>
-#include <fmt/ranges.h>
-#include <ldap.h>
-#include <seastar/core/seastar.hh>
-#include <seastar/core/sstring.hh>
-#include <seastar/net/dns.hh>
-#include <seastar/util/log.hh>
-#include <seastar/core/coroutine.hh>
-#include <vector>
-
-#include "common.hh"
-#include "cql3/query_processor.hh"
-#include "exceptions/exceptions.hh"
-#include "seastarx.hh"
-#include "service/raft/raft_group0_client.hh"
-#include "utils/class_registrator.hh"
-#include "db/config.hh"
-#include "utils/exponential_backoff_retry.hh"
-
-namespace {
-
-logger mylog{"ldap_role_manager"}; // `log` is taken by math.
-
-struct url_desc_deleter {
-    void operator()(LDAPURLDesc *p) {
-        ldap_free_urldesc(p);
-    }
-};
-
-using url_desc_ptr = std::unique_ptr<LDAPURLDesc, url_desc_deleter>;
-
-url_desc_ptr parse_url(std::string_view url) {
-    LDAPURLDesc *desc = nullptr;
-    if (ldap_url_parse(url.data(), &desc)) {
-        mylog.error("error in ldap_url_parse({})", url);
-    }
-    return url_desc_ptr(desc);
-}
-
-/// Extracts attribute \p attr from all entries in \p res.
-std::vector<sstring> get_attr_values(LDAP* ld, LDAPMessage* res, const char* attr) {
-    std::vector<sstring> values;
-    mylog.debug("Analyzing search results");
-    for (auto e = ldap_first_entry(ld, res); e; e = ldap_next_entry(ld, e)) {
-        struct deleter {
-            void operator()(berval** p) { ldap_value_free_len(p); }
-            void operator()(char* p) { ldap_memfree(p); }
-        };
-        const std::unique_ptr<char, deleter> dname(ldap_get_dn(ld, e));
-        mylog.debug("Analyzing entry {}", dname.get());
-        const std::unique_ptr<berval*, deleter> vals(ldap_get_values_len(ld, e, attr));
-        if (!vals) {
-            mylog.warn("LDAP entry {} has no attribute {}", dname.get(), attr);
-            continue;
-        }
-        for (size_t i = 0; vals.get()[i]; ++i) {
-            values.emplace_back(vals.get()[i]->bv_val, vals.get()[i]->bv_len);
-        }
-    }
-    mylog.debug("Done analyzing search results; extracted roles {}", values);
-    return values;
-}
-
-const char* ldap_role_manager_full_name = "com.scylladb.auth.LDAPRoleManager";
-
-} // anonymous namespace
-
-namespace auth {
-
-static const class_registrator<
-    role_manager,
-    ldap_role_manager,
-    cql3::query_processor&,
-    ::service::raft_group0_client&,
-    ::service::migration_manager&> registration(ldap_role_manager_full_name);
-
-ldap_role_manager::ldap_role_manager(
-        std::string_view query_template, std::string_view target_attr, std::string_view bind_name, std::string_view bind_password,
-        cql3::query_processor& qp, ::service::raft_group0_client& rg0c, ::service::migration_manager& mm)
-        : _std_mgr(qp, rg0c, mm), _group0_client(rg0c), _query_template(query_template), _target_attr(target_attr), _bind_name(bind_name)
-        , _bind_password(bind_password)
-        , _connection_factory(bind(std::mem_fn(&ldap_role_manager::reconnect), std::ref(*this))) {
-}
-
-ldap_role_manager::ldap_role_manager(cql3::query_processor& qp, ::service::raft_group0_client& rg0c, ::service::migration_manager& mm)
-    : ldap_role_manager(
-            qp.db().get_config().ldap_url_template(),
-            qp.db().get_config().ldap_attr_role(),
-            qp.db().get_config().ldap_bind_dn(),
-            qp.db().get_config().ldap_bind_passwd(),
-            qp,
-            rg0c,
-            mm) {
-}
-
-std::string_view ldap_role_manager::qualified_java_name() const noexcept {
-    return ldap_role_manager_full_name;
-}
-
-const resource_set& ldap_role_manager::protected_resources() const {
-    return _std_mgr.protected_resources();
-}
-
-future<> ldap_role_manager::start() {
-    if (!parse_url(get_url("dummy-user"))) { // Just need host and port -- any user should do.
-        return make_exception_future(
-                std::runtime_error(fmt::format("error getting LDAP server address from template {}", _query_template)));
-    }
-    return _std_mgr.start();
-}
-
-using conn_ptr = lw_shared_ptr<ldap_connection>;
-
-future<conn_ptr> ldap_role_manager::connect() {
-    const auto desc = parse_url(get_url("dummy-user")); // Just need host and port -- any user should do.
-    if (!desc) {
-        co_return coroutine::exception(std::make_exception_ptr(std::runtime_error("connect attempted before a successful start")));
-    }
-    net::inet_address host = co_await net::dns::resolve_name(desc->lud_host);
-    const socket_address addr(host, uint16_t(desc->lud_port));
-    connected_socket sock = co_await seastar::connect(addr);
-    auto conn = make_lw_shared<ldap_connection>(std::move(sock));
-    sstring error;
-    try {
-        ldap_msg_ptr response = co_await conn->simple_bind(_bind_name.c_str(), _bind_password.c_str());
-        if (!response || ldap_msgtype(response.get()) != LDAP_RES_BIND) {
-            error = format("simple_bind error: {}", conn->get_error());
-        }
-    } catch (...) {
-        error = format("connect error: {}", std::current_exception());
-    }
-    if (!error.empty()) {
-        co_await conn->close();
-        co_return coroutine::exception(std::make_exception_ptr(std::runtime_error(std::move(error))));
-    }
-    co_return std::move(conn);
-}
-
-future<conn_ptr> ldap_role_manager::reconnect() {
-    unsigned retries_left = 5;
-    using namespace std::literals::chrono_literals;
-    conn_ptr conn = co_await exponential_backoff_retry::do_until_value(1s, 32s, _as, [this, &retries_left] () -> future<std::optional<conn_ptr>> {
-        if (!retries_left) {
-            co_return conn_ptr{};
-        }
-        mylog.trace("reconnect() retrying ({} attempts left)", retries_left);
-        --retries_left;
-        try {
-            co_return co_await connect();
-        } catch (...) {
-            mylog.error("error in reconnect: {}", std::current_exception());
-        }
-        co_return std::nullopt;
-    });
-
-    mylog.trace("reconnect() finished backoff, conn={}", reinterpret_cast<void*>(conn.get()));
-    if (conn) {
-        co_return std::move(conn);
-    }
-    co_return coroutine::exception(std::make_exception_ptr(std::runtime_error("reconnect failed after 5 attempts")));
-}
-
-future<> ldap_role_manager::stop() {
-    _as.request_abort();
-    return _std_mgr.stop().then([this] { return _connection_factory.stop(); });
-}
-
-future<> ldap_role_manager::create(std::string_view name, const role_config& config, ::service::group0_batch& mc) {
-    return _std_mgr.create(name, config, mc);
-}
-
-future<> ldap_role_manager::drop(std::string_view name, ::service::group0_batch& mc) {
-    return _std_mgr.drop(name, mc);
-}
-
-future<> ldap_role_manager::alter(std::string_view name, const role_config_update& config, ::service::group0_batch& mc) {
-    return _std_mgr.alter(name, config, mc);
-}
-
-future<> ldap_role_manager::grant(std::string_view, std::string_view, ::service::group0_batch& mc) {
-    return make_exception_future<>(exceptions::invalid_request_exception("Cannot grant roles with LDAPRoleManager."));
-}
-
-future<> ldap_role_manager::revoke(std::string_view, std::string_view, ::service::group0_batch& mc) {
-    return make_exception_future<>(exceptions::invalid_request_exception("Cannot revoke roles with LDAPRoleManager."));
-}
-
-future<role_set> ldap_role_manager::query_granted(std::string_view grantee_name, recursive_role_query) {
-    const auto url = get_url(grantee_name.data());
-    auto desc = parse_url(url);
-    if (!desc) {
-        return make_exception_future<role_set>(std::runtime_error(format("Error parsing URL {}", url)));
-    }
-    return _connection_factory.with_connection([this, desc = std::move(desc), grantee_name_ = sstring(grantee_name)]
-                                               (ldap_connection& conn) -> future<role_set> {
-        sstring grantee_name = std::move(grantee_name_);
-        ldap_msg_ptr res = co_await conn.search(desc->lud_dn, desc->lud_scope, desc->lud_filter, desc->lud_attrs,
-                           /*attrsonly=*/0, /*serverctrls=*/nullptr, /*clientctrls=*/nullptr,
-                           /*timeout=*/nullptr, /*sizelimit=*/0);
-        mylog.trace("query_granted: got search results");
-        const auto mtype = ldap_msgtype(res.get());
-        if (mtype != LDAP_RES_SEARCH_ENTRY && mtype != LDAP_RES_SEARCH_RESULT && mtype != LDAP_RES_SEARCH_REFERENCE) {
-            mylog.error("ldap search yielded result {} of type {}", static_cast<const void*>(res.get()), mtype);
-            co_return coroutine::exception(std::make_exception_ptr(std::runtime_error("ldap_role_manager: search result has wrong type")));
-        }
-        std::vector<sstring> values = get_attr_values(conn.get_ldap(), res.get(), _target_attr.c_str());
-        auth::role_set valid_roles{grantee_name};
-
-        // Each value is a role to be granted.
-        co_await parallel_for_each(values, [this, &valid_roles] (const sstring& ldap_role) {
-            return _std_mgr.exists(ldap_role).then([&valid_roles, &ldap_role] (bool exists) {
-                if (exists) {
-                    valid_roles.insert(ldap_role);
-                } else {
-                    mylog.error("unrecognized role received from LDAP: {}", ldap_role);
-                }
-            });
-        });
-
-        co_return std::move(valid_roles);
-    });
-}
-
-future<role_to_directly_granted_map>
-ldap_role_manager::query_all_directly_granted(::service::query_state& qs) {
-    role_to_directly_granted_map result;
-    auto roles = co_await query_all(qs);
-    for (auto& role: roles) {
-        auto granted_set = co_await query_granted(role, recursive_role_query::no);
-        for (auto& granted: granted_set) {
-            if (granted != role) {
-                result.insert({role, granted});
-            }
-        }
-    }
-    co_return result;
-}
-
-future<role_set> ldap_role_manager::query_all(::service::query_state& qs) {
-    return _std_mgr.query_all(qs);
-}
-
-future<> ldap_role_manager::create_role(std::string_view role_name) {
-    return smp::submit_to(0, [this, role_name] () -> future<> {
-        int retries = 10;
-        while (true) {
-            auto guard = co_await _group0_client.start_operation(_as, ::service::raft_timeout{});
-            ::service::group0_batch batch(std::move(guard));
-            auto cfg = role_config{.can_login = true};
-            try {
-                co_await create(role_name, cfg, batch);
-                co_await std::move(batch).commit(_group0_client, _as, ::service::raft_timeout{});
-            } catch (const role_already_exists&) {
-                // ok
-            } catch (const ::service::group0_concurrent_modification& ex) {
-                mylog.warn("Failed to auto-create role \"{}\" due to guard conflict.{}.",
-                        role_name, retries ? " Retrying" : " Number of retries exceeded, giving up");
-                if (retries--) {
-                    continue;
-                }
-                throw;
-            }
-            break;
-        }
-        // make sure to wait until create mutations are applied locally
-        (void)(co_await _group0_client.start_operation(_as, ::service::raft_timeout{}));
-    });
-}
-
-future<bool> ldap_role_manager::exists(std::string_view role_name) {
-    bool exists = co_await _std_mgr.exists(role_name);
-    if (exists) {
-        co_return true;
-    }
-    role_set roles = co_await query_granted(role_name, recursive_role_query::yes);
-    // A role will get auto-created if it's already assigned any permissions.
-    // The role set will always contains at least a single entry (the role itself),
-    // so auto-creation is only triggered if at least one more external role is assigned.
-    if (roles.size() > 1) {
-        mylog.info("Auto-creating user {}", role_name);
-        try {
-            co_await create_role(role_name);
-            exists = true;
-        } catch (...) {
-            mylog.error("Failed to auto-create role {}: {}", role_name, std::current_exception());
-            exists = false;
-        }
-        co_return exists;
-    }
-    mylog.debug("Role {} will not be auto-created", role_name);
-    co_return false;
-}
-
-future<bool> ldap_role_manager::is_superuser(std::string_view role_name) {
-    return _std_mgr.is_superuser(role_name);
-}
-
-future<bool> ldap_role_manager::can_login(std::string_view role_name) {
-    return _std_mgr.can_login(role_name);
-}
-
-future<std::optional<sstring>> ldap_role_manager::get_attribute(
-        std::string_view role_name, std::string_view attribute_name, ::service::query_state& qs) {
-    return _std_mgr.get_attribute(role_name, attribute_name, qs);
-}
-
-future<role_manager::attribute_vals> ldap_role_manager::query_attribute_for_all(std::string_view attribute_name, ::service::query_state& qs) {
-    return _std_mgr.query_attribute_for_all(attribute_name, qs);
-}
-
-future<> ldap_role_manager::set_attribute(
-        std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value, ::service::group0_batch& mc) {
-    return _std_mgr.set_attribute(role_name, attribute_value, attribute_value, mc);
-}
-
-future<> ldap_role_manager::remove_attribute(std::string_view role_name, std::string_view attribute_name, ::service::group0_batch& mc) {
-    return _std_mgr.remove_attribute(role_name, attribute_name, mc);
-}
-
-sstring ldap_role_manager::get_url(std::string_view user) const {
-    return boost::replace_all_copy(_query_template, "{USER}", user);
-}
-
-future<std::vector<cql3::description>> ldap_role_manager::describe_role_grants() {
-    // Since grants are performed by the ldap admin, we shouldn't echo them back
-    co_return std::vector<cql3::description>();
-}
-
-future<> ldap_role_manager::ensure_superuser_is_created() {
-    return _std_mgr.ensure_superuser_is_created();
-}
-
-} // namespace auth

auth/ldap_role_manager.hh

@@ -1,114 +0,0 @@
-/*
- * Copyright (C) 2019 ScyllaDB
- */
-
-/*
- * SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
- */
-
-
-#pragma once
-
-#include <seastar/core/abort_source.hh>
-#include <stdexcept>
-
-#include "ent/ldap/ldap_connection.hh"
-#include "standard_role_manager.hh"
-
-namespace auth {
-
-/// Queries an LDAP server for roles.
-///
-/// Since LDAP grants and revokes roles, calling grant() and revoke() is disallowed.
-///
-/// We query LDAP for a list of a particular user's roles, and the results must match roles that exist in the
-/// database.  Furthermore, the user must have already authenticated to Scylla, meaning it, too, exists in the
-/// database.  Therefore, some of the role_manager functionality is provided by a standard_role_manager under
-/// the hood.  For example, listing all roles or checking if the user can login cannot currently be determined
-/// by querying LDAP, so they are delegated to the standard_role_manager.
-class ldap_role_manager : public role_manager {
-    standard_role_manager _std_mgr;
-    ::service::raft_group0_client& _group0_client;
-    seastar::sstring _query_template; ///< LDAP URL dictating which query to make.
-    seastar::sstring _target_attr; ///< LDAP entry attribute containing the Scylla role name.
-    seastar::sstring _bind_name; ///< Username for LDAP simple bind.
-    seastar::sstring _bind_password; ///< Password for LDAP simple bind.
-    mutable ldap_reuser _connection_factory; // Potentially modified by query_granted().
-    seastar::abort_source _as;
-  public:
-    ldap_role_manager(
-            std::string_view query_template, ///< LDAP query template as described in Scylla documentation.
-            std::string_view target_attr, ///< LDAP entry attribute containing the Scylla role name.
-            std::string_view bind_name, ///< LDAP bind credentials.
-            std::string_view bind_password, ///< LDAP bind credentials.
-            cql3::query_processor& qp, ///< Passed to standard_role_manager.
-            ::service::raft_group0_client& rg0c, ///< Passed to standard_role_manager.
-            ::service::migration_manager& mm ///< Passed to standard_role_manager.
-    );
-
-    /// Retrieves LDAP configuration entries from qp and invokes the other constructor.  Required by
-    /// class_registrator<role_manager>.
-    ldap_role_manager(cql3::query_processor& qp, ::service::raft_group0_client& rg0c, ::service::migration_manager& mm);
-
-    /// Thrown when query-template parsing fails.
-    struct url_error : public std::runtime_error {
-        using runtime_error::runtime_error;
-    };
-
-    std::string_view qualified_java_name() const noexcept override;
-
-    const resource_set& protected_resources() const override;
-
-    future<> start() override;
-
-    future<> stop() override;
-
-    future<> create(std::string_view, const role_config&, ::service::group0_batch& mc) override;
-
-    future<> drop(std::string_view, ::service::group0_batch& mc) override;
-
-    future<> alter(std::string_view, const role_config_update&, ::service::group0_batch& mc) override;
-
-    future<> grant(std::string_view, std::string_view, ::service::group0_batch& mc) override;
-
-    future<> revoke(std::string_view, std::string_view, ::service::group0_batch& mc) override;
-
-    future<role_set> query_granted(std::string_view, recursive_role_query) override;
-
-    future<role_to_directly_granted_map> query_all_directly_granted(::service::query_state&) override;
-
-    future<role_set> query_all(::service::query_state&) override;
-
-    future<bool> exists(std::string_view) override;
-
-    future<bool> is_superuser(std::string_view) override;
-
-    future<bool> can_login(std::string_view) override;
-
-    future<std::optional<sstring>> get_attribute(std::string_view, std::string_view, ::service::query_state&) override;
-
-    future<role_manager::attribute_vals> query_attribute_for_all(std::string_view, ::service::query_state&) override;
-
-    future<> set_attribute(std::string_view, std::string_view, std::string_view, ::service::group0_batch& mc) override;
-
-    future<> remove_attribute(std::string_view, std::string_view, ::service::group0_batch& mc) override;
-
-    future<std::vector<cql3::description>> describe_role_grants() override;
-  private:
-    /// Connects to the LDAP server indicated by _query_template and executes LDAP bind using _bind_name and
-    /// _bind_password.  Returns the resulting ldap_connection.
-    future<lw_shared_ptr<ldap_connection>> connect();
-
-    /// Invokes connect() repeatedly with backoff, until it succeeds or retry limit is reached.
-    future<seastar::lw_shared_ptr<ldap_connection>> reconnect();
-
-    /// Macro-expands _query_template, returning the result.
-    sstring get_url(std::string_view user) const;
-
-    /// Used to auto-create roles returned by ldap.
-    future<> create_role(std::string_view role_name);
-
-    future<> ensure_superuser_is_created() override;
-};
-
-} // namespace auth

auth/maintenance_socket_role_manager.cc

@@ -78,11 +78,11 @@ future<role_set> maintenance_socket_role_manager::query_granted(std::string_view
     return operation_not_supported_exception<role_set>("QUERY GRANTED");
 }
 
-future<role_to_directly_granted_map> maintenance_socket_role_manager::query_all_directly_granted(::service::query_state&) {
+future<role_to_directly_granted_map> maintenance_socket_role_manager::query_all_directly_granted() {
     return operation_not_supported_exception<role_to_directly_granted_map>("QUERY ALL DIRECTLY GRANTED");
 }
 
-future<role_set> maintenance_socket_role_manager::query_all(::service::query_state&) {
+future<role_set> maintenance_socket_role_manager::query_all() {
     return operation_not_supported_exception<role_set>("QUERY ALL");
 }
 
@@ -98,11 +98,11 @@ future<bool> maintenance_socket_role_manager::can_login(std::string_view role_na
     return make_ready_future<bool>(true);
 }
 
-future<std::optional<sstring>> maintenance_socket_role_manager::get_attribute(std::string_view role_name, std::string_view attribute_name, ::service::query_state&) {
+future<std::optional<sstring>> maintenance_socket_role_manager::get_attribute(std::string_view role_name, std::string_view attribute_name) {
     return operation_not_supported_exception<std::optional<sstring>>("GET ATTRIBUTE");
 }
 
-future<role_manager::attribute_vals> maintenance_socket_role_manager::query_attribute_for_all(std::string_view attribute_name, ::service::query_state&) {
+future<role_manager::attribute_vals> maintenance_socket_role_manager::query_attribute_for_all(std::string_view attribute_name) {
     return operation_not_supported_exception<role_manager::attribute_vals>("QUERY ATTRIBUTE");
 }
 

auth/maintenance_socket_role_manager.hh

@@ -53,9 +53,9 @@ public:
 
     virtual future<role_set> query_granted(std::string_view grantee_name, recursive_role_query) override;
 
-    virtual future<role_to_directly_granted_map> query_all_directly_granted(::service::query_state&) override;
+    virtual future<role_to_directly_granted_map> query_all_directly_granted() override;
 
-    virtual future<role_set> query_all(::service::query_state&) override;
+    virtual future<role_set> query_all() override;
 
     virtual future<bool> exists(std::string_view role_name) override;
 
@@ -63,9 +63,9 @@ public:
 
     virtual future<bool> can_login(std::string_view role_name) override;
 
-    virtual future<std::optional<sstring>> get_attribute(std::string_view role_name, std::string_view attribute_name, ::service::query_state&) override;
+    virtual future<std::optional<sstring>> get_attribute(std::string_view role_name, std::string_view attribute_name) override;
 
-    virtual future<role_manager::attribute_vals> query_attribute_for_all(std::string_view attribute_name, ::service::query_state&) override;
+    virtual future<role_manager::attribute_vals> query_attribute_for_all(std::string_view attribute_name) override;
 
     virtual future<> set_attribute(std::string_view role_name, std::string_view attribute_name, std::string_view attribute_value, ::service::group0_batch& mc) override;
 

auth/password_authenticator.cc

@@ -48,14 +48,14 @@ static const class_registrator<
         password_authenticator,
         cql3::query_processor&,
         ::service::raft_group0_client&,
-        ::service::migration_manager&,
-        utils::alien_worker&> password_auth_reg("org.apache.cassandra.auth.PasswordAuthenticator");
+        ::service::migration_manager&> password_auth_reg("org.apache.cassandra.auth.PasswordAuthenticator");
 
 static thread_local auto rng_for_salt = std::default_random_engine(std::random_device{}());
 
 static std::string_view get_config_value(std::string_view value, std::string_view def) {
     return value.empty() ? def : value;
 }
+
 std::string password_authenticator::default_superuser(const db::config& cfg) {
     return std::string(get_config_value(cfg.auth_superuser_name(), DEFAULT_USER_NAME));
 }
@@ -63,13 +63,12 @@ std::string password_authenticator::default_superuser(const db::config& cfg) {
 password_authenticator::~password_authenticator() {
 }
 
-password_authenticator::password_authenticator(cql3::query_processor& qp, ::service::raft_group0_client& g0, ::service::migration_manager& mm, utils::alien_worker& hashing_worker)
+password_authenticator::password_authenticator(cql3::query_processor& qp, ::service::raft_group0_client& g0, ::service::migration_manager& mm)
     : _qp(qp)
     , _group0_client(g0)
     , _migration_manager(mm)
     , _stopped(make_ready_future<>()) 
     , _superuser(default_superuser(qp.db().get_config()))
-    , _hashing_worker(hashing_worker)
 {}
 
 static bool has_salted_hash(const cql3::untyped_result_set_row& row) {
@@ -118,101 +117,36 @@ future<> password_authenticator::migrate_legacy_metadata() const {
     });
 }
 
-future<> password_authenticator::legacy_create_default_if_missing() {
-    SCYLLA_ASSERT(legacy_mode(_qp));
+future<> password_authenticator::create_default_if_missing() {
     const auto exists = co_await default_role_row_satisfies(_qp, &has_salted_hash, _superuser);
     if (exists) {
         co_return;
     }
     std::string salted_pwd(get_config_value(_qp.db().get_config().auth_superuser_salted_password(), ""));
     if (salted_pwd.empty()) {
-        salted_pwd = passwords::hash(DEFAULT_USER_PASSWORD, rng_for_salt, _scheme);
+        salted_pwd = passwords::hash(DEFAULT_USER_PASSWORD, rng_for_salt);
     }
     const auto query = update_row_query();
-    co_await _qp.execute_internal(
+    if (legacy_mode(_qp)) {
+        co_await _qp.execute_internal(
             query,
             db::consistency_level::QUORUM,
             internal_distributed_query_state(),
             {salted_pwd, _superuser},
             cql3::query_processor::cache_internal::no);
-    plogger.info("Created default superuser authentication record.");
-}
-
-future<> password_authenticator::maybe_create_default_password() {
-    auto needs_password = [this] () -> future<bool> {
-        const sstring query = seastar::format("SELECT * FROM {}.{} WHERE is_superuser = true ALLOW FILTERING", get_auth_ks_name(_qp), meta::roles_table::name);
-        auto results = co_await _qp.execute_internal(query,
-                db::consistency_level::LOCAL_ONE,
-                internal_distributed_query_state(), cql3::query_processor::cache_internal::yes);
-        // Don't add default password if
-        // - there is no default superuser
-        // - there is a superuser with a password.
-        bool has_default = false;
-        bool has_superuser_with_password = false;
-        for (auto& result : *results) {
-            if (result.get_as<sstring>(meta::roles_table::role_col_name) == _superuser) {
-                has_default = true;
-            }
-            if (has_salted_hash(result)) {
-                has_superuser_with_password = true;
-            }
-        }
-        co_return has_default && !has_superuser_with_password;
-    };
-    if (!co_await needs_password()) {
-        co_return;
-    }
-    // We don't want to start operation earlier to avoid quorum requirement in
-    // a common case.
-    ::service::group0_batch batch(
-            co_await _group0_client.start_operation(_as, get_raft_timeout()));
-    // Check again as the state may have changed before we took the guard (batch).
-    if (!co_await needs_password()) {
-        co_return;
-    }
-    // Set default superuser's password.
-    std::string salted_pwd(get_config_value(_qp.db().get_config().auth_superuser_salted_password(), ""));
-    if (salted_pwd.empty()) {
-        salted_pwd = passwords::hash(DEFAULT_USER_PASSWORD, rng_for_salt, _scheme);
-    }
-    const auto update_query = update_row_query();
-    co_await collect_mutations(_qp, batch, update_query, {salted_pwd, _superuser});
-    co_await std::move(batch).commit(_group0_client, _as, get_raft_timeout());
-    plogger.info("Created default superuser authentication record.");
-}
-
-future<> password_authenticator::maybe_create_default_password_with_retries() {
-    size_t retries = _migration_manager.get_concurrent_ddl_retries();
-    while (true)  {
-        try {
-            co_return co_await maybe_create_default_password();
-        } catch (const ::service::group0_concurrent_modification& ex) {
-            plogger.warn("Failed to execute maybe_create_default_password due to guard conflict.{}.", retries ? " Retrying" : " Number of retries exceeded, giving up");
-            if (retries--) {
-                continue;
-            }
-            // Log error but don't crash the whole node startup sequence.
-            plogger.error("Failed to create default superuser password due to guard conflict.");
-            co_return;
-        } catch (const ::service::raft_operation_timeout_error& ex) {
-            plogger.error("Failed to create default superuser password due to exception: {}", ex.what());
-            co_return;
-        }
+        plogger.info("Created default superuser authentication record.");
+    } else {
+        co_await announce_mutations(_qp, _group0_client, query,
+            {salted_pwd, _superuser}, _as, ::service::raft_timeout{});
+        plogger.info("Created default superuser authentication record.");
     }
 }
 
 future<> password_authenticator::start() {
     return once_among_shards([this] {
-        // Verify that at least one hashing scheme is supported.
-        passwords::detail::verify_scheme(_scheme);
-        plogger.info("Using password hashing scheme: {}", passwords::detail::prefix_for_scheme(_scheme));
-
         _stopped = do_after_system_ready(_as, [this] {
             return async([this] {
                 if (legacy_mode(_qp)) {
-                    if (!_superuser_created_promise.available()) {
-                        _superuser_created_promise.set_value();
-                    }
                     _migration_manager.wait_for_schema_agreement(_qp.db().real_database(), db::timeout_clock::time_point::max(), &_as).get();
 
                     if (any_nondefault_role_row_satisfies(_qp, &has_salted_hash, _superuser).get()) {
@@ -227,15 +161,8 @@ future<> password_authenticator::start() {
                         migrate_legacy_metadata().get();
                         return;
                     }
-                    legacy_create_default_if_missing().get();
-                }
-                utils::get_local_injector().inject("password_authenticator_start_pause", utils::wait_for_message(5min)).get();
-                if (!legacy_mode(_qp)) {
-                    maybe_create_default_password_with_retries().get();
-                    if (!_superuser_created_promise.available()) {
-                        _superuser_created_promise.set_value();
-                    }
                 }
+                create_default_if_missing().get();
             });
         });
 
@@ -294,13 +221,7 @@ future<authenticated_user> password_authenticator::authenticate(
 
     try {
         const std::optional<sstring> salted_hash = co_await get_password_hash(username);
-        if (!salted_hash) {
-            throw exceptions::authentication_exception("Username and/or password are incorrect");
-        }
-        const bool password_match = co_await _hashing_worker.submit<bool>([password = std::move(password), salted_hash = std::move(salted_hash)]{
-            return passwords::check(password, *salted_hash);
-        });
-        if (!password_match) {
+        if (!salted_hash || !passwords::check(password, *salted_hash)) {
             throw exceptions::authentication_exception("Username and/or password are incorrect");
         }
         co_return username;
@@ -324,7 +245,7 @@ future<> password_authenticator::create(std::string_view role_name, const authen
     auto maybe_hash = options.credentials.transform([&] (const auto& creds) -> sstring {
         return std::visit(make_visitor(
                 [&] (const password_option& opt) {
-                    return passwords::hash(opt.password, rng_for_salt, _scheme);
+                    return passwords::hash(opt.password, rng_for_salt);
                 },
                 [] (const hashed_password_option& opt) {
                     return opt.hashed_password;
@@ -367,11 +288,11 @@ future<> password_authenticator::alter(std::string_view role_name, const authent
                 query,
                 consistency_for_user(role_name),
                 internal_distributed_query_state(),
-                {passwords::hash(password, rng_for_salt, _scheme), sstring(role_name)},
+                {passwords::hash(password, rng_for_salt), sstring(role_name)},
                 cql3::query_processor::cache_internal::no).discard_result();
     } else {
         co_await collect_mutations(_qp, mc, query,
-                {passwords::hash(password, rng_for_salt, _scheme), sstring(role_name)});
+                {passwords::hash(password, rng_for_salt), sstring(role_name)});
     }
 }
 
@@ -440,8 +361,4 @@ const resource_set& password_authenticator::protected_resources() const {
     });
 }
 
-future<> password_authenticator::ensure_superuser_is_created() const {
-    return _superuser_created_promise.get_shared_future();
-}
-
 }

auth/password_authenticator.hh

@@ -11,13 +11,10 @@
 #pragma once
 
 #include <seastar/core/abort_source.hh>
-#include <seastar/core/shared_future.hh>
 
 #include "db/consistency_level_type.hh"
 #include "auth/authenticator.hh"
-#include "auth/passwords.hh"
 #include "service/raft/raft_group0_client.hh"
-#include "utils/alien_worker.hh"
 
 namespace db {
     class config;
@@ -43,17 +40,13 @@ class password_authenticator : public authenticator {
     ::service::migration_manager& _migration_manager;
     future<> _stopped;
     abort_source _as;
-    std::string _superuser; // default superuser name from the config (may or may not be present in roles table)
-    shared_promise<> _superuser_created_promise;
-    // We used to also support bcrypt, SHA-256, and MD5 (ref. scylladb#24524).
-    constexpr static auth::passwords::scheme _scheme = passwords::scheme::sha_512;
-    utils::alien_worker& _hashing_worker;
+    std::string _superuser;
 
 public:
     static db::consistency_level consistency_for_user(std::string_view role_name);
     static std::string default_superuser(const db::config&);
 
-    password_authenticator(cql3::query_processor&, ::service::raft_group0_client&, ::service::migration_manager&, utils::alien_worker&);
+    password_authenticator(cql3::query_processor&, ::service::raft_group0_client&, ::service::migration_manager&);
 
     ~password_authenticator();
 
@@ -87,17 +80,12 @@ public:
 
     virtual ::shared_ptr<sasl_challenge> new_sasl_challenge() const override;
 
-    virtual future<> ensure_superuser_is_created() const override;
-
 private:
     bool legacy_metadata_exists() const;
 
     future<> migrate_legacy_metadata() const;
 
-    future<> legacy_create_default_if_missing();
-
-    future<> maybe_create_default_password();
-    future<> maybe_create_default_password_with_retries();
+    future<> create_default_if_missing();
 
     sstring update_row_query() const;
 };

auth/passwords.cc

@@ -21,14 +21,18 @@ static thread_local crypt_data tlcrypt = {};
 
 namespace detail {
 
-void verify_scheme(scheme scheme) {
+scheme identify_best_supported_scheme() {
+    const auto all_schemes = { scheme::bcrypt_y, scheme::bcrypt_a, scheme::sha_512, scheme::sha_256, scheme::md5 };
+    // "Random", for testing schemes.
     const sstring random_part_of_salt = "aaaabbbbccccdddd";
 
-    const sstring salt = sstring(prefix_for_scheme(scheme)) + random_part_of_salt;
-    const char* e = crypt_r("fisk", salt.c_str(), &tlcrypt);
+    for (scheme c : all_schemes) {
+        const sstring salt = sstring(prefix_for_scheme(c)) + random_part_of_salt;
+        const char* e = crypt_r("fisk", salt.c_str(), &tlcrypt);
 
-    if (e && (e[0] != '*')) {
-        return;
+        if (e && (e[0] != '*')) {
+            return c;
+        }
     }
 
     throw no_supported_schemes();

auth/passwords.hh

@@ -21,11 +21,10 @@ class no_supported_schemes : public std::runtime_error {
 public:
     no_supported_schemes();
 };
+
 ///
-/// Apache Cassandra uses a library to provide the bcrypt scheme. In ScyllaDB, we use SHA-512
-/// instead of bcrypt for performance and for historical reasons (see scylladb#24524).
-/// Currently, SHA-512 is always chosen as the hashing scheme for new passwords, but the other
-/// algorithms remain supported for CREATE ROLE WITH HASHED PASSWORD and backward compatibility.
+/// Apache Cassandra uses a library to provide the bcrypt scheme. Many Linux implementations do not support bcrypt, so
+/// we support alternatives. The cost is loss of direct compatibility with Apache Cassandra system tables.
 ///
 enum class scheme {
     bcrypt_y,
@@ -52,11 +51,11 @@ sstring generate_random_salt_bytes(RandomNumberEngine& g) {
 }
 
 ///
-/// Test given hashing scheme on the current system.
+/// Test each allowed hashing scheme and report the best supported one on the current system.
 ///
-/// \throws \ref no_supported_schemes when scheme is unsupported.
+/// \throws \ref no_supported_schemes when none of the known schemes is supported.
 ///
-void verify_scheme(scheme scheme);
+scheme identify_best_supported_scheme();
 
 std::string_view prefix_for_scheme(scheme) noexcept;
 
@@ -68,7 +67,8 @@ std::string_view prefix_for_scheme(scheme) noexcept;
 /// \throws \ref no_supported_schemes when no known hashing schemes are supported on the system.
 ///
 template <typename RandomNumberEngine>
-sstring generate_salt(RandomNumberEngine& g, scheme scheme) {
+sstring generate_salt(RandomNumberEngine& g) {
+    static const scheme scheme = identify_best_supported_scheme();
     static const sstring prefix = sstring(prefix_for_scheme(scheme));
     return prefix + generate_random_salt_bytes(g);
 }
@@ -93,8 +93,8 @@ sstring hash_with_salt(const sstring& pass, const sstring& salt);
 /// \throws \ref std::system_error when the implementation-specific implementation fails to hash the cleartext.
 ///
 template <typename RandomNumberEngine>
-sstring hash(const sstring& pass, RandomNumberEngine& g, scheme scheme) {
-    return detail::hash_with_salt(pass, detail::generate_salt(g, scheme));
+sstring hash(const sstring& pass, RandomNumberEngine& g) {
+    return detail::hash_with_salt(pass, detail::generate_salt(g));
 }
 
 ///

auth/resource.cc

@@ -15,6 +15,7 @@
 #include <iterator>
 #include <unordered_map>
 
+#include <boost/algorithm/string/join.hpp>
 #include <boost/algorithm/string/split.hpp>
 #include <boost/algorithm/string/classification.hpp>
 
@@ -147,7 +148,7 @@ resource::resource(functions_resource_t, std::string_view keyspace, std::string_
 }
 
 sstring resource::name() const {
-    return fmt::to_string(fmt::join(_parts, "/"));
+    return boost::algorithm::join(_parts, "/");
 }
 
 std::optional<resource> resource::parent() const {

auth/role_manager.hh

@@ -17,17 +17,12 @@
 #include <seastar/core/format.hh>
 #include <seastar/core/sstring.hh>
 
-#include "auth/common.hh"
 #include "auth/resource.hh"
 #include "cql3/description.hh"
 #include "seastarx.hh"
 #include "exceptions/exceptions.hh"
 #include "service/raft/raft_group0_client.hh"
 
-namespace service {
-class query_state;
-};
-
 namespace auth {
 
 struct role_config final {
@@ -172,9 +167,9 @@ public:
     ///   (role2, role3)
     /// }
     ///  
-    virtual future<role_to_directly_granted_map> query_all_directly_granted(::service::query_state& = internal_distributed_query_state()) = 0;
+    virtual future<role_to_directly_granted_map> query_all_directly_granted() = 0;
 
-    virtual future<role_set> query_all(::service::query_state& = internal_distributed_query_state()) = 0;
+    virtual future<role_set> query_all() = 0;
 
     virtual future<bool> exists(std::string_view role_name) = 0;
 
@@ -191,12 +186,12 @@ public:
     ///
     /// \returns the value of the named attribute, if one is set.
     ///
-    virtual future<std::optional<sstring>> get_attribute(std::string_view role_name, std::string_view attribute_name, ::service::query_state& = internal_distributed_query_state()) = 0;
+    virtual future<std::optional<sstring>> get_attribute(std::string_view role_name, std::string_view attribute_name) = 0;
 
     ///
     /// \returns a mapping of each role's value for the named attribute, if one is set for the role.
     ///
-    virtual future<attribute_vals> query_attribute_for_all(std::string_view attribute_name, ::service::query_state& = internal_distributed_query_state()) = 0;
+    virtual future<attribute_vals> query_attribute_for_all(std::string_view attribute_name) = 0;
 
     /// Sets `attribute_name` with `attribute_value` for `role_name`.
     /// \returns an exceptional future with nonexistant_role if the role does not exist.