Compare commits
352 Commits
mykaul-pat
...
next-5.1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
790a36155b | ||
|
|
6be2ba8a0b | ||
|
|
f7d6364712 | ||
|
|
b6d0949ab5 | ||
|
|
74ebd484ae | ||
|
|
0f2269afbf | ||
|
|
3a66260231 | ||
|
|
3ffd3e6636 | ||
|
|
370ffc80b0 | ||
|
|
c9fa077c82 | ||
|
|
2ea211db69 | ||
|
|
dc4be20609 | ||
|
|
7a2f9fb48f | ||
|
|
801687f185 | ||
|
|
e3de2187ef | ||
|
|
c8fdd595e3 | ||
|
|
abf62e5b7f | ||
|
|
f886581bee | ||
|
|
2071b70394 | ||
|
|
83ca111398 | ||
|
|
58f1ecdddb | ||
|
|
18014f1d9a | ||
|
|
2c50655835 | ||
|
|
6acb1916f0 | ||
|
|
72522849c1 | ||
|
|
e3bd23a429 | ||
|
|
f2f590d197 | ||
|
|
67d6bd49e6 | ||
|
|
d20989470e | ||
|
|
3854de1656 | ||
|
|
af66363067 | ||
|
|
387c567125 | ||
|
|
553658ef6f | ||
|
|
6628bee308 | ||
|
|
407585cd40 | ||
|
|
3d1218bacb | ||
|
|
23f9fdfbba | ||
|
|
eaf93b3953 | ||
|
|
5591bb15a3 | ||
|
|
3f3ecbe727 | ||
|
|
fafc10e89e | ||
|
|
c6c05b8a40 | ||
|
|
f5c542de13 | ||
|
|
bc62963e61 | ||
|
|
d9134003d5 | ||
|
|
ebc9aed74e | ||
|
|
1c3751b251 | ||
|
|
5e876c6614 | ||
|
|
ccbce78b1c | ||
|
|
690b5579ef | ||
|
|
069c79f79f | ||
|
|
f7b754ce0e | ||
|
|
2563d9469f | ||
|
|
a61e99b774 | ||
|
|
f16e1d3e31 | ||
|
|
86d6204d9a | ||
|
|
526a3847f8 | ||
|
|
ac2bb46d5e | ||
|
|
12a3c20f4f | ||
|
|
eae8a18fa5 | ||
|
|
d645016687 | ||
|
|
504f4ae397 | ||
|
|
b912887cea | ||
|
|
e76ad7d064 | ||
|
|
81d5243706 | ||
|
|
a3dc1a36d5 | ||
|
|
dae4e2738f | ||
|
|
63d0e976d1 | ||
|
|
8f0e7db96e | ||
|
|
3cc5c80571 | ||
|
|
17cb69e791 | ||
|
|
37be8884e4 | ||
|
|
b76bfe58c9 | ||
|
|
a98d5206a5 | ||
|
|
12966e8435 | ||
|
|
e741de0c05 | ||
|
|
4c85375abb | ||
|
|
16c6d626fe | ||
|
|
3219981367 | ||
|
|
737213a679 | ||
|
|
7d11a34377 | ||
|
|
753c9a4769 | ||
|
|
602cd20698 | ||
|
|
ee24ef649a | ||
|
|
7ab70a2420 | ||
|
|
b4dbd77bb5 | ||
|
|
ecbe07cafa | ||
|
|
a58323766b | ||
|
|
9d587193fb | ||
|
|
b3f1789c5c | ||
|
|
d45dec49d3 | ||
|
|
32645e6e3e | ||
|
|
488d36f77e | ||
|
|
3dd0cb3221 | ||
|
|
e7a5c13aab | ||
|
|
da5926c080 | ||
|
|
f588b46f55 | ||
|
|
0a6676a594 | ||
|
|
f13f8954a4 | ||
|
|
b635a30b59 | ||
|
|
342d13e26a | ||
|
|
db01be31c6 | ||
|
|
87531915d9 | ||
|
|
4075daf96d | ||
|
|
b27c5567fa | ||
|
|
8877f0b28a | ||
|
|
fabc7df720 | ||
|
|
bdafe2b98c | ||
|
|
d78bc60a74 | ||
|
|
97985a68a1 | ||
|
|
c1278994d8 | ||
|
|
c33fb41802 | ||
|
|
bca4bf6c11 | ||
|
|
cf08b19dad | ||
|
|
0d5751b4b6 | ||
|
|
1c7cdf68d6 | ||
|
|
8fc0591f98 | ||
|
|
a0fa2d043d | ||
|
|
50a3cc6b90 | ||
|
|
cfa8fa1d77 | ||
|
|
6cdd5ccabd | ||
|
|
204baa0c1e | ||
|
|
3556d2b4e8 | ||
|
|
6b699c9667 | ||
|
|
316ea63ea0 | ||
|
|
bafde878ba | ||
|
|
88eeab7838 | ||
|
|
f6230b5eec | ||
|
|
737285d342 | ||
|
|
f1ee68e128 | ||
|
|
7d4abd9e64 | ||
|
|
0b4b5c21ad | ||
|
|
3c7d2a3284 | ||
|
|
d7e65a1a0a | ||
|
|
73d80d55d1 | ||
|
|
37538f00f5 | ||
|
|
1cb11e7e2f | ||
|
|
f4200098ce | ||
|
|
f751613924 | ||
|
|
b38d56367f | ||
|
|
991f4ab104 | ||
|
|
c72058199f | ||
|
|
c69e1a4e12 | ||
|
|
62c737aa49 | ||
|
|
edeec94a89 | ||
|
|
95a63edea8 | ||
|
|
e04eef29a8 | ||
|
|
c14ccceec9 | ||
|
|
29e3cd80fb | ||
|
|
067f94d3f8 | ||
|
|
f2f9f26b79 | ||
|
|
41cfe1c103 | ||
|
|
1de577d696 | ||
|
|
a6a23acd9b | ||
|
|
efba2c38ef | ||
|
|
ba1a57bd55 | ||
|
|
1891ad2551 | ||
|
|
2ea3d5ebf0 | ||
|
|
d7d9300453 | ||
|
|
ab2817b4a6 | ||
|
|
73a340033e | ||
|
|
fec4f88744 | ||
|
|
802beb972e | ||
|
|
1101694169 | ||
|
|
d361bce0f6 | ||
|
|
7e0dcf9bc5 | ||
|
|
011c5ac37e | ||
|
|
9430465a52 | ||
|
|
6f28b77962 | ||
|
|
06d5557c42 | ||
|
|
bc31472469 | ||
|
|
fcab382e37 | ||
|
|
be6c3022ce | ||
|
|
6fa78b90b5 | ||
|
|
8b5a342a92 | ||
|
|
6905f5056f | ||
|
|
c90807d29f | ||
|
|
c468c61ddc | ||
|
|
05a3e97077 | ||
|
|
41e93b2e69 | ||
|
|
9f00af9395 | ||
|
|
7e78b29609 | ||
|
|
7007c94f5d | ||
|
|
5c5a9633ea | ||
|
|
c75359d664 | ||
|
|
24f76f40b7 | ||
|
|
3f133cfa87 | ||
|
|
d66e23b265 | ||
|
|
378e8761b9 | ||
|
|
fc26f6b850 | ||
|
|
7663dc31b8 | ||
|
|
53b6e720f6 | ||
|
|
46d6145b37 | ||
|
|
772ac59299 | ||
|
|
1f334e48b2 | ||
|
|
389050e421 | ||
|
|
2152b765a6 | ||
|
|
d43b6db152 | ||
|
|
a514e60e65 | ||
|
|
6eb70caba9 | ||
|
|
dd094f1230 | ||
|
|
530600a646 | ||
|
|
c8e5f8c66b | ||
|
|
ca7a13cad2 | ||
|
|
d401551020 | ||
|
|
d952bf4035 | ||
|
|
b346136e98 | ||
|
|
87e267213d | ||
|
|
5bda9356d5 | ||
|
|
f9fe48ad89 | ||
|
|
0c9a0faf0d | ||
|
|
47dcfd866c | ||
|
|
5c9ecd5604 | ||
|
|
1a37b85d14 | ||
|
|
5070ddb723 | ||
|
|
7480af58e5 | ||
|
|
43d46a241f | ||
|
|
3807020a7b | ||
|
|
d6fb20f30e | ||
|
|
33c20eebe6 | ||
|
|
f3a6af663d | ||
|
|
1685af9829 | ||
|
|
bb880c7658 | ||
|
|
f952d397e8 | ||
|
|
5e88421360 | ||
|
|
1945102ca0 | ||
|
|
be3f6f8c7b | ||
|
|
94735f63a3 | ||
|
|
b0d28919c0 | ||
|
|
addc4666d5 | ||
|
|
a14ffbd5e2 | ||
|
|
ea56ecace0 | ||
|
|
fe8d8f97e2 | ||
|
|
44e920cbb0 | ||
|
|
b114551d53 | ||
|
|
099145fe9a | ||
|
|
0bdbce90f4 | ||
|
|
8ffdb8546b | ||
|
|
db382697f1 | ||
|
|
3c02e5d263 | ||
|
|
4c0f7ea098 | ||
|
|
c14a0340ca | ||
|
|
aa523141f9 | ||
|
|
86240d6344 | ||
|
|
95a94a2687 | ||
|
|
9173a3d808 | ||
|
|
7942041b95 | ||
|
|
1cfedc5b59 | ||
|
|
606ed61263 | ||
|
|
796e4c39f8 | ||
|
|
960434f784 | ||
|
|
05aed0417a | ||
|
|
9a1fc200e1 | ||
|
|
c7e9bbc377 | ||
|
|
a78dac7ae9 | ||
|
|
0debb419f7 | ||
|
|
0cfb950569 | ||
|
|
eebe77b5b8 | ||
|
|
aa206a6b6a | ||
|
|
9baf72b049 | ||
|
|
8e62405117 | ||
|
|
15421e45a0 | ||
|
|
cf6bcffc1b | ||
|
|
a50b7f3d6a | ||
|
|
0e06025487 | ||
|
|
820b79d56e | ||
|
|
fde4a6e92d | ||
|
|
34b8d4306c | ||
|
|
69e8fb997c | ||
|
|
fbfd91e02f | ||
|
|
4c19b48495 | ||
|
|
9deca4250f | ||
|
|
97ab2a4eb3 | ||
|
|
16a941db3b | ||
|
|
8f60a464a7 | ||
|
|
1e72f9cb5e | ||
|
|
d91da87313 | ||
|
|
c73d59c1cb | ||
|
|
9a6c0a89a0 | ||
|
|
5b7dd00b14 | ||
|
|
237df3b935 | ||
|
|
993d0371d9 | ||
|
|
04167eba68 | ||
|
|
ddf8eaba04 | ||
|
|
d5e5d27929 | ||
|
|
bb69ece13d | ||
|
|
d4268863cd | ||
|
|
8b05c67226 | ||
|
|
3afc58de7d | ||
|
|
496696140b | ||
|
|
3a070380a7 | ||
|
|
7b551d8ce4 | ||
|
|
2018b8fcfd | ||
|
|
20b5aa938e | ||
|
|
f519580252 | ||
|
|
46c0a1cc0a | ||
|
|
a822282fde | ||
|
|
bc5af9fdea | ||
|
|
c8bb147f84 | ||
|
|
dc92ec4c8b | ||
|
|
8f7e3275a2 | ||
|
|
40a1905a2d | ||
|
|
4e2c436222 | ||
|
|
68be369f93 | ||
|
|
0f7adb5f47 | ||
|
|
82dc8357ef | ||
|
|
12a58957e2 | ||
|
|
3423ad6e38 | ||
|
|
64001719fa | ||
|
|
cc3d368bc8 | ||
|
|
d957b0044b | ||
|
|
d4ed67bd47 | ||
|
|
0cd6341cae | ||
|
|
23d8852a82 | ||
|
|
88016de43e | ||
|
|
bdecf4318a | ||
|
|
72bf244ad1 | ||
|
|
ee82323599 | ||
|
|
2f78df92ab | ||
|
|
e2809674d2 | ||
|
|
0295d0c5c8 | ||
|
|
fa94222662 | ||
|
|
dff7f3c5ba | ||
|
|
3723713130 | ||
|
|
03f8411e38 | ||
|
|
0e391d67d1 | ||
|
|
f76989285e | ||
|
|
9deeeb4db1 | ||
|
|
1f3196735f | ||
|
|
abb6817261 | ||
|
|
d3fd090429 | ||
|
|
3e7c57d162 | ||
|
|
f878a34da3 | ||
|
|
eaded57b2e | ||
|
|
25d2da08d1 | ||
|
|
9b1a570f6f | ||
|
|
426d045249 | ||
|
|
86dbbf12cc | ||
|
|
b05903eddd | ||
|
|
26ead53304 | ||
|
|
f60bab9471 | ||
|
|
66f34245fc | ||
|
|
4047528bd9 | ||
|
|
1a82c61452 | ||
|
|
3d9800eb1c | ||
|
|
c48e9b47dd | ||
|
|
2eadaad9f7 | ||
|
|
d10aee15e7 | ||
|
|
9e017cb1e6 | ||
|
|
b8504cc9b2 | ||
|
|
856703a85e | ||
|
|
86a6c1fb2b |
24
.github/CODEOWNERS
vendored
24
.github/CODEOWNERS
vendored
@@ -12,7 +12,7 @@ test/cql/cdc_* @kbr- @elcallio @piodul @jul-stas
|
||||
test/boost/cdc_* @kbr- @elcallio @piodul @jul-stas
|
||||
|
||||
# COMMITLOG / BATCHLOG
|
||||
db/commitlog/* @elcallio @eliransin
|
||||
db/commitlog/* @elcallio
|
||||
db/batch* @elcallio
|
||||
|
||||
# COORDINATOR
|
||||
@@ -25,7 +25,7 @@ compaction/* @raphaelsc @nyh
|
||||
transport/*
|
||||
|
||||
# CQL QUERY LANGUAGE
|
||||
cql3/* @tgrabiec @cvybhu @nyh
|
||||
cql3/* @tgrabiec @psarna @cvybhu
|
||||
|
||||
# COUNTERS
|
||||
counters* @jul-stas
|
||||
@@ -33,7 +33,7 @@ tests/counter_test* @jul-stas
|
||||
|
||||
# DOCS
|
||||
docs/* @annastuchlik @tzach
|
||||
docs/alternator @annastuchlik @tzach @nyh @havaker @nuivall
|
||||
docs/alternator @annastuchlik @tzach @nyh @psarna
|
||||
|
||||
# GOSSIP
|
||||
gms/* @tgrabiec @asias
|
||||
@@ -45,9 +45,9 @@ dist/docker/*
|
||||
utils/logalloc* @tgrabiec
|
||||
|
||||
# MATERIALIZED VIEWS
|
||||
db/view/* @nyh @cvybhu @piodul
|
||||
cql3/statements/*view* @nyh @cvybhu @piodul
|
||||
test/boost/view_* @nyh @cvybhu @piodul
|
||||
db/view/* @nyh @psarna
|
||||
cql3/statements/*view* @nyh @psarna
|
||||
test/boost/view_* @nyh @psarna
|
||||
|
||||
# PACKAGING
|
||||
dist/* @syuu1228
|
||||
@@ -62,9 +62,9 @@ service/migration* @tgrabiec @nyh
|
||||
schema* @tgrabiec @nyh
|
||||
|
||||
# SECONDARY INDEXES
|
||||
index/* @nyh @cvybhu @piodul
|
||||
cql3/statements/*index* @nyh @cvybhu @piodul
|
||||
test/boost/*index* @nyh @cvybhu @piodul
|
||||
db/index/* @nyh @psarna
|
||||
cql3/statements/*index* @nyh @psarna
|
||||
test/boost/*index* @nyh @psarna
|
||||
|
||||
# SSTABLES
|
||||
sstables/* @tgrabiec @raphaelsc @nyh
|
||||
@@ -74,11 +74,11 @@ streaming/* @tgrabiec @asias
|
||||
service/storage_service.* @tgrabiec @asias
|
||||
|
||||
# ALTERNATOR
|
||||
alternator/* @nyh @havaker @nuivall
|
||||
test/alternator/* @nyh @havaker @nuivall
|
||||
alternator/* @nyh @psarna
|
||||
test/alternator/* @nyh @psarna
|
||||
|
||||
# HINTED HANDOFF
|
||||
db/hints/* @piodul @vladzcloudius @eliransin
|
||||
db/hints/* @piodul @vladzcloudius
|
||||
|
||||
# REDIS
|
||||
redis/* @nyh @syuu1228
|
||||
|
||||
17
.github/workflows/docs-amplify-enhanced.yaml
vendored
17
.github/workflows/docs-amplify-enhanced.yaml
vendored
@@ -1,17 +0,0 @@
|
||||
name: "Docs / Amplify enhanced"
|
||||
|
||||
on: issue_comment
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ github.event.issue.pull_request }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Amplify enhanced
|
||||
env:
|
||||
TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
uses: scylladb/sphinx-scylladb-theme/.github/actions/amplify-enhanced@master
|
||||
13
.github/workflows/docs-pages.yaml
vendored
13
.github/workflows/docs-pages.yaml
vendored
@@ -2,14 +2,10 @@ name: "Docs / Publish"
|
||||
# For more information,
|
||||
# see https://sphinx-theme.scylladb.com/stable/deployment/production.html#available-workflows
|
||||
|
||||
env:
|
||||
FLAG: ${{ github.repository == 'scylladb/scylla-enterprise' && 'enterprise' || 'opensource' }}
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'master'
|
||||
- 'enterprise'
|
||||
- master
|
||||
paths:
|
||||
- "docs/**"
|
||||
workflow_dispatch:
|
||||
@@ -28,13 +24,12 @@ jobs:
|
||||
with:
|
||||
python-version: 3.7
|
||||
- name: Set up env
|
||||
run: make -C docs FLAG="${{ env.FLAG }}" setupenv
|
||||
run: make -C docs setupenv
|
||||
- name: Build docs
|
||||
run: make -C docs FLAG="${{ env.FLAG }}" multiversion
|
||||
run: make -C docs multiversion
|
||||
- name: Build redirects
|
||||
run: make -C docs FLAG="${{ env.FLAG }}" redirects
|
||||
run: make -C docs redirects
|
||||
- name: Deploy docs to GitHub Pages
|
||||
run: ./docs/_utils/deploy.sh
|
||||
if: (github.ref_name == 'master' && env.FLAG == 'opensource') || (github.ref_name == 'enterprise' && env.FLAG == 'enterprise')
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
8
.github/workflows/docs-pr.yaml
vendored
8
.github/workflows/docs-pr.yaml
vendored
@@ -2,14 +2,10 @@ name: "Docs / Build PR"
|
||||
# For more information,
|
||||
# see https://sphinx-theme.scylladb.com/stable/deployment/production.html#available-workflows
|
||||
|
||||
env:
|
||||
FLAG: ${{ github.repository == 'scylladb/scylla-enterprise' && 'enterprise' || 'opensource' }}
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- enterprise
|
||||
paths:
|
||||
- "docs/**"
|
||||
|
||||
@@ -27,6 +23,6 @@ jobs:
|
||||
with:
|
||||
python-version: 3.7
|
||||
- name: Set up env
|
||||
run: make -C docs FLAG="${{ env.FLAG }}" setupenv
|
||||
run: make -C docs setupenv
|
||||
- name: Build docs
|
||||
run: make -C docs FLAG="${{ env.FLAG }}" test
|
||||
run: make -C docs test
|
||||
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -26,7 +26,8 @@ tags
|
||||
testlog
|
||||
test/*/*.reject
|
||||
.vscode
|
||||
docs/_build
|
||||
docs/poetry.lock
|
||||
compile_commands.json
|
||||
.ccls-cache/
|
||||
.mypy_cache
|
||||
.envrc
|
||||
|
||||
11
.gitmodules
vendored
11
.gitmodules
vendored
@@ -1,11 +1,17 @@
|
||||
[submodule "seastar"]
|
||||
path = seastar
|
||||
url = ../seastar
|
||||
url = ../scylla-seastar
|
||||
ignore = dirty
|
||||
[submodule "swagger-ui"]
|
||||
path = swagger-ui
|
||||
url = ../scylla-swagger-ui
|
||||
ignore = dirty
|
||||
[submodule "libdeflate"]
|
||||
path = libdeflate
|
||||
url = ../libdeflate
|
||||
[submodule "abseil"]
|
||||
path = abseil
|
||||
url = ../abseil-cpp
|
||||
[submodule "scylla-jmx"]
|
||||
path = tools/jmx
|
||||
url = ../scylla-jmx
|
||||
@@ -15,6 +21,3 @@
|
||||
[submodule "scylla-python3"]
|
||||
path = tools/python3
|
||||
url = ../scylla-python3
|
||||
[submodule "tools/cqlsh"]
|
||||
path = tools/cqlsh
|
||||
url = ../scylla-cqlsh
|
||||
|
||||
887
CMakeLists.txt
887
CMakeLists.txt
@@ -1,201 +1,796 @@
|
||||
cmake_minimum_required(VERSION 3.27)
|
||||
cmake_minimum_required(VERSION 3.18)
|
||||
|
||||
project(scylla)
|
||||
|
||||
include(CTest)
|
||||
|
||||
list(APPEND CMAKE_MODULE_PATH
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/cmake
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/seastar/cmake)
|
||||
|
||||
# Set the possible values of build type for cmake-gui
|
||||
set(scylla_build_types
|
||||
"Debug" "Release" "Dev" "Sanitize" "Coverage")
|
||||
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS
|
||||
${scylla_build_types})
|
||||
if(NOT CMAKE_BUILD_TYPE)
|
||||
set(CMAKE_BUILD_TYPE "Release" CACHE
|
||||
STRING "Choose the type of build." FORCE)
|
||||
message(WARNING "CMAKE_BUILD_TYPE not specified, Using 'Release'")
|
||||
elseif(NOT CMAKE_BUILD_TYPE IN_LIST scylla_build_types)
|
||||
message(FATAL_ERROR "Unknown CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}. "
|
||||
"Following types are supported: ${scylla_build_types}")
|
||||
if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
|
||||
message(STATUS "Setting build type to 'Release' as none was specified.")
|
||||
set(CMAKE_BUILD_TYPE "Release" CACHE
|
||||
STRING "Choose the type of build." FORCE)
|
||||
# Set the possible values of build type for cmake-gui
|
||||
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS
|
||||
"Debug" "Release" "Dev" "Sanitize")
|
||||
endif()
|
||||
string(TOUPPER "${CMAKE_BUILD_TYPE}" build_mode)
|
||||
include(mode.${build_mode})
|
||||
include(mode.common)
|
||||
add_compile_definitions(
|
||||
${Seastar_DEFINITIONS_${build_mode}}
|
||||
FMT_DEPRECATED_OSTREAM)
|
||||
include(limit_jobs)
|
||||
# Configure Seastar compile options to align with Scylla
|
||||
set(CMAKE_CXX_STANDARD "20" CACHE INTERNAL "")
|
||||
set(CMAKE_CXX_EXTENSIONS ON CACHE INTERNAL "")
|
||||
set(CMAKE_CXX_VISIBILITY_PRESET hidden)
|
||||
|
||||
set(Seastar_TESTING ON CACHE BOOL "" FORCE)
|
||||
set(Seastar_API_LEVEL 7 CACHE STRING "" FORCE)
|
||||
set(Seastar_APPS ON CACHE BOOL "" FORCE)
|
||||
set(Seastar_EXCLUDE_APPS_FROM_ALL ON CACHE BOOL "" FORCE)
|
||||
if(CMAKE_BUILD_TYPE)
|
||||
string(TOLOWER "${CMAKE_BUILD_TYPE}" BUILD_TYPE)
|
||||
else()
|
||||
set(BUILD_TYPE "release")
|
||||
endif()
|
||||
|
||||
function(default_target_arch arch)
|
||||
set(x86_instruction_sets i386 i686 x86_64)
|
||||
if(CMAKE_SYSTEM_PROCESSOR IN_LIST x86_instruction_sets)
|
||||
set(${arch} "westmere" PARENT_SCOPE)
|
||||
elseif(CMAKE_SYSTEM_PROCESSOR EQUAL "aarch64")
|
||||
set(${arch} "armv8-a+crc+crypto" PARENT_SCOPE)
|
||||
else()
|
||||
set(${arch} "" PARENT_SCOPE)
|
||||
endif()
|
||||
endfunction()
|
||||
default_target_arch(target_arch)
|
||||
if(target_arch)
|
||||
set(target_arch_flag "-march=${target_arch}")
|
||||
endif()
|
||||
|
||||
set(cxx_coro_flag)
|
||||
if (CMAKE_CXX_COMPILER_ID MATCHES GNU)
|
||||
set(cxx_coro_flag -fcoroutines)
|
||||
endif()
|
||||
|
||||
# Configure Seastar compile options to align with Scylla
|
||||
set(Seastar_CXX_FLAGS ${cxx_coro_flag} ${target_arch_flag} CACHE INTERNAL "" FORCE)
|
||||
set(Seastar_CXX_DIALECT gnu++20 CACHE INTERNAL "" FORCE)
|
||||
|
||||
add_subdirectory(seastar)
|
||||
add_subdirectory(abseil)
|
||||
# Exclude absl::strerror from the default "all" target since it's not
|
||||
# used in Scylla build and, moreover, makes use of deprecated glibc APIs,
|
||||
# such as sys_nerr, which are not exposed from "stdio.h" since glibc 2.32,
|
||||
# which happens to be the case for recent Fedora distribution versions.
|
||||
#
|
||||
# Need to use the internal "absl_strerror" target name instead of namespaced
|
||||
# variant because `set_target_properties` does not understand the latter form,
|
||||
# unfortunately.
|
||||
set_target_properties(absl_strerror PROPERTIES EXCLUDE_FROM_ALL TRUE)
|
||||
|
||||
# System libraries dependencies
|
||||
find_package(Boost REQUIRED
|
||||
COMPONENTS filesystem program_options system thread regex unit_test_framework)
|
||||
find_package(Boost COMPONENTS filesystem program_options system thread regex REQUIRED)
|
||||
find_package(Lua REQUIRED)
|
||||
find_package(ZLIB REQUIRED)
|
||||
find_package(ICU COMPONENTS uc i18n REQUIRED)
|
||||
find_package(absl COMPONENTS hash raw_hash_set REQUIRED)
|
||||
find_package(libdeflate REQUIRED)
|
||||
find_package(libxcrypt REQUIRED)
|
||||
find_package(Snappy REQUIRED)
|
||||
find_package(RapidJSON REQUIRED)
|
||||
find_package(Thrift REQUIRED)
|
||||
find_package(xxHash REQUIRED)
|
||||
find_package(ICU COMPONENTS uc REQUIRED)
|
||||
|
||||
set(scylla_gen_build_dir "${CMAKE_BINARY_DIR}/gen")
|
||||
file(MAKE_DIRECTORY "${scylla_gen_build_dir}")
|
||||
set(scylla_build_dir "${CMAKE_BINARY_DIR}/build/${BUILD_TYPE}")
|
||||
set(scylla_gen_build_dir "${scylla_build_dir}/gen")
|
||||
file(MAKE_DIRECTORY "${scylla_build_dir}" "${scylla_gen_build_dir}")
|
||||
|
||||
include(add_version_library)
|
||||
generate_scylla_version()
|
||||
# Place libraries, executables and archives in ${buildroot}/build/${mode}/
|
||||
foreach(mode RUNTIME LIBRARY ARCHIVE)
|
||||
set(CMAKE_${mode}_OUTPUT_DIRECTORY "${scylla_build_dir}")
|
||||
endforeach()
|
||||
|
||||
add_library(scylla-main STATIC)
|
||||
target_sources(scylla-main
|
||||
PRIVATE
|
||||
# Generate C++ source files from thrift definitions
|
||||
function(scylla_generate_thrift)
|
||||
set(one_value_args TARGET VAR IN_FILE OUT_DIR SERVICE)
|
||||
cmake_parse_arguments(args "" "${one_value_args}" "" ${ARGN})
|
||||
|
||||
get_filename_component(in_file_name ${args_IN_FILE} NAME_WE)
|
||||
|
||||
set(aux_out_file_name ${args_OUT_DIR}/${in_file_name})
|
||||
set(outputs
|
||||
${aux_out_file_name}_types.cpp
|
||||
${aux_out_file_name}_types.h
|
||||
${aux_out_file_name}_constants.cpp
|
||||
${aux_out_file_name}_constants.h
|
||||
${args_OUT_DIR}/${args_SERVICE}.cpp
|
||||
${args_OUT_DIR}/${args_SERVICE}.h)
|
||||
|
||||
add_custom_command(
|
||||
DEPENDS
|
||||
${args_IN_FILE}
|
||||
thrift
|
||||
OUTPUT ${outputs}
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory ${args_OUT_DIR}
|
||||
COMMAND thrift -gen cpp:cob_style,no_skeleton -out "${args_OUT_DIR}" "${args_IN_FILE}")
|
||||
|
||||
add_custom_target(${args_TARGET}
|
||||
DEPENDS ${outputs})
|
||||
|
||||
set(${args_VAR} ${outputs} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
scylla_generate_thrift(
|
||||
TARGET scylla_thrift_gen_cassandra
|
||||
VAR scylla_thrift_gen_cassandra_files
|
||||
IN_FILE "${CMAKE_SOURCE_DIR}/interface/cassandra.thrift"
|
||||
OUT_DIR ${scylla_gen_build_dir}
|
||||
SERVICE Cassandra)
|
||||
|
||||
# Parse antlr3 grammar files and generate C++ sources
|
||||
function(scylla_generate_antlr3)
|
||||
set(one_value_args TARGET VAR IN_FILE OUT_DIR)
|
||||
cmake_parse_arguments(args "" "${one_value_args}" "" ${ARGN})
|
||||
|
||||
get_filename_component(in_file_pure_name ${args_IN_FILE} NAME)
|
||||
get_filename_component(stem ${in_file_pure_name} NAME_WE)
|
||||
|
||||
set(outputs
|
||||
"${args_OUT_DIR}/${stem}Lexer.hpp"
|
||||
"${args_OUT_DIR}/${stem}Lexer.cpp"
|
||||
"${args_OUT_DIR}/${stem}Parser.hpp"
|
||||
"${args_OUT_DIR}/${stem}Parser.cpp")
|
||||
|
||||
add_custom_command(
|
||||
DEPENDS
|
||||
${args_IN_FILE}
|
||||
OUTPUT ${outputs}
|
||||
# Remove #ifdef'ed code from the grammar source code
|
||||
COMMAND sed -e "/^#if 0/,/^#endif/d" "${args_IN_FILE}" > "${args_OUT_DIR}/${in_file_pure_name}"
|
||||
COMMAND antlr3 "${args_OUT_DIR}/${in_file_pure_name}"
|
||||
# We replace many local `ExceptionBaseType* ex` variables with a single function-scope one.
|
||||
# Because we add such a variable to every function, and because `ExceptionBaseType` is not a global
|
||||
# name, we also add a global typedef to avoid compilation errors.
|
||||
COMMAND sed -i -e "/^.*On :.*$/d" "${args_OUT_DIR}/${stem}Lexer.hpp"
|
||||
COMMAND sed -i -e "/^.*On :.*$/d" "${args_OUT_DIR}/${stem}Lexer.cpp"
|
||||
COMMAND sed -i -e "/^.*On :.*$/d" "${args_OUT_DIR}/${stem}Parser.hpp"
|
||||
COMMAND sed -i
|
||||
-e "s/^\\( *\\)\\(ImplTraits::CommonTokenType\\* [a-zA-Z0-9_]* = NULL;\\)$/\\1const \\2/"
|
||||
-e "/^.*On :.*$/d"
|
||||
-e "1i using ExceptionBaseType = int;"
|
||||
-e "s/^{/{ ExceptionBaseType\\* ex = nullptr;/; s/ExceptionBaseType\\* ex = new/ex = new/; s/exceptions::syntax_exception e/exceptions::syntax_exception\\& e/"
|
||||
"${args_OUT_DIR}/${stem}Parser.cpp"
|
||||
VERBATIM)
|
||||
|
||||
add_custom_target(${args_TARGET}
|
||||
DEPENDS ${outputs})
|
||||
|
||||
set(${args_VAR} ${outputs} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
set(antlr3_grammar_files
|
||||
cql3/Cql.g
|
||||
alternator/expressions.g)
|
||||
|
||||
set(antlr3_gen_files)
|
||||
|
||||
foreach(f ${antlr3_grammar_files})
|
||||
get_filename_component(grammar_file_name "${f}" NAME_WE)
|
||||
get_filename_component(f_dir "${f}" DIRECTORY)
|
||||
scylla_generate_antlr3(
|
||||
TARGET scylla_antlr3_gen_${grammar_file_name}
|
||||
VAR scylla_antlr3_gen_${grammar_file_name}_files
|
||||
IN_FILE "${CMAKE_SOURCE_DIR}/${f}"
|
||||
OUT_DIR ${scylla_gen_build_dir}/${f_dir})
|
||||
list(APPEND antlr3_gen_files "${scylla_antlr3_gen_${grammar_file_name}_files}")
|
||||
endforeach()
|
||||
|
||||
# Generate C++ sources from ragel grammar files
|
||||
seastar_generate_ragel(
|
||||
TARGET scylla_ragel_gen_protocol_parser
|
||||
VAR scylla_ragel_gen_protocol_parser_file
|
||||
IN_FILE "${CMAKE_SOURCE_DIR}/redis/protocol_parser.rl"
|
||||
OUT_FILE ${scylla_gen_build_dir}/redis/protocol_parser.hh)
|
||||
|
||||
# Generate C++ sources from Swagger definitions
|
||||
set(swagger_files
|
||||
api/api-doc/cache_service.json
|
||||
api/api-doc/collectd.json
|
||||
api/api-doc/column_family.json
|
||||
api/api-doc/commitlog.json
|
||||
api/api-doc/compaction_manager.json
|
||||
api/api-doc/config.json
|
||||
api/api-doc/endpoint_snitch_info.json
|
||||
api/api-doc/error_injection.json
|
||||
api/api-doc/failure_detector.json
|
||||
api/api-doc/gossiper.json
|
||||
api/api-doc/hinted_handoff.json
|
||||
api/api-doc/lsa.json
|
||||
api/api-doc/messaging_service.json
|
||||
api/api-doc/storage_proxy.json
|
||||
api/api-doc/storage_service.json
|
||||
api/api-doc/stream_manager.json
|
||||
api/api-doc/system.json
|
||||
api/api-doc/utils.json)
|
||||
|
||||
set(swagger_gen_files)
|
||||
|
||||
foreach(f ${swagger_files})
|
||||
get_filename_component(fname "${f}" NAME_WE)
|
||||
get_filename_component(dir "${f}" DIRECTORY)
|
||||
seastar_generate_swagger(
|
||||
TARGET scylla_swagger_gen_${fname}
|
||||
VAR scylla_swagger_gen_${fname}_files
|
||||
IN_FILE "${CMAKE_SOURCE_DIR}/${f}"
|
||||
OUT_DIR "${scylla_gen_build_dir}/${dir}")
|
||||
list(APPEND swagger_gen_files "${scylla_swagger_gen_${fname}_files}")
|
||||
endforeach()
|
||||
|
||||
# Create C++ bindings for IDL serializers
|
||||
function(scylla_generate_idl_serializer)
|
||||
set(one_value_args TARGET VAR IN_FILE OUT_FILE)
|
||||
cmake_parse_arguments(args "" "${one_value_args}" "" ${ARGN})
|
||||
get_filename_component(out_dir ${args_OUT_FILE} DIRECTORY)
|
||||
set(idl_compiler "${CMAKE_SOURCE_DIR}/idl-compiler.py")
|
||||
|
||||
find_package(Python3 COMPONENTS Interpreter)
|
||||
|
||||
add_custom_command(
|
||||
DEPENDS
|
||||
${args_IN_FILE}
|
||||
${idl_compiler}
|
||||
OUTPUT ${args_OUT_FILE}
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory ${out_dir}
|
||||
COMMAND Python3::Interpreter ${idl_compiler} --ns ser -f ${args_IN_FILE} -o ${args_OUT_FILE})
|
||||
|
||||
add_custom_target(${args_TARGET}
|
||||
DEPENDS ${args_OUT_FILE})
|
||||
|
||||
set(${args_VAR} ${args_OUT_FILE} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
set(idl_serializers
|
||||
idl/cache_temperature.idl.hh
|
||||
idl/commitlog.idl.hh
|
||||
idl/consistency_level.idl.hh
|
||||
idl/frozen_mutation.idl.hh
|
||||
idl/frozen_schema.idl.hh
|
||||
idl/gossip_digest.idl.hh
|
||||
idl/hinted_handoff.idl.hh
|
||||
idl/idl_test.idl.hh
|
||||
idl/keys.idl.hh
|
||||
idl/messaging_service.idl.hh
|
||||
idl/mutation.idl.hh
|
||||
idl/paging_state.idl.hh
|
||||
idl/partition_checksum.idl.hh
|
||||
idl/paxos.idl.hh
|
||||
idl/query.idl.hh
|
||||
idl/raft.idl.hh
|
||||
idl/range.idl.hh
|
||||
idl/read_command.idl.hh
|
||||
idl/reconcilable_result.idl.hh
|
||||
idl/replay_position.idl.hh
|
||||
idl/result.idl.hh
|
||||
idl/ring_position.idl.hh
|
||||
idl/streaming.idl.hh
|
||||
idl/token.idl.hh
|
||||
idl/tracing.idl.hh
|
||||
idl/truncation_record.idl.hh
|
||||
idl/uuid.idl.hh
|
||||
idl/view.idl.hh)
|
||||
|
||||
set(idl_gen_files)
|
||||
|
||||
foreach(f ${idl_serializers})
|
||||
get_filename_component(idl_name "${f}" NAME)
|
||||
get_filename_component(idl_target "${idl_name}" NAME_WE)
|
||||
get_filename_component(idl_dir "${f}" DIRECTORY)
|
||||
string(REPLACE ".idl.hh" ".dist.hh" idl_out_hdr_name "${idl_name}")
|
||||
scylla_generate_idl_serializer(
|
||||
TARGET scylla_idl_gen_${idl_target}
|
||||
VAR scylla_idl_gen_${idl_target}_files
|
||||
IN_FILE "${CMAKE_SOURCE_DIR}/${f}"
|
||||
OUT_FILE ${scylla_gen_build_dir}/${idl_dir}/${idl_out_hdr_name})
|
||||
list(APPEND idl_gen_files "${scylla_idl_gen_${idl_target}_files}")
|
||||
endforeach()
|
||||
|
||||
set(scylla_sources
|
||||
absl-flat_hash_map.cc
|
||||
alternator/auth.cc
|
||||
alternator/conditions.cc
|
||||
alternator/controller.cc
|
||||
alternator/executor.cc
|
||||
alternator/expressions.cc
|
||||
alternator/serialization.cc
|
||||
alternator/server.cc
|
||||
alternator/stats.cc
|
||||
alternator/streams.cc
|
||||
api/api.cc
|
||||
api/cache_service.cc
|
||||
api/collectd.cc
|
||||
api/column_family.cc
|
||||
api/commitlog.cc
|
||||
api/compaction_manager.cc
|
||||
api/config.cc
|
||||
api/endpoint_snitch.cc
|
||||
api/error_injection.cc
|
||||
api/failure_detector.cc
|
||||
api/gossiper.cc
|
||||
api/hinted_handoff.cc
|
||||
api/lsa.cc
|
||||
api/messaging_service.cc
|
||||
api/storage_proxy.cc
|
||||
api/storage_service.cc
|
||||
api/stream_manager.cc
|
||||
api/system.cc
|
||||
atomic_cell.cc
|
||||
auth/allow_all_authenticator.cc
|
||||
auth/allow_all_authorizer.cc
|
||||
auth/authenticated_user.cc
|
||||
auth/authentication_options.cc
|
||||
auth/authenticator.cc
|
||||
auth/common.cc
|
||||
auth/default_authorizer.cc
|
||||
auth/password_authenticator.cc
|
||||
auth/passwords.cc
|
||||
auth/permission.cc
|
||||
auth/permissions_cache.cc
|
||||
auth/resource.cc
|
||||
auth/role_or_anonymous.cc
|
||||
auth/roles-metadata.cc
|
||||
auth/sasl_challenge.cc
|
||||
auth/service.cc
|
||||
auth/standard_role_manager.cc
|
||||
auth/transitional.cc
|
||||
bytes.cc
|
||||
client_data.cc
|
||||
caching_options.cc
|
||||
canonical_mutation.cc
|
||||
cdc/cdc_partitioner.cc
|
||||
cdc/generation.cc
|
||||
cdc/log.cc
|
||||
cdc/metadata.cc
|
||||
cdc/split.cc
|
||||
clocks-impl.cc
|
||||
collection_mutation.cc
|
||||
compaction/compaction.cc
|
||||
compaction/compaction_manager.cc
|
||||
compaction/compaction_strategy.cc
|
||||
compaction/leveled_compaction_strategy.cc
|
||||
compaction/size_tiered_compaction_strategy.cc
|
||||
compaction/time_window_compaction_strategy.cc
|
||||
compress.cc
|
||||
converting_mutation_partition_applier.cc
|
||||
counters.cc
|
||||
direct_failure_detector/failure_detector.cc
|
||||
cql3/abstract_marker.cc
|
||||
cql3/attributes.cc
|
||||
cql3/cf_name.cc
|
||||
cql3/column_condition.cc
|
||||
cql3/column_identifier.cc
|
||||
cql3/column_specification.cc
|
||||
cql3/constants.cc
|
||||
cql3/cql3_type.cc
|
||||
cql3/expr/expression.cc
|
||||
cql3/expr/prepare_expr.cc
|
||||
cql3/expr/restrictions.cc
|
||||
cql3/functions/aggregate_fcts.cc
|
||||
cql3/functions/castas_fcts.cc
|
||||
cql3/functions/error_injection_fcts.cc
|
||||
cql3/functions/functions.cc
|
||||
cql3/functions/user_function.cc
|
||||
cql3/index_name.cc
|
||||
cql3/keyspace_element_name.cc
|
||||
cql3/lists.cc
|
||||
cql3/maps.cc
|
||||
cql3/operation.cc
|
||||
cql3/prepare_context.cc
|
||||
cql3/query_options.cc
|
||||
cql3/query_processor.cc
|
||||
cql3/restrictions/statement_restrictions.cc
|
||||
cql3/result_set.cc
|
||||
cql3/role_name.cc
|
||||
cql3/selection/abstract_function_selector.cc
|
||||
cql3/selection/selectable.cc
|
||||
cql3/selection/selection.cc
|
||||
cql3/selection/selector.cc
|
||||
cql3/selection/selector_factories.cc
|
||||
cql3/selection/simple_selector.cc
|
||||
cql3/sets.cc
|
||||
cql3/statements/alter_keyspace_statement.cc
|
||||
cql3/statements/alter_service_level_statement.cc
|
||||
cql3/statements/alter_table_statement.cc
|
||||
cql3/statements/alter_type_statement.cc
|
||||
cql3/statements/alter_view_statement.cc
|
||||
cql3/statements/attach_service_level_statement.cc
|
||||
cql3/statements/authentication_statement.cc
|
||||
cql3/statements/authorization_statement.cc
|
||||
cql3/statements/batch_statement.cc
|
||||
cql3/statements/cas_request.cc
|
||||
cql3/statements/cf_prop_defs.cc
|
||||
cql3/statements/cf_statement.cc
|
||||
cql3/statements/create_aggregate_statement.cc
|
||||
cql3/statements/create_function_statement.cc
|
||||
cql3/statements/create_index_statement.cc
|
||||
cql3/statements/create_keyspace_statement.cc
|
||||
cql3/statements/create_service_level_statement.cc
|
||||
cql3/statements/create_table_statement.cc
|
||||
cql3/statements/create_type_statement.cc
|
||||
cql3/statements/create_view_statement.cc
|
||||
cql3/statements/delete_statement.cc
|
||||
cql3/statements/detach_service_level_statement.cc
|
||||
cql3/statements/drop_aggregate_statement.cc
|
||||
cql3/statements/drop_function_statement.cc
|
||||
cql3/statements/drop_index_statement.cc
|
||||
cql3/statements/drop_keyspace_statement.cc
|
||||
cql3/statements/drop_service_level_statement.cc
|
||||
cql3/statements/drop_table_statement.cc
|
||||
cql3/statements/drop_type_statement.cc
|
||||
cql3/statements/drop_view_statement.cc
|
||||
cql3/statements/function_statement.cc
|
||||
cql3/statements/grant_statement.cc
|
||||
cql3/statements/index_prop_defs.cc
|
||||
cql3/statements/index_target.cc
|
||||
cql3/statements/ks_prop_defs.cc
|
||||
cql3/statements/list_permissions_statement.cc
|
||||
cql3/statements/list_service_level_attachments_statement.cc
|
||||
cql3/statements/list_service_level_statement.cc
|
||||
cql3/statements/list_users_statement.cc
|
||||
cql3/statements/modification_statement.cc
|
||||
cql3/statements/permission_altering_statement.cc
|
||||
cql3/statements/property_definitions.cc
|
||||
cql3/statements/raw/parsed_statement.cc
|
||||
cql3/statements/revoke_statement.cc
|
||||
cql3/statements/role-management-statements.cc
|
||||
cql3/statements/schema_altering_statement.cc
|
||||
cql3/statements/select_statement.cc
|
||||
cql3/statements/service_level_statement.cc
|
||||
cql3/statements/sl_prop_defs.cc
|
||||
cql3/statements/truncate_statement.cc
|
||||
cql3/statements/update_statement.cc
|
||||
cql3/statements/use_statement.cc
|
||||
cql3/type_json.cc
|
||||
cql3/untyped_result_set.cc
|
||||
cql3/update_parameters.cc
|
||||
cql3/user_types.cc
|
||||
cql3/util.cc
|
||||
cql3/ut_name.cc
|
||||
cql3/values.cc
|
||||
data_dictionary/data_dictionary.cc
|
||||
db/batchlog_manager.cc
|
||||
db/commitlog/commitlog.cc
|
||||
db/commitlog/commitlog_entry.cc
|
||||
db/commitlog/commitlog_replayer.cc
|
||||
db/config.cc
|
||||
db/consistency_level.cc
|
||||
db/cql_type_parser.cc
|
||||
db/data_listeners.cc
|
||||
db/extensions.cc
|
||||
db/heat_load_balance.cc
|
||||
db/hints/host_filter.cc
|
||||
db/hints/manager.cc
|
||||
db/hints/resource_manager.cc
|
||||
db/hints/sync_point.cc
|
||||
db/large_data_handler.cc
|
||||
db/legacy_schema_migrator.cc
|
||||
db/marshal/type_parser.cc
|
||||
db/rate_limiter.cc
|
||||
db/schema_tables.cc
|
||||
db/size_estimates_virtual_reader.cc
|
||||
db/snapshot-ctl.cc
|
||||
db/sstables-format-selector.cc
|
||||
db/system_distributed_keyspace.cc
|
||||
db/system_keyspace.cc
|
||||
db/view/row_locking.cc
|
||||
db/view/view.cc
|
||||
db/view/view_update_generator.cc
|
||||
db/virtual_table.cc
|
||||
dht/boot_strapper.cc
|
||||
dht/i_partitioner.cc
|
||||
dht/murmur3_partitioner.cc
|
||||
dht/range_streamer.cc
|
||||
dht/token.cc
|
||||
replica/distributed_loader.cc
|
||||
duration.cc
|
||||
exceptions/exceptions.cc
|
||||
readers/mutation_readers.cc
|
||||
frozen_mutation.cc
|
||||
frozen_schema.cc
|
||||
generic_server.cc
|
||||
debug.cc
|
||||
gms/application_state.cc
|
||||
gms/endpoint_state.cc
|
||||
gms/failure_detector.cc
|
||||
gms/feature_service.cc
|
||||
gms/gossip_digest_ack2.cc
|
||||
gms/gossip_digest_ack.cc
|
||||
gms/gossip_digest_syn.cc
|
||||
gms/gossiper.cc
|
||||
gms/inet_address.cc
|
||||
gms/versioned_value.cc
|
||||
gms/version_generator.cc
|
||||
hashers.cc
|
||||
index/secondary_index.cc
|
||||
index/secondary_index_manager.cc
|
||||
init.cc
|
||||
keys.cc
|
||||
utils/lister.cc
|
||||
locator/abstract_replication_strategy.cc
|
||||
locator/azure_snitch.cc
|
||||
locator/ec2_multi_region_snitch.cc
|
||||
locator/ec2_snitch.cc
|
||||
locator/everywhere_replication_strategy.cc
|
||||
locator/gce_snitch.cc
|
||||
locator/gossiping_property_file_snitch.cc
|
||||
locator/local_strategy.cc
|
||||
locator/network_topology_strategy.cc
|
||||
locator/production_snitch_base.cc
|
||||
locator/rack_inferring_snitch.cc
|
||||
locator/simple_snitch.cc
|
||||
locator/simple_strategy.cc
|
||||
locator/snitch_base.cc
|
||||
locator/token_metadata.cc
|
||||
lang/lua.cc
|
||||
main.cc
|
||||
replica/memtable.cc
|
||||
message/messaging_service.cc
|
||||
multishard_mutation_query.cc
|
||||
mutation.cc
|
||||
mutation_fragment.cc
|
||||
mutation_partition.cc
|
||||
mutation_partition_serializer.cc
|
||||
mutation_partition_view.cc
|
||||
mutation_query.cc
|
||||
readers/mutation_reader.cc
|
||||
mutation_writer/feed_writers.cc
|
||||
mutation_writer/multishard_writer.cc
|
||||
mutation_writer/partition_based_splitting_writer.cc
|
||||
mutation_writer/shard_based_splitting_writer.cc
|
||||
mutation_writer/timestamp_based_splitting_writer.cc
|
||||
partition_slice_builder.cc
|
||||
partition_version.cc
|
||||
querier.cc
|
||||
query.cc
|
||||
query_ranges_to_vnodes.cc
|
||||
query-result-set.cc
|
||||
raft/fsm.cc
|
||||
raft/log.cc
|
||||
raft/raft.cc
|
||||
raft/server.cc
|
||||
raft/tracker.cc
|
||||
range_tombstone.cc
|
||||
range_tombstone_list.cc
|
||||
tombstone_gc_options.cc
|
||||
tombstone_gc.cc
|
||||
reader_concurrency_semaphore.cc
|
||||
redis/abstract_command.cc
|
||||
redis/command_factory.cc
|
||||
redis/commands.cc
|
||||
redis/keyspace_utils.cc
|
||||
redis/lolwut.cc
|
||||
redis/mutation_utils.cc
|
||||
redis/options.cc
|
||||
redis/query_processor.cc
|
||||
redis/query_utils.cc
|
||||
redis/server.cc
|
||||
redis/service.cc
|
||||
redis/stats.cc
|
||||
release.cc
|
||||
repair/repair.cc
|
||||
repair/row_level.cc
|
||||
replica/database.cc
|
||||
replica/table.cc
|
||||
row_cache.cc
|
||||
schema.cc
|
||||
schema_mutations.cc
|
||||
schema_registry.cc
|
||||
serializer.cc
|
||||
sstables_loader.cc
|
||||
service/client_state.cc
|
||||
service/forward_service.cc
|
||||
service/migration_manager.cc
|
||||
service/misc_services.cc
|
||||
service/pager/paging_state.cc
|
||||
service/pager/query_pagers.cc
|
||||
service/paxos/paxos_state.cc
|
||||
service/paxos/prepare_response.cc
|
||||
service/paxos/prepare_summary.cc
|
||||
service/paxos/proposal.cc
|
||||
service/priority_manager.cc
|
||||
service/qos/qos_common.cc
|
||||
service/qos/service_level_controller.cc
|
||||
service/qos/standard_service_level_distributed_data_accessor.cc
|
||||
service/raft/raft_group_registry.cc
|
||||
service/raft/raft_rpc.cc
|
||||
service/raft/raft_sys_table_storage.cc
|
||||
service/raft/group0_state_machine.cc
|
||||
service/storage_proxy.cc
|
||||
service/storage_service.cc
|
||||
sstables/compress.cc
|
||||
sstables/integrity_checked_file_impl.cc
|
||||
sstables/kl/reader.cc
|
||||
sstables/metadata_collector.cc
|
||||
sstables/m_format_read_helpers.cc
|
||||
sstables/mx/reader.cc
|
||||
sstables/mx/writer.cc
|
||||
sstables/prepended_input_stream.cc
|
||||
sstables/random_access_reader.cc
|
||||
sstables/sstable_directory.cc
|
||||
sstables/sstable_mutation_reader.cc
|
||||
sstables/sstables.cc
|
||||
sstables/sstable_set.cc
|
||||
sstables/sstables_manager.cc
|
||||
sstables/sstable_version.cc
|
||||
sstables/writer.cc
|
||||
streaming/consumer.cc
|
||||
streaming/progress_info.cc
|
||||
streaming/session_info.cc
|
||||
streaming/stream_coordinator.cc
|
||||
streaming/stream_manager.cc
|
||||
streaming/stream_plan.cc
|
||||
streaming/stream_reason.cc
|
||||
streaming/stream_receive_task.cc
|
||||
streaming/stream_request.cc
|
||||
streaming/stream_result_future.cc
|
||||
streaming/stream_session.cc
|
||||
streaming/stream_session_state.cc
|
||||
streaming/stream_summary.cc
|
||||
streaming/stream_task.cc
|
||||
streaming/stream_transfer_task.cc
|
||||
table_helper.cc
|
||||
tasks/task_manager.cc
|
||||
thrift/controller.cc
|
||||
thrift/handler.cc
|
||||
thrift/server.cc
|
||||
thrift/thrift_validation.cc
|
||||
timeout_config.cc
|
||||
tools/scylla-sstable-index.cc
|
||||
tools/scylla-types.cc
|
||||
tracing/traced_file.cc
|
||||
tracing/trace_keyspace_helper.cc
|
||||
tracing/trace_state.cc
|
||||
tracing/tracing_backend_registry.cc
|
||||
tracing/tracing.cc
|
||||
transport/controller.cc
|
||||
transport/cql_protocol_extension.cc
|
||||
transport/event.cc
|
||||
transport/event_notifier.cc
|
||||
transport/messages/result_message.cc
|
||||
transport/server.cc
|
||||
types.cc
|
||||
unimplemented.cc
|
||||
utils/arch/powerpc/crc32-vpmsum/crc32_wrapper.cc
|
||||
utils/array-search.cc
|
||||
utils/ascii.cc
|
||||
utils/base64.cc
|
||||
utils/big_decimal.cc
|
||||
utils/bloom_calculations.cc
|
||||
utils/bloom_filter.cc
|
||||
utils/buffer_input_stream.cc
|
||||
utils/build_id.cc
|
||||
utils/config_file.cc
|
||||
utils/directories.cc
|
||||
utils/disk-error-handler.cc
|
||||
utils/dynamic_bitset.cc
|
||||
utils/error_injection.cc
|
||||
utils/exceptions.cc
|
||||
utils/file_lock.cc
|
||||
utils/generation-number.cc
|
||||
utils/gz/crc_combine.cc
|
||||
utils/gz/gen_crc_combine_table.cc
|
||||
utils/human_readable.cc
|
||||
utils/i_filter.cc
|
||||
utils/large_bitset.cc
|
||||
utils/like_matcher.cc
|
||||
utils/limiting_data_source.cc
|
||||
utils/logalloc.cc
|
||||
utils/managed_bytes.cc
|
||||
utils/multiprecision_int.cc
|
||||
utils/murmur_hash.cc
|
||||
utils/rate_limiter.cc
|
||||
utils/rjson.cc
|
||||
utils/runtime.cc
|
||||
utils/updateable_value.cc
|
||||
utils/utf8.cc
|
||||
utils/uuid.cc
|
||||
utils/UUID_gen.cc
|
||||
validation.cc
|
||||
vint-serialization.cc
|
||||
zstd.cc)
|
||||
target_link_libraries(scylla-main
|
||||
PRIVATE
|
||||
db
|
||||
absl::hash
|
||||
absl::raw_hash_set
|
||||
Seastar::seastar
|
||||
Snappy::snappy
|
||||
systemd
|
||||
ZLIB::ZLIB)
|
||||
add_subdirectory(api)
|
||||
add_subdirectory(alternator)
|
||||
add_subdirectory(db)
|
||||
add_subdirectory(auth)
|
||||
add_subdirectory(cdc)
|
||||
add_subdirectory(compaction)
|
||||
add_subdirectory(cql3)
|
||||
add_subdirectory(data_dictionary)
|
||||
add_subdirectory(dht)
|
||||
add_subdirectory(gms)
|
||||
add_subdirectory(idl)
|
||||
add_subdirectory(index)
|
||||
add_subdirectory(interface)
|
||||
add_subdirectory(lang)
|
||||
add_subdirectory(locator)
|
||||
add_subdirectory(message)
|
||||
add_subdirectory(mutation)
|
||||
add_subdirectory(mutation_writer)
|
||||
add_subdirectory(node_ops)
|
||||
add_subdirectory(readers)
|
||||
add_subdirectory(redis)
|
||||
add_subdirectory(replica)
|
||||
add_subdirectory(raft)
|
||||
add_subdirectory(repair)
|
||||
add_subdirectory(rust)
|
||||
add_subdirectory(schema)
|
||||
add_subdirectory(service)
|
||||
add_subdirectory(sstables)
|
||||
add_subdirectory(streaming)
|
||||
add_subdirectory(test)
|
||||
add_subdirectory(thrift)
|
||||
add_subdirectory(tools)
|
||||
add_subdirectory(tracing)
|
||||
add_subdirectory(transport)
|
||||
add_subdirectory(types)
|
||||
add_subdirectory(utils)
|
||||
add_version_library(scylla_version
|
||||
release.cc)
|
||||
|
||||
set(scylla_gen_sources
|
||||
"${scylla_thrift_gen_cassandra_files}"
|
||||
"${scylla_ragel_gen_protocol_parser_file}"
|
||||
"${swagger_gen_files}"
|
||||
"${idl_gen_files}"
|
||||
"${antlr3_gen_files}")
|
||||
|
||||
add_executable(scylla
|
||||
main.cc)
|
||||
target_link_libraries(scylla PRIVATE
|
||||
scylla-main
|
||||
api
|
||||
auth
|
||||
alternator
|
||||
db
|
||||
cdc
|
||||
compaction
|
||||
cql3
|
||||
data_dictionary
|
||||
dht
|
||||
gms
|
||||
idl
|
||||
index
|
||||
lang
|
||||
locator
|
||||
message
|
||||
mutation
|
||||
mutation_writer
|
||||
raft
|
||||
readers
|
||||
redis
|
||||
repair
|
||||
replica
|
||||
schema
|
||||
scylla_version
|
||||
service
|
||||
sstables
|
||||
streaming
|
||||
test-perf
|
||||
thrift
|
||||
tools
|
||||
tracing
|
||||
transport
|
||||
types
|
||||
utils)
|
||||
target_link_libraries(Boost::regex
|
||||
INTERFACE
|
||||
ICU::i18n
|
||||
ICU::uc)
|
||||
${scylla_sources}
|
||||
${scylla_gen_sources})
|
||||
|
||||
target_link_libraries(scylla PRIVATE
|
||||
seastar
|
||||
Boost::program_options)
|
||||
# Boost dependencies
|
||||
Boost::filesystem
|
||||
Boost::program_options
|
||||
Boost::system
|
||||
Boost::thread
|
||||
Boost::regex
|
||||
Boost::headers
|
||||
# Abseil libs
|
||||
absl::hashtablez_sampler
|
||||
absl::raw_hash_set
|
||||
absl::synchronization
|
||||
absl::graphcycles_internal
|
||||
absl::stacktrace
|
||||
absl::symbolize
|
||||
absl::debugging_internal
|
||||
absl::demangle_internal
|
||||
absl::time
|
||||
absl::time_zone
|
||||
absl::int128
|
||||
absl::city
|
||||
absl::hash
|
||||
absl::malloc_internal
|
||||
absl::spinlock_wait
|
||||
absl::base
|
||||
absl::dynamic_annotations
|
||||
absl::raw_logging_internal
|
||||
absl::exponential_biased
|
||||
absl::throw_delegate
|
||||
# System libs
|
||||
ZLIB::ZLIB
|
||||
ICU::uc
|
||||
systemd
|
||||
zstd
|
||||
snappy
|
||||
${LUA_LIBRARIES}
|
||||
thrift
|
||||
crypt)
|
||||
|
||||
target_link_libraries(scylla PRIVATE
|
||||
-Wl,--build-id=sha1 # Force SHA1 build-id generation
|
||||
# TODO: Use lld linker if it's available, otherwise gold, else bfd
|
||||
-fuse-ld=lld)
|
||||
# TODO: patch dynamic linker to match configure.py behavior
|
||||
|
||||
target_compile_options(scylla PRIVATE
|
||||
-std=gnu++20
|
||||
${cxx_coro_flag}
|
||||
${target_arch_flag})
|
||||
# Hacks needed to expose internal APIs for xxhash dependencies
|
||||
target_compile_definitions(scylla PRIVATE XXH_PRIVATE_API HAVE_LZ4_COMPRESS_DEFAULT)
|
||||
|
||||
target_include_directories(scylla PRIVATE
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}"
|
||||
libdeflate
|
||||
abseil
|
||||
"${scylla_gen_build_dir}")
|
||||
|
||||
add_subdirectory(dist)
|
||||
###
|
||||
### Create crc_combine_table helper executable.
|
||||
### Use it to generate crc_combine_table.cc to be used in scylla at build time.
|
||||
###
|
||||
add_executable(crc_combine_table utils/gz/gen_crc_combine_table.cc)
|
||||
target_link_libraries(crc_combine_table PRIVATE seastar)
|
||||
target_include_directories(crc_combine_table PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}")
|
||||
target_compile_options(crc_combine_table PRIVATE
|
||||
-std=gnu++20
|
||||
${cxx_coro_flag}
|
||||
${target_arch_flag})
|
||||
add_dependencies(scylla crc_combine_table)
|
||||
|
||||
# Generate an additional source file at build time that is needed for Scylla compilation
|
||||
add_custom_command(OUTPUT "${scylla_gen_build_dir}/utils/gz/crc_combine_table.cc"
|
||||
COMMAND $<TARGET_FILE:crc_combine_table> > "${scylla_gen_build_dir}/utils/gz/crc_combine_table.cc"
|
||||
DEPENDS crc_combine_table)
|
||||
target_sources(scylla PRIVATE "${scylla_gen_build_dir}/utils/gz/crc_combine_table.cc")
|
||||
|
||||
###
|
||||
### Generate version file and supply appropriate compile definitions for release.cc
|
||||
###
|
||||
execute_process(COMMAND ${CMAKE_SOURCE_DIR}/SCYLLA-VERSION-GEN --output-dir "${CMAKE_BINARY_DIR}/gen" RESULT_VARIABLE scylla_version_gen_res)
|
||||
if(scylla_version_gen_res)
|
||||
message(SEND_ERROR "Version file generation failed. Return code: ${scylla_version_gen_res}")
|
||||
endif()
|
||||
|
||||
file(READ "${CMAKE_BINARY_DIR}/gen/SCYLLA-VERSION-FILE" scylla_version)
|
||||
string(STRIP "${scylla_version}" scylla_version)
|
||||
|
||||
file(READ "${CMAKE_BINARY_DIR}/gen/SCYLLA-RELEASE-FILE" scylla_release)
|
||||
string(STRIP "${scylla_release}" scylla_release)
|
||||
|
||||
get_property(release_cdefs SOURCE "${CMAKE_SOURCE_DIR}/release.cc" PROPERTY COMPILE_DEFINITIONS)
|
||||
list(APPEND release_cdefs "SCYLLA_VERSION=\"${scylla_version}\"" "SCYLLA_RELEASE=\"${scylla_release}\"")
|
||||
set_source_files_properties("${CMAKE_SOURCE_DIR}/release.cc" PROPERTIES COMPILE_DEFINITIONS "${release_cdefs}")
|
||||
|
||||
###
|
||||
### Custom command for building libdeflate. Link the library to scylla.
|
||||
###
|
||||
set(libdeflate_lib "${scylla_build_dir}/libdeflate/libdeflate.a")
|
||||
add_custom_command(OUTPUT "${libdeflate_lib}"
|
||||
COMMAND make -C "${CMAKE_SOURCE_DIR}/libdeflate"
|
||||
BUILD_DIR=../build/${BUILD_TYPE}/libdeflate/
|
||||
CC=${CMAKE_C_COMPILER}
|
||||
"CFLAGS=${target_arch_flag}"
|
||||
../build/${BUILD_TYPE}/libdeflate//libdeflate.a) # Two backslashes are important!
|
||||
# Hack to force generating custom command to produce libdeflate.a
|
||||
add_custom_target(libdeflate DEPENDS "${libdeflate_lib}")
|
||||
target_link_libraries(scylla PRIVATE "${libdeflate_lib}")
|
||||
|
||||
# TODO: create cmake/ directory and move utilities (generate functions etc) there
|
||||
# TODO: Build tests if BUILD_TESTING=on (using CTest module)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
## Asking questions or requesting help
|
||||
|
||||
Use the [ScyllaDB Community Forum](https://forum.scylladb.com) or the [Slack workspace](http://slack.scylladb.com) for general questions and help.
|
||||
Use the [Scylla Users mailing list](https://groups.google.com/g/scylladb-users) or the [Slack workspace](http://slack.scylladb.com) for general questions and help.
|
||||
|
||||
Join the [Scylla Developers mailing list](https://groups.google.com/g/scylladb-dev) for deeper technical discussions and to discuss your ideas for contributions.
|
||||
|
||||
|
||||
@@ -195,7 +195,7 @@ $ # Edit configuration options as appropriate
|
||||
$ SCYLLA_HOME=$HOME/scylla build/release/scylla
|
||||
```
|
||||
|
||||
The `scylla.yaml` file in the repository by default writes all database data to `/var/lib/scylla`, which likely requires root access. Change the `data_file_directories`, `commitlog_directory` and `schema_commitlog_directory` fields as appropriate.
|
||||
The `scylla.yaml` file in the repository by default writes all database data to `/var/lib/scylla`, which likely requires root access. Change the `data_file_directories` and `commitlog_directory` fields as appropriate.
|
||||
|
||||
Scylla has a number of requirements for the file-system and operating system to operate ideally and at peak performance. However, during development, these requirements can be relaxed with the `--developer-mode` flag.
|
||||
|
||||
|
||||
12
README.md
12
README.md
@@ -30,9 +30,9 @@ requirements - you just need to meet the frozen toolchain's prerequisites
|
||||
Building Scylla with the frozen toolchain `dbuild` is as easy as:
|
||||
|
||||
```bash
|
||||
$ git submodule update --init --force --recursive
|
||||
$ ./tools/toolchain/dbuild ./configure.py
|
||||
$ ./tools/toolchain/dbuild ninja build/release/scylla
|
||||
$ git submodule update --init --force --recursive
|
||||
$ ./tools/toolchain/dbuild ./configure.py
|
||||
$ ./tools/toolchain/dbuild ninja build/release/scylla
|
||||
```
|
||||
|
||||
For further information, please see:
|
||||
@@ -60,7 +60,7 @@ Please note that you need to run Scylla with `dbuild` if you built it with the f
|
||||
For more run options, run:
|
||||
|
||||
```bash
|
||||
$ ./tools/toolchain/dbuild ./build/release/scylla --help
|
||||
$ ./tools/toolchain/dbuild ./build/release/scylla --help
|
||||
```
|
||||
|
||||
## Testing
|
||||
@@ -100,10 +100,10 @@ If you are a developer working on Scylla, please read the [developer guidelines]
|
||||
|
||||
## Contact
|
||||
|
||||
* The [community forum] and [Slack channel] are for users to discuss configuration, management, and operations of the ScyllaDB open source.
|
||||
* The [users mailing list] and [Slack channel] are for users to discuss configuration, management, and operations of the ScyllaDB open source.
|
||||
* The [developers mailing list] is for developers and people interested in following the development of ScyllaDB to discuss technical topics.
|
||||
|
||||
[Community forum]: https://forum.scylladb.com/
|
||||
[Users mailing list]: https://groups.google.com/forum/#!forum/scylladb-users
|
||||
|
||||
[Slack channel]: http://slack.scylladb.com/
|
||||
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
#!/bin/sh
|
||||
|
||||
USAGE=$(cat <<-END
|
||||
Usage: $(basename "$0") [-h|--help] [-o|--output-dir PATH] [--date-stamp DATE] -- generate Scylla version and build information files.
|
||||
Usage: $(basename "$0") [-h|--help] [-o|--output-dir PATH] -- generate Scylla version and build information files.
|
||||
|
||||
Options:
|
||||
-h|--help show this help message.
|
||||
-o|--output-dir PATH specify destination path at which the version files are to be created.
|
||||
-d|--date-stamp DATE manually set date for release parameter
|
||||
-v|--verbose also print out the version number
|
||||
|
||||
By default, the script will attempt to parse 'version' file
|
||||
in the current directory, which should contain a string of
|
||||
@@ -33,10 +31,7 @@ using '-o PATH' option.
|
||||
END
|
||||
)
|
||||
|
||||
DATE=""
|
||||
PRINT_VERSION=false
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
while [[ $# -gt 0 ]]; do
|
||||
opt="$1"
|
||||
case $opt in
|
||||
-h|--help)
|
||||
@@ -48,15 +43,6 @@ while [ $# -gt 0 ]; do
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
--date-stamp)
|
||||
DATE="$2"
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
-v|--verbose)
|
||||
PRINT_VERSION=true
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo "Unexpected argument found: $1"
|
||||
echo
|
||||
@@ -72,33 +58,24 @@ if [ -z "$OUTPUT_DIR" ]; then
|
||||
OUTPUT_DIR="$SCRIPT_DIR/build"
|
||||
fi
|
||||
|
||||
if [ -z "$DATE" ]; then
|
||||
DATE=$(date --utc +%Y%m%d)
|
||||
fi
|
||||
|
||||
# Default scylla product/version tags
|
||||
PRODUCT=scylla
|
||||
VERSION=5.5.0-dev
|
||||
VERSION=5.1.19
|
||||
|
||||
if test -f version
|
||||
then
|
||||
SCYLLA_VERSION=$(cat version | awk -F'-' '{print $1}')
|
||||
SCYLLA_RELEASE=$(cat version | awk -F'-' '{print $2}')
|
||||
else
|
||||
DATE=$(date --utc +%Y%m%d)
|
||||
GIT_COMMIT=$(git -C "$SCRIPT_DIR" log --pretty=format:'%h' -n 1 --abbrev=12)
|
||||
SCYLLA_VERSION=$VERSION
|
||||
if [ -z "$SCYLLA_RELEASE" ]; then
|
||||
DATE=$(date --utc +%Y%m%d)
|
||||
GIT_COMMIT=$(git -C "$SCRIPT_DIR" log --pretty=format:'%h' -n 1 --abbrev=12)
|
||||
# For custom package builds, replace "0" with "counter.your_name",
|
||||
# where counter starts at 1 and increments for successive versions.
|
||||
# This ensures that the package manager will select your custom
|
||||
# package over the standard release.
|
||||
SCYLLA_BUILD=0
|
||||
SCYLLA_RELEASE=$SCYLLA_BUILD.$DATE.$GIT_COMMIT
|
||||
elif [ -f "$OUTPUT_DIR/SCYLLA-RELEASE-FILE" ]; then
|
||||
echo "setting SCYLLA_RELEASE only makes sense in clean builds" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
# For custom package builds, replace "0" with "counter.your_name",
|
||||
# where counter starts at 1 and increments for successive versions.
|
||||
# This ensures that the package manager will select your custom
|
||||
# package over the standard release.
|
||||
SCYLLA_BUILD=0
|
||||
SCYLLA_RELEASE=$SCYLLA_BUILD.$DATE.$GIT_COMMIT
|
||||
fi
|
||||
|
||||
if [ -f "$OUTPUT_DIR/SCYLLA-RELEASE-FILE" ]; then
|
||||
@@ -108,9 +85,7 @@ if [ -f "$OUTPUT_DIR/SCYLLA-RELEASE-FILE" ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
if $PRINT_VERSION; then
|
||||
echo "$SCYLLA_VERSION-$SCYLLA_RELEASE"
|
||||
fi
|
||||
echo "$SCYLLA_VERSION-$SCYLLA_RELEASE"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
echo "$SCYLLA_VERSION" > "$OUTPUT_DIR/SCYLLA-VERSION-FILE"
|
||||
echo "$SCYLLA_RELEASE" > "$OUTPUT_DIR/SCYLLA-RELEASE-FILE"
|
||||
|
||||
1
abseil
Submodule
1
abseil
Submodule
Submodule abseil added at 9e408e050f
@@ -1,30 +0,0 @@
|
||||
include(generate_cql_grammar)
|
||||
generate_cql_grammar(
|
||||
GRAMMAR expressions.g
|
||||
SOURCES cql_grammar_srcs)
|
||||
|
||||
add_library(alternator STATIC)
|
||||
target_sources(alternator
|
||||
PRIVATE
|
||||
controller.cc
|
||||
server.cc
|
||||
executor.cc
|
||||
stats.cc
|
||||
serialization.cc
|
||||
expressions.cc
|
||||
conditions.cc
|
||||
auth.cc
|
||||
streams.cc
|
||||
ttl.cc
|
||||
${cql_grammar_srcs})
|
||||
target_include_directories(alternator
|
||||
PUBLIC
|
||||
${CMAKE_SOURCE_DIR}
|
||||
${CMAKE_BINARY_DIR}
|
||||
PRIVATE
|
||||
${RAPIDJSON_INCLUDE_DIRS})
|
||||
target_link_libraries(alternator
|
||||
cql3
|
||||
idl
|
||||
Seastar::seastar
|
||||
xxHash::xxhash)
|
||||
@@ -10,6 +10,8 @@
|
||||
#include "log.hh"
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <gnutls/crypto.h>
|
||||
#include "hashers.hh"
|
||||
#include "bytes.hh"
|
||||
#include "alternator/auth.hh"
|
||||
#include <fmt/format.h>
|
||||
@@ -27,6 +29,99 @@ namespace alternator {
|
||||
|
||||
static logging::logger alogger("alternator-auth");
|
||||
|
||||
static hmac_sha256_digest hmac_sha256(std::string_view key, std::string_view msg) {
|
||||
hmac_sha256_digest digest;
|
||||
int ret = gnutls_hmac_fast(GNUTLS_MAC_SHA256, key.data(), key.size(), msg.data(), msg.size(), digest.data());
|
||||
if (ret) {
|
||||
throw std::runtime_error(fmt::format("Computing HMAC failed ({}): {}", ret, gnutls_strerror(ret)));
|
||||
}
|
||||
return digest;
|
||||
}
|
||||
|
||||
static hmac_sha256_digest get_signature_key(std::string_view key, std::string_view date_stamp, std::string_view region_name, std::string_view service_name) {
|
||||
auto date = hmac_sha256("AWS4" + std::string(key), date_stamp);
|
||||
auto region = hmac_sha256(std::string_view(date.data(), date.size()), region_name);
|
||||
auto service = hmac_sha256(std::string_view(region.data(), region.size()), service_name);
|
||||
auto signing = hmac_sha256(std::string_view(service.data(), service.size()), "aws4_request");
|
||||
return signing;
|
||||
}
|
||||
|
||||
static std::string apply_sha256(std::string_view msg) {
|
||||
sha256_hasher hasher;
|
||||
hasher.update(msg.data(), msg.size());
|
||||
return to_hex(hasher.finalize());
|
||||
}
|
||||
|
||||
static std::string apply_sha256(const std::vector<temporary_buffer<char>>& msg) {
|
||||
sha256_hasher hasher;
|
||||
for (const temporary_buffer<char>& buf : msg) {
|
||||
hasher.update(buf.get(), buf.size());
|
||||
}
|
||||
return to_hex(hasher.finalize());
|
||||
}
|
||||
|
||||
static std::string format_time_point(db_clock::time_point tp) {
|
||||
time_t time_point_repr = db_clock::to_time_t(tp);
|
||||
std::string time_point_str;
|
||||
time_point_str.resize(17);
|
||||
::tm time_buf;
|
||||
// strftime prints the terminating null character as well
|
||||
std::strftime(time_point_str.data(), time_point_str.size(), "%Y%m%dT%H%M%SZ", ::gmtime_r(&time_point_repr, &time_buf));
|
||||
time_point_str.resize(16);
|
||||
return time_point_str;
|
||||
}
|
||||
|
||||
void check_expiry(std::string_view signature_date) {
|
||||
//FIXME: The default 15min can be changed with X-Amz-Expires header - we should honor it
|
||||
std::string expiration_str = format_time_point(db_clock::now() - 15min);
|
||||
std::string validity_str = format_time_point(db_clock::now() + 15min);
|
||||
if (signature_date < expiration_str) {
|
||||
throw api_error::invalid_signature(
|
||||
fmt::format("Signature expired: {} is now earlier than {} (current time - 15 min.)",
|
||||
signature_date, expiration_str));
|
||||
}
|
||||
if (signature_date > validity_str) {
|
||||
throw api_error::invalid_signature(
|
||||
fmt::format("Signature not yet current: {} is still later than {} (current time + 15 min.)",
|
||||
signature_date, validity_str));
|
||||
}
|
||||
}
|
||||
|
||||
std::string get_signature(std::string_view access_key_id, std::string_view secret_access_key, std::string_view host, std::string_view method,
|
||||
std::string_view orig_datestamp, std::string_view signed_headers_str, const std::map<std::string_view, std::string_view>& signed_headers_map,
|
||||
const std::vector<temporary_buffer<char>>& body_content, std::string_view region, std::string_view service, std::string_view query_string) {
|
||||
auto amz_date_it = signed_headers_map.find("x-amz-date");
|
||||
if (amz_date_it == signed_headers_map.end()) {
|
||||
throw api_error::invalid_signature("X-Amz-Date header is mandatory for signature verification");
|
||||
}
|
||||
std::string_view amz_date = amz_date_it->second;
|
||||
check_expiry(amz_date);
|
||||
std::string_view datestamp = amz_date.substr(0, 8);
|
||||
if (datestamp != orig_datestamp) {
|
||||
throw api_error::invalid_signature(
|
||||
format("X-Amz-Date date does not match the provided datestamp. Expected {}, got {}",
|
||||
orig_datestamp, datestamp));
|
||||
}
|
||||
std::string_view canonical_uri = "/";
|
||||
|
||||
std::stringstream canonical_headers;
|
||||
for (const auto& header : signed_headers_map) {
|
||||
canonical_headers << fmt::format("{}:{}", header.first, header.second) << '\n';
|
||||
}
|
||||
|
||||
std::string payload_hash = apply_sha256(body_content);
|
||||
std::string canonical_request = fmt::format("{}\n{}\n{}\n{}\n{}\n{}", method, canonical_uri, query_string, canonical_headers.str(), signed_headers_str, payload_hash);
|
||||
|
||||
std::string_view algorithm = "AWS4-HMAC-SHA256";
|
||||
std::string credential_scope = fmt::format("{}/{}/{}/aws4_request", datestamp, region, service);
|
||||
std::string string_to_sign = fmt::format("{}\n{}\n{}\n{}", algorithm, amz_date, credential_scope, apply_sha256(canonical_request));
|
||||
|
||||
hmac_sha256_digest signing_key = get_signature_key(secret_access_key, datestamp, region, service);
|
||||
hmac_sha256_digest signature = hmac_sha256(std::string_view(signing_key.data(), signing_key.size()), string_to_sign);
|
||||
|
||||
return to_hex(bytes_view(reinterpret_cast<const int8_t*>(signature.data()), signature.size()));
|
||||
}
|
||||
|
||||
future<std::string> get_key_from_roles(service::storage_proxy& proxy, std::string username) {
|
||||
schema_ptr schema = proxy.data_dictionary().find_schema("system_auth", "roles");
|
||||
partition_key pk = partition_key::from_single_value(*schema, utf8_type->decompose(username));
|
||||
@@ -38,22 +133,21 @@ future<std::string> get_key_from_roles(service::storage_proxy& proxy, std::strin
|
||||
}
|
||||
auto selection = cql3::selection::selection::for_columns(schema, {salted_hash_col});
|
||||
auto partition_slice = query::partition_slice(std::move(bounds), {}, query::column_id_vector{salted_hash_col->id}, selection->get_query_options());
|
||||
auto command = ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice,
|
||||
proxy.get_max_result_size(partition_slice), query::tombstone_limit(proxy.get_tombstone_limit()));
|
||||
auto command = ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice, proxy.get_max_result_size(partition_slice));
|
||||
auto cl = auth::password_authenticator::consistency_for_user(username);
|
||||
|
||||
service::client_state client_state{service::client_state::internal_tag()};
|
||||
service::storage_proxy::coordinator_query_result qr = co_await proxy.query(schema, std::move(command), std::move(partition_ranges), cl,
|
||||
service::storage_proxy::coordinator_query_options(executor::default_timeout(), empty_service_permit(), client_state));
|
||||
|
||||
cql3::selection::result_set_builder builder(*selection, gc_clock::now());
|
||||
cql3::selection::result_set_builder builder(*selection, gc_clock::now(), cql_serialization_format::latest());
|
||||
query::result_view::consume(*qr.query_result, partition_slice, cql3::selection::result_set_builder::visitor(builder, *schema, *selection));
|
||||
|
||||
auto result_set = builder.build();
|
||||
if (result_set->empty()) {
|
||||
co_await coroutine::return_exception(api_error::unrecognized_client(format("User not found: {}", username)));
|
||||
}
|
||||
const managed_bytes_opt& salted_hash = result_set->rows().front().front(); // We only asked for 1 row and 1 column
|
||||
const bytes_opt& salted_hash = result_set->rows().front().front(); // We only asked for 1 row and 1 column
|
||||
if (!salted_hash) {
|
||||
co_await coroutine::return_exception(api_error::unrecognized_client(format("No password found for user: {}", username)));
|
||||
}
|
||||
|
||||
@@ -20,8 +20,14 @@ class storage_proxy;
|
||||
|
||||
namespace alternator {
|
||||
|
||||
using hmac_sha256_digest = std::array<char, 32>;
|
||||
|
||||
using key_cache = utils::loading_cache<std::string, std::string, 1>;
|
||||
|
||||
std::string get_signature(std::string_view access_key_id, std::string_view secret_access_key, std::string_view host, std::string_view method,
|
||||
std::string_view orig_datestamp, std::string_view signed_headers_str, const std::map<std::string_view, std::string_view>& signed_headers_map,
|
||||
const std::vector<temporary_buffer<char>>& body_content, std::string_view region, std::string_view service, std::string_view query_string);
|
||||
|
||||
future<std::string> get_key_from_roles(service::storage_proxy& proxy, std::string username);
|
||||
|
||||
}
|
||||
|
||||
@@ -232,14 +232,7 @@ bool check_BEGINS_WITH(const rjson::value* v1, const rjson::value& v2,
|
||||
if (it2->name == "S") {
|
||||
return rjson::to_string_view(it1->value).starts_with(rjson::to_string_view(it2->value));
|
||||
} else /* it2->name == "B" */ {
|
||||
try {
|
||||
return base64_begins_with(rjson::to_string_view(it1->value), rjson::to_string_view(it2->value));
|
||||
} catch(std::invalid_argument&) {
|
||||
// determine if any of the malformed values is from query and raise an exception if so
|
||||
unwrap_bytes(it1->value, v1_from_query);
|
||||
unwrap_bytes(it2->value, v2_from_query);
|
||||
return false;
|
||||
}
|
||||
return base64_begins_with(rjson::to_string_view(it1->value), rjson::to_string_view(it2->value));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -248,7 +241,7 @@ static bool is_set_of(const rjson::value& type1, const rjson::value& type2) {
|
||||
}
|
||||
|
||||
// Check if two JSON-encoded values match with the CONTAINS relation
|
||||
bool check_CONTAINS(const rjson::value* v1, const rjson::value& v2, bool v1_from_query, bool v2_from_query) {
|
||||
bool check_CONTAINS(const rjson::value* v1, const rjson::value& v2) {
|
||||
if (!v1) {
|
||||
return false;
|
||||
}
|
||||
@@ -257,12 +250,7 @@ bool check_CONTAINS(const rjson::value* v1, const rjson::value& v2, bool v1_from
|
||||
if (kv1.name == "S" && kv2.name == "S") {
|
||||
return rjson::to_string_view(kv1.value).find(rjson::to_string_view(kv2.value)) != std::string_view::npos;
|
||||
} else if (kv1.name == "B" && kv2.name == "B") {
|
||||
auto d_kv1 = unwrap_bytes(kv1.value, v1_from_query);
|
||||
auto d_kv2 = unwrap_bytes(kv2.value, v2_from_query);
|
||||
if (!d_kv1 || !d_kv2) {
|
||||
return false;
|
||||
}
|
||||
return d_kv1->find(*d_kv2) != bytes::npos;
|
||||
return rjson::base64_decode(kv1.value).find(rjson::base64_decode(kv2.value)) != bytes::npos;
|
||||
} else if (is_set_of(kv1.name, kv2.name)) {
|
||||
for (auto i = kv1.value.Begin(); i != kv1.value.End(); ++i) {
|
||||
if (*i == kv2.value) {
|
||||
@@ -285,11 +273,11 @@ bool check_CONTAINS(const rjson::value* v1, const rjson::value& v2, bool v1_from
|
||||
}
|
||||
|
||||
// Check if two JSON-encoded values match with the NOT_CONTAINS relation
|
||||
static bool check_NOT_CONTAINS(const rjson::value* v1, const rjson::value& v2, bool v1_from_query, bool v2_from_query) {
|
||||
static bool check_NOT_CONTAINS(const rjson::value* v1, const rjson::value& v2) {
|
||||
if (!v1) {
|
||||
return false;
|
||||
}
|
||||
return !check_CONTAINS(v1, v2, v1_from_query, v2_from_query);
|
||||
return !check_CONTAINS(v1, v2);
|
||||
}
|
||||
|
||||
// Check if a JSON-encoded value equals any element of an array, which must have at least one element.
|
||||
@@ -386,12 +374,7 @@ bool check_compare(const rjson::value* v1, const rjson::value& v2, const Compara
|
||||
std::string_view(kv2.value.GetString(), kv2.value.GetStringLength()));
|
||||
}
|
||||
if (kv1.name == "B") {
|
||||
auto d_kv1 = unwrap_bytes(kv1.value, v1_from_query);
|
||||
auto d_kv2 = unwrap_bytes(kv2.value, v2_from_query);
|
||||
if(!d_kv1 || !d_kv2) {
|
||||
return false;
|
||||
}
|
||||
return cmp(*d_kv1, *d_kv2);
|
||||
return cmp(rjson::base64_decode(kv1.value), rjson::base64_decode(kv2.value));
|
||||
}
|
||||
// cannot reach here, as check_comparable_type() verifies the type is one
|
||||
// of the above options.
|
||||
@@ -481,13 +464,7 @@ static bool check_BETWEEN(const rjson::value* v, const rjson::value& lb, const r
|
||||
bounds_from_query);
|
||||
}
|
||||
if (kv_v.name == "B") {
|
||||
auto d_kv_v = unwrap_bytes(kv_v.value, v_from_query);
|
||||
auto d_kv_lb = unwrap_bytes(kv_lb.value, lb_from_query);
|
||||
auto d_kv_ub = unwrap_bytes(kv_ub.value, ub_from_query);
|
||||
if(!d_kv_v || !d_kv_lb || !d_kv_ub) {
|
||||
return false;
|
||||
}
|
||||
return check_BETWEEN(*d_kv_v, *d_kv_lb, *d_kv_ub, bounds_from_query);
|
||||
return check_BETWEEN(rjson::base64_decode(kv_v.value), rjson::base64_decode(kv_lb.value), rjson::base64_decode(kv_ub.value), bounds_from_query);
|
||||
}
|
||||
if (v_from_query) {
|
||||
throw api_error::validation(
|
||||
@@ -580,7 +557,7 @@ static bool verify_expected_one(const rjson::value& condition, const rjson::valu
|
||||
format("CONTAINS operator requires a single AttributeValue of type String, Number, or Binary, "
|
||||
"got {} instead", argtype));
|
||||
}
|
||||
return check_CONTAINS(got, arg, false, true);
|
||||
return check_CONTAINS(got, arg);
|
||||
}
|
||||
case comparison_operator_type::NOT_CONTAINS:
|
||||
{
|
||||
@@ -594,7 +571,7 @@ static bool verify_expected_one(const rjson::value& condition, const rjson::valu
|
||||
format("CONTAINS operator requires a single AttributeValue of type String, Number, or Binary, "
|
||||
"got {} instead", argtype));
|
||||
}
|
||||
return check_NOT_CONTAINS(got, arg, false, true);
|
||||
return check_NOT_CONTAINS(got, arg);
|
||||
}
|
||||
}
|
||||
throw std::logic_error(format("Internal error: corrupted operator enum: {}", int(op)));
|
||||
|
||||
@@ -38,7 +38,7 @@ conditional_operator_type get_conditional_operator(const rjson::value& req);
|
||||
bool verify_expected(const rjson::value& req, const rjson::value* previous_item);
|
||||
bool verify_condition(const rjson::value& condition, bool require_all, const rjson::value* previous_item);
|
||||
|
||||
bool check_CONTAINS(const rjson::value* v1, const rjson::value& v2, bool v1_from_query, bool v2_from_query);
|
||||
bool check_CONTAINS(const rjson::value* v1, const rjson::value& v2);
|
||||
bool check_BEGINS_WITH(const rjson::value* v1, const rjson::value& v2, bool v1_from_query, bool v2_from_query);
|
||||
|
||||
bool verify_condition_expression(
|
||||
|
||||
@@ -14,8 +14,6 @@
|
||||
#include "db/config.hh"
|
||||
#include "cdc/generation_service.hh"
|
||||
#include "service/memory_limiter.hh"
|
||||
#include "auth/service.hh"
|
||||
#include "service/qos/service_level_controller.hh"
|
||||
|
||||
using namespace seastar;
|
||||
|
||||
@@ -30,8 +28,6 @@ controller::controller(
|
||||
sharded<db::system_distributed_keyspace>& sys_dist_ks,
|
||||
sharded<cdc::generation_service>& cdc_gen_svc,
|
||||
sharded<service::memory_limiter>& memory_limiter,
|
||||
sharded<auth::service>& auth_service,
|
||||
sharded<qos::service_level_controller>& sl_controller,
|
||||
const db::config& config)
|
||||
: _gossiper(gossiper)
|
||||
, _proxy(proxy)
|
||||
@@ -39,8 +35,6 @@ controller::controller(
|
||||
, _sys_dist_ks(sys_dist_ks)
|
||||
, _cdc_gen_svc(cdc_gen_svc)
|
||||
, _memory_limiter(memory_limiter)
|
||||
, _auth_service(auth_service)
|
||||
, _sl_controller(sl_controller)
|
||||
, _config(config)
|
||||
{
|
||||
}
|
||||
@@ -76,17 +70,14 @@ future<> controller::start_server() {
|
||||
_ssg = create_smp_service_group(c).get0();
|
||||
|
||||
rmw_operation::set_default_write_isolation(_config.alternator_write_isolation());
|
||||
executor::set_default_timeout(std::chrono::milliseconds(_config.alternator_timeout_in_ms()));
|
||||
|
||||
net::inet_address addr = utils::resolve(_config.alternator_address, family).get0();
|
||||
|
||||
auto get_cdc_metadata = [] (cdc::generation_service& svc) { return std::ref(svc.get_cdc_metadata()); };
|
||||
auto get_timeout_in_ms = [] (const db::config& cfg) -> utils::updateable_value<uint32_t> {
|
||||
return cfg.alternator_timeout_in_ms;
|
||||
};
|
||||
_executor.start(std::ref(_gossiper), std::ref(_proxy), std::ref(_mm), std::ref(_sys_dist_ks),
|
||||
sharded_parameter(get_cdc_metadata, std::ref(_cdc_gen_svc)), _ssg.value(),
|
||||
sharded_parameter(get_timeout_in_ms, std::ref(_config))).get();
|
||||
_server.start(std::ref(_executor), std::ref(_proxy), std::ref(_gossiper), std::ref(_auth_service), std::ref(_sl_controller)).get();
|
||||
|
||||
_executor.start(std::ref(_gossiper), std::ref(_proxy), std::ref(_mm), std::ref(_sys_dist_ks), sharded_parameter(get_cdc_metadata, std::ref(_cdc_gen_svc)), _ssg.value()).get();
|
||||
_server.start(std::ref(_executor), std::ref(_proxy), std::ref(_gossiper)).get();
|
||||
// Note: from this point on, if start_server() throws for any reason,
|
||||
// it must first call stop_server() to stop the executor and server
|
||||
// services we just started - or Scylla will cause an assertion
|
||||
|
||||
@@ -34,14 +34,6 @@ class gossiper;
|
||||
|
||||
}
|
||||
|
||||
namespace auth {
|
||||
class service;
|
||||
}
|
||||
|
||||
namespace qos {
|
||||
class service_level_controller;
|
||||
}
|
||||
|
||||
namespace alternator {
|
||||
|
||||
// This is the official DynamoDB API version.
|
||||
@@ -61,8 +53,6 @@ class controller : public protocol_server {
|
||||
sharded<db::system_distributed_keyspace>& _sys_dist_ks;
|
||||
sharded<cdc::generation_service>& _cdc_gen_svc;
|
||||
sharded<service::memory_limiter>& _memory_limiter;
|
||||
sharded<auth::service>& _auth_service;
|
||||
sharded<qos::service_level_controller>& _sl_controller;
|
||||
const db::config& _config;
|
||||
|
||||
std::vector<socket_address> _listen_addresses;
|
||||
@@ -78,8 +68,6 @@ public:
|
||||
sharded<db::system_distributed_keyspace>& sys_dist_ks,
|
||||
sharded<cdc::generation_service>& cdc_gen_svc,
|
||||
sharded<service::memory_limiter>& memory_limiter,
|
||||
sharded<auth::service>& auth_service,
|
||||
sharded<qos::service_level_controller>& sl_controller,
|
||||
const db::config& config);
|
||||
|
||||
virtual sstring name() const override;
|
||||
|
||||
@@ -23,7 +23,7 @@ namespace alternator {
|
||||
// api_error into a JSON object, and that is returned to the user.
|
||||
class api_error final : public std::exception {
|
||||
public:
|
||||
using status_type = http::reply::status_type;
|
||||
using status_type = httpd::reply::status_type;
|
||||
status_type _http_code;
|
||||
std::string _type;
|
||||
std::string _msg;
|
||||
@@ -77,7 +77,7 @@ public:
|
||||
return api_error("TableNotFoundException", std::move(msg));
|
||||
}
|
||||
static api_error internal(std::string msg) {
|
||||
return api_error("InternalServerError", std::move(msg), http::reply::status_type::internal_server_error);
|
||||
return api_error("InternalServerError", std::move(msg), reply::status_type::internal_server_error);
|
||||
}
|
||||
|
||||
// Provide the "std::exception" interface, to make it easier to print this
|
||||
|
||||
@@ -6,17 +6,19 @@
|
||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
*/
|
||||
|
||||
#include <regex>
|
||||
|
||||
#include "utils/base64.hh"
|
||||
|
||||
#include <seastar/core/sleep.hh>
|
||||
#include "alternator/executor.hh"
|
||||
#include "log.hh"
|
||||
#include "schema/schema_builder.hh"
|
||||
#include "schema_builder.hh"
|
||||
#include "data_dictionary/keyspace_metadata.hh"
|
||||
#include "exceptions/exceptions.hh"
|
||||
#include "timestamp.hh"
|
||||
#include "types/map.hh"
|
||||
#include "schema/schema.hh"
|
||||
#include "schema.hh"
|
||||
#include "query-request.hh"
|
||||
#include "query-result-reader.hh"
|
||||
#include "cql3/selection/selection.hh"
|
||||
@@ -38,7 +40,8 @@
|
||||
#include <seastar/json/json_elements.hh>
|
||||
#include <boost/algorithm/cxx11/any_of.hpp>
|
||||
#include "collection_mutation.hh"
|
||||
#include "schema/schema.hh"
|
||||
#include "db/query_context.hh"
|
||||
#include "schema.hh"
|
||||
#include "db/tags/extension.hh"
|
||||
#include "db/tags/utils.hh"
|
||||
#include "alternator/rmw_operation.hh"
|
||||
@@ -48,39 +51,16 @@
|
||||
#include <unordered_set>
|
||||
#include "service/storage_proxy.hh"
|
||||
#include "gms/gossiper.hh"
|
||||
#include "schema/schema_registry.hh"
|
||||
#include "schema_registry.hh"
|
||||
#include "utils/error_injection.hh"
|
||||
#include "db/schema_tables.hh"
|
||||
#include "utils/rjson.hh"
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
logging::logger elogger("alternator-executor");
|
||||
|
||||
namespace alternator {
|
||||
|
||||
enum class table_status {
|
||||
active = 0,
|
||||
creating,
|
||||
updating,
|
||||
deleting
|
||||
};
|
||||
|
||||
static sstring_view table_status_to_sstring(table_status tbl_status) {
|
||||
switch(tbl_status) {
|
||||
case table_status::active:
|
||||
return "ACTIVE";
|
||||
case table_status::creating:
|
||||
return "CREATING";
|
||||
case table_status::updating:
|
||||
return "UPDATING";
|
||||
case table_status::deleting:
|
||||
return "DELETING";
|
||||
}
|
||||
return "UKNOWN";
|
||||
}
|
||||
|
||||
static lw_shared_ptr<keyspace_metadata> create_keyspace_metadata(std::string_view keyspace_name, service::storage_proxy& sp, gms::gossiper& gossiper, api::timestamp_type);
|
||||
static future<std::vector<mutation>> create_keyspace(std::string_view keyspace_name, service::storage_proxy& sp, service::migration_manager& mm, gms::gossiper& gossiper, api::timestamp_type);
|
||||
|
||||
static map_type attrs_type() {
|
||||
static thread_local auto t = map_type_impl::get_instance(utf8_type, bytes_type, true);
|
||||
@@ -138,7 +118,8 @@ std::string json_string::to_json() const {
|
||||
void executor::supplement_table_info(rjson::value& descr, const schema& schema, service::storage_proxy& sp) {
|
||||
rjson::add(descr, "CreationDateTime", rjson::value(std::chrono::duration_cast<std::chrono::seconds>(gc_clock::now().time_since_epoch()).count()));
|
||||
rjson::add(descr, "TableStatus", "ACTIVE");
|
||||
rjson::add(descr, "TableId", rjson::from_string(schema.id().to_sstring()));
|
||||
auto schema_id_str = schema.id().to_sstring();
|
||||
rjson::add(descr, "TableId", rjson::from_string(schema_id_str));
|
||||
|
||||
executor::supplement_table_stream_info(descr, schema, sp);
|
||||
}
|
||||
@@ -150,20 +131,6 @@ void executor::supplement_table_info(rjson::value& descr, const schema& schema,
|
||||
// See https://github.com/scylladb/scylla/issues/4480
|
||||
static constexpr int max_table_name_length = 222;
|
||||
|
||||
static bool valid_table_name_chars(std::string_view name) {
|
||||
for (auto c : name) {
|
||||
if ((c < 'a' || c > 'z') &&
|
||||
(c < 'A' || c > 'Z') &&
|
||||
(c < '0' || c > '9') &&
|
||||
c != '_' &&
|
||||
c != '-' &&
|
||||
c != '.') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// The DynamoDB developer guide, https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.NamingRulesDataTypes.html#HowItWorks.NamingRules
|
||||
// specifies that table names "names must be between 3 and 255 characters long
|
||||
// and can contain only the following characters: a-z, A-Z, 0-9, _ (underscore), - (dash), . (dot)
|
||||
@@ -173,7 +140,8 @@ static void validate_table_name(const std::string& name) {
|
||||
throw api_error::validation(
|
||||
format("TableName must be at least 3 characters long and at most {} characters long", max_table_name_length));
|
||||
}
|
||||
if (!valid_table_name_chars(name)) {
|
||||
static const std::regex valid_table_name_chars ("[a-zA-Z0-9_.-]*");
|
||||
if (!std::regex_match(name.c_str(), valid_table_name_chars)) {
|
||||
throw api_error::validation(
|
||||
"TableName must satisfy regular expression pattern: [a-zA-Z0-9_.-]+");
|
||||
}
|
||||
@@ -189,10 +157,11 @@ static void validate_table_name(const std::string& name) {
|
||||
// The view_name() function assumes the table_name has already been validated
|
||||
// but validates the legality of index_name and the combination of both.
|
||||
static std::string view_name(const std::string& table_name, std::string_view index_name, const std::string& delim = ":") {
|
||||
static const std::regex valid_index_name_chars ("[a-zA-Z0-9_.-]*");
|
||||
if (index_name.length() < 3) {
|
||||
throw api_error::validation("IndexName must be at least 3 characters long");
|
||||
}
|
||||
if (!valid_table_name_chars(index_name)) {
|
||||
if (!std::regex_match(index_name.data(), valid_index_name_chars)) {
|
||||
throw api_error::validation(
|
||||
format("IndexName '{}' must satisfy regular expression pattern: [a-zA-Z0-9_.-]+", index_name));
|
||||
}
|
||||
@@ -211,8 +180,9 @@ static std::string lsi_name(const std::string& table_name, std::string_view inde
|
||||
|
||||
/** Extract table name from a request.
|
||||
* Most requests expect the table's name to be listed in a "TableName" field.
|
||||
* This convenience function returns the name or api_error in case the
|
||||
* table name is missing or not a string.
|
||||
* This convenience function returns the name, with appropriate validation
|
||||
* and api_error in case the table name is missing or not a string, or
|
||||
* doesn't pass validate_table_name().
|
||||
*/
|
||||
static std::optional<std::string> find_table_name(const rjson::value& request) {
|
||||
const rjson::value* table_name_value = rjson::find(request, "TableName");
|
||||
@@ -223,6 +193,7 @@ static std::optional<std::string> find_table_name(const rjson::value& request) {
|
||||
throw api_error::validation("Non-string TableName field in request");
|
||||
}
|
||||
std::string table_name = table_name_value->GetString();
|
||||
validate_table_name(table_name);
|
||||
return table_name;
|
||||
}
|
||||
|
||||
@@ -249,10 +220,6 @@ schema_ptr executor::find_table(service::storage_proxy& proxy, const rjson::valu
|
||||
try {
|
||||
return proxy.data_dictionary().find_schema(sstring(executor::KEYSPACE_NAME_PREFIX) + sstring(*table_name), *table_name);
|
||||
} catch(data_dictionary::no_such_column_family&) {
|
||||
// DynamoDB returns validation error even when table does not exist
|
||||
// and the table name is invalid.
|
||||
validate_table_name(table_name.value());
|
||||
|
||||
throw api_error::resource_not_found(
|
||||
format("Requested resource not found: Table: {} not found", *table_name));
|
||||
}
|
||||
@@ -303,10 +270,6 @@ get_table_or_view(service::storage_proxy& proxy, const rjson::value& request) {
|
||||
try {
|
||||
return { proxy.data_dictionary().find_schema(sstring(internal_ks_name), sstring(internal_table_name)), type };
|
||||
} catch (data_dictionary::no_such_column_family&) {
|
||||
// DynamoDB returns validation error even when table does not exist
|
||||
// and the table name is invalid.
|
||||
validate_table_name(table_name);
|
||||
|
||||
throw api_error::resource_not_found(
|
||||
format("Requested resource not found: Internal table: {}.{} not found", internal_ks_name, internal_table_name));
|
||||
}
|
||||
@@ -442,91 +405,6 @@ static rjson::value generate_arn_for_index(const schema& schema, std::string_vie
|
||||
schema.ks_name(), schema.cf_name(), index_name));
|
||||
}
|
||||
|
||||
static rjson::value fill_table_description(schema_ptr schema, table_status tbl_status, service::storage_proxy const& proxy)
|
||||
{
|
||||
rjson::value table_description = rjson::empty_object();
|
||||
rjson::add(table_description, "TableName", rjson::from_string(schema->cf_name()));
|
||||
// FIXME: take the tables creation time, not the current time!
|
||||
size_t creation_date_seconds = std::chrono::duration_cast<std::chrono::seconds>(gc_clock::now().time_since_epoch()).count();
|
||||
// FIXME: In DynamoDB the CreateTable implementation is asynchronous, and
|
||||
// the table may be in "Creating" state until creating is finished.
|
||||
// We don't currently do this in Alternator - instead CreateTable waits
|
||||
// until the table is really available. So/ DescribeTable returns either
|
||||
// ACTIVE or doesn't exist at all (and DescribeTable returns an error).
|
||||
// The states CREATING and UPDATING are not currently returned.
|
||||
rjson::add(table_description, "TableStatus", rjson::from_string(table_status_to_sstring(tbl_status)));
|
||||
rjson::add(table_description, "TableArn", generate_arn_for_table(*schema));
|
||||
rjson::add(table_description, "TableId", rjson::from_string(schema->id().to_sstring()));
|
||||
// FIXME: Instead of hardcoding, we should take into account which mode was chosen
|
||||
// when the table was created. But, Spark jobs expect something to be returned
|
||||
// and PAY_PER_REQUEST seems closer to reality than PROVISIONED.
|
||||
rjson::add(table_description, "BillingModeSummary", rjson::empty_object());
|
||||
rjson::add(table_description["BillingModeSummary"], "BillingMode", "PAY_PER_REQUEST");
|
||||
rjson::add(table_description["BillingModeSummary"], "LastUpdateToPayPerRequestDateTime", rjson::value(creation_date_seconds));
|
||||
// In PAY_PER_REQUEST billing mode, provisioned capacity should return 0
|
||||
rjson::add(table_description, "ProvisionedThroughput", rjson::empty_object());
|
||||
rjson::add(table_description["ProvisionedThroughput"], "ReadCapacityUnits", 0);
|
||||
rjson::add(table_description["ProvisionedThroughput"], "WriteCapacityUnits", 0);
|
||||
rjson::add(table_description["ProvisionedThroughput"], "NumberOfDecreasesToday", 0);
|
||||
|
||||
|
||||
|
||||
data_dictionary::table t = proxy.data_dictionary().find_column_family(schema);
|
||||
|
||||
if (tbl_status != table_status::deleting) {
|
||||
rjson::add(table_description, "CreationDateTime", rjson::value(creation_date_seconds));
|
||||
std::unordered_map<std::string,std::string> key_attribute_types;
|
||||
// Add base table's KeySchema and collect types for AttributeDefinitions:
|
||||
executor::describe_key_schema(table_description, *schema, key_attribute_types);
|
||||
if (!t.views().empty()) {
|
||||
rjson::value gsi_array = rjson::empty_array();
|
||||
rjson::value lsi_array = rjson::empty_array();
|
||||
for (const view_ptr& vptr : t.views()) {
|
||||
rjson::value view_entry = rjson::empty_object();
|
||||
const sstring& cf_name = vptr->cf_name();
|
||||
size_t delim_it = cf_name.find(':');
|
||||
if (delim_it == sstring::npos) {
|
||||
elogger.error("Invalid internal index table name: {}", cf_name);
|
||||
continue;
|
||||
}
|
||||
sstring index_name = cf_name.substr(delim_it + 1);
|
||||
rjson::add(view_entry, "IndexName", rjson::from_string(index_name));
|
||||
rjson::add(view_entry, "IndexArn", generate_arn_for_index(*schema, index_name));
|
||||
// Add indexes's KeySchema and collect types for AttributeDefinitions:
|
||||
executor::describe_key_schema(view_entry, *vptr, key_attribute_types);
|
||||
// Add projection type
|
||||
rjson::value projection = rjson::empty_object();
|
||||
rjson::add(projection, "ProjectionType", "ALL");
|
||||
// FIXME: we have to get ProjectionType from the schema when it is added
|
||||
rjson::add(view_entry, "Projection", std::move(projection));
|
||||
// Local secondary indexes are marked by an extra '!' sign occurring before the ':' delimiter
|
||||
rjson::value& index_array = (delim_it > 1 && cf_name[delim_it-1] == '!') ? lsi_array : gsi_array;
|
||||
rjson::push_back(index_array, std::move(view_entry));
|
||||
}
|
||||
if (!lsi_array.Empty()) {
|
||||
rjson::add(table_description, "LocalSecondaryIndexes", std::move(lsi_array));
|
||||
}
|
||||
if (!gsi_array.Empty()) {
|
||||
rjson::add(table_description, "GlobalSecondaryIndexes", std::move(gsi_array));
|
||||
}
|
||||
}
|
||||
// Use map built by describe_key_schema() for base and indexes to produce
|
||||
// AttributeDefinitions for all key columns:
|
||||
rjson::value attribute_definitions = rjson::empty_array();
|
||||
for (auto& type : key_attribute_types) {
|
||||
rjson::value key = rjson::empty_object();
|
||||
rjson::add(key, "AttributeName", rjson::from_string(type.first));
|
||||
rjson::add(key, "AttributeType", rjson::from_string(type.second));
|
||||
rjson::push_back(attribute_definitions, std::move(key));
|
||||
}
|
||||
rjson::add(table_description, "AttributeDefinitions", std::move(attribute_definitions));
|
||||
}
|
||||
executor::supplement_table_stream_info(table_description, *schema, proxy);
|
||||
|
||||
// FIXME: still missing some response fields (issue #5026)
|
||||
return table_description;
|
||||
}
|
||||
|
||||
bool is_alternator_keyspace(const sstring& ks_name) {
|
||||
return ks_name.find(executor::KEYSPACE_NAME_PREFIX) == 0;
|
||||
}
|
||||
@@ -543,7 +421,85 @@ future<executor::request_return_type> executor::describe_table(client_state& cli
|
||||
|
||||
tracing::add_table_name(trace_state, schema->ks_name(), schema->cf_name());
|
||||
|
||||
rjson::value table_description = fill_table_description(schema, table_status::active, _proxy);
|
||||
rjson::value table_description = rjson::empty_object();
|
||||
rjson::add(table_description, "TableName", rjson::from_string(schema->cf_name()));
|
||||
// FIXME: take the tables creation time, not the current time!
|
||||
size_t creation_date_seconds = std::chrono::duration_cast<std::chrono::seconds>(gc_clock::now().time_since_epoch()).count();
|
||||
rjson::add(table_description, "CreationDateTime", rjson::value(creation_date_seconds));
|
||||
// FIXME: In DynamoDB the CreateTable implementation is asynchronous, and
|
||||
// the table may be in "Creating" state until creating is finished.
|
||||
// We don't currently do this in Alternator - instead CreateTable waits
|
||||
// until the table is really available. So/ DescribeTable returns either
|
||||
// ACTIVE or doesn't exist at all (and DescribeTable returns an error).
|
||||
// The other states (CREATING, UPDATING, DELETING) are not currently
|
||||
// returned.
|
||||
rjson::add(table_description, "TableStatus", "ACTIVE");
|
||||
rjson::add(table_description, "TableArn", generate_arn_for_table(*schema));
|
||||
rjson::add(table_description, "TableId", rjson::from_string(schema->id().to_sstring()));
|
||||
// FIXME: Instead of hardcoding, we should take into account which mode was chosen
|
||||
// when the table was created. But, Spark jobs expect something to be returned
|
||||
// and PAY_PER_REQUEST seems closer to reality than PROVISIONED.
|
||||
rjson::add(table_description, "BillingModeSummary", rjson::empty_object());
|
||||
rjson::add(table_description["BillingModeSummary"], "BillingMode", "PAY_PER_REQUEST");
|
||||
rjson::add(table_description["BillingModeSummary"], "LastUpdateToPayPerRequestDateTime", rjson::value(creation_date_seconds));
|
||||
// In PAY_PER_REQUEST billing mode, provisioned capacity should return 0
|
||||
rjson::add(table_description, "ProvisionedThroughput", rjson::empty_object());
|
||||
rjson::add(table_description["ProvisionedThroughput"], "ReadCapacityUnits", 0);
|
||||
rjson::add(table_description["ProvisionedThroughput"], "WriteCapacityUnits", 0);
|
||||
rjson::add(table_description["ProvisionedThroughput"], "NumberOfDecreasesToday", 0);
|
||||
|
||||
std::unordered_map<std::string,std::string> key_attribute_types;
|
||||
// Add base table's KeySchema and collect types for AttributeDefinitions:
|
||||
describe_key_schema(table_description, *schema, key_attribute_types);
|
||||
|
||||
data_dictionary::table t = _proxy.data_dictionary().find_column_family(schema);
|
||||
if (!t.views().empty()) {
|
||||
rjson::value gsi_array = rjson::empty_array();
|
||||
rjson::value lsi_array = rjson::empty_array();
|
||||
for (const view_ptr& vptr : t.views()) {
|
||||
rjson::value view_entry = rjson::empty_object();
|
||||
const sstring& cf_name = vptr->cf_name();
|
||||
size_t delim_it = cf_name.find(':');
|
||||
if (delim_it == sstring::npos) {
|
||||
elogger.error("Invalid internal index table name: {}", cf_name);
|
||||
continue;
|
||||
}
|
||||
sstring index_name = cf_name.substr(delim_it + 1);
|
||||
rjson::add(view_entry, "IndexName", rjson::from_string(index_name));
|
||||
rjson::add(view_entry, "IndexArn", generate_arn_for_index(*schema, index_name));
|
||||
// Add indexes's KeySchema and collect types for AttributeDefinitions:
|
||||
describe_key_schema(view_entry, *vptr, key_attribute_types);
|
||||
// Add projection type
|
||||
rjson::value projection = rjson::empty_object();
|
||||
rjson::add(projection, "ProjectionType", "ALL");
|
||||
// FIXME: we have to get ProjectionType from the schema when it is added
|
||||
rjson::add(view_entry, "Projection", std::move(projection));
|
||||
// Local secondary indexes are marked by an extra '!' sign occurring before the ':' delimiter
|
||||
rjson::value& index_array = (delim_it > 1 && cf_name[delim_it-1] == '!') ? lsi_array : gsi_array;
|
||||
rjson::push_back(index_array, std::move(view_entry));
|
||||
}
|
||||
if (!lsi_array.Empty()) {
|
||||
rjson::add(table_description, "LocalSecondaryIndexes", std::move(lsi_array));
|
||||
}
|
||||
if (!gsi_array.Empty()) {
|
||||
rjson::add(table_description, "GlobalSecondaryIndexes", std::move(gsi_array));
|
||||
}
|
||||
}
|
||||
// Use map built by describe_key_schema() for base and indexes to produce
|
||||
// AttributeDefinitions for all key columns:
|
||||
rjson::value attribute_definitions = rjson::empty_array();
|
||||
for (auto& type : key_attribute_types) {
|
||||
rjson::value key = rjson::empty_object();
|
||||
rjson::add(key, "AttributeName", rjson::from_string(type.first));
|
||||
rjson::add(key, "AttributeType", rjson::from_string(type.second));
|
||||
rjson::push_back(attribute_definitions, std::move(key));
|
||||
}
|
||||
rjson::add(table_description, "AttributeDefinitions", std::move(attribute_definitions));
|
||||
|
||||
supplement_table_stream_info(table_description, *schema, _proxy);
|
||||
|
||||
// FIXME: still missing some response fields (issue #5026)
|
||||
|
||||
rjson::value response = rjson::empty_object();
|
||||
rjson::add(response, "Table", std::move(table_description));
|
||||
elogger.trace("returning {}", response);
|
||||
@@ -555,17 +511,10 @@ future<executor::request_return_type> executor::delete_table(client_state& clien
|
||||
elogger.trace("Deleting table {}", request);
|
||||
|
||||
std::string table_name = get_table_name(request);
|
||||
// DynamoDB returns validation error even when table does not exist
|
||||
// and the table name is invalid.
|
||||
validate_table_name(table_name);
|
||||
|
||||
std::string keyspace_name = executor::KEYSPACE_NAME_PREFIX + table_name;
|
||||
tracing::add_table_name(trace_state, keyspace_name, table_name);
|
||||
auto& p = _proxy.container();
|
||||
|
||||
schema_ptr schema = get_table(_proxy, request);
|
||||
rjson::value table_description = fill_table_description(schema, table_status::deleting, _proxy);
|
||||
|
||||
co_await _mm.container().invoke_on(0, [&] (service::migration_manager& mm) -> future<> {
|
||||
// FIXME: the following needs to be in a loop. If mm.announce() below
|
||||
// fails, we need to retry the whole thing.
|
||||
@@ -575,14 +524,18 @@ future<executor::request_return_type> executor::delete_table(client_state& clien
|
||||
throw api_error::resource_not_found(format("Requested resource not found: Table: {} not found", table_name));
|
||||
}
|
||||
|
||||
auto m = co_await service::prepare_column_family_drop_announcement(_proxy, keyspace_name, table_name, group0_guard.write_timestamp(), service::drop_views::yes);
|
||||
auto m2 = co_await service::prepare_keyspace_drop_announcement(_proxy.local_db(), keyspace_name, group0_guard.write_timestamp());
|
||||
auto m = co_await mm.prepare_column_family_drop_announcement(keyspace_name, table_name, group0_guard.write_timestamp(), service::migration_manager::drop_views::yes);
|
||||
auto m2 = mm.prepare_keyspace_drop_announcement(keyspace_name, group0_guard.write_timestamp());
|
||||
|
||||
std::move(m2.begin(), m2.end(), std::back_inserter(m));
|
||||
|
||||
co_await mm.announce(std::move(m), std::move(group0_guard), format("alternator-executor: delete {} table", table_name));
|
||||
co_await mm.announce(std::move(m), std::move(group0_guard));
|
||||
});
|
||||
|
||||
// FIXME: need more attributes?
|
||||
rjson::value table_description = rjson::empty_object();
|
||||
rjson::add(table_description, "TableName", rjson::from_string(table_name));
|
||||
rjson::add(table_description, "TableStatus", "DELETING");
|
||||
rjson::value response = rjson::empty_object();
|
||||
rjson::add(response, "TableDescription", std::move(table_description));
|
||||
elogger.trace("returning {}", response);
|
||||
@@ -811,6 +764,7 @@ future<executor::request_return_type> executor::tag_resource(client_state& clien
|
||||
co_return api_error::access_denied("Incorrect resource identifier");
|
||||
}
|
||||
schema_ptr schema = get_table_from_arn(_proxy, rjson::to_string_view(*arn));
|
||||
std::map<sstring, sstring> tags_map = get_tags_of_table_or_throw(schema);
|
||||
const rjson::value* tags = rjson::find(request, "Tags");
|
||||
if (!tags || !tags->IsArray()) {
|
||||
co_return api_error::validation("Cannot parse tags");
|
||||
@@ -818,9 +772,8 @@ future<executor::request_return_type> executor::tag_resource(client_state& clien
|
||||
if (tags->Size() < 1) {
|
||||
co_return api_error::validation("The number of tags must be at least 1") ;
|
||||
}
|
||||
co_await db::modify_tags(_mm, schema->ks_name(), schema->cf_name(), [tags](std::map<sstring, sstring>& tags_map) {
|
||||
update_tags_map(*tags, tags_map, update_tags_action::add_tags);
|
||||
});
|
||||
update_tags_map(*tags, tags_map, update_tags_action::add_tags);
|
||||
co_await db::update_tags(_mm, schema, std::move(tags_map));
|
||||
co_return json_string("");
|
||||
}
|
||||
|
||||
@@ -838,9 +791,9 @@ future<executor::request_return_type> executor::untag_resource(client_state& cli
|
||||
|
||||
schema_ptr schema = get_table_from_arn(_proxy, rjson::to_string_view(*arn));
|
||||
|
||||
co_await db::modify_tags(_mm, schema->ks_name(), schema->cf_name(), [tags](std::map<sstring, sstring>& tags_map) {
|
||||
update_tags_map(*tags, tags_map, update_tags_action::delete_tags);
|
||||
});
|
||||
std::map<sstring, sstring> tags_map = get_tags_of_table_or_throw(schema);
|
||||
update_tags_map(*tags, tags_map, update_tags_action::delete_tags);
|
||||
co_await db::update_tags(_mm, schema, std::move(tags_map));
|
||||
co_return json_string("");
|
||||
}
|
||||
|
||||
@@ -867,6 +820,17 @@ future<executor::request_return_type> executor::list_tags_of_resource(client_sta
|
||||
return make_ready_future<executor::request_return_type>(make_jsonable(std::move(ret)));
|
||||
}
|
||||
|
||||
static future<> wait_for_schema_agreement(service::migration_manager& mm, db::timeout_clock::time_point deadline) {
|
||||
return do_until([&mm, deadline] {
|
||||
if (db::timeout_clock::now() > deadline) {
|
||||
throw std::runtime_error("Unable to reach schema agreement");
|
||||
}
|
||||
return mm.have_schema_agreement();
|
||||
}, [] {
|
||||
return seastar::sleep(500ms);
|
||||
});
|
||||
}
|
||||
|
||||
static void verify_billing_mode(const rjson::value& request) {
|
||||
// Alternator does not yet support billing or throughput limitations, but
|
||||
// let's verify that BillingMode is at least legal.
|
||||
@@ -884,38 +848,6 @@ static void verify_billing_mode(const rjson::value& request) {
|
||||
}
|
||||
}
|
||||
|
||||
// Validate that a AttributeDefinitions parameter in CreateTable is valid, and
|
||||
// throws user-facing api_error::validation if it's not.
|
||||
// In particular, verify that the same AttributeName doesn't appear more than
|
||||
// once (Issue #13870).
|
||||
static void validate_attribute_definitions(const rjson::value& attribute_definitions){
|
||||
if (!attribute_definitions.IsArray()) {
|
||||
throw api_error::validation("AttributeDefinitions must be an array");
|
||||
}
|
||||
std::unordered_set<std::string> seen_attribute_names;
|
||||
for (auto it = attribute_definitions.Begin(); it != attribute_definitions.End(); ++it) {
|
||||
const rjson::value* attribute_name = rjson::find(*it, "AttributeName");
|
||||
if (!attribute_name) {
|
||||
throw api_error::validation("AttributeName missing in AttributeDefinitions");
|
||||
}
|
||||
if (!attribute_name->IsString()) {
|
||||
throw api_error::validation("AttributeName in AttributeDefinitions must be a string");
|
||||
}
|
||||
auto [it2, added] = seen_attribute_names.emplace(rjson::to_string_view(*attribute_name));
|
||||
if (!added) {
|
||||
throw api_error::validation(format("Duplicate AttributeName={} in AttributeDefinitions",
|
||||
rjson::to_string_view(*attribute_name)));
|
||||
}
|
||||
const rjson::value* attribute_type = rjson::find(*it, "AttributeType");
|
||||
if (!attribute_type) {
|
||||
throw api_error::validation("AttributeType missing in AttributeDefinitions");
|
||||
}
|
||||
if (!attribute_type->IsString()) {
|
||||
throw api_error::validation("AttributeType in AttributeDefinitions must be a string");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static future<executor::request_return_type> create_table_on_shard0(tracing::trace_state_ptr trace_state, rjson::value request, service::storage_proxy& sp, service::migration_manager& mm, gms::gossiper& gossiper) {
|
||||
assert(this_shard_id() == 0);
|
||||
|
||||
@@ -924,14 +856,11 @@ static future<executor::request_return_type> create_table_on_shard0(tracing::tra
|
||||
// (e.g., verify that this table doesn't already exist) - we can only
|
||||
// do this further down - after taking group0_guard.
|
||||
std::string table_name = get_table_name(request);
|
||||
validate_table_name(table_name);
|
||||
|
||||
if (table_name.find(executor::INTERNAL_TABLE_PREFIX) == 0) {
|
||||
co_return api_error::validation(format("Prefix {} is reserved for accessing internal tables", executor::INTERNAL_TABLE_PREFIX));
|
||||
}
|
||||
std::string keyspace_name = executor::KEYSPACE_NAME_PREFIX + table_name;
|
||||
const rjson::value& attribute_definitions = request["AttributeDefinitions"];
|
||||
validate_attribute_definitions(attribute_definitions);
|
||||
|
||||
tracing::add_table_name(trace_state, keyspace_name, table_name);
|
||||
|
||||
@@ -1121,9 +1050,8 @@ static future<executor::request_return_type> create_table_on_shard0(tracing::tra
|
||||
auto group0_guard = co_await mm.start_group0_operation();
|
||||
auto ts = group0_guard.write_timestamp();
|
||||
std::vector<mutation> schema_mutations;
|
||||
auto ksm = create_keyspace_metadata(keyspace_name, sp, gossiper, ts);
|
||||
try {
|
||||
schema_mutations = service::prepare_new_keyspace_announcement(sp.local_db(), ksm, ts);
|
||||
schema_mutations = co_await create_keyspace(keyspace_name, sp, mm, gossiper, ts);
|
||||
} catch (exceptions::already_exists_exception&) {
|
||||
if (sp.data_dictionary().has_schema(keyspace_name, table_name)) {
|
||||
co_return api_error::resource_in_use(format("Table {} already exists", table_name));
|
||||
@@ -1133,14 +1061,22 @@ static future<executor::request_return_type> create_table_on_shard0(tracing::tra
|
||||
// This should never happen, the ID is supposed to be unique
|
||||
co_return api_error::internal(format("Table with ID {} already exists", schema->id()));
|
||||
}
|
||||
co_await service::prepare_new_column_family_announcement(schema_mutations, sp, *ksm, schema, ts);
|
||||
db::schema_tables::add_table_or_view_to_schema_mutation(schema, ts, true, schema_mutations);
|
||||
// we must call before_create_column_family callbacks - which allow
|
||||
// listeners to modify our schema_mutations. For example, CDC may add
|
||||
// another table (the CDC log table) to the same keyspace.
|
||||
// Unfortunately the convention is that this callback must be run in
|
||||
// a Seastar thread.
|
||||
co_await seastar::async([&] {
|
||||
mm.get_notifier().before_create_column_family(*schema, schema_mutations, ts);
|
||||
});
|
||||
for (schema_builder& view_builder : view_builders) {
|
||||
db::schema_tables::add_table_or_view_to_schema_mutation(
|
||||
view_ptr(view_builder.build()), ts, true, schema_mutations);
|
||||
}
|
||||
co_await mm.announce(std::move(schema_mutations), std::move(group0_guard), format("alternator-executor: create {} table", table_name));
|
||||
co_await mm.announce(std::move(schema_mutations), std::move(group0_guard));
|
||||
|
||||
co_await mm.wait_for_schema_agreement(sp.local_db(), db::timeout_clock::now() + 10s, nullptr);
|
||||
co_await wait_for_schema_agreement(mm, db::timeout_clock::now() + 10s);
|
||||
rjson::value status = rjson::empty_object();
|
||||
executor::supplement_table_info(request, *schema, sp);
|
||||
rjson::add(status, "TableDescription", std::move(request));
|
||||
@@ -1162,6 +1098,7 @@ future<executor::request_return_type> executor::update_table(client_state& clien
|
||||
elogger.trace("Updating table {}", request);
|
||||
|
||||
static const std::vector<sstring> unsupported = {
|
||||
"AttributeDefinitions",
|
||||
"GlobalSecondaryIndexUpdates",
|
||||
"ProvisionedThroughput",
|
||||
"ReplicaUpdates",
|
||||
@@ -1203,11 +1140,11 @@ future<executor::request_return_type> executor::update_table(client_state& clien
|
||||
|
||||
auto schema = builder.build();
|
||||
|
||||
auto m = co_await service::prepare_column_family_update_announcement(p.local(), schema, false, std::vector<view_ptr>(), group0_guard.write_timestamp());
|
||||
auto m = co_await mm.prepare_column_family_update_announcement(schema, false, std::vector<view_ptr>(), group0_guard.write_timestamp());
|
||||
|
||||
co_await mm.announce(std::move(m), std::move(group0_guard), format("alternator-executor: update {} table", tab->cf_name()));
|
||||
co_await mm.announce(std::move(m), std::move(group0_guard));
|
||||
|
||||
co_await mm.wait_for_schema_agreement(p.local().local_db(), db::timeout_clock::now() + 10s, nullptr);
|
||||
co_await wait_for_schema_agreement(mm, db::timeout_clock::now() + 10s);
|
||||
|
||||
rjson::value status = rjson::empty_object();
|
||||
supplement_table_info(request, *schema, p.local());
|
||||
@@ -1334,22 +1271,6 @@ put_or_delete_item::put_or_delete_item(const rjson::value& key, schema_ptr schem
|
||||
check_key(key, schema);
|
||||
}
|
||||
|
||||
// find_attribute() checks whether the named attribute is stored in the
|
||||
// schema as a real column (we do this for key attribute, and for a GSI key)
|
||||
// and if so, returns that column. If not, the function returns nullptr,
|
||||
// telling the caller that the attribute is stored serialized in the
|
||||
// ATTRS_COLUMN_NAME map - not in a stand-alone column in the schema.
|
||||
static inline const column_definition* find_attribute(const schema& schema, const bytes& attribute_name) {
|
||||
const column_definition* cdef = schema.get_column_definition(attribute_name);
|
||||
// Although ATTRS_COLUMN_NAME exists as an actual column, when used as an
|
||||
// attribute name it should refer to an attribute inside ATTRS_COLUMN_NAME
|
||||
// not to ATTRS_COLUMN_NAME itself. This if() is needed for #5009.
|
||||
if (cdef && cdef->name() == executor::ATTRS_COLUMN_NAME) {
|
||||
return nullptr;
|
||||
}
|
||||
return cdef;
|
||||
}
|
||||
|
||||
put_or_delete_item::put_or_delete_item(const rjson::value& item, schema_ptr schema, put_item)
|
||||
: _pk(pk_from_json(item, schema)), _ck(ck_from_json(item, schema)) {
|
||||
_cells = std::vector<cell>();
|
||||
@@ -1357,7 +1278,7 @@ put_or_delete_item::put_or_delete_item(const rjson::value& item, schema_ptr sche
|
||||
for (auto it = item.MemberBegin(); it != item.MemberEnd(); ++it) {
|
||||
bytes column_name = to_bytes(it->name.GetString());
|
||||
validate_value(it->value, "PutItem");
|
||||
const column_definition* cdef = find_attribute(*schema, column_name);
|
||||
const column_definition* cdef = schema->get_column_definition(column_name);
|
||||
if (!cdef) {
|
||||
bytes value = serialize_item(it->value);
|
||||
_cells->push_back({std::move(column_name), serialize_item(it->value)});
|
||||
@@ -1389,7 +1310,7 @@ mutation put_or_delete_item::build(schema_ptr schema, api::timestamp_type ts) co
|
||||
auto& row = m.partition().clustered_row(*schema, _ck);
|
||||
attribute_collector attrs_collector;
|
||||
for (auto& c : *_cells) {
|
||||
const column_definition* cdef = find_attribute(*schema, c.column_name);
|
||||
const column_definition* cdef = schema->get_column_definition(c.column_name);
|
||||
if (!cdef) {
|
||||
attrs_collector.put(c.column_name, c.value, ts);
|
||||
} else {
|
||||
@@ -1419,11 +1340,14 @@ mutation put_or_delete_item::build(schema_ptr schema, api::timestamp_type ts) co
|
||||
|
||||
// The DynamoDB API doesn't let the client control the server's timeout, so
|
||||
// we have a global default_timeout() for Alternator requests. The value of
|
||||
// s_default_timeout_ms is overwritten in alternator::controller::start_server()
|
||||
// s_default_timeout is overwritten in alternator::controller::start_server()
|
||||
// based on the "alternator_timeout_in_ms" configuration parameter.
|
||||
thread_local utils::updateable_value<uint32_t> executor::s_default_timeout_in_ms{10'000};
|
||||
db::timeout_clock::duration executor::s_default_timeout = 10s;
|
||||
void executor::set_default_timeout(db::timeout_clock::duration timeout) {
|
||||
s_default_timeout = timeout;
|
||||
}
|
||||
db::timeout_clock::time_point executor::default_timeout() {
|
||||
return db::timeout_clock::now() + std::chrono::milliseconds(s_default_timeout_in_ms);
|
||||
return db::timeout_clock::now() + s_default_timeout;
|
||||
}
|
||||
|
||||
static future<std::unique_ptr<rjson::value>> get_previous_item(
|
||||
@@ -1451,8 +1375,7 @@ static lw_shared_ptr<query::read_command> previous_item_read_command(service::st
|
||||
auto regular_columns = boost::copy_range<query::column_id_vector>(
|
||||
schema->regular_columns() | boost::adaptors::transformed([] (const column_definition& cdef) { return cdef.id; }));
|
||||
auto partition_slice = query::partition_slice(std::move(bounds), {}, std::move(regular_columns), selection->get_query_options());
|
||||
return ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice, proxy.get_max_result_size(partition_slice),
|
||||
query::tombstone_limit(proxy.get_tombstone_limit()));
|
||||
return ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice, proxy.get_max_result_size(partition_slice));
|
||||
}
|
||||
|
||||
static dht::partition_range_vector to_partition_ranges(const schema& schema, const partition_key& pk) {
|
||||
@@ -1596,7 +1519,7 @@ future<executor::request_return_type> rmw_operation::execute(service::storage_pr
|
||||
// This is the old, unsafe, read before write which does first
|
||||
// a read, then a write. TODO: remove this mode entirely.
|
||||
return get_previous_item(proxy, client_state, schema(), _pk, _ck, permit, stats).then(
|
||||
[this, &proxy, trace_state, permit = std::move(permit)] (std::unique_ptr<rjson::value> previous_item) mutable {
|
||||
[this, &client_state, &proxy, trace_state, permit = std::move(permit)] (std::unique_ptr<rjson::value> previous_item) mutable {
|
||||
std::optional<mutation> m = apply(std::move(previous_item), api::new_timestamp());
|
||||
if (!m) {
|
||||
return make_ready_future<executor::request_return_type>(api_error::conditional_check_failed("Failed condition."));
|
||||
@@ -1643,7 +1566,7 @@ static parsed::condition_expression get_parsed_condition_expression(rjson::value
|
||||
throw api_error::validation("ConditionExpression must not be empty");
|
||||
}
|
||||
try {
|
||||
return parse_condition_expression(rjson::to_string_view(*condition_expression), "ConditionExpression");
|
||||
return parse_condition_expression(rjson::to_string_view(*condition_expression));
|
||||
} catch(expressions_syntax_error& e) {
|
||||
throw api_error::validation(e.what());
|
||||
}
|
||||
@@ -1658,16 +1581,17 @@ static bool check_needs_read_before_write(const parsed::condition_expression& co
|
||||
|
||||
// Fail the expression if it has unused attribute names or values. This is
|
||||
// how DynamoDB behaves, so we do too.
|
||||
static void verify_all_are_used(const rjson::value* field,
|
||||
const std::unordered_set<std::string>& used, const char* field_name, const char* operation) {
|
||||
if (!field) {
|
||||
static void verify_all_are_used(const rjson::value& req, const char* field,
|
||||
const std::unordered_set<std::string>& used, const char* operation) {
|
||||
const rjson::value* attribute_names = rjson::find(req, field);
|
||||
if (!attribute_names) {
|
||||
return;
|
||||
}
|
||||
for (auto it = field->MemberBegin(); it != field->MemberEnd(); ++it) {
|
||||
for (auto it = attribute_names->MemberBegin(); it != attribute_names->MemberEnd(); ++it) {
|
||||
if (!used.contains(it->name.GetString())) {
|
||||
throw api_error::validation(
|
||||
format("{} has spurious '{}', not used in {}",
|
||||
field_name, it->name.GetString(), operation));
|
||||
field, it->name.GetString(), operation));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1694,8 +1618,8 @@ public:
|
||||
resolve_condition_expression(_condition_expression,
|
||||
expression_attribute_names, expression_attribute_values,
|
||||
used_attribute_names, used_attribute_values);
|
||||
verify_all_are_used(expression_attribute_names, used_attribute_names,"ExpressionAttributeNames", "PutItem");
|
||||
verify_all_are_used(expression_attribute_values, used_attribute_values,"ExpressionAttributeValues", "PutItem");
|
||||
verify_all_are_used(_request, "ExpressionAttributeNames", used_attribute_names, "PutItem");
|
||||
verify_all_are_used(_request, "ExpressionAttributeValues", used_attribute_values, "PutItem");
|
||||
} else {
|
||||
if (expression_attribute_names) {
|
||||
throw api_error::validation("ExpressionAttributeNames cannot be used without ConditionExpression");
|
||||
@@ -1779,8 +1703,8 @@ public:
|
||||
resolve_condition_expression(_condition_expression,
|
||||
expression_attribute_names, expression_attribute_values,
|
||||
used_attribute_names, used_attribute_values);
|
||||
verify_all_are_used(expression_attribute_names, used_attribute_names,"ExpressionAttributeNames", "DeleteItem");
|
||||
verify_all_are_used(expression_attribute_values, used_attribute_values, "ExpressionAttributeValues", "DeleteItem");
|
||||
verify_all_are_used(_request, "ExpressionAttributeNames", used_attribute_names, "DeleteItem");
|
||||
verify_all_are_used(_request, "ExpressionAttributeValues", used_attribute_values, "DeleteItem");
|
||||
} else {
|
||||
if (expression_attribute_names) {
|
||||
throw api_error::validation("ExpressionAttributeNames cannot be used without ConditionExpression");
|
||||
@@ -2350,14 +2274,14 @@ static std::optional<attrs_to_get> calculate_attrs_to_get(const rjson::value& re
|
||||
* as before.
|
||||
*/
|
||||
void executor::describe_single_item(const cql3::selection::selection& selection,
|
||||
const std::vector<managed_bytes_opt>& result_row,
|
||||
const std::vector<bytes_opt>& result_row,
|
||||
const std::optional<attrs_to_get>& attrs_to_get,
|
||||
rjson::value& item,
|
||||
bool include_all_embedded_attributes)
|
||||
{
|
||||
const auto& columns = selection.get_columns();
|
||||
auto column_it = columns.begin();
|
||||
for (const managed_bytes_opt& cell : result_row) {
|
||||
for (const bytes_opt& cell : result_row) {
|
||||
std::string column_name = (*column_it)->name_as_text();
|
||||
if (cell && column_name != executor::ATTRS_COLUMN_NAME) {
|
||||
if (!attrs_to_get || attrs_to_get->contains(column_name)) {
|
||||
@@ -2365,12 +2289,10 @@ void executor::describe_single_item(const cql3::selection::selection& selection,
|
||||
// so add() makes sense
|
||||
rjson::add_with_string_name(item, column_name, rjson::empty_object());
|
||||
rjson::value& field = item[column_name.c_str()];
|
||||
cell->with_linearized([&] (bytes_view linearized_cell) {
|
||||
rjson::add_with_string_name(field, type_to_string((*column_it)->type), json_key_column_value(linearized_cell, **column_it));
|
||||
});
|
||||
rjson::add_with_string_name(field, type_to_string((*column_it)->type), json_key_column_value(*cell, **column_it));
|
||||
}
|
||||
} else if (cell) {
|
||||
auto deserialized = attrs_type()->deserialize(*cell);
|
||||
auto deserialized = attrs_type()->deserialize(*cell, cql_serialization_format::latest());
|
||||
auto keys_and_values = value_cast<map_type_impl::native_type>(deserialized);
|
||||
for (auto entry : keys_and_values) {
|
||||
std::string attr_name = value_cast<sstring>(entry.first);
|
||||
@@ -2405,7 +2327,7 @@ std::optional<rjson::value> executor::describe_single_item(schema_ptr schema,
|
||||
const std::optional<attrs_to_get>& attrs_to_get) {
|
||||
rjson::value item = rjson::empty_object();
|
||||
|
||||
cql3::selection::result_set_builder builder(selection, gc_clock::now());
|
||||
cql3::selection::result_set_builder builder(selection, gc_clock::now(), cql_serialization_format::latest());
|
||||
query::result_view::consume(query_result, slice, cql3::selection::result_set_builder::visitor(builder, *schema, selection));
|
||||
|
||||
auto result_set = builder.build();
|
||||
@@ -2428,7 +2350,7 @@ future<std::vector<rjson::value>> executor::describe_multi_item(schema_ptr schem
|
||||
shared_ptr<cql3::selection::selection> selection,
|
||||
foreign_ptr<lw_shared_ptr<query::result>> query_result,
|
||||
shared_ptr<const std::optional<attrs_to_get>> attrs_to_get) {
|
||||
cql3::selection::result_set_builder builder(*selection, gc_clock::now());
|
||||
cql3::selection::result_set_builder builder(*selection, gc_clock::now(), cql_serialization_format::latest());
|
||||
query::result_view::consume(*query_result, slice, cql3::selection::result_set_builder::visitor(builder, *schema, *selection));
|
||||
auto result_set = builder.build();
|
||||
std::vector<rjson::value> ret;
|
||||
@@ -2553,8 +2475,8 @@ update_item_operation::update_item_operation(service::storage_proxy& proxy, rjso
|
||||
expression_attribute_names, expression_attribute_values,
|
||||
used_attribute_names, used_attribute_values);
|
||||
|
||||
verify_all_are_used(expression_attribute_names, used_attribute_names, "ExpressionAttributeNames", "UpdateItem");
|
||||
verify_all_are_used(expression_attribute_values, used_attribute_values, "ExpressionAttributeValues", "UpdateItem");
|
||||
verify_all_are_used(_request, "ExpressionAttributeNames", used_attribute_names, "UpdateItem");
|
||||
verify_all_are_used(_request, "ExpressionAttributeValues", used_attribute_values, "UpdateItem");
|
||||
|
||||
// DynamoDB forbids having both old-style AttributeUpdates or Expected
|
||||
// and new-style UpdateExpression or ConditionExpression in the same request
|
||||
@@ -2843,7 +2765,7 @@ update_item_operation::apply(std::unique_ptr<rjson::value> previous_item, api::t
|
||||
}
|
||||
}
|
||||
}
|
||||
const column_definition* cdef = find_attribute(*_schema, column_name);
|
||||
const column_definition* cdef = _schema->get_column_definition(column_name);
|
||||
if (cdef) {
|
||||
bytes column_value = get_key_from_typed_value(json_value, *cdef);
|
||||
row.cells().apply(*cdef, atomic_cell::make_live(*cdef->type, ts, column_value));
|
||||
@@ -2865,7 +2787,7 @@ update_item_operation::apply(std::unique_ptr<rjson::value> previous_item, api::t
|
||||
rjson::add_with_string_name(_return_attributes, cn, rjson::copy(*col));
|
||||
}
|
||||
}
|
||||
const column_definition* cdef = find_attribute(*_schema, column_name);
|
||||
const column_definition* cdef = _schema->get_column_definition(column_name);
|
||||
if (cdef) {
|
||||
row.cells().apply(*cdef, atomic_cell::make_dead(ts, gc_clock::now()));
|
||||
} else {
|
||||
@@ -3158,13 +3080,11 @@ future<executor::request_return_type> executor::get_item(client_state& client_st
|
||||
auto selection = cql3::selection::selection::wildcard(schema);
|
||||
|
||||
auto partition_slice = query::partition_slice(std::move(bounds), {}, std::move(regular_columns), selection->get_query_options());
|
||||
auto command = ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice, _proxy.get_max_result_size(partition_slice),
|
||||
query::tombstone_limit(_proxy.get_tombstone_limit()));
|
||||
auto command = ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice, _proxy.get_max_result_size(partition_slice));
|
||||
|
||||
std::unordered_set<std::string> used_attribute_names;
|
||||
auto attrs_to_get = calculate_attrs_to_get(request, used_attribute_names);
|
||||
const rjson::value* expression_attribute_names = rjson::find(request, "ExpressionAttributeNames");
|
||||
verify_all_are_used(expression_attribute_names, used_attribute_names, "ExpressionAttributeNames", "GetItem");
|
||||
verify_all_are_used(request, "ExpressionAttributeNames", used_attribute_names, "GetItem");
|
||||
|
||||
return _proxy.query(schema, std::move(command), std::move(partition_ranges), cl,
|
||||
service::storage_proxy::coordinator_query_options(executor::default_timeout(), std::move(permit), client_state, trace_state)).then(
|
||||
@@ -3174,10 +3094,20 @@ future<executor::request_return_type> executor::get_item(client_state& client_st
|
||||
});
|
||||
}
|
||||
|
||||
// is_big() checks approximately if the given JSON value is "bigger" than
|
||||
// the given big_size number of bytes. The goal is to *quickly* detect
|
||||
// oversized JSON that, for example, is too large to be serialized to a
|
||||
// contiguous string - we don't need an accurate size for that. Moreover,
|
||||
// as soon as we detect that the JSON is indeed "big", we can return true
|
||||
// and don't need to continue calculating its exact size.
|
||||
// For simplicity, we use a recursive implementation. This is fine because
|
||||
// Alternator limits the depth of JSONs it reads from inputs, and doesn't
|
||||
// add more than a couple of levels in its own output construction.
|
||||
|
||||
static void check_big_object(const rjson::value& val, int& size_left);
|
||||
static void check_big_array(const rjson::value& val, int& size_left);
|
||||
|
||||
bool is_big(const rjson::value& val, int big_size) {
|
||||
static bool is_big(const rjson::value& val, int big_size = 100'000) {
|
||||
if (val.IsString()) {
|
||||
return ssize_t(val.GetStringLength()) > big_size;
|
||||
} else if (val.IsObject()) {
|
||||
@@ -3275,8 +3205,7 @@ future<executor::request_return_type> executor::batch_get_item(client_state& cli
|
||||
rs.cl = get_read_consistency(it->value);
|
||||
std::unordered_set<std::string> used_attribute_names;
|
||||
rs.attrs_to_get = ::make_shared<const std::optional<attrs_to_get>>(calculate_attrs_to_get(it->value, used_attribute_names));
|
||||
const rjson::value* expression_attribute_names = rjson::find(it->value, "ExpressionAttributeNames");
|
||||
verify_all_are_used(expression_attribute_names, used_attribute_names,"ExpressionAttributeNames", "GetItem");
|
||||
verify_all_are_used(request, "ExpressionAttributeNames", used_attribute_names, "GetItem");
|
||||
auto& keys = (it->value)["Keys"];
|
||||
for (rjson::value& key : keys.GetArray()) {
|
||||
rs.add(key);
|
||||
@@ -3305,8 +3234,7 @@ future<executor::request_return_type> executor::batch_get_item(client_state& cli
|
||||
rs.schema->regular_columns() | boost::adaptors::transformed([] (const column_definition& cdef) { return cdef.id; }));
|
||||
auto selection = cql3::selection::selection::wildcard(rs.schema);
|
||||
auto partition_slice = query::partition_slice(std::move(bounds), {}, std::move(regular_columns), selection->get_query_options());
|
||||
auto command = ::make_lw_shared<query::read_command>(rs.schema->id(), rs.schema->version(), partition_slice, _proxy.get_max_result_size(partition_slice),
|
||||
query::tombstone_limit(_proxy.get_tombstone_limit()));
|
||||
auto command = ::make_lw_shared<query::read_command>(rs.schema->id(), rs.schema->version(), partition_slice, _proxy.get_max_result_size(partition_slice));
|
||||
command->allow_limit = db::allow_per_partition_rate_limit::yes;
|
||||
future<std::vector<rjson::value>> f = _proxy.query(rs.schema, std::move(command), std::move(partition_ranges), rs.cl,
|
||||
service::storage_proxy::coordinator_query_options(executor::default_timeout(), permit, client_state, trace_state)).then(
|
||||
@@ -3445,7 +3373,7 @@ filter::filter(const rjson::value& request, request_type rt,
|
||||
throw api_error::validation("Cannot use both old-style and new-style parameters in same request: FilterExpression and AttributesToGet");
|
||||
}
|
||||
try {
|
||||
auto parsed = parse_condition_expression(rjson::to_string_view(*expression), "FilterExpression");
|
||||
auto parsed = parse_condition_expression(rjson::to_string_view(*expression));
|
||||
const rjson::value* expression_attribute_names = rjson::find(request, "ExpressionAttributeNames");
|
||||
const rjson::value* expression_attribute_values = rjson::find(request, "ExpressionAttributeValues");
|
||||
resolve_condition_expression(parsed,
|
||||
@@ -3552,7 +3480,7 @@ public:
|
||||
_column_it = _columns.begin();
|
||||
}
|
||||
|
||||
void accept_value(managed_bytes_view_opt result_bytes_view) {
|
||||
void accept_value(const std::optional<query::result_bytes_view>& result_bytes_view) {
|
||||
if (!result_bytes_view) {
|
||||
++_column_it;
|
||||
return;
|
||||
@@ -3568,7 +3496,7 @@ public:
|
||||
rjson::add_with_string_name(field, type_to_string((*_column_it)->type), json_key_column_value(bv, **_column_it));
|
||||
}
|
||||
} else {
|
||||
auto deserialized = attrs_type()->deserialize(bv);
|
||||
auto deserialized = attrs_type()->deserialize(bv, cql_serialization_format::latest());
|
||||
auto keys_and_values = value_cast<map_type_impl::native_type>(deserialized);
|
||||
for (auto entry : keys_and_values) {
|
||||
std::string attr_name = value_cast<sstring>(entry.first);
|
||||
@@ -3625,7 +3553,7 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
static std::tuple<rjson::value, size_t> describe_items(const cql3::selection::selection& selection, std::unique_ptr<cql3::result_set> result_set, std::optional<attrs_to_get>&& attrs_to_get, filter&& filter) {
|
||||
static std::tuple<rjson::value, size_t> describe_items(schema_ptr schema, const query::partition_slice& slice, const cql3::selection::selection& selection, std::unique_ptr<cql3::result_set> result_set, std::optional<attrs_to_get>&& attrs_to_get, filter&& filter) {
|
||||
describe_items_visitor visitor(selection.get_columns(), attrs_to_get, filter);
|
||||
result_set->visit(visitor);
|
||||
auto scanned_count = visitor.get_scanned_count();
|
||||
@@ -3675,7 +3603,7 @@ static rjson::value encode_paging_state(const schema& schema, const service::pag
|
||||
// We conditionally include these fields when reading CQL tables through alternator.
|
||||
if (!is_alternator_keyspace(schema.ks_name()) && (!pos.has_key() || pos.get_bound_weight() != bound_weight::equal)) {
|
||||
rjson::add_with_string_name(last_evaluated_key, scylla_paging_region, rjson::empty_object());
|
||||
rjson::add(last_evaluated_key[scylla_paging_region.data()], "S", rjson::from_string(fmt::to_string(pos.region())));
|
||||
rjson::add(last_evaluated_key[scylla_paging_region.data()], "S", rjson::from_string(to_string(pos.region())));
|
||||
rjson::add_with_string_name(last_evaluated_key, scylla_paging_weight, rjson::empty_object());
|
||||
rjson::add(last_evaluated_key[scylla_paging_weight.data()], "N", static_cast<int>(pos.get_bound_weight()));
|
||||
}
|
||||
@@ -3702,11 +3630,11 @@ static future<executor::request_return_type> do_query(service::storage_proxy& pr
|
||||
|
||||
if (exclusive_start_key) {
|
||||
partition_key pk = pk_from_json(*exclusive_start_key, schema);
|
||||
auto pos = position_in_partition::for_partition_start();
|
||||
auto pos = position_in_partition(position_in_partition::partition_start_tag_t());
|
||||
if (schema->clustering_key_size() > 0) {
|
||||
pos = pos_from_json(*exclusive_start_key, schema);
|
||||
}
|
||||
paging_state = make_lw_shared<service::pager::paging_state>(pk, pos, query::max_partitions, query_id::create_null_id(), service::pager::paging_state::replicas_per_token_range{}, std::nullopt, 0);
|
||||
paging_state = make_lw_shared<service::pager::paging_state>(pk, pos, query::max_partitions, utils::UUID(), service::pager::paging_state::replicas_per_token_range{}, std::nullopt, 0);
|
||||
}
|
||||
|
||||
auto regular_columns = boost::copy_range<query::column_id_vector>(
|
||||
@@ -3717,8 +3645,7 @@ static future<executor::request_return_type> do_query(service::storage_proxy& pr
|
||||
query::partition_slice::option_set opts = selection->get_query_options();
|
||||
opts.add(custom_opts);
|
||||
auto partition_slice = query::partition_slice(std::move(ck_bounds), std::move(static_columns), std::move(regular_columns), opts);
|
||||
auto command = ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice, proxy.get_max_result_size(partition_slice),
|
||||
query::tombstone_limit(proxy.get_tombstone_limit()));
|
||||
auto command = ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice, proxy.get_max_result_size(partition_slice));
|
||||
|
||||
auto query_state_ptr = std::make_unique<service::query_state>(client_state, trace_state, std::move(permit));
|
||||
|
||||
@@ -3739,7 +3666,7 @@ static future<executor::request_return_type> do_query(service::storage_proxy& pr
|
||||
}
|
||||
auto paging_state = rs->get_metadata().paging_state();
|
||||
bool has_filter = filter;
|
||||
auto [items, size] = describe_items(*selection, std::move(rs), std::move(attrs_to_get), std::move(filter));
|
||||
auto [items, size] = describe_items(schema, partition_slice, *selection, std::move(rs), std::move(attrs_to_get), std::move(filter));
|
||||
if (paging_state) {
|
||||
rjson::add(items, "LastEvaluatedKey", encode_paging_state(*schema, *paging_state));
|
||||
}
|
||||
@@ -3748,7 +3675,8 @@ static future<executor::request_return_type> do_query(service::storage_proxy& pr
|
||||
// update our "filtered_row_matched_total" for all the rows matched, despited the filter
|
||||
cql_stats.filtered_rows_matched_total += size;
|
||||
}
|
||||
if (is_big(items)) {
|
||||
// TODO: better threshold
|
||||
if (size > 10) {
|
||||
return make_ready_future<executor::request_return_type>(make_streamed(std::move(items)));
|
||||
}
|
||||
return make_ready_future<executor::request_return_type>(make_jsonable(std::move(items)));
|
||||
@@ -3849,10 +3777,8 @@ future<executor::request_return_type> executor::scan(client_state& client_state,
|
||||
// optimized the filtering by modifying partition_ranges and/or
|
||||
// ck_bounds. We haven't done this optimization yet.
|
||||
|
||||
const rjson::value* expression_attribute_names = rjson::find(request, "ExpressionAttributeNames");
|
||||
const rjson::value* expression_attribute_values = rjson::find(request, "ExpressionAttributeValues");
|
||||
verify_all_are_used(expression_attribute_names, used_attribute_names, "ExpressionAttributeNames", "Scan");
|
||||
verify_all_are_used(expression_attribute_values, used_attribute_values, "ExpressionAttributeValues", "Scan");
|
||||
verify_all_are_used(request, "ExpressionAttributeNames", used_attribute_names, "Scan");
|
||||
verify_all_are_used(request, "ExpressionAttributeValues", used_attribute_values, "Scan");
|
||||
|
||||
return do_query(_proxy, schema, exclusive_start_key, std::move(partition_ranges), std::move(ck_bounds), std::move(attrs_to_get), limit, cl,
|
||||
std::move(filter), query::partition_slice::option_set(), client_state, _stats.cql_stats, trace_state, std::move(permit));
|
||||
@@ -4073,7 +3999,7 @@ calculate_bounds_condition_expression(schema_ptr schema,
|
||||
// sort-key range.
|
||||
parsed::condition_expression p;
|
||||
try {
|
||||
p = parse_condition_expression(rjson::to_string_view(expression), "KeyConditionExpression");
|
||||
p = parse_condition_expression(rjson::to_string_view(expression));
|
||||
} catch(expressions_syntax_error& e) {
|
||||
throw api_error::validation(e.what());
|
||||
}
|
||||
@@ -4293,17 +4219,13 @@ future<executor::request_return_type> executor::query(client_state& client_state
|
||||
throw api_error::validation("Query must have one of "
|
||||
"KeyConditions or KeyConditionExpression");
|
||||
}
|
||||
|
||||
const rjson::value* expression_attribute_names = rjson::find(request, "ExpressionAttributeNames");
|
||||
const rjson::value* expression_attribute_values = rjson::find(request, "ExpressionAttributeValues");
|
||||
|
||||
// exactly one of key_conditions or key_condition_expression
|
||||
auto [partition_ranges, ck_bounds] = key_conditions
|
||||
? calculate_bounds_conditions(schema, *key_conditions)
|
||||
: calculate_bounds_condition_expression(schema, *key_condition_expression,
|
||||
expression_attribute_values,
|
||||
rjson::find(request, "ExpressionAttributeValues"),
|
||||
used_attribute_values,
|
||||
expression_attribute_names,
|
||||
rjson::find(request, "ExpressionAttributeNames"),
|
||||
used_attribute_names);
|
||||
|
||||
filter filter(request, filter::request_type::QUERY,
|
||||
@@ -4330,8 +4252,8 @@ future<executor::request_return_type> executor::query(client_state& client_state
|
||||
select_type select = parse_select(request, table_type);
|
||||
|
||||
auto attrs_to_get = calculate_attrs_to_get(request, used_attribute_names, select);
|
||||
verify_all_are_used(expression_attribute_names, used_attribute_names, "ExpressionAttributeNames", "Query");
|
||||
verify_all_are_used(expression_attribute_values, used_attribute_values, "ExpressionAttributeValues", "Query");
|
||||
verify_all_are_used(request, "ExpressionAttributeValues", used_attribute_values, "Query");
|
||||
verify_all_are_used(request, "ExpressionAttributeNames", used_attribute_names, "Query");
|
||||
query::partition_slice::option_set opts;
|
||||
opts.set_if<query::partition_slice::option::reversed>(!forward);
|
||||
return do_query(_proxy, schema, exclusive_start_key, std::move(partition_ranges), std::move(ck_bounds), std::move(attrs_to_get), limit, cl,
|
||||
@@ -4392,17 +4314,6 @@ future<executor::request_return_type> executor::list_tables(client_state& client
|
||||
|
||||
future<executor::request_return_type> executor::describe_endpoints(client_state& client_state, service_permit permit, rjson::value request, std::string host_header) {
|
||||
_stats.api_operations.describe_endpoints++;
|
||||
// The alternator_describe_endpoints configuration can be used to disable
|
||||
// the DescribeEndpoints operation, or set it to return a fixed string
|
||||
std::string override = _proxy.data_dictionary().get_config().alternator_describe_endpoints();
|
||||
if (!override.empty()) {
|
||||
if (override == "disabled") {
|
||||
_stats.unsupported_operations++;
|
||||
return make_ready_future<request_return_type>(api_error::unknown_operation(
|
||||
"DescribeEndpoints disabled by configuration (alternator_describe_endpoints=disabled)"));
|
||||
}
|
||||
host_header = std::move(override);
|
||||
}
|
||||
rjson::value response = rjson::empty_object();
|
||||
// Without having any configuration parameter to say otherwise, we tell
|
||||
// the user to return to the same endpoint they used to reach us. The only
|
||||
@@ -4440,10 +4351,6 @@ future<executor::request_return_type> executor::describe_continuous_backups(clie
|
||||
try {
|
||||
schema = _proxy.data_dictionary().find_schema(sstring(executor::KEYSPACE_NAME_PREFIX) + table_name, table_name);
|
||||
} catch(data_dictionary::no_such_column_family&) {
|
||||
// DynamoDB returns validation error even when table does not exist
|
||||
// and the table name is invalid.
|
||||
validate_table_name(table_name);
|
||||
|
||||
throw api_error::table_not_found(
|
||||
format("Table {} not found", table_name));
|
||||
}
|
||||
@@ -4457,23 +4364,25 @@ future<executor::request_return_type> executor::describe_continuous_backups(clie
|
||||
co_return make_jsonable(std::move(response));
|
||||
}
|
||||
|
||||
// Create the metadata for the keyspace in which we put the alternator
|
||||
// table if it doesn't already exist.
|
||||
// Create the keyspace in which we put the alternator table, if it doesn't
|
||||
// already exist.
|
||||
// Currently, we automatically configure the keyspace based on the number
|
||||
// of nodes in the cluster: A cluster with 3 or more live nodes, gets RF=3.
|
||||
// A smaller cluster (presumably, a test only), gets RF=1. The user may
|
||||
// manually create the keyspace to override this predefined behavior.
|
||||
static lw_shared_ptr<keyspace_metadata> create_keyspace_metadata(std::string_view keyspace_name, service::storage_proxy& sp, gms::gossiper& gossiper, api::timestamp_type ts) {
|
||||
int endpoint_count = gossiper.num_endpoints();
|
||||
static future<std::vector<mutation>> create_keyspace(std::string_view keyspace_name, service::storage_proxy& sp, service::migration_manager& mm, gms::gossiper& gossiper, api::timestamp_type ts) {
|
||||
sstring keyspace_name_str(keyspace_name);
|
||||
int endpoint_count = gossiper.get_endpoint_states().size();
|
||||
int rf = 3;
|
||||
if (endpoint_count < rf) {
|
||||
rf = 1;
|
||||
elogger.warn("Creating keyspace '{}' for Alternator with unsafe RF={} because cluster only has {} nodes.",
|
||||
keyspace_name, rf, endpoint_count);
|
||||
keyspace_name_str, rf, endpoint_count);
|
||||
}
|
||||
auto opts = get_network_topology_options(sp, gossiper, rf);
|
||||
auto ksm = keyspace_metadata::new_keyspace(keyspace_name_str, "org.apache.cassandra.locator.NetworkTopologyStrategy", std::move(opts), true);
|
||||
|
||||
return keyspace_metadata::new_keyspace(keyspace_name, "org.apache.cassandra.locator.NetworkTopologyStrategy", std::move(opts), true);
|
||||
co_return mm.prepare_new_keyspace_announcement(ksm, ts);
|
||||
}
|
||||
|
||||
future<> executor::start() {
|
||||
|
||||
@@ -22,7 +22,6 @@
|
||||
#include "alternator/error.hh"
|
||||
#include "stats.hh"
|
||||
#include "utils/rjson.hh"
|
||||
#include "utils/updateable_value.hh"
|
||||
|
||||
namespace db {
|
||||
class system_distributed_keyspace;
|
||||
@@ -171,16 +170,8 @@ public:
|
||||
static constexpr auto KEYSPACE_NAME_PREFIX = "alternator_";
|
||||
static constexpr std::string_view INTERNAL_TABLE_PREFIX = ".scylla.alternator.";
|
||||
|
||||
executor(gms::gossiper& gossiper,
|
||||
service::storage_proxy& proxy,
|
||||
service::migration_manager& mm,
|
||||
db::system_distributed_keyspace& sdks,
|
||||
cdc::metadata& cdc_metadata,
|
||||
smp_service_group ssg,
|
||||
utils::updateable_value<uint32_t> default_timeout_in_ms)
|
||||
: _gossiper(gossiper), _proxy(proxy), _mm(mm), _sdks(sdks), _cdc_metadata(cdc_metadata), _ssg(ssg) {
|
||||
s_default_timeout_in_ms = std::move(default_timeout_in_ms);
|
||||
}
|
||||
executor(gms::gossiper& gossiper, service::storage_proxy& proxy, service::migration_manager& mm, db::system_distributed_keyspace& sdks, cdc::metadata& cdc_metadata, smp_service_group ssg)
|
||||
: _gossiper(gossiper), _proxy(proxy), _mm(mm), _sdks(sdks), _cdc_metadata(cdc_metadata), _ssg(ssg) {}
|
||||
|
||||
future<request_return_type> create_table(client_state& client_state, tracing::trace_state_ptr trace_state, service_permit permit, rjson::value request);
|
||||
future<request_return_type> describe_table(client_state& client_state, tracing::trace_state_ptr trace_state, service_permit permit, rjson::value request);
|
||||
@@ -208,16 +199,13 @@ public:
|
||||
future<request_return_type> describe_continuous_backups(client_state& client_state, service_permit permit, rjson::value request);
|
||||
|
||||
future<> start();
|
||||
future<> stop() {
|
||||
// disconnect from the value source, but keep the value unchanged.
|
||||
s_default_timeout_in_ms = utils::updateable_value<uint32_t>{s_default_timeout_in_ms()};
|
||||
return make_ready_future<>();
|
||||
}
|
||||
future<> stop() { return make_ready_future<>(); }
|
||||
|
||||
static sstring table_name(const schema&);
|
||||
static db::timeout_clock::time_point default_timeout();
|
||||
static void set_default_timeout(db::timeout_clock::duration timeout);
|
||||
private:
|
||||
static thread_local utils::updateable_value<uint32_t> s_default_timeout_in_ms;
|
||||
static db::timeout_clock::duration s_default_timeout;
|
||||
public:
|
||||
static schema_ptr find_table(service::storage_proxy&, const rjson::value& request);
|
||||
|
||||
@@ -225,10 +213,9 @@ private:
|
||||
friend class rmw_operation;
|
||||
|
||||
static void describe_key_schema(rjson::value& parent, const schema&, std::unordered_map<std::string,std::string> * = nullptr);
|
||||
static void describe_key_schema(rjson::value& parent, const schema& schema, std::unordered_map<std::string,std::string>&);
|
||||
|
||||
public:
|
||||
static void describe_key_schema(rjson::value& parent, const schema& schema, std::unordered_map<std::string,std::string>&);
|
||||
|
||||
static std::optional<rjson::value> describe_single_item(schema_ptr,
|
||||
const query::partition_slice&,
|
||||
const cql3::selection::selection&,
|
||||
@@ -242,25 +229,14 @@ public:
|
||||
shared_ptr<const std::optional<attrs_to_get>> attrs_to_get);
|
||||
|
||||
static void describe_single_item(const cql3::selection::selection&,
|
||||
const std::vector<managed_bytes_opt>&,
|
||||
const std::vector<bytes_opt>&,
|
||||
const std::optional<attrs_to_get>&,
|
||||
rjson::value&,
|
||||
bool = false);
|
||||
|
||||
static void add_stream_options(const rjson::value& stream_spec, schema_builder&, service::storage_proxy& sp);
|
||||
static void supplement_table_info(rjson::value& descr, const schema& schema, service::storage_proxy& sp);
|
||||
static void supplement_table_stream_info(rjson::value& descr, const schema& schema, const service::storage_proxy& sp);
|
||||
static void supplement_table_stream_info(rjson::value& descr, const schema& schema, service::storage_proxy& sp);
|
||||
};
|
||||
|
||||
// is_big() checks approximately if the given JSON value is "bigger" than
|
||||
// the given big_size number of bytes. The goal is to *quickly* detect
|
||||
// oversized JSON that, for example, is too large to be serialized to a
|
||||
// contiguous string - we don't need an accurate size for that. Moreover,
|
||||
// as soon as we detect that the JSON is indeed "big", we can return true
|
||||
// and don't need to continue calculating its exact size.
|
||||
// For simplicity, we use a recursive implementation. This is fine because
|
||||
// Alternator limits the depth of JSONs it reads from inputs, and doesn't
|
||||
// add more than a couple of levels in its own output construction.
|
||||
bool is_big(const rjson::value& val, int big_size = 100'000);
|
||||
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
namespace alternator {
|
||||
|
||||
template <typename Func, typename Result = std::result_of_t<Func(expressionsParser&)>>
|
||||
static Result do_with_parser(std::string_view input, Func&& f) {
|
||||
Result do_with_parser(std::string_view input, Func&& f) {
|
||||
expressionsLexer::InputStreamType input_stream{
|
||||
reinterpret_cast<const ANTLR_UINT8*>(input.data()),
|
||||
ANTLR_ENC_UTF8,
|
||||
@@ -43,41 +43,31 @@ static Result do_with_parser(std::string_view input, Func&& f) {
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename Func, typename Result = std::result_of_t<Func(expressionsParser&)>>
|
||||
static Result parse(const char* input_name, std::string_view input, Func&& f) {
|
||||
if (input.length() > 4096) {
|
||||
throw expressions_syntax_error(format("{} expression size {} exceeds allowed maximum 4096.",
|
||||
input_name, input.length()));
|
||||
}
|
||||
try {
|
||||
return do_with_parser(input, f);
|
||||
} catch (expressions_syntax_error& e) {
|
||||
// If already an expressions_syntax_error, don't print the type's
|
||||
// name (it's just ugly), just the message.
|
||||
// TODO: displayRecognitionError could set a position inside the
|
||||
// expressions_syntax_error in throws, and we could use it here to
|
||||
// mark the broken position in 'input'.
|
||||
throw expressions_syntax_error(format("Failed parsing {} '{}': {}",
|
||||
input_name, input, e.what()));
|
||||
} catch (...) {
|
||||
throw expressions_syntax_error(format("Failed parsing {} '{}': {}",
|
||||
input_name, input, std::current_exception()));
|
||||
}
|
||||
}
|
||||
|
||||
parsed::update_expression
|
||||
parse_update_expression(std::string_view query) {
|
||||
return parse("UpdateExpression", query, std::mem_fn(&expressionsParser::update_expression));
|
||||
try {
|
||||
return do_with_parser(query, std::mem_fn(&expressionsParser::update_expression));
|
||||
} catch (...) {
|
||||
throw expressions_syntax_error(format("Failed parsing UpdateExpression '{}': {}", query, std::current_exception()));
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<parsed::path>
|
||||
parse_projection_expression(std::string_view query) {
|
||||
return parse ("ProjectionExpression", query, std::mem_fn(&expressionsParser::projection_expression));
|
||||
try {
|
||||
return do_with_parser(query, std::mem_fn(&expressionsParser::projection_expression));
|
||||
} catch (...) {
|
||||
throw expressions_syntax_error(format("Failed parsing ProjectionExpression '{}': {}", query, std::current_exception()));
|
||||
}
|
||||
}
|
||||
|
||||
parsed::condition_expression
|
||||
parse_condition_expression(std::string_view query, const char* caller) {
|
||||
return parse(caller, query, std::mem_fn(&expressionsParser::condition_expression));
|
||||
parse_condition_expression(std::string_view query) {
|
||||
try {
|
||||
return do_with_parser(query, std::mem_fn(&expressionsParser::condition_expression));
|
||||
} catch (...) {
|
||||
throw expressions_syntax_error(format("Failed parsing ConditionExpression '{}': {}", query, std::current_exception()));
|
||||
}
|
||||
}
|
||||
|
||||
namespace parsed {
|
||||
@@ -428,14 +418,9 @@ void for_condition_expression_on(const parsed::condition_expression& ce, const n
|
||||
// calculate_size() is ConditionExpression's size() function, i.e., it takes
|
||||
// a JSON-encoded value and returns its "size" as defined differently for the
|
||||
// different types - also as a JSON-encoded number.
|
||||
// If the value's type (e.g. number) has no size defined, there are two cases:
|
||||
// 1. If from_data (the value came directly from an attribute of the data),
|
||||
// It returns a JSON-encoded "null" value. Comparisons against this
|
||||
// non-numeric value will later fail, so eventually the application will
|
||||
// get a ConditionalCheckFailedException.
|
||||
// 2. Otherwise (the value came from a constant in the query or some other
|
||||
// calculation), throw a ValidationException.
|
||||
static rjson::value calculate_size(const rjson::value& v, bool from_data) {
|
||||
// It return a JSON-encoded "null" value if this value's type has no size
|
||||
// defined. Comparisons against this non-numeric value will later fail.
|
||||
static rjson::value calculate_size(const rjson::value& v) {
|
||||
// NOTE: If v is improperly formatted for our JSON value encoding, it
|
||||
// must come from the request itself, not from the database, so it makes
|
||||
// sense to throw a ValidationException if we see such a problem.
|
||||
@@ -464,12 +449,10 @@ static rjson::value calculate_size(const rjson::value& v, bool from_data) {
|
||||
throw api_error::validation(format("invalid byte string: {}", v));
|
||||
}
|
||||
ret = base64_decoded_len(rjson::to_string_view(it->value));
|
||||
} else if (from_data) {
|
||||
} else {
|
||||
rjson::value json_ret = rjson::empty_object();
|
||||
rjson::add(json_ret, "null", rjson::value(true));
|
||||
return json_ret;
|
||||
} else {
|
||||
throw api_error::validation(format("Unsupported operand type {} for function size()", it->name));
|
||||
}
|
||||
rjson::value json_ret = rjson::empty_object();
|
||||
rjson::add(json_ret, "N", rjson::from_string(std::to_string(ret)));
|
||||
@@ -551,7 +534,7 @@ std::unordered_map<std::string_view, function_handler_type*> function_handlers {
|
||||
format("{}: size() accepts 1 parameter, got {}", caller, f._parameters.size()));
|
||||
}
|
||||
rjson::value v = calculate_value(f._parameters[0], caller, previous_item);
|
||||
return calculate_size(v, f._parameters[0].is_path());
|
||||
return calculate_size(v);
|
||||
}
|
||||
},
|
||||
{"attribute_exists", [] (calculate_value_caller caller, const rjson::value* previous_item, const parsed::value::function_call& f) {
|
||||
@@ -651,8 +634,7 @@ std::unordered_map<std::string_view, function_handler_type*> function_handlers {
|
||||
}
|
||||
rjson::value v1 = calculate_value(f._parameters[0], caller, previous_item);
|
||||
rjson::value v2 = calculate_value(f._parameters[1], caller, previous_item);
|
||||
return to_bool_json(check_CONTAINS(v1.IsNull() ? nullptr : &v1, v2,
|
||||
f._parameters[0].is_constant(), f._parameters[1].is_constant()));
|
||||
return to_bool_json(check_CONTAINS(v1.IsNull() ? nullptr : &v1, v2));
|
||||
}
|
||||
},
|
||||
};
|
||||
@@ -679,7 +661,7 @@ static rjson::value extract_path(const rjson::value* item,
|
||||
// objects. But today Alternator does not validate the structure
|
||||
// of nested documents before storing them, so this can happen on
|
||||
// read.
|
||||
throw api_error::validation(format("{}: malformed item read: {}", caller, *item));
|
||||
throw api_error::validation(format("{}: malformed item read: {}", *item));
|
||||
}
|
||||
const char* type = v->MemberBegin()->name.GetString();
|
||||
v = &(v->MemberBegin()->value);
|
||||
|
||||
@@ -74,22 +74,7 @@ options {
|
||||
*/
|
||||
@parser::context {
|
||||
void displayRecognitionError(ANTLR_UINT8** token_names, ExceptionBaseType* ex) {
|
||||
const char* err;
|
||||
switch (ex->getType()) {
|
||||
case antlr3::ExceptionType::FAILED_PREDICATE_EXCEPTION:
|
||||
err = "expression nested too deeply";
|
||||
break;
|
||||
default:
|
||||
err = "syntax error";
|
||||
break;
|
||||
}
|
||||
// Alternator expressions are always single line so ex->get_line()
|
||||
// is always 1, no sense to print it.
|
||||
// TODO: return the position as part of the exception, so the
|
||||
// caller in expressions.cc that knows the expression string can
|
||||
// mark the error position in the final error message.
|
||||
throw expressions_syntax_error(format("{} at char {}", err,
|
||||
ex->get_charPositionInLine()));
|
||||
throw expressions_syntax_error("syntax error");
|
||||
}
|
||||
}
|
||||
@lexer::context {
|
||||
@@ -98,23 +83,6 @@ options {
|
||||
}
|
||||
}
|
||||
|
||||
/* Unfortunately, ANTLR uses recursion - not the heap - to parse recursive
|
||||
* expressions. To make things even worse, ANTLR has no way to limit the
|
||||
* depth of this recursion (unlike Yacc which has YYMAXDEPTH). So deeply-
|
||||
* nested expression like "(((((((((((((..." can easily crash Scylla on a
|
||||
* stack overflow (see issue #14477).
|
||||
*
|
||||
* We are lucky that in the grammar for DynamoDB expressions (below),
|
||||
* only a few specific rules can recurse, so it was fairly easy to add a
|
||||
* "depth" counter to a few specific rules, and then use a predicate
|
||||
* "{depth<MAX_DEPTH}?" to avoid parsing if the depth exceeds this limit,
|
||||
* and throw a FAILED_PREDICATE_EXCEPTION in that case, which we will
|
||||
* report to the user as a "expression nested too deeply" error.
|
||||
*/
|
||||
@parser::members {
|
||||
static constexpr int MAX_DEPTH = 400;
|
||||
}
|
||||
|
||||
/*
|
||||
* Lexical analysis phase, i.e., splitting the input up to tokens.
|
||||
* Lexical analyzer rules have names starting in capital letters.
|
||||
@@ -187,20 +155,19 @@ path returns [parsed::path p]:
|
||||
| '[' INTEGER ']' { $p.add_index(std::stoi($INTEGER.text)); }
|
||||
)*;
|
||||
|
||||
/* See comment above why the "depth" counter was needed here */
|
||||
value[int depth] returns [parsed::value v]:
|
||||
value returns [parsed::value v]:
|
||||
VALREF { $v.set_valref($VALREF.text); }
|
||||
| path { $v.set_path($path.p); }
|
||||
| {depth<MAX_DEPTH}? NAME { $v.set_func_name($NAME.text); }
|
||||
'(' x=value[depth+1] { $v.add_func_parameter($x.v); }
|
||||
(',' x=value[depth+1] { $v.add_func_parameter($x.v); })*
|
||||
| NAME { $v.set_func_name($NAME.text); }
|
||||
'(' x=value { $v.add_func_parameter($x.v); }
|
||||
(',' x=value { $v.add_func_parameter($x.v); })*
|
||||
')'
|
||||
;
|
||||
|
||||
update_expression_set_rhs returns [parsed::set_rhs rhs]:
|
||||
v=value[0] { $rhs.set_value(std::move($v.v)); }
|
||||
( '+' v=value[0] { $rhs.set_plus(std::move($v.v)); }
|
||||
| '-' v=value[0] { $rhs.set_minus(std::move($v.v)); }
|
||||
v=value { $rhs.set_value(std::move($v.v)); }
|
||||
( '+' v=value { $rhs.set_plus(std::move($v.v)); }
|
||||
| '-' v=value { $rhs.set_minus(std::move($v.v)); }
|
||||
)?
|
||||
;
|
||||
|
||||
@@ -238,7 +205,7 @@ projection_expression returns [std::vector<parsed::path> v]:
|
||||
|
||||
|
||||
primitive_condition returns [parsed::primitive_condition c]:
|
||||
v=value[0] { $c.add_value(std::move($v.v));
|
||||
v=value { $c.add_value(std::move($v.v));
|
||||
$c.set_operator(parsed::primitive_condition::type::VALUE); }
|
||||
( ( '=' { $c.set_operator(parsed::primitive_condition::type::EQ); }
|
||||
| '<' '>' { $c.set_operator(parsed::primitive_condition::type::NE); }
|
||||
@@ -247,14 +214,14 @@ primitive_condition returns [parsed::primitive_condition c]:
|
||||
| '>' { $c.set_operator(parsed::primitive_condition::type::GT); }
|
||||
| '>' '=' { $c.set_operator(parsed::primitive_condition::type::GE); }
|
||||
)
|
||||
v=value[0] { $c.add_value(std::move($v.v)); }
|
||||
v=value { $c.add_value(std::move($v.v)); }
|
||||
| BETWEEN { $c.set_operator(parsed::primitive_condition::type::BETWEEN); }
|
||||
v=value[0] { $c.add_value(std::move($v.v)); }
|
||||
v=value { $c.add_value(std::move($v.v)); }
|
||||
AND
|
||||
v=value[0] { $c.add_value(std::move($v.v)); }
|
||||
v=value { $c.add_value(std::move($v.v)); }
|
||||
| IN '(' { $c.set_operator(parsed::primitive_condition::type::IN); }
|
||||
v=value[0] { $c.add_value(std::move($v.v)); }
|
||||
(',' v=value[0] { $c.add_value(std::move($v.v)); })*
|
||||
v=value { $c.add_value(std::move($v.v)); }
|
||||
(',' v=value { $c.add_value(std::move($v.v)); })*
|
||||
')'
|
||||
)?
|
||||
;
|
||||
@@ -264,20 +231,19 @@ primitive_condition returns [parsed::primitive_condition c]:
|
||||
// common rule prefixes, and (lack of) support for operator precedence.
|
||||
// These rules could have been written more clearly using a more powerful
|
||||
// parser generator - such as Yacc.
|
||||
// See comment above why the "depth" counter was needed here.
|
||||
boolean_expression[int depth] returns [parsed::condition_expression e]:
|
||||
b=boolean_expression_1[depth] { $e.append(std::move($b.e), '|'); }
|
||||
(OR b=boolean_expression_1[depth] { $e.append(std::move($b.e), '|'); } )*
|
||||
boolean_expression returns [parsed::condition_expression e]:
|
||||
b=boolean_expression_1 { $e.append(std::move($b.e), '|'); }
|
||||
(OR b=boolean_expression_1 { $e.append(std::move($b.e), '|'); } )*
|
||||
;
|
||||
boolean_expression_1[int depth] returns [parsed::condition_expression e]:
|
||||
b=boolean_expression_2[depth] { $e.append(std::move($b.e), '&'); }
|
||||
(AND b=boolean_expression_2[depth] { $e.append(std::move($b.e), '&'); } )*
|
||||
boolean_expression_1 returns [parsed::condition_expression e]:
|
||||
b=boolean_expression_2 { $e.append(std::move($b.e), '&'); }
|
||||
(AND b=boolean_expression_2 { $e.append(std::move($b.e), '&'); } )*
|
||||
;
|
||||
boolean_expression_2[int depth] returns [parsed::condition_expression e]:
|
||||
boolean_expression_2 returns [parsed::condition_expression e]:
|
||||
p=primitive_condition { $e.set_primitive(std::move($p.c)); }
|
||||
| {depth<MAX_DEPTH}? NOT b=boolean_expression_2[depth+1] { $e = std::move($b.e); $e.apply_not(); }
|
||||
| {depth<MAX_DEPTH}? '(' b=boolean_expression[depth+1] ')' { $e = std::move($b.e); }
|
||||
| NOT b=boolean_expression_2 { $e = std::move($b.e); $e.apply_not(); }
|
||||
| '(' b=boolean_expression ')' { $e = std::move($b.e); }
|
||||
;
|
||||
|
||||
condition_expression returns [parsed::condition_expression e]:
|
||||
boolean_expression[0] { e=std::move($boolean_expression.e); } EOF;
|
||||
boolean_expression { e=std::move($boolean_expression.e); } EOF;
|
||||
|
||||
@@ -28,7 +28,7 @@ public:
|
||||
|
||||
parsed::update_expression parse_update_expression(std::string_view query);
|
||||
std::vector<parsed::path> parse_projection_expression(std::string_view query);
|
||||
parsed::condition_expression parse_condition_expression(std::string_view query, const char* caller);
|
||||
parsed::condition_expression parse_condition_expression(std::string_view query);
|
||||
|
||||
void resolve_update_expression(parsed::update_expression& ue,
|
||||
const rjson::value* expression_attribute_names,
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
/*
|
||||
* Parsed representation of expressions and their components.
|
||||
*
|
||||
* Types in alternator::parsed namespace are used for holding the parse
|
||||
* Types in alternator::parse namespace are used for holding the parse
|
||||
* tree - objects generated by the Antlr rules after parsing an expression.
|
||||
* Because of the way Antlr works, all these objects are default-constructed
|
||||
* first, and then assigned when the rule is completed, so all these types
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
#include "rapidjson/writer.h"
|
||||
#include "concrete_types.hh"
|
||||
#include "cql3/type_json.hh"
|
||||
#include "mutation/position_in_partition.hh"
|
||||
#include "position_in_partition.hh"
|
||||
|
||||
static logging::logger slogger("alternator-serialization");
|
||||
|
||||
@@ -50,115 +50,6 @@ type_representation represent_type(alternator_type atype) {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
// Get the magnitude and precision of a big_decimal - as these concepts are
|
||||
// defined by DynamoDB - to allow us to enforce limits on those as explained
|
||||
// in ssue #6794. The "magnitude" of 9e123 is 123 and of -9e-123 is -123,
|
||||
// the "precision" of 12.34e56 is the number of significant digits - 4.
|
||||
//
|
||||
// Unfortunately it turned out to be quite difficult to take a big_decimal and
|
||||
// calculate its magnitude and precision from its scale() and unscaled_value().
|
||||
// So in the following ugly implementation we calculate them from the string
|
||||
// representation instead. We assume the number was already parsed
|
||||
// sucessfully to a big_decimal to it follows its syntax rules.
|
||||
//
|
||||
// FIXME: rewrite this function to take a big_decimal, not a string.
|
||||
// Maybe a snippet like this can help:
|
||||
// boost::multiprecision::cpp_int digits = boost::multiprecision::log10(num.unscaled_value().convert_to<boost::multiprecision::mpf_float_50>()).convert_to<boost::multiprecision::cpp_int>() + 1;
|
||||
|
||||
|
||||
internal::magnitude_and_precision internal::get_magnitude_and_precision(std::string_view s) {
|
||||
size_t e_or_end = s.find_first_of("eE");
|
||||
std::string_view base = s.substr(0, e_or_end);
|
||||
if (s[0]=='-' || s[0]=='+') {
|
||||
base = base.substr(1);
|
||||
}
|
||||
int magnitude = 0;
|
||||
int precision = 0;
|
||||
size_t dot_or_end = base.find_first_of(".");
|
||||
size_t nonzero = base.find_first_not_of("0");
|
||||
if (dot_or_end != std::string_view::npos) {
|
||||
if (nonzero == dot_or_end) {
|
||||
// 0.000031 => magnitude = -5 (like 3.1e-5), precision = 2.
|
||||
std::string_view fraction = base.substr(dot_or_end + 1);
|
||||
size_t nonzero2 = fraction.find_first_not_of("0");
|
||||
if (nonzero2 != std::string_view::npos) {
|
||||
magnitude = -nonzero2 - 1;
|
||||
precision = fraction.size() - nonzero2;
|
||||
}
|
||||
} else {
|
||||
// 000123.45678 => magnitude = 2, precision = 8.
|
||||
magnitude = dot_or_end - nonzero - 1;
|
||||
precision = base.size() - nonzero - 1;
|
||||
}
|
||||
// trailing zeros don't count to precision, e.g., precision
|
||||
// of 1000.0, 1.0 or 1.0000 are just 1.
|
||||
size_t last_significant = base.find_last_not_of(".0");
|
||||
if (last_significant == std::string_view::npos) {
|
||||
precision = 0;
|
||||
} else if (last_significant < dot_or_end) {
|
||||
// e.g., 1000.00 reduce 5 = 7 - (0+1) - 1 from precision
|
||||
precision -= base.size() - last_significant - 2;
|
||||
} else {
|
||||
// e.g., 1235.60 reduce 5 = 7 - (5+1) from precision
|
||||
precision -= base.size() - last_significant - 1;
|
||||
}
|
||||
} else if (nonzero == std::string_view::npos) {
|
||||
// all-zero integer 000000
|
||||
magnitude = 0;
|
||||
precision = 0;
|
||||
} else {
|
||||
magnitude = base.size() - 1 - nonzero;
|
||||
precision = base.size() - nonzero;
|
||||
// trailing zeros don't count to precision, e.g., precision
|
||||
// of 1000 is just 1.
|
||||
size_t last_significant = base.find_last_not_of("0");
|
||||
if (last_significant == std::string_view::npos) {
|
||||
precision = 0;
|
||||
} else {
|
||||
// e.g., 1000 reduce 3 = 4 - (0+1)
|
||||
precision -= base.size() - last_significant - 1;
|
||||
}
|
||||
}
|
||||
if (precision && e_or_end != std::string_view::npos) {
|
||||
std::string_view exponent = s.substr(e_or_end + 1);
|
||||
if (exponent.size() > 4) {
|
||||
// don't even bother atoi(), exponent is too large
|
||||
magnitude = exponent[0]=='-' ? -9999 : 9999;
|
||||
} else {
|
||||
try {
|
||||
magnitude += boost::lexical_cast<int32_t>(exponent);
|
||||
} catch (...) {
|
||||
magnitude = 9999;
|
||||
}
|
||||
}
|
||||
}
|
||||
return magnitude_and_precision {magnitude, precision};
|
||||
}
|
||||
|
||||
// Parse a number read from user input, validating that it has a valid
|
||||
// numeric format and also in the allowed magnitude and precision ranges
|
||||
// (see issue #6794). Throws an api_error::validation if the validation
|
||||
// failed.
|
||||
static big_decimal parse_and_validate_number(std::string_view s) {
|
||||
try {
|
||||
big_decimal ret(s);
|
||||
auto [magnitude, precision] = internal::get_magnitude_and_precision(s);
|
||||
if (magnitude > 125) {
|
||||
throw api_error::validation(format("Number overflow: {}. Attempting to store a number with magnitude larger than supported range.", s));
|
||||
}
|
||||
if (magnitude < -130) {
|
||||
throw api_error::validation(format("Number underflow: {}. Attempting to store a number with magnitude lower than supported range.", s));
|
||||
}
|
||||
if (precision > 38) {
|
||||
throw api_error::validation(format("Number too precise: {}. Attempting to store a number with more significant digits than supported.", s));
|
||||
}
|
||||
return ret;
|
||||
} catch (const marshal_exception& e) {
|
||||
throw api_error::validation(format("The parameter cannot be converted to a numeric value: {}", s));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
struct from_json_visitor {
|
||||
const rjson::value& v;
|
||||
bytes_ostream& bo;
|
||||
@@ -168,19 +59,21 @@ struct from_json_visitor {
|
||||
bo.write(t.from_string(rjson::to_string_view(v)));
|
||||
}
|
||||
void operator()(const bytes_type_impl& t) const {
|
||||
// FIXME: it's difficult at this point to get information if value was provided
|
||||
// in request or comes from the storage, for now we assume it's user's fault.
|
||||
bo.write(*unwrap_bytes(v, true));
|
||||
bo.write(rjson::base64_decode(v));
|
||||
}
|
||||
void operator()(const boolean_type_impl& t) const {
|
||||
bo.write(boolean_type->decompose(v.GetBool()));
|
||||
}
|
||||
void operator()(const decimal_type_impl& t) const {
|
||||
bo.write(decimal_type->decompose(parse_and_validate_number(rjson::to_string_view(v))));
|
||||
try {
|
||||
bo.write(t.from_string(rjson::to_string_view(v)));
|
||||
} catch (const marshal_exception& e) {
|
||||
throw api_error::validation(format("The parameter cannot be converted to a numeric value: {}", v));
|
||||
}
|
||||
}
|
||||
// default
|
||||
void operator()(const abstract_type& t) const {
|
||||
bo.write(from_json_object(t, v));
|
||||
bo.write(from_json_object(t, v, cql_serialization_format::internal()));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -305,11 +198,7 @@ bytes get_key_from_typed_value(const rjson::value& key_typed_value, const column
|
||||
format("The AttributeValue for a key attribute cannot contain an empty string value. Key: {}", column.name_as_text()));
|
||||
}
|
||||
if (column.type == bytes_type) {
|
||||
// FIXME: it's difficult at this point to get information if value was provided
|
||||
// in request or comes from the storage, for now we assume it's user's fault.
|
||||
return *unwrap_bytes(value, true);
|
||||
} else if (column.type == decimal_type) {
|
||||
return decimal_type->decompose(parse_and_validate_number(rjson::to_string_view(value)));
|
||||
return rjson::base64_decode(value);
|
||||
} else {
|
||||
return column.type->from_string(value_view);
|
||||
}
|
||||
@@ -321,7 +210,7 @@ rjson::value json_key_column_value(bytes_view cell, const column_definition& col
|
||||
std::string b64 = base64_encode(cell);
|
||||
return rjson::from_string(b64);
|
||||
} if (column.type == utf8_type) {
|
||||
return rjson::from_string(reinterpret_cast<const char*>(cell.data()), cell.size());
|
||||
return rjson::from_string(std::string(reinterpret_cast<const char*>(cell.data()), cell.size()));
|
||||
} else if (column.type == decimal_type) {
|
||||
// FIXME: use specialized Alternator number type, not the more
|
||||
// general "decimal_type". A dedicated type can be more efficient
|
||||
@@ -372,6 +261,7 @@ position_in_partition pos_from_json(const rjson::value& item, schema_ptr schema)
|
||||
if (bool(region_item) != bool(weight_item)) {
|
||||
throw api_error::validation("Malformed value object: region and weight has to be either both missing or both present");
|
||||
}
|
||||
partition_region region;
|
||||
bound_weight weight;
|
||||
if (region_item) {
|
||||
auto region_view = rjson::to_string_view(get_typed_value(*region_item, "S", scylla_paging_region, "key region"));
|
||||
@@ -389,7 +279,7 @@ position_in_partition pos_from_json(const rjson::value& item, schema_ptr schema)
|
||||
return position_in_partition(region, weight, region == partition_region::clustered ? std::optional(std::move(ck)) : std::nullopt);
|
||||
}
|
||||
if (ck.is_empty()) {
|
||||
return position_in_partition::for_partition_start();
|
||||
return position_in_partition(position_in_partition::partition_start_tag_t());
|
||||
}
|
||||
return position_in_partition::for_key(std::move(ck));
|
||||
}
|
||||
@@ -402,13 +292,16 @@ big_decimal unwrap_number(const rjson::value& v, std::string_view diagnostic) {
|
||||
if (it->name != "N") {
|
||||
throw api_error::validation(format("{}: expected number, found type '{}'", diagnostic, it->name));
|
||||
}
|
||||
if (!it->value.IsString()) {
|
||||
// We shouldn't reach here. Callers normally validate their input
|
||||
// earlier with validate_value().
|
||||
throw api_error::validation(format("{}: improperly formatted number constant", diagnostic));
|
||||
try {
|
||||
if (!it->value.IsString()) {
|
||||
// We shouldn't reach here. Callers normally validate their input
|
||||
// earlier with validate_value().
|
||||
throw api_error::validation(format("{}: improperly formatted number constant", diagnostic));
|
||||
}
|
||||
return big_decimal(rjson::to_string_view(it->value));
|
||||
} catch (const marshal_exception& e) {
|
||||
throw api_error::validation(format("The parameter cannot be converted to a numeric value: {}", it->value));
|
||||
}
|
||||
big_decimal ret = parse_and_validate_number(rjson::to_string_view(it->value));
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::optional<big_decimal> try_unwrap_number(const rjson::value& v) {
|
||||
@@ -420,19 +313,8 @@ std::optional<big_decimal> try_unwrap_number(const rjson::value& v) {
|
||||
return std::nullopt;
|
||||
}
|
||||
try {
|
||||
return parse_and_validate_number(rjson::to_string_view(it->value));
|
||||
} catch (api_error&) {
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<bytes> unwrap_bytes(const rjson::value& value, bool from_query) {
|
||||
try {
|
||||
return rjson::base64_decode(value);
|
||||
} catch (...) {
|
||||
if (from_query) {
|
||||
throw api_error::serialization(format("Invalid base64 data"));
|
||||
}
|
||||
return big_decimal(rjson::to_string_view(it->value));
|
||||
} catch (const marshal_exception& e) {
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
@@ -466,7 +348,7 @@ rjson::value number_add(const rjson::value& v1, const rjson::value& v2) {
|
||||
auto n1 = unwrap_number(v1, "UpdateExpression");
|
||||
auto n2 = unwrap_number(v2, "UpdateExpression");
|
||||
rjson::value ret = rjson::empty_object();
|
||||
sstring str_ret = (n1 + n2).to_string();
|
||||
std::string str_ret = std::string((n1 + n2).to_string());
|
||||
rjson::add(ret, "N", rjson::from_string(str_ret));
|
||||
return ret;
|
||||
}
|
||||
@@ -475,7 +357,7 @@ rjson::value number_subtract(const rjson::value& v1, const rjson::value& v2) {
|
||||
auto n1 = unwrap_number(v1, "UpdateExpression");
|
||||
auto n2 = unwrap_number(v2, "UpdateExpression");
|
||||
rjson::value ret = rjson::empty_object();
|
||||
sstring str_ret = (n1 - n2).to_string();
|
||||
std::string str_ret = std::string((n1 - n2).to_string());
|
||||
rjson::add(ret, "N", rjson::from_string(str_ret));
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -11,8 +11,8 @@
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <optional>
|
||||
#include "types/types.hh"
|
||||
#include "schema/schema_fwd.hh"
|
||||
#include "types.hh"
|
||||
#include "schema_fwd.hh"
|
||||
#include "keys.hh"
|
||||
#include "utils/rjson.hh"
|
||||
#include "utils/big_decimal.hh"
|
||||
@@ -62,11 +62,6 @@ big_decimal unwrap_number(const rjson::value& v, std::string_view diagnostic);
|
||||
// when the given v does not encode a number.
|
||||
std::optional<big_decimal> try_unwrap_number(const rjson::value& v);
|
||||
|
||||
// unwrap_bytes decodes byte value, on decoding failure it either raises api_error::serialization
|
||||
// iff from_query is true or returns unset optional iff from_query is false.
|
||||
// Therefore it's safe to dereference returned optional when called with from_query equal true.
|
||||
std::optional<bytes> unwrap_bytes(const rjson::value& value, bool from_query);
|
||||
|
||||
// Check if a given JSON object encodes a set (i.e., it is a {"SS": [...]}, or "NS", "BS"
|
||||
// and returns set's type and a pointer to that set. If the object does not encode a set,
|
||||
// returned value is {"", nullptr}
|
||||
@@ -94,12 +89,5 @@ std::optional<rjson::value> set_diff(const rjson::value& v1, const rjson::value&
|
||||
// Returns a null value if one of the arguments is not actually a list.
|
||||
rjson::value list_concatenate(const rjson::value& v1, const rjson::value& v2);
|
||||
|
||||
namespace internal {
|
||||
struct magnitude_and_precision {
|
||||
int magnitude;
|
||||
int precision;
|
||||
};
|
||||
magnitude_and_precision get_magnitude_and_precision(std::string_view);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
#include <seastar/util/short_streams.hh>
|
||||
#include "seastarx.hh"
|
||||
#include "error.hh"
|
||||
#include "service/qos/service_level_controller.hh"
|
||||
#include "utils/rjson.hh"
|
||||
#include "auth.hh"
|
||||
#include <cctype>
|
||||
@@ -24,13 +23,10 @@
|
||||
#include "gms/gossiper.hh"
|
||||
#include "utils/overloaded_functor.hh"
|
||||
#include "utils/fb_utilities.hh"
|
||||
#include "utils/aws_sigv4.hh"
|
||||
|
||||
static logging::logger slogger("alternator-server");
|
||||
|
||||
using namespace httpd;
|
||||
using request = http::request;
|
||||
using reply = http::reply;
|
||||
|
||||
namespace alternator {
|
||||
|
||||
@@ -146,7 +142,7 @@ public:
|
||||
std::unique_ptr<request> req, std::unique_ptr<reply> rep) override {
|
||||
handle_CORS(*req, *rep, false);
|
||||
return _f_handle(std::move(req), std::move(rep)).then(
|
||||
[](std::unique_ptr<reply> rep) {
|
||||
[this](std::unique_ptr<reply> rep) {
|
||||
rep->set_mime_type("application/x-amz-json-1.0");
|
||||
rep->done();
|
||||
return make_ready_future<std::unique_ptr<reply>>(std::move(rep));
|
||||
@@ -238,7 +234,7 @@ protected:
|
||||
future<std::string> server::verify_signature(const request& req, const chunked_content& content) {
|
||||
if (!_enforce_authorization) {
|
||||
slogger.debug("Skipping authorization");
|
||||
return make_ready_future<std::string>();
|
||||
return make_ready_future<std::string>("<unauthenticated request>");
|
||||
}
|
||||
auto host_it = req._headers.find("Host");
|
||||
if (host_it == req._headers.end()) {
|
||||
@@ -320,13 +316,8 @@ future<std::string> server::verify_signature(const request& req, const chunked_c
|
||||
region = std::move(region),
|
||||
service = std::move(service),
|
||||
user_signature = std::move(user_signature)] (key_cache::value_ptr key_ptr) {
|
||||
std::string signature;
|
||||
try {
|
||||
signature = utils::aws::get_signature(user, *key_ptr, std::string_view(host), "/", req._method,
|
||||
datestamp, signed_headers_str, signed_headers_map, &content, region, service, "");
|
||||
} catch (const std::exception& e) {
|
||||
throw api_error::invalid_signature(e.what());
|
||||
}
|
||||
std::string signature = get_signature(user, *key_ptr, std::string_view(host), req._method,
|
||||
datestamp, signed_headers_str, signed_headers_map, content, region, service, "");
|
||||
|
||||
if (signature != std::string_view(user_signature)) {
|
||||
_key_cache.remove(user);
|
||||
@@ -373,9 +364,7 @@ static tracing::trace_state_ptr maybe_trace_query(service::client_state& client_
|
||||
tracing::add_session_param(trace_state, "alternator_op", op);
|
||||
tracing::add_query(trace_state, truncated_content_view(query, buf));
|
||||
tracing::begin(trace_state, format("Alternator {}", op), client_state.get_client_address());
|
||||
if (!username.empty()) {
|
||||
tracing::set_username(trace_state, auth::authenticated_user(username));
|
||||
}
|
||||
tracing::set_username(trace_state, auth::authenticated_user(username));
|
||||
}
|
||||
return trace_state;
|
||||
}
|
||||
@@ -418,13 +407,9 @@ future<executor::request_return_type> server::handle_api_request(std::unique_ptr
|
||||
auto leave = defer([this] () noexcept { _pending_requests.leave(); });
|
||||
//FIXME: Client state can provide more context, e.g. client's endpoint address
|
||||
// We use unique_ptr because client_state cannot be moved or copied
|
||||
executor::client_state client_state = username.empty()
|
||||
? service::client_state{service::client_state::internal_tag()}
|
||||
: service::client_state{service::client_state::internal_tag(), _auth_service, _sl_controller, username};
|
||||
co_await client_state.maybe_update_per_service_level_params();
|
||||
|
||||
executor::client_state client_state{executor::client_state::internal_tag()};
|
||||
tracing::trace_state_ptr trace_state = maybe_trace_query(client_state, username, op, content);
|
||||
tracing::trace(trace_state, "{}", op);
|
||||
tracing::trace(trace_state, op);
|
||||
rjson::value json_request = co_await _json_parser.parse(std::move(content));
|
||||
co_return co_await callback_it->second(_executor, client_state, trace_state,
|
||||
make_service_permit(std::move(units)), std::move(json_request), std::move(req));
|
||||
@@ -455,14 +440,12 @@ void server::set_routes(routes& r) {
|
||||
//FIXME: A way to immediately invalidate the cache should be considered,
|
||||
// e.g. when the system table which stores the keys is changed.
|
||||
// For now, this propagation may take up to 1 minute.
|
||||
server::server(executor& exec, service::storage_proxy& proxy, gms::gossiper& gossiper, auth::service& auth_service, qos::service_level_controller& sl_controller)
|
||||
server::server(executor& exec, service::storage_proxy& proxy, gms::gossiper& gossiper)
|
||||
: _http_server("http-alternator")
|
||||
, _https_server("https-alternator")
|
||||
, _executor(exec)
|
||||
, _proxy(proxy)
|
||||
, _gossiper(gossiper)
|
||||
, _auth_service(auth_service)
|
||||
, _sl_controller(sl_controller)
|
||||
, _key_cache(1024, 1min, slogger)
|
||||
, _enforce_authorization(false)
|
||||
, _enabled_servers{}
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
#include <seastar/net/tls.hh>
|
||||
#include <optional>
|
||||
#include "alternator/auth.hh"
|
||||
#include "service/qos/service_level_controller.hh"
|
||||
#include "utils/small_vector.hh"
|
||||
#include "utils/updateable_value.hh"
|
||||
#include <seastar/core/units.hh>
|
||||
@@ -27,16 +26,14 @@ using chunked_content = rjson::chunked_content;
|
||||
class server {
|
||||
static constexpr size_t content_length_limit = 16*MB;
|
||||
using alternator_callback = std::function<future<executor::request_return_type>(executor&, executor::client_state&,
|
||||
tracing::trace_state_ptr, service_permit, rjson::value, std::unique_ptr<http::request>)>;
|
||||
tracing::trace_state_ptr, service_permit, rjson::value, std::unique_ptr<request>)>;
|
||||
using alternator_callbacks_map = std::unordered_map<std::string_view, alternator_callback>;
|
||||
|
||||
httpd::http_server _http_server;
|
||||
httpd::http_server _https_server;
|
||||
http_server _http_server;
|
||||
http_server _https_server;
|
||||
executor& _executor;
|
||||
service::storage_proxy& _proxy;
|
||||
gms::gossiper& _gossiper;
|
||||
auth::service& _auth_service;
|
||||
qos::service_level_controller& _sl_controller;
|
||||
|
||||
key_cache _key_cache;
|
||||
bool _enforce_authorization;
|
||||
@@ -68,7 +65,7 @@ class server {
|
||||
json_parser _json_parser;
|
||||
|
||||
public:
|
||||
server(executor& executor, service::storage_proxy& proxy, gms::gossiper& gossiper, auth::service& service, qos::service_level_controller& sl_controller);
|
||||
server(executor& executor, service::storage_proxy& proxy, gms::gossiper& gossiper);
|
||||
|
||||
future<> init(net::inet_address addr, std::optional<uint16_t> port, std::optional<uint16_t> https_port, std::optional<tls::credentials_builder> creds,
|
||||
bool enforce_authorization, semaphore* memory_limiter, utils::updateable_value<uint32_t> max_concurrent_requests);
|
||||
@@ -76,8 +73,8 @@ public:
|
||||
private:
|
||||
void set_routes(seastar::httpd::routes& r);
|
||||
// If verification succeeds, returns the authenticated user's username
|
||||
future<std::string> verify_signature(const seastar::http::request&, const chunked_content&);
|
||||
future<executor::request_return_type> handle_api_request(std::unique_ptr<http::request> req);
|
||||
future<std::string> verify_signature(const seastar::httpd::request&, const chunked_content&);
|
||||
future<executor::request_return_type> handle_api_request(std::unique_ptr<request> req);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@@ -27,14 +27,13 @@
|
||||
#include "cql3/result_set.hh"
|
||||
#include "cql3/type_json.hh"
|
||||
#include "cql3/column_identifier.hh"
|
||||
#include "schema/schema_builder.hh"
|
||||
#include "schema_builder.hh"
|
||||
#include "service/storage_proxy.hh"
|
||||
#include "gms/feature.hh"
|
||||
#include "gms/feature_service.hh"
|
||||
|
||||
#include "executor.hh"
|
||||
#include "rmw_operation.hh"
|
||||
#include "data_dictionary/data_dictionary.hh"
|
||||
|
||||
/**
|
||||
* Base template type to implement rapidjson::internal::TypeHelper<...>:s
|
||||
@@ -75,8 +74,8 @@ struct rapidjson::internal::TypeHelper<ValueType, utils::UUID>
|
||||
: public from_string_helper<ValueType, utils::UUID>
|
||||
{};
|
||||
|
||||
static db_clock::time_point as_timepoint(const table_id& tid) {
|
||||
return db_clock::time_point{utils::UUID_gen::unix_timestamp(tid.uuid())};
|
||||
static db_clock::time_point as_timepoint(const utils::UUID& uuid) {
|
||||
return db_clock::time_point{utils::UUID_gen::unix_timestamp(uuid)};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -107,9 +106,6 @@ public:
|
||||
stream_arn(const UUID& uuid)
|
||||
: UUID(uuid)
|
||||
{}
|
||||
stream_arn(const table_id& tid)
|
||||
: UUID(tid.uuid())
|
||||
{}
|
||||
stream_arn(std::string_view v)
|
||||
: UUID(v.substr(1))
|
||||
{
|
||||
@@ -141,44 +137,30 @@ namespace alternator {
|
||||
future<alternator::executor::request_return_type> alternator::executor::list_streams(client_state& client_state, service_permit permit, rjson::value request) {
|
||||
_stats.api_operations.list_streams++;
|
||||
|
||||
auto limit = rjson::get_opt<int>(request, "Limit").value_or(100);
|
||||
auto limit = rjson::get_opt<int>(request, "Limit").value_or(std::numeric_limits<int>::max());
|
||||
auto streams_start = rjson::get_opt<stream_arn>(request, "ExclusiveStartStreamArn");
|
||||
auto table = find_table(_proxy, request);
|
||||
auto db = _proxy.data_dictionary();
|
||||
auto cfs = db.get_tables();
|
||||
|
||||
if (limit < 1) {
|
||||
throw api_error::validation("Limit must be 1 or more");
|
||||
}
|
||||
|
||||
std::vector<data_dictionary::table> cfs;
|
||||
|
||||
if (table) {
|
||||
auto log_name = cdc::log_name(table->cf_name());
|
||||
try {
|
||||
cfs.emplace_back(db.find_table(table->ks_name(), log_name));
|
||||
} catch (data_dictionary::no_such_column_family&) {
|
||||
cfs.clear();
|
||||
}
|
||||
} else {
|
||||
cfs = db.get_tables();
|
||||
}
|
||||
|
||||
// # 12601 (maybe?) - sort the set of tables on ID. This should ensure we never
|
||||
// generate duplicates in a paged listing here. Can obviously miss things if they
|
||||
// are added between paged calls and end up with a "smaller" UUID/ARN, but that
|
||||
// is to be expected.
|
||||
if (std::cmp_less(limit, cfs.size()) || streams_start) {
|
||||
std::sort(cfs.begin(), cfs.end(), [](const data_dictionary::table& t1, const data_dictionary::table& t2) {
|
||||
return t1.schema()->id().uuid() < t2.schema()->id().uuid();
|
||||
});
|
||||
}
|
||||
std::sort(cfs.begin(), cfs.end(), [](const data_dictionary::table& t1, const data_dictionary::table& t2) {
|
||||
return t1.schema()->id() < t2.schema()->id();
|
||||
});
|
||||
|
||||
auto i = cfs.begin();
|
||||
auto e = cfs.end();
|
||||
|
||||
if (streams_start) {
|
||||
i = std::find_if(i, e, [&](const data_dictionary::table& t) {
|
||||
return t.schema()->id().uuid() == streams_start
|
||||
return t.schema()->id() == streams_start
|
||||
&& cdc::get_base_table(db.real_database(), *t.schema())
|
||||
&& is_alternator_keyspace(t.schema()->ks_name())
|
||||
;
|
||||
@@ -201,7 +183,14 @@ future<alternator::executor::request_return_type> alternator::executor::list_str
|
||||
if (!is_alternator_keyspace(ks_name)) {
|
||||
continue;
|
||||
}
|
||||
if (table && ks_name != table->ks_name()) {
|
||||
continue;
|
||||
}
|
||||
if (cdc::is_log_for_some_table(db.real_database(), ks_name, cf_name)) {
|
||||
if (table && table != cdc::get_base_table(db.real_database(), *s)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
rjson::value new_entry = rjson::empty_object();
|
||||
|
||||
last = i->schema()->id();
|
||||
@@ -429,8 +418,6 @@ static std::chrono::seconds confidence_interval(data_dictionary::database db) {
|
||||
return std::chrono::seconds(db.get_config().alternator_streams_time_window_s());
|
||||
}
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
// Dynamo docs says no data shall live longer than 24h.
|
||||
static constexpr auto dynamodb_streams_max_window = 24h;
|
||||
|
||||
@@ -448,7 +435,7 @@ future<executor::request_return_type> executor::describe_stream(client_state& cl
|
||||
auto db = _proxy.data_dictionary();
|
||||
|
||||
try {
|
||||
auto cf = db.find_column_family(table_id(stream_arn));
|
||||
auto cf = db.find_column_family(stream_arn);
|
||||
schema = cf.schema();
|
||||
bs = cdc::get_base_table(db.real_database(), *schema);
|
||||
} catch (...) {
|
||||
@@ -508,7 +495,7 @@ future<executor::request_return_type> executor::describe_stream(client_state& cl
|
||||
// filter out cdc generations older than the table or now() - cdc::ttl (typically dynamodb_streams_max_window - 24h)
|
||||
auto low_ts = std::max(as_timepoint(schema->id()), db_clock::now() - ttl);
|
||||
|
||||
return _sdks.cdc_get_versioned_streams(low_ts, { normal_token_owners }).then([db, shard_start, limit, ret = std::move(ret), stream_desc = std::move(stream_desc)] (std::map<db_clock::time_point, cdc::streams_version> topologies) mutable {
|
||||
return _sdks.cdc_get_versioned_streams(low_ts, { normal_token_owners }).then([this, db, shard_start, limit, ret = std::move(ret), stream_desc = std::move(stream_desc)] (std::map<db_clock::time_point, cdc::streams_version> topologies) mutable {
|
||||
|
||||
auto e = topologies.end();
|
||||
auto prev = e;
|
||||
@@ -735,7 +722,7 @@ future<executor::request_return_type> executor::get_shard_iterator(client_state&
|
||||
std::optional<shard_id> sid;
|
||||
|
||||
try {
|
||||
auto cf = db.find_column_family(table_id(stream_arn));
|
||||
auto cf = db.find_column_family(stream_arn);
|
||||
schema = cf.schema();
|
||||
sid = rjson::get<shard_id>(request, "ShardId");
|
||||
} catch (...) {
|
||||
@@ -820,14 +807,14 @@ future<executor::request_return_type> executor::get_records(client_state& client
|
||||
auto db = _proxy.data_dictionary();
|
||||
schema_ptr schema, base;
|
||||
try {
|
||||
auto log_table = db.find_column_family(table_id(iter.table));
|
||||
auto log_table = db.find_column_family(iter.table);
|
||||
schema = log_table.schema();
|
||||
base = cdc::get_base_table(db.real_database(), *schema);
|
||||
} catch (...) {
|
||||
}
|
||||
|
||||
if (!schema || !base || !is_alternator_keyspace(schema->ks_name())) {
|
||||
throw api_error::resource_not_found(fmt::to_string(iter.table));
|
||||
throw api_error::resource_not_found(boost::lexical_cast<std::string>(iter.table));
|
||||
}
|
||||
|
||||
tracing::add_table_name(trace_state, schema->ks_name(), schema->cf_name());
|
||||
@@ -894,11 +881,11 @@ future<executor::request_return_type> executor::get_records(client_state& client
|
||||
++mul;
|
||||
}
|
||||
auto command = ::make_lw_shared<query::read_command>(schema->id(), schema->version(), partition_slice, _proxy.get_max_result_size(partition_slice),
|
||||
query::tombstone_limit(_proxy.get_tombstone_limit()), query::row_limit(limit * mul));
|
||||
query::row_limit(limit * mul));
|
||||
|
||||
return _proxy.query(schema, std::move(command), std::move(partition_ranges), cl, service::storage_proxy::coordinator_query_options(default_timeout(), std::move(permit), client_state)).then(
|
||||
[this, schema, partition_slice = std::move(partition_slice), selection = std::move(selection), start_time = std::move(start_time), limit, key_names = std::move(key_names), attr_names = std::move(attr_names), type, iter, high_ts] (service::storage_proxy::coordinator_query_result qr) mutable {
|
||||
cql3::selection::result_set_builder builder(*selection, gc_clock::now());
|
||||
cql3::selection::result_set_builder builder(*selection, gc_clock::now(), cql_serialization_format::latest());
|
||||
query::result_view::consume(*qr.query_result, partition_slice, cql3::selection::result_set_builder::visitor(builder, *schema, *selection));
|
||||
|
||||
auto result_set = builder.build();
|
||||
@@ -1027,7 +1014,7 @@ future<executor::request_return_type> executor::get_records(client_state& client
|
||||
// ugh. figure out if we are and end-of-shard
|
||||
auto normal_token_owners = _proxy.get_token_metadata_ptr()->count_normal_token_owners();
|
||||
|
||||
return _sdks.cdc_current_generation_timestamp({ normal_token_owners }).then([this, iter, high_ts, start_time, ret = std::move(ret)](db_clock::time_point ts) mutable {
|
||||
return _sdks.cdc_current_generation_timestamp({ normal_token_owners }).then([this, iter, high_ts, start_time, ret = std::move(ret), nrecords](db_clock::time_point ts) mutable {
|
||||
auto& shard = iter.shard;
|
||||
|
||||
if (shard.time < ts && ts < high_ts) {
|
||||
@@ -1044,7 +1031,8 @@ future<executor::request_return_type> executor::get_records(client_state& client
|
||||
rjson::add(ret, "NextShardIterator", iter);
|
||||
}
|
||||
_stats.api_operations.get_records_latency.add(std::chrono::steady_clock::now() - start_time);
|
||||
if (is_big(ret)) {
|
||||
// TODO: determine a better threshold...
|
||||
if (nrecords > 10) {
|
||||
return make_ready_future<executor::request_return_type>(make_streamed(std::move(ret)));
|
||||
}
|
||||
return make_ready_future<executor::request_return_type>(make_jsonable(std::move(ret)));
|
||||
@@ -1096,7 +1084,7 @@ void executor::add_stream_options(const rjson::value& stream_specification, sche
|
||||
}
|
||||
}
|
||||
|
||||
void executor::supplement_table_stream_info(rjson::value& descr, const schema& schema, const service::storage_proxy& sp) {
|
||||
void executor::supplement_table_stream_info(rjson::value& descr, const schema& schema, service::storage_proxy& sp) {
|
||||
auto& opts = schema.cdc_options();
|
||||
if (opts.enabled()) {
|
||||
auto db = sp.data_dictionary();
|
||||
|
||||
@@ -8,7 +8,6 @@
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <exception>
|
||||
#include <optional>
|
||||
#include <seastar/core/sstring.hh>
|
||||
#include <seastar/core/coroutine.hh>
|
||||
@@ -18,7 +17,6 @@
|
||||
#include <seastar/coroutine/maybe_yield.hh>
|
||||
#include <boost/multiprecision/cpp_int.hpp>
|
||||
|
||||
#include "exceptions/exceptions.hh"
|
||||
#include "gms/gossiper.hh"
|
||||
#include "gms/inet_address.hh"
|
||||
#include "inet_address_vectors.hh"
|
||||
@@ -33,8 +31,8 @@
|
||||
#include "service/pager/query_pagers.hh"
|
||||
#include "gms/feature_service.hh"
|
||||
#include "sstables/types.hh"
|
||||
#include "mutation/mutation.hh"
|
||||
#include "types/types.hh"
|
||||
#include "mutation.hh"
|
||||
#include "types.hh"
|
||||
#include "types/map.hh"
|
||||
#include "utils/rjson.hh"
|
||||
#include "utils/big_decimal.hh"
|
||||
@@ -94,25 +92,24 @@ future<executor::request_return_type> executor::update_time_to_live(client_state
|
||||
}
|
||||
sstring attribute_name(v->GetString(), v->GetStringLength());
|
||||
|
||||
co_await db::modify_tags(_mm, schema->ks_name(), schema->cf_name(), [&](std::map<sstring, sstring>& tags_map) {
|
||||
if (enabled) {
|
||||
if (tags_map.contains(TTL_TAG_KEY)) {
|
||||
throw api_error::validation("TTL is already enabled");
|
||||
}
|
||||
tags_map[TTL_TAG_KEY] = attribute_name;
|
||||
} else {
|
||||
auto i = tags_map.find(TTL_TAG_KEY);
|
||||
if (i == tags_map.end()) {
|
||||
throw api_error::validation("TTL is already disabled");
|
||||
} else if (i->second != attribute_name) {
|
||||
throw api_error::validation(format(
|
||||
"Requested to disable TTL on attribute {}, but a different attribute {} is enabled.",
|
||||
attribute_name, i->second));
|
||||
}
|
||||
tags_map.erase(TTL_TAG_KEY);
|
||||
std::map<sstring, sstring> tags_map = get_tags_of_table_or_throw(schema);
|
||||
if (enabled) {
|
||||
if (tags_map.contains(TTL_TAG_KEY)) {
|
||||
co_return api_error::validation("TTL is already enabled");
|
||||
}
|
||||
});
|
||||
|
||||
tags_map[TTL_TAG_KEY] = attribute_name;
|
||||
} else {
|
||||
auto i = tags_map.find(TTL_TAG_KEY);
|
||||
if (i == tags_map.end()) {
|
||||
co_return api_error::validation("TTL is already disabled");
|
||||
} else if (i->second != attribute_name) {
|
||||
co_return api_error::validation(format(
|
||||
"Requested to disable TTL on attribute {}, but a different attribute {} is enabled.",
|
||||
attribute_name, i->second));
|
||||
}
|
||||
tags_map.erase(TTL_TAG_KEY);
|
||||
}
|
||||
co_await db::update_tags(_mm, schema, std::move(tags_map));
|
||||
// Prepare the response, which contains a TimeToLiveSpecification
|
||||
// basically identical to the request's
|
||||
rjson::value response = rjson::empty_object();
|
||||
@@ -139,7 +136,7 @@ future<executor::request_return_type> executor::describe_time_to_live(client_sta
|
||||
|
||||
// expiration_service is a sharded service responsible for cleaning up expired
|
||||
// items in all tables with per-item expiration enabled. Currently, this means
|
||||
// Alternator tables with TTL configured via a UpdateTimeToLive request.
|
||||
// Alternator tables with TTL configured via a UpdateTimeToLeave request.
|
||||
//
|
||||
// Here is a brief overview of how the expiration service works:
|
||||
//
|
||||
@@ -153,25 +150,25 @@ future<executor::request_return_type> executor::describe_time_to_live(client_sta
|
||||
// To avoid scanning the same items RF times in RF replicas, only one node is
|
||||
// responsible for scanning a token range at a time. Normally, this is the
|
||||
// node owning this range as a "primary range" (the first node in the ring
|
||||
// with this range), but when this node is down, the secondary owner (the
|
||||
// second in the ring) may take over.
|
||||
// with this range), but when this node is down, other nodes may take over
|
||||
// (FIXME: this is not implemented yet).
|
||||
// An expiration thread is reponsible for all tables which need expiration
|
||||
// scans. Currently, the different tables are scanned sequentially (not in
|
||||
// parallel).
|
||||
// scans. FIXME: explain how this is done with multiple tables - parallel,
|
||||
// staggered, or what?
|
||||
// The expiration thread scans item using CL=QUORUM to ensures that it reads
|
||||
// a consistent expiration-time attribute. This means that the items are read
|
||||
// locally and in addition QUORUM-1 additional nodes (one additional node
|
||||
// when RF=3) need to read the data and send digests.
|
||||
// FIXME: explain if we can read the exact attribute or the entire map.
|
||||
// When the expiration thread decides that an item has expired and wants
|
||||
// to delete it, it does it using a CL=QUORUM write. This allows this
|
||||
// deletion to be visible for consistent (quorum) reads. The deletion,
|
||||
// like user deletions, will also appear on the CDC log and therefore
|
||||
// Alternator Streams if enabled - currently as ordinary deletes (the
|
||||
// userIdentity flag is currently missing this is issue #11523).
|
||||
expiration_service::expiration_service(data_dictionary::database db, service::storage_proxy& proxy, gms::gossiper& g)
|
||||
// Alternator Streams if enabled (FIXME: explain how we mark the
|
||||
// deletion different from user deletes. We don't do it yet.).
|
||||
expiration_service::expiration_service(data_dictionary::database db, service::storage_proxy& proxy)
|
||||
: _db(db)
|
||||
, _proxy(proxy)
|
||||
, _gossiper(g)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -241,7 +238,7 @@ static bool is_expired(const rjson::value& expiration_time, gc_clock::time_point
|
||||
// understands it is an expiration event - not a user-initiated deletion.
|
||||
static future<> expire_item(service::storage_proxy& proxy,
|
||||
const service::query_state& qs,
|
||||
const std::vector<managed_bytes_opt>& row,
|
||||
const std::vector<bytes_opt>& row,
|
||||
schema_ptr schema,
|
||||
api::timestamp_type ts) {
|
||||
// Prepare the row key to delete
|
||||
@@ -260,7 +257,7 @@ static future<> expire_item(service::storage_proxy& proxy,
|
||||
// FIXME: log or increment a metric if this happens.
|
||||
return make_ready_future<>();
|
||||
}
|
||||
exploded_pk.push_back(to_bytes(*row_c));
|
||||
exploded_pk.push_back(*row_c);
|
||||
}
|
||||
auto pk = partition_key::from_exploded(exploded_pk);
|
||||
mutation m(schema, pk);
|
||||
@@ -280,14 +277,12 @@ static future<> expire_item(service::storage_proxy& proxy,
|
||||
// FIXME: log or increment a metric if this happens.
|
||||
return make_ready_future<>();
|
||||
}
|
||||
exploded_ck.push_back(to_bytes(*row_c));
|
||||
exploded_ck.push_back(*row_c);
|
||||
}
|
||||
auto ck = clustering_key::from_exploded(exploded_ck);
|
||||
m.partition().clustered_row(*schema, ck).apply(tombstone(ts, gc_clock::now()));
|
||||
}
|
||||
std::vector<mutation> mutations;
|
||||
mutations.push_back(std::move(m));
|
||||
return proxy.mutate(std::move(mutations),
|
||||
return proxy.mutate(std::vector<mutation>{std::move(m)},
|
||||
db::consistency_level::LOCAL_QUORUM,
|
||||
executor::default_timeout(), // FIXME - which timeout?
|
||||
qs.get_trace_state(), qs.get_permit(),
|
||||
@@ -370,7 +365,7 @@ static std::vector<std::pair<dht::token_range, gms::inet_address>> get_secondary
|
||||
// 2. The primary replica for this token is currently marked down.
|
||||
// 3. In this node, this shard is responsible for this token.
|
||||
// We use the <secondary> case to handle the possibility that some of the
|
||||
// nodes in the system are down. A dead node will not be expiring
|
||||
// nodes in the system are down. A dead node will not be expiring expiring
|
||||
// the tokens owned by it, so we want the secondary owner to take over its
|
||||
// primary ranges.
|
||||
//
|
||||
@@ -387,7 +382,7 @@ class token_ranges_owned_by_this_shard {
|
||||
class ranges_holder_primary {
|
||||
const dht::token_range_vector _token_ranges;
|
||||
public:
|
||||
ranges_holder_primary(const locator::vnode_effective_replication_map_ptr& erm, gms::gossiper& g, gms::inet_address ep)
|
||||
ranges_holder_primary(const locator::effective_replication_map_ptr& erm, gms::gossiper& g, gms::inet_address ep)
|
||||
: _token_ranges(erm->get_primary_ranges(ep)) {}
|
||||
std::size_t size() const { return _token_ranges.size(); }
|
||||
const dht::token_range& operator[](std::size_t i) const {
|
||||
@@ -430,7 +425,6 @@ class token_ranges_owned_by_this_shard {
|
||||
size_t _range_idx;
|
||||
size_t _end_idx;
|
||||
std::optional<dht::selective_token_range_sharder> _intersecter;
|
||||
locator::effective_replication_map_ptr _erm;
|
||||
public:
|
||||
token_ranges_owned_by_this_shard(replica::database& db, gms::gossiper& g, schema_ptr s)
|
||||
: _s(s)
|
||||
@@ -438,7 +432,6 @@ public:
|
||||
g, utils::fb_utilities::get_broadcast_address())
|
||||
, _range_idx(random_offset(0, _token_ranges.size() - 1))
|
||||
, _end_idx(_range_idx + _token_ranges.size())
|
||||
, _erm(s->table().get_effective_replication_map())
|
||||
{
|
||||
tlogger.debug("Generating token ranges starting from base range {} of {}", _range_idx, _token_ranges.size());
|
||||
}
|
||||
@@ -471,7 +464,7 @@ public:
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
_intersecter.emplace(_erm->get_sharder(*_s), _token_ranges[_range_idx % _token_ranges.size()], this_shard_id());
|
||||
_intersecter.emplace(_s->get_sharder(), _token_ranges[_range_idx % _token_ranges.size()], this_shard_id());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -518,7 +511,7 @@ struct scan_ranges_context {
|
||||
opts.set<query::partition_slice::option::bypass_cache>();
|
||||
std::vector<query::clustering_range> ck_bounds{query::clustering_range::make_open_ended_both_sides()};
|
||||
auto partition_slice = query::partition_slice(std::move(ck_bounds), {}, std::move(regular_columns), opts);
|
||||
command = ::make_lw_shared<query::read_command>(s->id(), s->version(), partition_slice, proxy.get_max_result_size(partition_slice), query::tombstone_limit(proxy.get_tombstone_limit()));
|
||||
command = ::make_lw_shared<query::read_command>(s->id(), s->version(), partition_slice, proxy.get_max_result_size(partition_slice));
|
||||
executor::client_state client_state{executor::client_state::internal_tag()};
|
||||
tracing::trace_state_ptr trace_state;
|
||||
// NOTICE: empty_service_permit is used because the TTL service has fixed parallelism
|
||||
@@ -553,34 +546,13 @@ static future<> scan_table_ranges(
|
||||
co_return;
|
||||
}
|
||||
auto units = co_await get_units(page_sem, 1);
|
||||
// We don't need to limit page size in number of rows because there is
|
||||
// a builtin limit of the page's size in bytes. Setting this limit to
|
||||
// 1 is useful for debugging the paging code with moderate-size data.
|
||||
// We don't to limit page size in number of rows because there is a
|
||||
// builtin limit of the page's size in bytes. Setting this limit to 1
|
||||
// is useful for debugging the paging code with moderate-size data.
|
||||
uint32_t limit = std::numeric_limits<uint32_t>::max();
|
||||
// Read a page, and if that times out, try again after a small sleep.
|
||||
// If we didn't catch the timeout exception, it would cause the scan
|
||||
// be aborted and only be restarted at the next scanning period.
|
||||
// If we retry too many times, give up and restart the scan later.
|
||||
std::unique_ptr<cql3::result_set> rs;
|
||||
for (int retries=0; ; retries++) {
|
||||
try {
|
||||
// FIXME: which timeout?
|
||||
rs = co_await p->fetch_page(limit, gc_clock::now(), executor::default_timeout());
|
||||
break;
|
||||
} catch(exceptions::read_timeout_exception&) {
|
||||
tlogger.warn("expiration scanner read timed out, will retry: {}",
|
||||
std::current_exception());
|
||||
}
|
||||
// If we didn't break out of this loop, add a minimal sleep
|
||||
if (retries >= 10) {
|
||||
// Don't get stuck forever asking the same page, maybe there's
|
||||
// a bug or a real problem in several replicas. Give up on
|
||||
// this scan an retry the scan from a random position later,
|
||||
// in the next scan period.
|
||||
throw runtime_exception("scanner thread failed after too many timeouts for the same page");
|
||||
}
|
||||
co_await sleep_abortable(std::chrono::seconds(1), abort_source);
|
||||
}
|
||||
// FIXME: which timeout?
|
||||
// FIXME: if read times out, need to retry it.
|
||||
std::unique_ptr<cql3::result_set> rs = co_await p->fetch_page(limit, gc_clock::now(), executor::default_timeout());
|
||||
auto rows = rs->rows();
|
||||
auto meta = rs->get_metadata().get_names();
|
||||
std::optional<unsigned> expiration_column;
|
||||
@@ -595,7 +567,7 @@ static future<> scan_table_ranges(
|
||||
continue;
|
||||
}
|
||||
for (const auto& row : rows) {
|
||||
const managed_bytes_opt& cell = row[*expiration_column];
|
||||
const bytes_opt& cell = row[*expiration_column];
|
||||
if (!cell) {
|
||||
continue;
|
||||
}
|
||||
@@ -665,7 +637,6 @@ static future<> scan_table_ranges(
|
||||
static future<bool> scan_table(
|
||||
service::storage_proxy& proxy,
|
||||
data_dictionary::database db,
|
||||
gms::gossiper& gossiper,
|
||||
schema_ptr s,
|
||||
abort_source& abort_source,
|
||||
named_semaphore& page_sem,
|
||||
@@ -718,7 +689,7 @@ static future<bool> scan_table(
|
||||
expiration_stats.scan_table++;
|
||||
// FIXME: need to pace the scan, not do it all at once.
|
||||
scan_ranges_context scan_ctx{s, proxy, std::move(column_name), std::move(member)};
|
||||
token_ranges_owned_by_this_shard<primary> my_ranges(db.real_database(), gossiper, s);
|
||||
token_ranges_owned_by_this_shard<primary> my_ranges(db.real_database(), proxy.gossiper(), s);
|
||||
while (std::optional<dht::partition_range> range = my_ranges.next_partition_range()) {
|
||||
// Note that because of issue #9167 we need to run a separate
|
||||
// query on each partition range, and can't pass several of
|
||||
@@ -738,7 +709,7 @@ static future<bool> scan_table(
|
||||
// by tasking another node to take over scanning of the dead node's primary
|
||||
// ranges. What we do here is that this node will also check expiration
|
||||
// on its *secondary* ranges - but only those whose primary owner is down.
|
||||
token_ranges_owned_by_this_shard<secondary> my_secondary_ranges(db.real_database(), gossiper, s);
|
||||
token_ranges_owned_by_this_shard<secondary> my_secondary_ranges(db.real_database(), proxy.gossiper(), s);
|
||||
while (std::optional<dht::partition_range> range = my_secondary_ranges.next_partition_range()) {
|
||||
expiration_stats.secondary_ranges_scanned++;
|
||||
dht::partition_range_vector partition_ranges;
|
||||
@@ -770,7 +741,7 @@ future<> expiration_service::run() {
|
||||
co_return;
|
||||
}
|
||||
try {
|
||||
co_await scan_table(_proxy, _db, _gossiper, s, _abort_source, _page_sem, _expiration_stats);
|
||||
co_await scan_table(_proxy, _db, s, _abort_source, _page_sem, _expiration_stats);
|
||||
} catch (...) {
|
||||
// The scan of a table may fail in the middle for many
|
||||
// reasons, including network failure and even the table
|
||||
@@ -796,15 +767,13 @@ future<> expiration_service::run() {
|
||||
// in the next iteration by reducing the scanner's scheduling-group
|
||||
// share (if using a separate scheduling group), or introduce
|
||||
// finer-grain sleeps into the scanning code.
|
||||
std::chrono::milliseconds scan_duration(std::chrono::duration_cast<std::chrono::milliseconds>(lowres_clock::now() - start));
|
||||
std::chrono::milliseconds period(long(_db.get_config().alternator_ttl_period_in_seconds() * 1000));
|
||||
std::chrono::seconds scan_duration(std::chrono::duration_cast<std::chrono::seconds>(lowres_clock::now() - start));
|
||||
std::chrono::seconds period(_db.get_config().alternator_ttl_period_in_seconds());
|
||||
if (scan_duration < period) {
|
||||
try {
|
||||
tlogger.info("sleeping {} seconds until next period", (period - scan_duration).count()/1000.0);
|
||||
tlogger.info("sleeping {} seconds until next period", (period - scan_duration).count());
|
||||
co_await seastar::sleep_abortable(period - scan_duration, _abort_source);
|
||||
} catch(seastar::sleep_aborted&) {}
|
||||
} else {
|
||||
tlogger.warn("scan took {} seconds, longer than period - not sleeping", scan_duration.count()/1000.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,10 +14,6 @@
|
||||
#include <seastar/core/semaphore.hh>
|
||||
#include "data_dictionary/data_dictionary.hh"
|
||||
|
||||
namespace gms {
|
||||
class gossiper;
|
||||
}
|
||||
|
||||
namespace replica {
|
||||
class database;
|
||||
}
|
||||
@@ -51,7 +47,6 @@ public:
|
||||
private:
|
||||
data_dictionary::database _db;
|
||||
service::storage_proxy& _proxy;
|
||||
gms::gossiper& _gossiper;
|
||||
// _end is set by start(), and resolves when the the background service
|
||||
// started by it ends. To ask the background service to end, _abort_source
|
||||
// should be triggered. stop() below uses both _abort_source and _end.
|
||||
@@ -65,7 +60,7 @@ public:
|
||||
// sharded_service<expiration_service>::start() creates this object on
|
||||
// all shards, so calls this constructor on each shard. Later, the
|
||||
// additional start() function should be invoked on all shards.
|
||||
expiration_service(data_dictionary::database, service::storage_proxy&, gms::gossiper&);
|
||||
expiration_service(data_dictionary::database, service::storage_proxy&);
|
||||
future<> start();
|
||||
future<> run();
|
||||
// sharded_service<expiration_service>::stop() calls the following stop()
|
||||
|
||||
15
amplify.yml
15
amplify.yml
@@ -1,15 +0,0 @@
|
||||
version: 1
|
||||
applications:
|
||||
- frontend:
|
||||
phases:
|
||||
build:
|
||||
commands:
|
||||
- make setupenv
|
||||
- make dirhtml
|
||||
artifacts:
|
||||
baseDirectory: _build/dirhtml
|
||||
files:
|
||||
- '**/*'
|
||||
cache:
|
||||
paths: []
|
||||
appRoot: docs
|
||||
@@ -1,71 +0,0 @@
|
||||
# Generate C++ sources from Swagger definitions
|
||||
set(swagger_files
|
||||
api-doc/authorization_cache.json
|
||||
api-doc/cache_service.json
|
||||
api-doc/collectd.json
|
||||
api-doc/column_family.json
|
||||
api-doc/commitlog.json
|
||||
api-doc/compaction_manager.json
|
||||
api-doc/config.json
|
||||
api-doc/endpoint_snitch_info.json
|
||||
api-doc/error_injection.json
|
||||
api-doc/failure_detector.json
|
||||
api-doc/gossiper.json
|
||||
api-doc/hinted_handoff.json
|
||||
api-doc/lsa.json
|
||||
api-doc/messaging_service.json
|
||||
api-doc/metrics.json
|
||||
api-doc/storage_proxy.json
|
||||
api-doc/storage_service.json
|
||||
api-doc/stream_manager.json
|
||||
api-doc/system.json
|
||||
api-doc/task_manager.json
|
||||
api-doc/task_manager_test.json
|
||||
api-doc/utils.json)
|
||||
|
||||
foreach(f ${swagger_files})
|
||||
get_filename_component(fname "${f}" NAME_WE)
|
||||
get_filename_component(dir "${f}" DIRECTORY)
|
||||
seastar_generate_swagger(
|
||||
TARGET scylla_swagger_gen_${fname}
|
||||
VAR scylla_swagger_gen_${fname}_files
|
||||
IN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${f}"
|
||||
OUT_DIR "${scylla_gen_build_dir}/api/${dir}")
|
||||
list(APPEND swagger_gen_files "${scylla_swagger_gen_${fname}_files}")
|
||||
endforeach()
|
||||
|
||||
add_library(api)
|
||||
target_sources(api
|
||||
PRIVATE
|
||||
api.cc
|
||||
cache_service.cc
|
||||
collectd.cc
|
||||
column_family.cc
|
||||
commitlog.cc
|
||||
compaction_manager.cc
|
||||
config.cc
|
||||
endpoint_snitch.cc
|
||||
error_injection.cc
|
||||
authorization_cache.cc
|
||||
failure_detector.cc
|
||||
gossiper.cc
|
||||
hinted_handoff.cc
|
||||
lsa.cc
|
||||
messaging_service.cc
|
||||
storage_proxy.cc
|
||||
storage_service.cc
|
||||
stream_manager.cc
|
||||
system.cc
|
||||
task_manager.cc
|
||||
task_manager_test.cc
|
||||
${swagger_gen_files})
|
||||
target_include_directories(api
|
||||
PUBLIC
|
||||
${CMAKE_SOURCE_DIR}
|
||||
${scylla_gen_build_dir})
|
||||
target_link_libraries(api
|
||||
idl
|
||||
wasmtime_bindings
|
||||
|
||||
Seastar::seastar
|
||||
xxHash::xxhash)
|
||||
@@ -437,68 +437,6 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/column_family/tombstone_gc/{name}",
|
||||
"operations":[
|
||||
{
|
||||
"method":"GET",
|
||||
"summary":"Check if tombstone GC is enabled for a given table",
|
||||
"type":"boolean",
|
||||
"nickname":"get_tombstone_gc",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"name":"name",
|
||||
"description":"The table name in keyspace:name format",
|
||||
"required":true,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"path"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"method":"POST",
|
||||
"summary":"Enable tombstone GC for a given table",
|
||||
"type":"void",
|
||||
"nickname":"enable_tombstone_gc",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"name":"name",
|
||||
"description":"The table name in keyspace:name format",
|
||||
"required":true,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"path"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"method":"DELETE",
|
||||
"summary":"Disable tombstone GC for a given table",
|
||||
"type":"void",
|
||||
"nickname":"disable_tombstone_gc",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"name":"name",
|
||||
"description":"The table name in keyspace:name format",
|
||||
"required":true,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"path"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/column_family/estimate_keys/{name}",
|
||||
"operations":[
|
||||
|
||||
@@ -34,14 +34,6 @@
|
||||
"allowMultiple":false,
|
||||
"type":"boolean",
|
||||
"paramType":"query"
|
||||
},
|
||||
{
|
||||
"name":"parameters",
|
||||
"description":"dict of parameters to pass to the injection (json format)",
|
||||
"required":false,
|
||||
"allowMultiple":false,
|
||||
"type":"dict",
|
||||
"paramType":"body"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -66,30 +58,6 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/v2/error_injection/injection/{injection}/message",
|
||||
"operations":[
|
||||
{
|
||||
"method":"POST",
|
||||
"summary":"Send message to trigger an event in injection's code",
|
||||
"type":"void",
|
||||
"nickname":"message_injection",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"name":"injection",
|
||||
"description":"injection name, should correspond to an injection added in code",
|
||||
"required":true,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"path"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/v2/error_injection/injection",
|
||||
"operations":[
|
||||
@@ -118,15 +86,5 @@
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"components":{
|
||||
"schemas": {
|
||||
"dict": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -245,7 +245,7 @@
|
||||
"GOSSIP_SHUTDOWN",
|
||||
"DEFINITIONS_UPDATE",
|
||||
"TRUNCATE",
|
||||
"UNUSED__REPLICATION_FINISHED",
|
||||
"REPLICATION_FINISHED",
|
||||
"MIGRATION_REQUEST",
|
||||
"PREPARE_MESSAGE",
|
||||
"PREPARE_DONE_MESSAGE",
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
"metrics_config": {
|
||||
"id": "metrics_config",
|
||||
"summary": "An entry in the metrics configuration",
|
||||
"properties": {
|
||||
"source_labels": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": "The source labels, a match is based on concatination of the labels"
|
||||
},
|
||||
"action": {
|
||||
"type": "string",
|
||||
"description": "The action to perfrom on match",
|
||||
"enum": ["skip_when_empty", "report_when_empty", "replace", "keep", "drop", "drop_label"]
|
||||
},
|
||||
"target_label": {
|
||||
"type": "string",
|
||||
"description": "The application state version"
|
||||
},
|
||||
"replacement": {
|
||||
"type": "string",
|
||||
"description": "The replacement string to use when replacing a value"
|
||||
},
|
||||
"regex": {
|
||||
"type": "string",
|
||||
"description": "The regex string to use when replacing a value"
|
||||
},
|
||||
"separator": {
|
||||
"type": "string",
|
||||
"description": "The separator string to use when concatinating the labels"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
"/v2/metrics-config/":{
|
||||
"get":{
|
||||
"description":"Return the metrics layer configuration",
|
||||
"operationId":"get_metrics_config",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"tags":[
|
||||
"metrics"
|
||||
],
|
||||
"parameters":[
|
||||
],
|
||||
"responses":{
|
||||
"200":{
|
||||
"schema": {
|
||||
"type":"array",
|
||||
"items":{
|
||||
"$ref":"#/definitions/metrics_config",
|
||||
"description":"metrics Config value"
|
||||
}
|
||||
}
|
||||
},
|
||||
"default":{
|
||||
"description":"unexpected error",
|
||||
"schema":{
|
||||
"$ref":"#/definitions/ErrorModel"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"post": {
|
||||
"description":"Set the metrics layer relabel configuration",
|
||||
"operationId":"set_metrics_config",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"tags":[
|
||||
"metrics"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"in":"body",
|
||||
"name":"conf",
|
||||
"description":"An array of relabel_config objects",
|
||||
"schema": {
|
||||
"type":"array",
|
||||
"items":{
|
||||
"$ref":"#/definitions/metrics_config",
|
||||
"description":"metrics Config value"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses":{
|
||||
"200":{
|
||||
"description": "OK"
|
||||
},
|
||||
"default":{
|
||||
"description":"unexpected error",
|
||||
"schema":{
|
||||
"$ref":"#/definitions/ErrorModel"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -465,7 +465,7 @@
|
||||
"operations":[
|
||||
{
|
||||
"method":"GET",
|
||||
"summary":"Retrieve the mapping of endpoint to host ID of all nodes that own tokens",
|
||||
"summary":"Retrieve the mapping of endpoint to host ID",
|
||||
"type":"array",
|
||||
"items":{
|
||||
"type":"mapper"
|
||||
@@ -1114,14 +1114,6 @@
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"query"
|
||||
},
|
||||
{
|
||||
"name":"ranges_parallelism",
|
||||
"description":"An integer specifying the number of ranges to repair in parallel by user request. If this number is bigger than the max_repair_ranges_in_parallel calculated by Scylla core, the smaller one will be used.",
|
||||
"required":false,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"query"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1236,7 +1228,7 @@
|
||||
"operations":[
|
||||
{
|
||||
"method":"POST",
|
||||
"summary":"Removes a node from the cluster. Replicated data that logically belonged to this node is redistributed among the remaining nodes.",
|
||||
"summary":"Removes token (and all data associated with enpoint that had it) from the ring",
|
||||
"type":"void",
|
||||
"nickname":"remove_node",
|
||||
"produces":[
|
||||
@@ -1253,7 +1245,7 @@
|
||||
},
|
||||
{
|
||||
"name":"ignore_nodes",
|
||||
"description":"Comma-separated list of dead nodes to ignore in removenode operation. Use the same method for all nodes to ignore: either Host IDs or ip addresses.",
|
||||
"description":"List of dead nodes to ingore in removenode operation",
|
||||
"required":false,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
@@ -2118,65 +2110,6 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/storage_service/tombstone_gc/{keyspace}",
|
||||
"operations":[
|
||||
{
|
||||
"method":"POST",
|
||||
"summary":"Enable tombstone GC",
|
||||
"type":"void",
|
||||
"nickname":"enable_tombstone_gc",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"name":"keyspace",
|
||||
"description":"The keyspace",
|
||||
"required":true,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"path"
|
||||
},
|
||||
{
|
||||
"name":"cf",
|
||||
"description":"Comma-separated column family names",
|
||||
"required":false,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"query"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"method":"DELETE",
|
||||
"summary":"Disable tombstone GC",
|
||||
"type":"void",
|
||||
"nickname":"disable_tombstone_gc",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"name":"keyspace",
|
||||
"description":"The keyspace",
|
||||
"required":true,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"path"
|
||||
},
|
||||
{
|
||||
"name":"cf",
|
||||
"description":"Comma-separated column family names",
|
||||
"required":false,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"query"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/storage_service/deliver_hints",
|
||||
"operations":[
|
||||
@@ -2495,23 +2428,7 @@
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/storage_service/raft_topology/reload",
|
||||
"operations":[
|
||||
{
|
||||
"method":"POST",
|
||||
"summary":"Reload Raft topology state from disk.",
|
||||
"type":"void",
|
||||
"nickname":"reload_raft_topology_state",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"models":{
|
||||
"mapper":{
|
||||
@@ -2714,7 +2631,7 @@
|
||||
"description":"File creation time"
|
||||
},
|
||||
"generation":{
|
||||
"type":"string",
|
||||
"type":"long",
|
||||
"description":"SSTable generation"
|
||||
},
|
||||
"level":{
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
}
|
||||
},
|
||||
"host": "{{Host}}",
|
||||
"basePath": "/",
|
||||
"basePath": "/v2",
|
||||
"schemes": [
|
||||
"http"
|
||||
],
|
||||
|
||||
@@ -52,45 +52,6 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/system/log",
|
||||
"operations":[
|
||||
{
|
||||
"method":"POST",
|
||||
"summary":"Write a message to the Scylla log",
|
||||
"type":"void",
|
||||
"nickname":"write_log_message",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"name":"message",
|
||||
"description":"The message to write to the log",
|
||||
"required":true,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"query"
|
||||
},
|
||||
{
|
||||
"name":"level",
|
||||
"description":"The logging level to use",
|
||||
"required":true,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"enum":[
|
||||
"error",
|
||||
"warn",
|
||||
"info",
|
||||
"debug",
|
||||
"trace"
|
||||
],
|
||||
"paramType":"query"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/system/drop_sstable_caches",
|
||||
"operations":[
|
||||
|
||||
@@ -1,337 +0,0 @@
|
||||
{
|
||||
"apiVersion":"0.0.1",
|
||||
"swaggerVersion":"1.2",
|
||||
"basePath":"{{Protocol}}://{{Host}}",
|
||||
"resourcePath":"/task_manager",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"apis":[
|
||||
{
|
||||
"path":"/task_manager/list_modules",
|
||||
"operations":[
|
||||
{
|
||||
"method":"GET",
|
||||
"summary":"Get all modules names",
|
||||
"type":"array",
|
||||
"items":{
|
||||
"type":"string"
|
||||
},
|
||||
"nickname":"get_modules",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/task_manager/list_module_tasks/{module}",
|
||||
"operations":[
|
||||
{
|
||||
"method":"GET",
|
||||
"summary":"Get a list of tasks",
|
||||
"type":"array",
|
||||
"items":{
|
||||
"type":"task_stats"
|
||||
},
|
||||
"nickname":"get_tasks",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"name":"module",
|
||||
"description":"The module to query about",
|
||||
"required":true,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"path"
|
||||
},
|
||||
{
|
||||
"name":"internal",
|
||||
"description":"Boolean flag indicating whether internal tasks should be shown (false by default)",
|
||||
"required":false,
|
||||
"allowMultiple":false,
|
||||
"type":"boolean",
|
||||
"paramType":"query"
|
||||
},
|
||||
{
|
||||
"name":"keyspace",
|
||||
"description":"The keyspace to query about",
|
||||
"required":false,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"query"
|
||||
},
|
||||
{
|
||||
"name":"table",
|
||||
"description":"The table to query about",
|
||||
"required":false,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"query"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/task_manager/task_status/{task_id}",
|
||||
"operations":[
|
||||
{
|
||||
"method":"GET",
|
||||
"summary":"Get task status",
|
||||
"type":"task_status",
|
||||
"nickname":"get_task_status",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"name":"task_id",
|
||||
"description":"The uuid of a task to query about",
|
||||
"required":true,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"path"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/task_manager/abort_task/{task_id}",
|
||||
"operations":[
|
||||
{
|
||||
"method":"POST",
|
||||
"summary":"Abort running task and its descendants",
|
||||
"type":"void",
|
||||
"nickname":"abort_task",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"name":"task_id",
|
||||
"description":"The uuid of a task to abort",
|
||||
"required":true,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"path"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/task_manager/wait_task/{task_id}",
|
||||
"operations":[
|
||||
{
|
||||
"method":"GET",
|
||||
"summary":"Wait for a task to complete",
|
||||
"type":"task_status",
|
||||
"nickname":"wait_task",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"name":"task_id",
|
||||
"description":"The uuid of a task to wait for",
|
||||
"required":true,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"path"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/task_manager/task_status_recursive/{task_id}",
|
||||
"operations":[
|
||||
{
|
||||
"method":"GET",
|
||||
"summary":"Get statuses of the task and all its descendants",
|
||||
"type":"array",
|
||||
"items":{
|
||||
"type":"task_status"
|
||||
},
|
||||
"nickname":"get_task_status_recursively",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"name":"task_id",
|
||||
"description":"The uuid of a task to query about",
|
||||
"required":true,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"path"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/task_manager/ttl",
|
||||
"operations":[
|
||||
{
|
||||
"method":"POST",
|
||||
"summary":"Set ttl in seconds and get last value",
|
||||
"type":"long",
|
||||
"nickname":"get_and_update_ttl",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"name":"ttl",
|
||||
"description":"The number of seconds for which the tasks will be kept in memory after it finishes",
|
||||
"required":true,
|
||||
"allowMultiple":false,
|
||||
"type":"long",
|
||||
"paramType":"query"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"models":{
|
||||
"task_stats" :{
|
||||
"id": "task_stats",
|
||||
"description":"A task statistics object",
|
||||
"properties":{
|
||||
"task_id":{
|
||||
"type":"string",
|
||||
"description":"The uuid of a task"
|
||||
},
|
||||
"state":{
|
||||
"type":"string",
|
||||
"enum":[
|
||||
"created",
|
||||
"running",
|
||||
"done",
|
||||
"failed"
|
||||
],
|
||||
"description":"The state of a task"
|
||||
},
|
||||
"type":{
|
||||
"type":"string",
|
||||
"description":"The description of the task"
|
||||
},
|
||||
"scope":{
|
||||
"type":"string",
|
||||
"description":"The scope of the task"
|
||||
},
|
||||
"keyspace":{
|
||||
"type":"string",
|
||||
"description":"The keyspace the task is working on (if applicable)"
|
||||
},
|
||||
"table":{
|
||||
"type":"string",
|
||||
"description":"The table the task is working on (if applicable)"
|
||||
},
|
||||
"entity":{
|
||||
"type":"string",
|
||||
"description":"Task-specific entity description"
|
||||
},
|
||||
"sequence_number":{
|
||||
"type":"long",
|
||||
"description":"The running sequence number of the task"
|
||||
}
|
||||
}
|
||||
},
|
||||
"task_status":{
|
||||
"id":"task_status",
|
||||
"description":"A task status object",
|
||||
"properties":{
|
||||
"id":{
|
||||
"type":"string",
|
||||
"description":"The uuid of the task"
|
||||
},
|
||||
"type":{
|
||||
"type":"string",
|
||||
"description":"The description of the task"
|
||||
},
|
||||
"scope":{
|
||||
"type":"string",
|
||||
"description":"The scope of the task"
|
||||
},
|
||||
"state":{
|
||||
"type":"string",
|
||||
"enum":[
|
||||
"created",
|
||||
"running",
|
||||
"done",
|
||||
"failed"
|
||||
],
|
||||
"description":"The state of the task"
|
||||
},
|
||||
"is_abortable":{
|
||||
"type":"boolean",
|
||||
"description":"Boolean flag indicating whether the task can be aborted"
|
||||
},
|
||||
"start_time":{
|
||||
"type":"datetime",
|
||||
"description":"The start time of the task"
|
||||
},
|
||||
"end_time":{
|
||||
"type":"datetime",
|
||||
"description":"The end time of the task (unspecified when the task is not completed)"
|
||||
},
|
||||
"error":{
|
||||
"type":"string",
|
||||
"description":"Error string, if the task failed"
|
||||
},
|
||||
"parent_id":{
|
||||
"type":"string",
|
||||
"description":"The uuid of the parent task"
|
||||
},
|
||||
"sequence_number":{
|
||||
"type":"long",
|
||||
"description":"The running sequence number of the task"
|
||||
},
|
||||
"shard":{
|
||||
"type":"long",
|
||||
"description":"The number of a shard the task is running on"
|
||||
},
|
||||
"keyspace":{
|
||||
"type":"string",
|
||||
"description":"The keyspace the task is working on (if applicable)"
|
||||
},
|
||||
"table":{
|
||||
"type":"string",
|
||||
"description":"The table the task is working on (if applicable)"
|
||||
},
|
||||
"entity":{
|
||||
"type":"string",
|
||||
"description":"Task-specific entity description"
|
||||
},
|
||||
"progress_units":{
|
||||
"type":"string",
|
||||
"description":"A description of the progress units"
|
||||
},
|
||||
"progress_total":{
|
||||
"type":"double",
|
||||
"description":"The total number of units to complete for the task"
|
||||
},
|
||||
"progress_completed":{
|
||||
"type":"double",
|
||||
"description":"The number of units completed so far"
|
||||
},
|
||||
"children_ids":{
|
||||
"type":"array",
|
||||
"items":{
|
||||
"type":"string"
|
||||
},
|
||||
"description":"Task IDs of children of this task"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
{
|
||||
"apiVersion":"0.0.1",
|
||||
"swaggerVersion":"1.2",
|
||||
"basePath":"{{Protocol}}://{{Host}}",
|
||||
"resourcePath":"/task_manager_test",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"apis":[
|
||||
{
|
||||
"path":"/task_manager_test/test_module",
|
||||
"operations":[
|
||||
{
|
||||
"method":"POST",
|
||||
"summary":"Register test module in task manager",
|
||||
"type":"void",
|
||||
"nickname":"register_test_module",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
]
|
||||
},
|
||||
{
|
||||
"method":"DELETE",
|
||||
"summary":"Unregister test module in task manager",
|
||||
"type":"void",
|
||||
"nickname":"unregister_test_module",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/task_manager_test/test_task",
|
||||
"operations":[
|
||||
{
|
||||
"method":"POST",
|
||||
"summary":"Register test task",
|
||||
"type":"string",
|
||||
"nickname":"register_test_task",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"name":"task_id",
|
||||
"description":"The uuid of a task to register",
|
||||
"required":false,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"query"
|
||||
},
|
||||
{
|
||||
"name":"shard",
|
||||
"description":"The shard of the task",
|
||||
"required":false,
|
||||
"allowMultiple":false,
|
||||
"type":"long",
|
||||
"paramType":"query"
|
||||
},
|
||||
{
|
||||
"name":"parent_id",
|
||||
"description":"The uuid of a parent task",
|
||||
"required":false,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"query"
|
||||
},
|
||||
{
|
||||
"name":"keyspace",
|
||||
"description":"The keyspace the task is working on",
|
||||
"required":false,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"query"
|
||||
},
|
||||
{
|
||||
"name":"table",
|
||||
"description":"The table the task is working on",
|
||||
"required":false,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"query"
|
||||
},
|
||||
{
|
||||
"name":"entity",
|
||||
"description":"Task-specific entity description",
|
||||
"required":false,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"query"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"method":"DELETE",
|
||||
"summary":"Unregister test task",
|
||||
"type":"void",
|
||||
"nickname":"unregister_test_task",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"name":"task_id",
|
||||
"description":"The uuid of a task to register",
|
||||
"required":true,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"query"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"path":"/task_manager_test/finish_test_task/{task_id}",
|
||||
"operations":[
|
||||
{
|
||||
"method":"POST",
|
||||
"summary":"Finish test task",
|
||||
"type":"void",
|
||||
"nickname":"finish_test_task",
|
||||
"produces":[
|
||||
"application/json"
|
||||
],
|
||||
"parameters":[
|
||||
{
|
||||
"name":"task_id",
|
||||
"description":"The uuid of a task to finish",
|
||||
"required":true,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"path"
|
||||
},
|
||||
{
|
||||
"name":"error",
|
||||
"description":"The error with which task fails (if it does)",
|
||||
"required":false,
|
||||
"allowMultiple":false,
|
||||
"type":"string",
|
||||
"paramType":"query"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
85
api/api.cc
85
api/api.cc
@@ -29,13 +29,10 @@
|
||||
#include "stream_manager.hh"
|
||||
#include "system.hh"
|
||||
#include "api/config.hh"
|
||||
#include "task_manager.hh"
|
||||
#include "task_manager_test.hh"
|
||||
|
||||
logging::logger apilog("api");
|
||||
|
||||
namespace api {
|
||||
using namespace seastar::httpd;
|
||||
|
||||
static std::unique_ptr<reply> exception_reply(std::exception_ptr eptr) {
|
||||
try {
|
||||
@@ -60,10 +57,8 @@ future<> set_server_init(http_context& ctx) {
|
||||
rb->set_api_doc(r);
|
||||
rb02->set_api_doc(r);
|
||||
rb02->register_api_file(r, "swagger20_header");
|
||||
rb02->register_api_file(r, "metrics");
|
||||
rb->register_function(r, "system",
|
||||
"The system related API");
|
||||
rb02->add_definitions_file(r, "metrics");
|
||||
set_system(ctx, r);
|
||||
});
|
||||
}
|
||||
@@ -71,7 +66,7 @@ future<> set_server_init(http_context& ctx) {
|
||||
future<> set_server_config(http_context& ctx, const db::config& cfg) {
|
||||
auto rb02 = std::make_shared < api_registry_builder20 > (ctx.api_doc, "/v2");
|
||||
return ctx.http_server.set_routes([&ctx, &cfg, rb02](routes& r) {
|
||||
set_config(rb02, ctx, r, cfg, false);
|
||||
set_config(rb02, ctx, r, cfg);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -102,16 +97,12 @@ future<> unset_rpc_controller(http_context& ctx) {
|
||||
return ctx.http_server.set_routes([&ctx] (routes& r) { unset_rpc_controller(ctx, r); });
|
||||
}
|
||||
|
||||
future<> set_server_storage_service(http_context& ctx, sharded<service::storage_service>& ss, service::raft_group0_client& group0_client) {
|
||||
return register_api(ctx, "storage_service", "The storage service API", [&ss, &group0_client] (http_context& ctx, routes& r) {
|
||||
set_storage_service(ctx, r, ss, group0_client);
|
||||
future<> set_server_storage_service(http_context& ctx, sharded<service::storage_service>& ss, sharded<gms::gossiper>& g, sharded<cdc::generation_service>& cdc_gs, sharded<db::system_keyspace>& sys_ks) {
|
||||
return register_api(ctx, "storage_service", "The storage service API", [&ss, &g, &cdc_gs, &sys_ks] (http_context& ctx, routes& r) {
|
||||
set_storage_service(ctx, r, ss, g.local(), cdc_gs, sys_ks);
|
||||
});
|
||||
}
|
||||
|
||||
future<> unset_server_storage_service(http_context& ctx) {
|
||||
return ctx.http_server.set_routes([&ctx] (routes& r) { unset_storage_service(ctx, r); });
|
||||
}
|
||||
|
||||
future<> set_server_sstables_loader(http_context& ctx, sharded<sstables_loader>& sst_loader) {
|
||||
return ctx.http_server.set_routes([&ctx, &sst_loader] (routes& r) { set_sstables_loader(ctx, r, sst_loader); });
|
||||
}
|
||||
@@ -155,14 +146,8 @@ future<> unset_server_snapshot(http_context& ctx) {
|
||||
return ctx.http_server.set_routes([&ctx] (routes& r) { unset_snapshot(ctx, r); });
|
||||
}
|
||||
|
||||
future<> set_server_snitch(http_context& ctx, sharded<locator::snitch_ptr>& snitch) {
|
||||
return register_api(ctx, "endpoint_snitch_info", "The endpoint snitch info API", [&snitch] (http_context& ctx, routes& r) {
|
||||
set_endpoint_snitch(ctx, r, snitch);
|
||||
});
|
||||
}
|
||||
|
||||
future<> unset_server_snitch(http_context& ctx) {
|
||||
return ctx.http_server.set_routes([&ctx] (routes& r) { unset_endpoint_snitch(ctx, r); });
|
||||
future<> set_server_snitch(http_context& ctx) {
|
||||
return register_api(ctx, "endpoint_snitch_info", "The endpoint snitch info API", set_endpoint_snitch);
|
||||
}
|
||||
|
||||
future<> set_server_gossip(http_context& ctx, sharded<gms::gossiper>& g) {
|
||||
@@ -172,15 +157,9 @@ future<> set_server_gossip(http_context& ctx, sharded<gms::gossiper>& g) {
|
||||
});
|
||||
}
|
||||
|
||||
future<> set_server_load_sstable(http_context& ctx, sharded<db::system_keyspace>& sys_ks) {
|
||||
future<> set_server_load_sstable(http_context& ctx) {
|
||||
return register_api(ctx, "column_family",
|
||||
"The column family API", [&sys_ks] (http_context& ctx, routes& r) {
|
||||
set_column_family(ctx, r, sys_ks);
|
||||
});
|
||||
}
|
||||
|
||||
future<> unset_server_load_sstable(http_context& ctx) {
|
||||
return ctx.http_server.set_routes([&ctx] (routes& r) { unset_column_family(ctx, r); });
|
||||
"The column family API", set_column_family);
|
||||
}
|
||||
|
||||
future<> set_server_messaging_service(http_context& ctx, sharded<netw::messaging_service>& ms) {
|
||||
@@ -193,17 +172,13 @@ future<> unset_server_messaging_service(http_context& ctx) {
|
||||
return ctx.http_server.set_routes([&ctx] (routes& r) { unset_messaging_service(ctx, r); });
|
||||
}
|
||||
|
||||
future<> set_server_storage_proxy(http_context& ctx, sharded<service::storage_proxy>& proxy) {
|
||||
future<> set_server_storage_proxy(http_context& ctx, sharded<service::storage_service>& ss) {
|
||||
return register_api(ctx, "storage_proxy",
|
||||
"The storage proxy API", [&proxy] (http_context& ctx, routes& r) {
|
||||
set_storage_proxy(ctx, r, proxy);
|
||||
"The storage proxy API", [&ss] (http_context& ctx, routes& r) {
|
||||
set_storage_proxy(ctx, r, ss);
|
||||
});
|
||||
}
|
||||
|
||||
future<> unset_server_storage_proxy(http_context& ctx) {
|
||||
return ctx.http_server.set_routes([&ctx] (routes& r) { unset_storage_proxy(ctx, r); });
|
||||
}
|
||||
|
||||
future<> set_server_stream_manager(http_context& ctx, sharded<streaming::stream_manager>& sm) {
|
||||
return register_api(ctx, "stream_manager",
|
||||
"The stream manager API", [&sm] (http_context& ctx, routes& r) {
|
||||
@@ -220,10 +195,10 @@ future<> set_server_cache(http_context& ctx) {
|
||||
"The cache service API", set_cache_service);
|
||||
}
|
||||
|
||||
future<> set_hinted_handoff(http_context& ctx, sharded<service::storage_proxy>& proxy) {
|
||||
future<> set_hinted_handoff(http_context& ctx, sharded<gms::gossiper>& g) {
|
||||
return register_api(ctx, "hinted_handoff",
|
||||
"The hinted handoff API", [&proxy] (http_context& ctx, routes& r) {
|
||||
set_hinted_handoff(ctx, r, proxy);
|
||||
"The hinted handoff API", [&g] (http_context& ctx, routes& r) {
|
||||
set_hinted_handoff(ctx, r, g.local());
|
||||
});
|
||||
}
|
||||
|
||||
@@ -270,38 +245,6 @@ future<> set_server_done(http_context& ctx) {
|
||||
});
|
||||
}
|
||||
|
||||
future<> set_server_task_manager(http_context& ctx, sharded<tasks::task_manager>& tm, lw_shared_ptr<db::config> cfg) {
|
||||
auto rb = std::make_shared < api_registry_builder > (ctx.api_doc);
|
||||
|
||||
return ctx.http_server.set_routes([rb, &ctx, &tm, &cfg = *cfg](routes& r) {
|
||||
rb->register_function(r, "task_manager",
|
||||
"The task manager API");
|
||||
set_task_manager(ctx, r, tm, cfg);
|
||||
});
|
||||
}
|
||||
|
||||
future<> unset_server_task_manager(http_context& ctx) {
|
||||
return ctx.http_server.set_routes([&ctx] (routes& r) { unset_task_manager(ctx, r); });
|
||||
}
|
||||
|
||||
#ifndef SCYLLA_BUILD_MODE_RELEASE
|
||||
|
||||
future<> set_server_task_manager_test(http_context& ctx, sharded<tasks::task_manager>& tm) {
|
||||
auto rb = std::make_shared < api_registry_builder > (ctx.api_doc);
|
||||
|
||||
return ctx.http_server.set_routes([rb, &ctx, &tm](routes& r) mutable {
|
||||
rb->register_function(r, "task_manager_test",
|
||||
"The task manager test API");
|
||||
set_task_manager_test(ctx, r, tm);
|
||||
});
|
||||
}
|
||||
|
||||
future<> unset_server_task_manager_test(http_context& ctx) {
|
||||
return ctx.http_server.set_routes([&ctx] (routes& r) { unset_task_manager_test(ctx, r); });
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
void req_params::process(const request& req) {
|
||||
// Process mandatory parameters
|
||||
for (auto& [name, ent] : params) {
|
||||
|
||||
22
api/api.hh
22
api/api.hh
@@ -27,7 +27,7 @@ template<class T>
|
||||
std::vector<sstring> container_to_vec(const T& container) {
|
||||
std::vector<sstring> res;
|
||||
for (auto i : container) {
|
||||
res.push_back(fmt::to_string(i));
|
||||
res.push_back(boost::lexical_cast<std::string>(i));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
@@ -47,8 +47,8 @@ template<class T, class MAP>
|
||||
std::vector<T>& map_to_key_value(const MAP& map, std::vector<T>& res) {
|
||||
for (auto i : map) {
|
||||
T val;
|
||||
val.key = fmt::to_string(i.first);
|
||||
val.value = fmt::to_string(i.second);
|
||||
val.key = boost::lexical_cast<std::string>(i.first);
|
||||
val.value = boost::lexical_cast<std::string>(i.second);
|
||||
res.push_back(val);
|
||||
}
|
||||
return res;
|
||||
@@ -65,7 +65,7 @@ template <typename MAP>
|
||||
std::vector<sstring> map_keys(const MAP& map) {
|
||||
std::vector<sstring> res;
|
||||
for (const auto& i : map) {
|
||||
res.push_back(fmt::to_string(i.first));
|
||||
res.push_back(boost::lexical_cast<std::string>(i.first));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
@@ -137,14 +137,6 @@ future<json::json_return_type> sum_timer_stats(distributed<T>& d, utils::timed_
|
||||
});
|
||||
}
|
||||
|
||||
template<class T, class F>
|
||||
future<json::json_return_type> sum_timer_stats(distributed<T>& d, utils::timed_rate_moving_average_summary_and_histogram F::*f) {
|
||||
return d.map_reduce0([f](const T& p) {return (p.get_stats().*f).rate();}, utils::rate_moving_average_and_histogram(),
|
||||
std::plus<utils::rate_moving_average_and_histogram>()).then([](const utils::rate_moving_average_and_histogram& val) {
|
||||
return make_ready_future<json::json_return_type>(timer_to_json(val));
|
||||
});
|
||||
}
|
||||
|
||||
inline int64_t min_int64(int64_t a, int64_t b) {
|
||||
return std::min(a,b);
|
||||
}
|
||||
@@ -189,7 +181,7 @@ struct basic_ratio_holder : public json::jsonable {
|
||||
typedef basic_ratio_holder<double> ratio_holder;
|
||||
typedef basic_ratio_holder<int64_t> integral_ratio_holder;
|
||||
|
||||
class unimplemented_exception : public httpd::base_exception {
|
||||
class unimplemented_exception : public base_exception {
|
||||
public:
|
||||
unimplemented_exception()
|
||||
: base_exception("API call is not supported yet", reply::status_type::internal_server_error) {
|
||||
@@ -238,7 +230,7 @@ public:
|
||||
value = T{boost::lexical_cast<Base>(param)};
|
||||
}
|
||||
} catch (boost::bad_lexical_cast&) {
|
||||
throw httpd::bad_param_exception(format("{} ({}): type error - should be {}", name, param, boost::units::detail::demangle(typeid(Base).name())));
|
||||
throw bad_param_exception(format("{} ({}): type error - should be {}", name, param, boost::units::detail::demangle(typeid(Base).name())));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -306,6 +298,6 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
httpd::utils_json::estimated_histogram time_to_json_histogram(const utils::time_estimated_histogram& val);
|
||||
utils_json::estimated_histogram time_to_json_histogram(const utils::time_estimated_histogram& val);
|
||||
|
||||
}
|
||||
|
||||
@@ -11,18 +11,13 @@
|
||||
#include <seastar/core/future.hh>
|
||||
|
||||
#include "replica/database_fwd.hh"
|
||||
#include "tasks/task_manager.hh"
|
||||
#include "seastarx.hh"
|
||||
|
||||
using request = http::request;
|
||||
using reply = http::reply;
|
||||
|
||||
namespace service {
|
||||
|
||||
class load_meter;
|
||||
class storage_proxy;
|
||||
class storage_service;
|
||||
class raft_group0_client;
|
||||
|
||||
} // namespace service
|
||||
|
||||
@@ -36,7 +31,6 @@ namespace locator {
|
||||
|
||||
class token_metadata;
|
||||
class shared_token_metadata;
|
||||
class snitch_ptr;
|
||||
|
||||
} // namespace locator
|
||||
|
||||
@@ -52,6 +46,7 @@ class system_keyspace;
|
||||
}
|
||||
namespace netw { class messaging_service; }
|
||||
class repair_service;
|
||||
namespace cdc { class generation_service; }
|
||||
|
||||
namespace gms {
|
||||
|
||||
@@ -61,10 +56,6 @@ class gossiper;
|
||||
|
||||
namespace auth { class service; }
|
||||
|
||||
namespace tasks {
|
||||
class task_manager;
|
||||
}
|
||||
|
||||
namespace api {
|
||||
|
||||
struct http_context {
|
||||
@@ -72,12 +63,14 @@ struct http_context {
|
||||
sstring api_doc;
|
||||
httpd::http_server_control http_server;
|
||||
distributed<replica::database>& db;
|
||||
distributed<service::storage_proxy>& sp;
|
||||
service::load_meter& lmeter;
|
||||
const sharded<locator::shared_token_metadata>& shared_token_metadata;
|
||||
|
||||
http_context(distributed<replica::database>& _db,
|
||||
distributed<service::storage_proxy>& _sp,
|
||||
service::load_meter& _lm, const sharded<locator::shared_token_metadata>& _stm)
|
||||
: db(_db), lmeter(_lm), shared_token_metadata(_stm) {
|
||||
: db(_db), sp(_sp), lmeter(_lm), shared_token_metadata(_stm) {
|
||||
}
|
||||
|
||||
const locator::token_metadata& get_token_metadata();
|
||||
@@ -85,10 +78,8 @@ struct http_context {
|
||||
|
||||
future<> set_server_init(http_context& ctx);
|
||||
future<> set_server_config(http_context& ctx, const db::config& cfg);
|
||||
future<> set_server_snitch(http_context& ctx, sharded<locator::snitch_ptr>& snitch);
|
||||
future<> unset_server_snitch(http_context& ctx);
|
||||
future<> set_server_storage_service(http_context& ctx, sharded<service::storage_service>& ss, service::raft_group0_client&);
|
||||
future<> unset_server_storage_service(http_context& ctx);
|
||||
future<> set_server_snitch(http_context& ctx);
|
||||
future<> set_server_storage_service(http_context& ctx, sharded<service::storage_service>& ss, sharded<gms::gossiper>& g, sharded<cdc::generation_service>& cdc_gs, sharded<db::system_keyspace>& sys_ks);
|
||||
future<> set_server_sstables_loader(http_context& ctx, sharded<sstables_loader>& sst_loader);
|
||||
future<> unset_server_sstables_loader(http_context& ctx);
|
||||
future<> set_server_view_builder(http_context& ctx, sharded<db::view::view_builder>& vb);
|
||||
@@ -104,23 +95,17 @@ future<> unset_server_authorization_cache(http_context& ctx);
|
||||
future<> set_server_snapshot(http_context& ctx, sharded<db::snapshot_ctl>& snap_ctl);
|
||||
future<> unset_server_snapshot(http_context& ctx);
|
||||
future<> set_server_gossip(http_context& ctx, sharded<gms::gossiper>& g);
|
||||
future<> set_server_load_sstable(http_context& ctx, sharded<db::system_keyspace>& sys_ks);
|
||||
future<> unset_server_load_sstable(http_context& ctx);
|
||||
future<> set_server_load_sstable(http_context& ctx);
|
||||
future<> set_server_messaging_service(http_context& ctx, sharded<netw::messaging_service>& ms);
|
||||
future<> unset_server_messaging_service(http_context& ctx);
|
||||
future<> set_server_storage_proxy(http_context& ctx, sharded<service::storage_proxy>& proxy);
|
||||
future<> unset_server_storage_proxy(http_context& ctx);
|
||||
future<> set_server_storage_proxy(http_context& ctx, sharded<service::storage_service>& ss);
|
||||
future<> set_server_stream_manager(http_context& ctx, sharded<streaming::stream_manager>& sm);
|
||||
future<> unset_server_stream_manager(http_context& ctx);
|
||||
future<> set_hinted_handoff(http_context& ctx, sharded<service::storage_proxy>& p);
|
||||
future<> set_hinted_handoff(http_context& ctx, sharded<gms::gossiper>& g);
|
||||
future<> unset_hinted_handoff(http_context& ctx);
|
||||
future<> set_server_gossip_settle(http_context& ctx, sharded<gms::gossiper>& g);
|
||||
future<> set_server_cache(http_context& ctx);
|
||||
future<> set_server_compaction_manager(http_context& ctx);
|
||||
future<> set_server_done(http_context& ctx);
|
||||
future<> set_server_task_manager(http_context& ctx, sharded<tasks::task_manager>& tm, lw_shared_ptr<db::config> cfg);
|
||||
future<> unset_server_task_manager(http_context& ctx);
|
||||
future<> set_server_task_manager_test(http_context& ctx, sharded<tasks::task_manager>& tm);
|
||||
future<> unset_server_task_manager_test(http_context& ctx);
|
||||
|
||||
}
|
||||
|
||||
@@ -11,14 +11,12 @@
|
||||
#include "api/authorization_cache.hh"
|
||||
#include "api/api.hh"
|
||||
#include "auth/common.hh"
|
||||
#include "auth/service.hh"
|
||||
|
||||
namespace api {
|
||||
using namespace json;
|
||||
using namespace seastar::httpd;
|
||||
|
||||
void set_authorization_cache(http_context& ctx, routes& r, sharded<auth::service> &auth_service) {
|
||||
httpd::authorization_cache_json::authorization_cache_reset.set(r, [&auth_service] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
httpd::authorization_cache_json::authorization_cache_reset.set(r, [&auth_service] (std::unique_ptr<request> req) -> future<json::json_return_type> {
|
||||
co_await auth_service.invoke_on_all([] (auth::service& auth) -> future<> {
|
||||
auth.reset_authorization_cache();
|
||||
return make_ready_future<>();
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_authorization_cache(http_context& ctx, httpd::routes& r, sharded<auth::service> &auth_service);
|
||||
void unset_authorization_cache(http_context& ctx, httpd::routes& r);
|
||||
void set_authorization_cache(http_context& ctx, routes& r, sharded<auth::service> &auth_service);
|
||||
void unset_authorization_cache(http_context& ctx, routes& r);
|
||||
|
||||
}
|
||||
|
||||
@@ -12,128 +12,127 @@
|
||||
|
||||
namespace api {
|
||||
using namespace json;
|
||||
using namespace seastar::httpd;
|
||||
namespace cs = httpd::cache_service_json;
|
||||
|
||||
void set_cache_service(http_context& ctx, routes& r) {
|
||||
cs::get_row_cache_save_period_in_seconds.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::get_row_cache_save_period_in_seconds.set(r, [](std::unique_ptr<request> req) {
|
||||
// We never save the cache
|
||||
// Origin uses 0 for never
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::set_row_cache_save_period_in_seconds.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::set_row_cache_save_period_in_seconds.set(r, [](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
auto period = req->get_query_param("period");
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
cs::get_key_cache_save_period_in_seconds.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::get_key_cache_save_period_in_seconds.set(r, [](std::unique_ptr<request> req) {
|
||||
// We never save the cache
|
||||
// Origin uses 0 for never
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::set_key_cache_save_period_in_seconds.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::set_key_cache_save_period_in_seconds.set(r, [](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
auto period = req->get_query_param("period");
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
cs::get_counter_cache_save_period_in_seconds.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::get_counter_cache_save_period_in_seconds.set(r, [](std::unique_ptr<request> req) {
|
||||
// We never save the cache
|
||||
// Origin uses 0 for never
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::set_counter_cache_save_period_in_seconds.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::set_counter_cache_save_period_in_seconds.set(r, [](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
auto ccspis = req->get_query_param("ccspis");
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
cs::get_row_cache_keys_to_save.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::get_row_cache_keys_to_save.set(r, [](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::set_row_cache_keys_to_save.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::set_row_cache_keys_to_save.set(r, [](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
auto rckts = req->get_query_param("rckts");
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
cs::get_key_cache_keys_to_save.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::get_key_cache_keys_to_save.set(r, [](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::set_key_cache_keys_to_save.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::set_key_cache_keys_to_save.set(r, [](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
auto kckts = req->get_query_param("kckts");
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
cs::get_counter_cache_keys_to_save.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::get_counter_cache_keys_to_save.set(r, [](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::set_counter_cache_keys_to_save.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::set_counter_cache_keys_to_save.set(r, [](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
auto cckts = req->get_query_param("cckts");
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
cs::invalidate_key_cache.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::invalidate_key_cache.set(r, [](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
cs::invalidate_counter_cache.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::invalidate_counter_cache.set(r, [](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
cs::set_row_cache_capacity_in_mb.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::set_row_cache_capacity_in_mb.set(r, [](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
auto capacity = req->get_query_param("capacity");
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
cs::set_key_cache_capacity_in_mb.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::set_key_cache_capacity_in_mb.set(r, [](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
auto period = req->get_query_param("period");
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
cs::set_counter_cache_capacity_in_mb.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::set_counter_cache_capacity_in_mb.set(r, [](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
auto capacity = req->get_query_param("capacity");
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
cs::save_caches.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cs::save_caches.set(r, [](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
cs::get_key_capacity.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cs::get_key_capacity.set(r, [] (std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
// FIXME
|
||||
// we don't support keys cache,
|
||||
@@ -141,7 +140,7 @@ void set_cache_service(http_context& ctx, routes& r) {
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::get_key_hits.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cs::get_key_hits.set(r, [] (std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
// FIXME
|
||||
// we don't support keys cache,
|
||||
@@ -149,7 +148,7 @@ void set_cache_service(http_context& ctx, routes& r) {
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::get_key_requests.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cs::get_key_requests.set(r, [] (std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
// FIXME
|
||||
// we don't support keys cache,
|
||||
@@ -157,7 +156,7 @@ void set_cache_service(http_context& ctx, routes& r) {
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::get_key_hit_rate.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cs::get_key_hit_rate.set(r, [] (std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
// FIXME
|
||||
// we don't support keys cache,
|
||||
@@ -165,21 +164,21 @@ void set_cache_service(http_context& ctx, routes& r) {
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::get_key_hits_moving_avrage.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cs::get_key_hits_moving_avrage.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
// FIXME
|
||||
// See above
|
||||
return make_ready_future<json::json_return_type>(meter_to_json(utils::rate_moving_average()));
|
||||
});
|
||||
|
||||
cs::get_key_requests_moving_avrage.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cs::get_key_requests_moving_avrage.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
// FIXME
|
||||
// See above
|
||||
return make_ready_future<json::json_return_type>(meter_to_json(utils::rate_moving_average()));
|
||||
});
|
||||
|
||||
cs::get_key_size.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cs::get_key_size.set(r, [] (std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
// FIXME
|
||||
// we don't support keys cache,
|
||||
@@ -187,7 +186,7 @@ void set_cache_service(http_context& ctx, routes& r) {
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::get_key_entries.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cs::get_key_entries.set(r, [] (std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
// FIXME
|
||||
// we don't support keys cache,
|
||||
@@ -195,7 +194,7 @@ void set_cache_service(http_context& ctx, routes& r) {
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::get_row_capacity.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cs::get_row_capacity.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return ctx.db.map_reduce0([](replica::database& db) -> uint64_t {
|
||||
return db.row_cache_tracker().region().occupancy().used_space();
|
||||
}, uint64_t(0), std::plus<uint64_t>()).then([](const int64_t& res) {
|
||||
@@ -203,26 +202,26 @@ void set_cache_service(http_context& ctx, routes& r) {
|
||||
});
|
||||
});
|
||||
|
||||
cs::get_row_hits.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cs::get_row_hits.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, uint64_t(0), [](const replica::column_family& cf) {
|
||||
return cf.get_row_cache().stats().hits.count();
|
||||
}, std::plus<uint64_t>());
|
||||
});
|
||||
|
||||
cs::get_row_requests.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cs::get_row_requests.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, uint64_t(0), [](const replica::column_family& cf) {
|
||||
return cf.get_row_cache().stats().hits.count() + cf.get_row_cache().stats().misses.count();
|
||||
}, std::plus<uint64_t>());
|
||||
});
|
||||
|
||||
cs::get_row_hit_rate.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cs::get_row_hit_rate.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, ratio_holder(), [](const replica::column_family& cf) {
|
||||
return ratio_holder(cf.get_row_cache().stats().hits.count() + cf.get_row_cache().stats().misses.count(),
|
||||
cf.get_row_cache().stats().hits.count());
|
||||
}, std::plus<ratio_holder>());
|
||||
});
|
||||
|
||||
cs::get_row_hits_moving_avrage.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cs::get_row_hits_moving_avrage.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf_raw(ctx, utils::rate_moving_average(), [](const replica::column_family& cf) {
|
||||
return cf.get_row_cache().stats().hits.rate();
|
||||
}, std::plus<utils::rate_moving_average>()).then([](const utils::rate_moving_average& m) {
|
||||
@@ -230,7 +229,7 @@ void set_cache_service(http_context& ctx, routes& r) {
|
||||
});
|
||||
});
|
||||
|
||||
cs::get_row_requests_moving_avrage.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cs::get_row_requests_moving_avrage.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf_raw(ctx, utils::rate_moving_average(), [](const replica::column_family& cf) {
|
||||
return cf.get_row_cache().stats().hits.rate() + cf.get_row_cache().stats().misses.rate();
|
||||
}, std::plus<utils::rate_moving_average>()).then([](const utils::rate_moving_average& m) {
|
||||
@@ -238,7 +237,7 @@ void set_cache_service(http_context& ctx, routes& r) {
|
||||
});
|
||||
});
|
||||
|
||||
cs::get_row_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cs::get_row_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
// In origin row size is the weighted size.
|
||||
// We currently do not support weights, so we use num entries instead
|
||||
return ctx.db.map_reduce0([](replica::database& db) -> uint64_t {
|
||||
@@ -248,7 +247,7 @@ void set_cache_service(http_context& ctx, routes& r) {
|
||||
});
|
||||
});
|
||||
|
||||
cs::get_row_entries.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cs::get_row_entries.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return ctx.db.map_reduce0([](replica::database& db) -> uint64_t {
|
||||
return db.row_cache_tracker().partitions();
|
||||
}, uint64_t(0), std::plus<uint64_t>()).then([](const int64_t& res) {
|
||||
@@ -256,7 +255,7 @@ void set_cache_service(http_context& ctx, routes& r) {
|
||||
});
|
||||
});
|
||||
|
||||
cs::get_counter_capacity.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cs::get_counter_capacity.set(r, [] (std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
// FIXME
|
||||
// we don't support counter cache,
|
||||
@@ -264,7 +263,7 @@ void set_cache_service(http_context& ctx, routes& r) {
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::get_counter_hits.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cs::get_counter_hits.set(r, [] (std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
// FIXME
|
||||
// we don't support counter cache,
|
||||
@@ -272,7 +271,7 @@ void set_cache_service(http_context& ctx, routes& r) {
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::get_counter_requests.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cs::get_counter_requests.set(r, [] (std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
// FIXME
|
||||
// we don't support counter cache,
|
||||
@@ -280,7 +279,7 @@ void set_cache_service(http_context& ctx, routes& r) {
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::get_counter_hit_rate.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cs::get_counter_hit_rate.set(r, [] (std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
// FIXME
|
||||
// we don't support counter cache,
|
||||
@@ -288,21 +287,21 @@ void set_cache_service(http_context& ctx, routes& r) {
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::get_counter_hits_moving_avrage.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cs::get_counter_hits_moving_avrage.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
// FIXME
|
||||
// See above
|
||||
return make_ready_future<json::json_return_type>(meter_to_json(utils::rate_moving_average()));
|
||||
});
|
||||
|
||||
cs::get_counter_requests_moving_avrage.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cs::get_counter_requests_moving_avrage.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
// FIXME
|
||||
// See above
|
||||
return make_ready_future<json::json_return_type>(meter_to_json(utils::rate_moving_average()));
|
||||
});
|
||||
|
||||
cs::get_counter_size.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cs::get_counter_size.set(r, [] (std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
// FIXME
|
||||
// we don't support counter cache,
|
||||
@@ -310,7 +309,7 @@ void set_cache_service(http_context& ctx, routes& r) {
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cs::get_counter_entries.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cs::get_counter_entries.set(r, [] (std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
// FIXME
|
||||
// we don't support counter cache,
|
||||
|
||||
@@ -12,6 +12,6 @@
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_cache_service(http_context& ctx, httpd::routes& r);
|
||||
void set_cache_service(http_context& ctx, routes& r);
|
||||
|
||||
}
|
||||
|
||||
@@ -52,7 +52,7 @@ static const char* str_to_regex(const sstring& v) {
|
||||
}
|
||||
|
||||
void set_collectd(http_context& ctx, routes& r) {
|
||||
cd::get_collectd.set(r, [](std::unique_ptr<request> req) {
|
||||
cd::get_collectd.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
|
||||
auto id = ::make_shared<scollectd::type_instance_id>(req->param["pluginid"],
|
||||
req->get_query_param("instance"), req->get_query_param("type"),
|
||||
|
||||
@@ -12,6 +12,6 @@
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_collectd(http_context& ctx, httpd::routes& r);
|
||||
void set_collectd(http_context& ctx, routes& r);
|
||||
|
||||
}
|
||||
|
||||
@@ -14,10 +14,9 @@
|
||||
#include "sstables/metadata_collector.hh"
|
||||
#include "utils/estimated_histogram.hh"
|
||||
#include <algorithm>
|
||||
#include "db/system_keyspace.hh"
|
||||
#include "db/system_keyspace_view_types.hh"
|
||||
#include "db/data_listeners.hh"
|
||||
#include "storage_service.hh"
|
||||
#include "compaction/compaction_manager.hh"
|
||||
#include "unimplemented.hh"
|
||||
|
||||
extern logging::logger apilog;
|
||||
@@ -25,6 +24,7 @@ extern logging::logger apilog;
|
||||
namespace api {
|
||||
using namespace httpd;
|
||||
|
||||
using namespace std;
|
||||
using namespace json;
|
||||
namespace cf = httpd::column_family_json;
|
||||
|
||||
@@ -43,7 +43,7 @@ std::tuple<sstring, sstring> parse_fully_qualified_cf_name(sstring name) {
|
||||
return std::make_tuple(name.substr(0, pos), name.substr(end));
|
||||
}
|
||||
|
||||
table_id get_uuid(const sstring& ks, const sstring& cf, const replica::database& db) {
|
||||
const utils::UUID& get_uuid(const sstring& ks, const sstring& cf, const replica::database& db) {
|
||||
try {
|
||||
return db.find_uuid(ks, cf);
|
||||
} catch (replica::no_such_column_family& e) {
|
||||
@@ -51,12 +51,12 @@ table_id get_uuid(const sstring& ks, const sstring& cf, const replica::database&
|
||||
}
|
||||
}
|
||||
|
||||
table_id get_uuid(const sstring& name, const replica::database& db) {
|
||||
const utils::UUID& get_uuid(const sstring& name, const replica::database& db) {
|
||||
auto [ks, cf] = parse_fully_qualified_cf_name(name);
|
||||
return get_uuid(ks, cf, db);
|
||||
}
|
||||
|
||||
future<> foreach_column_family(http_context& ctx, const sstring& name, std::function<void(replica::column_family&)> f) {
|
||||
future<> foreach_column_family(http_context& ctx, const sstring& name, function<void(replica::column_family&)> f) {
|
||||
auto uuid = get_uuid(name, ctx.db.local());
|
||||
|
||||
return ctx.db.invoke_on_all([f, uuid](replica::database& db) {
|
||||
@@ -110,7 +110,7 @@ static future<json::json_return_type> get_cf_stats_count(http_context& ctx,
|
||||
|
||||
static future<json::json_return_type> get_cf_histogram(http_context& ctx, const sstring& name,
|
||||
utils::timed_rate_moving_average_and_histogram replica::column_family_stats::*f) {
|
||||
auto uuid = get_uuid(name, ctx.db.local());
|
||||
utils::UUID uuid = get_uuid(name, ctx.db.local());
|
||||
return ctx.db.map_reduce0([f, uuid](const replica::database& p) {
|
||||
return (p.find_column_family(uuid).get_stats().*f).hist;},
|
||||
utils::ihistogram(),
|
||||
@@ -122,7 +122,7 @@ static future<json::json_return_type> get_cf_histogram(http_context& ctx, const
|
||||
|
||||
static future<json::json_return_type> get_cf_histogram(http_context& ctx, const sstring& name,
|
||||
utils::timed_rate_moving_average_summary_and_histogram replica::column_family_stats::*f) {
|
||||
auto uuid = get_uuid(name, ctx.db.local());
|
||||
utils::UUID uuid = get_uuid(name, ctx.db.local());
|
||||
return ctx.db.map_reduce0([f, uuid](const replica::database& p) {
|
||||
return (p.find_column_family(uuid).get_stats().*f).hist;},
|
||||
utils::ihistogram(),
|
||||
@@ -135,9 +135,9 @@ static future<json::json_return_type> get_cf_histogram(http_context& ctx, const
|
||||
static future<json::json_return_type> get_cf_histogram(http_context& ctx, utils::timed_rate_moving_average_summary_and_histogram replica::column_family_stats::*f) {
|
||||
std::function<utils::ihistogram(const replica::database&)> fun = [f] (const replica::database& db) {
|
||||
utils::ihistogram res;
|
||||
db.get_tables_metadata().for_each_table([&] (table_id, lw_shared_ptr<replica::table> table) mutable {
|
||||
res += (table->get_stats().*f).hist;
|
||||
});
|
||||
for (auto i : db.get_column_families()) {
|
||||
res += (i.second->get_stats().*f).hist;
|
||||
}
|
||||
return res;
|
||||
};
|
||||
return ctx.db.map(fun).then([](const std::vector<utils::ihistogram> &res) {
|
||||
@@ -149,7 +149,7 @@ static future<json::json_return_type> get_cf_histogram(http_context& ctx, utils:
|
||||
|
||||
static future<json::json_return_type> get_cf_rate_and_histogram(http_context& ctx, const sstring& name,
|
||||
utils::timed_rate_moving_average_summary_and_histogram replica::column_family_stats::*f) {
|
||||
auto uuid = get_uuid(name, ctx.db.local());
|
||||
utils::UUID uuid = get_uuid(name, ctx.db.local());
|
||||
return ctx.db.map_reduce0([f, uuid](const replica::database& p) {
|
||||
return (p.find_column_family(uuid).get_stats().*f).rate();},
|
||||
utils::rate_moving_average_and_histogram(),
|
||||
@@ -162,9 +162,9 @@ static future<json::json_return_type> get_cf_rate_and_histogram(http_context& c
|
||||
static future<json::json_return_type> get_cf_rate_and_histogram(http_context& ctx, utils::timed_rate_moving_average_summary_and_histogram replica::column_family_stats::*f) {
|
||||
std::function<utils::rate_moving_average_and_histogram(const replica::database&)> fun = [f] (const replica::database& db) {
|
||||
utils::rate_moving_average_and_histogram res;
|
||||
db.get_tables_metadata().for_each_table([&] (table_id, lw_shared_ptr<replica::table> table) {
|
||||
res += (table->get_stats().*f).rate();
|
||||
});
|
||||
for (auto i : db.get_column_families()) {
|
||||
res += (i.second->get_stats().*f).rate();
|
||||
}
|
||||
return res;
|
||||
};
|
||||
return ctx.db.map(fun).then([](const std::vector<utils::rate_moving_average_and_histogram> &res) {
|
||||
@@ -303,44 +303,44 @@ ratio_holder filter_recent_false_positive_as_ratio_holder(const sstables::shared
|
||||
return ratio_holder(f + sst->filter_get_recent_true_positive(), f);
|
||||
}
|
||||
|
||||
void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace>& sys_ks) {
|
||||
void set_column_family(http_context& ctx, routes& r) {
|
||||
cf::get_column_family_name.set(r, [&ctx] (const_req req){
|
||||
std::vector<sstring> res;
|
||||
ctx.db.local().get_tables_metadata().for_each_table_id([&] (const std::pair<sstring, sstring>& kscf, table_id) {
|
||||
res.push_back(kscf.first + ":" + kscf.second);
|
||||
});
|
||||
vector<sstring> res;
|
||||
for (auto i: ctx.db.local().get_column_families_mapping()) {
|
||||
res.push_back(i.first.first + ":" + i.first.second);
|
||||
}
|
||||
return res;
|
||||
});
|
||||
|
||||
cf::get_column_family.set(r, [&ctx] (std::unique_ptr<http::request> req){
|
||||
std::list<cf::column_family_info> res;
|
||||
ctx.db.local().get_tables_metadata().for_each_table_id([&] (const std::pair<sstring, sstring>& kscf, table_id) {
|
||||
cf::get_column_family.set(r, [&ctx] (std::unique_ptr<request> req){
|
||||
std::list<cf::column_family_info> res;
|
||||
for (auto i: ctx.db.local().get_column_families_mapping()) {
|
||||
cf::column_family_info info;
|
||||
info.ks = kscf.first;
|
||||
info.cf = kscf.second;
|
||||
info.ks = i.first.first;
|
||||
info.cf = i.first.second;
|
||||
info.type = "ColumnFamilies";
|
||||
res.push_back(info);
|
||||
});
|
||||
}
|
||||
return make_ready_future<json::json_return_type>(json::stream_range_as_array(std::move(res), std::identity()));
|
||||
});
|
||||
|
||||
cf::get_column_family_name_keyspace.set(r, [&ctx] (const_req req){
|
||||
std::vector<sstring> res;
|
||||
vector<sstring> res;
|
||||
for (auto i = ctx.db.local().get_keyspaces().cbegin(); i!= ctx.db.local().get_keyspaces().cend(); i++) {
|
||||
res.push_back(i->first);
|
||||
}
|
||||
return res;
|
||||
});
|
||||
|
||||
cf::get_memtable_columns_count.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_memtable_columns_count.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], uint64_t{0}, [](replica::column_family& cf) {
|
||||
return boost::accumulate(cf.active_memtables() | boost::adaptors::transformed(std::mem_fn(&replica::memtable::partition_count)), uint64_t(0));
|
||||
return cf.active_memtable().partition_count();
|
||||
}, std::plus<>());
|
||||
});
|
||||
|
||||
cf::get_all_memtable_columns_count.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_memtable_columns_count.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, uint64_t{0}, [](replica::column_family& cf) {
|
||||
return boost::accumulate(cf.active_memtables() | boost::adaptors::transformed(std::mem_fn(&replica::memtable::partition_count)), uint64_t(0));
|
||||
return cf.active_memtable().partition_count();
|
||||
}, std::plus<>());
|
||||
});
|
||||
|
||||
@@ -352,35 +352,27 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
return 0;
|
||||
});
|
||||
|
||||
cf::get_memtable_off_heap_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_memtable_off_heap_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](replica::column_family& cf) {
|
||||
return boost::accumulate(cf.active_memtables() | boost::adaptors::transformed([] (replica::memtable* active_memtable) {
|
||||
return active_memtable->region().occupancy().total_space();
|
||||
}), uint64_t(0));
|
||||
return cf.active_memtable().region().occupancy().total_space();
|
||||
}, std::plus<int64_t>());
|
||||
});
|
||||
|
||||
cf::get_all_memtable_off_heap_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_memtable_off_heap_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, int64_t(0), [](replica::column_family& cf) {
|
||||
return boost::accumulate(cf.active_memtables() | boost::adaptors::transformed([] (replica::memtable* active_memtable) {
|
||||
return active_memtable->region().occupancy().total_space();
|
||||
}), uint64_t(0));
|
||||
return cf.active_memtable().region().occupancy().total_space();
|
||||
}, std::plus<int64_t>());
|
||||
});
|
||||
|
||||
cf::get_memtable_live_data_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_memtable_live_data_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](replica::column_family& cf) {
|
||||
return boost::accumulate(cf.active_memtables() | boost::adaptors::transformed([] (replica::memtable* active_memtable) {
|
||||
return active_memtable->region().occupancy().used_space();
|
||||
}), uint64_t(0));
|
||||
return cf.active_memtable().region().occupancy().used_space();
|
||||
}, std::plus<int64_t>());
|
||||
});
|
||||
|
||||
cf::get_all_memtable_live_data_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_memtable_live_data_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, int64_t(0), [](replica::column_family& cf) {
|
||||
return boost::accumulate(cf.active_memtables() | boost::adaptors::transformed([] (replica::memtable* active_memtable) {
|
||||
return active_memtable->region().occupancy().used_space();
|
||||
}), uint64_t(0));
|
||||
return cf.active_memtable().region().occupancy().used_space();
|
||||
}, std::plus<int64_t>());
|
||||
});
|
||||
|
||||
@@ -392,48 +384,46 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
return 0;
|
||||
});
|
||||
|
||||
cf::get_cf_all_memtables_off_heap_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_cf_all_memtables_off_heap_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
warn(unimplemented::cause::INDEXES);
|
||||
return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](replica::column_family& cf) {
|
||||
return cf.occupancy().total_space();
|
||||
}, std::plus<int64_t>());
|
||||
});
|
||||
|
||||
cf::get_all_cf_all_memtables_off_heap_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_cf_all_memtables_off_heap_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
warn(unimplemented::cause::INDEXES);
|
||||
return ctx.db.map_reduce0([](const replica::database& db){
|
||||
return db.dirty_memory_region_group().real_memory_used();
|
||||
return db.dirty_memory_region_group().memory_used();
|
||||
}, int64_t(0), std::plus<int64_t>()).then([](int res) {
|
||||
return make_ready_future<json::json_return_type>(res);
|
||||
});
|
||||
});
|
||||
|
||||
cf::get_cf_all_memtables_live_data_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_cf_all_memtables_live_data_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
warn(unimplemented::cause::INDEXES);
|
||||
return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](replica::column_family& cf) {
|
||||
return cf.occupancy().used_space();
|
||||
}, std::plus<int64_t>());
|
||||
});
|
||||
|
||||
cf::get_all_cf_all_memtables_live_data_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_cf_all_memtables_live_data_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
warn(unimplemented::cause::INDEXES);
|
||||
return map_reduce_cf(ctx, int64_t(0), [](replica::column_family& cf) {
|
||||
return boost::accumulate(cf.active_memtables() | boost::adaptors::transformed([] (replica::memtable* active_memtable) {
|
||||
return active_memtable->region().occupancy().used_space();
|
||||
}), uint64_t(0));
|
||||
return cf.active_memtable().region().occupancy().used_space();
|
||||
}, std::plus<int64_t>());
|
||||
});
|
||||
|
||||
cf::get_memtable_switch_count.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_memtable_switch_count.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_stats(ctx,req->param["name"] ,&replica::column_family_stats::memtable_switch_count);
|
||||
});
|
||||
|
||||
cf::get_all_memtable_switch_count.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_memtable_switch_count.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_stats(ctx, &replica::column_family_stats::memtable_switch_count);
|
||||
});
|
||||
|
||||
// FIXME: this refers to partitions, not rows.
|
||||
cf::get_estimated_row_size_histogram.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_estimated_row_size_histogram.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], utils::estimated_histogram(0), [](replica::column_family& cf) {
|
||||
utils::estimated_histogram res(0);
|
||||
for (auto sstables = cf.get_sstables(); auto& i : *sstables) {
|
||||
@@ -445,7 +435,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
});
|
||||
|
||||
// FIXME: this refers to partitions, not rows.
|
||||
cf::get_estimated_row_count.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_estimated_row_count.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](replica::column_family& cf) {
|
||||
uint64_t res = 0;
|
||||
for (auto sstables = cf.get_sstables(); auto& i : *sstables) {
|
||||
@@ -456,7 +446,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
std::plus<uint64_t>());
|
||||
});
|
||||
|
||||
cf::get_estimated_column_count_histogram.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_estimated_column_count_histogram.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], utils::estimated_histogram(0), [](replica::column_family& cf) {
|
||||
utils::estimated_histogram res(0);
|
||||
for (auto sstables = cf.get_sstables(); auto& i : *sstables) {
|
||||
@@ -467,149 +457,149 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
utils::estimated_histogram_merge, utils_json::estimated_histogram());
|
||||
});
|
||||
|
||||
cf::get_all_compression_ratio.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_compression_ratio.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cf::get_pending_flushes.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_pending_flushes.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_stats(ctx,req->param["name"] ,&replica::column_family_stats::pending_flushes);
|
||||
});
|
||||
|
||||
cf::get_all_pending_flushes.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_pending_flushes.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_stats(ctx, &replica::column_family_stats::pending_flushes);
|
||||
});
|
||||
|
||||
cf::get_read.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_read.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_stats_count(ctx,req->param["name"] ,&replica::column_family_stats::reads);
|
||||
});
|
||||
|
||||
cf::get_all_read.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_read.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_stats_count(ctx, &replica::column_family_stats::reads);
|
||||
});
|
||||
|
||||
cf::get_write.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_write.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_stats_count(ctx, req->param["name"] ,&replica::column_family_stats::writes);
|
||||
});
|
||||
|
||||
cf::get_all_write.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_write.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_stats_count(ctx, &replica::column_family_stats::writes);
|
||||
});
|
||||
|
||||
cf::get_read_latency_histogram_depricated.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_read_latency_histogram_depricated.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_histogram(ctx, req->param["name"], &replica::column_family_stats::reads);
|
||||
});
|
||||
|
||||
cf::get_read_latency_histogram.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_read_latency_histogram.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_rate_and_histogram(ctx, req->param["name"], &replica::column_family_stats::reads);
|
||||
});
|
||||
|
||||
cf::get_read_latency.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_read_latency.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_stats_sum(ctx,req->param["name"] ,&replica::column_family_stats::reads);
|
||||
});
|
||||
|
||||
cf::get_write_latency.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_write_latency.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_stats_sum(ctx, req->param["name"] ,&replica::column_family_stats::writes);
|
||||
});
|
||||
|
||||
cf::get_all_read_latency_histogram_depricated.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_read_latency_histogram_depricated.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_histogram(ctx, &replica::column_family_stats::writes);
|
||||
});
|
||||
|
||||
cf::get_all_read_latency_histogram.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_read_latency_histogram.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_rate_and_histogram(ctx, &replica::column_family_stats::writes);
|
||||
});
|
||||
|
||||
cf::get_write_latency_histogram_depricated.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_write_latency_histogram_depricated.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_histogram(ctx, req->param["name"], &replica::column_family_stats::writes);
|
||||
});
|
||||
|
||||
cf::get_write_latency_histogram.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_write_latency_histogram.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_rate_and_histogram(ctx, req->param["name"], &replica::column_family_stats::writes);
|
||||
});
|
||||
|
||||
cf::get_all_write_latency_histogram_depricated.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_write_latency_histogram_depricated.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_histogram(ctx, &replica::column_family_stats::writes);
|
||||
});
|
||||
|
||||
cf::get_all_write_latency_histogram.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_write_latency_histogram.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_rate_and_histogram(ctx, &replica::column_family_stats::writes);
|
||||
});
|
||||
|
||||
cf::get_pending_compactions.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_pending_compactions.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](replica::column_family& cf) {
|
||||
return cf.estimate_pending_compactions();
|
||||
return cf.get_compaction_strategy().estimated_pending_compactions(cf.as_table_state());
|
||||
}, std::plus<int64_t>());
|
||||
});
|
||||
|
||||
cf::get_all_pending_compactions.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_pending_compactions.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, int64_t(0), [](replica::column_family& cf) {
|
||||
return cf.estimate_pending_compactions();
|
||||
return cf.get_compaction_strategy().estimated_pending_compactions(cf.as_table_state());
|
||||
}, std::plus<int64_t>());
|
||||
});
|
||||
|
||||
cf::get_live_ss_table_count.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_live_ss_table_count.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_stats(ctx, req->param["name"], &replica::column_family_stats::live_sstable_count);
|
||||
});
|
||||
|
||||
cf::get_all_live_ss_table_count.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_live_ss_table_count.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_stats(ctx, &replica::column_family_stats::live_sstable_count);
|
||||
});
|
||||
|
||||
cf::get_unleveled_sstables.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_unleveled_sstables.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_unleveled_sstables(ctx, req->param["name"]);
|
||||
});
|
||||
|
||||
cf::get_live_disk_space_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_live_disk_space_used.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return sum_sstable(ctx, req->param["name"], false);
|
||||
});
|
||||
|
||||
cf::get_all_live_disk_space_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_live_disk_space_used.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return sum_sstable(ctx, false);
|
||||
});
|
||||
|
||||
cf::get_total_disk_space_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_total_disk_space_used.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return sum_sstable(ctx, req->param["name"], true);
|
||||
});
|
||||
|
||||
cf::get_all_total_disk_space_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_total_disk_space_used.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return sum_sstable(ctx, true);
|
||||
});
|
||||
|
||||
// FIXME: this refers to partitions, not rows.
|
||||
cf::get_min_row_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_min_row_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], INT64_MAX, min_partition_size, min_int64);
|
||||
});
|
||||
|
||||
// FIXME: this refers to partitions, not rows.
|
||||
cf::get_all_min_row_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_min_row_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, INT64_MAX, min_partition_size, min_int64);
|
||||
});
|
||||
|
||||
// FIXME: this refers to partitions, not rows.
|
||||
cf::get_max_row_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_max_row_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], int64_t(0), max_partition_size, max_int64);
|
||||
});
|
||||
|
||||
// FIXME: this refers to partitions, not rows.
|
||||
cf::get_all_max_row_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_max_row_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, int64_t(0), max_partition_size, max_int64);
|
||||
});
|
||||
|
||||
// FIXME: this refers to partitions, not rows.
|
||||
cf::get_mean_row_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_mean_row_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
// Cassandra 3.x mean values are truncated as integrals.
|
||||
return map_reduce_cf(ctx, req->param["name"], integral_ratio_holder(), mean_partition_size, std::plus<integral_ratio_holder>());
|
||||
});
|
||||
|
||||
// FIXME: this refers to partitions, not rows.
|
||||
cf::get_all_mean_row_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_mean_row_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
// Cassandra 3.x mean values are truncated as integrals.
|
||||
return map_reduce_cf(ctx, integral_ratio_holder(), mean_partition_size, std::plus<integral_ratio_holder>());
|
||||
});
|
||||
|
||||
cf::get_bloom_filter_false_positives.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_bloom_filter_false_positives.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], uint64_t(0), [] (replica::column_family& cf) {
|
||||
auto sstables = cf.get_sstables();
|
||||
return std::accumulate(sstables->begin(), sstables->end(), uint64_t(0), [](uint64_t s, auto& sst) {
|
||||
@@ -618,7 +608,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
}, std::plus<uint64_t>());
|
||||
});
|
||||
|
||||
cf::get_all_bloom_filter_false_positives.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_bloom_filter_false_positives.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, uint64_t(0), [] (replica::column_family& cf) {
|
||||
auto sstables = cf.get_sstables();
|
||||
return std::accumulate(sstables->begin(), sstables->end(), uint64_t(0), [](uint64_t s, auto& sst) {
|
||||
@@ -627,7 +617,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
}, std::plus<uint64_t>());
|
||||
});
|
||||
|
||||
cf::get_recent_bloom_filter_false_positives.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_recent_bloom_filter_false_positives.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], uint64_t(0), [] (replica::column_family& cf) {
|
||||
auto sstables = cf.get_sstables();
|
||||
return std::accumulate(sstables->begin(), sstables->end(), uint64_t(0), [](uint64_t s, auto& sst) {
|
||||
@@ -636,7 +626,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
}, std::plus<uint64_t>());
|
||||
});
|
||||
|
||||
cf::get_all_recent_bloom_filter_false_positives.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_recent_bloom_filter_false_positives.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, uint64_t(0), [] (replica::column_family& cf) {
|
||||
auto sstables = cf.get_sstables();
|
||||
return std::accumulate(sstables->begin(), sstables->end(), uint64_t(0), [](uint64_t s, auto& sst) {
|
||||
@@ -645,31 +635,31 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
}, std::plus<uint64_t>());
|
||||
});
|
||||
|
||||
cf::get_bloom_filter_false_ratio.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_bloom_filter_false_ratio.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], ratio_holder(), [] (replica::column_family& cf) {
|
||||
return boost::accumulate(*cf.get_sstables() | boost::adaptors::transformed(filter_false_positive_as_ratio_holder), ratio_holder());
|
||||
}, std::plus<>());
|
||||
});
|
||||
|
||||
cf::get_all_bloom_filter_false_ratio.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_bloom_filter_false_ratio.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, ratio_holder(), [] (replica::column_family& cf) {
|
||||
return boost::accumulate(*cf.get_sstables() | boost::adaptors::transformed(filter_false_positive_as_ratio_holder), ratio_holder());
|
||||
}, std::plus<>());
|
||||
});
|
||||
|
||||
cf::get_recent_bloom_filter_false_ratio.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_recent_bloom_filter_false_ratio.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], ratio_holder(), [] (replica::column_family& cf) {
|
||||
return boost::accumulate(*cf.get_sstables() | boost::adaptors::transformed(filter_recent_false_positive_as_ratio_holder), ratio_holder());
|
||||
}, std::plus<>());
|
||||
});
|
||||
|
||||
cf::get_all_recent_bloom_filter_false_ratio.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_recent_bloom_filter_false_ratio.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, ratio_holder(), [] (replica::column_family& cf) {
|
||||
return boost::accumulate(*cf.get_sstables() | boost::adaptors::transformed(filter_recent_false_positive_as_ratio_holder), ratio_holder());
|
||||
}, std::plus<>());
|
||||
});
|
||||
|
||||
cf::get_bloom_filter_disk_space_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_bloom_filter_disk_space_used.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], uint64_t(0), [] (replica::column_family& cf) {
|
||||
auto sstables = cf.get_sstables();
|
||||
return std::accumulate(sstables->begin(), sstables->end(), uint64_t(0), [](uint64_t s, auto& sst) {
|
||||
@@ -678,7 +668,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
}, std::plus<uint64_t>());
|
||||
});
|
||||
|
||||
cf::get_all_bloom_filter_disk_space_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_bloom_filter_disk_space_used.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, uint64_t(0), [] (replica::column_family& cf) {
|
||||
auto sstables = cf.get_sstables();
|
||||
return std::accumulate(sstables->begin(), sstables->end(), uint64_t(0), [](uint64_t s, auto& sst) {
|
||||
@@ -687,7 +677,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
}, std::plus<uint64_t>());
|
||||
});
|
||||
|
||||
cf::get_bloom_filter_off_heap_memory_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_bloom_filter_off_heap_memory_used.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], uint64_t(0), [] (replica::column_family& cf) {
|
||||
auto sstables = cf.get_sstables();
|
||||
return std::accumulate(sstables->begin(), sstables->end(), uint64_t(0), [](uint64_t s, auto& sst) {
|
||||
@@ -696,7 +686,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
}, std::plus<uint64_t>());
|
||||
});
|
||||
|
||||
cf::get_all_bloom_filter_off_heap_memory_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_bloom_filter_off_heap_memory_used.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, uint64_t(0), [] (replica::column_family& cf) {
|
||||
auto sstables = cf.get_sstables();
|
||||
return std::accumulate(sstables->begin(), sstables->end(), uint64_t(0), [](uint64_t s, auto& sst) {
|
||||
@@ -705,7 +695,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
}, std::plus<uint64_t>());
|
||||
});
|
||||
|
||||
cf::get_index_summary_off_heap_memory_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_index_summary_off_heap_memory_used.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], uint64_t(0), [] (replica::column_family& cf) {
|
||||
auto sstables = cf.get_sstables();
|
||||
return std::accumulate(sstables->begin(), sstables->end(), uint64_t(0), [](uint64_t s, auto& sst) {
|
||||
@@ -714,7 +704,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
}, std::plus<uint64_t>());
|
||||
});
|
||||
|
||||
cf::get_all_index_summary_off_heap_memory_used.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_index_summary_off_heap_memory_used.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, uint64_t(0), [] (replica::column_family& cf) {
|
||||
auto sstables = cf.get_sstables();
|
||||
return std::accumulate(sstables->begin(), sstables->end(), uint64_t(0), [](uint64_t s, auto& sst) {
|
||||
@@ -723,7 +713,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
}, std::plus<uint64_t>());
|
||||
});
|
||||
|
||||
cf::get_compression_metadata_off_heap_memory_used.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cf::get_compression_metadata_off_heap_memory_used.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
// FIXME
|
||||
// We are missing the off heap memory calculation
|
||||
@@ -733,33 +723,33 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cf::get_all_compression_metadata_off_heap_memory_used.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_compression_metadata_off_heap_memory_used.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cf::get_speculative_retries.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cf::get_speculative_retries.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
//auto id = get_uuid(req->param["name"], ctx.db.local());
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cf::get_all_speculative_retries.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_speculative_retries.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cf::get_key_cache_hit_rate.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cf::get_key_cache_hit_rate.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
//auto id = get_uuid(req->param["name"], ctx.db.local());
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cf::get_true_snapshots_size.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_true_snapshots_size.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
auto uuid = get_uuid(req->param["name"], ctx.db.local());
|
||||
return ctx.db.local().find_column_family(uuid).get_snapshot_details().then([](
|
||||
const std::unordered_map<sstring, replica::column_family::snapshot_details>& sd) {
|
||||
@@ -771,26 +761,26 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
});
|
||||
});
|
||||
|
||||
cf::get_all_true_snapshots_size.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_true_snapshots_size.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cf::get_row_cache_hit_out_of_range.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cf::get_row_cache_hit_out_of_range.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
//auto id = get_uuid(req->param["name"], ctx.db.local());
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cf::get_all_row_cache_hit_out_of_range.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_row_cache_hit_out_of_range.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cf::get_row_cache_hit.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_row_cache_hit.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf_raw(ctx, req->param["name"], utils::rate_moving_average(), [](const replica::column_family& cf) {
|
||||
return cf.get_row_cache().stats().hits.rate();
|
||||
}, std::plus<utils::rate_moving_average>()).then([](const utils::rate_moving_average& m) {
|
||||
@@ -798,7 +788,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
});
|
||||
});
|
||||
|
||||
cf::get_all_row_cache_hit.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_row_cache_hit.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf_raw(ctx, utils::rate_moving_average(), [](const replica::column_family& cf) {
|
||||
return cf.get_row_cache().stats().hits.rate();
|
||||
}, std::plus<utils::rate_moving_average>()).then([](const utils::rate_moving_average& m) {
|
||||
@@ -806,7 +796,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
});
|
||||
});
|
||||
|
||||
cf::get_row_cache_miss.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_row_cache_miss.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf_raw(ctx, req->param["name"], utils::rate_moving_average(), [](const replica::column_family& cf) {
|
||||
return cf.get_row_cache().stats().misses.rate();
|
||||
}, std::plus<utils::rate_moving_average>()).then([](const utils::rate_moving_average& m) {
|
||||
@@ -814,7 +804,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
});
|
||||
});
|
||||
|
||||
cf::get_all_row_cache_miss.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_all_row_cache_miss.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf_raw(ctx, utils::rate_moving_average(), [](const replica::column_family& cf) {
|
||||
return cf.get_row_cache().stats().misses.rate();
|
||||
}, std::plus<utils::rate_moving_average>()).then([](const utils::rate_moving_average& m) {
|
||||
@@ -823,40 +813,40 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
|
||||
});
|
||||
|
||||
cf::get_cas_prepare.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_cas_prepare.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf_time_histogram(ctx, req->param["name"], [](const replica::column_family& cf) {
|
||||
return cf.get_stats().cas_prepare.histogram();
|
||||
});
|
||||
});
|
||||
|
||||
cf::get_cas_propose.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_cas_propose.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf_time_histogram(ctx, req->param["name"], [](const replica::column_family& cf) {
|
||||
return cf.get_stats().cas_accept.histogram();
|
||||
});
|
||||
});
|
||||
|
||||
cf::get_cas_commit.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_cas_commit.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf_time_histogram(ctx, req->param["name"], [](const replica::column_family& cf) {
|
||||
return cf.get_stats().cas_learn.histogram();
|
||||
});
|
||||
});
|
||||
|
||||
cf::get_sstables_per_read_histogram.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_sstables_per_read_histogram.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, req->param["name"], utils::estimated_histogram(0), [](replica::column_family& cf) {
|
||||
return cf.get_stats().estimated_sstable_per_read;
|
||||
},
|
||||
utils::estimated_histogram_merge, utils_json::estimated_histogram());
|
||||
});
|
||||
|
||||
cf::get_tombstone_scanned_histogram.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_tombstone_scanned_histogram.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_histogram(ctx, req->param["name"], &replica::column_family_stats::tombstone_scanned);
|
||||
});
|
||||
|
||||
cf::get_live_scanned_histogram.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::get_live_scanned_histogram.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cf_histogram(ctx, req->param["name"], &replica::column_family_stats::live_scanned);
|
||||
});
|
||||
|
||||
cf::get_col_update_time_delta_histogram.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cf::get_col_update_time_delta_histogram.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
//auto id = get_uuid(req->param["name"], ctx.db.local());
|
||||
@@ -865,13 +855,12 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
});
|
||||
|
||||
cf::get_auto_compaction.set(r, [&ctx] (const_req req) {
|
||||
auto uuid = get_uuid(req.param["name"], ctx.db.local());
|
||||
const utils::UUID& uuid = get_uuid(req.param["name"], ctx.db.local());
|
||||
replica::column_family& cf = ctx.db.local().find_column_family(uuid);
|
||||
return !cf.is_auto_compaction_disabled_by_user();
|
||||
});
|
||||
|
||||
cf::enable_auto_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) {
|
||||
apilog.info("column_family/enable_auto_compaction: name={}", req->param["name"]);
|
||||
cf::enable_auto_compaction.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return ctx.db.invoke_on(0, [&ctx, req = std::move(req)] (replica::database& db) {
|
||||
auto g = replica::database::autocompaction_toggle_guard(db);
|
||||
return foreach_column_family(ctx, req->param["name"], [](replica::column_family &cf) {
|
||||
@@ -882,8 +871,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
});
|
||||
});
|
||||
|
||||
cf::disable_auto_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) {
|
||||
apilog.info("column_family/disable_auto_compaction: name={}", req->param["name"]);
|
||||
cf::disable_auto_compaction.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return ctx.db.invoke_on(0, [&ctx, req = std::move(req)] (replica::database& db) {
|
||||
auto g = replica::database::autocompaction_toggle_guard(db);
|
||||
return foreach_column_family(ctx, req->param["name"], [](replica::column_family &cf) {
|
||||
@@ -894,35 +882,11 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
});
|
||||
});
|
||||
|
||||
cf::get_tombstone_gc.set(r, [&ctx] (const_req req) {
|
||||
auto uuid = get_uuid(req.param["name"], ctx.db.local());
|
||||
replica::table& t = ctx.db.local().find_column_family(uuid);
|
||||
return t.tombstone_gc_enabled();
|
||||
});
|
||||
|
||||
cf::enable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
|
||||
apilog.info("column_family/enable_tombstone_gc: name={}", req->param["name"]);
|
||||
return foreach_column_family(ctx, req->param["name"], [](replica::table& t) {
|
||||
t.set_tombstone_gc_enabled(true);
|
||||
}).then([] {
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
});
|
||||
|
||||
cf::disable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
|
||||
apilog.info("column_family/disable_tombstone_gc: name={}", req->param["name"]);
|
||||
return foreach_column_family(ctx, req->param["name"], [](replica::table& t) {
|
||||
t.set_tombstone_gc_enabled(false);
|
||||
}).then([] {
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
});
|
||||
|
||||
cf::get_built_indexes.set(r, [&ctx, &sys_ks](std::unique_ptr<http::request> req) {
|
||||
cf::get_built_indexes.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
auto ks_cf = parse_fully_qualified_cf_name(req->param["name"]);
|
||||
auto&& ks = std::get<0>(ks_cf);
|
||||
auto&& cf_name = std::get<1>(ks_cf);
|
||||
return sys_ks.local().load_view_build_progress().then([ks, cf_name, &ctx](const std::vector<db::system_keyspace_view_build_progress>& vb) mutable {
|
||||
return db::system_keyspace::load_view_build_progress().then([ks, cf_name, &ctx](const std::vector<db::system_keyspace_view_build_progress>& vb) mutable {
|
||||
std::set<sstring> vp;
|
||||
for (auto b : vb) {
|
||||
if (b.view.first == ks) {
|
||||
@@ -956,7 +920,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
return std::vector<sstring>();
|
||||
});
|
||||
|
||||
cf::get_compression_ratio.set(r, [&ctx](std::unique_ptr<http::request> req) {
|
||||
cf::get_compression_ratio.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
auto uuid = get_uuid(req->param["name"], ctx.db.local());
|
||||
|
||||
return ctx.db.map_reduce(sum_ratio<double>(), [uuid](replica::database& db) {
|
||||
@@ -967,21 +931,20 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
});
|
||||
});
|
||||
|
||||
cf::get_read_latency_estimated_histogram.set(r, [&ctx](std::unique_ptr<http::request> req) {
|
||||
cf::get_read_latency_estimated_histogram.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return map_reduce_cf_time_histogram(ctx, req->param["name"], [](const replica::column_family& cf) {
|
||||
return cf.get_stats().reads.histogram();
|
||||
});
|
||||
});
|
||||
|
||||
cf::get_write_latency_estimated_histogram.set(r, [&ctx](std::unique_ptr<http::request> req) {
|
||||
cf::get_write_latency_estimated_histogram.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return map_reduce_cf_time_histogram(ctx, req->param["name"], [](const replica::column_family& cf) {
|
||||
return cf.get_stats().writes.histogram();
|
||||
});
|
||||
});
|
||||
|
||||
cf::set_compaction_strategy_class.set(r, [&ctx](std::unique_ptr<http::request> req) {
|
||||
cf::set_compaction_strategy_class.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
sstring strategy = req->get_query_param("class_name");
|
||||
apilog.info("column_family/set_compaction_strategy_class: name={} strategy={}", req->param["name"], strategy);
|
||||
return foreach_column_family(ctx, req->param["name"], [strategy](replica::column_family& cf) {
|
||||
cf.set_compaction_strategy(sstables::compaction_strategy::type(strategy));
|
||||
}).then([] {
|
||||
@@ -993,19 +956,19 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
return ctx.db.local().find_column_family(get_uuid(req.param["name"], ctx.db.local())).get_compaction_strategy().name();
|
||||
});
|
||||
|
||||
cf::set_compression_parameters.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cf::set_compression_parameters.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
cf::set_crc_check_chance.set(r, [](std::unique_ptr<http::request> req) {
|
||||
cf::set_crc_check_chance.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
cf::get_sstable_count_per_level.set(r, [&ctx](std::unique_ptr<http::request> req) {
|
||||
cf::get_sstable_count_per_level.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return map_reduce_cf_raw(ctx, req->param["name"], std::vector<uint64_t>(), [](const replica::column_family& cf) {
|
||||
return cf.sstable_count_per_level();
|
||||
}, concat_sstable_count_per_level).then([](const std::vector<uint64_t>& res) {
|
||||
@@ -1013,16 +976,15 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
});
|
||||
});
|
||||
|
||||
cf::get_sstables_for_key.set(r, [&ctx](std::unique_ptr<http::request> req) {
|
||||
cf::get_sstables_for_key.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
auto key = req->get_query_param("key");
|
||||
auto uuid = get_uuid(req->param["name"], ctx.db.local());
|
||||
|
||||
return ctx.db.map_reduce0([key, uuid] (replica::database& db) -> future<std::unordered_set<sstring>> {
|
||||
auto sstables = co_await db.find_column_family(uuid).get_sstables_by_partition_key(key);
|
||||
co_return boost::copy_range<std::unordered_set<sstring>>(sstables | boost::adaptors::transformed([] (auto s) { return s->get_filename(); }));
|
||||
return ctx.db.map_reduce0([key, uuid] (replica::database& db) {
|
||||
return db.find_column_family(uuid).get_sstables_by_partition_key(key);
|
||||
}, std::unordered_set<sstring>(),
|
||||
[](std::unordered_set<sstring> a, std::unordered_set<sstring>&& b) mutable {
|
||||
a.merge(b);
|
||||
[](std::unordered_set<sstring> a, std::unordered_set<sstring>&& b) mutable {
|
||||
a.insert(b.begin(),b.end());
|
||||
return a;
|
||||
}).then([](const std::unordered_set<sstring>& res) {
|
||||
return make_ready_future<json::json_return_type>(container_to_vec(res));
|
||||
@@ -1030,7 +992,7 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
});
|
||||
|
||||
|
||||
cf::toppartitions.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cf::toppartitions.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
auto name = req->param["name"];
|
||||
auto [ks, cf] = parse_fully_qualified_cf_name(name);
|
||||
|
||||
@@ -1046,131 +1008,15 @@ void set_column_family(http_context& ctx, routes& r, sharded<db::system_keyspace
|
||||
});
|
||||
});
|
||||
|
||||
cf::force_major_compaction.set(r, [&ctx](std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
cf::force_major_compaction.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
if (req->get_query_param("split_output") != "") {
|
||||
fail(unimplemented::cause::API);
|
||||
}
|
||||
|
||||
apilog.info("column_family/force_major_compaction: name={}", req->param["name"]);
|
||||
auto [ks, cf] = parse_fully_qualified_cf_name(req->param["name"]);
|
||||
auto keyspace = validate_keyspace(ctx, ks);
|
||||
std::vector<table_info> table_infos = {table_info{
|
||||
.name = cf,
|
||||
.id = ctx.db.local().find_uuid(ks, cf)
|
||||
}};
|
||||
|
||||
auto& compaction_module = ctx.db.local().get_compaction_manager().get_task_manager_module();
|
||||
auto task = co_await compaction_module.make_and_start_task<major_keyspace_compaction_task_impl>({}, std::move(keyspace), ctx.db, std::move(table_infos));
|
||||
co_await task->done();
|
||||
co_return json_void();
|
||||
return foreach_column_family(ctx, req->param["name"], [](replica::column_family &cf) {
|
||||
return cf.compact_all_sstables();
|
||||
}).then([] {
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void unset_column_family(http_context& ctx, routes& r) {
|
||||
cf::get_column_family_name.unset(r);
|
||||
cf::get_column_family.unset(r);
|
||||
cf::get_column_family_name_keyspace.unset(r);
|
||||
cf::get_memtable_columns_count.unset(r);
|
||||
cf::get_all_memtable_columns_count.unset(r);
|
||||
cf::get_memtable_on_heap_size.unset(r);
|
||||
cf::get_all_memtable_on_heap_size.unset(r);
|
||||
cf::get_memtable_off_heap_size.unset(r);
|
||||
cf::get_all_memtable_off_heap_size.unset(r);
|
||||
cf::get_memtable_live_data_size.unset(r);
|
||||
cf::get_all_memtable_live_data_size.unset(r);
|
||||
cf::get_cf_all_memtables_on_heap_size.unset(r);
|
||||
cf::get_all_cf_all_memtables_on_heap_size.unset(r);
|
||||
cf::get_cf_all_memtables_off_heap_size.unset(r);
|
||||
cf::get_all_cf_all_memtables_off_heap_size.unset(r);
|
||||
cf::get_cf_all_memtables_live_data_size.unset(r);
|
||||
cf::get_all_cf_all_memtables_live_data_size.unset(r);
|
||||
cf::get_memtable_switch_count.unset(r);
|
||||
cf::get_all_memtable_switch_count.unset(r);
|
||||
cf::get_estimated_row_size_histogram.unset(r);
|
||||
cf::get_estimated_row_count.unset(r);
|
||||
cf::get_estimated_column_count_histogram.unset(r);
|
||||
cf::get_all_compression_ratio.unset(r);
|
||||
cf::get_pending_flushes.unset(r);
|
||||
cf::get_all_pending_flushes.unset(r);
|
||||
cf::get_read.unset(r);
|
||||
cf::get_all_read.unset(r);
|
||||
cf::get_write.unset(r);
|
||||
cf::get_all_write.unset(r);
|
||||
cf::get_read_latency_histogram_depricated.unset(r);
|
||||
cf::get_read_latency_histogram.unset(r);
|
||||
cf::get_read_latency.unset(r);
|
||||
cf::get_write_latency.unset(r);
|
||||
cf::get_all_read_latency_histogram_depricated.unset(r);
|
||||
cf::get_all_read_latency_histogram.unset(r);
|
||||
cf::get_write_latency_histogram_depricated.unset(r);
|
||||
cf::get_write_latency_histogram.unset(r);
|
||||
cf::get_all_write_latency_histogram_depricated.unset(r);
|
||||
cf::get_all_write_latency_histogram.unset(r);
|
||||
cf::get_pending_compactions.unset(r);
|
||||
cf::get_all_pending_compactions.unset(r);
|
||||
cf::get_live_ss_table_count.unset(r);
|
||||
cf::get_all_live_ss_table_count.unset(r);
|
||||
cf::get_unleveled_sstables.unset(r);
|
||||
cf::get_live_disk_space_used.unset(r);
|
||||
cf::get_all_live_disk_space_used.unset(r);
|
||||
cf::get_total_disk_space_used.unset(r);
|
||||
cf::get_all_total_disk_space_used.unset(r);
|
||||
cf::get_min_row_size.unset(r);
|
||||
cf::get_all_min_row_size.unset(r);
|
||||
cf::get_max_row_size.unset(r);
|
||||
cf::get_all_max_row_size.unset(r);
|
||||
cf::get_mean_row_size.unset(r);
|
||||
cf::get_all_mean_row_size.unset(r);
|
||||
cf::get_bloom_filter_false_positives.unset(r);
|
||||
cf::get_all_bloom_filter_false_positives.unset(r);
|
||||
cf::get_recent_bloom_filter_false_positives.unset(r);
|
||||
cf::get_all_recent_bloom_filter_false_positives.unset(r);
|
||||
cf::get_bloom_filter_false_ratio.unset(r);
|
||||
cf::get_all_bloom_filter_false_ratio.unset(r);
|
||||
cf::get_recent_bloom_filter_false_ratio.unset(r);
|
||||
cf::get_all_recent_bloom_filter_false_ratio.unset(r);
|
||||
cf::get_bloom_filter_disk_space_used.unset(r);
|
||||
cf::get_all_bloom_filter_disk_space_used.unset(r);
|
||||
cf::get_bloom_filter_off_heap_memory_used.unset(r);
|
||||
cf::get_all_bloom_filter_off_heap_memory_used.unset(r);
|
||||
cf::get_index_summary_off_heap_memory_used.unset(r);
|
||||
cf::get_all_index_summary_off_heap_memory_used.unset(r);
|
||||
cf::get_compression_metadata_off_heap_memory_used.unset(r);
|
||||
cf::get_all_compression_metadata_off_heap_memory_used.unset(r);
|
||||
cf::get_speculative_retries.unset(r);
|
||||
cf::get_all_speculative_retries.unset(r);
|
||||
cf::get_key_cache_hit_rate.unset(r);
|
||||
cf::get_true_snapshots_size.unset(r);
|
||||
cf::get_all_true_snapshots_size.unset(r);
|
||||
cf::get_row_cache_hit_out_of_range.unset(r);
|
||||
cf::get_all_row_cache_hit_out_of_range.unset(r);
|
||||
cf::get_row_cache_hit.unset(r);
|
||||
cf::get_all_row_cache_hit.unset(r);
|
||||
cf::get_row_cache_miss.unset(r);
|
||||
cf::get_all_row_cache_miss.unset(r);
|
||||
cf::get_cas_prepare.unset(r);
|
||||
cf::get_cas_propose.unset(r);
|
||||
cf::get_cas_commit.unset(r);
|
||||
cf::get_sstables_per_read_histogram.unset(r);
|
||||
cf::get_tombstone_scanned_histogram.unset(r);
|
||||
cf::get_live_scanned_histogram.unset(r);
|
||||
cf::get_col_update_time_delta_histogram.unset(r);
|
||||
cf::get_auto_compaction.unset(r);
|
||||
cf::enable_auto_compaction.unset(r);
|
||||
cf::disable_auto_compaction.unset(r);
|
||||
cf::get_built_indexes.unset(r);
|
||||
cf::get_compression_metadata_off_heap_memory_used.unset(r);
|
||||
cf::get_compression_parameters.unset(r);
|
||||
cf::get_compression_ratio.unset(r);
|
||||
cf::get_read_latency_estimated_histogram.unset(r);
|
||||
cf::get_write_latency_estimated_histogram.unset(r);
|
||||
cf::set_compaction_strategy_class.unset(r);
|
||||
cf::get_compaction_strategy_class.unset(r);
|
||||
cf::set_compression_parameters.unset(r);
|
||||
cf::set_crc_check_chance.unset(r);
|
||||
cf::get_sstable_count_per_level.unset(r);
|
||||
cf::get_sstables_for_key.unset(r);
|
||||
cf::toppartitions.unset(r);
|
||||
cf::force_major_compaction.unset(r);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,16 +14,11 @@
|
||||
#include <seastar/core/future-util.hh>
|
||||
#include <any>
|
||||
|
||||
namespace db {
|
||||
class system_keyspace;
|
||||
}
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_column_family(http_context& ctx, httpd::routes& r, sharded<db::system_keyspace>& sys_ks);
|
||||
void unset_column_family(http_context& ctx, httpd::routes& r);
|
||||
void set_column_family(http_context& ctx, routes& r);
|
||||
|
||||
table_id get_uuid(const sstring& name, const replica::database& db);
|
||||
const utils::UUID& get_uuid(const sstring& name, const replica::database& db);
|
||||
future<> foreach_column_family(http_context& ctx, const sstring& name, std::function<void(replica::column_family&)> f);
|
||||
|
||||
|
||||
@@ -68,10 +63,9 @@ struct map_reduce_column_families_locally {
|
||||
std::function<std::unique_ptr<std::any>(std::unique_ptr<std::any>, std::unique_ptr<std::any>)> reducer;
|
||||
future<std::unique_ptr<std::any>> operator()(replica::database& db) const {
|
||||
auto res = seastar::make_lw_shared<std::unique_ptr<std::any>>(std::make_unique<std::any>(init));
|
||||
return db.get_tables_metadata().for_each_table_gently([res, this] (table_id, seastar::lw_shared_ptr<replica::table> table) {
|
||||
*res = reducer(std::move(*res), mapper(*table.get()));
|
||||
return make_ready_future();
|
||||
}).then([res] () {
|
||||
return do_for_each(db.get_column_families(), [res, this](const std::pair<utils::UUID, seastar::lw_shared_ptr<replica::table>>& i) {
|
||||
*res = reducer(std::move(*res), mapper(*i.second.get()));
|
||||
}).then([res] {
|
||||
return std::move(*res);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
#include <vector>
|
||||
|
||||
namespace api {
|
||||
using namespace seastar::httpd;
|
||||
|
||||
template<typename T>
|
||||
static auto acquire_cl_metric(http_context& ctx, std::function<T (db::commitlog*)> func) {
|
||||
|
||||
@@ -12,6 +12,6 @@
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_commitlog(http_context& ctx, httpd::routes& r);
|
||||
void set_commitlog(http_context& ctx, routes& r);
|
||||
|
||||
}
|
||||
|
||||
@@ -22,7 +22,6 @@ namespace api {
|
||||
|
||||
namespace cm = httpd::compaction_manager_json;
|
||||
using namespace json;
|
||||
using namespace seastar::httpd;
|
||||
|
||||
static future<json::json_return_type> get_cm_stats(http_context& ctx,
|
||||
int64_t compaction_manager::stats::*f) {
|
||||
@@ -42,8 +41,9 @@ static std::unordered_map<std::pair<sstring, sstring>, uint64_t, utils::tuple_ha
|
||||
return std::move(a);
|
||||
}
|
||||
|
||||
|
||||
void set_compaction_manager(http_context& ctx, routes& r) {
|
||||
cm::get_compactions.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cm::get_compactions.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return ctx.db.map_reduce0([](replica::database& db) {
|
||||
std::vector<cm::summary> summaries;
|
||||
const compaction_manager& cm = db.get_compaction_manager();
|
||||
@@ -65,12 +65,12 @@ void set_compaction_manager(http_context& ctx, routes& r) {
|
||||
});
|
||||
});
|
||||
|
||||
cm::get_pending_tasks_by_table.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
return ctx.db.map_reduce0([](replica::database& db) {
|
||||
return do_with(std::unordered_map<std::pair<sstring, sstring>, uint64_t, utils::tuple_hash>(), [&db](std::unordered_map<std::pair<sstring, sstring>, uint64_t, utils::tuple_hash>& tasks) {
|
||||
return db.get_tables_metadata().for_each_table_gently([&tasks] (table_id, lw_shared_ptr<replica::table> table) {
|
||||
replica::table& cf = *table.get();
|
||||
tasks[std::make_pair(cf.schema()->ks_name(), cf.schema()->cf_name())] = cf.estimate_pending_compactions();
|
||||
cm::get_pending_tasks_by_table.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return ctx.db.map_reduce0([&ctx](replica::database& db) {
|
||||
return do_with(std::unordered_map<std::pair<sstring, sstring>, uint64_t, utils::tuple_hash>(), [&ctx, &db](std::unordered_map<std::pair<sstring, sstring>, uint64_t, utils::tuple_hash>& tasks) {
|
||||
return do_for_each(db.get_column_families(), [&tasks](const std::pair<utils::UUID, seastar::lw_shared_ptr<replica::table>>& i) {
|
||||
replica::table& cf = *i.second.get();
|
||||
tasks[std::make_pair(cf.schema()->ks_name(), cf.schema()->cf_name())] = cf.get_compaction_strategy().estimated_pending_compactions(cf.as_table_state());
|
||||
return make_ready_future<>();
|
||||
}).then([&tasks] {
|
||||
return std::move(tasks);
|
||||
@@ -91,14 +91,14 @@ void set_compaction_manager(http_context& ctx, routes& r) {
|
||||
});
|
||||
});
|
||||
|
||||
cm::force_user_defined_compaction.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cm::force_user_defined_compaction.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
// FIXME
|
||||
warn(unimplemented::cause::API);
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
cm::stop_compaction.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cm::stop_compaction.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
auto type = req->get_query_param("type");
|
||||
return ctx.db.invoke_on_all([type] (replica::database& db) {
|
||||
auto& cm = db.get_compaction_manager();
|
||||
@@ -108,7 +108,7 @@ void set_compaction_manager(http_context& ctx, routes& r) {
|
||||
});
|
||||
});
|
||||
|
||||
cm::stop_keyspace_compaction.set(r, [&ctx] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
cm::stop_keyspace_compaction.set(r, [&ctx] (std::unique_ptr<request> req) -> future<json::json_return_type> {
|
||||
auto ks_name = validate_keyspace(ctx, req->param);
|
||||
auto table_names = parse_tables(ks_name, ctx, req->query_parameters, "tables");
|
||||
if (table_names.empty()) {
|
||||
@@ -119,43 +119,41 @@ void set_compaction_manager(http_context& ctx, routes& r) {
|
||||
auto& cm = db.get_compaction_manager();
|
||||
return parallel_for_each(table_names, [&db, &cm, &ks_name, type] (sstring& table_name) {
|
||||
auto& t = db.find_column_family(ks_name, table_name);
|
||||
return t.parallel_foreach_table_state([&] (compaction::table_state& ts) {
|
||||
return cm.stop_compaction(type, &ts);
|
||||
});
|
||||
return cm.stop_compaction(type, &t.as_table_state());
|
||||
});
|
||||
});
|
||||
co_return json_void();
|
||||
});
|
||||
|
||||
cm::get_pending_tasks.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cm::get_pending_tasks.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return map_reduce_cf(ctx, int64_t(0), [](replica::column_family& cf) {
|
||||
return cf.estimate_pending_compactions();
|
||||
return cf.get_compaction_strategy().estimated_pending_compactions(cf.as_table_state());
|
||||
}, std::plus<int64_t>());
|
||||
});
|
||||
|
||||
cm::get_completed_tasks.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
cm::get_completed_tasks.set(r, [&ctx] (std::unique_ptr<request> req) {
|
||||
return get_cm_stats(ctx, &compaction_manager::stats::completed_tasks);
|
||||
});
|
||||
|
||||
cm::get_total_compactions_completed.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cm::get_total_compactions_completed.set(r, [] (std::unique_ptr<request> req) {
|
||||
// FIXME
|
||||
// We are currently dont have an API for compaction
|
||||
// so returning a 0 as the number of total compaction is ok
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cm::get_bytes_compacted.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cm::get_bytes_compacted.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
// FIXME
|
||||
warn(unimplemented::cause::API);
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
cm::get_compaction_history.set(r, [&ctx] (std::unique_ptr<http::request> req) {
|
||||
std::function<future<>(output_stream<char>&&)> f = [&ctx](output_stream<char>&& s) {
|
||||
return do_with(output_stream<char>(std::move(s)), true, [&ctx] (output_stream<char>& s, bool& first){
|
||||
return s.write("[").then([&ctx, &s, &first] {
|
||||
return ctx.db.local().get_compaction_manager().get_compaction_history([&s, &first](const db::compaction_history_entry& entry) mutable {
|
||||
cm::get_compaction_history.set(r, [] (std::unique_ptr<request> req) {
|
||||
std::function<future<>(output_stream<char>&&)> f = [](output_stream<char>&& s) {
|
||||
return do_with(output_stream<char>(std::move(s)), true, [] (output_stream<char>& s, bool& first){
|
||||
return s.write("[").then([&s, &first] {
|
||||
return db::system_keyspace::get_compaction_history([&s, &first](const db::system_keyspace::compaction_history_entry& entry) mutable {
|
||||
cm::history h;
|
||||
h.id = entry.id.to_sstring();
|
||||
h.ks = std::move(entry.ks);
|
||||
@@ -185,7 +183,7 @@ void set_compaction_manager(http_context& ctx, routes& r) {
|
||||
return make_ready_future<json::json_return_type>(std::move(f));
|
||||
});
|
||||
|
||||
cm::get_compaction_info.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
cm::get_compaction_info.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
// FIXME
|
||||
warn(unimplemented::cause::API);
|
||||
|
||||
@@ -12,6 +12,6 @@
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_compaction_manager(http_context& ctx, httpd::routes& r);
|
||||
void set_compaction_manager(http_context& ctx, routes& r);
|
||||
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
#include <boost/algorithm/string/replace.hpp>
|
||||
|
||||
namespace api {
|
||||
using namespace seastar::httpd;
|
||||
|
||||
template<class T>
|
||||
json::json_return_type get_json_return_type(const T& val) {
|
||||
@@ -45,7 +44,7 @@ future<> get_config_swagger_entry(std::string_view name, const std::string& desc
|
||||
} else {
|
||||
ss <<',';
|
||||
};
|
||||
ss << "\"/v2/config/" << name <<"\": {"
|
||||
ss << "\"/config/" << name <<"\": {"
|
||||
"\"get\": {"
|
||||
"\"description\": \"" << boost::replace_all_copy(boost::replace_all_copy(boost::replace_all_copy(description,"\n","\\n"),"\"", "''"), "\t", " ") <<"\","
|
||||
"\"operationId\": \"find_config_"<< name <<"\","
|
||||
@@ -76,9 +75,9 @@ future<> get_config_swagger_entry(std::string_view name, const std::string& desc
|
||||
|
||||
namespace cs = httpd::config_json;
|
||||
|
||||
void set_config(std::shared_ptr < api_registry_builder20 > rb, http_context& ctx, routes& r, const db::config& cfg, bool first) {
|
||||
rb->register_function(r, [&cfg, first] (output_stream<char>& os) {
|
||||
return do_with(first, [&os, &cfg] (bool& first) {
|
||||
void set_config(std::shared_ptr < api_registry_builder20 > rb, http_context& ctx, routes& r, const db::config& cfg) {
|
||||
rb->register_function(r, [&cfg] (output_stream<char>& os) {
|
||||
return do_with(true, [&os, &cfg] (bool& first) {
|
||||
auto f = make_ready_future();
|
||||
for (auto&& cfg_ref : cfg.values()) {
|
||||
auto&& cfg = cfg_ref.get();
|
||||
|
||||
@@ -13,5 +13,5 @@
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_config(std::shared_ptr<httpd::api_registry_builder20> rb, http_context& ctx, httpd::routes& r, const db::config& cfg, bool first = false);
|
||||
void set_config(std::shared_ptr<api_registry_builder20> rb, http_context& ctx, routes& r, const db::config& cfg);
|
||||
}
|
||||
|
||||
@@ -8,16 +8,13 @@
|
||||
|
||||
#include "locator/token_metadata.hh"
|
||||
#include "locator/snitch_base.hh"
|
||||
#include "locator/production_snitch_base.hh"
|
||||
#include "endpoint_snitch.hh"
|
||||
#include "api/api-doc/endpoint_snitch_info.json.hh"
|
||||
#include "api/api-doc/storage_service.json.hh"
|
||||
#include "utils/fb_utilities.hh"
|
||||
|
||||
namespace api {
|
||||
using namespace seastar::httpd;
|
||||
|
||||
void set_endpoint_snitch(http_context& ctx, routes& r, sharded<locator::snitch_ptr>& snitch) {
|
||||
void set_endpoint_snitch(http_context& ctx, routes& r) {
|
||||
static auto host_or_broadcast = [](const_req req) {
|
||||
auto host = req.get_query_param("host");
|
||||
return host.empty() ? gms::inet_address(utils::fb_utilities::get_broadcast_address()) : gms::inet_address(host);
|
||||
@@ -25,45 +22,17 @@ void set_endpoint_snitch(http_context& ctx, routes& r, sharded<locator::snitch_p
|
||||
|
||||
httpd::endpoint_snitch_info_json::get_datacenter.set(r, [&ctx](const_req req) {
|
||||
auto& topology = ctx.shared_token_metadata.local().get()->get_topology();
|
||||
auto ep = host_or_broadcast(req);
|
||||
if (!topology.has_endpoint(ep)) {
|
||||
// Cannot return error here, nodetool status can race, request
|
||||
// info about just-left node and not handle it nicely
|
||||
return locator::endpoint_dc_rack::default_location.dc;
|
||||
}
|
||||
return topology.get_datacenter(ep);
|
||||
return topology.get_datacenter(host_or_broadcast(req));
|
||||
});
|
||||
|
||||
httpd::endpoint_snitch_info_json::get_rack.set(r, [&ctx](const_req req) {
|
||||
auto& topology = ctx.shared_token_metadata.local().get()->get_topology();
|
||||
auto ep = host_or_broadcast(req);
|
||||
if (!topology.has_endpoint(ep)) {
|
||||
// Cannot return error here, nodetool status can race, request
|
||||
// info about just-left node and not handle it nicely
|
||||
return locator::endpoint_dc_rack::default_location.rack;
|
||||
}
|
||||
return topology.get_rack(ep);
|
||||
return topology.get_rack(host_or_broadcast(req));
|
||||
});
|
||||
|
||||
httpd::endpoint_snitch_info_json::get_snitch_name.set(r, [&snitch] (const_req req) {
|
||||
return snitch.local()->get_name();
|
||||
httpd::endpoint_snitch_info_json::get_snitch_name.set(r, [] (const_req req) {
|
||||
return locator::i_endpoint_snitch::get_local_snitch_ptr()->get_name();
|
||||
});
|
||||
|
||||
httpd::storage_service_json::update_snitch.set(r, [&snitch](std::unique_ptr<request> req) {
|
||||
locator::snitch_config cfg;
|
||||
cfg.name = req->get_query_param("ep_snitch_class_name");
|
||||
return locator::i_endpoint_snitch::reset_snitch(snitch, cfg).then([] {
|
||||
return make_ready_future<json::json_return_type>(json::json_void());
|
||||
});
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
void unset_endpoint_snitch(http_context& ctx, routes& r) {
|
||||
httpd::endpoint_snitch_info_json::get_datacenter.unset(r);
|
||||
httpd::endpoint_snitch_info_json::get_rack.unset(r);
|
||||
httpd::endpoint_snitch_info_json::get_snitch_name.unset(r);
|
||||
httpd::storage_service_json::update_snitch.unset(r);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -10,13 +10,8 @@
|
||||
|
||||
#include "api.hh"
|
||||
|
||||
namespace locator {
|
||||
class snitch_ptr;
|
||||
}
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_endpoint_snitch(http_context& ctx, httpd::routes& r, sharded<locator::snitch_ptr>&);
|
||||
void unset_endpoint_snitch(http_context& ctx, httpd::routes& r);
|
||||
void set_endpoint_snitch(http_context& ctx, routes& r);
|
||||
|
||||
}
|
||||
|
||||
@@ -12,12 +12,9 @@
|
||||
#include <seastar/http/exception.hh>
|
||||
#include "log.hh"
|
||||
#include "utils/error_injection.hh"
|
||||
#include "utils/rjson.hh"
|
||||
#include <seastar/core/future-util.hh>
|
||||
#include <seastar/util/short_streams.hh>
|
||||
|
||||
namespace api {
|
||||
using namespace seastar::httpd;
|
||||
|
||||
namespace hf = httpd::error_injection_json;
|
||||
|
||||
@@ -26,27 +23,10 @@ void set_error_injection(http_context& ctx, routes& r) {
|
||||
hf::enable_injection.set(r, [](std::unique_ptr<request> req) {
|
||||
sstring injection = req->param["injection"];
|
||||
bool one_shot = req->get_query_param("one_shot") == "True";
|
||||
auto params = req->content;
|
||||
|
||||
const size_t max_params_size = 1024 * 1024;
|
||||
if (params.size() > max_params_size) {
|
||||
// This is a hard limit, because we don't want to allocate
|
||||
// too much memory or block the thread for too long.
|
||||
throw httpd::bad_param_exception(format("Injection parameters are too long, max length is {}", max_params_size));
|
||||
}
|
||||
|
||||
try {
|
||||
auto parameters = params.empty()
|
||||
? utils::error_injection_parameters{}
|
||||
: rjson::parse_to_map<utils::error_injection_parameters>(params);
|
||||
|
||||
auto& errinj = utils::get_local_injector();
|
||||
return errinj.enable_on_all(injection, one_shot, std::move(parameters)).then([] {
|
||||
return make_ready_future<json::json_return_type>(json::json_void());
|
||||
});
|
||||
} catch (const rjson::error& e) {
|
||||
throw httpd::bad_param_exception(format("Failed to parse injections parameters: {}", e.what()));
|
||||
}
|
||||
auto& errinj = utils::get_local_injector();
|
||||
return errinj.enable_on_all(injection, one_shot).then([] {
|
||||
return make_ready_future<json::json_return_type>(json::json_void());
|
||||
});
|
||||
});
|
||||
|
||||
hf::get_enabled_injections_on_all.set(r, [](std::unique_ptr<request> req) {
|
||||
@@ -71,13 +51,6 @@ void set_error_injection(http_context& ctx, routes& r) {
|
||||
});
|
||||
});
|
||||
|
||||
hf::message_injection.set(r, [](std::unique_ptr<request> req) {
|
||||
sstring injection = req->param["injection"];
|
||||
auto& errinj = utils::get_local_injector();
|
||||
return errinj.receive_message_on_all(injection).then([] {
|
||||
return make_ready_future<json::json_return_type>(json::json_void());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace api
|
||||
|
||||
@@ -12,6 +12,6 @@
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_error_injection(http_context& ctx, httpd::routes& r);
|
||||
void set_error_injection(http_context& ctx, routes& r);
|
||||
|
||||
}
|
||||
|
||||
@@ -8,47 +8,51 @@
|
||||
|
||||
#include "failure_detector.hh"
|
||||
#include "api/api-doc/failure_detector.json.hh"
|
||||
#include "gms/failure_detector.hh"
|
||||
#include "gms/application_state.hh"
|
||||
#include "gms/gossiper.hh"
|
||||
|
||||
namespace api {
|
||||
using namespace seastar::httpd;
|
||||
|
||||
namespace fd = httpd::failure_detector_json;
|
||||
|
||||
void set_failure_detector(http_context& ctx, routes& r, gms::gossiper& g) {
|
||||
fd::get_all_endpoint_states.set(r, [&g](std::unique_ptr<request> req) {
|
||||
std::vector<fd::endpoint_state> res;
|
||||
res.reserve(g.num_endpoints());
|
||||
g.for_each_endpoint_state([&] (const gms::inet_address& addr, const gms::endpoint_state& eps) {
|
||||
fd::endpoint_state val;
|
||||
val.addrs = fmt::to_string(addr);
|
||||
val.is_alive = g.is_alive(addr);
|
||||
val.generation = eps.get_heart_beat_state().get_generation().value();
|
||||
val.version = eps.get_heart_beat_state().get_heart_beat_version().value();
|
||||
val.update_time = eps.get_update_timestamp().time_since_epoch().count();
|
||||
for (const auto& [as_type, app_state] : eps.get_application_state_map()) {
|
||||
fd::version_value version_val;
|
||||
// We return the enum index and not it's name to stay compatible to origin
|
||||
// method that the state index are static but the name can be changed.
|
||||
version_val.application_state = static_cast<std::underlying_type<gms::application_state>::type>(as_type);
|
||||
version_val.value = app_state.value();
|
||||
version_val.version = app_state.version().value();
|
||||
val.application_state.push(version_val);
|
||||
return g.container().invoke_on(0, [] (gms::gossiper& g) {
|
||||
std::vector<fd::endpoint_state> res;
|
||||
for (auto i : g.get_endpoint_states()) {
|
||||
fd::endpoint_state val;
|
||||
val.addrs = boost::lexical_cast<std::string>(i.first);
|
||||
val.is_alive = i.second.is_alive();
|
||||
val.generation = i.second.get_heart_beat_state().get_generation();
|
||||
val.version = i.second.get_heart_beat_state().get_heart_beat_version();
|
||||
val.update_time = i.second.get_update_timestamp().time_since_epoch().count();
|
||||
for (auto a : i.second.get_application_state_map()) {
|
||||
fd::version_value version_val;
|
||||
// We return the enum index and not it's name to stay compatible to origin
|
||||
// method that the state index are static but the name can be changed.
|
||||
version_val.application_state = static_cast<std::underlying_type<gms::application_state>::type>(a.first);
|
||||
version_val.value = a.second.value;
|
||||
version_val.version = a.second.version;
|
||||
val.application_state.push(version_val);
|
||||
}
|
||||
res.push_back(val);
|
||||
}
|
||||
res.emplace_back(std::move(val));
|
||||
return make_ready_future<json::json_return_type>(res);
|
||||
});
|
||||
return make_ready_future<json::json_return_type>(res);
|
||||
});
|
||||
|
||||
fd::get_up_endpoint_count.set(r, [&g](std::unique_ptr<request> req) {
|
||||
int res = g.get_up_endpoint_count();
|
||||
return make_ready_future<json::json_return_type>(res);
|
||||
return g.container().invoke_on(0, [] (gms::gossiper& g) {
|
||||
int res = g.get_up_endpoint_count();
|
||||
return make_ready_future<json::json_return_type>(res);
|
||||
});
|
||||
});
|
||||
|
||||
fd::get_down_endpoint_count.set(r, [&g](std::unique_ptr<request> req) {
|
||||
int res = g.get_down_endpoint_count();
|
||||
return make_ready_future<json::json_return_type>(res);
|
||||
return g.container().invoke_on(0, [] (gms::gossiper& g) {
|
||||
int res = g.get_down_endpoint_count();
|
||||
return make_ready_future<json::json_return_type>(res);
|
||||
});
|
||||
});
|
||||
|
||||
fd::get_phi_convict_threshold.set(r, [] (std::unique_ptr<request> req) {
|
||||
@@ -56,34 +60,42 @@ void set_failure_detector(http_context& ctx, routes& r, gms::gossiper& g) {
|
||||
});
|
||||
|
||||
fd::get_simple_states.set(r, [&g] (std::unique_ptr<request> req) {
|
||||
std::map<sstring, sstring> nodes_status;
|
||||
g.for_each_endpoint_state([&] (const gms::inet_address& node, const gms::endpoint_state&) {
|
||||
nodes_status.emplace(node.to_sstring(), g.is_alive(node) ? "UP" : "DOWN");
|
||||
return g.container().invoke_on(0, [] (gms::gossiper& g) {
|
||||
std::map<sstring, sstring> nodes_status;
|
||||
for (auto& entry : g.get_endpoint_states()) {
|
||||
nodes_status.emplace(entry.first.to_sstring(), entry.second.is_alive() ? "UP" : "DOWN");
|
||||
}
|
||||
return make_ready_future<json::json_return_type>(map_to_key_value<fd::mapper>(nodes_status));
|
||||
});
|
||||
return make_ready_future<json::json_return_type>(map_to_key_value<fd::mapper>(nodes_status));
|
||||
});
|
||||
|
||||
fd::set_phi_convict_threshold.set(r, [](std::unique_ptr<request> req) {
|
||||
// TBD
|
||||
unimplemented();
|
||||
std::ignore = atof(req->get_query_param("phi").c_str());
|
||||
double phi = atof(req->get_query_param("phi").c_str());
|
||||
return make_ready_future<json::json_return_type>("");
|
||||
});
|
||||
|
||||
fd::get_endpoint_state.set(r, [&g] (std::unique_ptr<request> req) {
|
||||
auto state = g.get_endpoint_state_ptr(gms::inet_address(req->param["addr"]));
|
||||
if (!state) {
|
||||
return make_ready_future<json::json_return_type>(format("unknown endpoint {}", req->param["addr"]));
|
||||
}
|
||||
std::stringstream ss;
|
||||
g.append_endpoint_state(ss, *state);
|
||||
return make_ready_future<json::json_return_type>(sstring(ss.str()));
|
||||
return g.container().invoke_on(0, [req = std::move(req)] (gms::gossiper& g) {
|
||||
auto* state = g.get_endpoint_state_for_endpoint_ptr(gms::inet_address(req->param["addr"]));
|
||||
if (!state) {
|
||||
return make_ready_future<json::json_return_type>(format("unknown endpoint {}", req->param["addr"]));
|
||||
}
|
||||
std::stringstream ss;
|
||||
g.append_endpoint_state(ss, *state);
|
||||
return make_ready_future<json::json_return_type>(sstring(ss.str()));
|
||||
});
|
||||
});
|
||||
|
||||
fd::get_endpoint_phi_values.set(r, [](std::unique_ptr<request> req) {
|
||||
// We no longer have a phi failure detector,
|
||||
// just returning the empty value is good enough.
|
||||
std::map<gms::inet_address, gms::arrival_window> map;
|
||||
std::vector<fd::endpoint_phi_value> res;
|
||||
auto now = gms::arrival_window::clk::now();
|
||||
for (auto& p : map) {
|
||||
fd::endpoint_phi_value val;
|
||||
val.endpoint = p.first.to_sstring();
|
||||
val.phi = p.second.phi(now);
|
||||
res.emplace_back(std::move(val));
|
||||
}
|
||||
return make_ready_future<json::json_return_type>(res);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -18,6 +18,6 @@ class gossiper;
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_failure_detector(http_context& ctx, httpd::routes& r, gms::gossiper& g);
|
||||
void set_failure_detector(http_context& ctx, routes& r, gms::gossiper& g);
|
||||
|
||||
}
|
||||
|
||||
@@ -6,21 +6,17 @@
|
||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
*/
|
||||
|
||||
#include <seastar/core/coroutine.hh>
|
||||
|
||||
#include "gossiper.hh"
|
||||
#include "api/api-doc/gossiper.json.hh"
|
||||
#include "gms/endpoint_state.hh"
|
||||
#include "gms/gossiper.hh"
|
||||
|
||||
namespace api {
|
||||
using namespace seastar::httpd;
|
||||
using namespace json;
|
||||
|
||||
void set_gossiper(http_context& ctx, routes& r, gms::gossiper& g) {
|
||||
httpd::gossiper_json::get_down_endpoint.set(r, [&g] (std::unique_ptr<request> req) -> future<json::json_return_type> {
|
||||
auto res = co_await g.get_unreachable_members_synchronized();
|
||||
co_return json::json_return_type(container_to_vec(res));
|
||||
httpd::gossiper_json::get_down_endpoint.set(r, [&g] (const_req req) {
|
||||
auto res = g.get_unreachable_members();
|
||||
return container_to_vec(res);
|
||||
});
|
||||
|
||||
|
||||
@@ -30,28 +26,26 @@ void set_gossiper(http_context& ctx, routes& r, gms::gossiper& g) {
|
||||
});
|
||||
});
|
||||
|
||||
httpd::gossiper_json::get_endpoint_downtime.set(r, [&g] (std::unique_ptr<request> req) -> future<json::json_return_type> {
|
||||
gms::inet_address ep(req->param["addr"]);
|
||||
// synchronize unreachable_members on all shards
|
||||
co_await g.get_unreachable_members_synchronized();
|
||||
co_return g.get_endpoint_downtime(ep);
|
||||
httpd::gossiper_json::get_endpoint_downtime.set(r, [&g] (const_req req) {
|
||||
gms::inet_address ep(req.param["addr"]);
|
||||
return g.get_endpoint_downtime(ep);
|
||||
});
|
||||
|
||||
httpd::gossiper_json::get_current_generation_number.set(r, [&g] (std::unique_ptr<http::request> req) {
|
||||
httpd::gossiper_json::get_current_generation_number.set(r, [&g] (std::unique_ptr<request> req) {
|
||||
gms::inet_address ep(req->param["addr"]);
|
||||
return g.get_current_generation_number(ep).then([] (gms::generation_type res) {
|
||||
return make_ready_future<json::json_return_type>(res.value());
|
||||
return g.get_current_generation_number(ep).then([] (int res) {
|
||||
return make_ready_future<json::json_return_type>(res);
|
||||
});
|
||||
});
|
||||
|
||||
httpd::gossiper_json::get_current_heart_beat_version.set(r, [&g] (std::unique_ptr<http::request> req) {
|
||||
httpd::gossiper_json::get_current_heart_beat_version.set(r, [&g] (std::unique_ptr<request> req) {
|
||||
gms::inet_address ep(req->param["addr"]);
|
||||
return g.get_current_heart_beat_version(ep).then([] (gms::version_type res) {
|
||||
return make_ready_future<json::json_return_type>(res.value());
|
||||
return g.get_current_heart_beat_version(ep).then([] (int res) {
|
||||
return make_ready_future<json::json_return_type>(res);
|
||||
});
|
||||
});
|
||||
|
||||
httpd::gossiper_json::assassinate_endpoint.set(r, [&g](std::unique_ptr<http::request> req) {
|
||||
httpd::gossiper_json::assassinate_endpoint.set(r, [&g](std::unique_ptr<request> req) {
|
||||
if (req->get_query_param("unsafe") != "True") {
|
||||
return g.assassinate_endpoint(req->param["addr"]).then([] {
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
@@ -62,9 +56,9 @@ void set_gossiper(http_context& ctx, routes& r, gms::gossiper& g) {
|
||||
});
|
||||
});
|
||||
|
||||
httpd::gossiper_json::force_remove_endpoint.set(r, [&g](std::unique_ptr<http::request> req) {
|
||||
httpd::gossiper_json::force_remove_endpoint.set(r, [&g](std::unique_ptr<request> req) {
|
||||
gms::inet_address ep(req->param["addr"]);
|
||||
return g.force_remove_endpoint(ep, gms::null_permit_id).then([] {
|
||||
return g.force_remove_endpoint(ep).then([] {
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
});
|
||||
|
||||
@@ -18,6 +18,6 @@ class gossiper;
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_gossiper(http_context& ctx, httpd::routes& r, gms::gossiper& g);
|
||||
void set_gossiper(http_context& ctx, routes& r, gms::gossiper& g);
|
||||
|
||||
}
|
||||
|
||||
@@ -13,41 +13,46 @@
|
||||
#include "api/api-doc/hinted_handoff.json.hh"
|
||||
|
||||
#include "gms/inet_address.hh"
|
||||
#include "gms/gossiper.hh"
|
||||
#include "service/storage_proxy.hh"
|
||||
|
||||
namespace api {
|
||||
|
||||
using namespace json;
|
||||
using namespace seastar::httpd;
|
||||
namespace hh = httpd::hinted_handoff_json;
|
||||
|
||||
void set_hinted_handoff(http_context& ctx, routes& r, sharded<service::storage_proxy>& proxy) {
|
||||
hh::create_hints_sync_point.set(r, [&proxy] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
auto parse_hosts_list = [] (sstring arg) {
|
||||
void set_hinted_handoff(http_context& ctx, routes& r, gms::gossiper& g) {
|
||||
hh::create_hints_sync_point.set(r, [&ctx, &g] (std::unique_ptr<request> req) -> future<json::json_return_type> {
|
||||
auto parse_hosts_list = [&g] (sstring arg) {
|
||||
std::vector<sstring> hosts_str = split(arg, ",");
|
||||
std::vector<gms::inet_address> hosts;
|
||||
hosts.reserve(hosts_str.size());
|
||||
|
||||
for (const auto& host_str : hosts_str) {
|
||||
try {
|
||||
gms::inet_address host;
|
||||
host = gms::inet_address(host_str);
|
||||
hosts.push_back(host);
|
||||
} catch (std::exception& e) {
|
||||
throw httpd::bad_param_exception(format("Failed to parse host address {}: {}", host_str, e.what()));
|
||||
if (hosts_str.empty()) {
|
||||
// No target_hosts specified means that we should wait for hints for all nodes to be sent
|
||||
const auto members_set = g.get_live_members();
|
||||
std::copy(members_set.begin(), members_set.end(), std::back_inserter(hosts));
|
||||
} else {
|
||||
for (const auto& host_str : hosts_str) {
|
||||
try {
|
||||
gms::inet_address host;
|
||||
host = gms::inet_address(host_str);
|
||||
hosts.push_back(host);
|
||||
} catch (std::exception& e) {
|
||||
throw httpd::bad_param_exception(format("Failed to parse host address {}: {}", host_str, e.what()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return hosts;
|
||||
};
|
||||
|
||||
std::vector<gms::inet_address> target_hosts = parse_hosts_list(req->get_query_param("target_hosts"));
|
||||
return proxy.local().create_hint_sync_point(std::move(target_hosts)).then([] (db::hints::sync_point sync_point) {
|
||||
return ctx.sp.local().create_hint_sync_point(std::move(target_hosts)).then([] (db::hints::sync_point sync_point) {
|
||||
return json::json_return_type(sync_point.encode());
|
||||
});
|
||||
});
|
||||
|
||||
hh::get_hints_sync_point.set(r, [&proxy] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
hh::get_hints_sync_point.set(r, [&ctx] (std::unique_ptr<request> req) -> future<json::json_return_type> {
|
||||
db::hints::sync_point sync_point;
|
||||
const sstring encoded = req->get_query_param("id");
|
||||
try {
|
||||
@@ -81,49 +86,49 @@ void set_hinted_handoff(http_context& ctx, routes& r, sharded<service::storage_p
|
||||
using return_type = hh::ns_get_hints_sync_point::get_hints_sync_point_return_type;
|
||||
using return_type_wrapper = hh::ns_get_hints_sync_point::return_type_wrapper;
|
||||
|
||||
return proxy.local().wait_for_hint_sync_point(std::move(sync_point), deadline).then([] {
|
||||
return ctx.sp.local().wait_for_hint_sync_point(std::move(sync_point), deadline).then([] {
|
||||
return json::json_return_type(return_type_wrapper(return_type::DONE));
|
||||
}).handle_exception_type([] (const timed_out_error&) {
|
||||
return json::json_return_type(return_type_wrapper(return_type::IN_PROGRESS));
|
||||
});
|
||||
});
|
||||
|
||||
hh::list_endpoints_pending_hints.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
hh::list_endpoints_pending_hints.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
std::vector<sstring> res;
|
||||
return make_ready_future<json::json_return_type>(res);
|
||||
});
|
||||
|
||||
hh::truncate_all_hints.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
hh::truncate_all_hints.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
sstring host = req->get_query_param("host");
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
hh::schedule_hint_delivery.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
hh::schedule_hint_delivery.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
sstring host = req->get_query_param("host");
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
hh::pause_hints_delivery.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
hh::pause_hints_delivery.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
sstring pause = req->get_query_param("pause");
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
hh::get_create_hint_count.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
hh::get_create_hint_count.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
sstring host = req->get_query_param("host");
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
hh::get_not_stored_hints_count.set(r, [] (std::unique_ptr<http::request> req) {
|
||||
hh::get_not_stored_hints_count.set(r, [] (std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
sstring host = req->get_query_param("host");
|
||||
|
||||
@@ -8,14 +8,17 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <seastar/core/sharded.hh>
|
||||
#include "api.hh"
|
||||
|
||||
namespace service { class storage_proxy; }
|
||||
namespace gms {
|
||||
|
||||
class gossiper;
|
||||
|
||||
}
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_hinted_handoff(http_context& ctx, httpd::routes& r, sharded<service::storage_proxy>& p);
|
||||
void unset_hinted_handoff(http_context& ctx, httpd::routes& r);
|
||||
void set_hinted_handoff(http_context& ctx, routes& r, gms::gossiper& g);
|
||||
void unset_hinted_handoff(http_context& ctx, routes& r);
|
||||
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
#include "replica/database.hh"
|
||||
|
||||
namespace api {
|
||||
using namespace seastar::httpd;
|
||||
|
||||
static logging::logger alogger("lsa-api");
|
||||
|
||||
|
||||
@@ -12,6 +12,6 @@
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_lsa(http_context& ctx, httpd::routes& r);
|
||||
void set_lsa(http_context& ctx, routes& r);
|
||||
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
|
||||
using namespace seastar::httpd;
|
||||
using namespace httpd::messaging_service_json;
|
||||
using namespace netw;
|
||||
|
||||
@@ -29,7 +28,7 @@ std::vector<message_counter> map_to_message_counters(
|
||||
std::vector<message_counter> res;
|
||||
for (auto i : map) {
|
||||
res.push_back(message_counter());
|
||||
res.back().key = fmt::to_string(i.first);
|
||||
res.back().key = boost::lexical_cast<sstring>(i.first);
|
||||
res.back().value = i.second;
|
||||
}
|
||||
return res;
|
||||
|
||||
@@ -14,7 +14,7 @@ namespace netw { class messaging_service; }
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_messaging_service(http_context& ctx, httpd::routes& r, sharded<netw::messaging_service>& ms);
|
||||
void unset_messaging_service(http_context& ctx, httpd::routes& r);
|
||||
void set_messaging_service(http_context& ctx, routes& r, sharded<netw::messaging_service>& ms);
|
||||
void unset_messaging_service(http_context& ctx, routes& r);
|
||||
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
#include "service/storage_proxy.hh"
|
||||
#include "api/api-doc/storage_proxy.json.hh"
|
||||
#include "api/api-doc/utils.json.hh"
|
||||
#include "service/storage_service.hh"
|
||||
#include "db/config.hh"
|
||||
#include "utils/histogram.hh"
|
||||
#include "replica/database.hh"
|
||||
@@ -19,12 +20,8 @@ namespace api {
|
||||
|
||||
namespace sp = httpd::storage_proxy_json;
|
||||
using proxy = service::storage_proxy;
|
||||
using namespace seastar::httpd;
|
||||
using namespace json;
|
||||
|
||||
utils::time_estimated_histogram timed_rate_moving_average_summary_merge(utils::time_estimated_histogram a, const utils::timed_rate_moving_average_summary_and_histogram& b) {
|
||||
return a.merge(b.histogram());
|
||||
}
|
||||
|
||||
/**
|
||||
* This function implement a two dimentional map reduce where
|
||||
@@ -58,10 +55,10 @@ future<V> two_dimensional_map_reduce(distributed<service::storage_proxy>& d,
|
||||
* @param initial_value - the initial value to use for both aggregations* @return
|
||||
* @return A future that resolves to the result of the aggregation.
|
||||
*/
|
||||
template<typename V, typename Reducer, typename F, typename C>
|
||||
template<typename V, typename Reducer, typename F>
|
||||
future<V> two_dimensional_map_reduce(distributed<service::storage_proxy>& d,
|
||||
C F::*f, Reducer reducer, V initial_value) {
|
||||
return two_dimensional_map_reduce(d, [f] (F& stats) -> V {
|
||||
V F::*f, Reducer reducer, V initial_value) {
|
||||
return two_dimensional_map_reduce(d, [f] (F& stats) {
|
||||
return stats.*f;
|
||||
}, reducer, initial_value);
|
||||
}
|
||||
@@ -115,17 +112,17 @@ utils_json::estimated_histogram time_to_json_histogram(const utils::time_estimat
|
||||
return res;
|
||||
}
|
||||
|
||||
static future<json::json_return_type> sum_estimated_histogram(sharded<service::storage_proxy>& proxy, utils::timed_rate_moving_average_summary_and_histogram service::storage_proxy_stats::stats::*f) {
|
||||
return two_dimensional_map_reduce(proxy, [f] (service::storage_proxy_stats::stats& stats) {
|
||||
return (stats.*f).histogram();
|
||||
}, utils::time_estimated_histogram_merge, utils::time_estimated_histogram()).then([](const utils::time_estimated_histogram& val) {
|
||||
static future<json::json_return_type> sum_estimated_histogram(http_context& ctx, utils::time_estimated_histogram service::storage_proxy_stats::stats::*f) {
|
||||
|
||||
return two_dimensional_map_reduce(ctx.sp, f, utils::time_estimated_histogram_merge,
|
||||
utils::time_estimated_histogram()).then([](const utils::time_estimated_histogram& val) {
|
||||
return make_ready_future<json::json_return_type>(time_to_json_histogram(val));
|
||||
});
|
||||
}
|
||||
|
||||
static future<json::json_return_type> sum_estimated_histogram(sharded<service::storage_proxy>& proxy, utils::estimated_histogram service::storage_proxy_stats::stats::*f) {
|
||||
static future<json::json_return_type> sum_estimated_histogram(http_context& ctx, utils::estimated_histogram service::storage_proxy_stats::stats::*f) {
|
||||
|
||||
return two_dimensional_map_reduce(proxy, f, utils::estimated_histogram_merge,
|
||||
return two_dimensional_map_reduce(ctx.sp, f, utils::estimated_histogram_merge,
|
||||
utils::estimated_histogram()).then([](const utils::estimated_histogram& val) {
|
||||
utils_json::estimated_histogram res;
|
||||
res = val;
|
||||
@@ -133,8 +130,8 @@ static future<json::json_return_type> sum_estimated_histogram(sharded<service::
|
||||
});
|
||||
}
|
||||
|
||||
static future<json::json_return_type> total_latency(sharded<service::storage_proxy>& proxy, utils::timed_rate_moving_average_summary_and_histogram service::storage_proxy_stats::stats::*f) {
|
||||
return two_dimensional_map_reduce(proxy, [f] (service::storage_proxy_stats::stats& stats) {
|
||||
static future<json::json_return_type> total_latency(http_context& ctx, utils::timed_rate_moving_average_and_histogram service::storage_proxy_stats::stats::*f) {
|
||||
return two_dimensional_map_reduce(ctx.sp, [f] (service::storage_proxy_stats::stats& stats) {
|
||||
return (stats.*f).hist.mean * (stats.*f).hist.count;
|
||||
}, std::plus<double>(), 0.0).then([](double val) {
|
||||
int64_t res = val;
|
||||
@@ -153,7 +150,7 @@ static future<json::json_return_type> total_latency(sharded<service::storage_pr
|
||||
template<typename F>
|
||||
future<json::json_return_type>
|
||||
sum_histogram_stats_storage_proxy(distributed<proxy>& d,
|
||||
utils::timed_rate_moving_average_summary_and_histogram F::*f) {
|
||||
utils::timed_rate_moving_average_and_histogram F::*f) {
|
||||
return two_dimensional_map_reduce(d, [f] (service::storage_proxy_stats::stats& stats) {
|
||||
return (stats.*f).hist;
|
||||
}, std::plus<utils::ihistogram>(), utils::ihistogram()).
|
||||
@@ -173,7 +170,7 @@ sum_histogram_stats_storage_proxy(distributed<proxy>& d,
|
||||
template<typename F>
|
||||
future<json::json_return_type>
|
||||
sum_timer_stats_storage_proxy(distributed<proxy>& d,
|
||||
utils::timed_rate_moving_average_summary_and_histogram F::*f) {
|
||||
utils::timed_rate_moving_average_and_histogram F::*f) {
|
||||
|
||||
return two_dimensional_map_reduce(d, [f] (service::storage_proxy_stats::stats& stats) {
|
||||
return (stats.*f).rate();
|
||||
@@ -183,76 +180,76 @@ sum_timer_stats_storage_proxy(distributed<proxy>& d,
|
||||
});
|
||||
}
|
||||
|
||||
void set_storage_proxy(http_context& ctx, routes& r, sharded<service::storage_proxy>& proxy) {
|
||||
sp::get_total_hints.set(r, [](std::unique_ptr<http::request> req) {
|
||||
void set_storage_proxy(http_context& ctx, routes& r, sharded<service::storage_service>& ss) {
|
||||
sp::get_total_hints.set(r, [](std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
sp::get_hinted_handoff_enabled.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
const auto& filter = proxy.local().get_hints_host_filter();
|
||||
sp::get_hinted_handoff_enabled.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
const auto& filter = service::get_storage_proxy().local().get_hints_host_filter();
|
||||
return make_ready_future<json::json_return_type>(!filter.is_disabled_for_all());
|
||||
});
|
||||
|
||||
sp::set_hinted_handoff_enabled.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
sp::set_hinted_handoff_enabled.set(r, [](std::unique_ptr<request> req) {
|
||||
auto enable = req->get_query_param("enable");
|
||||
auto filter = (enable == "true" || enable == "1")
|
||||
? db::hints::host_filter(db::hints::host_filter::enabled_for_all_tag {})
|
||||
: db::hints::host_filter(db::hints::host_filter::disabled_for_all_tag {});
|
||||
return proxy.invoke_on_all([filter = std::move(filter)] (service::storage_proxy& sp) {
|
||||
return service::get_storage_proxy().invoke_on_all([filter = std::move(filter)] (service::storage_proxy& sp) {
|
||||
return sp.change_hints_host_filter(filter);
|
||||
}).then([] {
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
});
|
||||
|
||||
sp::get_hinted_handoff_enabled_by_dc.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
sp::get_hinted_handoff_enabled_by_dc.set(r, [](std::unique_ptr<request> req) {
|
||||
std::vector<sstring> res;
|
||||
const auto& filter = proxy.local().get_hints_host_filter();
|
||||
const auto& filter = service::get_storage_proxy().local().get_hints_host_filter();
|
||||
const auto& dcs = filter.get_dcs();
|
||||
res.reserve(res.size());
|
||||
std::copy(dcs.begin(), dcs.end(), std::back_inserter(res));
|
||||
return make_ready_future<json::json_return_type>(res);
|
||||
});
|
||||
|
||||
sp::set_hinted_handoff_enabled_by_dc_list.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
sp::set_hinted_handoff_enabled_by_dc_list.set(r, [](std::unique_ptr<request> req) {
|
||||
auto dcs = req->get_query_param("dcs");
|
||||
auto filter = db::hints::host_filter::parse_from_dc_list(std::move(dcs));
|
||||
return proxy.invoke_on_all([filter = std::move(filter)] (service::storage_proxy& sp) {
|
||||
return service::get_storage_proxy().invoke_on_all([filter = std::move(filter)] (service::storage_proxy& sp) {
|
||||
return sp.change_hints_host_filter(filter);
|
||||
}).then([] {
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
});
|
||||
|
||||
sp::get_max_hint_window.set(r, [](std::unique_ptr<http::request> req) {
|
||||
sp::get_max_hint_window.set(r, [](std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
});
|
||||
|
||||
sp::set_max_hint_window.set(r, [](std::unique_ptr<http::request> req) {
|
||||
sp::set_max_hint_window.set(r, [](std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
auto enable = req->get_query_param("ms");
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
sp::get_max_hints_in_progress.set(r, [](std::unique_ptr<http::request> req) {
|
||||
sp::get_max_hints_in_progress.set(r, [](std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(1);
|
||||
});
|
||||
|
||||
sp::set_max_hints_in_progress.set(r, [](std::unique_ptr<http::request> req) {
|
||||
sp::set_max_hints_in_progress.set(r, [](std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
auto enable = req->get_query_param("qs");
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
sp::get_hints_in_progress.set(r, [](std::unique_ptr<http::request> req) {
|
||||
sp::get_hints_in_progress.set(r, [](std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(0);
|
||||
@@ -262,7 +259,7 @@ void set_storage_proxy(http_context& ctx, routes& r, sharded<service::storage_pr
|
||||
return ctx.db.local().get_config().request_timeout_in_ms()/1000.0;
|
||||
});
|
||||
|
||||
sp::set_rpc_timeout.set(r, [](std::unique_ptr<http::request> req) {
|
||||
sp::set_rpc_timeout.set(r, [](std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
auto enable = req->get_query_param("timeout");
|
||||
@@ -273,7 +270,7 @@ void set_storage_proxy(http_context& ctx, routes& r, sharded<service::storage_pr
|
||||
return ctx.db.local().get_config().read_request_timeout_in_ms()/1000.0;
|
||||
});
|
||||
|
||||
sp::set_read_rpc_timeout.set(r, [](std::unique_ptr<http::request> req) {
|
||||
sp::set_read_rpc_timeout.set(r, [](std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
auto enable = req->get_query_param("timeout");
|
||||
@@ -284,7 +281,7 @@ void set_storage_proxy(http_context& ctx, routes& r, sharded<service::storage_pr
|
||||
return ctx.db.local().get_config().write_request_timeout_in_ms()/1000.0;
|
||||
});
|
||||
|
||||
sp::set_write_rpc_timeout.set(r, [](std::unique_ptr<http::request> req) {
|
||||
sp::set_write_rpc_timeout.set(r, [](std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
auto enable = req->get_query_param("timeout");
|
||||
@@ -295,7 +292,7 @@ void set_storage_proxy(http_context& ctx, routes& r, sharded<service::storage_pr
|
||||
return ctx.db.local().get_config().counter_write_request_timeout_in_ms()/1000.0;
|
||||
});
|
||||
|
||||
sp::set_counter_write_rpc_timeout.set(r, [](std::unique_ptr<http::request> req) {
|
||||
sp::set_counter_write_rpc_timeout.set(r, [](std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
auto enable = req->get_query_param("timeout");
|
||||
@@ -306,7 +303,7 @@ void set_storage_proxy(http_context& ctx, routes& r, sharded<service::storage_pr
|
||||
return ctx.db.local().get_config().cas_contention_timeout_in_ms()/1000.0;
|
||||
});
|
||||
|
||||
sp::set_cas_contention_timeout.set(r, [](std::unique_ptr<http::request> req) {
|
||||
sp::set_cas_contention_timeout.set(r, [](std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
auto enable = req->get_query_param("timeout");
|
||||
@@ -317,7 +314,7 @@ void set_storage_proxy(http_context& ctx, routes& r, sharded<service::storage_pr
|
||||
return ctx.db.local().get_config().range_request_timeout_in_ms()/1000.0;
|
||||
});
|
||||
|
||||
sp::set_range_rpc_timeout.set(r, [](std::unique_ptr<http::request> req) {
|
||||
sp::set_range_rpc_timeout.set(r, [](std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
auto enable = req->get_query_param("timeout");
|
||||
@@ -328,147 +325,160 @@ void set_storage_proxy(http_context& ctx, routes& r, sharded<service::storage_pr
|
||||
return ctx.db.local().get_config().truncate_request_timeout_in_ms()/1000.0;
|
||||
});
|
||||
|
||||
sp::set_truncate_rpc_timeout.set(r, [](std::unique_ptr<http::request> req) {
|
||||
sp::set_truncate_rpc_timeout.set(r, [](std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
auto enable = req->get_query_param("timeout");
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
sp::reload_trigger_classes.set(r, [](std::unique_ptr<http::request> req) {
|
||||
sp::reload_trigger_classes.set(r, [](std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
unimplemented();
|
||||
return make_ready_future<json::json_return_type>(json_void());
|
||||
});
|
||||
|
||||
sp::get_read_repair_attempted.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_stats_storage_proxy(proxy, &service::storage_proxy_stats::stats::read_repair_attempts);
|
||||
sp::get_read_repair_attempted.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_stats_storage_proxy(ctx.sp, &service::storage_proxy_stats::stats::read_repair_attempts);
|
||||
});
|
||||
|
||||
sp::get_read_repair_repaired_blocking.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_stats_storage_proxy(proxy, &service::storage_proxy_stats::stats::read_repair_repaired_blocking);
|
||||
sp::get_read_repair_repaired_blocking.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_stats_storage_proxy(ctx.sp, &service::storage_proxy_stats::stats::read_repair_repaired_blocking);
|
||||
});
|
||||
|
||||
sp::get_read_repair_repaired_background.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_stats_storage_proxy(proxy, &service::storage_proxy_stats::stats::read_repair_repaired_background);
|
||||
sp::get_read_repair_repaired_background.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_stats_storage_proxy(ctx.sp, &service::storage_proxy_stats::stats::read_repair_repaired_background);
|
||||
});
|
||||
|
||||
sp::get_cas_read_timeouts.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timed_rate_as_long(proxy, &proxy::stats::cas_read_timeouts);
|
||||
sp::get_schema_versions.set(r, [&ss](std::unique_ptr<request> req) {
|
||||
return ss.local().describe_schema_versions().then([] (auto result) {
|
||||
std::vector<sp::mapper_list> res;
|
||||
for (auto e : result) {
|
||||
sp::mapper_list entry;
|
||||
entry.key = std::move(e.first);
|
||||
entry.value = std::move(e.second);
|
||||
res.emplace_back(std::move(entry));
|
||||
}
|
||||
return make_ready_future<json::json_return_type>(std::move(res));
|
||||
});
|
||||
});
|
||||
|
||||
sp::get_cas_read_unavailables.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timed_rate_as_long(proxy, &proxy::stats::cas_read_unavailables);
|
||||
sp::get_cas_read_timeouts.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timed_rate_as_long(ctx.sp, &proxy::stats::cas_read_timeouts);
|
||||
});
|
||||
|
||||
sp::get_cas_write_timeouts.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timed_rate_as_long(proxy, &proxy::stats::cas_write_timeouts);
|
||||
sp::get_cas_read_unavailables.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timed_rate_as_long(ctx.sp, &proxy::stats::cas_read_unavailables);
|
||||
});
|
||||
|
||||
sp::get_cas_write_unavailables.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timed_rate_as_long(proxy, &proxy::stats::cas_write_unavailables);
|
||||
sp::get_cas_write_timeouts.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timed_rate_as_long(ctx.sp, &proxy::stats::cas_write_timeouts);
|
||||
});
|
||||
|
||||
sp::get_cas_write_metrics_unfinished_commit.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_stats(proxy, &proxy::stats::cas_write_unfinished_commit);
|
||||
sp::get_cas_write_unavailables.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timed_rate_as_long(ctx.sp, &proxy::stats::cas_write_unavailables);
|
||||
});
|
||||
|
||||
sp::get_cas_write_metrics_contention.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_estimated_histogram(proxy, &proxy::stats::cas_write_contention);
|
||||
sp::get_cas_write_metrics_unfinished_commit.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_stats(ctx.sp, &proxy::stats::cas_write_unfinished_commit);
|
||||
});
|
||||
|
||||
sp::get_cas_write_metrics_condition_not_met.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_stats(proxy, &proxy::stats::cas_write_condition_not_met);
|
||||
sp::get_cas_write_metrics_contention.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_estimated_histogram(ctx, &proxy::stats::cas_write_contention);
|
||||
});
|
||||
|
||||
sp::get_cas_write_metrics_failed_read_round_optimization.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_stats(proxy, &proxy::stats::cas_failed_read_round_optimization);
|
||||
sp::get_cas_write_metrics_condition_not_met.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_stats(ctx.sp, &proxy::stats::cas_write_condition_not_met);
|
||||
});
|
||||
|
||||
sp::get_cas_read_metrics_unfinished_commit.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_stats(proxy, &proxy::stats::cas_read_unfinished_commit);
|
||||
sp::get_cas_write_metrics_failed_read_round_optimization.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_stats(ctx.sp, &proxy::stats::cas_failed_read_round_optimization);
|
||||
});
|
||||
|
||||
sp::get_cas_read_metrics_contention.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_estimated_histogram(proxy, &proxy::stats::cas_read_contention);
|
||||
sp::get_cas_read_metrics_unfinished_commit.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_stats(ctx.sp, &proxy::stats::cas_read_unfinished_commit);
|
||||
});
|
||||
|
||||
sp::get_read_metrics_timeouts.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timed_rate_as_long(proxy, &service::storage_proxy_stats::stats::read_timeouts);
|
||||
sp::get_cas_read_metrics_contention.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_estimated_histogram(ctx, &proxy::stats::cas_read_contention);
|
||||
});
|
||||
|
||||
sp::get_read_metrics_unavailables.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timed_rate_as_long(proxy, &service::storage_proxy_stats::stats::read_unavailables);
|
||||
sp::get_read_metrics_timeouts.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timed_rate_as_long(ctx.sp, &service::storage_proxy_stats::stats::read_timeouts);
|
||||
});
|
||||
|
||||
sp::get_range_metrics_timeouts.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timed_rate_as_long(proxy, &service::storage_proxy_stats::stats::range_slice_timeouts);
|
||||
sp::get_read_metrics_unavailables.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timed_rate_as_long(ctx.sp, &service::storage_proxy_stats::stats::read_unavailables);
|
||||
});
|
||||
|
||||
sp::get_range_metrics_unavailables.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timed_rate_as_long(proxy, &service::storage_proxy_stats::stats::range_slice_unavailables);
|
||||
sp::get_range_metrics_timeouts.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timed_rate_as_long(ctx.sp, &service::storage_proxy_stats::stats::range_slice_timeouts);
|
||||
});
|
||||
|
||||
sp::get_write_metrics_timeouts.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timed_rate_as_long(proxy, &service::storage_proxy_stats::stats::write_timeouts);
|
||||
sp::get_range_metrics_unavailables.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timed_rate_as_long(ctx.sp, &service::storage_proxy_stats::stats::range_slice_unavailables);
|
||||
});
|
||||
|
||||
sp::get_write_metrics_unavailables.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timed_rate_as_long(proxy, &service::storage_proxy_stats::stats::write_unavailables);
|
||||
sp::get_write_metrics_timeouts.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timed_rate_as_long(ctx.sp, &service::storage_proxy_stats::stats::write_timeouts);
|
||||
});
|
||||
|
||||
sp::get_read_metrics_timeouts_rates.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timed_rate_as_obj(proxy, &service::storage_proxy_stats::stats::read_timeouts);
|
||||
sp::get_write_metrics_unavailables.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timed_rate_as_long(ctx.sp, &service::storage_proxy_stats::stats::write_unavailables);
|
||||
});
|
||||
|
||||
sp::get_read_metrics_unavailables_rates.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timed_rate_as_obj(proxy, &service::storage_proxy_stats::stats::read_unavailables);
|
||||
sp::get_read_metrics_timeouts_rates.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timed_rate_as_obj(ctx.sp, &service::storage_proxy_stats::stats::read_timeouts);
|
||||
});
|
||||
|
||||
sp::get_range_metrics_timeouts_rates.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timed_rate_as_obj(proxy, &service::storage_proxy_stats::stats::range_slice_timeouts);
|
||||
sp::get_read_metrics_unavailables_rates.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timed_rate_as_obj(ctx.sp, &service::storage_proxy_stats::stats::read_unavailables);
|
||||
});
|
||||
|
||||
sp::get_range_metrics_unavailables_rates.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timed_rate_as_obj(proxy, &service::storage_proxy_stats::stats::range_slice_unavailables);
|
||||
sp::get_range_metrics_timeouts_rates.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timed_rate_as_obj(ctx.sp, &service::storage_proxy_stats::stats::range_slice_timeouts);
|
||||
});
|
||||
|
||||
sp::get_write_metrics_timeouts_rates.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timed_rate_as_obj(proxy, &service::storage_proxy_stats::stats::write_timeouts);
|
||||
sp::get_range_metrics_unavailables_rates.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timed_rate_as_obj(ctx.sp, &service::storage_proxy_stats::stats::range_slice_unavailables);
|
||||
});
|
||||
|
||||
sp::get_write_metrics_unavailables_rates.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timed_rate_as_obj(proxy, &service::storage_proxy_stats::stats::write_unavailables);
|
||||
sp::get_write_metrics_timeouts_rates.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timed_rate_as_obj(ctx.sp, &service::storage_proxy_stats::stats::write_timeouts);
|
||||
});
|
||||
|
||||
sp::get_range_metrics_latency_histogram_depricated.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_histogram_stats_storage_proxy(proxy, &service::storage_proxy_stats::stats::range);
|
||||
sp::get_write_metrics_unavailables_rates.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timed_rate_as_obj(ctx.sp, &service::storage_proxy_stats::stats::write_unavailables);
|
||||
});
|
||||
|
||||
sp::get_write_metrics_latency_histogram_depricated.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_histogram_stats_storage_proxy(proxy, &service::storage_proxy_stats::stats::write);
|
||||
sp::get_range_metrics_latency_histogram_depricated.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_histogram_stats_storage_proxy(ctx.sp, &service::storage_proxy_stats::stats::range);
|
||||
});
|
||||
|
||||
sp::get_read_metrics_latency_histogram_depricated.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_histogram_stats_storage_proxy(proxy, &service::storage_proxy_stats::stats::read);
|
||||
sp::get_write_metrics_latency_histogram_depricated.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_histogram_stats_storage_proxy(ctx.sp, &service::storage_proxy_stats::stats::write);
|
||||
});
|
||||
|
||||
sp::get_range_metrics_latency_histogram.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timer_stats_storage_proxy(proxy, &service::storage_proxy_stats::stats::range);
|
||||
sp::get_read_metrics_latency_histogram_depricated.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_histogram_stats_storage_proxy(ctx.sp, &service::storage_proxy_stats::stats::read);
|
||||
});
|
||||
|
||||
sp::get_write_metrics_latency_histogram.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timer_stats_storage_proxy(proxy, &service::storage_proxy_stats::stats::write);
|
||||
});
|
||||
sp::get_cas_write_metrics_latency_histogram.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timer_stats(proxy, &proxy::stats::cas_write);
|
||||
sp::get_range_metrics_latency_histogram.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timer_stats_storage_proxy(ctx.sp, &service::storage_proxy_stats::stats::range);
|
||||
});
|
||||
|
||||
sp::get_cas_read_metrics_latency_histogram.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timer_stats(proxy, &proxy::stats::cas_read);
|
||||
sp::get_write_metrics_latency_histogram.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timer_stats_storage_proxy(ctx.sp, &service::storage_proxy_stats::stats::write);
|
||||
});
|
||||
sp::get_cas_write_metrics_latency_histogram.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timer_stats(ctx.sp, &proxy::stats::cas_write);
|
||||
});
|
||||
|
||||
sp::get_view_write_metrics_latency_histogram.set(r, [](std::unique_ptr<http::request> req) {
|
||||
sp::get_cas_read_metrics_latency_histogram.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timer_stats(ctx.sp, &proxy::stats::cas_read);
|
||||
});
|
||||
|
||||
sp::get_view_write_metrics_latency_histogram.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
//TBD
|
||||
// FIXME
|
||||
// No View metrics are available, so just return empty moving average
|
||||
@@ -476,100 +486,32 @@ void set_storage_proxy(http_context& ctx, routes& r, sharded<service::storage_pr
|
||||
return make_ready_future<json::json_return_type>(get_empty_moving_average());
|
||||
});
|
||||
|
||||
sp::get_read_metrics_latency_histogram.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timer_stats_storage_proxy(proxy, &service::storage_proxy_stats::stats::read);
|
||||
sp::get_read_metrics_latency_histogram.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timer_stats_storage_proxy(ctx.sp, &service::storage_proxy_stats::stats::read);
|
||||
});
|
||||
|
||||
sp::get_read_estimated_histogram.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_estimated_histogram(proxy, &service::storage_proxy_stats::stats::read);
|
||||
sp::get_read_estimated_histogram.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_estimated_histogram(ctx, &service::storage_proxy_stats::stats::estimated_read);
|
||||
});
|
||||
|
||||
sp::get_read_latency.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return total_latency(proxy, &service::storage_proxy_stats::stats::read);
|
||||
sp::get_read_latency.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return total_latency(ctx, &service::storage_proxy_stats::stats::read);
|
||||
});
|
||||
sp::get_write_estimated_histogram.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_estimated_histogram(proxy, &service::storage_proxy_stats::stats::write);
|
||||
sp::get_write_estimated_histogram.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_estimated_histogram(ctx, &service::storage_proxy_stats::stats::estimated_write);
|
||||
});
|
||||
|
||||
sp::get_write_latency.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return total_latency(proxy, &service::storage_proxy_stats::stats::write);
|
||||
sp::get_write_latency.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return total_latency(ctx, &service::storage_proxy_stats::stats::write);
|
||||
});
|
||||
|
||||
sp::get_range_estimated_histogram.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return sum_timer_stats_storage_proxy(proxy, &service::storage_proxy_stats::stats::range);
|
||||
sp::get_range_estimated_histogram.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return sum_timer_stats_storage_proxy(ctx.sp, &service::storage_proxy_stats::stats::range);
|
||||
});
|
||||
|
||||
sp::get_range_latency.set(r, [&proxy](std::unique_ptr<http::request> req) {
|
||||
return total_latency(proxy, &service::storage_proxy_stats::stats::range);
|
||||
sp::get_range_latency.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
return total_latency(ctx, &service::storage_proxy_stats::stats::range);
|
||||
});
|
||||
}
|
||||
|
||||
void unset_storage_proxy(http_context& ctx, routes& r) {
|
||||
sp::get_total_hints.unset(r);
|
||||
sp::get_hinted_handoff_enabled.unset(r);
|
||||
sp::set_hinted_handoff_enabled.unset(r);
|
||||
sp::get_hinted_handoff_enabled_by_dc.unset(r);
|
||||
sp::set_hinted_handoff_enabled_by_dc_list.unset(r);
|
||||
sp::get_max_hint_window.unset(r);
|
||||
sp::set_max_hint_window.unset(r);
|
||||
sp::get_max_hints_in_progress.unset(r);
|
||||
sp::set_max_hints_in_progress.unset(r);
|
||||
sp::get_hints_in_progress.unset(r);
|
||||
sp::get_rpc_timeout.unset(r);
|
||||
sp::set_rpc_timeout.unset(r);
|
||||
sp::get_read_rpc_timeout.unset(r);
|
||||
sp::set_read_rpc_timeout.unset(r);
|
||||
sp::get_write_rpc_timeout.unset(r);
|
||||
sp::set_write_rpc_timeout.unset(r);
|
||||
sp::get_counter_write_rpc_timeout.unset(r);
|
||||
sp::set_counter_write_rpc_timeout.unset(r);
|
||||
sp::get_cas_contention_timeout.unset(r);
|
||||
sp::set_cas_contention_timeout.unset(r);
|
||||
sp::get_range_rpc_timeout.unset(r);
|
||||
sp::set_range_rpc_timeout.unset(r);
|
||||
sp::get_truncate_rpc_timeout.unset(r);
|
||||
sp::set_truncate_rpc_timeout.unset(r);
|
||||
sp::reload_trigger_classes.unset(r);
|
||||
sp::get_read_repair_attempted.unset(r);
|
||||
sp::get_read_repair_repaired_blocking.unset(r);
|
||||
sp::get_read_repair_repaired_background.unset(r);
|
||||
sp::get_cas_read_timeouts.unset(r);
|
||||
sp::get_cas_read_unavailables.unset(r);
|
||||
sp::get_cas_write_timeouts.unset(r);
|
||||
sp::get_cas_write_unavailables.unset(r);
|
||||
sp::get_cas_write_metrics_unfinished_commit.unset(r);
|
||||
sp::get_cas_write_metrics_contention.unset(r);
|
||||
sp::get_cas_write_metrics_condition_not_met.unset(r);
|
||||
sp::get_cas_write_metrics_failed_read_round_optimization.unset(r);
|
||||
sp::get_cas_read_metrics_unfinished_commit.unset(r);
|
||||
sp::get_cas_read_metrics_contention.unset(r);
|
||||
sp::get_read_metrics_timeouts.unset(r);
|
||||
sp::get_read_metrics_unavailables.unset(r);
|
||||
sp::get_range_metrics_timeouts.unset(r);
|
||||
sp::get_range_metrics_unavailables.unset(r);
|
||||
sp::get_write_metrics_timeouts.unset(r);
|
||||
sp::get_write_metrics_unavailables.unset(r);
|
||||
sp::get_read_metrics_timeouts_rates.unset(r);
|
||||
sp::get_read_metrics_unavailables_rates.unset(r);
|
||||
sp::get_range_metrics_timeouts_rates.unset(r);
|
||||
sp::get_range_metrics_unavailables_rates.unset(r);
|
||||
sp::get_write_metrics_timeouts_rates.unset(r);
|
||||
sp::get_write_metrics_unavailables_rates.unset(r);
|
||||
sp::get_range_metrics_latency_histogram_depricated.unset(r);
|
||||
sp::get_write_metrics_latency_histogram_depricated.unset(r);
|
||||
sp::get_read_metrics_latency_histogram_depricated.unset(r);
|
||||
sp::get_range_metrics_latency_histogram.unset(r);
|
||||
sp::get_write_metrics_latency_histogram.unset(r);
|
||||
sp::get_cas_write_metrics_latency_histogram.unset(r);
|
||||
sp::get_cas_read_metrics_latency_histogram.unset(r);
|
||||
sp::get_view_write_metrics_latency_histogram.unset(r);
|
||||
sp::get_read_metrics_latency_histogram.unset(r);
|
||||
sp::get_read_estimated_histogram.unset(r);
|
||||
sp::get_read_latency.unset(r);
|
||||
sp::get_write_estimated_histogram.unset(r);
|
||||
sp::get_write_latency.unset(r);
|
||||
sp::get_range_estimated_histogram.unset(r);
|
||||
sp::get_range_latency.unset(r);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -11,11 +11,10 @@
|
||||
#include <seastar/core/sharded.hh>
|
||||
#include "api.hh"
|
||||
|
||||
namespace service { class storage_proxy; }
|
||||
namespace service { class storage_service; }
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_storage_proxy(http_context& ctx, httpd::routes& r, sharded<service::storage_proxy>& proxy);
|
||||
void unset_storage_proxy(http_context& ctx, httpd::routes& r);
|
||||
void set_storage_proxy(http_context& ctx, routes& r, sharded<service::storage_service>& ss);
|
||||
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -8,8 +8,6 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include <seastar/core/sharded.hh>
|
||||
#include "api.hh"
|
||||
#include "db/data_listeners.hh"
|
||||
@@ -25,6 +23,7 @@ class system_keyspace;
|
||||
}
|
||||
namespace netw { class messaging_service; }
|
||||
class repair_service;
|
||||
namespace cdc { class generation_service; }
|
||||
class sstables_loader;
|
||||
|
||||
namespace gms {
|
||||
@@ -35,42 +34,28 @@ class gossiper;
|
||||
|
||||
namespace api {
|
||||
|
||||
// verify that the keyspace is found, otherwise a bad_param_exception exception is thrown
|
||||
// containing the description of the respective keyspace error.
|
||||
sstring validate_keyspace(http_context& ctx, sstring ks_name);
|
||||
|
||||
// verify that the keyspace parameter is found, otherwise a bad_param_exception exception is thrown
|
||||
// containing the description of the respective keyspace error.
|
||||
sstring validate_keyspace(http_context& ctx, const httpd::parameters& param);
|
||||
sstring validate_keyspace(http_context& ctx, const parameters& param);
|
||||
|
||||
// splits a request parameter assumed to hold a comma-separated list of table names
|
||||
// verify that the tables are found, otherwise a bad_param_exception exception is thrown
|
||||
// containing the description of the respective no_such_column_family error.
|
||||
// Returns an empty vector if no parameter was found.
|
||||
// If the parameter is found and empty, returns a list of all table names in the keyspace.
|
||||
std::vector<sstring> parse_tables(const sstring& ks_name, http_context& ctx, const std::unordered_map<sstring, sstring>& query_params, sstring param_name);
|
||||
|
||||
// splits a request parameter assumed to hold a comma-separated list of table names
|
||||
// verify that the tables are found, otherwise a bad_param_exception exception is thrown
|
||||
// containing the description of the respective no_such_column_family error.
|
||||
// Returns a vector of all table infos given by the parameter, or
|
||||
// if the parameter is not found or is empty, returns a list of all table infos in the keyspace.
|
||||
std::vector<table_info> parse_table_infos(const sstring& ks_name, http_context& ctx, const std::unordered_map<sstring, sstring>& query_params, sstring param_name);
|
||||
|
||||
void set_storage_service(http_context& ctx, httpd::routes& r, sharded<service::storage_service>& ss, service::raft_group0_client&);
|
||||
void unset_storage_service(http_context& ctx, httpd::routes& r);
|
||||
void set_sstables_loader(http_context& ctx, httpd::routes& r, sharded<sstables_loader>& sst_loader);
|
||||
void unset_sstables_loader(http_context& ctx, httpd::routes& r);
|
||||
void set_view_builder(http_context& ctx, httpd::routes& r, sharded<db::view::view_builder>& vb);
|
||||
void unset_view_builder(http_context& ctx, httpd::routes& r);
|
||||
void set_repair(http_context& ctx, httpd::routes& r, sharded<repair_service>& repair);
|
||||
void unset_repair(http_context& ctx, httpd::routes& r);
|
||||
void set_transport_controller(http_context& ctx, httpd::routes& r, cql_transport::controller& ctl);
|
||||
void unset_transport_controller(http_context& ctx, httpd::routes& r);
|
||||
void set_rpc_controller(http_context& ctx, httpd::routes& r, thrift_controller& ctl);
|
||||
void unset_rpc_controller(http_context& ctx, httpd::routes& r);
|
||||
void set_snapshot(http_context& ctx, httpd::routes& r, sharded<db::snapshot_ctl>& snap_ctl);
|
||||
void unset_snapshot(http_context& ctx, httpd::routes& r);
|
||||
void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_service>& ss, gms::gossiper& g, sharded<cdc::generation_service>& cdc_gs, sharded<db::system_keyspace>& sys_ls);
|
||||
void set_sstables_loader(http_context& ctx, routes& r, sharded<sstables_loader>& sst_loader);
|
||||
void unset_sstables_loader(http_context& ctx, routes& r);
|
||||
void set_view_builder(http_context& ctx, routes& r, sharded<db::view::view_builder>& vb);
|
||||
void unset_view_builder(http_context& ctx, routes& r);
|
||||
void set_repair(http_context& ctx, routes& r, sharded<repair_service>& repair);
|
||||
void unset_repair(http_context& ctx, routes& r);
|
||||
void set_transport_controller(http_context& ctx, routes& r, cql_transport::controller& ctl);
|
||||
void unset_transport_controller(http_context& ctx, routes& r);
|
||||
void set_rpc_controller(http_context& ctx, routes& r, thrift_controller& ctl);
|
||||
void unset_rpc_controller(http_context& ctx, routes& r);
|
||||
void set_snapshot(http_context& ctx, routes& r, sharded<db::snapshot_ctl>& snap_ctl);
|
||||
void unset_snapshot(http_context& ctx, routes& r);
|
||||
seastar::future<json::json_return_type> run_toppartitions_query(db::toppartitions_query& q, http_context &ctx, bool legacy_request = false);
|
||||
|
||||
} // namespace api
|
||||
}
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
#include "gms/gossiper.hh"
|
||||
|
||||
namespace api {
|
||||
using namespace seastar::httpd;
|
||||
|
||||
namespace hs = httpd::stream_manager_json;
|
||||
|
||||
@@ -22,7 +21,7 @@ static void set_summaries(const std::vector<streaming::stream_summary>& from,
|
||||
json::json_list<hs::stream_summary>& to) {
|
||||
if (!from.empty()) {
|
||||
hs::stream_summary res;
|
||||
res.cf_id = fmt::to_string(from.front().cf_id);
|
||||
res.cf_id = boost::lexical_cast<std::string>(from.front().cf_id);
|
||||
// For each stream_session, we pretend we are sending/receiving one
|
||||
// file, to make it compatible with nodetool.
|
||||
res.files = 1;
|
||||
@@ -39,7 +38,7 @@ static hs::progress_info get_progress_info(const streaming::progress_info& info)
|
||||
res.current_bytes = info.current_bytes;
|
||||
res.direction = info.dir;
|
||||
res.file_name = info.file_name;
|
||||
res.peer = fmt::to_string(info.peer);
|
||||
res.peer = boost::lexical_cast<std::string>(info.peer);
|
||||
res.session_index = 0;
|
||||
res.total_bytes = info.total_bytes;
|
||||
return res;
|
||||
@@ -62,7 +61,7 @@ static hs::stream_state get_state(
|
||||
state.plan_id = result_future.plan_id.to_sstring();
|
||||
for (auto info : result_future.get_coordinator().get()->get_all_session_info()) {
|
||||
hs::stream_info si;
|
||||
si.peer = fmt::to_string(info.peer);
|
||||
si.peer = boost::lexical_cast<std::string>(info.peer);
|
||||
si.session_index = 0;
|
||||
si.state = info.state;
|
||||
si.connecting = si.peer;
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_stream_manager(http_context& ctx, httpd::routes& r, sharded<streaming::stream_manager>& sm);
|
||||
void unset_stream_manager(http_context& ctx, httpd::routes& r);
|
||||
void set_stream_manager(http_context& ctx, routes& r, sharded<streaming::stream_manager>& sm);
|
||||
void unset_stream_manager(http_context& ctx, routes& r);
|
||||
|
||||
}
|
||||
|
||||
@@ -7,98 +7,20 @@
|
||||
*/
|
||||
|
||||
#include "api/api-doc/system.json.hh"
|
||||
#include "api/api-doc/metrics.json.hh"
|
||||
|
||||
#include "api/api.hh"
|
||||
|
||||
#include <seastar/core/reactor.hh>
|
||||
#include <seastar/core/metrics_api.hh>
|
||||
#include <seastar/core/relabel_config.hh>
|
||||
#include <seastar/http/exception.hh>
|
||||
#include <seastar/util/short_streams.hh>
|
||||
#include <seastar/http/short_streams.hh>
|
||||
#include "utils/rjson.hh"
|
||||
|
||||
#include "log.hh"
|
||||
#include "replica/database.hh"
|
||||
|
||||
extern logging::logger apilog;
|
||||
|
||||
namespace api {
|
||||
using namespace seastar::httpd;
|
||||
|
||||
namespace hs = httpd::system_json;
|
||||
namespace hm = httpd::metrics_json;
|
||||
|
||||
void set_system(http_context& ctx, routes& r) {
|
||||
hm::get_metrics_config.set(r, [](const_req req) {
|
||||
std::vector<hm::metrics_config> res;
|
||||
res.resize(seastar::metrics::get_relabel_configs().size());
|
||||
size_t i = 0;
|
||||
for (auto&& r : seastar::metrics::get_relabel_configs()) {
|
||||
res[i].action = r.action;
|
||||
res[i].target_label = r.target_label;
|
||||
res[i].replacement = r.replacement;
|
||||
res[i].separator = r.separator;
|
||||
res[i].source_labels = r.source_labels;
|
||||
res[i].regex = r.expr.str();
|
||||
i++;
|
||||
}
|
||||
return res;
|
||||
});
|
||||
|
||||
hm::set_metrics_config.set(r, [](std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
rapidjson::Document doc;
|
||||
doc.Parse(req->content.c_str());
|
||||
if (!doc.IsArray()) {
|
||||
throw bad_param_exception("Expected a json array");
|
||||
}
|
||||
std::vector<seastar::metrics::relabel_config> relabels;
|
||||
relabels.resize(doc.Size());
|
||||
for (rapidjson::SizeType i = 0; i < doc.Size(); i++) {
|
||||
const auto& element = doc[i];
|
||||
if (element.HasMember("source_labels")) {
|
||||
std::vector<std::string> source_labels;
|
||||
source_labels.resize(element["source_labels"].Size());
|
||||
|
||||
for (size_t j = 0; j < element["source_labels"].Size(); j++) {
|
||||
source_labels[j] = element["source_labels"][j].GetString();
|
||||
}
|
||||
relabels[i].source_labels = source_labels;
|
||||
}
|
||||
if (element.HasMember("action")) {
|
||||
relabels[i].action = seastar::metrics::relabel_config_action(element["action"].GetString());
|
||||
}
|
||||
if (element.HasMember("replacement")) {
|
||||
relabels[i].replacement = element["replacement"].GetString();
|
||||
}
|
||||
if (element.HasMember("separator")) {
|
||||
relabels[i].separator = element["separator"].GetString();
|
||||
}
|
||||
if (element.HasMember("target_label")) {
|
||||
relabels[i].target_label = element["target_label"].GetString();
|
||||
}
|
||||
if (element.HasMember("regex")) {
|
||||
relabels[i].expr = element["regex"].GetString();
|
||||
}
|
||||
}
|
||||
return do_with(std::move(relabels), false, [](const std::vector<seastar::metrics::relabel_config>& relabels, bool& failed) {
|
||||
return smp::invoke_on_all([&relabels, &failed] {
|
||||
return metrics::set_relabel_configs(relabels).then([&failed](const metrics::metric_relabeling_result& result) {
|
||||
if (result.metrics_relabeled_due_to_collision > 0) {
|
||||
failed = true;
|
||||
}
|
||||
return;
|
||||
});
|
||||
}).then([&failed](){
|
||||
if (failed) {
|
||||
throw bad_param_exception("conflicts found during relabeling");
|
||||
}
|
||||
return make_ready_future<json::json_return_type>(seastar::json::json_void());
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
hs::get_system_uptime.set(r, [](const_req req) {
|
||||
return std::chrono::duration_cast<std::chrono::milliseconds>(engine().uptime()).count();
|
||||
});
|
||||
@@ -139,16 +61,6 @@ void set_system(http_context& ctx, routes& r) {
|
||||
return json::json_void();
|
||||
});
|
||||
|
||||
hs::write_log_message.set(r, [](const_req req) {
|
||||
try {
|
||||
logging::log_level level = boost::lexical_cast<logging::log_level>(std::string(req.get_query_param("level")));
|
||||
apilog.log(level, "/system/log: {}", std::string(req.get_query_param("message")));
|
||||
} catch (boost::bad_lexical_cast& e) {
|
||||
throw bad_param_exception("Unknown logging level " + req.get_query_param("level"));
|
||||
}
|
||||
return json::json_void();
|
||||
});
|
||||
|
||||
hs::drop_sstable_caches.set(r, [&ctx](std::unique_ptr<request> req) {
|
||||
apilog.info("Dropping sstable caches");
|
||||
return ctx.db.invoke_on_all([] (replica::database& db) {
|
||||
|
||||
@@ -12,6 +12,6 @@
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_system(http_context& ctx, httpd::routes& r);
|
||||
void set_system(http_context& ctx, routes& r);
|
||||
|
||||
}
|
||||
|
||||
@@ -1,277 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2022-present ScyllaDB
|
||||
*/
|
||||
|
||||
/*
|
||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
*/
|
||||
|
||||
#include <seastar/core/coroutine.hh>
|
||||
|
||||
#include "task_manager.hh"
|
||||
#include "api/api-doc/task_manager.json.hh"
|
||||
#include "db/system_keyspace.hh"
|
||||
#include "column_family.hh"
|
||||
#include "unimplemented.hh"
|
||||
#include "storage_service.hh"
|
||||
|
||||
#include <utility>
|
||||
#include <boost/range/adaptors.hpp>
|
||||
|
||||
namespace api {
|
||||
|
||||
namespace tm = httpd::task_manager_json;
|
||||
using namespace json;
|
||||
using namespace seastar::httpd;
|
||||
|
||||
inline bool filter_tasks(tasks::task_manager::task_ptr task, std::unordered_map<sstring, sstring>& query_params) {
|
||||
return (!query_params.contains("keyspace") || query_params["keyspace"] == task->get_status().keyspace) &&
|
||||
(!query_params.contains("table") || query_params["table"] == task->get_status().table);
|
||||
}
|
||||
|
||||
struct full_task_status {
|
||||
tasks::task_manager::task::status task_status;
|
||||
std::string type;
|
||||
tasks::task_manager::task::progress progress;
|
||||
std::string module;
|
||||
tasks::task_id parent_id;
|
||||
tasks::is_abortable abortable;
|
||||
std::vector<std::string> children_ids;
|
||||
};
|
||||
|
||||
struct task_stats {
|
||||
task_stats(tasks::task_manager::task_ptr task)
|
||||
: task_id(task->id().to_sstring())
|
||||
, state(task->get_status().state)
|
||||
, type(task->type())
|
||||
, scope(task->get_status().scope)
|
||||
, keyspace(task->get_status().keyspace)
|
||||
, table(task->get_status().table)
|
||||
, entity(task->get_status().entity)
|
||||
, sequence_number(task->get_status().sequence_number)
|
||||
{ }
|
||||
|
||||
sstring task_id;
|
||||
tasks::task_manager::task_state state;
|
||||
std::string type;
|
||||
std::string scope;
|
||||
std::string keyspace;
|
||||
std::string table;
|
||||
std::string entity;
|
||||
uint64_t sequence_number;
|
||||
};
|
||||
|
||||
tm::task_status make_status(full_task_status status) {
|
||||
auto start_time = db_clock::to_time_t(status.task_status.start_time);
|
||||
auto end_time = db_clock::to_time_t(status.task_status.end_time);
|
||||
::tm st, et;
|
||||
::gmtime_r(&end_time, &et);
|
||||
::gmtime_r(&start_time, &st);
|
||||
|
||||
tm::task_status res{};
|
||||
res.id = status.task_status.id.to_sstring();
|
||||
res.type = status.type;
|
||||
res.scope = status.task_status.scope;
|
||||
res.state = status.task_status.state;
|
||||
res.is_abortable = bool(status.abortable);
|
||||
res.start_time = st;
|
||||
res.end_time = et;
|
||||
res.error = status.task_status.error;
|
||||
res.parent_id = status.parent_id.to_sstring();
|
||||
res.sequence_number = status.task_status.sequence_number;
|
||||
res.shard = status.task_status.shard;
|
||||
res.keyspace = status.task_status.keyspace;
|
||||
res.table = status.task_status.table;
|
||||
res.entity = status.task_status.entity;
|
||||
res.progress_units = status.task_status.progress_units;
|
||||
res.progress_total = status.progress.total;
|
||||
res.progress_completed = status.progress.completed;
|
||||
res.children_ids = std::move(status.children_ids);
|
||||
return res;
|
||||
}
|
||||
|
||||
future<full_task_status> retrieve_status(const tasks::task_manager::foreign_task_ptr& task) {
|
||||
if (task.get() == nullptr) {
|
||||
co_return coroutine::return_exception(httpd::bad_param_exception("Task not found"));
|
||||
}
|
||||
auto progress = co_await task->get_progress();
|
||||
full_task_status s;
|
||||
s.task_status = task->get_status();
|
||||
s.type = task->type();
|
||||
s.parent_id = task->get_parent_id();
|
||||
s.abortable = task->is_abortable();
|
||||
s.module = task->get_module_name();
|
||||
s.progress.completed = progress.completed;
|
||||
s.progress.total = progress.total;
|
||||
std::vector<std::string> ct{task->get_children().size()};
|
||||
boost::transform(task->get_children(), ct.begin(), [] (const auto& child) {
|
||||
return child->id().to_sstring();
|
||||
});
|
||||
s.children_ids = std::move(ct);
|
||||
co_return s;
|
||||
}
|
||||
|
||||
void set_task_manager(http_context& ctx, routes& r, sharded<tasks::task_manager>& tm, db::config& cfg) {
|
||||
tm::get_modules.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
std::vector<std::string> v = boost::copy_range<std::vector<std::string>>(tm.local().get_modules() | boost::adaptors::map_keys);
|
||||
co_return v;
|
||||
});
|
||||
|
||||
tm::get_tasks.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
using chunked_stats = utils::chunked_vector<task_stats>;
|
||||
auto internal = tasks::is_internal{req_param<bool>(*req, "internal", false)};
|
||||
std::vector<chunked_stats> res = co_await tm.map([&req, internal] (tasks::task_manager& tm) {
|
||||
chunked_stats local_res;
|
||||
tasks::task_manager::module_ptr module;
|
||||
try {
|
||||
module = tm.find_module(req->param["module"]);
|
||||
} catch (...) {
|
||||
throw bad_param_exception(fmt::format("{}", std::current_exception()));
|
||||
}
|
||||
const auto& filtered_tasks = module->get_tasks() | boost::adaptors::filtered([¶ms = req->query_parameters, internal] (const auto& task) {
|
||||
return (internal || !task.second->is_internal()) && filter_tasks(task.second, params);
|
||||
});
|
||||
for (auto& [task_id, task] : filtered_tasks) {
|
||||
local_res.push_back(task_stats{task});
|
||||
}
|
||||
return local_res;
|
||||
});
|
||||
|
||||
std::function<future<>(output_stream<char>&&)> f = [r = std::move(res)] (output_stream<char>&& os) -> future<> {
|
||||
auto s = std::move(os);
|
||||
auto res = std::move(r);
|
||||
co_await s.write("[");
|
||||
std::string delim = "";
|
||||
for (auto& v: res) {
|
||||
for (auto& stats: v) {
|
||||
co_await s.write(std::exchange(delim, ", "));
|
||||
tm::task_stats ts;
|
||||
ts = stats;
|
||||
co_await formatter::write(s, ts);
|
||||
}
|
||||
}
|
||||
co_await s.write("]");
|
||||
co_await s.close();
|
||||
};
|
||||
co_return std::move(f);
|
||||
});
|
||||
|
||||
tm::get_task_status.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
auto id = tasks::task_id{utils::UUID{req->param["task_id"]}};
|
||||
tasks::task_manager::foreign_task_ptr task;
|
||||
try {
|
||||
task = co_await tasks::task_manager::invoke_on_task(tm, id, std::function([] (tasks::task_manager::task_ptr task) -> future<tasks::task_manager::foreign_task_ptr> {
|
||||
if (task->is_complete()) {
|
||||
task->unregister_task();
|
||||
}
|
||||
co_return std::move(task);
|
||||
}));
|
||||
} catch (tasks::task_manager::task_not_found& e) {
|
||||
throw bad_param_exception(e.what());
|
||||
}
|
||||
auto s = co_await retrieve_status(task);
|
||||
co_return make_status(s);
|
||||
});
|
||||
|
||||
tm::abort_task.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
auto id = tasks::task_id{utils::UUID{req->param["task_id"]}};
|
||||
try {
|
||||
co_await tasks::task_manager::invoke_on_task(tm, id, [] (tasks::task_manager::task_ptr task) -> future<> {
|
||||
if (!task->is_abortable()) {
|
||||
co_await coroutine::return_exception(std::runtime_error("Requested task cannot be aborted"));
|
||||
}
|
||||
co_await task->abort();
|
||||
});
|
||||
} catch (tasks::task_manager::task_not_found& e) {
|
||||
throw bad_param_exception(e.what());
|
||||
}
|
||||
co_return json_void();
|
||||
});
|
||||
|
||||
tm::wait_task.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
auto id = tasks::task_id{utils::UUID{req->param["task_id"]}};
|
||||
tasks::task_manager::foreign_task_ptr task;
|
||||
try {
|
||||
task = co_await tasks::task_manager::invoke_on_task(tm, id, std::function([] (tasks::task_manager::task_ptr task) {
|
||||
return task->done().then_wrapped([task] (auto f) {
|
||||
task->unregister_task();
|
||||
// done() is called only because we want the task to be complete before getting its status.
|
||||
// The future should be ignored here as the result does not matter.
|
||||
f.ignore_ready_future();
|
||||
return make_foreign(task);
|
||||
});
|
||||
}));
|
||||
} catch (tasks::task_manager::task_not_found& e) {
|
||||
throw bad_param_exception(e.what());
|
||||
}
|
||||
auto s = co_await retrieve_status(task);
|
||||
co_return make_status(s);
|
||||
});
|
||||
|
||||
tm::get_task_status_recursively.set(r, [&_tm = tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
auto& tm = _tm;
|
||||
auto id = tasks::task_id{utils::UUID{req->param["task_id"]}};
|
||||
std::queue<tasks::task_manager::foreign_task_ptr> q;
|
||||
utils::chunked_vector<full_task_status> res;
|
||||
|
||||
tasks::task_manager::foreign_task_ptr task;
|
||||
try {
|
||||
// Get requested task.
|
||||
task = co_await tasks::task_manager::invoke_on_task(tm, id, std::function([] (tasks::task_manager::task_ptr task) -> future<tasks::task_manager::foreign_task_ptr> {
|
||||
if (task->is_complete()) {
|
||||
task->unregister_task();
|
||||
}
|
||||
co_return task;
|
||||
}));
|
||||
} catch (tasks::task_manager::task_not_found& e) {
|
||||
throw bad_param_exception(e.what());
|
||||
}
|
||||
|
||||
// Push children's statuses in BFS order.
|
||||
q.push(co_await task.copy()); // Task cannot be moved since we need it to be alive during whole loop execution.
|
||||
while (!q.empty()) {
|
||||
auto& current = q.front();
|
||||
res.push_back(co_await retrieve_status(current));
|
||||
for (size_t i = 0; i < current->get_children().size(); ++i) {
|
||||
q.push(co_await current->get_children()[i].copy());
|
||||
}
|
||||
q.pop();
|
||||
}
|
||||
|
||||
std::function<future<>(output_stream<char>&&)> f = [r = std::move(res)] (output_stream<char>&& os) -> future<> {
|
||||
auto s = std::move(os);
|
||||
auto res = std::move(r);
|
||||
co_await s.write("[");
|
||||
std::string delim = "";
|
||||
for (auto& status: res) {
|
||||
co_await s.write(std::exchange(delim, ", "));
|
||||
co_await formatter::write(s, make_status(status));
|
||||
}
|
||||
co_await s.write("]");
|
||||
co_await s.close();
|
||||
};
|
||||
co_return f;
|
||||
});
|
||||
|
||||
tm::get_and_update_ttl.set(r, [&cfg] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
uint32_t ttl = cfg.task_ttl_seconds();
|
||||
try {
|
||||
co_await cfg.task_ttl_seconds.set_value_on_all_shards(req->query_parameters["ttl"], utils::config_file::config_source::API);
|
||||
} catch (...) {
|
||||
throw bad_param_exception(fmt::format("{}", std::current_exception()));
|
||||
}
|
||||
co_return json::json_return_type(ttl);
|
||||
});
|
||||
}
|
||||
|
||||
void unset_task_manager(http_context& ctx, routes& r) {
|
||||
tm::get_modules.unset(r);
|
||||
tm::get_tasks.unset(r);
|
||||
tm::get_task_status.unset(r);
|
||||
tm::abort_task.unset(r);
|
||||
tm::wait_task.unset(r);
|
||||
tm::get_task_status_recursively.unset(r);
|
||||
tm::get_and_update_ttl.unset(r);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2022-present ScyllaDB
|
||||
*/
|
||||
|
||||
/*
|
||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <seastar/core/sharded.hh>
|
||||
#include "api.hh"
|
||||
#include "db/config.hh"
|
||||
|
||||
namespace tasks {
|
||||
class task_manager;
|
||||
}
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_task_manager(http_context& ctx, httpd::routes& r, sharded<tasks::task_manager>& tm, db::config& cfg);
|
||||
void unset_task_manager(http_context& ctx, httpd::routes& r);
|
||||
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2022-present ScyllaDB
|
||||
*/
|
||||
|
||||
/*
|
||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
*/
|
||||
|
||||
#ifndef SCYLLA_BUILD_MODE_RELEASE
|
||||
|
||||
#include <seastar/core/coroutine.hh>
|
||||
|
||||
#include "task_manager_test.hh"
|
||||
#include "api/api-doc/task_manager_test.json.hh"
|
||||
#include "tasks/test_module.hh"
|
||||
|
||||
namespace api {
|
||||
|
||||
namespace tmt = httpd::task_manager_test_json;
|
||||
using namespace json;
|
||||
using namespace seastar::httpd;
|
||||
|
||||
void set_task_manager_test(http_context& ctx, routes& r, sharded<tasks::task_manager>& tm) {
|
||||
tmt::register_test_module.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
co_await tm.invoke_on_all([] (tasks::task_manager& tm) {
|
||||
auto m = make_shared<tasks::test_module>(tm);
|
||||
tm.register_module("test", m);
|
||||
});
|
||||
co_return json_void();
|
||||
});
|
||||
|
||||
tmt::unregister_test_module.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
co_await tm.invoke_on_all([] (tasks::task_manager& tm) -> future<> {
|
||||
auto module_name = "test";
|
||||
auto module = tm.find_module(module_name);
|
||||
co_await module->stop();
|
||||
});
|
||||
co_return json_void();
|
||||
});
|
||||
|
||||
tmt::register_test_task.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
sharded<tasks::task_manager>& tms = tm;
|
||||
auto it = req->query_parameters.find("task_id");
|
||||
auto id = it != req->query_parameters.end() ? tasks::task_id{utils::UUID{it->second}} : tasks::task_id::create_null_id();
|
||||
it = req->query_parameters.find("shard");
|
||||
unsigned shard = it != req->query_parameters.end() ? boost::lexical_cast<unsigned>(it->second) : 0;
|
||||
it = req->query_parameters.find("keyspace");
|
||||
std::string keyspace = it != req->query_parameters.end() ? it->second : "";
|
||||
it = req->query_parameters.find("table");
|
||||
std::string table = it != req->query_parameters.end() ? it->second : "";
|
||||
it = req->query_parameters.find("entity");
|
||||
std::string entity = it != req->query_parameters.end() ? it->second : "";
|
||||
it = req->query_parameters.find("parent_id");
|
||||
tasks::task_info data;
|
||||
if (it != req->query_parameters.end()) {
|
||||
data.id = tasks::task_id{utils::UUID{it->second}};
|
||||
auto parent_ptr = co_await tasks::task_manager::lookup_task_on_all_shards(tm, data.id);
|
||||
data.shard = parent_ptr->get_status().shard;
|
||||
}
|
||||
|
||||
auto module = tms.local().find_module("test");
|
||||
id = co_await module->make_task<tasks::test_task_impl>(shard, id, keyspace, table, entity, data);
|
||||
co_await tms.invoke_on(shard, [id] (tasks::task_manager& tm) {
|
||||
auto it = tm.get_all_tasks().find(id);
|
||||
if (it != tm.get_all_tasks().end()) {
|
||||
it->second->start();
|
||||
}
|
||||
});
|
||||
co_return id.to_sstring();
|
||||
});
|
||||
|
||||
tmt::unregister_test_task.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
auto id = tasks::task_id{utils::UUID{req->query_parameters["task_id"]}};
|
||||
try {
|
||||
co_await tasks::task_manager::invoke_on_task(tm, id, [] (tasks::task_manager::task_ptr task) -> future<> {
|
||||
tasks::test_task test_task{task};
|
||||
co_await test_task.unregister_task();
|
||||
});
|
||||
} catch (tasks::task_manager::task_not_found& e) {
|
||||
throw bad_param_exception(e.what());
|
||||
}
|
||||
co_return json_void();
|
||||
});
|
||||
|
||||
tmt::finish_test_task.set(r, [&tm] (std::unique_ptr<http::request> req) -> future<json::json_return_type> {
|
||||
auto id = tasks::task_id{utils::UUID{req->param["task_id"]}};
|
||||
auto it = req->query_parameters.find("error");
|
||||
bool fail = it != req->query_parameters.end();
|
||||
std::string error = fail ? it->second : "";
|
||||
|
||||
try {
|
||||
co_await tasks::task_manager::invoke_on_task(tm, id, [fail, error = std::move(error)] (tasks::task_manager::task_ptr task) {
|
||||
tasks::test_task test_task{task};
|
||||
if (fail) {
|
||||
test_task.finish_failed(std::make_exception_ptr(std::runtime_error(error)));
|
||||
} else {
|
||||
test_task.finish();
|
||||
}
|
||||
return make_ready_future<>();
|
||||
});
|
||||
} catch (tasks::task_manager::task_not_found& e) {
|
||||
throw bad_param_exception(e.what());
|
||||
}
|
||||
co_return json_void();
|
||||
});
|
||||
}
|
||||
|
||||
void unset_task_manager_test(http_context& ctx, routes& r) {
|
||||
tmt::register_test_module.unset(r);
|
||||
tmt::unregister_test_module.unset(r);
|
||||
tmt::register_test_task.unset(r);
|
||||
tmt::unregister_test_task.unset(r);
|
||||
tmt::finish_test_task.unset(r);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -1,27 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2022-present ScyllaDB
|
||||
*/
|
||||
|
||||
/*
|
||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
*/
|
||||
|
||||
#ifndef SCYLLA_BUILD_MODE_RELEASE
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <seastar/core/sharded.hh>
|
||||
#include "api.hh"
|
||||
|
||||
namespace tasks {
|
||||
class task_manager;
|
||||
}
|
||||
|
||||
namespace api {
|
||||
|
||||
void set_task_manager_test(http_context& ctx, httpd::routes& r, sharded<tasks::task_manager>& tm);
|
||||
void unset_task_manager_test(http_context& ctx, httpd::routes& r);
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -9,7 +9,7 @@
|
||||
#include "atomic_cell.hh"
|
||||
#include "atomic_cell_or_collection.hh"
|
||||
#include "counters.hh"
|
||||
#include "types/types.hh"
|
||||
#include "types.hh"
|
||||
|
||||
atomic_cell atomic_cell::make_dead(api::timestamp_type timestamp, gc_clock::time_point deletion_time) {
|
||||
return atomic_cell_type::make_dead(timestamp, deletion_time);
|
||||
@@ -206,13 +206,14 @@ operator<<(std::ostream& os, const atomic_cell_view::printer& acvp) {
|
||||
std::ostringstream cell_value_string_builder;
|
||||
if (type.is_counter()) {
|
||||
if (acv.is_counter_update()) {
|
||||
fmt::print(cell_value_string_builder, "counter_update_value={}", acv.counter_update_value());
|
||||
cell_value_string_builder << "counter_update_value=" << acv.counter_update_value();
|
||||
} else {
|
||||
cell_value_string_builder << "shards: ";
|
||||
auto ccv = counter_cell_view(acv);
|
||||
fmt::print(cell_value_string_builder, "shards: {}", fmt::join(ccv.shards(), ", "));
|
||||
cell_value_string_builder << ::join(", ", ccv.shards());
|
||||
}
|
||||
} else {
|
||||
fmt::print(cell_value_string_builder, "{}", type.to_string(to_bytes(acv.value())));
|
||||
cell_value_string_builder << type.to_string(to_bytes(acv.value()));
|
||||
}
|
||||
fmt::print(os, "atomic_cell{{{},ts={:d},expiry={:d},ttl={:d}}}",
|
||||
cell_value_string_builder.str(),
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
#include "bytes.hh"
|
||||
#include "timestamp.hh"
|
||||
#include "mutation/tombstone.hh"
|
||||
#include "tombstone.hh"
|
||||
#include "gc_clock.hh"
|
||||
#include "utils/managed_bytes.hh"
|
||||
#include <seastar/net//byteorder.hh>
|
||||
@@ -10,11 +10,11 @@
|
||||
|
||||
// Not part of atomic_cell.hh to avoid cyclic dependency between types.hh and atomic_cell.hh
|
||||
|
||||
#include "types/types.hh"
|
||||
#include "types.hh"
|
||||
#include "types/collection.hh"
|
||||
#include "atomic_cell.hh"
|
||||
#include "atomic_cell_or_collection.hh"
|
||||
#include "utils/hashing.hh"
|
||||
#include "hashing.hh"
|
||||
#include "counters.hh"
|
||||
|
||||
template<>
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
#include "atomic_cell.hh"
|
||||
#include "collection_mutation.hh"
|
||||
#include "schema/schema.hh"
|
||||
#include "schema.hh"
|
||||
|
||||
// A variant type that can hold either an atomic_cell, or a serialized collection.
|
||||
// Which type is stored is determined by the schema.
|
||||
@@ -1,37 +0,0 @@
|
||||
include(add_whole_archive)
|
||||
|
||||
add_library(scylla_auth STATIC)
|
||||
target_sources(scylla_auth
|
||||
PRIVATE
|
||||
allow_all_authenticator.cc
|
||||
allow_all_authorizer.cc
|
||||
authenticated_user.cc
|
||||
authenticator.cc
|
||||
certificate_authenticator.cc
|
||||
common.cc
|
||||
default_authorizer.cc
|
||||
password_authenticator.cc
|
||||
passwords.cc
|
||||
permission.cc
|
||||
permissions_cache.cc
|
||||
resource.cc
|
||||
role_or_anonymous.cc
|
||||
roles-metadata.cc
|
||||
sasl_challenge.cc
|
||||
service.cc
|
||||
standard_role_manager.cc
|
||||
transitional.cc)
|
||||
target_include_directories(scylla_auth
|
||||
PUBLIC
|
||||
${CMAKE_SOURCE_DIR})
|
||||
target_link_libraries(scylla_auth
|
||||
PUBLIC
|
||||
Seastar::seastar
|
||||
xxHash::xxhash
|
||||
PRIVATE
|
||||
cql3
|
||||
idl
|
||||
wasmtime_bindings
|
||||
libxcrypt::libxcrypt)
|
||||
|
||||
add_whole_archive(auth scylla_auth)
|
||||
@@ -10,12 +10,24 @@
|
||||
|
||||
#include "auth/authenticated_user.hh"
|
||||
|
||||
#include <iostream>
|
||||
|
||||
namespace auth {
|
||||
|
||||
authenticated_user::authenticated_user(std::string_view name)
|
||||
: name(sstring(name)) {
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream& os, const authenticated_user& u) {
|
||||
if (!u.name) {
|
||||
os << "anonymous";
|
||||
} else {
|
||||
os << *u.name;
|
||||
}
|
||||
|
||||
return os;
|
||||
}
|
||||
|
||||
static const authenticated_user the_anonymous_user{};
|
||||
|
||||
const authenticated_user& anonymous_user() noexcept {
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
|
||||
#include <string_view>
|
||||
#include <functional>
|
||||
#include <iosfwd>
|
||||
#include <optional>
|
||||
|
||||
#include <seastar/core/sstring.hh>
|
||||
@@ -35,9 +36,21 @@ public:
|
||||
///
|
||||
authenticated_user() = default;
|
||||
explicit authenticated_user(std::string_view name);
|
||||
friend bool operator==(const authenticated_user&, const authenticated_user&) noexcept = default;
|
||||
};
|
||||
|
||||
///
|
||||
/// The user name, or "anonymous".
|
||||
///
|
||||
std::ostream& operator<<(std::ostream&, const authenticated_user&);
|
||||
|
||||
inline bool operator==(const authenticated_user& u1, const authenticated_user& u2) noexcept {
|
||||
return u1.name == u2.name;
|
||||
}
|
||||
|
||||
inline bool operator!=(const authenticated_user& u1, const authenticated_user& u2) noexcept {
|
||||
return !(u1 == u2);
|
||||
}
|
||||
|
||||
const authenticated_user& anonymous_user() noexcept;
|
||||
|
||||
inline bool is_anonymous(const authenticated_user& u) noexcept {
|
||||
@@ -46,21 +59,6 @@ inline bool is_anonymous(const authenticated_user& u) noexcept {
|
||||
|
||||
}
|
||||
|
||||
///
|
||||
/// The user name, or "anonymous".
|
||||
///
|
||||
template <>
|
||||
struct fmt::formatter<auth::authenticated_user> : fmt::formatter<std::string_view> {
|
||||
template <typename FormatContext>
|
||||
auto format(const auth::authenticated_user& u, FormatContext& ctx) const {
|
||||
if (u.name) {
|
||||
return fmt::format_to(ctx.out(), "{}", *u.name);
|
||||
} else {
|
||||
return fmt::format_to(ctx.out(), "{}", "anonymous");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
namespace std {
|
||||
|
||||
template <>
|
||||
|
||||
24
auth/authentication_options.cc
Normal file
24
auth/authentication_options.cc
Normal file
@@ -0,0 +1,24 @@
|
||||
/*
|
||||
* Copyright (C) 2018-present ScyllaDB
|
||||
*/
|
||||
|
||||
/*
|
||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
*/
|
||||
|
||||
#include "auth/authentication_options.hh"
|
||||
|
||||
#include <iostream>
|
||||
|
||||
namespace auth {
|
||||
|
||||
std::ostream& operator<<(std::ostream& os, authentication_option a) {
|
||||
switch (a) {
|
||||
case authentication_option::password: os << "PASSWORD"; break;
|
||||
case authentication_option::options: os << "OPTIONS"; break;
|
||||
}
|
||||
|
||||
return os;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -26,6 +26,8 @@ enum class authentication_option {
|
||||
options
|
||||
};
|
||||
|
||||
std::ostream& operator<<(std::ostream&, authentication_option);
|
||||
|
||||
using authentication_option_set = std::unordered_set<authentication_option>;
|
||||
|
||||
using custom_options = std::unordered_map<sstring, sstring>;
|
||||
@@ -47,18 +49,3 @@ public:
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
template <>
|
||||
struct fmt::formatter<auth::authentication_option> : fmt::formatter<std::string_view> {
|
||||
template <typename FormatContext>
|
||||
auto format(const auth::authentication_option a, FormatContext& ctx) const {
|
||||
using enum auth::authentication_option;
|
||||
switch (a) {
|
||||
case password:
|
||||
return formatter<std::string_view>::format("PASSWORD", ctx);
|
||||
case options:
|
||||
return formatter<std::string_view>::format("OPTIONS", ctx);
|
||||
}
|
||||
std::abort();
|
||||
}
|
||||
};
|
||||
|
||||
@@ -18,7 +18,3 @@
|
||||
|
||||
const sstring auth::authenticator::USERNAME_KEY("username");
|
||||
const sstring auth::authenticator::PASSWORD_KEY("password");
|
||||
|
||||
future<std::optional<auth::authenticated_user>> auth::authenticator::authenticate(session_dn_func) const {
|
||||
return make_ready_future<std::optional<auth::authenticated_user>>(std::nullopt);
|
||||
}
|
||||
|
||||
@@ -15,8 +15,6 @@
|
||||
#include <set>
|
||||
#include <stdexcept>
|
||||
#include <unordered_map>
|
||||
#include <optional>
|
||||
#include <functional>
|
||||
|
||||
#include <seastar/core/enum.hh>
|
||||
#include <seastar/core/future.hh>
|
||||
@@ -38,16 +36,6 @@ namespace auth {
|
||||
|
||||
class authenticated_user;
|
||||
|
||||
// Query alt name info as a single (subject style) string
|
||||
using alt_name_func = std::function<future<std::string>()>;
|
||||
|
||||
struct certificate_info {
|
||||
std::string subject;
|
||||
alt_name_func get_alt_names;
|
||||
};
|
||||
|
||||
using session_dn_func = std::function<future<std::optional<certificate_info>>()>;
|
||||
|
||||
///
|
||||
/// Abstract client for authenticating role identity.
|
||||
///
|
||||
@@ -99,13 +87,6 @@ public:
|
||||
///
|
||||
virtual future<authenticated_user> authenticate(const credentials_map& credentials) const = 0;
|
||||
|
||||
///
|
||||
/// Authenticate (early) using transport info
|
||||
///
|
||||
/// \returns nullopt if not supported/required. exceptional future if failed
|
||||
///
|
||||
virtual future<std::optional<authenticated_user>> authenticate(session_dn_func) const;
|
||||
|
||||
///
|
||||
/// Create an authentication record for a new user. This is required before the user can log-in.
|
||||
///
|
||||
|
||||
@@ -39,6 +39,10 @@ inline bool operator==(const permission_details& pd1, const permission_details&
|
||||
== std::forward_as_tuple(pd2.role_name, pd2.resource, pd2.permissions.mask());
|
||||
}
|
||||
|
||||
inline bool operator!=(const permission_details& pd1, const permission_details& pd2) {
|
||||
return !(pd1 == pd2);
|
||||
}
|
||||
|
||||
inline bool operator<(const permission_details& pd1, const permission_details& pd2) {
|
||||
return std::forward_as_tuple(pd1.role_name, pd1.resource, pd1.permissions)
|
||||
< std::forward_as_tuple(pd2.role_name, pd2.resource, pd2.permissions);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user