From 20fffcdcf536aabe7409abf39e28c9034fee3598 Mon Sep 17 00:00:00 2001 From: Aleksandra Martyniuk Date: Thu, 4 Jul 2024 16:49:20 +0200 Subject: [PATCH] nodetool: tasks: add nodetool commands to track task manager tasks --- .../nodetool-commands/tasks/abort.rst | 22 + .../nodetool-commands/tasks/index.rst | 29 ++ .../nodetool-commands/tasks/list.rst | 64 +++ .../nodetool-commands/tasks/modules.rst | 17 + .../nodetool-commands/tasks/status.rst | 47 ++ .../nodetool-commands/tasks/tree.rst | 51 +++ .../nodetool-commands/tasks/ttl.rst | 34 ++ .../nodetool-commands/tasks/wait.rst | 68 +++ docs/operating-scylla/nodetool.rst | 2 + tools/scylla-nodetool.cc | 415 +++++++++++++++++- 10 files changed, 748 insertions(+), 1 deletion(-) create mode 100644 docs/operating-scylla/nodetool-commands/tasks/abort.rst create mode 100644 docs/operating-scylla/nodetool-commands/tasks/index.rst create mode 100644 docs/operating-scylla/nodetool-commands/tasks/list.rst create mode 100644 docs/operating-scylla/nodetool-commands/tasks/modules.rst create mode 100644 docs/operating-scylla/nodetool-commands/tasks/status.rst create mode 100644 docs/operating-scylla/nodetool-commands/tasks/tree.rst create mode 100644 docs/operating-scylla/nodetool-commands/tasks/ttl.rst create mode 100644 docs/operating-scylla/nodetool-commands/tasks/wait.rst diff --git a/docs/operating-scylla/nodetool-commands/tasks/abort.rst b/docs/operating-scylla/nodetool-commands/tasks/abort.rst new file mode 100644 index 0000000000..aeb2d34332 --- /dev/null +++ b/docs/operating-scylla/nodetool-commands/tasks/abort.rst @@ -0,0 +1,22 @@ +Nodetool tasks abort +==================== +**tasks abort** - Aborts a task manager task with the provided id if the task +is abortable. If the task is not abortable, the appropriate message with failure +reason will be printed. + +Syntax +------- +.. code-block:: console + + nodetool tasks abort + +For example: + +.. code-block:: shell + + > nodetool tasks abort ef1b7a61-66c8-494c-bb03-6f65724e6eee + +See also +-------- + +- :doc:`tasks list ` diff --git a/docs/operating-scylla/nodetool-commands/tasks/index.rst b/docs/operating-scylla/nodetool-commands/tasks/index.rst new file mode 100644 index 0000000000..de57a11c91 --- /dev/null +++ b/docs/operating-scylla/nodetool-commands/tasks/index.rst @@ -0,0 +1,29 @@ +Nodetool tasks +============== + +.. toctree:: + :hidden: + + abort + list + modules + status + tree + ttl + wait + +**tasks** - Nodetool supercommand for managing task manager tasks. + +Task manager is an API-based tool for tracking long-running background operations, such as repair or compaction, +which makes them observable and controllable. Task manager operates per node. + +Supported tasks suboperations +----------------------------- + +* :doc:`abort ` - Aborts the task. +* :doc:`list ` - Lists tasks in the module. +* :doc:`modules ` - Lists supported modules. +* :doc:`status ` - Gets status of the task. +* :doc:`tree ` - Gets statuses of the task and all its descendants. +* :doc:`ttl ` - Gets or sets task_ttl value. +* :doc:`wait ` - Waits for the task and gets its status. diff --git a/docs/operating-scylla/nodetool-commands/tasks/list.rst b/docs/operating-scylla/nodetool-commands/tasks/list.rst new file mode 100644 index 0000000000..fa104aeaea --- /dev/null +++ b/docs/operating-scylla/nodetool-commands/tasks/list.rst @@ -0,0 +1,64 @@ +Nodetool tasks list +========================= +**tasks list** - Gets the list of task manager tasks in a provided module. +An operation may be repeated if appropriate flags are set. + + +Syntax +------- +.. code-block:: console + + nodetool tasks list [--internal] [(--keyspace | -ks )] + [(--table | -t
)] [--interval ] [(--iterations | -i )] + +Options +------- + +* ``--internal`` - if set, internal tasks are listed. Internal tasks are the ones that + have a parent or cover an operation that is invoked internally. +* ``--keyspace`` or ``-ks`` - shows only tasks on the specified keyspace. +* ``--table`` or ``-t`` - shows only tasks on the specified table. +* ``--interval`` - repeats the operation periodically at specified time intervals. +* ``--iterations`` or ``-i`` - repeats the operation specified number of times. + +For example: + +Shows all repair tasks on keyspace `myks` and table `mytable`: + +.. code-block:: shell + + > nodetool tasks list repair --internal -ks myks --table mytable + +Shows all non-internal compaction tasks and repeats the operation 3 times every 5 seconds: + +.. code-block:: shell + + > nodetool tasks list compaction --interval 5 --i 3 + +Example output +-------------- + +For single list: + +.. code-block:: shell + + task_id type kind scope state sequence_number keyspace table entity + 5116ddb6-85b5-4c3e-94fb-72128f15d7b4 repair node keyspace done 3 abc + +With repetition: + +.. code-block:: shell + + task_id type kind scope state sequence_number keyspace table entity + d8926ee7-0faf-47b7-bfeb-82477e0c7b33 repair node keyspace done 5 abc + 1e028cb8-31a3-45ed-8728-af7a1ab586f6 repair node keyspace done 4 abc + + task_id type kind scope state sequence_number keyspace table entity + 1e535f9b-97fa-4788-a956-8f3216a6ea8d repair node keyspace done 6 abc + d8926ee7-0faf-47b7-bfeb-82477e0c7b33 repair node keyspace done 5 abc + 1e028cb8-31a3-45ed-8728-af7a1ab586f6 repair node keyspace done 4 abc + +See also +-------- + +- :doc:`tasks status ` diff --git a/docs/operating-scylla/nodetool-commands/tasks/modules.rst b/docs/operating-scylla/nodetool-commands/tasks/modules.rst new file mode 100644 index 0000000000..292990f797 --- /dev/null +++ b/docs/operating-scylla/nodetool-commands/tasks/modules.rst @@ -0,0 +1,17 @@ +Nodetool tasks modules +=========================== +**tasks modules** - Lists all modules supported by task manager. + +Syntax +------- +.. code-block:: console + + nodetool tasks modules + +Example output +-------------- + +.. code-block:: shell + + repair + compaction diff --git a/docs/operating-scylla/nodetool-commands/tasks/status.rst b/docs/operating-scylla/nodetool-commands/tasks/status.rst new file mode 100644 index 0000000000..c02964b098 --- /dev/null +++ b/docs/operating-scylla/nodetool-commands/tasks/status.rst @@ -0,0 +1,47 @@ +Nodetool tasks status +========================= +**tasks status** - Gets the status of a task manager task. If the task was finished it is unregistered. + +Syntax +------- +.. code-block:: console + + nodetool tasks status + +For example: + +.. code-block:: shell + + > nodetool tasks status ef1b7a61-66c8-494c-bb03-6f65724e6eee + +Example output +-------------- + +.. code-block:: shell + + id: 52b69817-aac7-47bf-9d8d-0b487dd6866a + type: repair + kind: node + scope: keyspace + state: done + is_abortable: true + start_time: 2024-07-29T15:48:55Z + end_time: 2024-07-29T15:48:55Z + error: + parent_id: none + sequence_number: 5 + shard: 0 + keyspace: abc + table: + entity: + progress_units: ranges + progress_total: 4 + progress_completed: 4 + children_ids: [{task_id: 9de80fd8-296e-432b-837e-63daf7c93e17, node: 127.0.0.1 }, {task_id: fa1c1338-abd1-4f0c-8ce1-8a806486e5c3, node: 127.0.0.1 }] + +See also +-------- + +- :doc:`tasks tree ` +- :doc:`tasks list ` +- :doc:`tasks wait ` diff --git a/docs/operating-scylla/nodetool-commands/tasks/tree.rst b/docs/operating-scylla/nodetool-commands/tasks/tree.rst new file mode 100644 index 0000000000..e1d84a60d2 --- /dev/null +++ b/docs/operating-scylla/nodetool-commands/tasks/tree.rst @@ -0,0 +1,51 @@ +Nodetool tasks tree +======================= +**tasks tree** - Gets the statuses of a task manager task and all its descendants. +The statuses are listed in BFS order. If the task was finished it is unregistered. + +If task_id isn't specified, trees of all non-internal tasks are printed +(internal tasks are the ones that have a parent or cover an operation that +is invoked internally). + +Syntax +------- +.. code-block:: console + + nodetool tasks tree [] + +For example: + +.. code-block:: shell + + > nodetool tasks tree 2ef0a5b6-f243-4c01-876f-54539b453766 + +Example output +-------------- + +For single task: + +.. code-block:: shell + + id type kind scope state is_abortable start_time end_time error parent_id sequence_number shard keyspace table entity progress_units total completed children_ids + be5559ea-bc5a-428c-b8ce-d14eac7a1765 repair node keyspace done true 2024-07-29T16:06:46Z 2024-07-29T16:06:46Z none 1 0 abc ranges 4 4 [{task_id: 542e38cb-9ad4-40aa-9010-de2630004e55, node: 127.0.0.1 }, {task_id: 8974ebcc-1e87-4040-88fe-f2438261f7fb, node: 127.0.0.1 }] + 542e38cb-9ad4-40aa-9010-de2630004e55 repair node shard done false 2024-07-29T16:06:46Z 2024-07-29T16:06:46Z be5559ea-bc5a-428c-b8ce-d14eac7a1765 1 0 abc ranges 2 2 [] + 8974ebcc-1e87-4040-88fe-f2438261f7fb repair node shard done false 2024-07-29T16:06:46Z 2024-07-29T16:06:46Z be5559ea-bc5a-428c-b8ce-d14eac7a1765 1 1 abc ranges 2 2 [] + +For all tasks: + +.. code-block:: shell + + id type kind scope state is_abortable start_time end_time error parent_id sequence_number shard keyspace table entity progress_units total completed children_ids + 16eafb1e-8b2e-48e6-bd7a-432ca3d8b9fc repair node keyspace done true 2024-07-29T16:34:46Z 2024-07-29T16:34:46Z none 1 0 abc ranges 4 4 [{task_id: e0aa1aa4-58ca-4bfb-b3e6-74e5f3a0f6ee, node: 127.0.0.1 }, {task_id: 49eb5797-b67e-46b0-9365-4460f7cf988a, node: 127.0.0.1 }] + e0aa1aa4-58ca-4bfb-b3e6-74e5f3a0f6ee repair node shard done false 2024-07-29T16:34:46Z 2024-07-29T16:34:46Z 16eafb1e-8b2e-48e6-bd7a-432ca3d8b9fc 1 0 abc ranges 2 2 [] + 49eb5797-b67e-46b0-9365-4460f7cf988a repair node shard done false 2024-07-29T16:34:46Z 2024-07-29T16:34:46Z 16eafb1e-8b2e-48e6-bd7a-432ca3d8b9fc 1 1 abc ranges 2 2 [] + 82d7b2a4-146e-4a72-ba93-c66d5b4e9867 offstrategy compaction node keyspace done true 2024-07-29T16:34:16Z 2024-07-29T16:34:16Z none 954 0 abc 1 1 [{task_id: 9818277b-238d-4298-a56b-c0d2153bf140, node: 127.0.0.1 }, {task_id: c1eb0701-ad7a-45ff-956f-7b8d671fc5db, node: 127.0.0.1 } + 9818277b-238d-4298-a56b-c0d2153bf140 offstrategy compaction node shard done false 2024-07-29T16:34:16Z 2024-07-29T16:34:16Z 82d7b2a4-146e-4a72-ba93-c66d5b4e9867 954 0 abc 1 1 [] + c1eb0701-ad7a-45ff-956f-7b8d671fc5db offstrategy compaction node shard done false 2024-07-29T16:34:16Z 2024-07-29T16:34:16Z 82d7b2a4-146e-4a72-ba93-c66d5b4e9867 954 1 abc 1 1 [] + +See also +-------- + +- :doc:`tasks status ` +- :doc:`tasks list ` +- :doc:`tasks wait ` diff --git a/docs/operating-scylla/nodetool-commands/tasks/ttl.rst b/docs/operating-scylla/nodetool-commands/tasks/ttl.rst new file mode 100644 index 0000000000..ccda2d3d63 --- /dev/null +++ b/docs/operating-scylla/nodetool-commands/tasks/ttl.rst @@ -0,0 +1,34 @@ +Nodetool tasks ttl +================== +**tasks ttl** - Gets or sets task_ttl value in seconds, i.e. time for which tasks are kept in task manager after +they are finished. + +The new value is set only in memory. To change the configuration, modify ``task_ttl_in_seconds`` in ``scylla.yaml``. +You can also run Scylla with ``--task-ttl-in-seconds`` parameter. + +If task_ttl == 0, tasks are unregistered from task manager immediately after they are finished. + +Syntax +------- +.. code-block:: console + + nodetool tasks ttl [--set ] + +Options +------- + +* ``--set`` - sets task_ttl to the specified value. + +For example: + +Gets task_ttl value: + +.. code-block:: shell + + > nodetool tasks ttl + +Sets task_ttl value to 10 seconds: + +.. code-block:: shell + + > nodetool tasks ttl --set 10 diff --git a/docs/operating-scylla/nodetool-commands/tasks/wait.rst b/docs/operating-scylla/nodetool-commands/tasks/wait.rst new file mode 100644 index 0000000000..f82c71e545 --- /dev/null +++ b/docs/operating-scylla/nodetool-commands/tasks/wait.rst @@ -0,0 +1,68 @@ +Nodetool tasks wait +=================== +**tasks wait** - Waits until a task manager task is finished and gets its status. +If timeout is set and it expires, the appropriate message with failure reason +will be printed. + +Syntax +------- +.. code-block:: console + + nodetool tasks wait [(--quiet|-q)] [--timeout ] + +Options +------- + +* ``--quiet`` or ``-q`` - if set, status of a task isn't printed. Instead the proper exit code is returned: + + * 0 - if task finished successfully + * 123 - if task failed + * 124 - if request timed out + * 125 - if an error occurred and the task status is undetermined + +* ``--timeout`` - timeout in seconds. + +Syntax +------- +.. code-block:: console + + nodetool tasks wait + +For example: + +.. code-block:: shell + + > nodetool tasks wait ef1b7a61-66c8-494c-bb03-6f65724e6eee + +Example output +-------------- + +.. code-block:: shell + + id : 29dd6552-1e9a-4f17-b2c9-231d088fbee6 + type : repair + kind : node + scope : keyspace + state : done + is_abortable : true + start_time : 2024-07-29T17:00:53Z + end_time : 2024-07-29T17:00:53Z + error : + parent_id : none + sequence_number : 2 + shard : 0 + keyspace : abc + table : + entity : + progress_units : ranges + progress_total : 4 + progress_completed : 4 + children_ids : [{task_id: 5d74a62e-aaf6-4993-b0f1-973899d89cd0, node: 127.0.0.1 }, {task_id: 055d5a59-d97c-45a8-a7e6-dcad39ed61ca, node: 127.0.0.1 }] + + +See also +-------- + +- :doc:`tasks status ` +- :doc:`tasks tree ` +- :doc:`tasks list ` diff --git a/docs/operating-scylla/nodetool.rst b/docs/operating-scylla/nodetool.rst index f5b34bf591..549242c13a 100644 --- a/docs/operating-scylla/nodetool.rst +++ b/docs/operating-scylla/nodetool.rst @@ -54,6 +54,7 @@ Nodetool nodetool-commands/status Nodetool stop compaction nodetool-commands/tablestats + nodetool-commands/tasks/index nodetool-commands/toppartitions nodetool-commands/upgradesstables nodetool-commands/viewbuildstatus @@ -136,6 +137,7 @@ Operations that are not listed below are currently not available. * :doc:`stop ` - Stop compaction operation. * **tablehistograms** see :doc:`cfhistograms ` * :doc:`tablestats ` - Provides in-depth diagnostics regard table. +* :doc:`tasks ` - Manage tasks manager tasks. * :doc:`toppartitions ` - Samples cluster writes and reads and reports the most active partitions in a specified table and time frame. * :doc:`upgradesstables ` - Upgrades each table that is not running the latest ScyllaDB version, by rewriting SSTables. * :doc:`viewbuildstatus ` - Shows the progress of a materialized view build. diff --git a/tools/scylla-nodetool.cc b/tools/scylla-nodetool.cc index c1776278c8..83fdf0bf59 100644 --- a/tools/scylla-nodetool.cc +++ b/tools/scylla-nodetool.cc @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -124,7 +125,13 @@ struct operation_failed_with_status : public std::runtime_error { }; struct api_request_failed : public std::runtime_error { + http::reply::status_type status; + using std::runtime_error::runtime_error; + api_request_failed(http::reply::status_type s, sstring message) + : std::runtime_error(std::move(message)) + , status(s) + {} }; class scylla_rest_client { @@ -156,7 +163,7 @@ class scylla_rest_client { } catch (...) { message = res; } - throw api_request_failed(fmt::format("error executing {} request to {} with parameters {}: remote replied with status code {}:\n{}", + throw api_request_failed(status, fmt::format("error executing {} request to {} with parameters {}: remote replied with status code {}:\n{}", type, url, params, status, message)); } @@ -2680,6 +2687,267 @@ void table_stats_operation(scylla_rest_client& client, const bpo::variables_map& } } +void tasks_print_status(const rjson::value& res) { + auto status = res.GetObject(); + for (const auto& x: status) { + if (x.value.IsString()) { + fmt::print("{}: {}\n", x.name.GetString(), x.value.GetString()); + } else if (x.value.IsArray()) { + fmt::print("{}: [", x.name.GetString()); + sstring delim = ""; + for (const auto& el : x.value.GetArray()) { + fmt::print("{}{{", delim); + sstring ident_delim = ""; + for (const auto& child_info : el.GetObject()) { + fmt::print("{}{}: {}", ident_delim, child_info.name.GetString(), child_info.value.GetString()); + ident_delim = ", "; + } + fmt::print(" }}"); + delim = ", "; + } + fmt::print("]\n"); + } else { + fmt::print("{}: {}\n", x.name.GetString(), x.value); + } + } +} + +void tasks_add_tree_to_statuses_lists(Tabulate& table, const rjson::value& res) { + auto statuses = res.GetArray(); + for (auto& element : statuses) { + const auto& status = element.GetObject(); + + sstring children_ids = "["; + sstring delim = ""; + if (status.HasMember("children_ids")) { + for (const auto& el : status["children_ids"].GetArray()) { + children_ids += format("{}{{", delim); + sstring ident_delim = ""; + for (const auto& child_info : el.GetObject()) { + children_ids += format("{}{}: {}", ident_delim, child_info.name.GetString(), child_info.value.GetString()); + ident_delim = ", "; + } + children_ids += format(" }}"); + delim = ", "; + } + } + children_ids += "]"; + + table.add(rjson::to_string_view(status["id"]), + rjson::to_string_view(status["type"]), + rjson::to_string_view(status["kind"]), + rjson::to_string_view(status["scope"]), + rjson::to_string_view(status["state"]), + status["is_abortable"].GetBool(), + rjson::to_string_view(status["start_time"]), + rjson::to_string_view(status["end_time"]), + rjson::to_string_view(status["error"]), + rjson::to_string_view(status["parent_id"]), + status["sequence_number"].GetUint64(), + status["shard"].GetUint(), + rjson::to_string_view(status["keyspace"]), + rjson::to_string_view(status["table"]), + rjson::to_string_view(status["entity"]), + rjson::to_string_view(status["progress_units"]), + status["progress_total"].GetDouble(), + status["progress_completed"].GetDouble(), + children_ids); + } +} + +void tasks_print_trees(const std::vector& res) { + Tabulate table; + table.add("id", "type", "kind", "scope", "state", + "is_abortable", "start_time", "end_time", "error", "parent_id", + "sequence_number", "shard", "keyspace", "table", "entity", + "progress_units", "total", "completed", "children_ids"); + + for (const auto& res_el : res) { + tasks_add_tree_to_statuses_lists(table, res_el); + } + + table.print(); +} + +void tasks_print_stats_list(const rjson::value& res) { + auto stats = res.GetArray(); + Tabulate table; + table.add("task_id", "type", "kind", "scope", "state", "sequence_number", "keyspace", "table", "entity"); + for (auto& element : stats) { + const auto& s = element.GetObject(); + + table.add(rjson::to_string_view(s["task_id"]), + rjson::to_string_view(s["type"]), + rjson::to_string_view(s["kind"]), + rjson::to_string_view(s["scope"]), + rjson::to_string_view(s["state"]), + s["sequence_number"].GetUint64(), + rjson::to_string_view(s["keyspace"]), + rjson::to_string_view(s["table"]), + rjson::to_string_view(s["entity"])); + } + table.print(); +} + +void tasks_abort_operation(scylla_rest_client& client, const bpo::variables_map& vm) { + if (!vm.contains("id")) { + throw std::invalid_argument("required parameter is missing: id"); + } + auto id = vm["id"].as(); + + try { + auto res = client.post(format("/task_manager/abort_task/{}", id)); + } catch (const api_request_failed& e) { + if (e.status != http::reply::status_type::forbidden) { + throw; + } + + fmt::print("Task with id {} is not abortable\n", id); + } +} + +void tasks_list_operation(scylla_rest_client& client, const bpo::variables_map& vm) { + if (!vm.contains("module")) { + throw std::invalid_argument("required parameter is missing: module"); + } + auto module = vm["module"].as(); + std::unordered_map params; + if (vm.contains("keyspace")) { + params["keyspace"] = vm["keyspace"].as(); + } + if (vm.contains("table")) { + params["table"] = vm["table"].as(); + } + if (vm.contains("internal")) { + params["internal"] = "true"; + } + int interval = vm["interval"].as(); + int iterations = vm["iterations"].as(); + + auto res = client.get(format("/task_manager/list_module_tasks/{}", module), params); + tasks_print_stats_list(res); + + for (auto i = 0; i < iterations; ++i) { + sleep(interval); + auto res = client.get(format("/task_manager/list_module_tasks/{}", module), params); + fmt::print("\n\n"); + tasks_print_stats_list(res); + } +} + +void tasks_modules_operation(scylla_rest_client& client, const bpo::variables_map& vm) { + auto res = client.get("/task_manager/list_modules"); + + for (auto& x: res.GetArray()) { + fmt::print("{}\n", rjson::to_string_view(x)); + } +} + +void tasks_status_operation(scylla_rest_client& client, const bpo::variables_map& vm) { + if (!vm.contains("id")) { + throw std::invalid_argument("required parameter is missing: id"); + } + auto id = vm["id"].as(); + + auto res = client.get(format("/task_manager/task_status/{}", id)); + + tasks_print_status(res); +} + +void tasks_tree_operation(scylla_rest_client& client, const bpo::variables_map& vm) { + std::vector res; + if (vm.contains("id")) { + auto id = vm["id"].as(); + + res.push_back(client.get(format("/task_manager/task_status_recursive/{}", id))); + + tasks_print_trees(res); + return; + } + + auto module_res = client.get("/task_manager/list_modules"); + for (const auto& module : module_res.GetArray()) { + auto list_res = client.get(format("/task_manager/list_module_tasks/{}", module.GetString())); + for (const auto& stats : list_res.GetArray()) { + try { + res.push_back(client.get(format("/task_manager/task_status_recursive/{}", stats.GetObject()["task_id"].GetString()))); + } catch (const api_request_failed& e) { + if (e.status == http::reply::status_type::bad_request) { + // Task has already been unregistered. + continue; + } + throw; + } + } + } + tasks_print_trees(res); +} + +void tasks_ttl_operation(scylla_rest_client& client, const bpo::variables_map& vm) { + if (!vm.contains("set")) { + auto res = client.get("/task_manager/ttl"); + fmt::print("Current ttl: {}\n", res); + return; + } + auto new_ttl = vm["set"].as(); + std::unordered_map params = {{ "ttl", fmt::to_string(new_ttl) }}; + + auto res = client.post("/task_manager/ttl", std::move(params)); +} + +enum class tasks_exit_code { + ok = 0, + failed = 123, + timeout = 124, + request_error = 125 +}; + +void tasks_wait_operation(scylla_rest_client& client, const bpo::variables_map& vm) { + if (!vm.count("id")) { + throw std::invalid_argument("required parameter is missing: id"); + } + std::unordered_map params; + auto id = vm["id"].as(); + bool quiet = vm.count("quiet"); + if (vm.count("timeout")) { + params["timeout"] = fmt::format("{}", vm["timeout"].as()); + } + + std::exception_ptr ex = nullptr; + tasks_exit_code exit_code = tasks_exit_code::ok; + try { + auto res = client.get(format("/task_manager/wait_task/{}", id), params); + if (!quiet) { + tasks_print_status(res); + return; + } + exit_code = res.GetObject()["state"] == "failed" ? tasks_exit_code::failed : tasks_exit_code::ok; + } catch (const api_request_failed& e) { + if (e.status == http::reply::status_type::request_timeout) { + if (!quiet) { + fmt::print("Operation timed out"); + return; + } + exit_code = tasks_exit_code::timeout; + } else { + exit_code = tasks_exit_code::request_error; + ex = std::current_exception(); + } + } catch (...) { + exit_code = tasks_exit_code::request_error; + ex = std::current_exception(); + } + + if (quiet) { + if (exit_code == tasks_exit_code::ok) { + return; + } + throw operation_failed_with_status(int(exit_code)); + } else if (ex) { + std::rethrow_exception(ex); + } +} + void toppartitions_operation(scylla_rest_client& client, const bpo::variables_map& vm) { // sanity check the arguments sstring table_filters; @@ -3797,6 +4065,151 @@ For more information, see: {}" table_stats_operation } }, + { + { + "tasks", + "Manages task manager tasks", + "", + { }, + { + typed_option("command", "The uuid of a task", 1), + }, + { + { + "abort", + "Aborts the task", +fmt::format(R"( +Aborts a task with given id. If the task is not abortable, appropriate message +will be printed, depending on why the abort failed. + +For more information, see: {}" +)", doc_link("operating-scylla/nodetool-commands/tasks/abort.html")), + { }, + { + typed_option("id", "The uuid of a task", 1), + }, + }, + { + "list", + "Gets a list of tasks in a given module", +fmt::format(R"( +Lists short stats (including id, type, kind, scope, state, sequence_number, +keyspace, table, and entity) of tasks in a specified module. + +Allows to monitor tasks for extended time. + +For more information, see: {}" +)", doc_link("operating-scylla/nodetool-commands/tasks/list.html")), + { + typed_option<>("internal", "Show internal tasks"), + typed_option("keyspace,ks", "The keyspace name; if specified only the tasks for this keyspace are shown"), + typed_option("table,t", "The table name; if specified only the tasks for this table are shown"), + typed_option("interval", 10, "Re-print the task list after interval seconds. Can be used to monitor the task-list for extended period of time"), + typed_option("iterations,i", 0, "The number of times the task list will be re-printed. Use interval to control the frequency of re-printing the list"), + }, + { + typed_option("module", "The module to query about", 1), + }, + }, + { + "modules", + "Gets a list of modules supported by task manager", +fmt::format(R"( +For more information, see: {}" +)", doc_link("operating-scylla/nodetool-commands/tasks/modules.html")), + { }, + { }, + }, + { + "status", + "Gets a status of the task", +fmt::format(R"( +For more information, see: {}" +)", doc_link("operating-scylla/nodetool-commands/tasks/status.html")), + { }, + { + typed_option("id", "The uuid of a task", 1), + }, + }, + { + "tree", + "Gets statuses of the task tree with a given root", +fmt::format(R"( +Lists statuses of a specified task and all its descendants in BFS order. +If id param isn't specified, trees of all non-internal tasks are listed. + +For more information, see: {}" +)", doc_link("operating-scylla/nodetool-commands/tasks/tree.html")), + { }, + { + typed_option("id", "The uuid of a root task", -1), + }, + }, + { + "ttl", + "Gets or sets task ttl", +fmt::format(R"( +Gets or sets the time in seconds for which tasks will be kept in task manager after +they are finished. + +For more information, see: {}" +)", doc_link("operating-scylla/nodetool-commands/tasks/ttl.html")), + { + typed_option("set", "New task_ttl value", -1), + }, + { }, + }, + { + "wait", + "Waits for the task to finish and returns its status", +fmt::format(R"( +Waits until task is finished and returns it status. If task does not finish before +timeout, the appropriate message with failure reason will be printed. + +If quiet flag is set, nothing is printed. Instead the right exit code is returned: +- 0, if the task finished successfully; +- 123, if the task failed; +- 124, if the operation timed out; +- 125, if there was an error. + +For more information, see: {}" +)", doc_link("operating-scylla/nodetool-commands/tasks/wait.html")), + { + typed_option<>("quiet,q", "If set, status isn't printed"), + typed_option("timeout", "Timeout in seconds"), + }, + { + typed_option("id", "The uuid of a root task", 1), + }, + } + } + }, + { + { + { + "abort", { tasks_abort_operation } + }, + { + "list", { tasks_list_operation } + }, + { + "modules", { tasks_modules_operation } + }, + { + "status", { tasks_status_operation } + }, + { + "tree", { tasks_tree_operation } + }, + { + "ttl", { tasks_ttl_operation } + }, + { + "wait", { tasks_wait_operation } + }, + } + } + }, { { "toppartitions",