7dbcfe5a39
Autogenerates metrics documentation using the scripts/get_description.py script introduced in #17479
docs: add beta
(cherry picked from commit 9eef3d6139)
Closes scylladb/scylladb#18857
189 lines
7.8 KiB
Python
189 lines
7.8 KiB
Python
import os
|
|
import sys
|
|
import json
|
|
from sphinx import addnodes
|
|
from sphinx.directives import ObjectDescription
|
|
from sphinx.util.docfields import Field
|
|
from sphinx.util.docutils import switch_source_input
|
|
from sphinx.util.nodes import make_id
|
|
from sphinx.util import logging, ws_re
|
|
from docutils.parsers.rst import Directive, directives
|
|
from docutils.statemachine import StringList
|
|
from sphinxcontrib.datatemplates.directive import DataTemplateJSON
|
|
from utils import maybe_add_filters
|
|
|
|
sys.path.insert(0, os.path.abspath("../../scripts"))
|
|
import scripts.get_description as metrics
|
|
|
|
LOGGER = logging.getLogger(__name__)
|
|
|
|
|
|
class MetricsProcessor:
|
|
|
|
MARKER = "::description"
|
|
|
|
def _create_output_directory(self, app, metrics_directory):
|
|
output_directory = os.path.join(app.builder.srcdir, metrics_directory)
|
|
os.makedirs(output_directory, exist_ok=True)
|
|
return output_directory
|
|
|
|
def _process_single_file(self, file_path, destination_path, metrics_config_path):
|
|
with open(file_path, 'r', encoding='utf-8') as f:
|
|
content = f.read()
|
|
if self.MARKER in content and not os.path.exists(destination_path):
|
|
try:
|
|
metrics_file = metrics.get_metrics_from_file(file_path, "scylla", metrics.get_metrics_information(metrics_config_path))
|
|
with open(destination_path, 'w+', encoding='utf-8') as f:
|
|
json.dump(metrics_file, f, indent=4)
|
|
except SystemExit:
|
|
LOGGER.info(f'Skipping file: {file_path}')
|
|
except Exception as error:
|
|
LOGGER.info(error)
|
|
|
|
def _process_metrics_files(self, repo_dir, output_directory, metrics_config_path):
|
|
for root, _, files in os.walk(repo_dir):
|
|
for file in files:
|
|
if file.endswith(".cc"):
|
|
file_path = os.path.join(root, file)
|
|
file_name = os.path.splitext(file)[0] + ".json"
|
|
destination_path = os.path.join(output_directory, file_name)
|
|
self._process_single_file(file_path, destination_path, metrics_config_path)
|
|
|
|
def run(self, app, exception=None):
|
|
repo_dir = os.path.abspath(os.path.join(app.srcdir, ".."))
|
|
metrics_config_path = os.path.join(repo_dir, app.config.scylladb_metrics_config_path)
|
|
output_directory = self._create_output_directory(app, app.config.scylladb_metrics_directory)
|
|
|
|
self._process_metrics_files(repo_dir, output_directory, metrics_config_path)
|
|
|
|
|
|
class MetricsTemplateDirective(DataTemplateJSON):
|
|
option_spec = DataTemplateJSON.option_spec.copy()
|
|
option_spec["title"] = lambda x: x
|
|
|
|
def _make_context(self, data, config, env):
|
|
context = super()._make_context(data, config, env)
|
|
context["title"] = self.options.get("title")
|
|
return context
|
|
|
|
def run(self):
|
|
return super().run()
|
|
|
|
|
|
class MetricsOption(ObjectDescription):
|
|
has_content = True
|
|
required_arguments = 1
|
|
optional_arguments = 0
|
|
final_argument_whitespace = False
|
|
option_spec = {
|
|
'type': directives.unchanged,
|
|
'component': directives.unchanged,
|
|
'key': directives.unchanged,
|
|
'source': directives.unchanged,
|
|
}
|
|
|
|
doc_field_types = [
|
|
Field('type', label='Type', has_arg=False, names=('type',)),
|
|
Field('component', label='Component', has_arg=False, names=('component',)),
|
|
Field('key', label='Key', has_arg=False, names=('key',)),
|
|
Field('source', label='Source', has_arg=False, names=('source',)),
|
|
]
|
|
|
|
def handle_signature(self, sig: str, signode: addnodes.desc_signature):
|
|
signode.clear()
|
|
signode += addnodes.desc_name(sig, sig)
|
|
return ws_re.sub(' ', sig)
|
|
|
|
@property
|
|
def env(self):
|
|
return self.state.document.settings.env
|
|
|
|
def _render(self, name, option_type, component, key, source):
|
|
item = {'name': name, 'type': option_type, 'component': component, 'key': key, 'source': source }
|
|
template = self.config.scylladb_metrics_option_template
|
|
return self.env.app.builder.templates.render(template, item)
|
|
|
|
def transform_content(self, contentnode: addnodes.desc_content) -> None:
|
|
name = self.arguments[0]
|
|
option_type = self.options.get('type', '')
|
|
component = self.options.get('component', '')
|
|
key = self.options.get('key', '')
|
|
source_file = self.options.get('source', '')
|
|
_, lineno = self.get_source_info()
|
|
source = f'scylladb_metrics:{lineno}:<{name}>'
|
|
fields = StringList(self._render(name, option_type, component, key, source_file).splitlines(), source=source, parent_offset=lineno)
|
|
with switch_source_input(self.state, fields):
|
|
self.state.nested_parse(fields, 0, contentnode)
|
|
|
|
def add_target_and_index(self, name: str, sig: str, signode: addnodes.desc_signature) -> None:
|
|
node_id = make_id(self.env, self.state.document, self.objtype, name)
|
|
signode['ids'].append(node_id)
|
|
self.state.document.note_explicit_target(signode)
|
|
entry = f'{name}; metrics option'
|
|
self.indexnode['entries'].append(('pair', entry, node_id, '', None))
|
|
self.env.get_domain('std').note_object(self.objtype, name, node_id, location=signode)
|
|
|
|
class MetricsDirective(Directive):
|
|
TEMPLATE = 'metrics.tmpl'
|
|
required_arguments = 0
|
|
optional_arguments = 1
|
|
option_spec = {'template': directives.path}
|
|
has_content = True
|
|
|
|
def _process_file(self, file, relative_path_from_current_rst):
|
|
data_directive = MetricsTemplateDirective(
|
|
name=self.name,
|
|
arguments=[os.path.join(relative_path_from_current_rst, file)],
|
|
options=self.options,
|
|
content=self.content,
|
|
lineno=self.lineno,
|
|
content_offset=self.content_offset,
|
|
block_text=self.block_text,
|
|
state=self.state,
|
|
state_machine=self.state_machine,
|
|
)
|
|
data_directive.options["template"] = self.options.get('template', self.TEMPLATE)
|
|
data_directive.options["title"] = file.replace('_', ' ').replace('.json','').capitalize()
|
|
return data_directive.run()
|
|
|
|
def _get_relative_path(self, output_directory, app, docname):
|
|
current_rst_path = os.path.join(app.builder.srcdir, docname + ".rst")
|
|
return os.path.relpath(output_directory, os.path.dirname(current_rst_path))
|
|
|
|
|
|
def run(self):
|
|
maybe_add_filters(self.state.document.settings.env.app.builder)
|
|
app = self.state.document.settings.env.app
|
|
docname = self.state.document.settings.env.docname
|
|
metrics_directory = os.path.join(app.builder.srcdir, app.config.scylladb_metrics_directory)
|
|
output = []
|
|
try:
|
|
relative_path_from_current_rst = self._get_relative_path(metrics_directory, app, docname)
|
|
files = os.listdir(metrics_directory)
|
|
for _, file in enumerate(files):
|
|
output.extend(self._process_file(file, relative_path_from_current_rst))
|
|
except Exception as error:
|
|
LOGGER.info(error)
|
|
return output
|
|
|
|
def setup(app):
|
|
app.add_config_value("scylladb_metrics_directory", default="_data/metrics", rebuild="html")
|
|
app.add_config_value("scylladb_metrics_config_path", default='scripts/metrics-config.yml', rebuild="html")
|
|
app.add_config_value('scylladb_metrics_option_template', default='metrics_option.tmpl', rebuild='html', types=[str])
|
|
app.connect("builder-inited", MetricsProcessor().run)
|
|
app.add_object_type(
|
|
'metrics_option',
|
|
'metrics_option',
|
|
objname='metrics option')
|
|
app.add_directive_to_domain('std', 'metrics_option', MetricsOption, override=True)
|
|
app.add_directive("metrics_option", MetricsOption)
|
|
app.add_directive("scylladb_metrics", MetricsDirective)
|
|
|
|
|
|
return {
|
|
"version": "0.1",
|
|
"parallel_read_safe": True,
|
|
"parallel_write_safe": True,
|
|
}
|
|
|