scylladb/dist/common/scripts/scylla-housekeeping

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2016-present ScyllaDB
#

#
# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
#

import argparse
import json
import os
import sys
import subprocess
import configparser
import uuid
import re
import glob
import urllib.request
from pkg_resources import parse_version
import multiprocessing as mp

# Python 3.14 changed the default to 'forkserver', which is not compatible
# with our relocatable python. It execs our Python binary, but without our
# ld.so. Change it back to 'fork' to avoid issues.
mp.set_start_method('fork')

VERSION = "1.0"
quiet = False
# Temporary url for the review
version_url = "https://i6a5h9l1kl.execute-api.us-east-1.amazonaws.com/prod/check_version"


def trace(*vals):
    print(''.join(vals))


def traceln(*vals):
    trace(*(vals + ('\n',)))


def help(args):
    parser.print_help()


def sh_command(*args):
    p = subprocess.Popen(args, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    out, err = p.communicate()
    if err:
        raise Exception(err)
    return out

def get_url(path):
    # If server returns any error, like 403, or 500 urllib.request throws exception, which is not serializable.
    # When multiprocessing routines fail to serialize it, it throws ambiguous serialization exception
    #   from get_json_from_url.
    # In order to see legit error we catch it from the inside of process, convert to string and
    #   pass it as part of return value
    try:
        return 0, urllib.request.urlopen(path).read().decode('utf-8')
    except Exception as exc:
        return 1, str(exc)

def get_json_from_url(path):
    pool = mp.Pool(processes=1)
    # Unfortunately the timeout parameter to urlopen seems to be something internal and is not close or
    # near a wallclock timeout. In my experiments, I could see a factor of up to 4, with a 5-second timeout
    # passed to urlopen timing out in 20s, which creates a bad user experience. We will then use multiprocessing
    # to enforce a wallclock timeout.
    result = pool.apply_async(get_url, args=(path,))
    try:
        status, retval = result.get(timeout=5)
    except mp.TimeoutError as err:
        pool.terminate()
        pool.join()
        raise
    if status == 1:
        raise RuntimeError(f'Failed to get "{path}" due to the following error: {retval}')
    return json.loads(retval)


def get_api(path):
    return get_json_from_url("http://" + api_address + path)


def parse_scylla_version(version):
    # Newer setuptools does not accept ~dev in version strings, need to
    # replace it to X.Y.Z.dev0
    if version and version.endswith('~dev'):
        version = version.replace('~dev', '.dev0')
    # Newer setuptools does not accept ~rcN in version strings, need to
    # replace it to X.Y.ZrcN
    if version and '~rc' in version:
        version = version.replace('~', '')
    return parse_version(version)


def version_compare(a, b):
    return parse_scylla_version(a) < parse_scylla_version(b)


def create_uuid_file(fl):
    with open(args.uuid_file, 'w') as myfile:
        myfile.write(str(uuid.uuid1()) + "\n")
    os.chmod(args.uuid_file, 0o644)


def sanitize_version(version):
    """
    Newer setuptools don't like dashed version strings, trim it to avoid
    false negative version_compare() checks.
    """
    if version and '-' in version:
        return version.split('-', 1)[0]
    else:
        return version


def get_repo_file(dir):
    files = glob.glob(dir)
    files.sort(key=os.path.getmtime, reverse=True)
    for name in files:
        with open(name, 'r') as myfile:
            for line in myfile:
                match = re.search(r".*http.?://repositories.*/scylladb/([^/\s]+)/.*/([^/\s]+)/scylladb-.*", line)
                if match:
                    return match.group(2), match.group(1)
    return None, None


def check_version(ar):
    if config and (not config.has_option("housekeeping", "check-version") or not config.getboolean("housekeeping", "check-version")):
        return
    if ar.version and ar.version != '':
        current_version = sanitize_version(ar.version)
    else:
        current_version = sanitize_version(get_api('/storage_service/scylla_release_version'))
        if current_version == "":
            # API is down, nothing to do
            return
    try:
        params = "?version=" + current_version
        if ar.mode:
            # mode would accept any string.
            # use i for install, c (default) for running from the command line
            params = params + "&sts=" + ar.mode
        if uid:
            params = params + "&uu=" + uid
        if repo_id:
            params = params + "&rid=" + repo_id
        if repo_type:
            params = params + "&rtype=" + repo_type
        versions = get_json_from_url(version_url + params)
        latest_version = versions["version"]
        latest_patch_version = versions["latest_patch_version"]
    except Exception:
        traceln("Unable to retrieve version information")
        return

    if latest_patch_version != latest_version:
        # user is using an older minor version
        if version_compare(current_version, latest_patch_version):
            # user is also running an older patch version
            traceln("Your current Scylla release is " + current_version + ", while the latest patch release is " + latest_patch_version +
                    ", and the latest minor release is " + latest_version + " (recommended)")
        else:
            traceln("Your current Scylla release is ", current_version, " the latest minor release is ", latest_version, " go to http://www.scylladb.com for upgrade instructions")
    elif version_compare(current_version, latest_patch_version):
            traceln("Your current Scylla release is ", current_version, " while the latest patch release is ", latest_patch_version, ", update for the latest bug fixes and improvements")


parser = argparse.ArgumentParser(description='ScyllaDB help report tool', conflict_handler="resolve")
parser.add_argument('-q', '--quiet', action='store_true', default=False, help='Quiet mode')
parser.add_argument('-c', '--config', default="", help='An optional config file. Specifying a missing file will terminate the script')
parser.add_argument('--uuid', default="", help='A uuid for the requests')
parser.add_argument('--uuid-file', default="", help='A uuid file for the requests')
parser.add_argument('--repo-files', default="", help='The repository files that is been used for private repositories')
parser.add_argument('--api-address', default="localhost:10000", help='The ip and port of the scylla api')

subparsers = parser.add_subparsers(help='Available commands')
parser_help = subparsers.add_parser('help', help='Display help information')
parser_help.set_defaults(func=help)
parser_system = subparsers.add_parser('version', help='Check if the current running version is the latest one')
parser_system.add_argument('--mode', default="c", help='Which mode the version check runs')
parser_system.add_argument('--version', default="", help='Use a given version to compare to')
parser_system.set_defaults(func=check_version)

args = parser.parse_args()
quiet = args.quiet
config = None
repo_id = None
repo_type = None

if args.config != "":
    if not os.path.isfile(args.config):
        traceln("Config file ", args.config, " is missing, terminating")
        sys.exit(0)
    config = configparser.ConfigParser()
    config.read(args.config)
uid = None
if args.uuid != "":
    uid = args.uuid
if args.uuid_file != "":
    if not os.path.exists(args.uuid_file):
        create_uuid_file(args.uuid_file)
    with open(args.uuid_file, 'r') as myfile:
        uid = myfile.read().replace('\n', '')
api_address = args.api_address
if args.repo_files != "":
    repo_type, repo_id = get_repo_file(args.repo_files)
args.func(args)