a04dbac369
* This is a breaking change, which removes compatibility with the old backtrace
- See https://staging.backtrace.scylladb.com/api/docs#/default/search_by_build_id_search_build_id_post for the APIDoc
* Add timestamp field to log
* Tested locally
Closes scylladb/scylladb#28325
444 lines
14 KiB
Bash
Executable File
444 lines
14 KiB
Bash
Executable File
#!/bin/bash
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# Copyright (C) 2022-present ScyllaDB
|
|
#
|
|
|
|
#
|
|
# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
|
|
#
|
|
|
|
#
|
|
# Requirements:
|
|
# * eu-unstrip
|
|
# * curl
|
|
# * jq
|
|
# * git
|
|
|
|
set -e
|
|
|
|
trap 'echo "error $? in $0 line $LINENO"' ERR
|
|
|
|
SCRIPT_NAME=$(basename $0)
|
|
SCYLLA_S3_RELOC_SERVER_DEFAULT_URL=https://api.backtrace.scylladb.com
|
|
|
|
function print_usage {
|
|
cat << EOF
|
|
Usage: ./${SCRIPT_NAME} [options] COREFILE
|
|
|
|
Script for "one-click" opening of coredumps.
|
|
|
|
It extracts the build-id from the coredump, retrieves metadata for that
|
|
build, downloads the binary package, the source code and finally
|
|
launches the dbuild container, with everything ready to load the
|
|
coredump.
|
|
Sometimes the metadata retrieved from the build lacks the package URL.
|
|
In this case, the scylla package can be downloaded manually and extracted
|
|
into the artifact directory (see below) using the "scylla.package" name.
|
|
When this directory exists, the script will not attempt to extract the
|
|
package URL from the metadata and download the package itself, instead it
|
|
will use the provided package.
|
|
|
|
The script is idempotent: running it after the prepartory steps will
|
|
reuse what is already downloaded. It is *strongly* recommended to run
|
|
this from an empty directory, with nothing but the core-file present.
|
|
|
|
Options:
|
|
|
|
--help,-h
|
|
Print this help message and exit.
|
|
|
|
--quiet,-q
|
|
Only print errors (if any).
|
|
|
|
--verbose,-v
|
|
Print progress information when downloading the package and cloning
|
|
the git repo.
|
|
|
|
--ci
|
|
Designate the coredump as one coming from CI. These coredumps are
|
|
produced by a merge commit between the main branch (master or
|
|
enterprise) and the tested branch and this commit will not be
|
|
present in origin. Using this flag will force the main branch to
|
|
be checked out, instead of the commit hash obtained from the
|
|
ScyllaDB vesrsion.
|
|
|
|
--artifact-dir,-d ARTIFACT_DIR
|
|
Directory where the script will store all downloaded artifacts. If
|
|
the directory doesn't exist, it will be created.
|
|
It is strongly recommended that this is an empty directory, not used
|
|
for anything else, to avoid interference with the script.
|
|
To reset the state of the script, just delete this directory.
|
|
This can be the same as the directory the coredump is already in, or
|
|
some other path.
|
|
By default, a directory called ${SCRIPT_NAME}.dir will be created in
|
|
the script's working directory.
|
|
Can be also provided via env variable ARTIFACT_DIR.
|
|
If both are provided, command line has precedence.
|
|
|
|
--scylla-package-url,-p SCYLLA_PACKAGE_URL
|
|
Instead of querying the s3-reloc-server (see below) and downloading the
|
|
package from the obtained URL, download it directly from the provided URL
|
|
and extract the scylla version metadata from the package.
|
|
If the package is already downloaded, it is not downloaded again.
|
|
|
|
--scylla-s3-reloc-server-url,-u SCYLLA_S3_RELOC_SERVER_URL
|
|
The URL of the s3 reloc server to connect to. Needed to fetch the
|
|
build info based on the build-id.
|
|
Defaults to ${SCYLLA_S3_RELOC_SERVER_DEFAULT_URL}
|
|
Can be also provided via env variable SCYLLA_S3_RELOC_SERVER_URL.
|
|
If both are provided, command line has precedence.
|
|
|
|
--scylla-repo-path,-r SCYLLA_REPO_PATH
|
|
Path to an existing scylla repository to use. The repo is expected to have
|
|
the branch and commit the packages were built from.
|
|
If not provided, the appropriate git repository (either scylladb.git or
|
|
scylla-enterprise.git) will be cloned from https://github.com/scylladb.
|
|
The script assumes you have the appropriate access rights set up.
|
|
Can be also provided via env variable SCYLLA_REPO_PATH.
|
|
If both are provided, command line has precedence.
|
|
|
|
--scylla-gdb-py-source,-s repo|package|none
|
|
Where to get the scylla-gdb.py script from:
|
|
* repo - download the latest one from the appropriate main branch;
|
|
* package - use the one that comes with the relocatable package;
|
|
* none - don't try to obtain scylla-gdb.py;
|
|
Defaults to repo.
|
|
Unlike other artifacts, scylla-gdb.py is placed in the same directory
|
|
where the coredump is located. If one is already present, it is not
|
|
overwritten.
|
|
Can be also provided via env variable SCYLLA_GDB_PY_SOURCE.
|
|
If both are provided, command line has precedence.
|
|
EOF
|
|
}
|
|
|
|
function log {
|
|
if [[ $VERBOSE_LEVEL -ge 1 ]]
|
|
then
|
|
echo $@
|
|
fi
|
|
}
|
|
|
|
function get_json_field {
|
|
local json_obj=$1
|
|
local field_name=$2
|
|
local optional=${3:-0}
|
|
local field_val=$(jq -r ".${field_name}" <<< "$json_obj")
|
|
|
|
if [[ -z $field_val ]] && [[ $optional -eq 0 ]]
|
|
then
|
|
echo "error: failed to get field '$field_name' from: $json_obj" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo $field_val
|
|
}
|
|
|
|
function get_repo_name() {
|
|
local branch=$1
|
|
local sha1=$2
|
|
local repo_name
|
|
local remote_repos
|
|
|
|
pushd ${SCYLLA_REPO_PATH} > /dev/null
|
|
|
|
# First, try to find the repo from local information
|
|
repo_name=$(git branch -r --contains "$sha1" 2>/dev/null | \
|
|
sed -n 's|^ *\(.*\)/.*$|\1|p' | \
|
|
head -n 1)
|
|
if [ -n "$repo_name" ]; then
|
|
echo "$repo_name"
|
|
return 0
|
|
fi
|
|
|
|
# If not found locally, try fetching from each remote
|
|
remote_repos=$(git remote -v | awk "/scylladb\/${PRODUCT}.*fetch/{ print \$1 }")
|
|
remote_name=$(while read -r remote; do
|
|
git fetch -q "$remote" "$branch" 2>/dev/null
|
|
if git branch -r --contains "$sha1" | grep -q "$remote"; then
|
|
echo "$remote"
|
|
break
|
|
fi
|
|
done <<< "$remote_repos")
|
|
|
|
popd > /dev/null
|
|
|
|
if [ -n "$remote_name" ]; then
|
|
echo "$remote_name"
|
|
else
|
|
echo "error: could not determine remote repository for SHA1 $sha1" >& 2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
for required in eu-unstrip jq curl git; do
|
|
if ! type $required >& /dev/null; then
|
|
echo "error: missing required program $required, please install first" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
VERBOSE_LEVEL=1
|
|
CORE_FROM_CI=0
|
|
PACKAGE_URL="${SCYLLA_PACKAGE_URL}"
|
|
SCYLLA_S3_RELOC_SERVER_URL="${SCYLLA_S3_RELOC_SERVER_URL:-$SCYLLA_S3_RELOC_SERVER_DEFAULT_URL}"
|
|
SCYLLA_REPO_PATH="${SCYLLA_REPO_PATH}"
|
|
SCYLLA_BUILD_ID="${SCYLLA_BUILD_ID}"
|
|
SCYLLA_GDB_PY_SOURCE="${SCYLLA_GDB_PY_SOURCE:-repo}"
|
|
ARTIFACT_DIR=${ARTIFACT_DIR:-${SCRIPT_NAME}.dir}
|
|
|
|
if [[ $# -ge 1 && ( "$1" == "--help" || "$1" == "-h" ) ]]; then
|
|
print_usage
|
|
exit 0
|
|
fi
|
|
|
|
while [[ $# -gt 1 ]]
|
|
do
|
|
case $1 in
|
|
"--verbose"|"-v")
|
|
VERBOSE_LEVEL=2
|
|
shift 1
|
|
;;
|
|
"--quiet"|"-q")
|
|
VERBOSE_LEVEL=0
|
|
shift 1
|
|
;;
|
|
"--ci")
|
|
CORE_FROM_CI=1
|
|
shift 1
|
|
;;
|
|
"--artifact-dir"|"-d")
|
|
ARTIFACT_DIR=$2
|
|
shift 2
|
|
;;
|
|
"--scylla-package-url"|"-p")
|
|
PACKAGE_URL=$2
|
|
shift 2
|
|
;;
|
|
"--scylla-s3-reloc-server-url"|"-u")
|
|
SCYLLA_S3_RELOC_SERVER_URL=$2
|
|
shift 2
|
|
;;
|
|
"--scylla-repo-path"|"-r")
|
|
SCYLLA_REPO_PATH=$2
|
|
shift 2
|
|
;;
|
|
"--scylla-gdb-py-source"|"-s")
|
|
SCYLLA_GDB_PY_SOURCE=$2
|
|
shift 2
|
|
;;
|
|
"--scylla-build-id"|"-b")
|
|
SCYLLA_BUILD_ID=$2
|
|
shift 2
|
|
;;
|
|
*)
|
|
echo "error: unrecognized option: $1, see $0 -h for usage" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
COREFILE=$1
|
|
shift 1
|
|
if ! [[ -f $COREFILE ]]
|
|
then
|
|
echo "error: ${COREFILE} is not a valid path to a core file" >&2
|
|
exit 1
|
|
fi
|
|
COREDIR=$(dirname ${COREFILE})
|
|
|
|
ARTIFACT_DIR=$(realpath ${ARTIFACT_DIR})
|
|
mkdir -p $ARTIFACT_DIR
|
|
|
|
if [[ -z "${SCYLLA_REPO_PATH}" ]]
|
|
then
|
|
SCYLLA_REPO_PATH="${ARTIFACT_DIR}/scylla.repo"
|
|
fi
|
|
|
|
SCYLLA_REPO_PATH=$(realpath ${SCYLLA_REPO_PATH})
|
|
|
|
if [[ ${VERBOSE_LEVEL} -lt 2 ]]
|
|
then
|
|
CURL_QUIET_FLAG="-s"
|
|
GIT_QUIET_FLAG="-q"
|
|
else
|
|
CURL_QUIET_FLAG=""
|
|
GIT_QUIET_FLAG=""
|
|
fi
|
|
|
|
case "${SCYLLA_GDB_PY_SOURCE}" in
|
|
("repo"|"package"|"none")
|
|
;;
|
|
*)
|
|
echo "error: invalid value for option SCYLLA_GDB_PY_SOURCE, has to be one of repo, package or none, got: ${SCYLLA_GDB_PY_SOURCE}" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
if [[ -z "${PACKAGE_URL}" ]]
|
|
then
|
|
BUILD_ID="${SCYLLA_BUILD_ID}"
|
|
if [[ -z "${BUILD_ID}" ]]; then
|
|
BUILD_ID=$(eu-unstrip -n --core ${COREFILE} | grep 'scylla$' | cut -f2 -d' ' | cut -f1 -d@)
|
|
fi
|
|
|
|
log "Build id: ${BUILD_ID}"
|
|
|
|
# https://api.backtrace.scylladb.com/api/docs#/default/search_by_build_id_search_build_id_get
|
|
BUILD=$(curl "${SCYLLA_S3_RELOC_SERVER_URL}/api/search/build_id?build_id=${BUILD_ID}" -H 'accept: application/json')
|
|
|
|
if [[ -z "$BUILD" ]]
|
|
then
|
|
echo "error: failed to retrieve build information from ${SCYLLA_S3_RELOC_SERVER_URL}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
RESPONSE_BUILD_ID=$(get_json_field "$BUILD" "build_id")
|
|
BUILD_MODE=$(get_json_field "$BUILD" "build_type")
|
|
PACKAGE_URL=$(get_json_field "$BUILD" "unstripped_url")
|
|
BUILD_DATA=$(get_json_field "$BUILD" "build_data")
|
|
|
|
VERSION=$(get_json_field "$BUILD_DATA" "version")
|
|
PRODUCT=$(get_json_field "$BUILD_DATA" "product")
|
|
RELEASE=$(get_json_field "$BUILD_DATA" "release")
|
|
ARCH=$(get_json_field "$BUILD_DATA" "platform")
|
|
TIMESTAMP=$(get_json_field "$BUILD_DATA" "timestamp")
|
|
|
|
|
|
if [[ "$RESPONSE_BUILD_ID" != "$BUILD_ID" ]]
|
|
then
|
|
echo "error: mismatching build id: requested ${BUILD_ID} but got ${RESPONSE_BUILD_ID}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
log "Matching build is ${PRODUCT}-${VERSION} ${RELEASE} ${BUILD_MODE}-${ARCH} from ${TIMESTAMP}"
|
|
fi
|
|
|
|
if ! [[ -d ${ARTIFACT_DIR}/scylla.package ]]
|
|
then
|
|
if [[ -z $PACKAGE_URL ]]
|
|
then
|
|
echo "error: no package_url in build object: ${BUILD}" >&2
|
|
echo "" >&2
|
|
echo "The package can be provided manually by placing it (unpacked) to "${ARTIFACT_DIR}/scylla.package" to work around this problem." >&2
|
|
exit 1
|
|
fi
|
|
PACKAGE_FILE=$(basename ${PACKAGE_URL})
|
|
if ! [[ -f ${ARTIFACT_DIR}/${PACKAGE_FILE} ]]
|
|
then
|
|
log "Downloading relocatable package from ${PACKAGE_URL}"
|
|
curl -L ${CURL_QUIET_FLAG} --output ${ARTIFACT_DIR}/${PACKAGE_FILE} ${PACKAGE_URL}
|
|
else
|
|
log "Relocatable package ${PACKAGE_URL} already downloaded"
|
|
fi
|
|
|
|
log "Extracting package ${PACKAGE_FILE}"
|
|
pushd ${ARTIFACT_DIR} > /dev/null
|
|
tar -zxf $PACKAGE_FILE
|
|
mv scylla scylla.package
|
|
popd > /dev/null
|
|
else
|
|
log "Relocatable package ${PACKAGE_URL} already downloaded and extracted"
|
|
fi
|
|
|
|
# If the package was provided directly we bypassed talking to the S3 server and
|
|
# the version metadata has to be loaded from the package itself.
|
|
if [[ -z "$VERSION" ]]
|
|
then
|
|
VERSION=$(cat ${ARTIFACT_DIR}/scylla.package/SCYLLA-VERSION-FILE)
|
|
PRODUCT=$(cat ${ARTIFACT_DIR}/scylla.package/SCYLLA-PRODUCT-FILE)
|
|
RELEASE=$(cat ${ARTIFACT_DIR}/scylla.package/SCYLLA-RELEASE-FILE)
|
|
fi
|
|
|
|
if [[ "${PRODUCT}" == "scylla-enterprise" ]]
|
|
then
|
|
MAIN_BRANCH=enterprise
|
|
else
|
|
MAIN_BRANCH=master
|
|
fi
|
|
|
|
COMMIT_HASH=$(cut -f3 -d. <<< $RELEASE)
|
|
if [ $CORE_FROM_CI -eq 1 ]
|
|
then
|
|
COMMIT_HASH=${MAIN_BRANCH}
|
|
fi
|
|
if [ "$(grep -o ~dev <<< $VERSION)" == "~dev" ]
|
|
then
|
|
BRANCH=${MAIN_BRANCH}
|
|
JENKINS_WORKSPACE_DIR=/jenkins/workspace/scylla-${MAIN_BRANCH}/next
|
|
else
|
|
BASE_VERSION=$(grep -o "^[0-9]\+\.[0-9]\+" <<< $VERSION)
|
|
BRANCH=branch-${BASE_VERSION}
|
|
JENKINS_WORKSPACE_DIR=/jenkins/workspace/scylla-${BASE_VERSION}/build
|
|
fi
|
|
|
|
if ! [[ -d ${SCYLLA_REPO_PATH} ]]
|
|
then
|
|
log "Cloning ${PRODUCT}.git"
|
|
git clone ${GIT_QUIET_FLAG} -b ${BRANCH} git@github.com:scylladb/${PRODUCT}.git ${SCYLLA_REPO_PATH}
|
|
REMOTE_REPO_NAME=origin
|
|
else
|
|
log "${PRODUCT}.git already cloned"
|
|
REMOTE_REPO_NAME=$(get_repo_name ${BRANCH} ${COMMIT_HASH})
|
|
fi
|
|
|
|
# We do the checkout unconditionally, it is cheap anyway.
|
|
pushd ${SCYLLA_REPO_PATH} > /dev/null
|
|
# we should already have the commit in the local repo, so no need to fetch it
|
|
git checkout -q ${COMMIT_HASH}
|
|
# Skip the other submodules, they are not needed for debugging
|
|
git submodule -q sync
|
|
git submodule update -q --depth=1 --init seastar
|
|
popd > /dev/null
|
|
|
|
if ! [[ -f ${COREDIR}/scylla-gdb.py ]]
|
|
then
|
|
if [[ "${SCYLLA_GDB_PY_SOURCE}" == "repo" ]]
|
|
then
|
|
WORKDIR=$(pwd)
|
|
cd ${SCYLLA_REPO_PATH}
|
|
git checkout -q $MAIN_BRANCH
|
|
git pull -q --no-recurse-submodules ${REMOTE_REPO_NAME} $MAIN_BRANCH
|
|
log "Copying scylla-gdb.py from ${SCYLLA_REPO_PATH}"
|
|
cp scylla-gdb.py ${WORKDIR}/scylla-gdb.py
|
|
git checkout -q ${COMMIT_HASH}
|
|
cd $WORKDIR
|
|
elif [[ "${SCYLLA_GDB_PY_SOURCE}" == "package" ]]
|
|
then
|
|
log "Copying scylla-gdb.py from package"
|
|
cp -n ${ARTIFACT_DIR}/scylla.package/scylla-gdb.py .
|
|
else
|
|
log "scylla-gdb.py was not requested"
|
|
fi
|
|
else
|
|
log "scylla-gdb.py already present"
|
|
fi
|
|
|
|
if [[ ${VERBOSE_LEVEL} -ge 1 ]]
|
|
then
|
|
cat << EOF
|
|
Launching dbuild container.
|
|
|
|
To examine the coredump with gdb:
|
|
|
|
$ gdb -x scylla-gdb.py -ex 'set directories /src/scylla' \
|
|
-iex 'set substitute-path ${JENKINS_WORKSPACE_DIR}/scylla /src/scylla' \
|
|
-iex 'set solib-search-path /opt/scylladb/libreloc/' \
|
|
--core ${COREFILE} /opt/scylladb/libexec/scylla
|
|
|
|
See https://github.com/scylladb/scylladb/blob/master/docs/dev/debugging.md for more information on how to debug scylla.
|
|
|
|
Good luck!
|
|
EOF
|
|
fi
|
|
|
|
TOP_SRCDIR="$(dirname "$(dirname "$(readlink -f "$0")")")"
|
|
IMAGE="${SCYLLA_REPO_PATH}/tools/toolchain/image"
|
|
exec ${TOP_SRCDIR}/tools/toolchain/dbuild \
|
|
--image "$(<"$IMAGE")" \
|
|
-it \
|
|
-v $(pwd):/workdir \
|
|
-v ${SCYLLA_REPO_PATH}:/src/scylla \
|
|
-v ${ARTIFACT_DIR}/scylla.package:/opt/scylladb \
|
|
-w /workdir -- bash -l
|