From 9edf858d303f89814b5faad3bff94183c3e84b3c Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Mon, 24 Feb 2020 14:19:47 +0200 Subject: [PATCH 01/10] Add scylla-jmx submodule --- .gitmodules | 3 +++ scylla-jmx | 1 + 2 files changed, 4 insertions(+) create mode 160000 scylla-jmx diff --git a/.gitmodules b/.gitmodules index 4156116122..b3f5c643ba 100644 --- a/.gitmodules +++ b/.gitmodules @@ -12,3 +12,6 @@ [submodule "abseil"] path = abseil url = ../abseil-cpp +[submodule "scylla-jmx"] + path = scylla-jmx + url = ../scylla-jmx diff --git a/scylla-jmx b/scylla-jmx new file mode 160000 index 0000000000..b2195734cc --- /dev/null +++ b/scylla-jmx @@ -0,0 +1 @@ +Subproject commit b2195734cc99463bf9f4a53940d0bc4f13acfb92 From d759d7567b5807e025f459bd36ef781b736f61d9 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 17 Mar 2020 17:13:14 +0200 Subject: [PATCH 02/10] Add scylla-tools submodule --- .gitmodules | 3 +++ scylla-tools | 1 + 2 files changed, 4 insertions(+) create mode 160000 scylla-tools diff --git a/.gitmodules b/.gitmodules index b3f5c643ba..bebd2dfc28 100644 --- a/.gitmodules +++ b/.gitmodules @@ -15,3 +15,6 @@ [submodule "scylla-jmx"] path = scylla-jmx url = ../scylla-jmx +[submodule "scylla-tools"] + path = scylla-tools + url = ../scylla-tools-java diff --git a/scylla-tools b/scylla-tools new file mode 160000 index 0000000000..1639b12061 --- /dev/null +++ b/scylla-tools @@ -0,0 +1 @@ +Subproject commit 1639b1206192c4d770893f46b395d9c459dec290 From e3376472e861b733a93de7af71b89081707f125d Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 21 Apr 2020 11:27:34 +0300 Subject: [PATCH 03/10] sync-submodules.sh: Add script for syncing submodules --- scripts/sync-submodules.sh | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100755 scripts/sync-submodules.sh diff --git a/scripts/sync-submodules.sh b/scripts/sync-submodules.sh new file mode 100755 index 0000000000..90cae27ec8 --- /dev/null +++ b/scripts/sync-submodules.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# +# This file is part of Scylla. +# +# Scylla is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Scylla is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Scylla. If not, see . +# + +set -euo pipefail + +submodules=( + seastar + scylla-jmx + scylla-tools +) + +for submodule in "${submodules[@]}"; do + GIT_DIR="$submodule/.git" git pull --ff-only origin master + SUMMARY=$(git submodule summary $submodule) + if grep '^ *<' <<< "$SUMMARY"; then + echo "Non fast-forward changes detected! Fire three red flares from your flare pistol." + exit 1 + fi + if [ ! -z "$SUMMARY" ]; then + git commit --edit -m "Update $submodule submodule" -m "$SUMMARY" $submodule + fi +done From 71da4e6e795f2204bde76473d4876479c2e35692 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Mon, 15 Jun 2020 13:20:55 +0300 Subject: [PATCH 04/10] docs: Document sync-submodules.sh script in maintainer.md --- docs/maintainer.md | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/docs/maintainer.md b/docs/maintainer.md index 1f8313b9bf..c5e0974674 100644 --- a/docs/maintainer.md +++ b/docs/maintainer.md @@ -137,25 +137,20 @@ Note: git contains a bug where branch descriptions with the characters `['":\.]` confuse it. Best to search-and-replace those characters with nothing. -## Updating seastar submodule references +## Updating submodule references -Seastar is updated upstream independently of Scylla. We want -to periodically (and upon contributor request) syncrhonize -scylla.git to include the latest seastar.git. +Submodules are maintained in separate repositories. For example, Seastar +is developed upstream independently of Scylla. We want to periodically +(and upon contributor request) synchronize scylla.git to include the +latest submodules. 1. Check out the `next` branch and synchronize it using `git pull` - 2. `cd seastar`, check out `master`, and synchronize with upstream - using `git pull` - 3. Go back to the scylla directory, and issue the command - `git submodule summary seastar`; this generates a change log. - 4. Verify that all lines in the change log start with '>'. Lines - that start with '<' indicate non-fast-forward changes. If that - happens, fire three red flares from your flare pistol. - 5. Use `git commit seastar` to update the submodule. Use something - like "Update seastar submodule" for the subject and the log from - step 3 for the body. If any of those commits fix an issue, add - a "Fixes #wxyz" entry. - 6. Use `git push` to publish your work. + 2. Run the `scripts/sync-submodules.sh` script, which will open a git + commit log editor for every submodule to show that commits are being + updated. + 3. Edit the submodule update commits with any necessary additional + imformation. For example, amend the message with `Fixes` tags. + 4. Use `git push` to publish your work. ## Backporting patches From 013f87f3880def5e70678923c511c3498a432b00 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 7 Apr 2020 14:25:05 +0300 Subject: [PATCH 05/10] build: Add "-ffile-prefix-map" to cxxflags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch adds "-ffile-prefix-map" to cxxflags for all build modes. This has two benefits: 1, Relocatable packages no longer have any special build flags, which makes deeper integration with the build system possible (e.g. targets for packages). 2 Builds are now reproducible, which makes debugging easier in case you only have a backtrace, but no artifacts. Rafael explains: "BTW, I think I found another argument for why we should always build with -ffile-prefix-map=. There was user after free test failure on next promotion. I am unable to reproduce it locally, so it would be super nice to be able to decode the backtrace. I was able to do it, but I had to create a /jenkins/workspace/scylla-master/next/ directory and build from there to get the same results as the bot." Acked-by: Botond Dénes Acked-by: Nadav Har'El Acked-by: Rafael Avila de Espindola --- configure.py | 4 ++++ reloc/build_reloc.sh | 6 ++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/configure.py b/configure.py index ee2bdbdfb2..db7b51c4d6 100755 --- a/configure.py +++ b/configure.py @@ -32,6 +32,8 @@ import tempfile import textwrap from distutils.spawn import find_executable +curdir = os.getcwd() + tempfile.tempdir = "./build/tmp" configure_args = str.join(' ', [shlex.quote(x) for x in sys.argv[1:]]) @@ -1205,6 +1207,8 @@ args.user_ldflags = forced_ldflags + ' ' + args.user_ldflags args.user_cflags += ' -Wno-error=stack-usage=' +args.user_cflags += f"-ffile-prefix-map={curdir}=." + seastar_cflags = args.user_cflags if args.target != '': seastar_cflags += ' -march=' + args.target diff --git a/reloc/build_reloc.sh b/reloc/build_reloc.sh index 8830106d85..7e7c6b98ee 100755 --- a/reloc/build_reloc.sh +++ b/reloc/build_reloc.sh @@ -2,14 +2,12 @@ . /etc/os-release -COMMON_FLAGS="--cflags=-ffile-prefix-map=$PWD=." - DEFAULT_MODE="release" print_usage() { echo "Usage: build_reloc.sh [OPTION]..." echo "" - echo " --configure-flags FLAGS specify extra build flags passed to 'configure.py' (common: '$COMMON_FLAGS')" + echo " --configure-flags FLAGS specify extra build flags passed to 'configure.py'" echo " --mode MODE specify build mode (default: '$DEFAULT_MODE')" echo " --jobs JOBS specify number of jobs" echo " --clean clean build directory" @@ -59,7 +57,7 @@ while [ $# -gt 0 ]; do esac done -FLAGS="$COMMON_FLAGS $FLAGS" +FLAGS="$FLAGS" if [ ! -e reloc/build_reloc.sh ]; then echo "run build_reloc.sh in top of scylla dir" From 7b7c91a34b15051a1dd6f360bec0d0718864927a Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Mon, 24 Feb 2020 10:41:21 +0200 Subject: [PATCH 06/10] reloc: Add '--builddir' option to build_deb.sh The build system will call this script. It needs control over where the packages are built to allow building packages for the different build modes. --- reloc/build_deb.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/reloc/build_deb.sh b/reloc/build_deb.sh index e40de93b78..6c647e9ec7 100755 --- a/reloc/build_deb.sh +++ b/reloc/build_deb.sh @@ -5,10 +5,12 @@ print_usage() { echo "build_deb.sh -target --dist --rebuild-dep --reloc-pkg build/release/scylla-package.tar.gz" echo " --dist create a public distribution package" echo " --reloc-pkg specify relocatable package path" + echo " --builddir specify Debian package build path" exit 1 } RELOC_PKG=build/release/scylla-package.tar.gz +BUILDDIR=build/debian OPTS="" while [ $# -gt 0 ]; do case "$1" in @@ -21,6 +23,10 @@ while [ $# -gt 0 ]; do RELOC_PKG=$2 shift 2 ;; + "--builddir") + BUILDDIR="$2" + shift 2 + ;; *) print_usage ;; @@ -36,8 +42,8 @@ RELOC_PKG=$(readlink -f $RELOC_PKG) if [[ ! $OPTS =~ --reloc-pkg ]]; then OPTS="$OPTS --reloc-pkg $RELOC_PKG" fi -rm -rf build/debian -mkdir -p build/debian/scylla-package -tar -C build/debian/scylla-package -xpf $RELOC_PKG -cd build/debian/scylla-package +rm -rf $BUILDDIR +mkdir -p $BUILDDIR/scylla-package +tar -C $BUILDDIR/scylla-package -xpf $RELOC_PKG +cd $BUILDDIR/scylla-package exec ./scylla/dist/debian/build_deb.sh $OPTS From 8e1a561fbad03a6e0f5a279ce8fa6de71e388102 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Mon, 24 Feb 2020 10:25:50 +0200 Subject: [PATCH 07/10] build: Add "dist" target --- configure.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/configure.py b/configure.py index db7b51c4d6..09e7b11541 100755 --- a/configure.py +++ b/configure.py @@ -1431,6 +1431,10 @@ with open(buildfile_tmp, 'w') as f: description = COPY $out rule package command = scripts/create-relocatable-package.py --mode $mode $out + rule rpmbuild + command = reloc/build_rpm.sh --reloc-pkg $in --builddir $out + rule debbuild + command = reloc/build_deb.sh --reloc-pkg $in --builddir $out ''').format(**globals())) for mode in build_modes: modeval = modes[mode] @@ -1675,6 +1679,13 @@ with open(buildfile_tmp, 'w') as f: f.write('build build/{mode}/scylla-package.tar.gz: package build/{mode}/scylla build/{mode}/iotune build/SCYLLA-RELEASE-FILE build/SCYLLA-VERSION-FILE build/debian/debian | always\n'.format(**locals())) f.write(' pool = submodule_pool\n') f.write(' mode = {mode}\n'.format(**locals())) + f.write(f'build build/dist/{mode}/redhat: rpmbuild build/{mode}/scylla-package.tar.gz\n') + f.write(f' pool = submodule_pool\n') + f.write(f' mode = {mode}\n') + f.write(f'build build/dist/{mode}/debian: debbuild build/{mode}/scylla-package.tar.gz\n') + f.write(f' pool = submodule_pool\n') + f.write(f' mode = {mode}\n') + f.write(f'build dist-server-{mode}: phony build/dist/{mode}/redhat build/dist/{mode}/debian\n') f.write('rule libdeflate.{mode}\n'.format(**locals())) f.write(' command = make -C libdeflate BUILD_DIR=../build/{mode}/libdeflate/ CFLAGS="{libdeflate_cflags}" CC={args.cc} ../build/{mode}/libdeflate//libdeflate.a\n'.format(**locals())) f.write('build build/{mode}/libdeflate/libdeflate.a: libdeflate.{mode}\n'.format(**locals())) @@ -1696,6 +1707,54 @@ with open(buildfile_tmp, 'w') as f: 'build check: phony {}\n'.format(' '.join(['{mode}-check'.format(mode=mode) for mode in modes])) ) + f.write(textwrap.dedent(f'''\ + build dist-server-deb: phony {' '.join(['build/dist/{mode}/debian'.format(mode=mode) for mode in build_modes])} + build dist-server-rpm: phony {' '.join(['build/dist/{mode}/redhat'.format(mode=mode) for mode in build_modes])} + build dist-server: phony dist-server-rpm dist-server-deb + + rule build-submodule-reloc + command = cd $reloc_dir && ./reloc/build_reloc.sh + rule build-submodule-rpm + command = cd $dir && ./reloc/build_rpm.sh --reloc-pkg $artifact + rule build-submodule-deb + command = cd $dir && ./reloc/build_deb.sh --reloc-pkg $artifact + + build scylla-jmx/build/scylla-jmx-package.tar.gz: build-submodule-reloc + reloc_dir = scylla-jmx + build dist-jmx-rpm: build-submodule-rpm scylla-jmx/build/scylla-jmx-package.tar.gz + dir = scylla-jmx + artifact = build/scylla-jmx-package.tar.gz + build dist-jmx-deb: build-submodule-deb scylla-jmx/build/scylla-jmx-package.tar.gz + dir = scylla-jmx + artifact = build/scylla-jmx-package.tar.gz + build dist-jmx: phony dist-jmx-rpm dist-jmx-deb + + build scylla-tools/build/scylla-tools-package.tar.gz: build-submodule-reloc + reloc_dir = scylla-tools + build dist-tools-rpm: build-submodule-rpm scylla-tools/build/scylla-tools-package.tar.gz + dir = scylla-tools + artifact = build/scylla-tools-package.tar.gz + build dist-tools-deb: build-submodule-deb scylla-tools/build/scylla-tools-package.tar.gz + dir = scylla-tools + artifact = build/scylla-tools-package.tar.gz + build dist-tools: phony dist-tools-rpm dist-tools-deb + + rule build-python-reloc + command = ./reloc/python3/build_reloc.sh + rule build-python-rpm + command = ./reloc/python3/build_rpm.sh + rule build-python-deb + command = ./reloc/python3/build_deb.sh + + build build/release/scylla-python3-package.tar.gz: build-python-reloc + build dist-python-rpm: build-python-rpm build/release/scylla-python3-package.tar.gz + build dist-python-deb: build-python-deb build/release/scylla-python3-package.tar.gz + build dist-python: phony dist-python-rpm dist-python-deb + build dist-deb: phony dist-server-deb dist-python-deb dist-jmx-deb dist-tools-deb + build dist-rpm: phony dist-server-rpm dist-python-rpm dist-jmx-rpm dist-tools-rpm + build dist: phony dist-server dist-python dist-jmx dist-tools + ''')) + f.write(textwrap.dedent('''\ rule configure command = {python} configure.py $configure_args From 584c7130a119af2481651b8fbc7581246a31acf0 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Wed, 18 Mar 2020 14:06:14 +0200 Subject: [PATCH 08/10] scripts/testing: Add "dist-check" for package verification This adds a "dist-check.sh" script in tools/testing, which performs distribution package verification by installing packages under Docker. --- tools/testing/dist-check/dist-check.sh | 70 +++++++++++++++++++ .../testing/dist-check/docker.io/centos:7.sh | 31 ++++++++ tools/testing/dist-check/docker.io/util.sh | 65 +++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100755 tools/testing/dist-check/dist-check.sh create mode 100755 tools/testing/dist-check/docker.io/centos:7.sh create mode 100644 tools/testing/dist-check/docker.io/util.sh diff --git a/tools/testing/dist-check/dist-check.sh b/tools/testing/dist-check/dist-check.sh new file mode 100755 index 0000000000..1733869166 --- /dev/null +++ b/tools/testing/dist-check/dist-check.sh @@ -0,0 +1,70 @@ +#!/bin/bash -e + +# +# Copyright (C) 2020 ScyllaDB +# + +# +# This file is part of Scylla. +# +# Scylla is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Scylla is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Scylla. If not, see . +# + +PROGRAM=$(basename $0) + +print_usage() { + echo "Usage: $PROGRAM [OPTION]..." + echo "" + echo " --mode MODE The build mode of 'scylla' to verify (options: 'release', 'dev', and 'debug')." + exit 1 +} + +while [ $# -gt 0 ]; do + case "$1" in + "--mode") + MODE=$2 + shift 2 + ;; + "--help") + print_usage + ;; + *) + print_usage + ;; + esac +done + +if [ -z "$MODE" ]; then + print_usage +fi + +if [ -f /.dockerenv ]; then + echo "error: running $PROGRAM in Docker is not supported, please run on host." + exit 1 +fi + +docker_images=( + docker.io/centos:7 +) + +for docker_image in "${docker_images[@]}" +do + install_sh="$(pwd)/tools/testing/dist-check/$docker_image.sh" + if [ -f "$install_sh" ]; then + docker run -i --rm -v $(pwd):$(pwd) $docker_image /bin/bash -c "cd $(pwd) && $install_sh --mode $MODE" + else + echo "internal error: $install_sh does not exist, please create one to verify packages on $docker_image." + exit 1 + fi +done diff --git a/tools/testing/dist-check/docker.io/centos:7.sh b/tools/testing/dist-check/docker.io/centos:7.sh new file mode 100755 index 0000000000..345cd3d39e --- /dev/null +++ b/tools/testing/dist-check/docker.io/centos:7.sh @@ -0,0 +1,31 @@ +#!/bin/bash -e + +# +# Copyright (C) 2020 ScyllaDB +# + +# +# This file is part of Scylla. +# +# Scylla is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Scylla is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Scylla. If not, see . +# + +source "$(dirname $0)/util.sh" + +echo "Installing Scylla ($MODE) packages on $PRETTY_NAME..." + +rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 +yum install -y -q deltarpm +yum update -y -q +yum install -y "${SCYLLA_RPMS[@]}" diff --git a/tools/testing/dist-check/docker.io/util.sh b/tools/testing/dist-check/docker.io/util.sh new file mode 100644 index 0000000000..b59790cb53 --- /dev/null +++ b/tools/testing/dist-check/docker.io/util.sh @@ -0,0 +1,65 @@ +# +# Copyright (C) 2020 ScyllaDB +# + +# +# This file is part of Scylla. +# +# Scylla is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Scylla is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Scylla. If not, see . +# + +PROGRAM=$(basename $0) + +print_usage() { + echo "Usage: $PROGRAM [OPTION]..." + echo "" + echo " --mode MODE Specify the Scylla server build mode to install." + exit 1 +} + +while [ $# -gt 0 ]; do + case "$1" in + "--mode") + MODE=$2 + shift 2 + ;; + "--help") + print_usage + ;; + *) + print_usage + ;; + esac +done + +if [ -z "$MODE" ]; then + print_usage +fi + +SCYLLA_PRODUCT=$(cat build/SCYLLA-PRODUCT-FILE) +SCYLLA_RELEASE=$(cat build/SCYLLA-RELEASE-FILE) +SCYLLA_PYTHON_RELEASE=$(cat build/redhat/scylla-python3/SCYLLA-RELEASE-FILE) +SCYLLA_JMX_PRODUCT=$(cat scylla-jmx/build/SCYLLA-PRODUCT-FILE) +SCYLLA_JMX_RELEASE=$(cat scylla-jmx/build/SCYLLA-RELEASE-FILE) +SCYLLA_TOOLS_PRODUCT=$(cat scylla-tools/build/SCYLLA-PRODUCT-FILE) +SCYLLA_TOOLS_RELEASE=$(cat scylla-tools/build/SCYLLA-RELEASE-FILE) + +SCYLLA_RPMS=( + build/dist/$MODE/redhat/RPMS/x86_64/$SCYLLA_PRODUCT-*$SCYLLA_RELEASE*.rpm + build/redhat/RPMS/x86_64/$SCYLLA_PRODUCT-python3-*$SCYLLA_PYTHON_RELEASE*.rpm + scylla-jmx/build/redhat/RPMS/noarch/$SCYLLA_JMX_PRODUCT*$SCYLLA_JMX_RELEASE*.rpm + scylla-tools/build/redhat/RPMS/noarch/$SCYLLA_TOOLS_PRODUCT*$SCYLLA_TOOLS_RELEASE*.rpm +) + +source /etc/os-release From 9e279ec2a9e051562620cd07b694c022e939ab03 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Wed, 18 Mar 2020 15:13:51 +0200 Subject: [PATCH 09/10] build: Add "dist-check" target This adds a "dist-check" target to ninja build. The target needs to be run on the host because package verification is done with Docker. --- configure.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/configure.py b/configure.py index 09e7b11541..48c5ad64aa 100755 --- a/configure.py +++ b/configure.py @@ -1755,6 +1755,17 @@ with open(buildfile_tmp, 'w') as f: build dist: phony dist-server dist-python dist-jmx dist-tools ''')) + f.write(textwrap.dedent(f'''\ + build dist-check: phony {' '.join(['dist-check-{mode}'.format(mode=mode) for mode in build_modes])} + rule dist-check + command = ./tools/testing/dist-check/dist-check.sh --mode $mode + ''')) + for mode in build_modes: + f.write(textwrap.dedent(f'''\ + build dist-check-{mode}: dist-check + mode = {mode} + ''')) + f.write(textwrap.dedent('''\ rule configure command = {python} configure.py $configure_args From 4d48f22827ac81af846327c976971f1ad292fe2a Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Wed, 18 Mar 2020 14:07:05 +0200 Subject: [PATCH 10/10] docs: Update packaging documentation --- docs/building-packages.md | 114 ++++++++++++++------------------------ 1 file changed, 42 insertions(+), 72 deletions(-) diff --git a/docs/building-packages.md b/docs/building-packages.md index 80000783d0..8113115b6e 100644 --- a/docs/building-packages.md +++ b/docs/building-packages.md @@ -1,90 +1,60 @@ # Building Scylla Packages -This document describes how to build Scylla's packages for supported Linux distributions. +This document describes how to build Scylla's packages. The build system generates _relocatable packages_, which means that the packages contain all the dependencies they need, and you can, therefore, install and run the same binaries on all Linux distributions. -The packages are built using Scylla's `dbuild` tool, which builds the packages in a Docker container. +The relocatable package tarball is used as a base for building the Linux distribution specific packages, `.rpm`s and `.deb`s. -### Scylla Server +## Building -To build a tarball, which contains the full Scylla server, run (remember to substitute `` with, for example, `release`): +In these instructions, we use `dbuild` to build the packages, but you can also build packages without it. + +The first step is to run `configure.py`, which generates the `build.ninja` file (equivalent of a `Makefile`): ``` -./tools/toolchain/dbuild ./reloc/build_reloc.sh --mode +./tools/toolchain/dbuild ./configure.py --mode= ``` -This step generates the following tarball: +(where `mode` is `release`, `dev`, or `debug`) + +The second step is to build the packages with the `dist` target. ``` -build//scylla-package.tar.gz +./tools/toolchain/dbuild ninja-build dist ``` -You can then generate RPM packages with: +## Artifacts + +The `dist` target generates the following tarballs (*relocatable packages*), which can be installed on any Linux distribution: + +### Relocatable package artifacts + +* `build//scylla-package.tar.gz` +* `build/release/scylla-python3-package.tar.gz` +* `scylla-jmx/build/scylla-jmx-package.tar.gz` +* `scylla-tools/build/scylla-tools-package.tar.gz` + +### RPM package artifacts + +* `build/dist//redhat/RPMS/x86_64/scylla-*.rpm` +* `build/dist//redhat/RPMS/x86_64/scylla-conf-*.rpm` +* `build/dist//redhat/RPMS/x86_64/scylla-debuginfo-*.rpm` +* `build/dist//redhat/RPMS/x86_64/scylla-kernel-conf-*.rpm` +* `build/dist//redhat/RPMS/x86_64/scylla-server-*.rpm` +* `build/redhat/RPMS/x86_64/scylla-python3-*.rpm` +* `scylla-jmx/build/redhat/RPMS/noarch/scylla-jmx-*.rpm` +* `scylla-tools/build/redhat/RPMS/noarch/scylla-tools-*.rpm` +* `scylla-tools/build/redhat/RPMS/noarch/scylla-tools-core-*.rpm` + +### Debian package artifacts + +## Verifying + +To verify built Scylla packages, run the following command: ``` -./tools/toolchain/dbuild ./reloc/build_rpm.sh --reloc-pkg build//scylla-package.tar.gz +ninja-build dist-check ``` -This step generates the following RPM packages: - -``` -build/redhat/RPMS/x86_64/scylla-666.development-0.20190807.7d0c99e268.x86_64.rpm -build/redhat/RPMS/x86_64/scylla-server-666.development-0.20190807.7d0c99e268.x86_64.rpm -build/redhat/RPMS/x86_64/scylla-debuginfo-666.development-0.20190807.7d0c99e268.x86_64.rpm -build/redhat/RPMS/x86_64/scylla-conf-666.development-0.20190807.7d0c99e268.x86_64.rpm -build/redhat/RPMS/x86_64/scylla-kernel-conf-666.development-0.20190807.7d0c99e268.x86_64.rpm -``` - -You can also generate deb packages with: - -``` -./tools/toolchain/dbuild ./reloc/build_deb.sh --reloc-pkg build/debug/scylla-package.tar.gz -``` - -This step generates the following deb packages: - -``` -build/debian/scylla-server-dbg_666.development-0.20190807.7d0c99e268-1_amd64.deb -build/debian/scylla-conf_666.development-0.20190807.7d0c99e268-1_amd64.deb -build/debian/scylla-kernel-conf_666.development-0.20190807.7d0c99e268-1_amd64.deb -build/debian/scylla-server_666.development-0.20190807.7d0c99e268-1_amd64.deb -build/debian/scylla_666.development-0.20190807.7d0c99e268-1_amd64.deb -``` - -### Python interpreter - -To build a tarball, which contains a [portable Python interpreter](https://www.scylladb.com/2019/02/14/the-complex-path-for-a-simple-portable-python-interpreter-or-snakes-on-a-data-plane/), run: - -``` -./tools/toolchain/dbuild ./reloc/python3/build_reloc.sh -``` - -This step generates the following tarball: - -``` -build/release/scylla-python3-package.tar.gz -``` - -You can then generate a RPM package: - -``` -./tools/toolchain/dbuild ./reloc/python3/build_rpm.sh -``` - -This step generates the following RPM package: - -``` -build/redhat/RPMS/x86_64/scylla-python3-3.7.2-0.20190807.689fc72bab.x86_64.rpm -``` - -You can also generate a deb package with: - -``` -./tools/toolchain/dbuild ./reloc/python3/build_deb.sh -``` - -This step generates the following deb package: - -``` -build/debian/scylla-python3_3.7.2-0.20190807.689fc72bab-1_amd64.deb -``` +Please note that you need to run `dist-check` on the host because it +requires Docker to perform the verification.