From 7adae6b04afd60c285521c3e57687c22130dfcd4 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jul 2020 15:26:44 +0300 Subject: [PATCH 1/4] configure.py: Use copy-on-write copies when possible Pass the "--reflink=auto" command line option to "cp" to use copy-on-write copies whenever the filesystem supports it to reduce disk space usage. --- configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.py b/configure.py index e2cca06cf9..e041b6284f 100755 --- a/configure.py +++ b/configure.py @@ -1439,7 +1439,7 @@ with open(buildfile_tmp, 'w') as f: command = $in > $out description = GEN $out rule copy - command = cp $in $out + command = cp --reflink=auto $in $out description = COPY $out rule package command = scripts/create-relocatable-package.py --mode $mode $out From e4685020bae0aae851abdbc83f038e6c5d425df2 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Wed, 22 Jul 2020 14:06:28 +0300 Subject: [PATCH 2/4] reloc/python3: Add "--builddir" to build_deb.sh Add a "--builddir" command line option to build_deb.sh script of Python 3 so that we can use it to control artifact build location. --- reloc/python3/build_deb.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/reloc/python3/build_deb.sh b/reloc/python3/build_deb.sh index edf507dc02..e9969b21c3 100755 --- a/reloc/python3/build_deb.sh +++ b/reloc/python3/build_deb.sh @@ -4,16 +4,22 @@ print_usage() { echo "build_deb.sh --reloc-pkg build/release/scylla-python3-package.tar.gz" echo " --reloc-pkg specify relocatable package path" + echo " --builddir specify Debian package build path" exit 1 } RELOC_PKG=build/release/scylla-python3-package.tar.gz +BUILDDIR=build/debian while [ $# -gt 0 ]; do case "$1" in "--reloc-pkg") RELOC_PKG=$2 shift 2 ;; + "--builddir") + BUILDDIR="$2" + shift 2 + ;; *) print_usage ;; @@ -26,10 +32,10 @@ if [ ! -e $RELOC_PKG ]; then exit 1 fi RELOC_PKG=$(readlink -f $RELOC_PKG) -rm -rf build/debian/scylla-python3-package -mkdir -p build/debian/scylla-python3-package -tar -C build/debian/scylla-python3-package -xpf $RELOC_PKG -cd build/debian/scylla-python3-package +rm -rf "$BUILDDIR/scylla-python3-package" +mkdir -p "$BUILDDIR/scylla-python3-package" +tar -C "$BUILDDIR/scylla-python3-package" -xpf $RELOC_PKG +cd "$BUILDDIR/scylla-python3-package" PRODUCT=$(cat scylla-python3/SCYLLA-PRODUCT-FILE) RELOC_PKG_FULLPATH=$(readlink -f $RELOC_PKG) From 5a2d271df83290c887013ca3038c38a6c94576aa Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Fri, 7 Aug 2020 12:53:37 +0300 Subject: [PATCH 3/4] configure.py: Add "dist--tar" targets --- configure.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/configure.py b/configure.py index e041b6284f..c6e3f2a033 100755 --- a/configure.py +++ b/configure.py @@ -1733,7 +1733,8 @@ with open(buildfile_tmp, 'w') as f: f.write(textwrap.dedent(f'''\ build dist-server-deb: phony {' '.join(['$builddir/dist/{mode}/debian'.format(mode=mode) for mode in build_modes])} build dist-server-rpm: phony {' '.join(['$builddir/dist/{mode}/redhat'.format(mode=mode) for mode in build_modes])} - build dist-server: phony dist-server-rpm dist-server-deb + build dist-server-tar: phony {' '.join(['$builddir/{mode}/scylla-package.tar.gz'.format(mode=mode) for mode in build_modes])} + build dist-server: phony dist-server-tar dist-server-rpm dist-server-deb rule build-submodule-reloc command = cd $reloc_dir && ./reloc/build_reloc.sh --nodeps @@ -1750,7 +1751,8 @@ with open(buildfile_tmp, 'w') as f: build dist-jmx-deb: build-submodule-deb tools/jmx/build/scylla-jmx-package.tar.gz dir = tools/jmx artifact = $builddir/scylla-jmx-package.tar.gz - build dist-jmx: phony dist-jmx-rpm dist-jmx-deb + build dist-jmx-tar: phony tools/jmx/build/scylla-jmx-package.tar.gz + build dist-jmx: phony dist-jmx-tar dist-jmx-rpm dist-jmx-deb build tools/java/build/scylla-tools-package.tar.gz: build-submodule-reloc reloc_dir = tools/java @@ -1760,7 +1762,8 @@ with open(buildfile_tmp, 'w') as f: build dist-tools-deb: build-submodule-deb tools/java/build/scylla-tools-package.tar.gz dir = tools/java artifact = $builddir/scylla-tools-package.tar.gz - build dist-tools: phony dist-tools-rpm dist-tools-deb + build dist-tools-tar: phony tools/java/build/scylla-tools-package.tar.gz + build dist-tools: phony dist-tools-tar dist-tools-rpm dist-tools-deb rule build-python-reloc command = ./reloc/python3/build_reloc.sh @@ -1772,9 +1775,13 @@ with open(buildfile_tmp, 'w') as f: build $builddir/release/scylla-python3-package.tar.gz: build-python-reloc build dist-python-rpm: build-python-rpm $builddir/release/scylla-python3-package.tar.gz build dist-python-deb: build-python-deb $builddir/release/scylla-python3-package.tar.gz + build dist-python-tar: phony $builddir/release/scylla-python3-package.tar.gz build dist-python: phony dist-python-rpm dist-python-deb + build dist-deb: phony dist-server-deb dist-python-deb dist-jmx-deb dist-tools-deb build dist-rpm: phony dist-server-rpm dist-python-rpm dist-jmx-rpm dist-tools-rpm + build dist-tar: phony dist-server-tar dist-python-tar dist-jmx-tar dist-tools-tar + build dist: phony dist-server dist-python dist-jmx dist-tools ''')) From 3c1db2fb87a6cb9e760ff0a1df7d29c9cb88abbc Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Thu, 13 Aug 2020 13:27:22 +0300 Subject: [PATCH 4/4] configure.py: Copy tarballs to build/dist//tar directory --- configure.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/configure.py b/configure.py index c6e3f2a033..2f65d2c52d 100755 --- a/configure.py +++ b/configure.py @@ -1693,9 +1693,10 @@ with open(buildfile_tmp, 'w') as f: f.write(textwrap.dedent('''\ build $builddir/{mode}/iotune: copy $builddir/{mode}/seastar/apps/iotune/iotune ''').format(**locals())) - f.write('build $builddir/{mode}/scylla-package.tar.gz: package $builddir/{mode}/scylla $builddir/{mode}/iotune $builddir/SCYLLA-RELEASE-FILE $builddir/SCYLLA-VERSION-FILE $builddir/debian/debian | always\n'.format(**locals())) + f.write('build $builddir/dist/{mode}/tar/scylla-package.tar.gz: package $builddir/{mode}/scylla $builddir/{mode}/iotune $builddir/SCYLLA-RELEASE-FILE $builddir/SCYLLA-VERSION-FILE $builddir/debian/debian | always\n'.format(**locals())) f.write(' pool = submodule_pool\n') f.write(' mode = {mode}\n'.format(**locals())) + f.write(f'build $builddir/{mode}/scylla-package.tar.gz: copy $builddir/dist/{mode}/tar/scylla-package.tar.gz\n') f.write(f'build $builddir/dist/{mode}/redhat: rpmbuild $builddir/{mode}/scylla-package.tar.gz\n') f.write(f' pool = submodule_pool\n') f.write(f' mode = {mode}\n') @@ -1751,7 +1752,7 @@ with open(buildfile_tmp, 'w') as f: build dist-jmx-deb: build-submodule-deb tools/jmx/build/scylla-jmx-package.tar.gz dir = tools/jmx artifact = $builddir/scylla-jmx-package.tar.gz - build dist-jmx-tar: phony tools/jmx/build/scylla-jmx-package.tar.gz + build dist-jmx-tar: phony {' '.join(['$builddir/dist/{mode}/tar/scylla-jmx-package.tar.gz'.format(mode=mode) for mode in build_modes])} build dist-jmx: phony dist-jmx-tar dist-jmx-rpm dist-jmx-deb build tools/java/build/scylla-tools-package.tar.gz: build-submodule-reloc @@ -1762,7 +1763,7 @@ with open(buildfile_tmp, 'w') as f: build dist-tools-deb: build-submodule-deb tools/java/build/scylla-tools-package.tar.gz dir = tools/java artifact = $builddir/scylla-tools-package.tar.gz - build dist-tools-tar: phony tools/java/build/scylla-tools-package.tar.gz + build dist-tools-tar: phony {' '.join(['$builddir/dist/{mode}/tar/scylla-tools-package.tar.gz'.format(mode=mode) for mode in build_modes])} build dist-tools: phony dist-tools-tar dist-tools-rpm dist-tools-deb rule build-python-reloc @@ -1775,7 +1776,7 @@ with open(buildfile_tmp, 'w') as f: build $builddir/release/scylla-python3-package.tar.gz: build-python-reloc build dist-python-rpm: build-python-rpm $builddir/release/scylla-python3-package.tar.gz build dist-python-deb: build-python-deb $builddir/release/scylla-python3-package.tar.gz - build dist-python-tar: phony $builddir/release/scylla-python3-package.tar.gz + build dist-python-tar: phony {' '.join(['$builddir/dist/{mode}/tar/scylla-python3-package.tar.gz'.format(mode=mode) for mode in build_modes])} build dist-python: phony dist-python-rpm dist-python-deb build dist-deb: phony dist-server-deb dist-python-deb dist-jmx-deb dist-tools-deb @@ -1792,6 +1793,10 @@ with open(buildfile_tmp, 'w') as f: ''')) for mode in build_modes: f.write(textwrap.dedent(f'''\ + build $builddir/dist/{mode}/tar/scylla-python3-package.tar.gz: copy build/release/scylla-python3-package.tar.gz + build $builddir/dist/{mode}/tar/scylla-tools-package.tar.gz: copy tools/java/build/scylla-tools-package.tar.gz + build $builddir/dist/{mode}/tar/scylla-jmx-package.tar.gz: copy tools/jmx/build/scylla-jmx-package.tar.gz + build dist-{mode}: phony dist-server-{mode} dist-python dist-tools dist-jmx build dist-check-{mode}: dist-check mode = {mode}