From 536ab4ebe40266624baeb2f271481b8feb35f71a Mon Sep 17 00:00:00 2001 From: Takuya ASADA Date: Thu, 30 Apr 2020 03:01:53 +0900 Subject: [PATCH] reloc-pkg: move all files under project name directory To make unified relocatable package easily, we may want to merge tarballs to single tarball like this: zcat *.tar.gz | gzip -c > scylla-unified.tar.xz But it's not possible with current relocatable package format, since there are multiple files conflicts, install.sh, SCYLLA-*-FILE, dist/, README.md, etc.. To support this, we need to archive everything in the directory when building relocatable package. This is modifying relocatable package format, we need to provide a way to detect the format version. To do this, we added a new file ".relocatable_package_version" on the top of the archive, and set version number "2" to the file. Fixes #6315 --- dist/debian/build_deb.sh | 9 +-- dist/debian/debian/rules | 4 +- dist/debian/python3/build_deb.sh | 16 ++--- dist/debian/python3/rules.mustache | 2 +- dist/redhat/python3/python.spec | 2 +- dist/redhat/scylla.spec | 2 +- reloc/build_deb.sh | 2 +- reloc/build_rpm.sh | 8 +-- reloc/python3/build_deb.sh | 2 +- reloc/python3/build_rpm.sh | 6 +- scripts/create-relocatable-package-python3.py | 50 ++++++++++----- scripts/create-relocatable-package.py | 63 ++++++++++++------- 12 files changed, 102 insertions(+), 64 deletions(-) diff --git a/dist/debian/build_deb.sh b/dist/debian/build_deb.sh index 16931009d2..8d383bff18 100755 --- a/dist/debian/build_deb.sh +++ b/dist/debian/build_deb.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -PRODUCT=$(cat SCYLLA-PRODUCT-FILE) +PRODUCT=$(cat scylla/SCYLLA-PRODUCT-FILE) . /etc/os-release print_usage() { @@ -45,7 +45,7 @@ pkg_install() { fi } -if [ ! -e SCYLLA-RELOCATABLE-FILE ]; then +if [ ! -e scylla/SCYLLA-RELOCATABLE-FILE ]; then echo "do not directly execute build_deb.sh, use reloc/build_deb.sh instead." exit 1 fi @@ -103,11 +103,12 @@ fi RELOC_PKG_FULLPATH=$(readlink -f $RELOC_PKG) RELOC_PKG_BASENAME=$(basename $RELOC_PKG) -SCYLLA_VERSION=$(cat SCYLLA-VERSION-FILE | sed 's/\.rc/~rc/') -SCYLLA_RELEASE=$(cat SCYLLA-RELEASE-FILE) +SCYLLA_VERSION=$(cat scylla/SCYLLA-VERSION-FILE | sed 's/\.rc/~rc/') +SCYLLA_RELEASE=$(cat scylla/SCYLLA-RELEASE-FILE) ln -fv $RELOC_PKG_FULLPATH ../$PRODUCT-server_$SCYLLA_VERSION-$SCYLLA_RELEASE.orig.tar.gz +cp -al scylla/debian debian if $DIST; then export DEB_BUILD_OPTIONS="housekeeping" fi diff --git a/dist/debian/debian/rules b/dist/debian/debian/rules index 04d762c257..67f2701ac3 100755 --- a/dist/debian/debian/rules +++ b/dist/debian/debian/rules @@ -20,9 +20,9 @@ override_dh_auto_clean: override_dh_auto_install: dh_auto_install - ./install.sh --packaging --root "$(CURDIR)/debian/tmp" $(install_arg) --sysconfdir "/etc/default" + cd scylla; ./install.sh --packaging --root "$(CURDIR)/debian/tmp" $(install_arg) --sysconfdir "/etc/default" # don't use default sysconfig file, use Debian version - cp dist/debian/sysconfig/scylla-housekeeping $(CURDIR)/debian/tmp/etc/default/ + cp scylla/dist/debian/sysconfig/scylla-housekeeping $(CURDIR)/debian/tmp/etc/default/ override_dh_installinit: ifeq ($(product),scylla) diff --git a/dist/debian/python3/build_deb.sh b/dist/debian/python3/build_deb.sh index 674f33d813..4ddf44371e 100755 --- a/dist/debian/python3/build_deb.sh +++ b/dist/debian/python3/build_deb.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -PRODUCT=$(cat SCYLLA-PRODUCT-FILE) +PRODUCT=$(cat scylla-python3/SCYLLA-PRODUCT-FILE) . /etc/os-release print_usage() { @@ -40,7 +40,7 @@ pkg_install() { fi } -if [ ! -e SCYLLA-RELOCATABLE-FILE ]; then +if [ ! -e scylla-python3/SCYLLA-RELOCATABLE-FILE ]; then echo "do not directly execute build_deb.sh, use reloc/build_deb.sh instead." exit 1 fi @@ -119,12 +119,12 @@ if [ -z "$TARGET" ]; then fi RELOC_PKG_FULLPATH=$(readlink -f $RELOC_PKG) RELOC_PKG_BASENAME=$(basename $RELOC_PKG) -SCYLLA_VERSION=$(cat SCYLLA-VERSION-FILE) -SCYLLA_RELEASE=$(cat SCYLLA-RELEASE-FILE) +SCYLLA_VERSION=$(cat scylla-python3/SCYLLA-VERSION-FILE) +SCYLLA_RELEASE=$(cat scylla-python3/SCYLLA-RELEASE-FILE) ln -fv $RELOC_PKG_FULLPATH ../$PRODUCT-python3_$SCYLLA_VERSION-$SCYLLA_RELEASE.orig.tar.gz -cp -al dist/debian/python3/debian debian +cp -al scylla-python3/dist/debian/python3/debian debian if [ "$PRODUCT" != "scylla" ]; then # rename all 'scylla-' prefixed artifacts in the debian folder to have the # product name as a prefix @@ -132,9 +132,9 @@ if [ "$PRODUCT" != "scylla" ]; then fi REVISION="1" MUSTACHE_DIST="\"debian\": true, \"product\": \"$PRODUCT\", \"$PRODUCT\": true" -pystache dist/debian/python3/changelog.mustache "{ $MUSTACHE_DIST, \"version\": \"$SCYLLA_VERSION\", \"release\": \"$SCYLLA_RELEASE\", \"revision\": \"$REVISION\", \"codename\": \"$TARGET\" }" > debian/changelog -pystache dist/debian/python3/rules.mustache "{ $MUSTACHE_DIST }" > debian/rules -pystache dist/debian/python3/control.mustache "{ $MUSTACHE_DIST }" > debian/control +pystache scylla-python3/dist/debian/python3/changelog.mustache "{ $MUSTACHE_DIST, \"version\": \"$SCYLLA_VERSION\", \"release\": \"$SCYLLA_RELEASE\", \"revision\": \"$REVISION\", \"codename\": \"$TARGET\" }" > debian/changelog +pystache scylla-python3/dist/debian/python3/rules.mustache "{ $MUSTACHE_DIST }" > debian/rules +pystache scylla-python3/dist/debian/python3/control.mustache "{ $MUSTACHE_DIST }" > debian/control chmod a+rx debian/rules debuild -rfakeroot -us -uc diff --git a/dist/debian/python3/rules.mustache b/dist/debian/python3/rules.mustache index 1cf84d1139..dba165efc6 100755 --- a/dist/debian/python3/rules.mustache +++ b/dist/debian/python3/rules.mustache @@ -8,7 +8,7 @@ override_dh_auto_build: override_dh_auto_install: dh_auto_install - ./install.sh --root "$(CURDIR)/debian/{{product}}-python3" + cd scylla-python3; ./install.sh --root "$(CURDIR)/debian/{{product}}-python3" override_dh_strip: diff --git a/dist/redhat/python3/python.spec b/dist/redhat/python3/python.spec index 4d3f72b8d8..084317e928 100644 --- a/dist/redhat/python3/python.spec +++ b/dist/redhat/python3/python.spec @@ -23,7 +23,7 @@ functionality). All shared libraries needed for the interpreter to operate are shipped with it. %prep -%setup -q -c +%setup -n scylla-python3 %install ./install.sh --root "$RPM_BUILD_ROOT" diff --git a/dist/redhat/scylla.spec b/dist/redhat/scylla.spec index 51f0201881..233af25f8a 100644 --- a/dist/redhat/scylla.spec +++ b/dist/redhat/scylla.spec @@ -44,7 +44,7 @@ This package installs all required packages for ScyllaDB, including %defattr(-,root,root) %prep -%setup -c +%setup -n scylla %package server Group: Applications/Databases diff --git a/reloc/build_deb.sh b/reloc/build_deb.sh index 48d7553da9..6c6e307449 100755 --- a/reloc/build_deb.sh +++ b/reloc/build_deb.sh @@ -39,4 +39,4 @@ fi mkdir -p build/debian/scylla-package tar -C build/debian/scylla-package -xpf $RELOC_PKG cd build/debian/scylla-package -exec ./dist/debian/build_deb.sh $OPTS +exec ./scylla/dist/debian/build_deb.sh $OPTS diff --git a/reloc/build_rpm.sh b/reloc/build_rpm.sh index b38de058fc..c79b94743b 100755 --- a/reloc/build_rpm.sh +++ b/reloc/build_rpm.sh @@ -52,8 +52,8 @@ RELOC_PKG=$(readlink -f $RELOC_PKG) if [[ ! $OPTS =~ --reloc-pkg ]]; then OPTS="$OPTS --reloc-pkg $RELOC_PKG" fi -mkdir -p $BUILDDIR/scylla-package -tar -C $BUILDDIR/scylla-package -xpf $RELOC_PKG SCYLLA-RELOCATABLE-FILE SCYLLA-RELEASE-FILE SCYLLA-VERSION-FILE SCYLLA-PRODUCT-FILE dist/redhat -cd $BUILDDIR/scylla-package -echo "Running './dist/redhat/build_rpm.sh $OPTS' under $BUILDDIR/scylla-package directory" +mkdir -p $BUILDDIR/ +tar -C $BUILDDIR/ -xpf $RELOC_PKG scylla/SCYLLA-RELOCATABLE-FILE scylla/SCYLLA-RELEASE-FILE scylla/SCYLLA-VERSION-FILE scylla/SCYLLA-PRODUCT-FILE scylla/dist/redhat +cd $BUILDDIR/scylla +echo "Running './dist/redhat/build_rpm.sh $OPTS' under $BUILDDIR/scylla directory" exec ./dist/redhat/build_rpm.sh $OPTS diff --git a/reloc/python3/build_deb.sh b/reloc/python3/build_deb.sh index 05a0f7e566..86c9f2d78d 100755 --- a/reloc/python3/build_deb.sh +++ b/reloc/python3/build_deb.sh @@ -34,4 +34,4 @@ fi mkdir -p build/debian/scylla-python3-package tar -C build/debian/scylla-python3-package -xpf $RELOC_PKG cd build/debian/scylla-python3-package -exec ./dist/debian/python3/build_deb.sh $OPTS +exec ./scylla-python3/dist/debian/python3/build_deb.sh $OPTS diff --git a/reloc/python3/build_rpm.sh b/reloc/python3/build_rpm.sh index 5d9f115956..d82b382ed4 100755 --- a/reloc/python3/build_rpm.sh +++ b/reloc/python3/build_rpm.sh @@ -36,7 +36,7 @@ RELOC_PKG=$(readlink -f $RELOC_PKG) if [[ ! $OPTS =~ --reloc-pkg ]]; then OPTS="$OPTS --reloc-pkg $RELOC_PKG" fi -mkdir -p $BUILDDIR/scylla-python3-package -tar -C $BUILDDIR/scylla-python3-package -xpf $RELOC_PKG SCYLLA-RELOCATABLE-FILE SCYLLA-RELEASE-FILE SCYLLA-VERSION-FILE SCYLLA-PRODUCT-FILE dist/redhat/python3 -cd $BUILDDIR/scylla-python3-package +mkdir -p $BUILDDIR/scylla-python3 +tar -C $BUILDDIR -xpf $RELOC_PKG scylla-python3/SCYLLA-RELOCATABLE-FILE scylla-python3/SCYLLA-RELEASE-FILE scylla-python3/SCYLLA-VERSION-FILE scylla-python3/SCYLLA-PRODUCT-FILE scylla-python3/dist/redhat/python3 +cd $BUILDDIR/scylla-python3 exec ./dist/redhat/python3/build_rpm.sh $OPTS diff --git a/scripts/create-relocatable-package-python3.py b/scripts/create-relocatable-package-python3.py index 07bc8c49ba..6bd6faf46d 100755 --- a/scripts/create-relocatable-package-python3.py +++ b/scripts/create-relocatable-package-python3.py @@ -34,6 +34,21 @@ import tarfile from tempfile import mkstemp import magic + +RELOC_PREFIX='scylla-python3' +def reloc_add(self, name, arcname=None, recursive=True, *, filter=None): + if arcname: + return self.add(name, arcname="{}/{}".format(RELOC_PREFIX, arcname)) + else: + return self.add(name, arcname="{}/{}".format(RELOC_PREFIX, name)) + +def reloc_addfile(self, tarinfo, fileobj=None): + tarinfo.name = "{}/{}".format(RELOC_PREFIX, tarinfo.name) + return self.addfile(tarinfo, fileobj) + +tarfile.TarFile.reloc_add = reloc_add +tarfile.TarFile.reloc_addfile = reloc_addfile + def should_copy(f): '''Given a file, returns whether or not we are interested in copying this file. We want the actual python interepreter, and the files in /lib(64) and /usr/lib(64) @@ -95,14 +110,14 @@ def fix_python_binary(ar, binpath): interpreter = subprocess.check_output(['patchelf', '--print-interpreter', patched_binary], universal_newlines=True).splitlines()[0] - ar.add(os.path.realpath(interpreter), arcname=os.path.join("libexec", "ld.so")) - ar.add(patched_binary, arcname=os.path.join("libexec", pyname + ".bin")) + ar.reloc_add(os.path.realpath(interpreter), arcname=os.path.join("libexec", "ld.so")) + ar.reloc_add(patched_binary, arcname=os.path.join("libexec", pyname + ".bin")) os.remove(patched_binary) def fix_sharedlib(ar, binpath, targetpath): relpath = os.path.join(os.path.relpath("lib64", targetpath), "lib64") patched_binary = fix_binary(ar, binpath, '$ORIGIN/' + relpath) - ar.add(patched_binary, arcname=targetpath) + ar.reloc_add(patched_binary, arcname=targetpath) os.remove(patched_binary) def gen_python_thunk(ar, pybin): @@ -124,12 +139,12 @@ PYTHONPATH="$d/{sitepackages}:$d/{sitepackages64}:$PYTHONPATH" exec -a "$0" "$ld ti = tarfile.TarInfo(name=os.path.join("bin", pybin)) ti.size = len(thunk) ti.mode = 0o755 - ar.addfile(ti, fileobj=io.BytesIO(thunk)) + ar.reloc_addfile(ti, fileobj=io.BytesIO(thunk)) ti = tarfile.TarInfo(name=os.path.join("bin", "python3")) ti.type = tarfile.SYMTYPE ti.linkname = pybin - ar.addfile(ti) + ar.reloc_addfile(ti) def copy_file_to_python_env(ar, f): if f.startswith("/usr/bin/python"): @@ -154,7 +169,7 @@ def copy_file_to_python_env(ar, f): # links to the current directory are usually safe, but because we are manipulating # the directory structure, very likely links that transverse paths will break. if os.path.islink(f) and os.readlink(f) != os.path.basename(os.readlink(f)): - ar.add(os.path.realpath(f), arcname=libfile) + ar.reloc_add(os.path.realpath(f), arcname=libfile) else: m = magic.detect_from_filename(f) if m and (m.mime_type.startswith('application/x-sharedlib') or m.mime_type.startswith('application/x-pie-executable')): @@ -163,7 +178,7 @@ def copy_file_to_python_env(ar, f): # in case this is a directory that is listed, we don't want to include everything that is in that directory # for instance, the python3 package will own site-packages, but other packages that we are not packaging could have # filled it with stuff. - ar.add(f, arcname=libfile, recursive=False) + ar.reloc_add(f, arcname=libfile, recursive=False) def filter_basic_packages(package): '''Returns true if this package should be filtered out. We filter out packages that are too basic like the Fedora repos, @@ -228,19 +243,24 @@ packages= ["python3"] + args.modules file_list = generate_file_list(dependencies(packages)) ar = tarfile.open(args.output, mode='w|gz') +# relocatable package format version = 2 +with open('build/.relocatable_package_version', 'w') as f: + f.write('2\n') +ar.add('build/.relocatable_package_version', arcname='.relocatable_package_version') + pathlib.Path('build/SCYLLA-RELOCATABLE-FILE').touch() -ar.add('build/SCYLLA-RELOCATABLE-FILE', arcname='SCYLLA-RELOCATABLE-FILE') -ar.add('dist/redhat/python3') -ar.add('dist/debian/python3') -ar.add('build/python3/SCYLLA-RELEASE-FILE', arcname='SCYLLA-RELEASE-FILE') -ar.add('build/python3/SCYLLA-VERSION-FILE', arcname='SCYLLA-VERSION-FILE') -ar.add('build/SCYLLA-PRODUCT-FILE', arcname='SCYLLA-PRODUCT-FILE') -ar.add('python3/install.sh', arcname='install.sh') +ar.reloc_add('build/SCYLLA-RELOCATABLE-FILE', arcname='SCYLLA-RELOCATABLE-FILE') +ar.reloc_add('dist/redhat/python3') +ar.reloc_add('dist/debian/python3') +ar.reloc_add('build/python3/SCYLLA-RELEASE-FILE', arcname='SCYLLA-RELEASE-FILE') +ar.reloc_add('build/python3/SCYLLA-VERSION-FILE', arcname='SCYLLA-VERSION-FILE') +ar.reloc_add('build/SCYLLA-PRODUCT-FILE', arcname='SCYLLA-PRODUCT-FILE') +ar.reloc_add('python3/install.sh', arcname='install.sh') for p in ['pyhton3-libs'] + packages: pdir = pathlib.Path('/usr/share/licenses/{}/'.format(p)) if pdir.exists(): for f in pdir.glob('*'): - ar.add(f, arcname='licenses/{}/{}'.format(p, f.name)) + ar.reloc_add(f, arcname='licenses/{}/{}'.format(p, f.name)) for f in file_list: copy_file_to_python_env(ar, f) diff --git a/scripts/create-relocatable-package.py b/scripts/create-relocatable-package.py index 4c3c0b9011..ae9a47f4b1 100755 --- a/scripts/create-relocatable-package.py +++ b/scripts/create-relocatable-package.py @@ -29,6 +29,15 @@ import tarfile import pathlib +RELOC_PREFIX='scylla' +def reloc_add(self, name, arcname=None, recursive=True, *, filter=None): + if arcname: + return self.add(name, arcname="{}/{}".format(RELOC_PREFIX, arcname)) + else: + return self.add(name, arcname="{}/{}".format(RELOC_PREFIX, name)) + +tarfile.TarFile.reloc_add = reloc_add + def ldd(executable): '''Given an executable file, return a dictionary with the keys containing its shared library dependencies and the values pointing @@ -61,6 +70,10 @@ def filter_dist(info): return None return info +SCYLLA_DIR='scylla-package' +def reloc_add(ar, name, arcname=None): + ar.add(name, arcname="{}/{}".format(SCYLLA_DIR, arcname if arcname else name)) + ap = argparse.ArgumentParser(description='Create a relocatable scylla package.') ap.add_argument('dest', help='Destination file (tar format)') @@ -104,38 +117,42 @@ have_gnutls = any([lib.startswith('libgnutls.so') gzip_process = subprocess.Popen("pigz > "+output, shell=True, stdin=subprocess.PIPE) ar = tarfile.open(fileobj=gzip_process.stdin, mode='w|') -pathlib.Path('build/SCYLLA-RELOCATABLE-FILE').touch() -ar.add('build/SCYLLA-RELOCATABLE-FILE', arcname='SCYLLA-RELOCATABLE-FILE') +# relocatable package format version = 2 +with open('build/.relocatable_package_version', 'w') as f: + f.write('2\n') +ar.add('build/.relocatable_package_version', arcname='.relocatable_package_version') for exe in executables: basename = os.path.basename(exe) - ar.add(exe, arcname='libexec/' + basename) + ar.reloc_add(exe, arcname='libexec/' + basename) for lib, libfile in libs.items(): - ar.add(libfile, arcname='libreloc/' + lib) + ar.reloc_add(libfile, arcname='libreloc/' + lib) if have_gnutls: gnutls_config_nolink = os.path.realpath('/etc/crypto-policies/back-ends/gnutls.config') - ar.add(gnutls_config_nolink, arcname='libreloc/gnutls.config') -ar.add('conf') -ar.add('dist', filter=filter_dist) -ar.add('build/SCYLLA-RELEASE-FILE', arcname='SCYLLA-RELEASE-FILE') -ar.add('build/SCYLLA-VERSION-FILE', arcname='SCYLLA-VERSION-FILE') -ar.add('build/SCYLLA-PRODUCT-FILE', arcname='SCYLLA-PRODUCT-FILE') -ar.add('seastar/scripts') -ar.add('seastar/dpdk/usertools') -ar.add('install.sh') + ar.reloc_add(gnutls_config_nolink, arcname='libreloc/gnutls.config') + ar.reloc_add('conf') +ar.reloc_add('dist', filter=filter_dist) +pathlib.Path('build/SCYLLA-RELOCATABLE-FILE').touch() +ar.reloc_add('build/SCYLLA-RELOCATABLE-FILE', arcname='SCYLLA-RELOCATABLE-FILE') +ar.reloc_add('build/SCYLLA-RELEASE-FILE', arcname='SCYLLA-RELEASE-FILE') +ar.reloc_add('build/SCYLLA-VERSION-FILE', arcname='SCYLLA-VERSION-FILE') +ar.reloc_add('build/SCYLLA-PRODUCT-FILE', arcname='SCYLLA-PRODUCT-FILE') +ar.reloc_add('seastar/scripts') +ar.reloc_add('seastar/dpdk/usertools') +ar.reloc_add('install.sh') # scylla_post_install.sh lives at the top level together with install.sh in the src tree, but while install.sh is # not distributed in the .rpm and .deb packages, scylla_post_install is, so we'll add it in the package # together with the other scripts that will end up in /usr/lib/scylla -ar.add('scylla_post_install.sh', arcname="dist/common/scripts/scylla_post_install.sh") -ar.add('README.md') -ar.add('NOTICE.txt') -ar.add('ORIGIN') -ar.add('licenses') -ar.add('swagger-ui') -ar.add('api') -ar.add('tools') -ar.add('scylla-gdb.py') -ar.add('build/debian/debian', arcname='debian') +ar.reloc_add('scylla_post_install.sh', arcname="dist/common/scripts/scylla_post_install.sh") +ar.reloc_add('README.md') +ar.reloc_add('NOTICE.txt') +ar.reloc_add('ORIGIN') +ar.reloc_add('licenses') +ar.reloc_add('swagger-ui') +ar.reloc_add('api') +ar.reloc_add('tools') +ar.reloc_add('scylla-gdb.py') +ar.reloc_add('build/debian/debian', arcname='debian') # Complete the tar output, and wait for the gzip process to complete ar.close()