dist/debian: drop dependency on pystache

Same as 9d91ac345a, drop dependency on pystache
since it nolonger present in Fedora 32.

To implement it, simplified debian package build process.
It will be generate debian/ directory when building relocatable package,
we just need to run debuild using the package.

To generate debian/ directory this commit added debian_files_gen.py,
it construct whole directory including control and changelog files
from template files.
Since we need to stop pystache, these template files swiched to
string.Template class which is included python3 standard library.

see: https://github.com/scylladb/scylla/pull/6313

configure.py

@@ -1635,7 +1635,7 @@ with open(buildfile_tmp, 'w') as f:
         f.write(textwrap.dedent('''\
             build build/{mode}/iotune: copy build/{mode}/seastar/apps/iotune/iotune
             ''').format(**locals()))
-        f.write('build build/{mode}/scylla-package.tar.gz: package build/{mode}/scylla build/{mode}/iotune build/SCYLLA-RELEASE-FILE build/SCYLLA-VERSION-FILE | always\n'.format(**locals()))
+        f.write('build build/{mode}/scylla-package.tar.gz: package build/{mode}/scylla build/{mode}/iotune build/SCYLLA-RELEASE-FILE build/SCYLLA-VERSION-FILE build/debian/debian | always\n'.format(**locals()))
         f.write('  pool = submodule_pool\n')
         f.write('  mode = {mode}\n'.format(**locals()))
         f.write('rule libdeflate.{mode}\n'.format(**locals()))
@@ -1681,6 +1681,9 @@ with open(buildfile_tmp, 'w') as f:
         rule scylla_version_gen
             command = ./SCYLLA-VERSION-GEN
         build build/SCYLLA-RELEASE-FILE build/SCYLLA-VERSION-FILE: scylla_version_gen
+        rule debian_files_gen
+            command = ./dist/debian/debian_files_gen.py
+        build build/debian/debian: debian_files_gen | always
         ''').format(modes_list=' '.join(build_modes), **globals()))
 
 os.rename(buildfile_tmp, buildfile)

dist/debian/build_deb.sh

@@ -11,7 +11,6 @@ print_usage() {
 }
 
 DIST="false"
-TARGET=stable
 RELOC_PKG=
 while [ $# -gt 0 ]; do
     case "$1" in
@@ -65,9 +64,6 @@ if [ ! -f "$RELOC_PKG" ]; then
     exit 1
 fi
 
-if [ -e debian ]; then
-    rm -rf debian
-fi
 if is_debian_variant; then
     sudo apt-get -y update
 fi
@@ -91,13 +87,6 @@ fi
 if [ ! -f /usr/bin/fakeroot ]; then
     pkg_install fakeroot
 fi
-if [ ! -f /usr/bin/pystache ] && [ ! -f /usr/local/bin/pystache ]; then
-    if is_redhat_variant; then
-        sudo yum install -y /usr/bin/pystache
-    elif is_debian_variant; then
-        sudo apt-get install -y python-pystache
-    fi
-fi
 if [ ! -f /usr/bin/file ]; then
     pkg_install file
 fi
@@ -112,17 +101,6 @@ if [ "$ID" = "debian" ] && [ ! -f /usr/share/keyrings/ubuntu-archive-keyring.gpg
     sudo apt-get install -y ubuntu-archive-keyring
 fi
 
-if [ -z "$TARGET" ]; then
-    if is_debian_variant; then
-        if [ ! -f /usr/bin/lsb_release ]; then
-            pkg_install lsb-release
-        fi
-        TARGET=`lsb_release -c|awk '{print $2}'`
-    else
-        echo "Please specify target"
-        exit 1
-    fi
-fi
 RELOC_PKG_FULLPATH=$(readlink -f $RELOC_PKG)
 RELOC_PKG_BASENAME=$(basename $RELOC_PKG)
 SCYLLA_VERSION=$(cat SCYLLA-VERSION-FILE | sed 's/\.rc/~rc/')
@@ -130,28 +108,7 @@ SCYLLA_RELEASE=$(cat SCYLLA-RELEASE-FILE)
 
 ln -fv $RELOC_PKG_FULLPATH ../$PRODUCT-server_$SCYLLA_VERSION-$SCYLLA_RELEASE.orig.tar.gz
 
-cp -al dist/debian/debian debian
-if [ "$PRODUCT" != "scylla" ]; then
-    # rename all 'scylla-' prefixed artifacts in the debian folder to have the 
-    # product name as a prefix
-    find debian -maxdepth 1 -name "scylla-*" -exec bash -c 'mv $1 ${1/scylla-/$2-}' _ {} "$PRODUCT" \;
+if $DIST; then
+    export DEB_BUILD_OPTIONS="housekeeping"
 fi
-ln -fv dist/common/sysconfig/scylla-server debian/$PRODUCT-server.default
-REVISION="1"
-MUSTACHE_DIST="\"debian\": true, \"product\": \"$PRODUCT\", \"$PRODUCT\": true"
-pystache dist/debian/changelog.mustache "{ $MUSTACHE_DIST, \"version\": \"$SCYLLA_VERSION\", \"release\": \"$SCYLLA_RELEASE\", \"revision\": \"$REVISION\", \"codename\": \"$TARGET\" }" > debian/changelog
-pystache dist/debian/rules.mustache "{ $MUSTACHE_DIST, \"housekeeping\": $DIST }" > debian/rules
-pystache dist/debian/control.mustache "{ $MUSTACHE_DIST }" > debian/control
-pystache dist/debian/scylla-conf.preinst.mustache "{ \"version\": \"$SCYLLA_VERSION\" }" > debian/$PRODUCT-conf.preinst
-chmod a+rx debian/rules
-
-if [ "$PRODUCT" != "scylla" ]; then
-    SERVER_SERVICE_PREFIX="$PRODUCT-server."
-fi
-ln -fv dist/common/systemd/scylla-server.service debian/${SERVER_SERVICE_PREFIX}scylla-server.service
-ln -fv dist/common/systemd/scylla-housekeeping-daily.service debian/$PRODUCT-server.scylla-housekeeping-daily.service
-ln -fv dist/common/systemd/scylla-housekeeping-restart.service debian/$PRODUCT-server.scylla-housekeeping-restart.service
-ln -fv dist/common/systemd/scylla-fstrim.service debian/$PRODUCT-server.scylla-fstrim.service
-ln -fv dist/common/systemd/node-exporter.service debian/$PRODUCT-server.node-exporter.service
-
 debuild -rfakeroot -us -uc

dist/debian/changelog.template

@@ -1,4 +1,4 @@
-{{product}}-server ({{version}}-{{release}}-{{revision}}) {{codename}}; urgency=medium
+%{product}-server (%{version}-%{release}-%{revision}) %{codename}; urgency=medium
 
   * Initial release.
 

dist/debian/control.template

@@ -1,4 +1,4 @@
-Source: {{product}}-server
+Source: %{product}-server
 Maintainer: Takuya ASADA <syuu@scylladb.com>
 Homepage: http://scylladb.com
 Section: database
@@ -6,42 +6,42 @@ Priority: optional
 X-Python3-Version: >= 3.4
 Standards-Version: 3.9.5
 
-Package: {{product}}-conf
+Package: %{product}-conf
 Architecture: any
 Description: Scylla database main configuration file
  Scylla is a highly scalable, eventually consistent, distributed,
  partitioned row DB.
-Replaces: {{product}}-server (<< 1.1)
-Conflicts: {{product}}-server (<< 1.1)
+Replaces: %{product}-server (<< 1.1)
+Conflicts: %{product}-server (<< 1.1)
 
-Package: {{product}}-server
+Package: %{product}-server
 Architecture: amd64
-Depends: ${shlibs:Depends}, ${misc:Depends}, {{product}}-conf, {{product}}-python3
+Depends: ${shlibs:Depends}, ${misc:Depends}, %{product}-conf, %{product}-python3
 Description: Scylla database server binaries 
  Scylla is a highly scalable, eventually consistent, distributed,
  partitioned row DB.
 
-Package: {{product}}-server-dbg
+Package: %{product}-server-dbg
 Section: debug
 Priority: extra
 Architecture: amd64
-Depends: {{product}}-server (= ${binary:Version}), ${shlibs:Depends}, ${misc:Depends}
-Description: debugging symbols for {{product}}-server
+Depends: %{product}-server (= ${binary:Version}), ${shlibs:Depends}, ${misc:Depends}
+Description: debugging symbols for %{product}-server
  Scylla is a highly scalable, eventually consistent, distributed,
  partitioned row DB.
- This package contains the debugging symbols for {{product}}-server.
+ This package contains the debugging symbols for %{product}-server.
 
-Package: {{product}}-kernel-conf
+Package: %{product}-kernel-conf
 Architecture: any
 Depends: procps
 Description: Scylla kernel tuning configuration
  Scylla is a highly scalable, eventually consistent, distributed,
  partitioned row DB.
 
-Package: {{product}}
+Package: %{product}
 Section: metapackages
 Architecture: any
-Depends: {{product}}-server, {{product}}-jmx, {{product}}-tools, {{product}}-tools-core, {{product}}-kernel-conf
+Depends: %{product}-server, %{product}-jmx, %{product}-tools, %{product}-tools-core, %{product}-kernel-conf
 Description: Scylla database metapackage
  Scylla is a highly scalable, eventually consistent, distributed,
  partitioned row DB.

dist/debian/debian/rules

@@ -1,13 +1,16 @@
 #!/usr/bin/make -f
 
+include /usr/share/dpkg/pkg-info.mk
+
 export PYBUILD_DISABLE=1
 jobs := $(shell echo $$DEB_BUILD_OPTIONS | sed -r "s/.*parallel=([0-9]+).*/-j\1/")
-{{#housekeeping}}
-install_arg := --housekeeping
-{{/housekeeping}}
-{{^housekeeping}}
-install_arg :=
-{{/housekeeping}}
+ifneq ($(findstring housekeeping, $(DEB_BUILD_OPTIONS)),)
+    install_arg := --housekeeping
+else
+    install_arg :=
+endif
+
+product := $(subst -server,,$(DEB_SOURCE))
 
 override_dh_auto_configure:
 
@@ -22,12 +25,11 @@ override_dh_auto_install:
 	cp dist/debian/sysconfig/scylla-housekeeping $(CURDIR)/debian/tmp/etc/default/
 
 override_dh_installinit:
-{{#scylla}}
+ifeq ($(product),scylla)
 	dh_installinit --no-start
-{{/scylla}}
-{{^scylla}}
+else
 	dh_installinit --no-start --name scylla-server
-{{/scylla}}
+endif
 	dh_installinit --no-start --name scylla-housekeeping-daily
 	dh_installinit --no-start --name scylla-housekeeping-restart
 	dh_installinit --no-start --name scylla-fstrim
@@ -36,7 +38,7 @@ override_dh_installinit:
 override_dh_strip:
 	# The binaries (ethtool...patchelf) don't pass dh_strip after going through patchelf. Since they are
 	# already stripped, nothing is lost if we exclude them, so that's what we do.
-	dh_strip -Xlibprotobuf.so.15 -Xld.so -Xethtool -Xgawk -Xgzip -Xhwloc-calc -Xhwloc-distrib -Xifconfig -Xlscpu -Xnetstat -Xpatchelf --dbg-package={{product}}-server-dbg
+	dh_strip -Xlibprotobuf.so.15 -Xld.so -Xethtool -Xgawk -Xgzip -Xhwloc-calc -Xhwloc-distrib -Xifconfig -Xlscpu -Xnetstat -Xpatchelf --dbg-package=$(product)-server-dbg
 
 override_dh_makeshlibs:
 
@@ -44,7 +46,7 @@ override_dh_shlibdeps:
 
 override_dh_fixperms:
 	dh_fixperms
-	chmod 755 $(CURDIR)/debian/{{product}}-server/opt/scylladb/libreloc/ld.so
+	chmod 755 $(CURDIR)/debian/$(product)-server/opt/scylladb/libreloc/ld.so
 
 override_dh_strip_nondeterminism:
 

dist/debian/debian/scylla-conf.preinst

@@ -7,8 +7,8 @@ if [ -n "$ver" ]; then
         # for <scylla-1.2
         if [ ! -f /opt/scylladb/scripts/scylla_config_get.py ]; then
             echo
-            echo "Error: Upgrading from scylla-$ver to scylla-{{version}} is not supported."
-            echo "Please upgrade to scylla-1.7.3 or later, before upgrade to {{version}}."
+            echo "Error: Upgrading from scylla-$ver to this version of scylla is not supported."
+            echo "Please upgrade to scylla-1.7.3 or later, before upgrade to this version."
             echo
             exit 1
         fi
@@ -16,8 +16,8 @@ if [ -n "$ver" ]; then
         commitlog_files=$(ls $commitlog_directory | wc -l)
         if [ $commitlog_files -ne 0 ]; then
             echo
-            echo "Error: Upgrading from scylla-$ver to scylla-{{version}} is not supported when commitlog is not clean."
-            echo "Please upgrade to scylla-1.7.3 or later, before upgrade to {{version}}."
+            echo "Error: Upgrading from scylla-$ver to this version of scylla is not supported when commitlog is not clean."
+            echo "Please upgrade to scylla-1.7.3 or later, before upgrade to this version."
             echo "Also make sure $commitlog_directory is empty."
             echo
             exit 1

dist/debian/debian_files_gen.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020 ScyllaDB
+#
+
+#
+# This file is part of Scylla.
+#
+# Scylla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Scylla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import string
+import os
+import glob
+import shutil
+
+class DebianFilesTemplate(string.Template):
+    delimiter = '%'
+
+scriptdir = os.path.dirname(__file__)
+
+with open(os.path.join(scriptdir, 'changelog.template')) as f:
+    changelog_template = f.read()
+
+with open(os.path.join(scriptdir, 'control.template')) as f:
+    control_template = f.read()
+
+with open('build/SCYLLA-PRODUCT-FILE') as f:
+    product = f.read().strip()
+
+with open('build/SCYLLA-VERSION-FILE') as f:
+    version = f.read().strip()
+
+with open('build/SCYLLA-RELEASE-FILE') as f:
+    release = f.read().strip()
+
+shutil.rmtree('build/debian/debian', ignore_errors=True)
+shutil.copytree('dist/debian/debian', 'build/debian/debian')
+
+if product != 'scylla':
+    for p in glob.glob('build/debian/debian/scylla-*'):
+        shutil.move(p, p.replace('scylla-', '{}-'.format(product)))
+
+shutil.copy('dist/common/sysconfig/scylla-server', 'build/debian/debian/{}-server.default'.format(product))
+if product != 'scylla':
+    shutil.copy('dist/common/systemd/scylla-server.service', 'build/debian/debian/{}-server.scylla-server.service'.format(product))
+else:
+    shutil.copy('dist/common/systemd/scylla-server.service', 'build/debian/debian/scylla-server.service')
+shutil.copy('dist/common/systemd/scylla-housekeeping-daily.service', 'build/debian/debian/{}-server.scylla-housekeeping-daily.service'.format(product))
+shutil.copy('dist/common/systemd/scylla-housekeeping-restart.service', 'build/debian/debian/{}-server.scylla-housekeeping-restart.service'.format(product))
+shutil.copy('dist/common/systemd/scylla-fstrim.service', 'build/debian/debian/{}-server.scylla-fstrim.service'.format(product))
+shutil.copy('dist/common/systemd/node-exporter.service', 'build/debian/debian/{}-server.scylla-node-exporter.service'.format(product))
+
+s = DebianFilesTemplate(changelog_template)
+changelog_applied = s.substitute(product=product, version=version, release=release, revision='1', codename='stable')
+
+s = DebianFilesTemplate(control_template)
+control_applied = s.substitute(product=product)
+
+with open('build/debian/debian/changelog', 'w') as f:
+    f.write(changelog_applied)
+
+with open('build/debian/debian/control', 'w') as f:
+    f.write(control_applied)
+

scripts/create-relocatable-package.py

@@ -135,6 +135,7 @@ ar.add('swagger-ui')
 ar.add('api')
 ar.add('tools')
 ar.add('scylla-gdb.py')
+ar.add('build/debian/debian', arcname='debian')
 
 # Complete the tar output, and wait for the gzip process to complete
 ar.close()