17c1b15c81
before this change, we build multiple relocatable package for
different builds in parallel using ninja. all these relocatable
packages are built using the same script of
`create-relocatable-package.py`. but this script always use the
same directory and file for the `.relocatable_package_version`
file.
so there are chances that these jobs building the relocatable
package can race and writing / accessing the same file at the same
time. so, in this change, instead of using a fixed file path
for this temporary file, we use a NamedTemporaryFile for this purpose.
this should helps us avoid the build failures like
```
[2023-08-10T09:38:00.019Z] FAILED: build/debug/dist/tar/scylla-unstripped-5.4.0~dev-0.20230810.116c10a2b0c6.x86_64.tar.gz
[2023-08-10T09:38:00.019Z] scripts/create-relocatable-package.py --mode debug 'build/debug/dist/tar/scylla-unstripped-5.4.0~dev-0.20230810.116c10a2b0c6.x86_64.tar.gz'
[2023-08-10T09:38:00.019Z] Traceback (most recent call last):
[2023-08-10T09:38:00.019Z] File "/jenkins/workspace/scylla-master/scylla-ci/scylla/scripts/create-relocatable-package.py", line 130, in <module>
[2023-08-10T09:38:00.019Z] os.makedirs(f'build/{SCYLLA_DIR}')
[2023-08-10T09:38:00.019Z] File "<frozen os>", line 225, in makedirs
[2023-08-10T09:38:00.019Z] FileExistsError: [Errno 17] File exists:
'build/scylla-package'
```
Fixes #15018
Signed-off-by: Kefu Chai <kefu.chai@scylladb.com>
Closes #15007
70 lines
2.4 KiB
Python
Executable File
70 lines
2.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# Copyright (C) 2022-present ScyllaDB
|
|
#
|
|
|
|
#
|
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
#
|
|
|
|
import argparse
|
|
import os
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
|
|
RELOC_PREFIX='scylla-debuginfo'
|
|
def reloc_add(self, name, arcname=None, recursive=True, *, filter=None):
|
|
if arcname:
|
|
return self.add(name, arcname="{}/{}".format(RELOC_PREFIX, arcname),
|
|
filter=filter)
|
|
else:
|
|
return self.add(name, arcname="{}/{}".format(RELOC_PREFIX, name),
|
|
filter=filter)
|
|
|
|
tarfile.TarFile.reloc_add = reloc_add
|
|
|
|
SCYLLA_DIR='scylla-debuginfo-package'
|
|
def reloc_add(ar, name, arcname=None):
|
|
ar.add(name, arcname="{}/{}".format(SCYLLA_DIR, arcname if arcname else name))
|
|
|
|
ap = argparse.ArgumentParser(description='Create a relocatable scylla-debuginfo package.')
|
|
ap.add_argument('dest',
|
|
help='Destination file (tar format)')
|
|
ap.add_argument('--mode', dest='mode', default='release',
|
|
help='Build mode (debug/release) to use')
|
|
|
|
args = ap.parse_args()
|
|
|
|
executables_scylla = [
|
|
'build/{}/scylla'.format(args.mode),
|
|
'build/{}/iotune'.format(args.mode)]
|
|
|
|
output = args.dest
|
|
|
|
# Although tarfile.open() can write directly to a compressed tar by using
|
|
# the "w|gz" mode, it does so using a slow Python implementation. It is as
|
|
# much as 3 times faster (!) to output to a pipe running the external gzip
|
|
# command. We can complete the compression even faster by using the pigz
|
|
# command - a parallel implementation of gzip utilizing all processors
|
|
# instead of just one.
|
|
gzip_process = subprocess.Popen("pigz > "+output, shell=True, stdin=subprocess.PIPE)
|
|
|
|
ar = tarfile.open(fileobj=gzip_process.stdin, mode='w|')
|
|
# relocatable package format version = 2.1
|
|
with tempfile.NamedTemporaryFile('w+t') as version_file:
|
|
version_file.write('2.1\n')
|
|
version_file.flush()
|
|
ar.add(version_file.name, arcname='.relocatable_package_version')
|
|
|
|
for exe in executables_scylla:
|
|
basename = os.path.basename(exe)
|
|
ar.reloc_add(f'{exe}.debug', arcname=f'libexec/.debug/{basename}.debug')
|
|
ar.reloc_add('build/node_exporter/node_exporter.debug', arcname='node_exporter/.debug/node_exporter.debug')
|
|
ar.reloc_add('dist/debuginfo/install.sh', arcname='install.sh')
|
|
|
|
# Complete the tar output, and wait for the gzip process to complete
|
|
ar.close()
|
|
gzip_process.communicate()
|