From c065ae0dedbe3eb79b5240fda05bb230a1a2c280 Mon Sep 17 00:00:00 2001 From: Wojciech Mitros Date: Wed, 15 Mar 2023 19:01:38 +0100 Subject: [PATCH] build: prepare rules for compiling wasm files Currently, when we deal with a Wasm program, we store it in its final WebAssembly Text form. This causes a lot of code bloat and is hard to read. Instead, we would like to store only the (C/Rust) source codes, and build Wasm when neccessary. This patch adds build commands that compile C/Rust sources to Wasm. After these changes, adding a new program that should be compiled to Rust, requires only adding the source code of it and updating the wasms and wasm_deps lists in configure.py. All Wasm programs are build by default when building all artifacts, all artifacts in a given mode, or when building tests. Additionally, a ninja wasm target is added, so that it's possible to build just the wasm files. The generated files are saved in $builddir/wasm. --- configure.py | 44 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/configure.py b/configure.py index 2947907724..85bda5ac37 100755 --- a/configure.py +++ b/configure.py @@ -563,6 +563,9 @@ raft_tests = set([ 'test/raft/failure_detector_test', ]) +wasms = set([ +]) + apps = set([ 'scylla', ]) @@ -573,7 +576,7 @@ other = set([ 'iotune', ]) -all_artifacts = apps | tests | other +all_artifacts = apps | tests | other | wasms arg_parser = argparse.ArgumentParser('Configure scylla') arg_parser.add_argument('--out', dest='buildfile', action='store', default='build.ninja', @@ -1345,6 +1348,7 @@ deps['test/raft/discovery_test'] = ['test/raft/discovery_test.cc', 'test/lib/log.cc', 'service/raft/discovery.cc'] + scylla_raft_dependencies +wasm_deps = {} warnings = [ '-Wall', @@ -1792,7 +1796,32 @@ with open(buildfile, 'w') as f: description = RUST_SOURCE $out rule cxxbridge_header command = cxxbridge --header > $out + rule c2wasm + command = clang --target=wasm32 --no-standard-libraries -Wl,--export-all -Wl,--no-entry $in -o $out + description = C2WASM $out + rule rust2wasm + # The default stack size in Rust is 1MB, which causes oversized allocation warnings, + # because it's allocated in a single chunk as a part of a Wasm Linear Memory. + # We change the stack size to 128KB using the RUSTFLAGS environment variable + # in the command below. + command = RUSTFLAGS="-C link-args=-zstack-size=131072" cargo build --target=wasm32-wasi --example=$example --locked --manifest-path=test/resource/wasm/rust/Cargo.toml --target-dir=$builddir/wasm/ $ + && wasm-opt -Oz $builddir/wasm/wasm32-wasi/debug/examples/$example.wasm -o $builddir/wasm/$example.wasm $ + && wasm-strip $builddir/wasm/$example.wasm + description = RUST2WASM $out + rule wasm2wat + command = wasm2wat $in > $out + description = WASM2WAT $out ''').format(**globals())) + for binary in sorted(wasms): + src = wasm_deps[binary] + wasm = binary[:-4] + '.wasm' + if src.endswith('.rs'): + f.write(f'build $builddir/{wasm}: rust2wasm {src} | test/resource/wasm/rust/Cargo.lock\n') + example_name = binary[binary.rindex('/')+1:-4] + f.write(f' example = {example_name}\n') + else: + f.write(f'build $builddir/{wasm}: c2wasm {src}\n') + f.write(f'build $builddir/{binary}: wasm2wat $builddir/{wasm}\n') for mode in build_modes: modeval = modes[mode] fmt_lib = 'fmt' @@ -1857,9 +1886,10 @@ with open(buildfile, 'w') as f: description = RUST_LIB $out ''').format(mode=mode, antlr3_exec=antlr3_exec, fmt_lib=fmt_lib, test_repeat=test_repeat, test_timeout=test_timeout, **modeval)) f.write( - 'build {mode}-build: phony {artifacts}\n'.format( + 'build {mode}-build: phony {artifacts} {wasms}\n'.format( mode=mode, - artifacts=str.join(' ', ['$builddir/' + mode + '/' + x for x in sorted(build_artifacts)]) + artifacts=str.join(' ', ['$builddir/' + mode + '/' + x for x in sorted(build_artifacts - wasms)]), + wasms = str.join(' ', ['$builddir/' + x for x in sorted(build_artifacts & wasms)]), ) ) include_cxx_target = f'{mode}-build' if not args.dist_only else '' @@ -1876,7 +1906,7 @@ with open(buildfile, 'w') as f: seastar_dep = f'$builddir/{mode}/seastar/libseastar.{seastar_lib_ext}' seastar_testing_dep = f'$builddir/{mode}/seastar/libseastar_testing.{seastar_lib_ext}' for binary in sorted(build_artifacts): - if binary in other: + if binary in other or binary in wasms: continue srcs = deps[binary] objs = ['$builddir/' + mode + '/' + src.replace('.cc', '.o') @@ -1964,9 +1994,10 @@ with open(buildfile, 'w') as f: ) f.write( - 'build {mode}-test: test.{mode} {test_executables} $builddir/{mode}/scylla\n'.format( + 'build {mode}-test: test.{mode} {test_executables} $builddir/{mode}/scylla {wasms}\n'.format( mode=mode, test_executables=' '.join(['$builddir/{}/{}'.format(mode, binary) for binary in sorted(tests)]), + wasms=' '.join([f'$builddir/{binary}' for binary in sorted(wasms)]), ) ) f.write( @@ -2110,6 +2141,9 @@ with open(buildfile, 'w') as f: f.write( 'build check: phony {}\n'.format(' '.join(['{mode}-check'.format(mode=mode) for mode in default_modes])) ) + f.write( + 'build wasm: phony {}\n'.format(' '.join([f'$builddir/{binary}' for binary in sorted(wasms)])) + ) f.write(textwrap.dedent(f'''\ build dist-unified-tar: phony {' '.join([f'$builddir/{mode}/dist/tar/{scylla_product}-unified-{scylla_version}-{scylla_release}.{arch}.tar.gz' for mode in default_modes])}