diff --git a/.gitignore b/.gitignore
index b518e4c7f1..ba54446190 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,4 @@ compile_commands.json
 .ccls-cache/
 .mypy_cache
 .envrc
+clang_build
diff --git a/configure.py b/configure.py
index c5485c1ef3..84e3175cca 100755
--- a/configure.py
+++ b/configure.py
@@ -20,9 +20,6 @@ import textwrap
 from shutil import which
 from typing import NamedTuple
 
-outdir = 'build'
-
-tempfile.tempdir = f"{outdir}/tmp"
 
 configure_args = str.join(' ', [shlex.quote(x) for x in sys.argv[1:] if not x.startswith('--out=')])
 
@@ -287,7 +284,7 @@ def generate_compdb(compdb, ninja, buildfile, modes):
             mode_out = outdir + '/' + mode
             submodule_compdbs = [mode_out + '/' + submodule + '/' + compdb for submodule in ['seastar']]
             with open(mode_out + '/' + compdb, 'w+b') as combined_mode_specific_compdb:
-                subprocess.run(['./scripts/merge-compdb.py', 'build/' + mode,
+                subprocess.run(['./scripts/merge-compdb.py', outdir + '/' + mode,
                                 ninja_compdb.name] + submodule_compdbs, stdout=combined_mode_specific_compdb)
 
     # sort modes by supposed indexing speed
@@ -780,10 +777,16 @@ arg_parser.add_argument('--date-stamp', dest='date_stamp', type=str,
                         help='Set datestamp for SCYLLA-VERSION-GEN')
 arg_parser.add_argument('--use-cmake', action='store_true', help='Use CMake as the build system')
 arg_parser.add_argument('--coverage', action = 'store_true', help = 'Compile scylla with coverage instrumentation')
+arg_parser.add_argument('--build-dir', action='store', default='build',
+                        help='Build directory path')
+
 args = arg_parser.parse_args()
 
 PROFILES_LIST_FILE_NAME = "coverage_sources.list"
 
+outdir = os.path.abspath(args.build_dir)
+tempfile.tempdir = f"{outdir}/tmp"
+
 if args.list_artifacts:
     for artifact in sorted(all_artifacts):
         print(artifact)
@@ -1613,7 +1616,7 @@ def generate_version(date_stamp):
     date_stamp_opt = ''
     if date_stamp:
         date_stamp_opt = f'--date-stamp {date_stamp}'
-    status = subprocess.call(f"./SCYLLA-VERSION-GEN {date_stamp_opt}", shell=True)
+    status = subprocess.call(f"./SCYLLA-VERSION-GEN --output-dir {outdir} {date_stamp_opt}", shell=True)
     if status != 0:
         print('Version file generation failed')
         sys.exit(1)
@@ -1885,17 +1888,17 @@ def write_build_file(f,
         rule strip
             command = scripts/strip.sh $in
         rule package
-            command = scripts/create-relocatable-package.py --build-dir build/$mode $out
+            command = scripts/create-relocatable-package.py --build-dir $builddir/$mode --node-exporter-dir $builddir/node_exporter --debian-dir $builddir/debian/debian $out
         rule stripped_package
-            command = scripts/create-relocatable-package.py --stripped --build-dir build/$mode $out
+            command = scripts/create-relocatable-package.py --stripped --build-dir $builddir/$mode --node-exporter-dir $builddir/node_exporter --debian-dir $builddir/debian/debian $out
         rule debuginfo_package
-            command = dist/debuginfo/scripts/create-relocatable-package.py --build-dir build/$mode $out
+            command = dist/debuginfo/scripts/create-relocatable-package.py --build-dir $builddir/$mode --node-exporter-dir $builddir/node_exporter $out
         rule rpmbuild
             command = reloc/build_rpm.sh --reloc-pkg $in --builddir $out
         rule debbuild
             command = reloc/build_deb.sh --reloc-pkg $in --builddir $out
         rule unified
-            command = unified/build_unified.sh --build-dir build/$mode --unified-pkg $out
+            command = unified/build_unified.sh --build-dir $builddir/$mode --unified-pkg $out
         rule rust_header
             command = cxxbridge --include rust/cxx.h --header $in > $out
             description = RUST_HEADER $out
@@ -2127,6 +2130,17 @@ def write_build_file(f,
                 mode=mode,
             )
         )
+        compiler_training_artifacts=[]
+        if mode == 'dev':
+            compiler_training_artifacts.append(f'$builddir/{mode}/scylla')
+        elif mode == 'release' or mode == 'debug':
+            compiler_training_artifacts.append(f'$builddir/{mode}/service/storage_proxy.o')
+        f.write(
+            'build {mode}-compiler-training: phony {artifacts}\n'.format(
+                mode=mode,
+                artifacts=str.join(' ', compiler_training_artifacts)
+            )
+        )
 
         gen_dir = '$builddir/{}/gen'.format(mode)
         gen_headers = []
@@ -2266,6 +2280,9 @@ def write_build_file(f,
     f.write(
             'build wasm: phony {}\n'.format(' '.join([f'$builddir/{binary}' for binary in sorted(wasms)]))
     )
+    f.write(
+            'build compiler-training: phony {}\n'.format(' '.join(['{mode}-compiler-training'.format(mode=mode) for mode in default_modes]))
+    )
 
     f.write(textwrap.dedent(f'''\
         build dist-unified-tar: phony {' '.join([f'$builddir/{mode}/dist/tar/{scylla_product}-unified-{scylla_version}-{scylla_release}.{arch}.tar.gz' for mode in default_modes])}
@@ -2278,54 +2295,54 @@ def write_build_file(f,
         build dist-server: phony dist-server-tar dist-server-debuginfo dist-server-rpm dist-server-deb
 
         rule build-submodule-reloc
-          command = cd $reloc_dir && ./reloc/build_reloc.sh --version $$(<../../build/SCYLLA-PRODUCT-FILE)-$$(sed 's/-/~/' <../../build/SCYLLA-VERSION-FILE)-$$(<../../build/SCYLLA-RELEASE-FILE) --nodeps $args
+          command = cd $reloc_dir && ./reloc/build_reloc.sh --version $$(<$builddir/SCYLLA-PRODUCT-FILE)-$$(sed 's/-/~/' <$builddir/SCYLLA-VERSION-FILE)-$$(<$builddir/SCYLLA-RELEASE-FILE) --nodeps $args
         rule build-submodule-rpm
           command = cd $dir && ./reloc/build_rpm.sh --reloc-pkg $artifact
         rule build-submodule-deb
           command = cd $dir && ./reloc/build_deb.sh --reloc-pkg $artifact
 
-        build tools/jmx/build/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz: build-submodule-reloc | build/SCYLLA-PRODUCT-FILE build/SCYLLA-VERSION-FILE build/SCYLLA-RELEASE-FILE
+        build tools/jmx/build/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz: build-submodule-reloc | $builddir/SCYLLA-PRODUCT-FILE $builddir/SCYLLA-VERSION-FILE $builddir/SCYLLA-RELEASE-FILE
           reloc_dir = tools/jmx
         build dist-jmx-rpm: build-submodule-rpm tools/jmx/build/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz
           dir = tools/jmx
-          artifact = $builddir/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz
+          artifact = build/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz
         build dist-jmx-deb: build-submodule-deb tools/jmx/build/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz
           dir = tools/jmx
-          artifact = $builddir/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz
+          artifact = build/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz
         build dist-jmx-tar: phony {' '.join(['$builddir/{mode}/dist/tar/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz'.format(mode=mode, scylla_product=scylla_product, scylla_version=scylla_version, scylla_release=scylla_release) for mode in default_modes])}
         build dist-jmx: phony dist-jmx-tar dist-jmx-rpm dist-jmx-deb
 
-        build tools/java/build/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz: build-submodule-reloc | build/SCYLLA-PRODUCT-FILE build/SCYLLA-VERSION-FILE build/SCYLLA-RELEASE-FILE
+        build tools/java/build/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz: build-submodule-reloc | $builddir/SCYLLA-PRODUCT-FILE $builddir/SCYLLA-VERSION-FILE $builddir/SCYLLA-RELEASE-FILE
           reloc_dir = tools/java
         build dist-tools-rpm: build-submodule-rpm tools/java/build/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz
           dir = tools/java
-          artifact = $builddir/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz
+          artifact = build/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz
         build dist-tools-deb: build-submodule-deb tools/java/build/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz
           dir = tools/java
-          artifact = $builddir/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz
+          artifact = build/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz
         build dist-tools-tar: phony {' '.join(['$builddir/{mode}/dist/tar/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz'.format(mode=mode, scylla_product=scylla_product, scylla_version=scylla_version, scylla_release=scylla_release) for mode in default_modes])}
         build dist-tools: phony dist-tools-tar dist-tools-rpm dist-tools-deb
 
-        build tools/cqlsh/build/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz: build-submodule-reloc | build/SCYLLA-PRODUCT-FILE build/SCYLLA-VERSION-FILE build/SCYLLA-RELEASE-FILE
+        build tools/cqlsh/build/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz: build-submodule-reloc | $builddir/SCYLLA-PRODUCT-FILE $builddir/SCYLLA-VERSION-FILE $builddir/SCYLLA-RELEASE-FILE
           reloc_dir = tools/cqlsh
         build dist-cqlsh-rpm: build-submodule-rpm tools/cqlsh/build/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz
           dir = tools/cqlsh
-          artifact = $builddir/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz
+          artifact = build/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz
         build dist-cqlsh-deb: build-submodule-deb tools/cqlsh/build/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz
           dir = tools/cqlsh
-          artifact = $builddir/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz
+          artifact = build/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz
         build dist-cqlsh-tar: phony {' '.join(['$builddir/{mode}/dist/tar/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz'.format(mode=mode, scylla_product=scylla_product, scylla_version=scylla_version, scylla_release=scylla_release) for mode in default_modes])}
         build dist-cqlsh: phony dist-cqlsh-tar dist-cqlsh-rpm dist-cqlsh-deb
 
-        build tools/python3/build/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz: build-submodule-reloc | build/SCYLLA-PRODUCT-FILE build/SCYLLA-VERSION-FILE build/SCYLLA-RELEASE-FILE
+        build tools/python3/build/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz: build-submodule-reloc | $builddir/SCYLLA-PRODUCT-FILE $builddir/SCYLLA-VERSION-FILE $builddir/SCYLLA-RELEASE-FILE
           reloc_dir = tools/python3
           args = --packages "{python3_dependencies}" --pip-packages "{pip_dependencies}" --pip-symlinks "{pip_symlinks}"
         build dist-python3-rpm: build-submodule-rpm tools/python3/build/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz
           dir = tools/python3
-          artifact = $builddir/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz
+          artifact = build/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz
         build dist-python3-deb: build-submodule-deb tools/python3/build/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz
           dir = tools/python3
-          artifact = $builddir/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz
+          artifact = build/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz
         build dist-python3-tar: phony {' '.join(['$builddir/{mode}/dist/tar/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz'.format(mode=mode, scylla_product=scylla_product, arch=arch, scylla_version=scylla_version, scylla_release=scylla_release) for mode in default_modes])}
         build dist-python3: phony dist-python3-tar dist-python3-rpm dist-python3-deb
         build dist-deb: phony dist-server-deb dist-python3-deb dist-jmx-deb dist-tools-deb dist-cqlsh-deb
@@ -2360,10 +2377,10 @@ def write_build_file(f,
 
     f.write(textwrap.dedent('''\
         rule configure
-          command = {python} configure.py --out=build.ninja.new $configure_args && mv build.ninja.new build.ninja
+          command = {python} configure.py --out={buildfile}.new $configure_args && mv {buildfile}.new {buildfile}
           generator = 1
           description = CONFIGURE $configure_args
-        build build.ninja {build_ninja_list}: configure | configure.py SCYLLA-VERSION-GEN $builddir/SCYLLA-PRODUCT-FILE $builddir/SCYLLA-VERSION-FILE $builddir/SCYLLA-RELEASE-FILE {args.seastar_path}/CMakeLists.txt
+        build {buildfile} {build_ninja_list}: configure | configure.py SCYLLA-VERSION-GEN $builddir/SCYLLA-PRODUCT-FILE $builddir/SCYLLA-VERSION-FILE $builddir/SCYLLA-RELEASE-FILE {args.seastar_path}/CMakeLists.txt
         rule cscope
             command = find -name '*.[chS]' -o -name "*.cc" -o -name "*.hh" | cscope -bq -i-
             description = CSCOPE
@@ -2377,7 +2394,7 @@ def write_build_file(f,
             description = List configured modes
         build mode_list: mode_list
         default {modes_list}
-        ''').format(modes_list=' '.join(default_modes), build_ninja_list=' '.join([f'build/{mode}/{dir}/build.ninja' for mode in build_modes for dir in ['seastar']]), **globals()))
+        ''').format(modes_list=' '.join(default_modes), build_ninja_list=' '.join([f'{outdir}/{mode}/{dir}/build.ninja' for mode in build_modes for dir in ['seastar']]), **globals()))
     unit_test_list = set(test for test in build_artifacts if test in set(tests))
     f.write(textwrap.dedent('''\
         rule unit_test_list
@@ -2388,14 +2405,14 @@ def write_build_file(f,
     f.write(textwrap.dedent('''\
         build always: phony
         rule scylla_version_gen
-            command = ./SCYLLA-VERSION-GEN
+            command = ./SCYLLA-VERSION-GEN --output-dir $builddir
             restat = 1
         build $builddir/SCYLLA-RELEASE-FILE $builddir/SCYLLA-VERSION-FILE: scylla_version_gen | always
         rule debian_files_gen
-            command = ./dist/debian/debian_files_gen.py
+            command = ./dist/debian/debian_files_gen.py --build-dir $builddir
         build $builddir/debian/debian: debian_files_gen | always
         rule extract_node_exporter
-            command = tar -C build -xvpf {node_exporter_filename} --no-same-owner && rm -rfv build/node_exporter && mv -v build/{node_exporter_dirname} build/node_exporter
+            command = tar -C $builddir -xvpf {node_exporter_filename} --no-same-owner && rm -rfv $builddir/node_exporter && mv -v $builddir/{node_exporter_dirname} $builddir/node_exporter
         build $builddir/node_exporter/node_exporter: extract_node_exporter | always
         build $builddir/node_exporter/node_exporter.stripped: strip $builddir/node_exporter/node_exporter
         build $builddir/node_exporter/node_exporter.debug: phony $builddir/node_exporter/node_exporter.stripped
diff --git a/install-dependencies.sh b/install-dependencies.sh
index eec2652aca..cbd932e430 100755
--- a/install-dependencies.sh
+++ b/install-dependencies.sh
@@ -113,6 +113,7 @@ fedora_packages=(
     wabt
     binaryen
     lcov
+    llvm-bolt
 )
 
 # lld is not available on s390x, see
diff --git a/tools/toolchain/Dockerfile b/tools/toolchain/Dockerfile
index b26df9358a..d3efed76fd 100644
--- a/tools/toolchain/Dockerfile
+++ b/tools/toolchain/Dockerfile
@@ -1,17 +1,18 @@
 FROM docker.io/fedora:38
-ADD ./install-dependencies.sh ./
-ADD ./seastar/install-dependencies.sh ./seastar/
-ADD ./tools/jmx/install-dependencies.sh ./tools/jmx/
-ADD ./tools/java/install-dependencies.sh ./tools/java/
-ADD ./tools/toolchain/system-auth ./
+
+ARG CLANG_BUILD="SKIP"
+ARG CLANG_ARCHIVE
+
+WORKDIR /mnt
 RUN dnf -y update \
     && dnf -y install 'dnf-command(copr)' \
     && dnf -y install ccache \
     && dnf -y install devscripts debhelper fakeroot file rpm-build \
     && ./install-dependencies.sh && dnf clean all \
     && echo 'ALL ALL=(ALL:ALL) NOPASSWD: ALL' >> /etc/sudoers \
-    && cp system-auth /etc/pam.d \
+    && cp ./tools/toolchain/system-auth /etc/pam.d \
     && echo 'Defaults !requiretty' >> /etc/sudoers
 RUN mkdir -p /root/.m2/repository
 ENV JAVA8_HOME=/usr/lib/jvm/java-1.8.0-openjdk
+RUN ./tools/toolchain/optimized_clang.sh
 CMD /bin/bash
diff --git a/tools/toolchain/README.md b/tools/toolchain/README.md
index 6b4432094e..dd2f125f14 100644
--- a/tools/toolchain/README.md
+++ b/tools/toolchain/README.md
@@ -85,45 +85,33 @@ there is a need to update a toolchain, the branch designation is added to
 the tag to avoid ambiguity.
 
 Publishing an image is complicated since multiple architectures are supported.
-There are two procedures, one using emulation (can run on any x86 machine) and
-another using native systems, which requires access to aarch64 and s390x machines.
 
 You will need credentials to push images. You either need to setup permanent
 credentials with `podman login`, or you can just append
 `--creds=yourname@scylladb.com` to the `podman manifest push` command and then
 type in your password.
 
-## Emulated publishing procedure (slow)
+## Publishing procedure
 
 1. Pick a new name for the image (in `tools/toolchain/image`) and
    commit it. The commit updating install-dependencies.sh should
    include the toolchain change, for atomicity. Do not push the commit
    to `next` yet.
-2. Run `tools/toolchain/prepare` and wait. It requires `buildah` and
-   `qemu-user-static` to be installed (and will complain if they are not).
-3. Publish the image using the instructions printed by the previous step.
-4. Push the `next` branch that refers to the new toolchain.
-
-## Native publishing procedure (complicated)
-
-1. Pick a new name for the image (in `tools/toolchain/image`) and
-   commit it. The commit updating install-dependencies.sh should
-   include the toolchain change, for atomicity. Do not push the commit
-   to `next` yet.
-2. Push the commit to a personal repository/branch.
-3. Perform the following on an x86 and an ARM machine:
+2. Run `tools/toolchain/prepare --clang-build-mode INSTALL` and wait. It requires `buildah` and to be installed (and will complain if they are not).
+3. Push the commit to a personal repository/branch.
+4. Perform the following on an x86 and an ARM machine:
     1. check out the branch containing the new toolchain name
     2. Run `git submodule update --init --recursive` to make sure
        all the submodules are synchronized
     3. Run `podman build --no-cache --pull --tag mytag-arch -f tools/toolchain/Dockerfile .`, where mytag-arch is a new, unique tag that is different for x86 and ARM.
     4. Push the resulting images to a personal docker repository.
-4. Now, create a multiarch image with the following:
+5. Now, create a multiarch image with the following:
     1. Pull the two images with `podman pull`. Let's call the two tags
        `mytag-x86` and `mytag-arm`.
     2. Create the new toolchain manifest with `podman manifest create $(<tools/toolchain/image)`
     3. Add each image with `podman manifest add --all $(<tools/toolchain/image) mytag-x86` and `podman manifest add --all $(<tools/toolchain/image) mytag-arm`
     4. Push the image with `podman manifest push --all $(<tools/toolchain/image) docker://$(<tools/toolchain/image)`
-5. Now push the commit that updated the toolchain with `git push`.
+6. Now push the commit that updated the toolchain with `git push`.
 
 ## Troubleshooting
 
diff --git a/tools/toolchain/optimized_clang.sh b/tools/toolchain/optimized_clang.sh
new file mode 100755
index 0000000000..5e8279d230
--- /dev/null
+++ b/tools/toolchain/optimized_clang.sh
@@ -0,0 +1,117 @@
+#!/bin/bash -uex
+
+case "${CLANG_BUILD}" in
+    "SKIP")
+        echo "CLANG_BUILD: ${CLANG_BUILD}"
+        exit 0
+        ;;
+    "INSTALL" | "INSTALL_FROM")
+        echo "CLANG_BUILD: ${CLANG_BUILD}"
+        if [[ -z "${CLANG_ARCHIVE}" ]]; then
+            echo "CLANG_ARCHIVE not specified"
+        fi
+        ;;
+    "")
+        echo "CLANG_BUILD not specified"
+        exit 1
+        ;;
+    *)
+        echo "Invalid mode specified on CLANG_BUILD: ${CLANG_BUILD}"
+        exit 1
+        ;;
+esac
+
+ARCH="$(arch)"
+if [[ "${ARCH}" = "x86_64" ]]; then
+    LLVM_TARGET_ARCH=X86
+elif [[ "${ARCH}" = "aarch64" ]]; then
+    LLVM_TARGET_ARCH=AArch64
+else
+    echo "Unsupported architecture: ${ARCH}"
+    exit 1
+fi
+
+SCYLLA_DIR=/mnt
+CLANG_ROOT_DIR="${SCYLLA_DIR}"/clang_build
+CLANG_CHECKOUT_NAME=llvm-project
+CLANG_BUILD_DIR="${CLANG_ROOT_DIR}"/"${CLANG_CHECKOUT_NAME}"
+
+SCYLLA_BUILD_DIR="${SCYLLA_DIR}"/build_profile
+SCYLLA_NINJA_FILE="${SCYLLA_DIR}"/build_profile.ninja
+
+# Which LLVM release to build in order to compile Scylla
+LLVM_CLANG_TAG=16.0.6
+CLANG_SUFFIX=16
+
+CLANG_ARCHIVE=$(cd "${SCYLLA_DIR}" && realpath -m "${CLANG_ARCHIVE}")
+
+CLANG_OPTS=(-DCMAKE_C_COMPILER="/usr/bin/clang" -DCMAKE_CXX_COMPILER="/usr/bin/clang++" -DLLVM_USE_LINKER="/usr/bin/ld.lld" -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD="${LLVM_TARGET_ARCH};WebAssembly" -DLLVM_TARGET_ARCH="${LLVM_TARGET_ARCH}" -G Ninja -DLLVM_INCLUDE_BENCHMARKS=OFF -DLLVM_INCLUDE_EXAMPLES=OFF -DLLVM_INCLUDE_TESTS=OFF -DLLVM_ENABLE_BINDINGS=OFF -DLLVM_ENABLE_PROJECTS="clang;lld" -DLLVM_ENABLE_RUNTIMES="compiler-rt" -DLLVM_ENABLE_LTO=Thin -DCLANG_DEFAULT_PIE_ON_LINUX=OFF -DLLVM_BUILD_TOOLS=OFF -DLLVM_VP_COUNTERS_PER_SITE=6)
+SCYLLA_OPTS=(--date-stamp "$(date "+%Y%m%d")" --debuginfo 1 --tests-debuginfo 1 --c-compiler="${CLANG_BUILD_DIR}/build/bin/clang" --compiler="${CLANG_BUILD_DIR}/build/bin/clang++" --build-dir="${SCYLLA_BUILD_DIR}" --out="${SCYLLA_NINJA_FILE}")
+
+if [[ "${CLANG_BUILD}" = "INSTALL" ]]; then
+    # Build a PGO-optimized compiler using the boostrapped compiler.
+    rm -rf "${CLANG_BUILD_DIR}"
+    git clone https://github.com/llvm/llvm-project --branch llvmorg-"${LLVM_CLANG_TAG}" --depth=1 "${CLANG_BUILD_DIR}"
+    cd "${CLANG_BUILD_DIR}"
+    cmake -B build -S llvm "${CLANG_OPTS[@]}" -DLLVM_BUILD_INSTRUMENTED=IR
+    ninja -C build
+
+    rm -rf "${SCYLLA_BUILD_DIR}" "${SCYLLA_NINJA_FILE}"
+    cd "${SCYLLA_DIR}"
+    ./configure.py "${SCYLLA_OPTS[@]}"
+    LLVM_PROFILE_FILE="${CLANG_BUILD_DIR}"/build/profiles/default_%p-%m.profraw ninja -f "${SCYLLA_NINJA_FILE}" compiler-training
+
+    cd "${CLANG_BUILD_DIR}"
+    llvm-profdata merge "${CLANG_BUILD_DIR}"/build/profiles/default_*.profraw -output=ir.prof
+    rm -rf build
+    cmake -B build -S llvm "${CLANG_OPTS[@]}" -DLLVM_BUILD_INSTRUMENTED=CSIR -DLLVM_PROFDATA_FILE="$(realpath ir.prof)"
+    ninja -C build
+
+    # 2nd compilation: gathering a clang profile for CSPGO
+    rm -rf "${SCYLLA_BUILD_DIR}" "${SCYLLA_NINJA_FILE}"
+    cd "${SCYLLA_DIR}"
+    ./configure.py "${SCYLLA_OPTS[@]}"
+    LLVM_PROFILE_FILE="${CLANG_BUILD_DIR}"/build/profiles/csir-%p-%m.profraw ninja -f "${SCYLLA_NINJA_FILE}" compiler-training
+
+    cd "${CLANG_BUILD_DIR}"
+    llvm-profdata merge build/csprofiles/default_*.profraw -output=csir.prof
+    llvm-profdata merge ir.prof csir.prof -output=combined.prof
+    rm -rf build
+    # -DLLVM_LIBDIR_SUFFIX=64 for Fedora compatibility
+    cmake -B build -S llvm "${CLANG_OPTS[@]}" -DLLVM_PROFDATA_FILE="$(realpath combined.prof)" -DCMAKE_EXE_LINKER_FLAGS="-Wl,--emit-relocs" -DCMAKE_INSTALL_PREFIX=/usr/local -DLLVM_LIBDIR_SUFFIX=64
+    ninja -C build
+
+    #TODO: skipping BOLT for aarch64 for now, since it causes segfault
+    if [[ "${ARCH}" != "aarch64" ]]; then
+        # BOLT phase
+        mv build/bin/clang-"${CLANG_SUFFIX}" build/bin/clang-"${CLANG_SUFFIX}".prebolt
+        mkdir -p build/profiles
+        llvm-bolt build/bin/clang-"${CLANG_SUFFIX}".prebolt -o build/bin/clang-"${CLANG_SUFFIX}" --instrument --instrumentation-file="${CLANG_BUILD_DIR}"/build/profiles/prof --instrumentation-file-append-pid --conservative-instrumentation
+
+        # 3rd ScyllaDB compilation: gathering a clang profile for BOLT
+        rm -rf "${SCYLLA_BUILD_DIR}" "${SCYLLA_NINJA_FILE}"
+        cd "${SCYLLA_DIR}"
+        ./configure.py "${SCYLLA_OPTS[@]}"
+        ninja -f "${SCYLLA_NINJA_FILE}" compiler-training
+
+        cd "${CLANG_BUILD_DIR}"
+        merge-fdata build/profiles/*.fdata > prof.fdata
+        llvm-bolt build/bin/clang-"${CLANG_SUFFIX}".prebolt -o build/bin/clang-"${CLANG_SUFFIX}" --data=prof.fdata --reorder-functions=hfsort --reorder-blocks=ext-tsp --split-functions --split-all-cold --split-eh --dyno-stats
+    fi
+
+    cd "${CLANG_ROOT_DIR}"
+    rm -rf "${CLANG_BUILD_DIR}"/{build/profiles,*.prof,prof.fdata}
+    tar -cpzf "${CLANG_ARCHIVE}" "${CLANG_CHECKOUT_NAME}"
+elif [[ "${CLANG_BUILD}" = "INSTALL_FROM" ]]; then
+    mkdir -p "${CLANG_ROOT_DIR}"
+    tar -C "${CLANG_ROOT_DIR}" -xpzf "${CLANG_ARCHIVE}"
+fi
+
+cd "${CLANG_BUILD_DIR}"
+mv /usr/bin/clang-"${CLANG_SUFFIX}" /usr/bin/clang-"${CLANG_SUFFIX}".orig
+mv /usr/bin/lld /usr/bin/lld.orig
+mv /usr/lib64/libLTO.so."${CLANG_SUFFIX}" /usr/lib64/libLTO.so."${CLANG_SUFFIX}".orig
+install -Z -m755 "${CLANG_BUILD_DIR}"/build/bin/clang-"${CLANG_SUFFIX}" /usr/bin/clang-"${CLANG_SUFFIX}"
+install -Z -m755 "${CLANG_BUILD_DIR}"/build/bin/lld /usr/bin/lld
+install -Z -m755 "${CLANG_BUILD_DIR}"/build/lib64/libLTO.so."${CLANG_SUFFIX}" /usr/lib64/libLTO.so."${CLANG_SUFFIX}"
+rm -rf "${CLANG_BUILD_DIR}" "${SCYLLA_BUILD_DIR}" "${SCYLLA_NINJA_FILE}"
diff --git a/tools/toolchain/prepare b/tools/toolchain/prepare
index 5129166b60..7b290d06ee 100755
--- a/tools/toolchain/prepare
+++ b/tools/toolchain/prepare
@@ -32,29 +32,95 @@ if reg digest $(<tools/toolchain/image) > /dev/null; then
     exit 1
 fi
 
-archs=(amd64 arm64)
-
 # docker arch has a diffrent spelling than uname arch
-declare -A arch_unames=(
-    [amd64]=x86_64
-    [arm64]=aarch64
+declare -A docker_arch=(
+    [x86_64]=amd64
+    [aarch64]=arm64
     [s390x]=s390x
 )
 
-for arch in "${archs[@]}"; do
-    # translate from docker arch to uname arch
-    arch_uname="${arch_unames[$arch]}"
-    if [[ "$(uname -m)" == "${arch_uname}" ]]; then
-        continue
-    fi
-    if [[ ! -f  /proc/sys/fs/binfmt_misc/qemu-"${arch_uname}" ]]; then
-        echo install qemu-user-static
-        exit 1
-    fi
+usage() {
+    cat <<EOF
+
+Options:
+  --clang-build-mode <mode> specify one of following build modes:
+                            SKIP: skip building optimized clang
+                            INSTALL: build and install an optimized clang compiler binary, and save it as a tarball.
+                            INSTALL_FROM: install optimized clang from a tarball.
+  --clang-archive <file>   specify optimized clang tarball path
+  --help                   this help snippet
+EOF
+}
+
+CLANG_BUILD="SKIP"
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        "--clang-build-mode")
+            if [[ -z "$2" ]]; then
+                usage
+                exit 1
+            fi
+            CLANG_BUILD="$2"
+            shift 2
+            ;;
+        "--clang-archive")
+            if [[ -z "$2" ]]; then
+                usage
+                exit 1
+            fi
+            CLANG_ARCHIVE="$(realpath -m --relative-to=. "$2")"
+            shift 2
+            ;;
+        "--help")
+            usage
+            exit 0
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
 done
 
-buildah bud "${archs[@]/#/--platform=linux/}" --jobs 0 --squash --no-cache --pull -f tools/toolchain/Dockerfile --manifest "$(<tools/toolchain/image)"
+case "${CLANG_BUILD}" in
+    "SKIP" | "INSTALL" | "INSTALL_FROM")
+        ;;
+    *)
+        echo "Invalid mode specified on CLANG_BUILD: ${CLANG_BUILD}"
+        exit 1
+        ;;
+esac
+
+# clang archive file need to be under scylla directory, since we only share
+# scylla directory with the container
+if [[ "${CLANG_ARCHIVE}" = "../"* ]]; then
+    echo "clang archive file need to be under scylla directory"
+    exit 1
+fi
+
+# set default archive path if not specified
+if [[ -z "${CLANG_ARCHIVE}" ]]; then
+    CURDIR="$(dirname "$0")"
+    LLVM_CLANG_TAG="$(sed -n -e 's/^LLVM_CLANG_TAG=\(.*\)/\1/p' "${CURDIR}"/optimized_clang.sh)"
+    IMAGE_ID="$(sed -e 's#docker.io/scylladb/scylla-toolchain:##' "${CURDIR}"/image)"
+    CLANG_ARCHIVE="clang_build/optimized_clang_${LLVM_CLANG_TAG}_${IMAGE_ID}.$(arch).tar.gz"
+fi
+
+if [[ "${CLANG_BUILD}" = "INSTALL_FROM" ]] && [[ ! -f "${CLANG_ARCHIVE}" ]]; then
+    echo "${CLANG_ARCHIVE}" does not exist.
+    echo "Please specify vaild file with --clang-archive"
+    exit 1
+fi
+
+buildah bud --platform="linux/${docker_arch[$(arch)]}" --jobs 0 --squash --no-cache --pull -f tools/toolchain/Dockerfile --manifest "$(<tools/toolchain/image)" -v "$(realpath ./):/mnt:Z" --build-arg CLANG_BUILD="${CLANG_BUILD}" --build-arg CLANG_ARCHIVE="${CLANG_ARCHIVE}"
 
 echo "Done building $(<tools/toolchain/image). You can now test it, and push with"
 echo ""
 echo "    podman manifest push --all $(<tools/toolchain/image) docker://$(<tools/toolchain/image)"
+if [[ "${CLANG_BUILD}" = "INSTALL" ]]; then
+    echo ""
+    echo "Optimized clang archive saved at:"
+    echo "    ${CLANG_ARCHIVE}"
+    echo "You can rebuild frozen toolchain without rebuilding optimized clang by:"
+    echo "    ./tools/toolchain/prepare --clang-build-mode INSTALL_FROM --clang-archive ${CLANG_ARCHIVE}"
+fi