mirrors/VeraCrypt
1
0
Fork 0
mirror of https://github.com/veracrypt/VeraCrypt.git synced 2026-10-21 13:52:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tests: read known data from the test volumes without mounting (#1907)

Browse Source 
The Linux workflow checks the content of the Tests/*.hc volumes by
mounting them, which needs root, FUSE and the installed package, and so
only runs on x86_64. Add a test that recovers the same data through the
volume layer instead, so it also runs on other architectures, e.g. on the
big-endian cross build under qemu-user.

Tests/volume_read_test.cpp opens a volume read-only and reads its data
area with Volume::ReadSectors: in one call, as multi-sector ranges at
nonzero offsets with odd sector counts, again with the encryption thread
pool running, and one sector at a time; every read must match the first.
CPU features are detected as in the application, and the character set
is taken from the environment (LC_CTYPE) so that paths with non-ASCII
characters work; if that locale is unavailable, the test continues in the
C locale and says so.

Tests/test_volume_read.py builds it against Volume.a and Platform.a of a
finished standard Linux build (CXX and NM can be set for a cross build)
and checks the plaintext of every Tests/test.*.hc volume: the FAT boot
sector, DUMMY.TXT with the content "Dummy" at the cluster given by the
root directory, and a known SHA-256 digest of the whole data area. A
volume without a digest, or a digest without a volume, is an error.

It runs in the Linux workflow after the .deb build and in the build job
of the big-endian workflow (test.sha512.hc only in the automatic check,
all volumes in the full run), and requires the thread pool there.

Signed-off-by: Ville Takio <ville+git@takio.fi>
This commit is contained in:
Ville Takio
2026-10-11 18:50:22 +09:00
committed by GitHub
parent 3f04c42db9
commit df2e9dcd30
4 changed files with 413 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build-linux-big-endian.yml
+23 -4
View File
@@ -1,12 +1,14 @@
name: Build and test Linux big-endian
# Cross-builds the console version for s390x (big-endian) and runs it under
# qemu-user: the algorithm self-tests and one of the Tests/*.hc volumes
# (created on little-endian).
# qemu-user: the algorithm self-tests, a read of known data from one of the
# Tests/*.hc volumes without mounting, and a mount of one of them (all
# created on little-endian).
#
# The full suite (manual run with "full" selected) also builds amd64, then
# tests the Tests/*.hc volumes on both builds and volume compatibility
# between them in both directions, in separate jobs that run concurrently.
# reads and tests all Tests/*.hc volumes on both builds and volume
# compatibility between them in both directions, in separate jobs that run
# concurrently.
on:
push:
@@ -25,6 +27,8 @@ on:
- 'src/Makefile'
- 'Tests/test.*.hc'
- 'Tests/test_byte_order_interop.py'
- 'Tests/test_volume_read.py'
- 'Tests/volume_read_test.cpp'
- '.github/workflows/build-linux-big-endian.yml'
pull_request:
branches: [ "master" ]
@@ -42,6 +46,8 @@ on:
- 'src/Makefile'
- 'Tests/test.*.hc'
- 'Tests/test_byte_order_interop.py'
- 'Tests/test_volume_read.py'
- 'Tests/volume_read_test.cpp'
- '.github/workflows/build-linux-big-endian.yml'
# Manual runs, e.g. on a fork before opening a pull request.
workflow_dispatch:
@@ -128,6 +134,19 @@ jobs:
- name: Algorithm self-tests
run: src/Main/veracrypt --text --test
# Needs Volume.a and Platform.a, so it runs here and not in the jobs that
# only receive the binary. One volume keeps the automatic check short.
- name: Read known data from the test volumes
run: |
set -- --build-dir src --require-thread-pool
if [ "${{ matrix.arch }}" = s390x ]; then
export CXX=s390x-linux-gnu-g++ NM=s390x-linux-gnu-nm
fi
if [ "${{ env.FULL }}" != true ]; then
set -- "$@" --fixture test.sha512.hc
fi
python3 Tests/test_volume_read.py "$@"
# One volume keeps this check short; the full suite opens all of them.
- name: Volume created on little-endian
if: env.FULL != 'true'
.github/workflows/build-linux.yml
+12
View File
@@ -22,6 +22,9 @@ on:
- 'src/Volume/**'
- 'src/Makefile'
- 'Tests/test_linux_privilege_helpers.py'
- 'Tests/test.*.hc'
- 'Tests/test_volume_read.py'
- 'Tests/volume_read_test.cpp'
- '.github/workflows/build-linux.yml'
pull_request:
branches: [ "master" ]
@@ -44,6 +47,9 @@ on:
- 'src/Volume/**'
- 'src/Makefile'
- 'Tests/test_linux_privilege_helpers.py'
- 'Tests/test.*.hc'
- 'Tests/test_volume_read.py'
- 'Tests/volume_read_test.cpp'
- '.github/workflows/build-linux.yml'
concurrency:
@@ -129,6 +135,12 @@ jobs:
name: veracrypt-console-debs-${{ matrix.os }}
path: /tmp/VeraCrypt_Packaging/Console/Packaging/veracrypt-console-*.*
# Decrypts the Tests/*.hc volumes through Volume::ReadSectors, without
# mounting, using the archives that the last build above (the console
# version) left in src, and checks known data.
- name: Read known data from the test volumes
run: python3 Tests/test_volume_read.py --build-dir src --require-thread-pool
- name: Install and test VeraCrypt GUI .deb packages
run: |
sudo apt install -y /tmp/VeraCrypt_Packaging/GUI/Packaging/veracrypt-*.deb
Tests/test_volume_read.py
+223
View File
@@ -0,0 +1,223 @@
#!/usr/bin/env python3
"""Read known data from the Tests/test.*.hc volumes without mounting.
Usage:
python3 Tests/test_volume_read.py --build-dir src [--require-thread-pool]
[--fixture test.sha512.hc ...]
Compiles Tests/volume_read_test.cpp against Volume/Volume.a and
Platform/Platform.a of a finished Linux build in --build-dir, then decrypts
the data area of each volume through Volume::ReadSectors (in one call, as
multi-sector ranges at nonzero offsets, with the encryption thread pool, and
one sector at a time). The master keys and data offsets decoded from a
real header must recover the volume's plaintext. For every volume, the decrypted data area must:
- start with a FAT boot sector,
- contain DUMMY.TXT with the content "Dummy" at the cluster given by the
root directory,
- match a known SHA-256 digest.
Without --fixture, every Tests/test.*.hc volume is checked, and a volume
without a digest here (or a digest without a volume) is an error, so a new
fixture cannot be left out by accident. The thread pool does not start with
fewer than two processors; that part is then reported as skipped, or fails
with --require-thread-pool (as in CI).
Supported builds: the standard Linux build (Crypto/ backend, not
WOLFCRYPT_BACKEND), native or cross-compiled. The test program is compiled
with the defines and libraries listed in build() below, not with the flags
the archives were built with; only the libstdc++ ABI is detected from the
archive. Other configurations, e.g. a wolfCrypt build, which also needs
WOLFCRYPT_BACKEND and the wolfSSL library, are not supported.
Unlike mounting the volumes, this needs no root, FUSE or installed package,
so it also runs on a cross build: set CXX (and NM) to the cross tools, e.g.
CXX=s390x-linux-gnu-g++ NM=s390x-linux-gnu-nm. The test program is then run
through qemu-user binfmt support, which needs the target's shared libraries
(e.g. libstdc++6:s390x from a multiarch install, as in the CI workflow).
"""
import argparse
import hashlib
import os
import shlex
import struct
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
PASSWORD = "test"
EXPECTED_FILE = ("DUMMY TXT", b"Dummy")
# SHA-256 of the decrypted data area of each volume.
DIGESTS = {
"test.blake2s.hc":
"21cc4f2f8db13fadca82bebb2ad574184dcbb441d447a5dee7363ceac729e0c0",
"test.sha256.hc":
"2b15dbe4544a853ca15311dd1dc83162c6cc06c6c470d98ca427b26093a9fb41",
"test.sha512.hc":
"eef460d147e613d70df39a8e6e83b624fc1d258d8ae35875caf6579a05cba95b",
"test.streebog.hc":
"4c890502857a6360ed059c4d1dc736179fe29bffc9af72bfbff63bb691d34d82",
"test.whirlpool.hc":
"cb194e676678b7c40eead878df102eee0c93d453ca9d0a9e4eab7a3847edd1e0",
}
class CheckFailed(Exception):
pass
def old_abi(archive):
"""Tell whether the archive was built with _GLIBCXX_USE_CXX11_ABI=0.
src/Makefile selects the old ABI for WXSTATIC builds with GCC 5 or later;
the test program must use the same one to link.
"""
nm = shlex.split(os.environ.get("NM", "nm"))
symbols = subprocess.run(nm + ["--defined-only", str(archive)],
stdout=subprocess.PIPE, text=True,
check=True).stdout
return "cxx11" not in symbols
def build(build_dir, workdir):
volume_lib = build_dir / "Volume" / "Volume.a"
platform_lib = build_dir / "Platform" / "Platform.a"
for lib in (volume_lib, platform_lib):
if not lib.is_file():
sys.exit(f"{lib} not found: build VeraCrypt in {build_dir} first")
program = workdir / "volume_read_test"
command = shlex.split(os.environ.get("CXX", "c++")) + [
"-std=c++11", "-O2", "-DTC_UNIX", "-DTC_LINUX", "-DARGON2_NO_THREADS",
"-D_FILE_OFFSET_BITS=64",
"-I" + str(build_dir), "-I" + str(build_dir / "Crypto"),
"-I" + str(build_dir / "Crypto" / "Argon2" / "include"),
"-I" + str(build_dir / "PKCS11"),
]
if old_abi(volume_lib):
command.append("-D_GLIBCXX_USE_CXX11_ABI=0")
command += [str(ROOT / "Tests" / "volume_read_test.cpp"), str(volume_lib),
str(platform_lib), "-pthread", "-ldl", "-o", str(program)]
subprocess.run(command, check=True, timeout=300)
return program
def find_file(data, name):
"""Return a small file (one cluster at most) from the root directory of a
FAT12/16 image."""
if data[510:512] != b"\x55\xaa":
raise CheckFailed("no FAT boot sector signature")
(sector_size, cluster_sectors, reserved_sectors, fat_count, root_entries,
_, _, fat_sectors) = struct.unpack_from("<HBHBHHBH", data, 11)
if sector_size not in (512, 1024, 2048, 4096) or not cluster_sectors \
or not fat_count or not fat_sectors or not root_entries:
raise CheckFailed("unexpected FAT boot sector")
root = (reserved_sectors + fat_count * fat_sectors) * sector_size
first_data = root + -(-root_entries * 32 // sector_size) * sector_size
if first_data > len(data):
raise CheckFailed("FAT root directory beyond the data area")
for offset in range(root, root + root_entries * 32, 32):
entry = data[offset:offset + 32]
if entry[0] == 0:
break
# Skip volume labels, directories and long file name entries.
if entry[:11].decode("latin-1") == name and not entry[11] & 0x18:
cluster, size = struct.unpack_from("<HI", entry, 26)
start = first_data + (cluster - 2) * cluster_sectors * sector_size
if cluster < 2 or size > cluster_sectors * sector_size \
or start + size > len(data):
raise CheckFailed(f"{name.strip()}: bad cluster {cluster} "
f"or size {size}")
return data[start:start + size]
raise CheckFailed(f"{name.strip()} not found in the root directory")
def check_volume(program, fixture, workdir, require_thread_pool):
kdf = fixture.name.split(".")[1]
output = workdir / (fixture.name + ".plain")
command = [str(program), str(fixture), PASSWORD, kdf, str(output)]
if require_thread_pool:
command.append("--require-thread-pool")
result = subprocess.run(command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, text=True, timeout=900,
check=False)
if result.returncode != 0:
raise CheckFailed(f"exit {result.returncode}: "
f"{result.stderr.strip() or result.stdout.strip()}")
data = output.read_bytes()
output.unlink()
name, content = EXPECTED_FILE
found = find_file(data, name)
if found != content:
raise CheckFailed(f"{name.strip()} contains {found!r}, "
f"expected {content!r}")
digest = hashlib.sha256(data).hexdigest()
if digest != DIGESTS[fixture.name]:
raise CheckFailed(f"data area SHA-256 {digest}, "
f"expected {DIGESTS[fixture.name]}")
return result.stdout.strip()
def main():
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--build-dir", type=Path, required=True,
help="VeraCrypt source directory of a finished build "
"(normally src)")
parser.add_argument("--fixture", action="append", default=[],
metavar="NAME",
help="check only this volume from Tests/, given by "
"file name, e.g. test.sha512.hc (repeatable; "
"default: all)")
parser.add_argument("--require-thread-pool", action="store_true",
help="fail if the encryption thread pool does not "
"start (instead of skipping the threaded reads)")
args = parser.parse_args()
if not args.fixture:
present = {p.name for p in (ROOT / "Tests").glob("test.*.hc")}
if present != set(DIGESTS):
missing = sorted(present - set(DIGESTS))
stale = sorted(set(DIGESTS) - present)
problems = []
if missing:
problems.append(f"no digest for {', '.join(missing)}")
if stale:
problems.append(f"no volume for {', '.join(stale)}")
sys.exit("Tests/test.*.hc and the digests in this script do not "
"match: " + "; ".join(problems))
names = args.fixture or sorted(DIGESTS)
fixtures = []
for name in names:
path = ROOT / "Tests" / name
if not path.is_file():
sys.exit(f"Missing fixture: {path}")
if name not in DIGESTS:
sys.exit(f"No digest for {name} in this script")
fixtures.append(path)
failures = 0
with tempfile.TemporaryDirectory(prefix="vc-volume-read-") as tmp:
workdir = Path(tmp)
program = build(args.build_dir.resolve(), workdir)
for fixture in fixtures:
try:
info = check_volume(program, fixture, workdir,
args.require_thread_pool)
print(f"ok {fixture.name}: {info}", flush=True)
except (CheckFailed, subprocess.TimeoutExpired) as e:
print(f"FAIL {fixture.name}: {e}", flush=True)
failures += 1
if failures:
sys.exit(f"Failed: {failures} check(s)")
print("All checks passed", flush=True)
if __name__ == "__main__":
main()
Tests/volume_read_test.cpp
+155
View File
@@ -0,0 +1,155 @@
// Decrypt the data area of a volume through the volume layer, without
// mounting. Usage:
// volume_read_test VOLUME PASSWORD KDF OUTPUT [--require-thread-pool]
//
// The volume is opened read-only with PIM 0 and the given KDF (as with
// --hash). CPU features are detected as in the application, so the
// accelerated cipher code is used where available. The data area is read in
// one call, then as ranges of several sectors at nonzero offsets (odd sector
// counts, so the work is split unevenly), again with the encryption thread
// pool running, and one sector at a time; every read must match the first.
// The pool does not start with fewer than two processors: that part is then
// reported as skipped, or fails with --require-thread-pool. The plaintext is
// written to OUTPUT, where Tests/test_volume_read.py checks it.
#include "Platform/Platform.h"
#include "Platform/Finally.h"
#include "Volume/EncryptionThreadPool.h"
#include "Volume/Pkcs5Kdf.h"
#include "Volume/Volume.h"
#include "Crypto/cpu.h"
#include <clocale>
#include <cstring>
#include <fstream>
#include <iostream>
using namespace VeraCrypt;
// Same matching as --hash on the command line: the KDF name, the hash name
// or its alternative name, or "argon2"/"argon2id".
static shared_ptr <Pkcs5Kdf> FindKdf (const string &name)
{
string lowerName = StringConverter::ToLower (name);
foreach (shared_ptr <Pkcs5Kdf> kdf, Pkcs5Kdf::GetAvailableAlgorithms())
{
if (StringConverter::ToLower (StringConverter::ToSingle (kdf->GetName())) == lowerName)
return kdf;
if (kdf->IsArgon2())
{
if (lowerName == "argon2" || lowerName == "argon2id")
return kdf;
continue;
}
if (StringConverter::ToLower (StringConverter::ToSingle (kdf->GetHash()->GetName())) == lowerName
|| StringConverter::ToLower (StringConverter::ToSingle (kdf->GetHash()->GetAltName())) == lowerName)
return kdf;
}
throw ParameterIncorrect (SRC_POS);
}
static void Require (bool condition, const char *message)
{
if (!condition)
throw std::runtime_error (message);
}
// Read several multi-sector ranges at nonzero offsets and compare them with
// the data area read in one call.
static void CheckRanges (Volume &volume, const SecureBuffer &data, const char *message)
{
const size_t sectorSize = volume.GetSectorSize();
const uint64 sectors = data.Size() / sectorSize;
Require (sectors >= 34, "data area too small for the range checks");
const uint64 ranges[][2] = {
{ 1, 3 }, { 7, 5 }, { sectors / 2 - 4, 9 }, { sectors - 17, 17 }
};
for (size_t i = 0; i < sizeof (ranges) / sizeof (ranges[0]); i++)
{
const uint64 offset = ranges[i][0] * sectorSize;
const size_t length = (size_t) (ranges[i][1] * sectorSize);
Require (ranges[i][0] > 0 && ranges[i][0] + ranges[i][1] <= sectors, "data area too small for the range checks");
SecureBuffer buffer (length);
volume.ReadSectors (buffer, offset);
Require (memcmp (buffer.Ptr(), data.Ptr() + offset, length) == 0, message);
}
}
int main (int argc, char **argv)
{
const bool requireThreadPool = (argc == 6 && strcmp (argv[5], "--require-thread-pool") == 0);
if (argc != 5 && !requireThreadPool)
{
std::cerr << "usage: volume_read_test VOLUME PASSWORD KDF OUTPUT [--require-thread-pool]\n";
return 2;
}
// The volume path is converted to a wide string with mbsrtowcs, which
// needs the character set of the environment (e.g. for "répertoire").
// Without it (e.g. the locale is not installed) only ASCII paths work.
const bool localeSet = setlocale (LC_CTYPE, "") != nullptr;
try
{
#ifdef CRYPTOPP_CPUID_AVAILABLE
DetectX86Features ();
#endif
#if CRYPTOPP_BOOL_ARMV8
DetectArmFeatures ();
#endif
shared_ptr <VolumePassword> password (new VolumePassword ((const uint8 *) argv[2], strlen (argv[2])));
Volume volume;
volume.Open (VolumePath (string (argv[1])), false, password, 0, FindKdf (argv[3]),
shared_ptr <KeyfileList> (), false, VolumeProtection::ReadOnly);
const uint64 size = volume.GetSize();
const size_t sectorSize = volume.GetSectorSize();
Require (size != 0 && size % sectorSize == 0, "unexpected data area size");
SecureBuffer data ((size_t) size);
volume.ReadSectors (data, 0);
CheckRanges (volume, data, "read of several sectors differs");
EncryptionThreadPool::Start();
finally_do ({ EncryptionThreadPool::Stop(); });
const bool threaded = EncryptionThreadPool::IsRunning();
if (threaded)
{
SecureBuffer all ((size_t) size);
volume.ReadSectors (all, 0);
Require (memcmp (all.Ptr(), data.Ptr(), (size_t) size) == 0, "read with the thread pool differs");
CheckRanges (volume, data, "read of several sectors with the thread pool differs");
}
else
Require (!requireThreadPool, "the encryption thread pool did not start (fewer than two processors?)");
SecureBuffer sector (sectorSize);
for (uint64 offset = 0; offset < size; offset += sectorSize)
{
volume.ReadSectors (sector, offset);
Require (memcmp (sector.Ptr(), data.Ptr() + offset, sectorSize) == 0, "read one sector at a time differs");
}
std::ofstream output (argv[4], std::ios::binary);
output.write ((const char *) data.Ptr(), (std::streamsize) data.Size());
output.close();
Require (!output.fail(), "cannot write the output file");
std::cout << StringConverter::ToSingle (volume.GetEncryptionAlgorithm()->GetName())
<< " " << size << " bytes" << (threaded ? "" : ", thread pool reads skipped (pool not started)")
<< (localeSet ? "" : ", locale from the environment unavailable, using C") << "\n";
volume.Close();
}
catch (std::exception &e)
{
std::cerr << "error: " << StringConverter::ToSingle (StringConverter::ToExceptionString (e)) << "\n";
return 1;
}
catch (...)
{
std::cerr << "error: unknown exception\n";
return 1;
}
return 0;
}