commit ed38ddaac22f0684bf444a76644267647e332853
parent be310c5ab67061fe0913fd988a26d44934d1af4d
Author: Jared Tobin <jared@jtobin.io>
Date: Tue, 20 Oct 2026 14:25:17 +0400
record: QEMU's allocations kept off the guest's; runs laid out apart refused
QEMU in user mode maps the guest's libraries and heap in its own
address space, wherever its own allocations leave room. Those vary
with how its threads are scheduled: a second malloc arena, or a large
block mapped rather than taken from the heap, can take the room the
guest's next mapping would have had. With address randomisation off,
runs of one secret still laid out apart under load. On pollux, six at
a time, 16 of 23 runs of a secp256k1 harness differed from the first,
some of them in every object, and 10 of 23 runs of a Penumbra harness
moved their heap. Such a run differs wherever it touches memory, which
buried the planted table-index leak among instructions counted as
downstream, and set the instructions of a moved heap apart as unsteady.
record now runs QEMU with one malloc arena and large blocks taken from
the heap (GLIBC_TUNABLES, set for QEMU and kept from the guest), and
the same parallel runs of the Penumbra harness then agree, 0 of 23
differing. diff refuses runs that loaded an object at different
addresses, naming the run and the object, rather than reporting their
differences as the secret's.
Diffstat:
5 files changed, 82 insertions(+), 7 deletions(-)
diff --git a/CHANGELOG b/CHANGELOG
@@ -20,8 +20,11 @@
designed to reach particular paths compared for control and
coverage alone (`--designed`); runs are recorded with
randomisation off (or not at all) and the guest's entropy,
- its getrandom included, seeded. A taint run that marked no
- secret is refused.
+ its getrandom included, seeded, and QEMU's own allocations kept
+ off the guest's mappings, which under load can otherwise take
+ the room of a guest library or heap; `diff` refuses runs that
+ loaded an object at different addresses. A taint run that
+ marked no secret is refused.
* `traceOf` and `diffOf` run either as a derivation;
`nix/lib.nix` holds them for any package set, with the version
the command reports.
diff --git a/README.md b/README.md
@@ -89,8 +89,12 @@ every instruction some run executed is listed as coverage.
`record` disables address-space randomisation and seeds the guest's
entropy, `getrandom` included (`nix/qemu-getrandom.patch`), so runs
-lay memory out alike. A guest reading `/dev/urandom` still reads the
-host's.
+lay memory out alike. QEMU maps the guest wherever its own
+allocations leave room, which under load can differ between runs, so
+`record` keeps QEMU's allocations off the guest's mappings (one malloc
+arena, large blocks from the heap), and `diff` refuses runs that
+loaded an object at different addresses. A guest reading
+`/dev/urandom` still reads the host's.
### As derivations
diff --git a/app/Main.hs b/app/Main.hs
@@ -17,6 +17,7 @@ import Control.Exception (IOException, try)
import Control.Monad (unless, when)
import qualified Data.ByteString as BS
import Data.Version (showVersion)
+import Numeric (showHex)
import qualified Data.Text as T
import qualified Data.Text.Encoding as TE
import qualified Data.Text.Encoding.Error as TEE
@@ -32,7 +33,7 @@ import qualified System.Process as P
import Paths_vestigia (version)
import Vestigia.Diff
(diffDesigned, divergenceJson, divergenceSummary, dvOrigins,
- inObject, inOffset, readDump, restrictTo)
+ inObject, inOffset, layoutApart, readDump, restrictTo)
import Vestigia.Json (encode)
import Vestigia.Layout (fixLayout)
import Vestigia.Memcheck (readMemcheck)
@@ -236,10 +237,27 @@ record o prog progArgs = do
exitWith (ExitFailure 2)
qemu <- tool (oQemu o) "VESTIGIA_QEMU" "qemu-x86_64"
plugin <- tool (oPlugin o) "VESTIGIA_PLUGIN" "libvestigia-qemu.so"
+ -- QEMU maps the guest's libraries and heap in its own address
+ -- space, wherever its own allocations leave room, and those vary
+ -- with how its threads run: a second malloc arena, or a large block
+ -- mapped rather than drawn from the heap, takes the room the
+ -- guest's next mapping would have had, and under load one run lays
+ -- out apart from another. One arena, and large blocks drawn from
+ -- the heap, keep QEMU's allocations off the guest's mappings. The
+ -- guest keeps its own setting of the variable, or has none
+ environment <- getEnvironment
+ let tunables = "GLIBC_TUNABLES"
+ host = "glibc.malloc.arena_max=1:glibc.malloc.mmap_threshold="
+ <> "33554432"
+ guest = maybe [ "-U", tunables ]
+ (\v -> [ "-E", tunables <> "=" <> v ]) (lookup tunables environment)
r <- try (withCreateProcess
- (proc qemu ([ "-seed", "0"
- , "-plugin", plugin <> ",out=" <> dump, prog ]
+ ((proc qemu (guest
+ <> [ "-seed", "0"
+ , "-plugin", plugin <> ",out=" <> dump, prog ]
<> progArgs))
+ { P.env = Just ((tunables, host)
+ : filter ((/= tunables) . fst) environment) })
(\_ _ _ ph -> waitForProcess ph))
case r of
Left e -> do
@@ -254,6 +272,23 @@ diff o paths = do
dumps <- traverse readOne paths
controls <- traverse readOne (oControls o)
designed <- traverse readOne (oDesigned o)
+ -- runs laid out apart differ wherever they touch memory, not by
+ -- the secret: such runs are refused, not compared
+ case zip paths dumps of
+ ((p0, first) : _) -> case
+ [ (p, obj, x, y)
+ | (p, d) <- zip (paths <> oControls o <> oDesigned o)
+ (dumps <> controls <> designed)
+ , (obj, x, y) <- take 1 (layoutApart first d) ] of
+ ((p, obj, x, y) : _) -> do
+ hPutStrLn stderr ("vestigia: " <> p <> " loaded "
+ <> T.unpack obj <> " at 0x" <> showHex y "" <> " where "
+ <> p0 <> " loaded it at 0x"
+ <> showHex x "" <> "; runs laid out apart differ wherever \
+ \they touch memory, not by the secret, and are not compared")
+ exitWith (ExitFailure 2)
+ [] -> pure ()
+ [] -> pure ()
let dv = restrictTo (oObjects o)
(diffDesigned controls dumps designed)
places <- symbolize addr2line
diff --git a/lib/Vestigia/Diff.hs b/lib/Vestigia/Diff.hs
@@ -66,6 +66,8 @@ module Vestigia.Diff (
, diffControlled
, diffDesigned
, restrictTo
+ , loadBases
+ , layoutApart
, divergenceJson
, divergenceSummary
) where
@@ -127,6 +129,25 @@ readDump t = go (T.lines t) M.empty M.empty
[ (v, "") ] -> pure v
_ -> Left ("not hexadecimal: " <> T.unpack x)
+-- | Where a run loaded each object it executed: any of its
+-- instructions' addresses less its offset in the object.
+loadBases :: Dump -> M.Map T.Text Integer
+loadBases d = M.fromList
+ [ (o, inPc i - inOffset i)
+ | i <- M.elems (dpInsns d), Just o <- [ inObject i ] ]
+
+-- | The objects two runs both executed but loaded at different
+-- addresses, each with where the first run put it and where the
+-- second did. Runs laid out apart touch other addresses wherever
+-- they touch memory, so comparing them says nothing of the
+-- secret.
+layoutApart :: Dump -> Dump -> [(T.Text, Integer, Integer)]
+layoutApart a b =
+ [ (o, x, y)
+ | (o, (x, y)) <- M.toList
+ (M.intersectionWith (,) (loadBases a) (loadBases b))
+ , x /= y ]
+
-- | How an origin depended on the secret.
data Origin =
ControlOrigin
diff --git a/test/Main.hs b/test/Main.hs
@@ -131,6 +131,18 @@ diffTests = testGroup "diff"
dpTransfers d @?= M.fromList
[ (0x10, M.fromList [ (0x30, 2) ]) ]
+ , testCase "a run's objects are where its instructions put them" $ do
+ a <- either assertFailure pure (readDump dumpA)
+ loadBases a @?= M.fromList [ ("/lib.so", 0) ]
+ -- the same run with /lib.so loaded 0x1000 higher
+ b <- either assertFailure pure (readDump (T.unlines
+ [ "I 1010 4 0 aaaa 0 /lib.so 10", "E 1010 1030 2"
+ , "I 1020 4 4 bbbb 0 /lib.so 20", "I 1030 2 0 cccc 0 /lib.so 30"
+ , "I 1040 1 1 dddd 0 /lib.so 40" ]))
+ loadBases b @?= M.fromList [ ("/lib.so", 0x1000) ]
+ layoutApart a b @?= [ ("/lib.so", 0, 0x1000) ]
+ layoutApart a a @?= []
+
, testCase "origins are where steady instructions differ" $ do
a <- either assertFailure pure (readDump dumpA)
b <- either assertFailure pure (readDump dumpB)
