commit 9113a280893f9493a01519d7af1510dbc4a36f62
parent be310c5ab67061fe0913fd988a26d44934d1af4d
Author: Jared Tobin <jared@jtobin.io>
Date: Tue, 20 Oct 2026 14:44:12 +0400
merge impl/layout: QEMU's allocations kept off the guest's; runs laid out apart refused
- record runs QEMU with one malloc arena and large blocks taken from
the heap (GLIBC_TUNABLES, set for QEMU and kept from the guest), so
QEMU's own allocations no longer take the room of the guest's
libraries or heap. Under load, runs of one secret had laid out
apart: on pollux, six at a time, 16 of 23 runs of a secp256k1
harness and 10 of 23 of a Penumbra harness differed from the first.
With the fix, the Penumbra runs all agree, and the secp256k1 runs
differ only in five glibc instructions' addresses, outside the
objects under audit.
- diff refuses runs that loaded an object at different addresses,
naming the run and the object; such runs differ wherever they touch
memory, which had buried a planted table-index leak among
instructions counted as downstream.
- loadBases and layoutApart in Vestigia.Diff, with tests; README and
CHANGELOG say what record and diff now do.
Diffstat:
5 files changed, 82 insertions(+), 7 deletions(-)
diff --git a/CHANGELOG b/CHANGELOG
@@ -20,8 +20,11 @@
designed to reach particular paths compared for control and
coverage alone (`--designed`); runs are recorded with
randomisation off (or not at all) and the guest's entropy,
- its getrandom included, seeded. A taint run that marked no
- secret is refused.
+ its getrandom included, seeded, and QEMU's own allocations kept
+ off the guest's mappings, which under load can otherwise take
+ the room of a guest library or heap; `diff` refuses runs that
+ loaded an object at different addresses. A taint run that
+ marked no secret is refused.
* `traceOf` and `diffOf` run either as a derivation;
`nix/lib.nix` holds them for any package set, with the version
the command reports.
diff --git a/README.md b/README.md
@@ -89,8 +89,12 @@ every instruction some run executed is listed as coverage.
`record` disables address-space randomisation and seeds the guest's
entropy, `getrandom` included (`nix/qemu-getrandom.patch`), so runs
-lay memory out alike. A guest reading `/dev/urandom` still reads the
-host's.
+lay memory out alike. QEMU maps the guest wherever its own
+allocations leave room, which under load can differ between runs, so
+`record` keeps QEMU's allocations off the guest's mappings (one malloc
+arena, large blocks from the heap), and `diff` refuses runs that
+loaded an object at different addresses. A guest reading
+`/dev/urandom` still reads the host's.
### As derivations
diff --git a/app/Main.hs b/app/Main.hs
@@ -17,6 +17,7 @@ import Control.Exception (IOException, try)
import Control.Monad (unless, when)
import qualified Data.ByteString as BS
import Data.Version (showVersion)
+import Numeric (showHex)
import qualified Data.Text as T
import qualified Data.Text.Encoding as TE
import qualified Data.Text.Encoding.Error as TEE
@@ -32,7 +33,7 @@ import qualified System.Process as P
import Paths_vestigia (version)
import Vestigia.Diff
(diffDesigned, divergenceJson, divergenceSummary, dvOrigins,
- inObject, inOffset, readDump, restrictTo)
+ inObject, inOffset, layoutApart, readDump, restrictTo)
import Vestigia.Json (encode)
import Vestigia.Layout (fixLayout)
import Vestigia.Memcheck (readMemcheck)
@@ -236,10 +237,27 @@ record o prog progArgs = do
exitWith (ExitFailure 2)
qemu <- tool (oQemu o) "VESTIGIA_QEMU" "qemu-x86_64"
plugin <- tool (oPlugin o) "VESTIGIA_PLUGIN" "libvestigia-qemu.so"
+ -- QEMU maps the guest's libraries and heap in its own address
+ -- space, wherever its own allocations leave room, and those vary
+ -- with how its threads run: a second malloc arena, or a large block
+ -- mapped rather than drawn from the heap, takes the room the
+ -- guest's next mapping would have had, and under load one run lays
+ -- out apart from another. One arena, and large blocks drawn from
+ -- the heap, keep QEMU's allocations off the guest's mappings. The
+ -- guest keeps its own setting of the variable, or has none
+ environment <- getEnvironment
+ let tunables = "GLIBC_TUNABLES"
+ host = "glibc.malloc.arena_max=1:glibc.malloc.mmap_threshold="
+ <> "33554432"
+ guest = maybe [ "-U", tunables ]
+ (\v -> [ "-E", tunables <> "=" <> v ]) (lookup tunables environment)
r <- try (withCreateProcess
- (proc qemu ([ "-seed", "0"
- , "-plugin", plugin <> ",out=" <> dump, prog ]
+ ((proc qemu (guest
+ <> [ "-seed", "0"
+ , "-plugin", plugin <> ",out=" <> dump, prog ]
<> progArgs))
+ { P.env = Just ((tunables, host)
+ : filter ((/= tunables) . fst) environment) })
(\_ _ _ ph -> waitForProcess ph))
case r of
Left e -> do
@@ -254,6 +272,23 @@ diff o paths = do
dumps <- traverse readOne paths
controls <- traverse readOne (oControls o)
designed <- traverse readOne (oDesigned o)
+ -- runs laid out apart differ wherever they touch memory, not by
+ -- the secret: such runs are refused, not compared
+ case zip paths dumps of
+ ((p0, first) : _) -> case
+ [ (p, obj, x, y)
+ | (p, d) <- zip (paths <> oControls o <> oDesigned o)
+ (dumps <> controls <> designed)
+ , (obj, x, y) <- take 1 (layoutApart first d) ] of
+ ((p, obj, x, y) : _) -> do
+ hPutStrLn stderr ("vestigia: " <> p <> " loaded "
+ <> T.unpack obj <> " at 0x" <> showHex y "" <> " where "
+ <> p0 <> " loaded it at 0x"
+ <> showHex x "" <> "; runs laid out apart differ wherever \
+ \they touch memory, not by the secret, and are not compared")
+ exitWith (ExitFailure 2)
+ [] -> pure ()
+ [] -> pure ()
let dv = restrictTo (oObjects o)
(diffDesigned controls dumps designed)
places <- symbolize addr2line
diff --git a/lib/Vestigia/Diff.hs b/lib/Vestigia/Diff.hs
@@ -66,6 +66,8 @@ module Vestigia.Diff (
, diffControlled
, diffDesigned
, restrictTo
+ , loadBases
+ , layoutApart
, divergenceJson
, divergenceSummary
) where
@@ -127,6 +129,25 @@ readDump t = go (T.lines t) M.empty M.empty
[ (v, "") ] -> pure v
_ -> Left ("not hexadecimal: " <> T.unpack x)
+-- | Where a run loaded each object it executed: any of its
+-- instructions' addresses less its offset in the object.
+loadBases :: Dump -> M.Map T.Text Integer
+loadBases d = M.fromList
+ [ (o, inPc i - inOffset i)
+ | i <- M.elems (dpInsns d), Just o <- [ inObject i ] ]
+
+-- | The objects two runs both executed but loaded at different
+-- addresses, each with where the first run put it and where the
+-- second did. Runs laid out apart touch other addresses wherever
+-- they touch memory, so comparing them says nothing of the
+-- secret.
+layoutApart :: Dump -> Dump -> [(T.Text, Integer, Integer)]
+layoutApart a b =
+ [ (o, x, y)
+ | (o, (x, y)) <- M.toList
+ (M.intersectionWith (,) (loadBases a) (loadBases b))
+ , x /= y ]
+
-- | How an origin depended on the secret.
data Origin =
ControlOrigin
diff --git a/test/Main.hs b/test/Main.hs
@@ -131,6 +131,18 @@ diffTests = testGroup "diff"
dpTransfers d @?= M.fromList
[ (0x10, M.fromList [ (0x30, 2) ]) ]
+ , testCase "a run's objects are where its instructions put them" $ do
+ a <- either assertFailure pure (readDump dumpA)
+ loadBases a @?= M.fromList [ ("/lib.so", 0) ]
+ -- the same run with /lib.so loaded 0x1000 higher
+ b <- either assertFailure pure (readDump (T.unlines
+ [ "I 1010 4 0 aaaa 0 /lib.so 10", "E 1010 1030 2"
+ , "I 1020 4 4 bbbb 0 /lib.so 20", "I 1030 2 0 cccc 0 /lib.so 30"
+ , "I 1040 1 1 dddd 0 /lib.so 40" ]))
+ loadBases b @?= M.fromList [ ("/lib.so", 0x1000) ]
+ layoutApart a b @?= [ ("/lib.so", 0, 0x1000) ]
+ layoutApart a a @?= []
+
, testCase "origins are where steady instructions differ" $ do
a <- either assertFailure pure (readDump dumpA)
b <- either assertFailure pure (readDump dumpB)
